Update README.md

Final Experiments and Plot adjustments
adjustments for cuda and auto ckpt cleanup
2023-11-14 16:25:07 +01:00 · 2022-03-12 11:39:28 +01:00 · 2022-03-07 11:29:06 +01:00 · 2022-03-06 22:24:00 +01:00 · 2022-03-05 17:32:30 +01:00 · 2022-03-05 16:51:19 +01:00
129 changed files with 3553 additions and 7890 deletions
--- a/.gitignore
+++ b/.gitignore
@ -1,572 +1 @@
-# Created by https://www.gitignore.io/api/data,linux,macos,python,windows,pycharm,database,jupyternotebook
+/output/
 # Edit at https://www.gitignore.io/?templates=data,linux,macos,python,windows,pycharm,database,jupyternotebook
 ### Local Datasets ###
 /experiments
 /setups/experiments
 ### Data ###
 *.csv
 *.dat
 *.efx
 *.gbr
 *.key
 *.pps
 *.ppt
 *.pptx
 *.sdf
 *.tax2010
 *.vcf
 *.xml
 ### Database ###
 *.accdb
 *.db
 *.dbf
 *.mdb
 *.pdb
 *.sqlite3
 ### JupyterNotebook ###
 .ipynb_checkpoints
 */.ipynb_checkpoints/*
 # Remove previous ipynb_checkpoints
 #   git rm -r .ipynb_checkpoints/
 #
 ### Linux ###
 *~
 # temporary files which can be created if a process still has a handle open of a deleted file
 .fuse_hidden*
 # KDE directory preferences
 .directory
 # Linux trash folder which might appear on any partition or disk
 .Trash-*
 # .nfs files are created when an open file is removed but is still being accessed
 .nfs*
 ### macOS ###
 # General
 .DS_Store
 .AppleDouble
 .LSOverride
 # Icon must end with two \r
 Icon
 # Thumbnails
 ._*
 # Files that might appear in the root of a volume
 .DocumentRevisions-V100
 .fseventsd
 .Spotlight-V100
 .TemporaryItems
 .Trashes
 .VolumeIcon.icns
 .com.apple.timemachine.donotpresent
 # Directories potentially created on remote AFP share
 .AppleDB
 .AppleDesktop
 Network Trash Folder
 Temporary Items
 .apdisk
 ### PyCharm ###
 # Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio and WebStorm
 # Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839
 # User-specific stuff
 .idea/**/workspace.xml
 .idea/**/tasks.xml
 .idea/**/usage.statistics.xml
 .idea/**/dictionaries
 .idea/**/shelf
 # Generated files
 .idea/**/contentModel.xml
 # Sensitive or high-churn files
 .idea/**/dataSources/
 .idea/**/dataSources.ids
 .idea/**/dataSources.local.xml
 .idea/**/sqlDataSources.xml
 .idea/**/dynamic.xml
 .idea/**/uiDesigner.xml
 .idea/**/dbnavigator.xml
 # Gradle
 .idea/**/gradle.xml
 .idea/**/libraries
 # Gradle and Maven with auto-import
 # When using Gradle or Maven with auto-import, you should exclude module files,
 # since they will be recreated, and may cause churn.  Uncomment if using
 # auto-import.
 # .idea/modules.xml
 # .idea/*.iml
 # .idea/modules
 # CMake
 cmake-build-*/
 # Mongo Explorer plugin
 .idea/**/mongoSettings.xml
 # File-based project format
 *.iws
 # IntelliJ
 out/
 # mpeltonen/sbt-idea plugin
 .idea_modules/
 # JIRA plugin
 atlassian-ide-plugin.xml
 # Cursive Clojure plugin
 .idea/replstate.xml
 # Crashlytics plugin (for Android Studio and IntelliJ)
 com_crashlytics_export_strings.xml
 crashlytics.properties
 crashlytics-build.properties
 fabric.properties
 # Editor-based Rest Client
 .idea/httpRequests
 # Android studio 3.1+ serialized cache file
 .idea/caches/build_file_checksums.ser
 ### PyCharm Patch ###
 # Comment Reason: https://github.com/joeblau/gitignore.io/issues/186#issuecomment-215987721
 # *.iml
 # modules.xml
 # .idea/misc.xml
 # *.ipr
 # Sonarlint plugin
 .idea/sonarlint
 ### Python ###
 # Byte-compiled / optimized / DLL files
 __pycache__/
 *.py[cod]
 *$py.class
 # C extensions
 *.so
 # Distribution / packaging
 .Python
 build/
 develop-eggs/
 dist/
 downloads/
 eggs/
 .eggs/
 lib/
 lib64/
 parts/
 sdist/
 var/
 wheels/
 share/python-wheels/
 *.egg-info/
 .installed.cfg
 *.egg
 MANIFEST
 # PyInstaller
 #  Usually these files are written by a python script from a template
 #  before PyInstaller builds the exe, so as to inject date/other infos into it.
 *.manifest
 *.spec
 # Installer logs
 pip-log.txt
 pip-delete-this-directory.txt
 # Unit test / coverage reports
 htmlcov/
 .tox/
 .nox/
 .coverage
 .coverage.*
 .cache
 nosetests.xml
 coverage.xml
 *.cover
 .hypothesis/
 .pytest_cache/
 # Translations
 *.mo
 *.pot
 # Django stuff:
 *.log
 local_settings.py
 db.sqlite3
 # Flask stuff:
 instance/
 .webassets-cache
 # Scrapy stuff:
 .scrapy
 # Sphinx documentation
 docs/_build/
 # PyBuilder
 target/
 # Jupyter Notebook
 # IPython
 profile_default/
 ipython_config.py
 # pyenv
 .python-version
 # celery beat schedule file
 celerybeat-schedule
 # SageMath parsed files
 *.sage.py
 # Environments
 .env
 .venv
 env/
 venv/
 ENV/
 env.bak/
 venv.bak/
 # Spyder project settings
 .spyderproject
 .spyproject
 # Rope project settings
 .ropeproject
 # mkdocs documentation
 /site
 # mypy
 .mypy_cache/
 .dmypy.json
 dmypy.json
 # Pyre type checker
 .pyre/
 ### Python Patch ###
 .venv/
 ### Windows ###
 # Windows thumbnail cache files
 Thumbs.db
 ehthumbs.db
 ehthumbs_vista.db
 # Dump file
 *.stackdump
 # Folder config file
 [Dd]esktop.ini
 # Recycle Bin used on file shares
 $RECYCLE.BIN/
 # Windows Installer files
 *.cab
 *.msi
 *.msix
 *.msm
 *.msp
 # Windows shortcuts
 *.lnk
 # pycharm
 .idea/
 #######################################
 #### Tex related
 ## Core latex/pdflatex auxiliary files:
 *.aux
 *.lof
 *.log
 *.lot
 *.fls
 *.out
 *.toc
 *.fmt
 *.fot
 *.cb
 *.cb2
 .*.lb
 ## Intermediate documents:
 *.dvi
 *.xdv
 *-converted-to.*
 # these rules might exclude image files for figures etc.
 # *.ps
 # *.eps
 # *.pdf
 ## Generated if empty string is given at "Please type another file name for output:"
 .pdf
 ## Bibliography auxiliary files (bibtex/biblatex/biber):
 *.bbl
 *.bcf
 *.blg
 *-blx.aux
 *-blx.bib
 *.run.xml
 ## Build tool auxiliary files:
 *.fdb_latexmk
 *.synctex
 *.synctex(busy)
 *.synctex.gz
 *.synctex.gz(busy)
 *.pdfsync
 ## Build tool directories for auxiliary files
 # latexrun
 latex.out/
 ## Auxiliary and intermediate files from other packages:
 # algorithms
 *.alg
 *.loa
 # achemso
 acs-*.bib
 # amsthm
 *.thm
 # beamer
 *.nav
 *.pre
 *.snm
 *.vrb
 # changes
 *.soc
 # comment
 *.cut
 # cprotect
 *.cpt
 # elsarticle (documentclass of Elsevier journals)
 *.spl
 # endnotes
 *.ent
 # fixme
 *.lox
 # feynmf/feynmp
 *.mf
 *.mp
 *.t[1-9]
 *.t[1-9][0-9]
 *.tfm
 #(r)(e)ledmac/(r)(e)ledpar
 *.end
 *.?end
 *.[1-9]
 *.[1-9][0-9]
 *.[1-9][0-9][0-9]
 *.[1-9]R
 *.[1-9][0-9]R
 *.[1-9][0-9][0-9]R
 *.eledsec[1-9]
 *.eledsec[1-9]R
 *.eledsec[1-9][0-9]
 *.eledsec[1-9][0-9]R
 *.eledsec[1-9][0-9][0-9]
 *.eledsec[1-9][0-9][0-9]R
 # glossaries
 *.acn
 *.acr
 *.glg
 *.glo
 *.gls
 *.glsdefs
 # gnuplottex
 *-gnuplottex-*
 # gregoriotex
 *.gaux
 *.gtex
 # htlatex
 *.4ct
 *.4tc
 *.idv
 *.lg
 *.trc
 *.xref
 # hyperref
 *.brf
 # knitr
 *-concordance.tex
 # TODO Comment the next line if you want to keep your tikz graphics files
 *.tikz
 *-tikzDictionary
 # listings
 *.lol
 # makeidx
 *.idx
 *.ilg
 *.ind
 *.ist
 # minitoc
 *.maf
 *.mlf
 *.mlt
 *.mtc[0-9]*
 *.slf[0-9]*
 *.slt[0-9]*
 *.stc[0-9]*
 # minted
 _minted*
 *.pyg
 # morewrites
 *.mw
 # nomencl
 *.nlg
 *.nlo
 *.nls
 # pax
 *.pax
 # pdfpcnotes
 *.pdfpc
 # sagetex
 *.sagetex.sage
 *.sagetex.py
 *.sagetex.scmd
 # scrwfile
 *.wrt
 # sympy
 *.sout
 *.sympy
 sympy-plots-for-*.tex/
 # pdfcomment
 *.upa
 *.upb
 # pythontex
 *.pytxcode
 pythontex-files-*/
 # tcolorbox
 *.listing
 # thmtools
 *.loe
 # TikZ & PGF
 *.dpth
 *.md5
 *.auxlock
 # todonotes
 *.tdo
 # vhistory
 *.hst
 *.ver
 # easy-todo
 *.lod
 # xcolor
 *.xcp
 # xmpincl
 *.xmpi
 # xindy
 *.xdy
 # xypic precompiled matrices
 *.xyc
 # endfloat
 *.ttt
 *.fff
 # Latexian
 TSWLatexianTemp*
 ## Editors:
 # WinEdt
 *.bak
 *.sav
 # Texpad
 .texpadtmp
 # LyX
 *.lyx~
 # Kile
 *.backup
 # KBibTeX
 *~[0-9]*
 # auto folder when using emacs and auctex
 ./auto/*
 *.el
 # expex forward references with \gathertags
 *-tags.tex
 # standalone packages
 *.sta
 # End of https://www.gitignore.io/api/data,linux,macos,python,windows,pycharm,database,jupyternotebook
--- a/README.md
+++ b/README.md
@ -1,2 +1,84 @@
-# bannana-networks
+# Bureaucratic Cohort Swarms
-
+### Pruning Networks by SRNN 
 ###### Deadline: 28.02.22
 ## Experimente
 ### Fixpoint Tests:
 - [X] Dropout Test 
  - (Macht das Partikel beim Goal mit oder ist es nur SRN)
  - Zero_ident diff = -00.04999637603759766 %
 - [ ] gnf(1) -> Aprox. Weight
  - Übersetung in ein Gewichtsskalar
    - Einbettung in ein Reguläres Netz
 - [ ] Übersetzung in ein Explainable AI Framework
  - Rückschlüsse auf Mikro Netze
 - [ ] Visualiserung
  - Der Zugehörigkeit 
  - Der Vernetzung
 - [ ] PCA()
  - Dataframe Epoch, Weight, dim_1, ..., dim_n
  - Visualisierung als Trajectory Cube
 - [ ] Recherche zu Makro Mikro Netze Strukturen 
  - gits das schon?
  - Hypernetwork?
  - arxiv: 1905.02898
  - Sparse Networks
  - Pruning 
 ---
 ### Tasks für Steffen:
 - [x] Sanity Check:
  - [x] Neuronen können lernen einen Eingabewert mit x zu multiplizieren?
    | SRNN x*n    3 Neurons            Identity_Func     | SRNN x*n    4 Neurons              Identity_Func  |
    |---------------------------------------------------|----------------------------------------------------|
    | ![](./figures/sanity/sanity_3hidden_xtimesn.png)   | ![](./figures/sanity/sanity_4hidden_xtimesn.png)  |
    | SRNN x*n    6 Neurons              Other_Func      | SRNN x*n    10 Neurons      Other_Func            |
    | ![](./figures/sanity/sanity_6hidden_xtimesn.png)   | ![](./figures/sanity/sanity_10hidden_xtimesn.png) |
 - [ ] Connectivity
  - Das Netz dünnt sich wirklich aus.
    |||
    |---------------------------------------------------|----------------------------------------------------|
    | 200 Epochs - 4 Neurons - \alpha 100     RES     |                                                           |
    | ![](./figures/connectivity/training_lineplot.png) | ![](./figures/connectivity/training_particle_type_lp.png) |
    | OTHER FUNTIONS                                    |                  IDENTITY FUNCTIONS                       |
    | ![](./figures/connectivity/other.png)             | ![](./figures/connectivity/identity.png)                  |
 - [ ] Training mit kleineren GNs
 - [ ] Weiter Trainieren -> 500 Epochs?
 - [x] Training ohne Residual Skip Connection
  - Ist anders:
     Self Training wird zunächst priorisiert, dann kommt langsam der eigentliche Task durch:
      | No Residual Skip connections 8 Neurons in SRNN  Alpha=100                                    | Residual Skip connections 8 Neurons in SRNN    Alpha=100                                 |
      |------------------------------------------------------------------------------------------|----------------------------------------------------------------------------------------------|
      | ![LinePlot](./figures/res_no_res/mn_st_200_8_alpha_100_no_res_training_particle_type_lp.png) | ![image info](./figures/res_no_res/mn_st_200_8_alpha_100_training_particle_type_lp.png)  |
      | ![image info](./figures/res_no_res/mn_st_200_8_alpha_100_no_res_training_lineplot.png)       | ![image info](./figures/res_no_res/mn_st_200_8_alpha_100_training_lineplot.png)          |
 - [ ] Test mit Baseline Dense Network 
  - [ ] mit vergleichbaren Neuron Count
  - [ ] mit gesamt Weight Count
 - [ ] Task/Goal statt SRNN-Task
 ---
 ### Für Menschen mit zu viel Zeit:
 - [ ] Sparse Network Training der Self Replication
  - Just for the lulz and speeeeeeed)
  - (Spaß bei Seite, wäre wichtig für schnellere Forschung)
    <https://pytorch.org/docs/stable/sparse.html>
--- a/code/experiment.py
+++ b/code/experiment.py
@ -1,277 +0,0 @@
 import os
 import time
 import dill
 from tqdm import tqdm
 from copy import copy
 from tensorflow.python.keras import backend as K
 from abc import ABC, abstractmethod
 class IllegalArgumentError(ValueError):
    pass
 class Experiment(ABC):
    @staticmethod
    def from_dill(path):
        with open(path, "rb") as dill_file:
            return dill.load(dill_file)
    @staticmethod
    def reset_model():
        K.clear_session()
    def __init__(self, name=None, ident=None, **kwargs):
        self.experiment_id = f'{ident or ""}_{time.time()}'
        self.experiment_name = name or 'unnamed_experiment'
        self.iteration = 0
        self.log_messages = list()
        self.historical_particles = dict()
        self.params = dict(exp_iterations=100, application_steps=100, prints=True, trains_per_application=100)
        self.with_params(**kwargs)
    def __copy__(self, *args, **kwargs):
        params = self.params
        params.update(name=self.experiment_name)
        params.update(**kwargs)
        self_copy = self.__class__(*args, **params)
        return self_copy
    def __enter__(self):
        self.dir = os.path.join('experiments', f'exp-{self.experiment_name}-{self.experiment_id}-{self.iteration}')
        os.makedirs(self.dir)
        print(f'** created {self.dir} **')
        return self
    def __exit__(self, exc_type, exc_value, traceback):
        self.save(experiment=self.without_particles())
        self.save_log()
        # Clean Exit
        self.reset_all()
        # self.iteration += 1  Taken From here!
    def with_params(self, **kwargs):
        # Make them your own
        self.params.update(kwargs)
        return self
    def log(self, message, **kwargs):
        self.log_messages.append(message)
        print(message, **kwargs)
    def save_log(self, log_name="log"):
        with open(os.path.join(self.dir, f"{log_name}.txt"), "w") as log_file:
            for log_message in self.log_messages:
                print(str(log_message), file=log_file)
    def without_particles(self):
        self_copy = copy(self)
        # Check if attribute exists
        if hasattr(self, 'historical_particles'):
            # Check if it is empty.
            if self.historical_particles:
                # Do the Update
                # self_copy.particles = [particle.states for particle in self.particles]
                self_copy.historical_particles = {key: val.states for key, val in self.historical_particles.items()}
        return self_copy
    def save(self, **kwargs):
        for name, value in kwargs.items():
            with open(os.path.join(self.dir, f"{name}.dill"), "wb") as dill_file:
                dill.dump(value, dill_file)
    def reset_log(self):
        self.log_messages = list()
    @abstractmethod
    def run_net(self, net, **kwargs):
        raise NotImplementedError
        pass
    def run_exp(self, network_generator, reset_model=False, **kwargs):
        # INFO Run_ID needs to be more than 0, so that exp stores the trajectories!
        for run_id in range(self.params.get('exp_iterations')):
            network = network_generator()
            self.run_net(network, **kwargs)
            self.historical_particles[run_id] = network
            if self.params.get('prints'):
                print("Fixpoint? " + str(network.is_fixpoint()))
            self.iteration += 1
            if reset_model:
                self.reset_model()
    def reset_all(self):
        self.reset_log()
        self.reset_model()
 class FixpointExperiment(Experiment):
    def __init__(self, **kwargs):
        kwargs['name'] = self.__class__.__name__ if 'name' not in kwargs else kwargs['name']
        super().__init__(**kwargs)
        self.counters = dict(divergent=0, fix_zero=0, fix_other=0, fix_sec=0, other=0)
        self.interesting_fixpoints = []
    def run_exp(self, network_generator, logging=True, reset_model=False, **kwargs):
        kwargs.update(reset_model=False)
        super(FixpointExperiment, self).run_exp(network_generator, **kwargs)
        if logging:
            self.log(self.counters)
        if reset_model:
            self.reset_model()
    def run_net(self, net, **kwargs):
        if len(kwargs):
            raise IllegalArgumentError
        for i in range(self.params.get('application_steps')):
            if net.is_diverged() or net.is_fixpoint():
                break
            net.set_weights(net.apply_to_weights(net.get_weights()))
            if self.iteration and hasattr(self, 'save_state'):
                net.save_state(time=i)
        self.count(net)
    def count(self, net):
        if net.is_diverged():
            self.counters['divergent'] += 1
        elif net.is_fixpoint():
            if net.is_zero():
                self.counters['fix_zero'] += 1
            else:
                self.counters['fix_other'] += 1
                self.interesting_fixpoints.append(net.get_weights())
        elif net.is_fixpoint(2):
            self.counters['fix_sec'] += 1
        else:
            self.counters['other'] += 1
    def reset_counters(self):
        for key in self.counters.keys():
            self.counters[key] = 0
        return True
    def reset_all(self):
        super(FixpointExperiment, self).reset_all()
        self.reset_counters()
 class MixedFixpointExperiment(FixpointExperiment):
    def __init__(self, **kwargs):
        kwargs['name'] = self.__class__.__name__ if 'name' not in kwargs else kwargs['name']
        super(MixedFixpointExperiment, self).__init__(**kwargs)
    def run_net(self, net, **kwargs):
        assert hasattr(net, 'train'), 'This Network must be trainable, i.e. use the "TrainingNeuralNetworkDecorator"!'
        for application in range(self.params.get('application_steps')):
            epoch_num = self.params.get('trains_per_application') * application
            net.set_weights(net.apply_to_weights(net.get_weights()))
            if net.is_diverged() or net.is_fixpoint():
                break
            barformat = "Experiment Iteration: {postfix[iteration]} | "
            barformat += "Evolution Step:{postfix[step]}| "
            barformat += "Training Epoch:{postfix[epoch]}| "
            barformat += "Loss: {postfix[loss]} | {bar}"
            with tqdm(total=self.params.get('trains_per_application'),
                      postfix={'step': 0, 'loss': 0, 'iteration': self.iteration, 'epoch': 0, None: None},
                      bar_format=barformat) as bar:
                # This iterates for self.trains_per_application times, the addition is just for  epoch enumeration
                for epoch in range(epoch_num, epoch_num + self.params.get('trains_per_application')):
                    if net.is_diverged():
                        print('Network diverged to either inf or nan... breaking')
                        break
                    loss = net.train(epoch=epoch)
                    if epoch % 10 == 0:
                        bar.postfix.update(step=application, epoch=epoch, loss=loss, iteration=self.iteration)
                    bar.update()
                    epoch_num += 1
            if self.iteration and hasattr(net, 'save_sate'):
                net.save_state()
        self.count(net)
 class TaskExperiment(MixedFixpointExperiment):
    def __init__(self, **kwargs):
        kwargs['name'] = self.__class__.__name__ if 'name' not in kwargs else kwargs['name']
        super(TaskExperiment, self).__init__(**kwargs)
    def run_exp(self, network_generator, reset_model=False, logging=True, **kwargs):
        kwargs.update(reset_model=False, logging=logging)
        super(FixpointExperiment, self).run_exp(network_generator, **kwargs)
        if reset_model:
            self.reset_model()
        pass
    def run_net(self, net, **kwargs):
        assert hasattr(net, 'evaluate')
        super(TaskExperiment, self).run_net(net, **kwargs)
        # Get Performance without Training
        task_performance = net.evaluate(*net.get_samples(task_samples=True),
                                        batchsize=net.get_amount_of_weights())
        self_performance = net.evaluate(*net.get_samples(self_samples=True),
                                        batchsize=net.get_amount_of_weights())
        current_performance = dict(task_performance=task_performance,
                                   self_performance=self_performance,
                                   counters=self.counters, id=self.iteration)
        self.log(current_performance)
        pass
 class SoupExperiment(Experiment):
    def __init__(self, soup_generator, **kwargs):
        kwargs['name'] = self.__class__.__name__ if 'name' not in kwargs else kwargs['name']
        self.soup_generator = soup_generator
        super(SoupExperiment, self).__init__(**kwargs)
    def run_exp(self, network_generator, **kwargs):
        for i in range(self.params.get('exp_iterations')):
            soup = self.soup_generator()
            soup.seed()
            for _ in tqdm(range(self.params.get('application_steps'))):
                soup.evolve()
            self.log(soup.count())
            self.save(soup=soup.without_particles())
            K.clear_session()
    def run_net(self, net, **kwargs):
        raise NotImplementedError
        pass
 class TaskingSoupExperiment(Experiment):
    def __init__(self, soup_generator, **kwargs):
        kwargs['name'] = self.__class__.__name__ if 'name' not in kwargs else kwargs['name']
        super(TaskingSoupExperiment, self).__init__(**kwargs)
        self.soup_generator = soup_generator
    def __copy__(self):
        super(TaskingSoupExperiment, self).__copy__(self.soup_generator)
    def run_exp(self, **kwargs):
        for i in range(self.params.get('exp_iterations')):
            soup = self.soup_generator()
            soup.seed()
            for _ in tqdm(range(self.params.get('application_steps'))):
                soup.evolve()
            self.log(soup.count())
            self.save(soup=soup.without_particles())
            K.clear_session()
    def run_net(self, net, **kwargs):
        raise NotImplementedError()
        pass
 if __name__ == '__main__':
    pass
--- a/code/fixpoint-2.ipynb
+++ b/code/fixpoint-2.ipynb
--- a/code/network.py
+++ b/code/network.py
@ -1,672 +0,0 @@
 # Librarys
 import numpy as np
 from abc import abstractmethod, ABC
 from typing import List, Tuple
 from types import FunctionType
 import warnings
 import os
 # Functions and Operators
 from operator import mul
 from functools import reduce
 from itertools import accumulate
 from copy import deepcopy
 # Deep learning Framework
 from tensorflow.python.keras.models import Sequential
 from tensorflow.python.keras.callbacks import Callback
 from tensorflow.python.keras.layers import SimpleRNN, Dense
 # Experiment Class
 from task import TaskAdditionOfN
 from experiment import TaskExperiment
 # Supress warnings and info messages
 os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2'
 class SaveStateCallback(Callback):
    def __init__(self, network, epoch=0):
        super(SaveStateCallback, self).__init__()
        self.net = network
        self.init_epoch = epoch
    def on_epoch_end(self, epoch, logs=None):
        description = dict(time=epoch+self.init_epoch)
        description['action'] = 'train_self'
        description['counterpart'] = None
        self.net.save_state(**description)
        return
 class EarlyStoppingByInfNanLoss(Callback):
    def __init__(self, monitor='loss', verbose=0):
        super(Callback, self).__init__()
        self.monitor = monitor
        self.verbose = verbose
    def on_epoch_end(self, epoch, logs: dict = None):
        logs = logs or dict()
        current = logs.get(self.monitor)
        if current is None:
            warnings.warn(f'Early stopping requires {self.monitor} available!', RuntimeWarning)
            pass
        if np.isnan(current) or np.isinf(current):
            if self.verbose > 0:
                print(f'Epoch {epoch}: early stopping THR')
            self.model.stop_training = True
 class NeuralNetwork(ABC):
    """
    This is the Base Network Class, including abstract functions that must be implemented.
    """
    @staticmethod
    def are_weights_diverged(weights: List[np.ndarray]) -> bool:
        return any([any((np.isnan(x).any(), np.isinf(x).any())) for x in weights])
    @staticmethod
    def are_weights_within_bounds(weights: List[np.ndarray], lower_bound: float, upper_bound: float) -> bool:
        return any([((lower_bound < x) & (x < upper_bound)).any() for x in weights])
    @staticmethod
    def get_weight_amount(weights: List[np.ndarray]):
        return sum([x.size for x in weights])
    @staticmethod
    def shapes(weights: List[np.ndarray]):
        return [x.shape for x in weights]
    @staticmethod
    def num_layers(weights: List[np.ndarray]):
        return len(weights)
    def repr(self, weights: List[np.ndarray]):
        return f'Weights({self.weights_to_flat_array(weights).tolist()})'
    @staticmethod
    def weights_to_flat_array(weights: List[np.ndarray]) -> np.ndarray:
        return np.concatenate([d.ravel() for d in weights])
    @staticmethod
    def reshape_flat_array(array, shapes: List[Tuple[int]]) -> List[np.ndarray]:
        # Same thing, but with an additional np call
        # sizes: List[int] = [int(np.prod(shape)) for shape in shapes]
        sizes = [reduce(mul, shape) for shape in shapes]
        # Split the incoming array into slices for layers
        slices = [array[x: y] for x, y in zip(accumulate([0] + sizes), accumulate(sizes))]
        # reshape them in accordance to the given shapes
        weights = [np.reshape(weight_slice, shape) for weight_slice, shape in zip(slices, shapes)]
        return weights
    def __init__(self, **params):
        super().__init__()
        self.params = dict(epsilon=0.00000000000001, early_nan_stopping=True, store_states=False)
        self.params.update(params)
        self.name = params.get('name', self.__class__.__name__)
        self.keras_params = dict(activation='linear', use_bias=False)
        self.states = []
        self.model: Sequential
    def get_params(self) -> dict:
        return self.params
    def get_keras_params(self) -> dict:
        return self.keras_params
    def with_params(self, **kwargs):
        self.params.update(kwargs)
        return self
    def with_keras_params(self, **kwargs):
        self.keras_params.update(kwargs)
        return self
    def print_weights(self, weights=None):
        print(self.repr(weights or self.get_weights()))
    def get_amount_of_weights(self):
        return self.get_weight_amount(self.get_weights())
    def get_model(self):
        return self.model
    def get_weights(self) -> List[np.ndarray]:
        return self.get_model().get_weights()
    def get_weights_flat(self) -> np.ndarray:
        return self.weights_to_flat_array(self.get_weights())
    def reshape_flat_array_like(self, array, weights: List[np.ndarray]) -> List[np.ndarray]:
        return self.reshape_flat_array(array, self.shapes(weights))
    def set_weights(self, new_weights: List[np.ndarray]):
        return self.model.set_weights(new_weights)
    def apply_to_network(self, other_network) -> List[np.ndarray]:
        """
        Take a networks weights and apply _this_ networks function.
        :param other_network:
        :return:
        """
        new_weights = self.apply_to_weights(other_network.get_weights())
        return new_weights
    def is_diverged(self):
        return self.are_weights_diverged(self.get_weights())
    def is_zero(self, epsilon=None):
        epsilon = epsilon or self.get_params().get('epsilon')
        return self.are_weights_within_bounds(self.get_weights(), -epsilon, epsilon)
    def is_fixpoint(self, degree: int = 1, epsilon: float = None) -> bool:
        assert degree >= 1, "degree must be >= 1"
        epsilon = epsilon or self.get_params().get('epsilon')
        new_weights = deepcopy(self.get_weights())
        for _ in range(degree):
            new_weights = self.apply_to_weights(new_weights)
            if self.are_weights_diverged(new_weights):
                return False
        flat_new = self.weights_to_flat_array(new_weights)
        flat_old = self.weights_to_flat_array(self.get_weights())
        biggerEpsilon = (np.abs(flat_new - flat_old) >= epsilon).any()
        # Boolean Value needs to be flipped to answer "is_fixpoint"
        return not biggerEpsilon
    def aggregate_weights_by(self, weights: List[np.ndarray], func: FunctionType, num_aggregates: int):
        collection_sizes = self.get_weight_amount(weights) // num_aggregates
        flat = self.weights_to_flat_array(weights)
        array_for_aggregation = flat[:collection_sizes * num_aggregates].reshape((num_aggregates, -1))
        left_overs = flat[collection_sizes * num_aggregates:]
        aggregated_weights = func(array_for_aggregation, num_aggregates)
        return aggregated_weights, left_overs
    def shuffle_weights(self, weights: List[np.ndarray]):
        flat = self.weights_to_flat_array(weights)
        np.random.shuffle(flat)
        return self.reshape_flat_array_like(flat, weights)
    @abstractmethod
    def get_samples(self, **kwargs):
        # TODO: add a dogstring, telling the user what this does, e.g. what is a sample?
        raise NotImplementedError
    @abstractmethod
    def apply_to_weights(self, old_weights) -> List[np.ndarray]:
        """
        Take weights as inputs; retunr the evaluation of _this_ network.
        "Apply this function".
        :param old_weights:
        :return:
        """
        raise NotImplementedError
 class ParticleDecorator:
    next_uid = 0
    def __init__(self, network):
        # ToDo: Add DocString, What does it do?
        self.uid = self.__class__.next_uid
        self.__class__.next_uid += 1
        self.network = network
        self.states = []
        self.save_state(time=0, action='init', counterpart=None)
    def __getattr__(self, name):
        return getattr(self.network, name)
    def get_uid(self):
        return self.uid
    def make_state(self, **kwargs):
        if self.network.is_diverged():
            return None
        state = {'class': self.network.__class__.__name__, 'weights': self.network.get_weights_flat()}
        state.update(kwargs)
        return state
    def save_state(self, **kwargs):
        state = self.make_state(**kwargs)
        if state is not None:
            self.states += [state]
        else:
            pass
        return True
    def update_state(self, number, **kwargs):
        raise NotImplementedError('Result is vague')
        # if number < len(self.states):
        #     self.states[number] = self.make_state(**kwargs)
        # else:
        #     for i in range(len(self.states), number):
        #         self.states += [None]
        #     self.states += self.make_state(**kwargs)
    def get_states(self):
        return self.states
    def attack(self, other_network, iterations: int = 1):
        """
        Set a networks weights based on the output of the application of my function to its weights.
        "Alter a networks weights based on my evaluation"
        :param other_network:
        :param iterations:
        :return:
        """
        for _ in range(iterations):
            other_network.set_weights(self.apply_to_network(other_network))
        return self
    def self_attack(self, iterations: int = 1):
        """
        Set my weights based on the output of the application of my function to its weights.
        "Alter my network weights based on my evaluation"
        :param iterations:
        :return:
        """
        for _ in range(iterations):
            self.attack(self)
        return self
 class TaskDecorator(TaskAdditionOfN):
    def __init__(self, network, **kwargs):
        super(TaskDecorator, self).__init__(**kwargs)
        self.network = network
        self.batchsize = self.network.get_amount_of_weights()
    def __getattr__(self, name):
        return getattr(self.network, name)
    def get_samples(self, task_samples=False, self_samples=False, **kwargs):
        # XOR, cannot be true at the same time
        assert not all([task_samples, self_samples])
        if task_samples:
            return super(TaskDecorator, self).get_samples()
        elif self_samples:
            return self.network.get_samples()
        else:
            self_x, self_y = self.network.get_samples()
            # Super class = Task
            task_x, task_y = super(TaskDecorator, self).get_samples()
            amount_of_weights = self.network.get_amount_of_weights()
            random_idx = np.random.choice(np.arange(amount_of_weights), amount_of_weights//2)
            x = self_x[random_idx] = task_x[random_idx]
            y = self_y[random_idx] = task_y[random_idx]
            return x, y
 class WeightwiseNeuralNetwork(NeuralNetwork):
    def __init__(self, width, depth, **kwargs):
        # ToDo: Insert Docstring
        super().__init__(**kwargs)
        self.width: int = width
        self.depth: int = depth
        self.model = Sequential()
        self.model.add(Dense(units=self.width, input_dim=4, **self.keras_params))
        for _ in range(self.depth-1):
            self.model.add(Dense(units=self.width, **self.keras_params))
        self.model.add(Dense(units=1, **self.keras_params))
    def apply(self, inputs):
        # TODO: Write about it... What does it do?
        return self.model.predict(inputs)
    def get_samples(self, **kwargs: List[np.ndarray]):
        weights = kwargs.get('weights', self.get_weights())
        sample = np.asarray([
            [weight, idx, *x] for idx, layer in enumerate(weights) for x, weight in np.ndenumerate(layer)
        ])
        # normalize [layer, cell, position]
        for idx in range(1, sample.shape[1]):
            sample[:, idx] = sample[:, idx] / np.max(sample[:, idx])
        return sample, sample[:, 0]
    def apply_to_weights(self, weights) -> List[np.ndarray]:
        # ToDo: Insert DocString
        # Transform the weight matrix in an horizontal stack as: array([[weight, layer, cell, position], ...])
        transformed_weights, _ = self.get_samples(weights=weights)
        new_flat_weights = self.apply(transformed_weights)
        # use the original weight shape to transform the new tensor
        return self.reshape_flat_array_like(new_flat_weights, weights)
 class AggregatingNeuralNetwork(NeuralNetwork):
    @staticmethod
    def aggregate_fft(array: np.ndarray, aggregates: int):
        flat = array.flatten()
        # noinspection PyTypeChecker
        fft_reduction = np.fft.fftn(flat, aggregates)
        return fft_reduction
    @staticmethod
    def aggregate_average(array, _):
        return np.average(array, axis=1)
    @staticmethod
    def aggregate_max(array, _):
        return np.max(array, axis=1)
    @staticmethod
    def deaggregate_identically(aggregate, amount):
        return np.repeat(aggregate, amount, axis=0)
    @staticmethod
    def shuffle_not(weights: List[np.ndarray]):
        """
        Doesn't do a thing. f(x)
        :param weights: A List of Weights
        :type weights: Weights
        :return: The same old weights.
        :rtype: Weights
        """
        return weights
    def shuffle_random(self, weights: List[np.ndarray]):
        weights = self.shuffle_weights(weights)
        return weights
    def __init__(self, aggregates, width, depth, **kwargs):
        super().__init__(**kwargs)
        self.aggregates = aggregates
        self.width = width
        self.depth = depth
        self.model = Sequential()
        self.model.add(Dense(units=width, input_dim=self.aggregates, **self.keras_params))
        for _ in range(depth-1):
            self.model.add(Dense(units=width, **self.keras_params))
        self.model.add(Dense(units=self.aggregates, **self.keras_params))
    def get_aggregator(self):
        return self.params.get('aggregator', self.aggregate_average)
    def get_deaggregator(self):
        return self.params.get('deaggregator', self.deaggregate_identically)
    def get_shuffler(self):
        return self.params.get('shuffler', self.shuffle_not)
    def apply(self, inputs):
        # You need to add an dimension here... "..." copies array values
        return self.model.predict(inputs[None, ...])
    def get_aggregated_weights(self):
        return self.aggregate_weights_by(self.get_weights(), self.get_aggregator(), self.aggregates)
    def apply_to_weights(self, old_weights) -> List[np.ndarray]:
        # build aggregations of old_weights
        old_aggregations, leftovers = self.get_aggregated_weights()
        # call network
        new_aggregations = self.apply(old_aggregations)
        collection_sizes = self.get_amount_of_weights() // self.aggregates
        new_aggregations = self.deaggregate_identically(new_aggregations, collection_sizes)
        # generate new weights
        # only include leftovers if there are some then coonvert them to Weight on base of th old shape
        complete_weights = new_aggregations if not leftovers.shape[0] else np.hstack((new_aggregations, leftovers))
        new_weights = self.reshape_flat_array_like(complete_weights, old_weights)
        # maybe shuffle
        new_weights = self.get_shuffler()(new_weights)
        return new_weights
    def get_samples(self, **kwargs):
        aggregations, _ = self.get_aggregated_weights()
        # What did that do?
        # sample = np.transpose(np.array([[aggregations[i]] for i in range(self.aggregates)]))
        return aggregations, aggregations
    def is_fixpoint_after_aggregation(self, degree=1, epsilon=None):
        assert degree >= 1, "degree must be >= 1"
        epsilon = epsilon or self.get_params().get('epsilon')
        old_aggregations, _ = self.get_aggregated_weights()
        new_weights = deepcopy(self.get_weights())
        for _ in range(degree):
            new_weights = self.apply_to_weights(new_weights)
            if self.are_weights_diverged(new_weights):
                return False
        new_aggregations, leftovers = self.get_aggregated_weights()
        # ToDo: Explain This, why are you additionally checking tolerances of aggregated weights?
        biggerEpsilon = (np.abs(np.asarray(old_aggregations) - np.asarray(new_aggregations)) >= epsilon).any()
        # Boolean value has to be flipped to answer the question.
        return True, not biggerEpsilon
 class RecurrentNeuralNetwork(NeuralNetwork):
    def __init__(self, width, depth, **kwargs):
        raise NotImplementedError
        super(RecurrentNeuralNetwork, self).__init__()
        self.features = 1
        self.width = width
        self.depth = depth
        self.model = Sequential()
        self.model.add(SimpleRNN(units=width, input_dim=self.features, return_sequences=True, **self.keras_params))
        for _ in range(depth-1):
            self.model.add(SimpleRNN(units=width, return_sequences=True, **self.keras_params))
        self.model.add(SimpleRNN(units=self.features, return_sequences=True, **self.keras_params))
    def apply(self, *inputs):
        stuff = np.transpose(np.array([[[inputs[i]] for i in range(len(inputs))]]))
        return self.model.predict(stuff)[0].flatten()
    def apply_to_weights(self, old_weights):
        # build list from old weights
        new_weights = deepcopy(old_weights)
        old_weights_list = []
        for layer_id, layer in enumerate(old_weights):
            for cell_id, cell in enumerate(layer):
                for weight_id, weight in enumerate(cell):
                    old_weights_list += [weight]
        # call network
        new_weights_list = self.apply(*old_weights_list)
        # write back new weights from list of rnn returns
        current_weight_id = 0
        for layer_id, layer in enumerate(new_weights):
            for cell_id, cell in enumerate(layer):
                for weight_id, weight in enumerate(cell):
                    new_weight = new_weights_list[current_weight_id]
                    new_weights[layer_id][cell_id][weight_id] = new_weight
                    current_weight_id += 1
        return new_weights
    def compute_samples(self):
        # build list from old weights
        old_weights_list = []
        for layer_id, layer in enumerate(self.get_weights()):
            for cell_id, cell in enumerate(layer):
                for weight_id, weight in enumerate(cell):
                    old_weights_list += [weight]
        sample = np.asarray(old_weights_list)[None, ..., None]
        return sample, sample
 class TrainingNeuralNetworkDecorator:
    def __init__(self, network):
        self.network = network
        self.compile_params = dict(loss='mse', optimizer='sgd')
        self.model_compiled = False
    def __getattr__(self, name):
        return getattr(self.network, name)
    def with_params(self, **kwargs):
        self.network.with_params(**kwargs)
        return self
    def with_keras_params(self, **kwargs):
        self.network.with_keras_params(**kwargs)
        return self
    def get_compile_params(self):
        return self.compile_params
    def with_compile_params(self, **kwargs):
        self.compile_params.update(kwargs)
        return self
    def compile_model(self, **kwargs):
        compile_params = deepcopy(self.compile_params)
        compile_params.update(kwargs)
        return self.network.model.compile(**compile_params)
    def compiled(self, **kwargs):
        if not self.model_compiled:
            self.compile_model(**kwargs)
            self.model_compiled = True
        return self
    def train(self, batchsize=1, epoch=0):
        self.compiled()
        x, y = self.network.get_samples()
        callbacks = []
        if self.get_params().get('store_states'):
            callbacks.append(SaveStateCallback(network=self, epoch=epoch))
        if self.get_params().get('early_nan_stopping'):
            callbacks.append(EarlyStoppingByInfNanLoss())
        # 'or' does not work on empty lists
        callbacks = callbacks if callbacks else None
        """
        Please Note:
        epochs: Integer. Number of epochs to train the model.
            An epoch is an iteration over the entire `x` and `y`
            data provided.
            Note that in conjunction with `initial_epoch`,
            `epochs` is to be understood as "final epoch".
            The model is not trained for a number of iterations
            given by `epochs`, but merely until the epoch
            of index `epochs` is reached."""
        history = self.network.model.fit(x=x, y=y, initial_epoch=epoch, epochs=epoch+1, verbose=0,
                                         batch_size=batchsize, callbacks=callbacks)
        return history.history['loss'][-1]
    def learn_from(self, other_network, batchsize=1):
        self.compiled()
        other_network.compiled()
        x, y = other_network.network.get_samples()
        history = self.network.model.fit(x=x, y=y, verbose=0, batch_size=batchsize)
        return history.history['loss'][-1]
    def evaluate(self, x=None, y=None, batchsize=1):
        self.compiled()
        x, y = x, y if x is not None and y is not None else self.network.get_samples()
        """
        Please Note:
        epochs: Integer. Number of epochs to train the model.
            An epoch is an iteration over the entire `x` and `y`
            data provided.
            Note that in conjunction with `initial_epoch`,
            `epochs` is to be understood as "final epoch".
            The model is not trained for a number of iterations
            given by `epochs`, but merely until the epoch
            of index `epochs` is reached."""
        loss = self.network.model.evaluate(x=x, y=y, verbose=0, batch_size=batchsize)
        return loss
 if __name__ == '__main__':
    if True:
        # WeightWise Neural Network
        with TaskExperiment().with_params(application_steps=10, trains_per_application=1000, exp_iterations=30) as exp:
            net_generator = lambda: TrainingNeuralNetworkDecorator(TaskDecorator(
                WeightwiseNeuralNetwork(width=2, depth=2))
            ).with_keras_params(activation='linear')
            exp.run_exp(net_generator, reset_model=True)
    if False:
        # Aggregating Neural Network
        net_generator = lambda: AggregatingNeuralNetwork(aggregates=4, width=2, depth=2)
        with MixedFixpointExperiment() as exp:
            exp.run_exp(net_generator, 10)
            exp.reset_all()
    if False:
        # FFT Aggregation
        net_generator = lambda: AggregatingNeuralNetwork(
            aggregates=4, width=2, depth=2, aggregator=AggregatingNeuralNetwork.aggregate_fft)
        with FixpointExperiment() as exp:
            exp.run_exp(net_generator, 10)
            exp.log(exp.counters)
            exp.reset_model()
            exp.reset_all()
    if False:
        # ok so this works quite realiably
        run_count = 1000
        net_generator = lambda: TrainingNeuralNetworkDecorator(WeightwiseNeuralNetwork(
            width=2, depth=2).with_params(epsilon=0.0001)).with_keras_params(optimizer='sgd')
        with MixedFixpointExperiment() as exp:
            for run_id in tqdm(range(run_count+1)):
                exp.run_exp(net_generator, 1)
                if run_id % 100 == 0:
                    exp.run_exp(net_generator, 1)
            K.clear_session()
    if False:
        with FixpointExperiment() as exp:
            run_count = 100
            net = TrainingNeuralNetworkDecorator(
                AggregatingNeuralNetwork(4, width=2, depth=2).with_params(epsilon=0.1e-6))
            for run_id in tqdm(range(run_count+1)):
                current_loss = net.compiled().train()
                if run_id % 100 == 0:
                    net.print_weights()
                    old_aggs, _ = net.get_aggregated_weights()
                    print("old weights agg: " + str(old_aggs))
                    fp, new_aggs = net.is_fixpoint_after_aggregation(epsilon=0.0001)
                    print("new weights agg: " + str(new_aggs))
                    print("Fixpoint? " + str(net.is_fixpoint()))
                    print("Fixpoint after Agg? " + str(fp))
                    print("Loss " + str(current_loss))
                    print()
    if False:
        # this explodes in our faces completely... NAN everywhere
        # TODO: Wtf is happening here?
        with FixpointExperiment() as exp:
            run_count = 10000
            net = TrainingNeuralNetworkDecorator(RecurrentNeuralNetwork(width=2, depth=2)
                                                 ).with_keras_params(optimizer='sgd', activation='linear')
            for run_id in tqdm(range(run_count+1)):
                current_loss = net.compiled().train()
                if run_id % 500 == 0:
                    net.print_weights()
                    # print(net.apply_to_network(net))
                    print("Fixpoint? " + str(net.is_fixpoint()))
                    print("Loss " + str(current_loss))
                    print()
--- a/code/plotting/plotting_class.py
+++ b/code/plotting/plotting_class.py
@ -1,66 +0,0 @@
 import os
 from experiment import Experiment
 # noinspection PyUnresolvedReferences
 from soup import Soup
 from argparse import ArgumentParser
 import numpy as np
 import plotly as pl
 import plotly.graph_objs as go
 import colorlover as cl
 import dill
 from sklearn.manifold.t_sne import TSNE, PCA
 def build_args():
    arg_parser = ArgumentParser()
    arg_parser.add_argument('-i', '--in_file', nargs=1, type=str)
    arg_parser.add_argument('-o', '--out_file', nargs='?', default='out', type=str)
    return arg_parser.parse_args()
 class DataPlotter:
    def __init__(self, path=None):
        self.path = path or os.getcwd()
        pass
    def search_and_apply(self, plotting_function, files_to_look_for=None, absolut_file_or_folder=None):
        absolut_file_or_folder, files_to_look_for = self.path or absolut_file_or_folder, list() or files_to_look_for
        if os.path.isdir(absolut_file_or_folder):
            for sub_file_or_folder in os.scandir(absolut_file_or_folder):
                self.search_and_apply(plotting_function, files_to_look_for=files_to_look_for,
                                      absolut_file_or_folder=sub_file_or_folder.path)
        elif absolut_file_or_folder.endswith('.dill'):
            file_or_folder = os.path.split(absolut_file_or_folder)[-1]
            if file_or_folder in files_to_look_for and not os.path.exists(
                    '{}.html'.format(absolut_file_or_folder[:-5])):
                print('Apply Plotting function "{func}" on file "{file}"'.format(func=plotting_function.__name__,
                                                                                 file=absolut_file_or_folder)
                      )
                with open(absolut_file_or_folder, 'rb') as in_f:
                    exp = dill.load(in_f)
                names_dill_location = os.path.join(*os.path.split(absolut_file_or_folder)[:-1], 'all_names.dill')
                with open(names_dill_location, 'rb') as in_f:
                    names = dill.load(in_f)
                try:
                    plotting_function((names, exp), filename='{}.html'.format(absolut_file_or_folder[:-5]))
                except ValueError:
                    pass
                except AttributeError:
                    pass
            else:
                # This was either another FilyType or Plot.html already exists.
                pass
 if __name__ == '__main__':
    plotter = DataPlotter
    pass
--- a/code/plotting/task_learning_curves.py
+++ b/code/plotting/task_learning_curves.py
@ -1,109 +0,0 @@
 import os
 from collections import defaultdict
 # noinspection PyUnresolvedReferences
 from soup import Soup
 from experiment import TaskExperiment
 from argparse import ArgumentParser
 import plotly as pl
 import plotly.graph_objs as go
 import colorlover as cl
 import dill
 import numpy as np
 def build_args():
    arg_parser = ArgumentParser()
    arg_parser.add_argument('-i', '--in_file', nargs=1, type=str)
    arg_parser.add_argument('-o', '--out_file', nargs='?', default='out', type=str)
    return arg_parser.parse_args()
 def line_plot(exp: TaskExperiment, filename='lineplot'):
    assert isinstance(exp, TaskExperiment), ' This has to be a TaskExperiment!'
    traces, data = [], defaultdict(list)
    color_scale = cl.scales['3']['div']['RdYlBu']
    # Sort data per Key
    for message in exp.log_messages:
        for key in message.keys():
            try:
                data[key].append(-0.1 if np.isnan(message[key]) or np.isinf(message[key]) else message[key])
            except:
                data[key].append(message[key])
    for line_id, key in enumerate(data.keys()):
        if key not in ['counters', 'id']:
            trace = go.Scatter(
                x=[x for x in range(len(data[key]))],
                y=data[key],
                name=key,
                line=dict(
                    color=color_scale[line_id],
                    width=5
                ),
            )
            traces.append(trace)
        else:
            continue
    layout = dict(xaxis=dict(title='Trains per self-application', titlefont=dict(size=20)),
                  yaxis=dict(title='Average amount of fixpoints found',
                             titlefont=dict(size=20),
                             # type='log',
                             # range=[0, 2]
                             ),
                  legend=dict(orientation='h', x=0.3, y=-0.3),
                  # height=800, width=800,
                  margin=dict(b=0)
                  )
    fig = go.Figure(data=traces, layout=layout)
    pl.offline.plot(fig, auto_open=True, filename=filename)
    pass
 def search_and_apply(absolut_file_or_folder, plotting_function, files_to_look_for=None, override=False):
    # ToDo: Clean this Mess
    assert os.path.exists(absolut_file_or_folder), f'The given path does not exist! Given: {absolut_file_or_folder}'
    files_to_look_for = files_to_look_for or list()
    if os.path.isdir(absolut_file_or_folder):
        for sub_file_or_folder in os.scandir(absolut_file_or_folder):
            search_and_apply(sub_file_or_folder.path, plotting_function,
                             files_to_look_for=files_to_look_for, override=override)
    elif absolut_file_or_folder.endswith('.dill'):
        file_or_folder = os.path.split(absolut_file_or_folder)[-1]
        if file_or_folder in files_to_look_for or not files_to_look_for:
            if not os.path.exists('{}.html'.format(absolut_file_or_folder[:-5])) or override:
                print('Apply Plotting function "{func}" on file "{file}"'.format(func=plotting_function.__name__,
                                                                                 file=absolut_file_or_folder)
                      )
                with open(absolut_file_or_folder, 'rb') as in_f:
                    exp = dill.load(in_f)
                try:
                    plotting_function(exp, filename='{}.html'.format(absolut_file_or_folder[:-5]))
                except ValueError:
                    pass
                except AttributeError:
                    pass
            else:
                # Plot.html already exists.
                pass
        else:
            # This was a wrong FilyType.
            pass
 if __name__ == '__main__':
    args = build_args()
    in_file = args.in_file[0]
    out_file = args.out_file
    search_and_apply(in_file, line_plot, override=True)
--- a/code/results/Soup/experiment.dill
+++ b/code/results/Soup/experiment.dill
--- a/code/results/Soup/log.txt
+++ b/code/results/Soup/log.txt
@ -1 +0,0 @@
 {'divergent': 0, 'fix_zero': 0, 'fix_other': 13, 'fix_sec': 0, 'other': 7}
--- a/code/results/Soup/soup.dill
+++ b/code/results/Soup/soup.dill
--- a/code/results/Soup/soup.html
+++ b/code/results/Soup/soup.html
--- a/code/results/Soup/weights.txt
+++ b/code/results/Soup/weights.txt
@ -1,30 +0,0 @@
 [-0.15321673  1.0428386  -0.7245892  -0.04343993  0.42338863  0.02538261
 -0.40465942 -0.0242596  -1.226809   -0.8168446   0.26588777 -1.0929432
  0.5383322  -0.73875046]
 [-0.03072096 -1.369665   -0.357126   -0.21180922  0.3853204   0.22853081
 -0.3705557  -0.21977347 -0.6684716   0.12849599  1.0226644  -0.0922638
 -0.7828449  -0.6572327 ]
 [-1.2444692   0.61213857  0.07965802  0.12361202  0.62641835  0.9720597
  0.3863232   0.59948945  1.0857513   0.49231085 -0.5319295   0.29433587
 -0.64177823  0.17603302]
 [-0.9938292  -0.4438207  -0.03172896  0.06261964 -0.3870194   0.7637992
  0.0244509  -0.04825407  0.91551745 -0.78740424  0.29226422 -0.52767307
 -0.41744384  0.5567152 ]
 [-0.39049304  0.8842579  -0.8447943  -0.19669186  0.7207061   0.16780053
  0.3728221   0.08680353  0.7535456  -0.1000197   0.02029054  0.8640245
 -0.15881588  1.1905665 ]
 [ 1.0482084   0.9248296  -0.26946014  0.57047915 -0.32660747  0.6914731
 -0.18025818  0.3816289  -0.69358927  0.21312684 -0.39932403 -0.02991759
 -0.83068466  0.45619962]
 [ 0.75814664  0.10328437  0.07867077 -0.0743314  -0.53440267  0.50492585
 -0.54172474  0.51184535  0.3462249   1.0527638  -0.9503541   0.9235086
 -0.1665241   1.1497779 ]
 [-0.77187353  1.1105504   0.24265823  0.53782856 -0.34098852 -0.75576884
 -0.25396293 -0.56288165  0.3851537  -0.67497945  0.14336896  0.763481
 -0.9224985   0.6374753 ]
 [-0.79123825  0.68166596 -0.30061013 -0.19360289  0.5632736   0.36276665
  0.7470975   0.48115698  0.10046808 -0.8064349  -1.036736   -0.68296516
 -1.156437    0.52633154]
 [ 0.1788832  -1.5321186  -0.62001514 -0.3870902   0.97524184  0.6088638
 -0.08297889 -0.05180515 -0.29096788  0.7519439   0.8803648   0.82771575
 -0.854887    0.1742936 ]
--- a/code/results/apply_fixpoints.png
+++ b/code/results/apply_fixpoints.png
--- a/code/results/exp-applying_fixpoint-_1552681870.3570378-0/all_counters.dill
+++ b/code/results/exp-applying_fixpoint-_1552681870.3570378-0/all_counters.dill
--- a/code/results/exp-applying_fixpoint-_1552681870.3570378-0/all_counters.html
+++ b/code/results/exp-applying_fixpoint-_1552681870.3570378-0/all_counters.html
--- a/code/results/exp-applying_fixpoint-_1552681870.3570378-0/all_names.dill
+++ b/code/results/exp-applying_fixpoint-_1552681870.3570378-0/all_names.dill
--- a/code/results/exp-applying_fixpoint-_1552681870.3570378-0/experiment.dill
+++ b/code/results/exp-applying_fixpoint-_1552681870.3570378-0/experiment.dill
--- a/code/results/exp-applying_fixpoint-_1552681870.3570378-0/log.txt
+++ b/code/results/exp-applying_fixpoint-_1552681870.3570378-0/log.txt
@ -1,12 +0,0 @@
 WeightwiseNeuralNetwork activiation='linear' use_bias=False
 {'divergent': 23, 'fix_zero': 27, 'fix_other': 0, 'fix_sec': 0, 'other': 0}
 AggregatingNeuralNetwork activiation='linear' use_bias=False
 {'divergent': 4, 'fix_zero': 46, 'fix_other': 0, 'fix_sec': 0, 'other': 0}
 RecurrentNeuralNetwork activiation='linear' use_bias=False
 {'divergent': 46, 'fix_zero': 4, 'fix_other': 0, 'fix_sec': 0, 'other': 0}
--- a/code/results/exp-applying_fixpoint-_1552681870.3570378-0/trajectorys.dill
+++ b/code/results/exp-applying_fixpoint-_1552681870.3570378-0/trajectorys.dill
--- a/code/results/exp-learn-from-soup-_1552658566.5572753-0/all_data.dill
+++ b/code/results/exp-learn-from-soup-_1552658566.5572753-0/all_data.dill
--- a/code/results/exp-learn-from-soup-_1552658566.5572753-0/all_data.html
+++ b/code/results/exp-learn-from-soup-_1552658566.5572753-0/all_data.html
--- a/code/results/exp-learn-from-soup-_1552658566.5572753-0/all_names.dill
+++ b/code/results/exp-learn-from-soup-_1552658566.5572753-0/all_names.dill
--- a/code/results/exp-learn-from-soup-_1552658566.5572753-0/experiment.dill
+++ b/code/results/exp-learn-from-soup-_1552658566.5572753-0/experiment.dill
--- a/code/results/exp-learn-from-soup-_1552658566.5572753-0/log.txt
+++ b/code/results/exp-learn-from-soup-_1552658566.5572753-0/log.txt
@ -1,4 +0,0 @@
 TrainingNeuralNetworkDecorator activiation='linear' use_bias=False
 {'xs': [0, 10, 20, 30, 40, 50, 60, 70, 80, 90, 100], 'ys': [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], 'zs': [0.0, 1.2, 5.2, 7.4, 8.1, 9.1, 9.6, 9.8, 10.0, 9.9, 9.9]}
--- a/code/results/exp-learn-from-soup-_1552658566.5572753-0/soup.dill
+++ b/code/results/exp-learn-from-soup-_1552658566.5572753-0/soup.dill
--- a/code/results/exp-learn-from-soup-_1552658566.5572753-0/soup.html
+++ b/code/results/exp-learn-from-soup-_1552658566.5572753-0/soup.html
--- a/code/results/exp-learn-from-soup-_1552658566.5572753-0/soup.png
+++ b/code/results/exp-learn-from-soup-_1552658566.5572753-0/soup.png
--- a/code/results/exp-mixed-self-fixpoints-_1552666977.5858653-0/all_data.dill
+++ b/code/results/exp-mixed-self-fixpoints-_1552666977.5858653-0/all_data.dill
--- a/code/results/exp-mixed-self-fixpoints-_1552666977.5858653-0/all_data.html
+++ b/code/results/exp-mixed-self-fixpoints-_1552666977.5858653-0/all_data.html
--- a/code/results/exp-mixed-self-fixpoints-_1552666977.5858653-0/all_names.dill
+++ b/code/results/exp-mixed-self-fixpoints-_1552666977.5858653-0/all_names.dill
--- a/code/results/exp-mixed-self-fixpoints-_1552666977.5858653-0/experiment.dill
+++ b/code/results/exp-mixed-self-fixpoints-_1552666977.5858653-0/experiment.dill
--- a/code/results/exp-mixed-self-fixpoints-_1552666977.5858653-0/log.txt
+++ b/code/results/exp-mixed-self-fixpoints-_1552666977.5858653-0/log.txt
@ -1,12 +0,0 @@
 WeightwiseNeuralNetwork activiation='linear' use_bias=False
 {'xs': [0, 50, 100, 150, 200, 250, 300, 350, 400, 450, 500], 'ys': [0.2, 0.3, 0.15, 0.55, 0.7, 0.85, 0.8, 0.95, 0.9, 1.0, 1.0]}
 AggregatingNeuralNetwork activiation='linear' use_bias=False
 {'xs': [0, 50, 100, 150, 200, 250, 300, 350, 400, 450, 500], 'ys': [1.0, 0.95, 1.0, 1.0, 0.95, 0.9, 0.8, 1.0, 0.85, 1.0, 0.9]}
 RecurrentNeuralNetwork activiation='linear' use_bias=False
 {'xs': [0, 50, 100, 150, 200, 250, 300, 350, 400, 450, 500], 'ys': [0.05, 0.0, 0.05, 0.0, 0.0, 0.1, 0.1, 0.05, 0.1, 0.0, 0.0]}
--- a/code/results/exp-mixed-soup-_1552674483.9866457-0/all_data.dill
+++ b/code/results/exp-mixed-soup-_1552674483.9866457-0/all_data.dill
--- a/code/results/exp-mixed-soup-_1552674483.9866457-0/all_data.html
+++ b/code/results/exp-mixed-soup-_1552674483.9866457-0/all_data.html
--- a/code/results/exp-mixed-soup-_1552674483.9866457-0/all_names.dill
+++ b/code/results/exp-mixed-soup-_1552674483.9866457-0/all_names.dill
--- a/code/results/exp-mixed-soup-_1552674483.9866457-0/experiment.dill
+++ b/code/results/exp-mixed-soup-_1552674483.9866457-0/experiment.dill
--- a/code/results/exp-mixed-soup-_1552674483.9866457-0/log.txt
+++ b/code/results/exp-mixed-soup-_1552674483.9866457-0/log.txt
@ -1,8 +0,0 @@
 TrainingNeuralNetworkDecorator activiation='linear' use_bias=False
 {'xs': [0, 10, 20, 30, 40, 50, 60, 70, 80, 90, 100], 'ys': [0.0, 0.0, 0.1, 0.0, 0.0, 0.0, 0.0, 0.1, 0.0, 0.0, 0.0], 'zs': [0.0, 0.0, 0.7, 1.9, 3.6, 4.3, 6.0, 6.1, 8.3, 7.7, 8.8]}
 TrainingNeuralNetworkDecorator activiation='linear' use_bias=False
 {'xs': [0, 10, 20, 30, 40, 50, 60, 70, 80, 90, 100], 'ys': [0.8, 0.4, 0.4, 0.3, 0.2, 0.2, 0.2, 0.2, 0.2, 0.4, 0.3], 'zs': [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]}
--- a/code/results/exp-training_fixpoint-_1552658296.0913951-0/all_counters.dill
+++ b/code/results/exp-training_fixpoint-_1552658296.0913951-0/all_counters.dill
--- a/code/results/exp-training_fixpoint-_1552658296.0913951-0/all_counters.html
+++ b/code/results/exp-training_fixpoint-_1552658296.0913951-0/all_counters.html
--- a/code/results/exp-training_fixpoint-_1552658296.0913951-0/all_names.dill
+++ b/code/results/exp-training_fixpoint-_1552658296.0913951-0/all_names.dill
--- a/code/results/exp-training_fixpoint-_1552658296.0913951-0/experiment.dill
+++ b/code/results/exp-training_fixpoint-_1552658296.0913951-0/experiment.dill
--- a/code/results/exp-training_fixpoint-_1552658296.0913951-0/log.txt
+++ b/code/results/exp-training_fixpoint-_1552658296.0913951-0/log.txt
@ -1,12 +0,0 @@
 WeightwiseNeuralNetwork activiation='linear' use_bias=False
 {'divergent': 0, 'fix_zero': 0, 'fix_other': 50, 'fix_sec': 0, 'other': 0}
 AggregatingNeuralNetwork activiation='linear' use_bias=False
 {'divergent': 0, 'fix_zero': 0, 'fix_other': 0, 'fix_sec': 0, 'other': 50}
 RecurrentNeuralNetwork activiation='linear' use_bias=False
 {'divergent': 38, 'fix_zero': 0, 'fix_other': 0, 'fix_sec': 0, 'other': 12}
--- a/code/results/exp-training_fixpoint-_1552658296.0913951-0/trajectorys.dill
+++ b/code/results/exp-training_fixpoint-_1552658296.0913951-0/trajectorys.dill
--- a/code/results/known_fixpoint_variation/experiment.dill
+++ b/code/results/known_fixpoint_variation/experiment.dill
--- a/code/results/known_fixpoint_variation/experiment.html
+++ b/code/results/known_fixpoint_variation/experiment.html
--- a/code/results/known_fixpoint_variation/log.txt
+++ b/code/results/known_fixpoint_variation/log.txt
@ -1,30 +0,0 @@
 variation 10e-0
 avg time to vergence 3.63
 avg time as fixpoint 0
 variation 10e-1
 avg time to vergence 5.02
 avg time as fixpoint 0
 variation 10e-2
 avg time to vergence 6.46
 avg time as fixpoint 0
 variation 10e-3
 avg time to vergence 8.04
 avg time as fixpoint 0
 variation 10e-4
 avg time to vergence 9.61
 avg time as fixpoint 0.04
 variation 10e-5
 avg time to vergence 11.23
 avg time as fixpoint 1.38
 variation 10e-6
 avg time to vergence 12.99
 avg time as fixpoint 3.23
 variation 10e-7
 avg time to vergence 14.58
 avg time as fixpoint 4.84
 variation 10e-8
 avg time to vergence 21.95
 avg time as fixpoint 11.91
 variation 10e-9
 avg time to vergence 26.45
 avg time as fixpoint 16.47
--- a/code/results/known_fixpoint_variation_box.png
+++ b/code/results/known_fixpoint_variation_box.png
--- a/code/results/learn_severity.png
+++ b/code/results/learn_severity.png
--- a/code/results/mixed_self_fixpoints.png
+++ b/code/results/mixed_self_fixpoints.png
--- a/code/results/mixed_soup.png
+++ b/code/results/mixed_soup.png
--- a/code/results/newplot
+++ b/code/results/newplot
--- a/code/results/newplot(2).png
+++ b/code/results/newplot(2).png
--- a/code/results/self_application_aggregation_network/experiment.dill
+++ b/code/results/self_application_aggregation_network/experiment.dill
--- a/code/results/self_application_aggregation_network/log.txt
+++ b/code/results/self_application_aggregation_network/log.txt
@ -1 +0,0 @@
 {'divergent': 0, 'fix_zero': 10, 'fix_other': 0, 'fix_sec': 0, 'other': 0}
--- a/code/results/self_application_aggregation_network/trajectorys.dill
+++ b/code/results/self_application_aggregation_network/trajectorys.dill
--- a/code/results/self_application_aggregation_network/trajectorys.html
+++ b/code/results/self_application_aggregation_network/trajectorys.html
--- a/code/results/self_apply1.png
+++ b/code/results/self_apply1.png
--- a/code/results/self_apply2.png
+++ b/code/results/self_apply2.png
--- a/code/results/self_train1.png
+++ b/code/results/self_train1.png
--- a/code/results/self_train2.png
+++ b/code/results/self_train2.png
--- a/code/results/self_training_weightwise_network/experiment.dill
+++ b/code/results/self_training_weightwise_network/experiment.dill
--- a/code/results/self_training_weightwise_network/trajectorys.dill
+++ b/code/results/self_training_weightwise_network/trajectorys.dill
--- a/code/results/self_training_weightwise_network/trajectorys.html
+++ b/code/results/self_training_weightwise_network/trajectorys.html
--- a/code/results/soup1.png
+++ b/code/results/soup1.png
--- a/code/results/soup2.png
+++ b/code/results/soup2.png
--- a/code/results/training_fixpoints.png
+++ b/code/results/training_fixpoints.png
--- a/code/setups/applying-fixpoints.py
+++ b/code/setups/applying-fixpoints.py
@ -1,70 +0,0 @@
 import sys
 import os
 # Concat top Level dir to system environmental variables
 sys.path += os.path.join('..', '.')
 from experiment import *
 from network import *
 def generate_counters():
    return {'divergent': 0, 'fix_zero': 0, 'fix_other': 0, 'fix_sec': 0, 'other': 0}
 def count(counters, net, notable_nets: list=None):
    notable_nets = notable_nets or list()
    if net.is_diverged():
        counters['divergent'] += 1
    elif net.is_fixpoint():
        if net.is_zero():
            counters['fix_zero'] += 1
        else:
            counters['fix_other'] += 1
            notable_nets += [net]
    elif net.is_fixpoint(2):
        counters['fix_sec'] += 1
        notable_nets += [net]
    else:
        counters['other'] += 1
    return counters, notable_nets
 if __name__ == '__main__':
    with FixpointExperiment(name='applying_fixpoint') as exp:
        exp.trials = 50
        exp.run_count = 100
        exp.epsilon = 1e-4
        net_generators = []
        for activation in ['linear']:  # , 'sigmoid', 'relu']:
            for use_bias in [False]:
                net_generators += [lambda activation=activation, use_bias=use_bias: WeightwiseNeuralNetwork(width=2, depth=2).with_keras_params(activation=activation, use_bias=use_bias)]
                net_generators += [lambda activation=activation, use_bias=use_bias: AggregatingNeuralNetwork(aggregates=4, width=2, depth=2).with_keras_params(activation=activation, use_bias=use_bias)]
        #       net_generators += [lambda activation=activation, use_bias=use_bias: RecurrentNeuralNetwork(width=2, depth=2).with_keras_params(activation=activation, use_bias=use_bias)]
        all_counters = []
        all_notable_nets = []
        all_names = []
        for net_generator_id, net_generator in enumerate(net_generators):
            counters = generate_counters()
            notable_nets = []
            for _ in tqdm(range(exp.trials)):
                net = ParticleDecorator(net_generator())
                net.with_params(epsilon=exp.epsilon)
                name = str(net.name) + " activiation='" + str(net.get_keras_params().get('activation')) + "' use_bias=" + str(net.get_keras_params().get('use_bias'))
                for run_id in range(exp.run_count):
                    loss = net.self_attack()
                count(counters, net, notable_nets)
            all_counters += [counters]
            all_notable_nets += [notable_nets]
            all_names += [name]
            exp.reset_model()
        exp.save(all_counters=all_counters)
        exp.save(trajectorys=exp.without_particles())
        # net types reached in the end
        # exp.save(all_notable_nets=all_notable_nets)
        exp.save(all_names=all_names) #experiment setups
        for exp_id, counter in enumerate(all_counters):
            exp.log(all_names[exp_id])
            exp.log(all_counters[exp_id])
            exp.log('\n')
--- a/code/setups/experiments/exp-training_fixpoint-_1552658296.0913951-0/all_counters.dill
+++ b/code/setups/experiments/exp-training_fixpoint-_1552658296.0913951-0/all_counters.dill
--- a/code/setups/experiments/exp-training_fixpoint-_1552658296.0913951-0/all_counters.html
+++ b/code/setups/experiments/exp-training_fixpoint-_1552658296.0913951-0/all_counters.html
--- a/code/setups/experiments/exp-training_fixpoint-_1552658296.0913951-0/all_names.dill
+++ b/code/setups/experiments/exp-training_fixpoint-_1552658296.0913951-0/all_names.dill
--- a/code/setups/experiments/exp-training_fixpoint-_1552658296.0913951-0/experiment.dill
+++ b/code/setups/experiments/exp-training_fixpoint-_1552658296.0913951-0/experiment.dill
--- a/code/setups/experiments/exp-training_fixpoint-_1552658296.0913951-0/log.txt
+++ b/code/setups/experiments/exp-training_fixpoint-_1552658296.0913951-0/log.txt
@ -1,12 +0,0 @@
 WeightwiseNeuralNetwork activiation='linear' use_bias=False
 {'divergent': 0, 'fix_zero': 0, 'fix_other': 50, 'fix_sec': 0, 'other': 0}
 AggregatingNeuralNetwork activiation='linear' use_bias=False
 {'divergent': 0, 'fix_zero': 0, 'fix_other': 0, 'fix_sec': 0, 'other': 50}
 RecurrentNeuralNetwork activiation='linear' use_bias=False
 {'divergent': 38, 'fix_zero': 0, 'fix_other': 0, 'fix_sec': 0, 'other': 12}
--- a/code/setups/experiments/exp-training_fixpoint-_1552658296.0913951-0/trajectorys.dill
+++ b/code/setups/experiments/exp-training_fixpoint-_1552658296.0913951-0/trajectorys.dill
--- a/code/setups/experiments/exp-weightwise_self_application-_1552664922.4501734-0/experiment.dill
+++ b/code/setups/experiments/exp-weightwise_self_application-_1552664922.4501734-0/experiment.dill
--- a/code/setups/experiments/exp-weightwise_self_application-_1552664922.4501734-0/log.txt
+++ b/code/setups/experiments/exp-weightwise_self_application-_1552664922.4501734-0/log.txt
@ -1 +0,0 @@
 {'divergent': 11, 'fix_zero': 9, 'fix_other': 0, 'fix_sec': 0, 'other': 0}
--- a/code/setups/experiments/exp-weightwise_self_application-_1552664922.4501734-0/trajectorys.dill
+++ b/code/setups/experiments/exp-weightwise_self_application-_1552664922.4501734-0/trajectorys.dill
--- a/code/setups/experiments/exp-weightwise_self_application-_1552664922.4501734-0/trajectorys.html
+++ b/code/setups/experiments/exp-weightwise_self_application-_1552664922.4501734-0/trajectorys.html
--- a/code/setups/fixpoint-density.py
+++ b/code/setups/fixpoint-density.py
@ -1,69 +0,0 @@
 import sys
 import os
 # Concat top Level dir to system environmental variables
 sys.path += os.path.join('..', '.')
 from experiment import *
 from network import *
 import tensorflow.python.keras.backend as K
 def generate_counters():
    return {'divergent': 0, 'fix_zero': 0, 'fix_other': 0, 'fix_sec': 0, 'other': 0}
 def count(counters, net, notable_nets=None):
    notable_nets = notable_nets or []
    if net.is_diverged():
        counters['divergent'] += 1
    elif net.is_fixpoint():
        if net.is_zero():
            counters['fix_zero'] += 1
        else:
            counters['fix_other'] += 1
            notable_nets += [net]
    elif net.is_fixpoint(2):
        counters['fix_sec'] += 1
        notable_nets += [net]
    else:
        counters['other'] += 1
    return counters, notable_nets
 if __name__ == '__main__':
    with Experiment('fixpoint-density') as exp:
        #NOTE: settings could/should stay this way
        #FFT doesn't work though
        exp.trials = 100000
        exp.epsilon = 1e-4
        net_generators = []
        for activation in ['linear']:
            net_generators += [lambda activation=activation: WeightwiseNeuralNetwork(width=2, depth=2).with_keras_params(activation=activation, use_bias=False)]
            net_generators += [lambda activation=activation: AggregatingNeuralNetwork(aggregates=4, width=2, depth=2).with_keras_params(activation=activation, use_bias=False)]
            # net_generators += [lambda activation=activation: FFTNeuralNetwork(aggregates=4, width=2, depth=2).with_keras_params(activation=activation, use_bias=False)]
            # net_generators += [lambda activation=activation: RecurrentNeuralNetwork(width=2, depth=2).with_keras_params(activation=activation, use_bias=False)]
        all_counters = []
        all_notable_nets = []
        all_names = []
        for net_generator_id, net_generator in enumerate(net_generators):
            counters = generate_counters()
            notable_nets = []
            for _ in tqdm(range(exp.trials)):
                net = net_generator().with_params(epsilon=exp.epsilon)
                net = ParticleDecorator(net)
                name = str(net.__class__.__name__) + " activiation='" + str(net.get_keras_params().get('activation')) + "' use_bias='" + str(net.get_keras_params().get('use_bias')) + "'"
                count(counters, net, notable_nets)
                K.clear_session()
            all_counters += [counters]
            # all_notable_nets += [notable_nets]
            all_names += [name]
        exp.save(all_counters=all_counters)
        exp.save(all_notable_nets=all_notable_nets)
        exp.save(all_names=all_names)
        for exp_id, counter in enumerate(all_counters):
            exp.log(all_names[exp_id])
            exp.log(all_counters[exp_id])
            exp.log('\n')
    print('Done')
--- a/code/setups/known-fixpoint-variation.py
+++ b/code/setups/known-fixpoint-variation.py
@ -1,92 +0,0 @@
 import sys
 import os
 # Concat top Level dir to system environmental variables
 sys.path += os.path.join('..', '.')
 from experiment import *
 from network import *
 from soup import prng
 import tensorflow.python.keras.backend as K
 from statistics import mean
 avg = mean
 def generate_fixpoint_weights():
    return [
        np.array([[1.0, 0.0], [0.0, 0.0], [0.0, 0.0], [0.0, 0.0]], dtype=np.float32),
        np.array([[1.0, 0.0], [0.0, 0.0]], dtype=np.float32),
        np.array([[1.0], [0.0]], dtype=np.float32)
    ]
 def generate_fixpoint_net():
    #NOTE: Weightwise only is all we can do right now IMO
    net = WeightwiseNeuralNetwork(width=2, depth=2).with_keras_params(activation='sigmoid')
    # I don't know if this work for aggregaeting. We don't actually need it, though.
    # net = AggregatingNeuralNetwork(aggregates=4, width=2, depth=2).with_keras_params(activation='sigmoid')
    net.set_weights(generate_fixpoint_weights())
    return net
 def vary(old_weights, e=1.0):
    new_weights = copy.deepcopy(old_weights)
    for layer_id, layer in enumerate(new_weights):
        for cell_id, cell in enumerate(layer):
            for weight_id, weight in enumerate(cell):
                if prng() < 0.5:
                    new_weights[layer_id][cell_id][weight_id] = weight + prng() * e
                else:
                    new_weights[layer_id][cell_id][weight_id] = weight - prng() * e
    return new_weights
 if __name__ == '__main__':
    with Experiment('known-fixpoint-variation') as exp:
        exp.depth = 10
        exp.trials = 100
        exp.max_steps = 100
        exp.epsilon = 1e-4
        exp.xs = []
        exp.ys = []
        exp.zs = []
        exp.notable_nets = []
        current_scale = 1.0
        for _ in range(exp.depth):
            print('variation scale ' + str(current_scale))
            for _ in tqdm(range(exp.trials)):
                net = generate_fixpoint_net().with_params(epsilon=exp.epsilon)
                net = ParticleDecorator(net)
                net.set_weights(vary(net.get_weights(), current_scale))
                time_to_something = 0
                time_as_fixpoint = 0
                still_fixpoint = True
                for _ in range(exp.max_steps):
                    net.self_attack()
                    if net.is_zero() or net.is_diverged():
                        break
                    if net.is_fixpoint():
                        if still_fixpoint:
                            time_as_fixpoint += 1
                        else:
                            print('remarkable')
                            exp.notable_nets += [net.get_weights()]
                            still_fixpoint = True
                    else:
                        still_fixpoint = False
                    time_to_something += 1
                exp.xs += [current_scale]
                # time steps taken to reach divergence or zero (reaching another fix-point is basically never happening)
                exp.ys += [time_to_something]
                # time steps still regarded as sthe initial fix-point
                exp.zs += [time_as_fixpoint]
                K.backend.clear_session()
            current_scale /= 10.0
        for d in range(exp.depth):
            exp.log('variation 10e-' + str(d))
            exp.log('avg time to vergence ' + str(avg(exp.ys[d*exp.trials:(d+1) * exp.trials])))
            exp.log('avg time as fixpoint ' + str(avg(exp.zs[d*exp.trials:(d+1) * exp.trials])))
--- a/code/setups/learn_from_soup.py
+++ b/code/setups/learn_from_soup.py
@ -1,110 +0,0 @@
 import sys
 import os
 # Concat top Level dir to system environmental variables
 sys.path += os.path.join('..', '.')
 from typing import Tuple
 from experiment import *
 from network import *
 from soup import *
 from tensorflow.python.keras import backend as K
 from statistics import mean
 avg = mean
 def generate_counters():
    """
    Initial build of the counter dict, to store counts.
    :rtype: dict
    :return: dictionary holding counter for: 'divergent', 'fix_zero', 'fix_sec', 'other'
    """
    return {'divergent': 0, 'fix_zero': 0, 'fix_other': 0, 'fix_sec': 0, 'other': 0}
 def count(counters, soup, notable_nets=None):
    """
    Count the occurences ot the types of weight trajectories.
    :param counters:      A counter dictionary.
    :param soup:          A Soup
    :param notable_nets:  A list to store and save intersting candidates
    :rtype      Tuple[dict, list]
    :return:    Both the counter dictionary and the list of interessting nets.
    """
    notable_nets = notable_nets or list()
    for net in soup.particles:
        if net.is_diverged():
            counters['divergent'] += 1
        elif net.is_fixpoint():
            if net.is_zero():
                counters['fix_zero'] += 1
            else:
                counters['fix_other'] += 1
                # notable_nets += [net]
        # elif net.is_fixpoint(2):
        #     counters['fix_sec'] += 1
        #     notable_nets += [net]
        else:
            counters['other'] += 1
    return counters, notable_nets
 if __name__ == '__main__':
    with SoupExperiment(name='learn-from-soup') as exp:
        exp.soup_size = 10
        exp.soup_life = 100
        exp.trials = 10
        exp.learn_from_severity_values = [10 * i for i in range(11)]
        exp.epsilon = 1e-4
        net_generators = []
        for activation in ['linear']:  # ['sigmoid', 'linear', 'relu']:
            for use_bias in [False]:
                net_generators += [lambda activation=activation, use_bias=use_bias: WeightwiseNeuralNetwork(width=2, depth=2).with_keras_params(activation=activation, use_bias=use_bias)]
                # net_generators += [lambda activation=activation, use_bias=use_bias: AggregatingNeuralNetwork(aggregates=4, width=2, depth=2).with_keras_params(activation=activation, use_bias=use_bias)]
                # net_generators += [lambda activation=activation, use_bias=use_bias: RecurrentNeuralNetwork(width=2, depth=2).with_keras_params(activation=activation, use_bias=use_bias)]
        all_names = []
        all_data = []
        for net_generator_id, net_generator in enumerate(net_generators):
            xs = []
            ys = []
            zs = []
            notable_nets = []
            for learn_from_severity in exp.learn_from_severity_values:
                counters = generate_counters()
                results = []
                for _ in tqdm(range(exp.trials)):
                    soup = Soup(exp.soup_size, lambda net_generator=net_generator, exp=exp: TrainingNeuralNetworkDecorator(net_generator()).with_params(epsilon=exp.epsilon))
                    soup.with_params(attacking_rate=-1, learn_from_rate=0.1, train=0, learn_from_severity=learn_from_severity)
                    soup.seed()
                    name = str(soup.particles[0].name) + " activiation='" + str(soup.particles[0].get_keras_params().get('activation')) + "' use_bias=" + str(soup.particles[0].get_keras_params().get('use_bias'))
                    for time in range(exp.soup_life):
                        soup.evolve()
                    count(counters, soup, notable_nets)
                    K.clear_session()
                xs += [learn_from_severity]
                ys += [float(counters['fix_zero']) / float(exp.trials)]
                zs += [float(counters['fix_other']) / float(exp.trials)]
            all_names += [name]
            # xs: learn_from_intensity according to exp.learn_from_intensity_values
            # ys: zero-fixpoints after life time
            # zs: non-zero-fixpoints after life time
            all_data += [{'xs':xs, 'ys':ys, 'zs':zs}]
        exp.save(all_names=all_names)
        exp.save(all_data=all_data)
        exp.save(soup=soup.without_particles())
        for exp_id, name in enumerate(all_names):
            exp.log(all_names[exp_id])
            exp.log(all_data[exp_id])
            exp.log('\n')
--- a/code/setups/mixed-self-fixpoints.py
+++ b/code/setups/mixed-self-fixpoints.py
@ -1,98 +0,0 @@
 import sys
 import os
 from typing import Tuple
 # Concat top Level dir to system environmental variables
 sys.path += os.path.join('..', '.')
 from experiment import *
 from network import *
 def generate_counters():
    """
    Initial build of the counter dict, to store counts.
    :rtype: dict
    :return: dictionary holding counter for: 'divergent', 'fix_zero', 'fix_sec', 'other'
    """
    return {'divergent': 0, 'fix_zero': 0, 'fix_other': 0, 'fix_sec': 0, 'other': 0}
 def count(counters, net, notable_nets=None):
    """
    Count the occurences ot the types of weight trajectories.
    :param counters:      A counter dictionary.
    :param net:           A Neural Network
    :param notable_nets:  A list to store and save intersting candidates
    :rtype      Tuple[dict, list]
    :return:    Both the counter dictionary and the list of interessting nets.
    """
    notable_nets = notable_nets or list()
    if net.is_diverged():
        counters['divergent'] += 1
    elif net.is_fixpoint():
        if net.is_zero():
            counters['fix_zero'] += 1
        else:
            counters['fix_other'] += 1
            notable_nets += [net]
    elif net.is_fixpoint(2):
        counters['fix_sec'] += 1
        notable_nets += [net]
    else:
        counters['other'] += 1
    return counters, notable_nets
 if __name__ == '__main__':
    with Experiment('mixed-self-fixpoints') as exp:
        exp.trials = 20
        exp.selfattacks = 4
        exp.trains_per_selfattack_values = [50 * i for i in range(11)]
        exp.epsilon = 1e-4
        net_generators = []
        for activation in ['linear']:  # , 'sigmoid', 'relu']:
            for use_bias in [False]:
                net_generators += [lambda activation=activation, use_bias=use_bias: WeightwiseNeuralNetwork(width=2, depth=2).with_keras_params(activation=activation, use_bias=use_bias)]
                net_generators += [lambda activation=activation, use_bias=use_bias: AggregatingNeuralNetwork(aggregates=4, width=2, depth=2).with_keras_params(activation=activation, use_bias=use_bias)]
                # net_generators += [lambda activation=activation, use_bias=use_bias: FFTNeuralNetwork(aggregates=4, width=2, depth=2).with_keras_params(activation=activation, use_bias=use_bias)]
                net_generators += [lambda activation=activation, use_bias=use_bias: RecurrentNeuralNetwork(width=2, depth=2).with_keras_params(activation=activation, use_bias=use_bias)]
        all_names = []
        all_data = []
        for net_generator_id, net_generator in enumerate(net_generators):
            xs = []
            ys = []
            for trains_per_selfattack in exp.trains_per_selfattack_values:
                counters = generate_counters()
                notable_nets = []
                for _ in tqdm(range(exp.trials)):
                    net = ParticleDecorator(net_generator())
                    net = TrainingNeuralNetworkDecorator(net).with_params(epsilon=exp.epsilon)
                    name = str(net.net.net.__class__.__name__) + " activiation='" + str(net.get_keras_params().get('activation')) + "' use_bias=" + str(net.get_keras_params().get('use_bias'))
                    for selfattack_id in range(exp.selfattacks):
                        net.self_attack()
                        for train_id in range(trains_per_selfattack):
                            loss = net.compiled().train(epoch=selfattack_id*trains_per_selfattack+train_id)
                        if net.is_diverged() or net.is_fixpoint():
                            break
                    count(counters, net, notable_nets)
                    exp.reset_model()
                xs += [trains_per_selfattack]
                ys += [float(counters['fix_zero'] + counters['fix_other']) / float(exp.trials)]
            all_names += [name]
            # xs: how many trains per self-attack from exp.trains_per_selfattack_values
            # ys: average amount of fixpoints found
            all_data += [{'xs': xs, 'ys': ys}]
        exp.save(all_names=all_names)
        exp.save(all_data=all_data)
        for exp_id, name in enumerate(all_names):
            exp.log(all_names[exp_id])
            exp.log(all_data[exp_id])
            exp.log('\n')
--- a/code/setups/mixed-soup.py
+++ b/code/setups/mixed-soup.py
@ -1,108 +0,0 @@
 import sys
 import os
 # Concat top Level dir to system environmental variables
 sys.path += os.path.join('..', '.')
 from typing import Tuple
 from experiment import *
 from network import *
 from soup import *
 import tensorflow.python.keras.backend as K
 def generate_counters():
    """
    Initial build of the counter dict, to store counts.
    :rtype: dict
    :return: dictionary holding counter for: 'divergent', 'fix_zero', 'fix_sec', 'other'
    """
    return {'divergent': 0, 'fix_zero': 0, 'fix_other': 0, 'fix_sec': 0, 'other': 0}
 def count(counters, soup, notable_nets=None):
    """
    Count the occurences ot the types of weight trajectories.
    :param counters:      A counter dictionary.
    :param soup:          A Soup
    :param notable_nets:  A list to store and save intersting candidates
    :rtype      Tuple[dict, list]
    :return:    Both the counter dictionary and the list of interessting nets.
    """
    notable_nets = notable_nets or list()
    for net in soup.particles:
        if net.is_diverged():
            counters['divergent'] += 1
        elif net.is_fixpoint():
            if net.is_zero():
                counters['fix_zero'] += 1
            else:
                counters['fix_other'] += 1
                # notable_nets += [net]
        # elif net.is_fixpoint(2):
        #     counters['fix_sec'] += 1
        #     notable_nets += [net]
        else:
            counters['other'] += 1
    return counters, notable_nets
 if __name__ == '__main__':
    with Experiment('mixed-soup') as exp:
        exp.trials = 10
        exp.soup_size = 10
        exp.soup_life = 5
        exp.trains_per_selfattack_values = [10 * i for i in range(11)]
        exp.epsilon = 1e-4
        net_generators = []
        for activation in ['linear']:  # ['linear', 'sigmoid', 'relu']:
            for use_bias in [False]:
                net_generators += [lambda activation=activation, use_bias=use_bias: WeightwiseNeuralNetwork(width=2, depth=2).with_keras_params(activation=activation, use_bias=use_bias)]
                net_generators += [lambda activation=activation, use_bias=use_bias: AggregatingNeuralNetwork(aggregates=4, width=2, depth=2).with_keras_params(activation=activation, use_bias=use_bias)]
                # net_generators += [lambda activation=activation, use_bias=use_bias: RecurrentNeuralNetwork(width=2, depth=2).with_keras_params(activation=activation, use_bias=use_bias)]
        all_names = []
        all_data = []
        for net_generator_id, net_generator in enumerate(net_generators):
            xs = []
            ys = []
            zs = []
            for trains_per_selfattack in exp.trains_per_selfattack_values:
                counters = generate_counters()
                notable_nets = []
                for soup_idx in tqdm(range(exp.trials)):
                    soup = Soup(exp.soup_size,
                                lambda net_generator=net_generator, exp=exp: TrainingNeuralNetworkDecorator(
                                    net_generator()).with_params(epsilon=exp.epsilon))
                    soup.with_params(attacking_rate=0.1, learn_from_rate=-1, train=trains_per_selfattack,
                                     learn_from_severity=-1)
                    soup.seed()
                    name = str(soup.particles[0].net.__class__.__name__) + " activiation='" + str(
                        soup.particles[0].get_keras_params().get('activation')) + "' use_bias=" + str(
                        soup.particles[0].get_keras_params().get('use_bias'))
                    for _ in range(exp.soup_life):
                        soup.evolve()
                    count(counters, soup, notable_nets)
                    K.clear_session()
                xs += [trains_per_selfattack]
                ys += [float(counters['fix_zero']) / float(exp.trials)]
                zs += [float(counters['fix_other']) / float(exp.trials)]
            all_names += [name]
            # xs: how many trains per self-attack from exp.trains_per_selfattack_values
            # ys: average amount of zero-fixpoints found
            # zs: average amount of non-zero fixpoints
            all_data += [{'xs': xs, 'ys': ys, 'zs': zs}]
        exp.save(all_names=all_names)
        exp.save(all_data=all_data)
        for exp_id, name in enumerate(all_names):
            exp.log(all_names[exp_id])
            exp.log(all_data[exp_id])
            exp.log('\n')
--- a/code/setups/network_trajectorys.py
+++ b/code/setups/network_trajectorys.py
@ -1,112 +0,0 @@
 import sys
 import os
 # Concat top Level dir to system environmental variables
 sys.path += os.path.join('..', '.')
 from soup import *
 from experiment import *
 if __name__ == '__main__':
    def run_exp(net, prints=False):
        # INFO Run_ID needs to be more than 0, so that exp stores the trajectories!
        exp.run_net(net, 100, run_id=run_id + 1)
        exp.historical_particles[run_id] = net
        if prints:
            print("Fixpoint? " + str(net.is_fixpoint()))
            print("Loss " + str(loss))
    if True:
        # WeightWise Neural Network
        with FixpointExperiment(name="weightwise_self_application") as exp:
            for run_id in tqdm(range(20)):
                net = ParticleDecorator(WeightwiseNeuralNetwork(width=2, depth=2)
                                        .with_keras_params(activation='linear'))
                run_exp(net)
                K.clear_session()
            exp.log(exp.counters)
            exp.save(trajectorys=exp.without_particles())
    if False:
        # Aggregating Neural Network
        with FixpointExperiment(name="aggregating_self_application") as exp:
            for run_id in tqdm(range(10)):
                net = ParticleDecorator(AggregatingNeuralNetwork(aggregates=4, width=2, depth=2)
                                        .with_keras_params(activation='linear'))
                run_exp(net)
                K.clear_session()
            exp.log(exp.counters)
            exp.save(trajectorys=exp.without_particles())
    if False:
        #FFT Neural Network
        with FixpointExperiment() as exp:
            for run_id in tqdm(range(10)):
                net = ParticleDecorator(FFTNeuralNetwork(aggregates=4, width=2, depth=2)
                                        .with_keras_params(activation='linear'))
                run_exp(net)
                K.clear_session()
            exp.log(exp.counters)
            exp.save(trajectorys=exp.without_particles())
    if False:
        # ok so this works quite realiably
        with FixpointExperiment(name="weightwise_learning") as exp:
            for i in range(10):
                run_count = 100
                net = TrainingNeuralNetworkDecorator(ParticleDecorator(WeightwiseNeuralNetwork(width=2, depth=2)))
                net.with_params(epsilon=0.0001).with_keras_params(activation='linear')
                exp.historical_particles[net.get_uid()] = net
                for run_id in tqdm(range(run_count+1)):
                    net.compiled()
                    loss = net.train(epoch=run_id)
                    # run_exp(net)
                    # net.save_state(time=run_id)
                K.clear_session()
            exp.save(trajectorys=exp.without_particles())
    if False:
        # ok so this works quite realiably
        with FixpointExperiment(name="aggregating_learning") as exp:
            for i in range(10):
                run_count = 100
                net = TrainingNeuralNetworkDecorator(ParticleDecorator(AggregatingNeuralNetwork(4, width=2, depth=2)))
                net.with_params(epsilon=0.0001).with_keras_params(activation='linear')
                exp.historical_particles[net.get_uid()] = net
                for run_id in tqdm(range(run_count+1)):
                    net.compiled()
                    loss = net.train(epoch=run_id)
                    # run_exp(net)
                    # net.save_state(time=run_id)
                K.clear_session()
            exp.save(trajectorys=exp.without_particles())
    if False:
        # this explodes in our faces completely... NAN everywhere
        # TODO: Wtf is happening here?
        with FixpointExperiment() as exp:
            run_count = 10000
            net = TrainingNeuralNetworkDecorator(RecurrentNeuralNetwork(width=2, depth=2))\
                .with_params(epsilon=0.1e-2).with_keras_params(optimizer='sgd', activation='linear')
            for run_id in tqdm(range(run_count+1)):
                loss = net.compiled().train()
                if run_id % 500 == 0:
                    net.print_weights()
                    # print(net.apply_to_network(net))
                    print("Fixpoint? " + str(net.is_fixpoint()))
                    print("Loss " + str(loss))
                    print()
    if False:
        # and this gets somewhat interesting... we can still achieve non-trivial fixpoints
        # over multiple applications when training enough in-between
        with MixedFixpointExperiment() as exp:
            for run_id in range(10):
                net = TrainingNeuralNetworkDecorator(FFTNeuralNetwork(2, width=2, depth=2))\
                    .with_params(epsilon=0.0001, activation='sigmoid')
                exp.run_net(net)
                net.print_weights()
                print("Fixpoint? " + str(net.is_fixpoint()))
            exp.log(exp.counters)
--- a/code/setups/soup_trajectorys.py
+++ b/code/setups/soup_trajectorys.py
@ -1,32 +0,0 @@
 import sys
 import os
 # Concat top Level dir to system environmental variables
 sys.path += os.path.join('..', '.')
 from soup import *
 from experiment import *
 if __name__ == '__main__':
    if True:
        with SoupExperiment(namne="soup") as exp:
            net_generator = lambda: TrainingNeuralNetworkDecorator(WeightwiseNeuralNetwork(2, 2)) \
                 .with_keras_params(activation='linear').with_params(epsilon=0.0001)
            # net_generator = lambda: TrainingNeuralNetworkDecorator(AggregatingNeuralNetwork(4, 2, 2))\
            #     .with_keras_params(activation='linear')
            # net_generator = lambda: TrainingNeuralNetworkDecorator(FFTNeuralNetwork(4, 2, 2))\
            #    .with_keras_params(activation='linear')
            # net_generator = lambda: RecurrentNeuralNetwork(2, 2).with_keras_params(activation='linear').with_params()
            soup = Soup(20, net_generator).with_params(remove_divergent=True, remove_zero=True,
                                                       train=30,
                                                       learn_from_rate=-1)
            soup.seed()
            for _ in tqdm(range(100)):
                soup.evolve()
            exp.log(soup.count())
            # you can access soup.historical_particles[particle_uid].states[time_step]['loss']
            #             or soup.historical_particles[particle_uid].states[time_step]['weights']
            # from soup.dill
            exp.save(soup=soup.without_particles())
            K.clear_session()
--- a/code/setups/training-fixpoints.py
+++ b/code/setups/training-fixpoints.py
@ -1,70 +0,0 @@
 import sys
 import os
 # Concat top Level dir to system environmental variables
 sys.path += os.path.join('..', '.')
 from experiment import *
 from network import *
 import tensorflow.python.keras.backend as K
 def generate_counters():
    return {'divergent': 0, 'fix_zero': 0, 'fix_other': 0, 'fix_sec': 0, 'other': 0}
 def count(counters, net, notable_nets=None):
    notable_nets = notable_nets or list()
    if net.is_diverged():
        counters['divergent'] += 1
    elif net.is_fixpoint():
        if net.is_zero():
            counters['fix_zero'] += 1
        else:
            counters['fix_other'] += 1
            notable_nets += [net]
    elif net.is_fixpoint(2):
        counters['fix_sec'] += 1
        notable_nets += [net]
    else:
        counters['other'] += 1
    return counters, notable_nets
 if __name__ == '__main__':
    with Experiment('training_fixpoint') as exp:
        exp.trials = 50
        exp.run_count = 1000
        exp.epsilon = 1e-4
        net_generators = []
        for activation in ['linear']:  # , 'sigmoid', 'relu']:
            for use_bias in [False]:
                net_generators += [lambda activation=activation, use_bias=use_bias: WeightwiseNeuralNetwork(width=2, depth=2).with_keras_params(activation=activation, use_bias=use_bias)]
                net_generators += [lambda activation=activation, use_bias=use_bias: AggregatingNeuralNetwork(aggregates=4, width=2, depth=2).with_keras_params(activation=activation, use_bias=use_bias)]
                net_generators += [lambda activation=activation, use_bias=use_bias: RecurrentNeuralNetwork(width=2, depth=2).with_keras_params(activation=activation, use_bias=use_bias)]
        all_counters = []
        all_notable_nets = []
        all_names = []
        for net_generator_id, net_generator in enumerate(net_generators):
            counters = generate_counters()
            notable_nets = []
            for _ in tqdm(range(exp.trials)):
                net = ParticleDecorator(net_generator())
                net = TrainingNeuralNetworkDecorator(net).with_params(epsilon=exp.epsilon)
                name = str(net.net.net.__class__.__name__) + " activiation='" + str(net.get_keras_params().get('activation')) + "' use_bias=" + str(net.get_keras_params().get('use_bias'))
                for run_id in range(exp.run_count):
                    loss = net.compiled().train(epoch=run_id+1)
                count(counters, net, notable_nets)
            all_counters += [counters]
            all_notable_nets += [notable_nets]
            all_names += [name]
            K.clear_session()
        exp.save(all_counters=all_counters)
        exp.save(trajectorys=exp.without_particles())
        # net types reached in the end
        # exp.save(all_notable_nets=all_notable_nets)
        exp.save(all_names=all_names) #experiment setups
        for exp_id, counter in enumerate(all_counters):
            exp.log(all_names[exp_id])
            exp.log(all_counters[exp_id])
            exp.log('\n')
--- a/code/soup.py
+++ b/code/soup.py
@ -1,376 +0,0 @@
 import random
 from tensorflow.python.keras.layers import Dense, Dropout, BatchNormalization
 from tensorflow.python.keras.layers import Input, Layer, Concatenate, RepeatVector, Reshape
 from tensorflow.python.keras.models import Sequential, Model
 from tensorflow.python.keras import backend as K
 from typing import List, Tuple
 # Functions and Operators
 from operator import mul
 from functools import reduce
 from itertools import accumulate
 import numpy as np
 from task import Task, TaskAdditionOfN
 from copy import copy, deepcopy
 from network import ParticleDecorator, WeightwiseNeuralNetwork, TrainingNeuralNetworkDecorator, \
    EarlyStoppingByInfNanLoss
 from experiment import TaskingSoupExperiment
 from math import sqrt
 def prng():
    return random.random()
 class SlicingLayer(Layer):
    def __init__(self):
        self.kernel: None
        self.inputs: int
        super(SlicingLayer, self).__init__()
    def build(self, input_shape):
        # Create a trainable weight variable for this layer.
        self.kernel = None
        self.inputs = input_shape[-1]
        super(SlicingLayer, self).build(input_shape)  # Be sure to call this at the end
    def call(self, x, **kwargs):
        concats = [Concatenate()([x[:, i][..., None]] * self.inputs) for i in range(x.shape[-1].value)]
        return concats
    def compute_output_shape(self, input_shape):
        return [Concatenate()([(None, 1)] * 4) for _ in range(input_shape[-1])]
 class Soup(object):
    def __init__(self, size, generator, **kwargs):
        self.size = size
        self.generator = generator
        self.particles = []
        self.historical_particles = {}
        self.soup_params = dict(attacking_rate=0.1, learn_from_rate=0.1, train=0, learn_from_severity=1)
        self.soup_params.update(kwargs)
        self.time = 0
        self.is_seeded = False
        self.is_compiled = False
    def __len__(self):
        return len(self.particles)
    def __copy__(self):
        copy_ = Soup(self.size, self.generator, **self.soup_params)
        copy_.__dict__ = {attr: self.__dict__[attr] for attr in self.__dict__ if
                          attr not in ['particles', 'historical_particles']}
        return copy_
    def without_particles(self):
        self_copy = copy(self)
        # self_copy.particles = [particle.states for particle in self.particles]
        self_copy.historical_particles = {key: val.states for key, val in self.historical_particles.items()}
        return self_copy
    def with_soup_params(self, **kwargs):
        self.soup_params.update(kwargs)
        return self
    def generate_particle(self):
        new_particle = ParticleDecorator(self.generator())
        self.historical_particles[new_particle.get_uid()] = new_particle
        return new_particle
    def get_particle(self, uid, otherwise=None):
        return self.historical_particles.get(uid, otherwise)
    def seed(self):
        if not self.is_seeded:
            self.particles = []
            for _ in range(self.size):
                self.particles += [self.generate_particle()]
        else:
            print('already seeded!')
        self.is_seeded = True
        return self
    def evolve(self, iterations=1):
        for _ in range(iterations):
            self.time += 1
            for particle_id, particle in enumerate(self.particles):
                description = {'time': self.time}
                if prng() < self.soup_params.get('attacking_rate'):
                    other_particle_id = int(prng() * len(self.particles))
                    other_particle = self.particles[other_particle_id]
                    particle.attack(other_particle)
                    description['action'] = 'attacking'
                    description['counterpart'] = other_particle.get_uid()
                if prng() < self.soup_params.get('learn_from_rate'):
                    other_particle_id = int(prng() * len(self.particles))
                    other_particle = self.particles[other_particle_id]
                    for _ in range(self.soup_params.get('learn_from_severity', 1)):
                        particle.learn_from(other_particle)
                    description['action'] = 'learn_from'
                    description['counterpart'] = other_particle.get_uid()
                for _ in range(self.soup_params.get('train', 0)):
                    # callbacks on save_state are broken for TrainingNeuralNetwork
                    loss = particle.train(store_states=False)
                    description['fitted'] = self.soup_params.get('train', 0)
                    description['loss'] = loss
                    description['action'] = 'train_self'
                    description['counterpart'] = None
                if self.soup_params.get('remove_divergent') and particle.is_diverged():
                    new_particle = self.generate_particle()
                    self.particles[particle_id] = new_particle
                    description['action'] = 'divergent_dead'
                    description['counterpart'] = new_particle.get_uid()
                if self.soup_params.get('remove_zero') and particle.is_zero():
                    new_particle = self.generate_particle()
                    self.particles[particle_id] = new_particle
                    description['action'] = 'zweo_dead'
                    description['counterpart'] = new_particle.get_uid()
                particle.save_state(**description)
    def count(self):
        counters = dict(divergent=0, fix_zero=0, fix_other=0, fix_sec=0, other=0)
        for particle in self.particles:
            if particle.is_diverged():
                counters['divergent'] += 1
            elif particle.is_fixpoint():
                if particle.is_zero():
                    counters['fix_zero'] += 1
                else:
                    counters['fix_other'] += 1
            elif particle.is_fixpoint(2):
                counters['fix_sec'] += 1
            else:
                counters['other'] += 1
        return counters
    def print_all(self):
        for particle in self.particles:
            particle.print_weights()
            print(particle.is_fixpoint())
 class TaskingSoup(Soup):
    @staticmethod
    def weights_to_flat_array(weights: List[np.ndarray]) -> np.ndarray:
        return np.concatenate([d.ravel() for d in weights])
    @staticmethod
    def reshape_flat_array(array, shapes: List[Tuple[int]]) -> List[np.ndarray]:
        # Same thing, but with an additional np call
        # sizes: List[int] = [int(np.prod(shape)) for shape in shapes]
        sizes = [reduce(mul, shape) for shape in shapes]
        # Split the incoming array into slices for layers
        slices = [array[x: y] for x, y in zip(accumulate([0] + sizes), accumulate(sizes))]
        # reshape them in accordance to the given shapes
        weights = [np.reshape(weight_slice, shape) for weight_slice, shape in zip(slices, shapes)]
        return weights
    def __init__(self, population_size: int, task: Task, particle_generator, sparsity_rate=0.1, use_bias=False,
                 safe=True, **kwargs):
        if safe:
            input_shape_error_message = f'The population size must be devideable by {task.input_shape[-1]}'
            assert population_size % task.input_shape[-1] == 0, input_shape_error_message
            assert population_size % 2 == 0, 'The population size needs to be of even value'
        super(TaskingSoup, self).__init__(population_size, particle_generator, **kwargs)
        self.task = task
        self.model: Sequential
        self.network_params = dict(sparsity_rate=sparsity_rate, early_nan_stopping=True, use_bias=use_bias,
                                   depth=population_size // task.input_shape[-1])
        self.network_params.update(kwargs.get('network_params', {}))
        self.compile_params = dict(loss='mse', optimizer='sgd')
        self.compile_params.update(kwargs.get('compile_params', {}))
    def with_network_params(self, **params):
        self.network_params.update(params)
    def _generate_model(self):
        particle_idx_list = list(range(len(self)))
        particles_per_layer = len(self) // self.network_params.get('depth')
        task_input = Input(self.task.input_shape, name='Task_Input')
        # First layer, which is conected to the input layer and independently trainable / not trainable at all.
        input_neurons = particles_per_layer * self.task.output_shape
        x = Dense(input_neurons, use_bias=self.network_params.get('use_bias'))(task_input)
        x = SlicingLayer()(x)
        for layer_num in range(self.network_params.get('depth')):
            # This needs to be tensors, because particles come as keras models that applicable
            x = [self.particles[layer_num*particles_per_layer + i].get_model()(x[i]) for
                 i in range(particles_per_layer)]
            x = [RepeatVector(particles_per_layer)(x[i]) for i in range(particles_per_layer)]
            x = [Reshape((particles_per_layer,))(x[i]) for i in range(particles_per_layer)]
        x = Concatenate()(x)
        x = Dense(self.task.output_shape, use_bias=self.network_params.get('use_bias'), activation='linear')(x)
        model = Model(inputs=task_input, outputs=x)
        return model
    def get_weights(self):
        return self.model.get_weights()
    def set_weights(self, weights: List[np.ndarray]):
        self.model.set_weights(weights)
    def set_intermediate_weights(self, weights: List[np.ndarray]):
        all_weights = self.get_weights()
        all_weights[1:-1] = weights
        self.set_weights(all_weights)
    def get_intermediate_weights(self):
        return self.get_weights()[1:-1]
    def seed(self):
        K.clear_session()
        self.is_compiled = False
        super(TaskingSoup, self).seed()
        self.model = self._generate_model()
        pass
    def compile_model(self, **kwargs):
        if not self.is_compiled:
            compile_params = deepcopy(self.compile_params)
            compile_params.update(kwargs)
            return self.model.compile(**compile_params)
        else:
            raise BrokenPipeError('This Model is not compiled yet! Something went wrong in the Pipeline!')
    def get_total_weight_amount(self):
        if self.is_seeded:
            return sum([x.get_amount_of_weights() for x in self.particles])
        else:
            return 0
    def get_shapes(self):
        return [x.shape for x in self.get_weights()]
    def get_intermediate_shapes(self):
        weights = [x.shape for x in self.get_weights()]
        return weights[2:-2] if self.network_params.get('use_bias') else weights[1:-1]
    def predict(self, x):
        return self.model.predict(x)
    def evolve(self, iterations=1):
        for iteration in range(iterations):
            super(TaskingSoup, self).evolve(iterations=1)
            self.train_particles()
    def get_particle_weights(self):
        return np.concatenate([x.get_weights_flat() for x in self.particles])
    def get_particle_input_shape(self):
        if self.is_seeded:
            return tuple([x if x else -1 for x in self.particles[0].get_model().input_shape])
        else:
            return -1
    def set_particle_weights(self, weights):
        particle_weight_shape = self.particles[0].shapes(self.particles[0].get_weights())
        sizes = [x.get_amount_of_weights() for x in self.particles]
        flat_weights = self.weights_to_flat_array(weights)
        slices = [flat_weights[x: y] for x, y in zip(accumulate([0] + sizes), accumulate(sizes))]
        for particle, slice in zip(self.particles, slices):
            new_weights = self.reshape_flat_array(slice, particle_weight_shape)
            particle.set_weights(new_weights)
        return True
    def compiled(self, **kwargs):
        if not self.is_compiled:
            self.compile_model(**kwargs)
            self.is_compiled = True
        return self
    def train(self, batchsize=1, epoch=0):
        self.compiled()
        x, y = self.task.get_samples()
        callbacks = []
        if self.network_params.get('early_nan_stopping'):
            callbacks.append(EarlyStoppingByInfNanLoss())
        # 'or' does not work on empty lists
        callbacks = callbacks if callbacks else None
        """
        Please Note:
        epochs: Integer. Number of epochs to train the model.
            An epoch is an iteration over the entire `x` and `y`
            data provided.
            Note that in conjunction with `initial_epoch`,
            `epochs` is to be understood as "final epoch".
            The model is not trained for a number of iterations
            given by `epochs`, but merely until the epoch
            of index `epochs` is reached."""
        history = self.model.fit(x=x, y=y, initial_epoch=epoch, epochs=epoch + 1, verbose=0,
                                 batch_size=batchsize, callbacks=callbacks)
        return history.history['loss'][-1]
    def train_particles(self, **kwargs):
        self.compiled()
        weights = self.get_particle_weights()
        shaped_weights = self.reshape_flat_array(weights, self.get_intermediate_shapes())
        self.set_intermediate_weights(shaped_weights)
        _ = self.train(**kwargs)  # This returns the loss values
        new_weights = self.get_intermediate_weights()
        self.set_particle_weights(new_weights)
        return
 if __name__ == '__main__':
    if True:
        from task import TaskAdditionOfN
        net_generator = lambda: TrainingNeuralNetworkDecorator(
            WeightwiseNeuralNetwork(2, 2).with_keras_params(activation='linear').with_params()
        )
        soup_generator = lambda: TaskingSoup(20, TaskAdditionOfN(4), net_generator)
        with TaskingSoupExperiment(soup_generator, name='solving_soup') as exp:
            exp.run_exp(reset_model=False)
    if False:
        soup_generator = lambda: Soup(10, net_generator).with_soup_params(remove_divergent=True, remove_zero=True)
        with SoupExperiment(soup_generator, name='soup') as exp:
            net_generator = lambda: TrainingNeuralNetworkDecorator(
                WeightwiseNeuralNetwork(2, 2).with_keras_params(activation='linear').with_params()
            )
            exp.run_exp(net_generator)
            # net_generator = lambda: FFTNeuralNetwork(2, 2).with_keras_params(activation='linear').with_params()
            # net_generator = lambda: AggregatingNeuralNetwork(4, 2, 2).with_keras_params(activation='sigmoid')\
            # .with_params(shuffler=AggregatingNeuralNetwork.shuffle_random)
            # net_generator = lambda: RecurrentNeuralNetwork(2, 2).with_keras_params(activation='linear').with_params()
    if False:
        soup_generator = lambda: Soup(10, net_generator).with_soup_params(remove_divergent=True, remove_zero=True)
        with SoupExperiment(soup_generator, name='soup') as exp:
            net_generator = lambda: TrainingNeuralNetworkDecorator(WeightwiseNeuralNetwork(2, 2)) \
                .with_keras_params(activation='linear').with_params(epsilon=0.0001)
            exp.run_exp(net_generator)
            # net_generator = lambda: TrainingNeuralNetworkDecorator(AggregatingNeuralNetwork(4, 2, 2))
            # .with_keras_params(activation='linear')\
            # .with_params(shuffler=AggregatingNeuralNetwork.shuffle_random)
            # net_generator = lambda: TrainingNeuralNetworkDecorator(FFTNeuralNetwork(4, 2, 2))\
            #     .with_keras_params(activation='linear')\
            # .with_params(shuffler=AggregatingNeuralNetwork.shuffle_random)
            # net_generator = lambda: RecurrentNeuralNetwork(2, 2).with_keras_params(activation='linear').with_params()
--- a/code/task.py
+++ b/code/task.py
@ -1,32 +0,0 @@
 from abc import ABC, abstractmethod
 import numpy as np
 from typing import Tuple
 class Task(ABC):
    def __init__(self, input_shape, output_shape, **kwargs):
        assert any([x not in kwargs.keys() for x in ["input_shape", "output_shape"]]), 'Dublicated arguments were given'
        self.input_shape = input_shape
        self.output_shape = output_shape
        self.batchsize = kwargs.get('batchsize', 100)
    def get_samples(self) -> Tuple[np.ndarray, np.ndarray]:
        raise NotImplementedError
 class TaskAdditionOfN(Task):
    def __init__(self, n: int, input_shape=(4,), output_shape=1, **kwargs):
        assert any([x not in kwargs.keys() for x in ["input_shape", "output_shape"]]), 'Dublicated arguments were given'
        assert n <= input_shape[0], f'You cannot Add more values (n={n}) than your input is long (in={input_shape}).'
        kwargs.update(input_shape=input_shape, output_shape=output_shape)
        super(TaskAdditionOfN, self).__init__(**kwargs)
        self.n = n
    def get_samples(self) -> Tuple[np.ndarray, np.ndarray]:
        x = np.zeros((self.batchsize, *self.input_shape))
        x[:, :self.n] = np.random.standard_normal((self.batchsize, self.n)) * 0.5
        y = np.sum(x, axis=1)
        return x, y
--- a/code/results/self_training_weightwise_network/log.txt
+++ b/code/results/self_training_weightwise_network/log.txt
--- a/experiments/helpers.py
+++ b/experiments/helpers.py
@ -0,0 +1,59 @@
 """ -------------------------------- Methods for summarizing the experiments --------------------------------- """
 from pathlib import Path
 from visualization import line_chart_fixpoints, bar_chart_fixpoints
 def summary_fixpoint_experiment(runs, population_size, epochs, experiments, net_learning_rate, directory,
                                summary_pre_title):
    avg_fixpoint_counters = {
        "avg_identity_func": 0,
        "avg_divergent": 0,
        "avg_fix_zero": 0,
        "avg_fix_weak": 0,
        "avg_fix_sec": 0,
        "avg_other_func": 0
    }
    for i in range(len(experiments)):
        fixpoint_counters = experiments[i].fixpoint_counters
        avg_fixpoint_counters["avg_identity_func"] += fixpoint_counters["identity_func"]
        avg_fixpoint_counters["avg_divergent"] += fixpoint_counters["divergent"]
        avg_fixpoint_counters["avg_fix_zero"] += fixpoint_counters["fix_zero"]
        avg_fixpoint_counters["avg_fix_weak"] += fixpoint_counters["fix_weak"]
        avg_fixpoint_counters["avg_fix_sec"] += fixpoint_counters["fix_sec"]
        avg_fixpoint_counters["avg_other_func"] += fixpoint_counters["other_func"]
    # Calculating the average for each fixpoint
    avg_fixpoint_counters.update((x, y / len(experiments)) for x, y in avg_fixpoint_counters.items())
    # Checking where the data is coming from to have a relevant title in the plot.
    if summary_pre_title not in ["ST", "SA", "soup", "mixed", "robustness"]:
        summary_pre_title = ""
    # Plotting the summary
    source_checker = "summary"
    exp_details = f"{summary_pre_title}: {runs} runs & {epochs} epochs each."
    bar_chart_fixpoints(avg_fixpoint_counters, population_size, directory, net_learning_rate, exp_details,
                        source_checker)
 def summary_fixpoint_percentage(runs, epochs, fixpoints_percentages, ST_steps, SA_steps, directory_name,
                                population_size):
    fixpoints_percentages = [round(fixpoints_percentages[i] / runs, 1) for i in range(len(fixpoints_percentages))]
    # Plotting summary
    if "soup" in directory_name:
        line_chart_fixpoints(fixpoints_percentages, epochs / ST_steps, ST_steps, SA_steps, directory_name,
                             population_size)
    else:
        line_chart_fixpoints(fixpoints_percentages, epochs, ST_steps, SA_steps, directory_name, population_size)
 """ -------------------------------------------- Miscellaneous --------------------------------------------------- """
 def check_folder(experiment_folder: str):
    exp_path = Path('experiments') / experiment_folder
    exp_path.mkdir(parents=True, exist_ok=True)
--- a/experiments/meta_task_small_utility.py
+++ b/experiments/meta_task_small_utility.py
@ -0,0 +1,38 @@
 import torch
 from torch.utils.data import Dataset
 DEVICE = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
 class AddTaskDataset(Dataset):
    def __init__(self, length=int(1e3)):
        super().__init__()
        self.length = length
    def __len__(self):
        return self.length
    def __getitem__(self, _):
        ab = torch.randn(size=(2,)).to(torch.float32)
        return ab, ab.sum(axis=-1, keepdims=True)
 def train_task(model, optimizer, loss_func, btch_x, btch_y) -> (dict, torch.Tensor):
    # Zero your gradients for every batch!
    optimizer.zero_grad()
    btch_x, btch_y = btch_x.to(DEVICE), btch_y.to(DEVICE)
    y_prd = model(btch_x)
    loss = loss_func(y_prd, btch_y.to(torch.float))
    loss.backward()
    # Adjust learning weights
    optimizer.step()
    stp_log = dict(Metric='Task Loss', Score=loss.item())
    return stp_log, y_prd
 if __name__ == '__main__':
    raise(NotImplementedError('Get out of here'))
--- a/experiments/meta_task_utility.py
+++ b/experiments/meta_task_utility.py
@ -0,0 +1,405 @@
 import pickle
 import re
 import shutil
 from collections import defaultdict
 from pathlib import Path
 import pandas as pd
 import numpy as np
 import torch
 import torchmetrics
 from matplotlib import pyplot as plt
 import seaborn as sns
 from torch.utils.data import Dataset
 from tqdm import tqdm
 from functionalities_test import test_for_fixpoints, FixTypes as ft
 from sanity_check_weights import test_weights_as_model, extract_weights_from_model
 WORKER = 10
 BATCHSIZE = 500
 DEVICE = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
 DATA_PATH = Path('data')
 DATA_PATH.mkdir(exist_ok=True, parents=True)
 PALETTE = sns.color_palette()
 PALETTE.insert(0, PALETTE.pop(1))  # Orange First
 FINAL_CHECKPOINT_NAME = f'trained_model_ckpt_FINAL.tp'
 class AddGaussianNoise(object):
    def __init__(self, ratio=1e-4):
        self.ratio = ratio
    def __call__(self, tensor: torch.Tensor):
        return tensor + (torch.randn_like(tensor, device=tensor.device) * self.ratio)
    def __repr__(self):
        return self.__class__.__name__ + f'(ratio={self.ratio}'
 class ToFloat:
    def __init__(self):
        pass
    def __call__(self, x):
        return x.to(torch.float32)
 class AddTaskDataset(Dataset):
    def __init__(self, length=int(5e5)):
        super().__init__()
        self.length = length
        self.prng = np.random.default_rng()
    def __len__(self):
        return self.length
    def __getitem__(self, _):
        ab = self.prng.normal(size=(2,)).astype(np.float32)
        return ab, ab.sum(axis=-1, keepdims=True)
 def set_checkpoint(model, out_path, epoch_n, final_model=False):
    if not final_model:
        epoch_n = str(epoch_n)
        ckpt_path = Path(out_path) / 'ckpt' / f'{epoch_n.zfill(4)}_model_ckpt.tp'
    else:
        if isinstance(epoch_n, str):
            ckpt_path = Path(out_path) / f'{Path(FINAL_CHECKPOINT_NAME).stem}_{epoch_n}.tp'
        else:
            ckpt_path = Path(out_path) / FINAL_CHECKPOINT_NAME
    ckpt_path.parent.mkdir(exist_ok=True, parents=True)
    torch.save(model, ckpt_path, pickle_protocol=pickle.HIGHEST_PROTOCOL)
    py_store_path = Path(out_path) / 'exp_py.txt'
    if not py_store_path.exists():
        shutil.copy(__file__, py_store_path)
    return ckpt_path
 # noinspection PyProtectedMember
 def validate(checkpoint_path, valid_loader, metric_class=torchmetrics.Accuracy):
    checkpoint_path = Path(checkpoint_path)
    # initialize metric
    validmetric = metric_class()
    model = torch.load(checkpoint_path, map_location=DEVICE).eval()
    with tqdm(total=len(valid_loader), desc='Validation Run: ') as pbar:
        for idx, (valid_batch_x, valid_batch_y) in enumerate(valid_loader):
            valid_batch_x, valid_batch_y = valid_batch_x.to(DEVICE), valid_batch_y.to(DEVICE)
            y_valid = model(valid_batch_x)
            # metric on current batch
            measure = validmetric(y_valid.cpu(), valid_batch_y.cpu())
            pbar.set_postfix_str(f'Measure: {measure}')
            pbar.update()
    # metric on all batches using custom accumulation
    measure = validmetric.compute()
    tqdm.write(f"Avg. {validmetric._get_name()} on all data: {measure}")
    return measure
 def new_storage_df(identifier, weight_count):
    if identifier == 'train':
        return pd.DataFrame(columns=['Epoch', 'Batch', 'Metric', 'Score'])
    elif identifier == 'weights':
        return pd.DataFrame(columns=['Epoch', 'Weight', *(f'weight_{x}' for x in range(weight_count))])
 def checkpoint_and_validate(model, valid_loader, out_path, epoch_n, keep_n=5, final_model=False,
                            validation_metric=torchmetrics.Accuracy):
    out_path = Path(out_path)
    ckpt_path = set_checkpoint(model, out_path, epoch_n, final_model=final_model)
    # Clean up Checkpoints
    if keep_n > 0:
        all_ckpts = sorted(list(ckpt_path.parent.iterdir()))
        while len(all_ckpts) > keep_n:
            all_ckpts.pop(0).unlink()
    elif keep_n == 0:
        pass
    else:
        raise ValueError(f'"keep_n" cannot be negative, but was: {keep_n}')
    result = validate(ckpt_path, valid_loader, metric_class=validation_metric)
    return result
 def plot_training_particle_types(path_to_dataframe):
    plt.close('all')
    plt.clf()
    # load from Drive
    df = pd.read_csv(path_to_dataframe, index_col=False).sort_values('Metric')
    # Set up figure
    fig, ax = plt.subplots()  # initializes figure and plots
    data = df.loc[df['Metric'].isin(ft.all_types())]
    fix_types = data['Metric'].unique()
    data = data.pivot(index='Epoch', columns='Metric', values='Score').reset_index().fillna(0)
    _ = plt.stackplot(data['Epoch'], *[data[fixtype] for fixtype in fix_types],
                      labels=fix_types.tolist(), colors=PALETTE)
    ax.set(ylabel='Particle Count', xlabel='Epoch')
    ax.yaxis.get_major_locator().set_params(integer=True)
    # ax.set_title('Particle Type Count')
    fig.legend(loc="center right", title='Particle Type', bbox_to_anchor=(0.85, 0.5))
    plt.tight_layout()
    plt.savefig(Path(path_to_dataframe.parent / 'training_particle_type_lp.png'), dpi=300)
 def plot_training_result(path_to_dataframe, metric_name='Accuracy', plot_name=None):
    plt.clf()
    # load from Drive
    df = pd.read_csv(path_to_dataframe, index_col=False).sort_values('Metric')
    # Check if this is a single lineplot or if aggregated
    group = ['Epoch', 'Metric']
    if 'Seed' in df.columns:
        group.append('Seed')
    # Set up figure
    fig, ax1 = plt.subplots()  # initializes figure and plots
    ax2 = ax1.twinx()  # applies twinx to ax2, which is the second y-axis.
    # plots the first set of data
    data = df[(df['Metric'] == 'Task Loss') | (df['Metric'] == 'Self Train Loss')].groupby(['Epoch', 'Metric']).mean()
    grouped_for_lineplot = data.groupby(group).mean()
    palette_len_1 = len(grouped_for_lineplot.droplevel(0).reset_index().Metric.unique())
    sns.lineplot(data=grouped_for_lineplot, x='Epoch', y='Score', hue='Metric',
                 palette=PALETTE[:palette_len_1], ax=ax1, ci='sd')
    # plots the second set of data
    data = df[(df['Metric'] == f'Test {metric_name}') | (df['Metric'] == f'Train {metric_name}')]
    palette_len_2 = len(data.Metric.unique())
    sns.lineplot(data=data, x='Epoch', y='Score', hue='Metric',
                 palette=PALETTE[palette_len_1:palette_len_2+palette_len_1], ci='sd')
    ax1.set(yscale='log', ylabel='Losses')
    # ax1.set_title('Training Lineplot')
    ax2.set(ylabel=metric_name)
    if metric_name != 'Accuracy':
        ax2.set(yscale='log')
    fig.legend(loc="center right", title='Metric', bbox_to_anchor=(0.85, 0.5))
    for ax in [ax1, ax2]:
        if legend := ax.get_legend():
            legend.remove()
    plt.tight_layout()
    plt.savefig(Path(path_to_dataframe.parent / ('training_lineplot.png' if plot_name is None else plot_name)), dpi=300)
 def plot_network_connectivity_by_fixtype(path_to_trained_model):
    m = torch.load(path_to_trained_model, map_location=DEVICE).eval()
    # noinspection PyProtectedMember
    particles = list(m.particles)
    df = pd.DataFrame(columns=['Type', 'Layer', 'Neuron', 'Name'])
    for prtcl in particles:
        l, c, w = [float(x) for x in re.sub("[^0-9|_]", "", prtcl.name).split('_')]
        df.loc[df.shape[0]] = (prtcl.is_fixpoint, l-1, w, prtcl.name)
        df.loc[df.shape[0]] = (prtcl.is_fixpoint, l, c, prtcl.name)
    for layer in list(df['Layer'].unique()):
        # Rescale
        divisor = df.loc[(df['Layer'] == layer), 'Neuron'].max()
        df.loc[(df['Layer'] == layer), 'Neuron'] /= divisor
    tqdm.write(f'Connectivity Data gathered')
    df = df.sort_values('Type')
    n = 0
    for fixtype in ft.all_types():
        if df[df['Type'] == fixtype].shape[0] > 0:
            plt.clf()
            ax = sns.lineplot(y='Neuron', x='Layer', hue='Name', data=df[df['Type'] == fixtype],
                              legend=False, estimator=None, lw=1)
            _ = sns.lineplot(y=[0, 1], x=[-1, df['Layer'].max()], legend=False, estimator=None, lw=0)
            ax.set_title(fixtype)
            ax.yaxis.get_major_locator().set_params(integer=True)
            ax.xaxis.get_major_locator().set_params(integer=True)
            ax.set_ylabel('Normalized Neuron Position (1/n)')  # XAXIS Label
            lines = ax.get_lines()
            for line in lines:
                line.set_color(PALETTE[n])
            plt.savefig(Path(path_to_trained_model.parent / f'net_connectivity_{fixtype}.png'), dpi=300)
            tqdm.write(f'Connectivity plottet: {fixtype} - n = {df[df["Type"] == fixtype].shape[0] // 2}')
            n += 1
        else:
            # tqdm.write(f'No Connectivity {fixtype}')
            pass
 # noinspection PyProtectedMember
 def run_particle_dropout_test(model_path, valid_loader, metric_class=torchmetrics.Accuracy):
    diff_store_path = model_path.parent / 'diff_store.csv'
    latest_model = torch.load(model_path, map_location=DEVICE).eval()
    prtcl_dict = defaultdict(lambda: 0)
    _ = test_for_fixpoints(prtcl_dict, list(latest_model.particles))
    tqdm.write(str(dict(prtcl_dict)))
    diff_df = pd.DataFrame(columns=['Particle Type', metric_class()._get_name(), 'Diff'])
    acc_pre = validate(model_path, valid_loader, metric_class=metric_class).item()
    diff_df.loc[diff_df.shape[0]] = ('All Organism', acc_pre, 0)
    for fixpoint_type in ft.all_types():
        new_model = torch.load(model_path, map_location=DEVICE).eval().replace_with_zero(fixpoint_type)
        if [x for x in new_model.particles if x.is_fixpoint == fixpoint_type]:
            new_ckpt = set_checkpoint(new_model, model_path.parent, fixpoint_type, final_model=True)
            acc_post = validate(new_ckpt, valid_loader, metric_class=metric_class).item()
            acc_diff = abs(acc_post - acc_pre)
            tqdm.write(f'Zero_ident diff = {acc_diff}')
            diff_df.loc[diff_df.shape[0]] = (fixpoint_type, acc_post, acc_diff)
    diff_df.to_csv(diff_store_path, mode='w', header=True, index=False)
    return diff_store_path
 # noinspection PyProtectedMember
 def plot_dropout_stacked_barplot(mdl_path, diff_store_path, metric_class=torchmetrics.Accuracy):
    metric_name = metric_class()._get_name()
    diff_df = pd.read_csv(diff_store_path).sort_values('Particle Type')
    particle_dict = defaultdict(lambda: 0)
    latest_model = torch.load(mdl_path, map_location=DEVICE).eval()
    _ = test_for_fixpoints(particle_dict, list(latest_model.particles))
    particle_dict = dict(particle_dict)
    sorted_particle_dict = dict(sorted(particle_dict.items()))
    tqdm.write(str(sorted_particle_dict))
    plt.clf()
    fig, ax = plt.subplots(ncols=2)
    colors = PALETTE.copy()
    colors.insert(0, colors.pop(-1))
    palette_len = len(diff_df['Particle Type'].unique())
    _ = sns.barplot(data=diff_df, y=metric_name, x='Particle Type', ax=ax[0], palette=colors[:palette_len], ci=None)
    ax[0].set_title(f'{metric_name} after particle dropout')
    # ax[0].set_xlabel('Particle Type')  # XAXIS Label
    ax[0].set_xticklabels(ax[0].get_xticklabels(), rotation=30)
    ax[1].pie(sorted_particle_dict.values(), labels=sorted_particle_dict.keys(),
              colors=PALETTE[:len(sorted_particle_dict)])
    ax[1].set_title('Particle Count')
    plt.tight_layout()
    plt.savefig(Path(diff_store_path.parent / 'dropout_stacked_barplot.png'), dpi=300)
 def run_particle_dropout_and_plot(model_path, valid_loader, metric_class=torchmetrics.Accuracy):
    diff_store_path = run_particle_dropout_test(model_path, valid_loader=valid_loader, metric_class=metric_class)
    plot_dropout_stacked_barplot(model_path, diff_store_path, metric_class=metric_class)
 def flat_for_store(parameters):
    return (x.item() for y in parameters for x in y.detach().flatten())
 def train_self_replication(model, st_stps, **kwargs) -> dict:
    self_train_loss = model.combined_self_train(st_stps, **kwargs)
    # noinspection PyUnboundLocalVariable
    stp_log = dict(Metric='Self Train Loss', Score=self_train_loss.item())
    return stp_log
 def train_task(model, optimizer, loss_func, btch_x, btch_y) -> (dict, torch.Tensor):
    # Zero your gradients for every batch!
    optimizer.zero_grad()
    btch_x, btch_y = btch_x.to(DEVICE), btch_y.to(DEVICE)
    y_prd = model(btch_x)
    loss = loss_func(y_prd, btch_y.to(torch.long))
    loss.backward()
    # Adjust learning weights
    optimizer.step()
    stp_log = dict(Metric='Task Loss', Score=loss.item())
    return stp_log, y_prd
 def highlight_fixpoints_vs_mnist_mean(mdl_path, dataloader):
    latest_model = torch.load(mdl_path, map_location=DEVICE).eval()
    activation_vector = torch.as_tensor([[0, 0, 0, 0, 1]], dtype=torch.float32, device=DEVICE)
    binary_images = []
    real_images = []
    with torch.no_grad():
        # noinspection PyProtectedMember
        for cell in latest_model._meta_layer_first.meta_cell_list:
            cell_image_binary = torch.zeros((len(cell.meta_weight_list)), device=DEVICE)
            cell_image_real = torch.zeros((len(cell.meta_weight_list)), device=DEVICE)
            for idx, particle in enumerate(cell.particles):
                if particle.is_fixpoint == ft.identity_func:
                    cell_image_binary[idx] += 1
                    cell_image_real[idx] = particle(activation_vector).abs().squeeze().item()
            binary_images.append(cell_image_binary.reshape((15, 15)))
            real_images.append(cell_image_real.reshape((15, 15)))
        binary_images = torch.stack(binary_images)
        real_images = torch.stack(real_images)
        binary_image = torch.sum(binary_images, keepdim=True, dim=0)
        real_image = torch.sum(real_images, keepdim=True, dim=0)
    mnist_images = [x for x, _ in dataloader]
    mnist_mean = torch.cat(mnist_images).reshape(10000, 15, 15).abs().sum(dim=0)
    fig, axs = plt.subplots(1, 3)
    for idx, (image, title) in enumerate(zip([binary_image, real_image, mnist_mean],
                                           ["Particle Count", "Particle Value", "MNIST mean"])):
        img = axs[idx].imshow(image.squeeze().detach().cpu())
        img.axes.axis('off')
        img.axes.set_title('Random Noise')
    plt.tight_layout()
    plt.savefig(mdl_path.parent / 'heatmap.png', dpi=300)
    plt.clf()
    plt.close('all')
 def plot_training_results_over_n_seeds(exp_path, df_train_store_name='train_store.csv', metric_name='Accuracy'):
    combined_df_store_path = exp_path / f'comb_train_{exp_path.stem[:-1]}n.csv'
    # noinspection PyUnboundLocalVariable
    found_train_stores = exp_path.rglob(df_train_store_name)
    train_dfs = []
    for found_train_store in found_train_stores:
        train_store_df = pd.read_csv(found_train_store, index_col=False)
        train_store_df['Seed'] = int(found_train_store.parent.name)
        train_dfs.append(train_store_df)
    combined_train_df = pd.concat(train_dfs)
    combined_train_df.to_csv(combined_df_store_path, index=False)
    plot_training_result(combined_df_store_path, metric_name=metric_name,
                         plot_name=f"{combined_df_store_path.stem}.png"
                         )
    plt.clf()
    plt.close('all')
 def sanity_weight_swap(exp_path, dataloader, metric_class=torchmetrics.Accuracy):
    # noinspection PyProtectedMember
    metric_name = metric_class()._get_name()
    found_models = exp_path.rglob(f'*{FINAL_CHECKPOINT_NAME}')
    df = pd.DataFrame(columns=['Seed', 'Model', metric_name])
    for model_idx, found_model in enumerate(found_models):
        model = torch.load(found_model, map_location=DEVICE).eval()
        weights = extract_weights_from_model(model)
        results = test_weights_as_model(model, weights, dataloader, metric_class=metric_class)
        for model_name, measurement in results.items():
            df.loc[df.shape[0]] = (model_idx, model_name, measurement)
        df.loc[df.shape[0]] = (model_idx, 'Difference', np.abs(np.subtract(*results.values())))
    df.to_csv(exp_path / 'sanity_weight_swap.csv', index=False)
    _ = sns.boxplot(data=df, x='Model', y=metric_name)
    plt.tight_layout()
    plt.savefig(exp_path / 'sanity_weight_swap.png', dpi=300)
    plt.clf()
    plt.close('all')
 if __name__ == '__main__':
    raise NotImplementedError('Test this here!!!')
--- a/experiments/robustness_tester.py
+++ b/experiments/robustness_tester.py
@ -0,0 +1,118 @@
 import pandas as pd
 import torch
 import random
 import copy
 from tqdm import tqdm
 from functionalities_test import (is_identity_function, is_zero_fixpoint, test_for_fixpoints, is_divergent,
                                  FixTypes as FT)
 from network import Net
 from torch.nn import functional as F
 import seaborn as sns
 from matplotlib import pyplot as plt
 def prng():
    return random.random()
 def generate_perfekt_synthetic_fixpoint_weights():
    return torch.tensor([[1.0], [0.0], [0.0], [0.0], [0.0], [0.0], [0.0], [0.0],
                         [1.0], [0.0], [0.0], [0.0],
                         [1.0], [0.0]
                         ], dtype=torch.float32)
 PALETTE = 10 * (
    "#377eb8",
    "#4daf4a",
    "#984ea3",
    "#e41a1c",
    "#ff7f00",
    "#a65628",
    "#f781bf",
    "#888888",
    "#a6cee3",
    "#b2df8a",
    "#cab2d6",
    "#fb9a99",
    "#fdbf6f",
 )
 def test_robustness(model_path, noise_levels=10, seeds=10, log_step_size=10):
    model = torch.load(model_path, map_location='cpu')
    networks = [x for x in model.particles if x.is_fixpoint == FT.identity_func]
    time_to_vergence = [[0 for _ in range(noise_levels)] for _ in range(len(networks))]
    time_as_fixpoint = [[0 for _ in range(noise_levels)] for _ in range(len(networks))]
    row_headers = []
    df = pd.DataFrame(columns=['setting', 'Noise Level', 'Self Train Steps', 'absolute_loss',
                               'Time to convergence', 'Time as fixpoint'])
    with tqdm(total=(seeds * noise_levels * len(networks)), desc='Per Particle Robustness') as pbar:
        for setting, fixpoint in enumerate(networks):  # 1 / n
            row_headers.append(fixpoint.name)
            for seed in range(seeds):  # n / 1
                for noise_level in range(noise_levels):
                    steps = 0
                    clone = Net(fixpoint.input_size, fixpoint.hidden_size, fixpoint.out_size,
                                f"{fixpoint.name}_clone_noise_1e-{noise_level}")
                    clone.load_state_dict(copy.deepcopy(fixpoint.state_dict()))
                    clone = clone.apply_noise(pow(10, -noise_level))
                    while not is_zero_fixpoint(clone) and not is_divergent(clone):
                        # -> before
                        clone_weight_pre_application = clone.input_weight_matrix()
                        target_data_pre_application = clone.create_target_weights(clone_weight_pre_application)
                        clone.self_application(1, log_step_size)
                        time_to_vergence[setting][noise_level] += 1
                        # -> after
                        clone_weight_post_application = clone.input_weight_matrix()
                        target_data_post_application = clone.create_target_weights(clone_weight_post_application)
                        absolute_loss = F.l1_loss(target_data_pre_application, target_data_post_application).item()
                        if is_identity_function(clone):
                            time_as_fixpoint[setting][noise_level] += 1
                            # When this raises a Type Error, we found a second order fixpoint!
                        steps += 1
                        df.loc[df.shape[0]] = [f'{setting}_{seed}', fr'$\mathregular{{10^{{-{noise_level}}}}}$',
                                               steps, absolute_loss,
                                               time_to_vergence[setting][noise_level],
                                               time_as_fixpoint[setting][noise_level]]
                    pbar.update(1)
    # Get the measuremts at the highest time_time_to_vergence
    df_sorted = df.sort_values('Self Train Steps', ascending=False).drop_duplicates(['setting', 'Noise Level'])
    df_melted = df_sorted.reset_index().melt(id_vars=['setting', 'Noise Level', 'Self Train Steps'],
                                             value_vars=['Time to convergence', 'Time as fixpoint'],
                                             var_name="Measurement",
                                             value_name="Steps").sort_values('Noise Level')
    df_melted.to_csv(model_path.parent / 'robustness_boxplot.csv', index=False)
    # Plotting
    # plt.rcParams.update({
    #    "text.usetex": True,
    #    "font.family": "sans-serif",
    #    "font.size": 12,
    #    "font.weight": 'bold',
    #    "font.sans-serif": ["Helvetica"]})
    plt.clf()
    sns.set(style='whitegrid', font_scale=1)
    _ = sns.boxplot(data=df_melted, y='Steps', x='Noise Level', hue='Measurement', palette=PALETTE)
    plt.tight_layout()
    # sns.set(rc={'figure.figsize': (10, 50)})
    # bx = sns.catplot(data=df[df['absolute_loss'] < 1], y='absolute_loss', x='application_step', kind='box',
    #                  col='noise_level', col_wrap=3, showfliers=False)
    filename = f"robustness_boxplot.png"
    filepath = model_path.parent / filename
    plt.savefig(str(filepath))
    plt.close('all')
    return time_as_fixpoint, time_to_vergence
 if __name__ == "__main__":
    raise NotImplementedError('Get out of here!')
--- a/figures/connectivity/identity.png
+++ b/figures/connectivity/identity.png
--- a/figures/connectivity/other.png
+++ b/figures/connectivity/other.png
--- a/figures/connectivity/training_lineplot.png
+++ b/figures/connectivity/training_lineplot.png
--- a/figures/connectivity/training_particle_type_lp.png
+++ b/figures/connectivity/training_particle_type_lp.png
--- a/figures/res_no_res/mn_st_200_8_alpha_100_no_res_training_lineplot.png
+++ b/figures/res_no_res/mn_st_200_8_alpha_100_no_res_training_lineplot.png
--- a/figures/res_no_res/mn_st_200_8_alpha_100_no_res_training_particle_type_lp.png
+++ b/figures/res_no_res/mn_st_200_8_alpha_100_no_res_training_particle_type_lp.png
--- a/figures/res_no_res/mn_st_200_8_alpha_100_training_lineplot.png
+++ b/figures/res_no_res/mn_st_200_8_alpha_100_training_lineplot.png
--- a/figures/res_no_res/mn_st_200_8_alpha_100_training_particle_type_lp.png
+++ b/figures/res_no_res/mn_st_200_8_alpha_100_training_particle_type_lp.png
--- a/Show More
+++ b/Show More
Author	SHA1	Message	Date
Steffen Illium	38a4af2baa	Update README.md	2023-11-14 16:25:07 +01:00
Steffen Illium	0ba3994325	Final Experiments and Plot adjustments	2022-03-12 11:39:28 +01:00
Steffen Illium	dd2458da4a	adjustments for cuda and auto ckpt cleanup	2022-03-07 11:29:06 +01:00
Steffen Illium	ce5a36c8f4	Late Evening Commit	2022-03-06 22:24:00 +01:00
Steffen Illium	b3d4987cb8	robustness	2022-03-05 17:32:30 +01:00
Steffen Illium	f3ff4c9239	refactoring and running experiments	2022-03-05 16:51:19 +01:00
Steffen Illium	69c904e156	refactoring and running experiments	2022-03-05 11:07:35 +01:00
Steffen Illium	b6c8859081	smaller task train	2022-03-03 21:53:46 +01:00
Steffen Illium	c52b398819	smaller task train	2022-03-03 21:43:38 +01:00
Steffen Illium	16c08d04d4	smaller task train	2022-03-03 21:16:33 +01:00
Steffen Illium	e167cc78c5	smaller task train	2022-03-03 14:57:26 +01:00
Steffen Illium	926b27b4ef	in between upload	2022-02-27 17:56:25 +01:00
Steffen Illium	78a919395b	reset diverged by xavier init	2022-02-26 16:42:37 +01:00
Steffen Illium	a3a587476c	Merge remote-tracking branch 'origin/swarm' into swarm	2022-02-26 16:01:22 +01:00
Steffen Illium	c0db8e19a3	sparse net training	2022-02-26 16:01:12 +01:00
Maximilian Zorn	e1a5383c04	Single or grouped 3d-PCA trajectory plotting functions.	2022-02-25 18:40:53 +01:00
Steffen Illium	9d8496a725	small fixes new parameters	2022-02-25 15:32:56 +01:00
Steffen Illium	5b2b5b5beb	residual skip in metacomparebaseline	2022-02-23 18:32:36 +01:00
Steffen Illium	3da00c793b	new sanity methode	2022-02-23 18:23:00 +01:00
Steffen Illium	ebf133414c	network test	2022-02-23 12:19:27 +01:00
Steffen Illium	0bc3b62340	network test	2022-02-23 12:08:49 +01:00
Steffen Illium	f0ad875e79	from state dict to direct parameter update	2022-02-22 10:08:27 +01:00
Steffen Illium	bb12176f72	Merge remote-tracking branch 'origin/swarm' into swarm # Conflicts: # sanity_check_weights.py	2022-02-22 09:55:10 +01:00
Steffen Illium	78e9c4d520	from state dict to direct parameter update	2022-02-22 09:54:54 +01:00
Maximilian Zorn	7166fa86ca	Merge branch 'swarm' of gitlab.lrz.de:mobile-ifi/bannana-networks into swarm	2022-02-21 20:14:33 +01:00
Maximilian Zorn	f033a2448e	Sanity check shape error fix.	2022-02-21 20:14:07 +01:00
Steffen Illium	2a710b40d7	apply networks are now loop free	2022-02-21 18:11:30 +01:00
Steffen Illium	f25cee5203	sparse network redo	2022-02-20 21:21:22 +01:00
Maximilian Zorn	52081d176e	sanity check for MetaNet weights and min-net hparam search	2022-02-19 17:44:39 +01:00
Steffen Illium	5aea4f9f55	New Images	2022-02-17 13:13:41 +01:00
Steffen Illium	4f99251e68	New Images	2022-02-15 14:52:01 +01:00
Steffen Illium	28b5e85697	New Residual Skip	2022-02-15 14:27:26 +01:00
Steffen Illium	a4d1ee86dd	New Self Replication	2022-02-15 14:25:05 +01:00
Steffen Illium	62e640e1f0	Dropout	2022-02-15 10:57:40 +01:00
Steffen Illium	8546cc7ddf	Merge remote-tracking branch 'origin/swarm' into swarm	2022-02-10 16:54:01 +01:00
Steffen Illium	14768ffc0a	README.md Update	2022-02-10 16:53:49 +01:00
Maximilian Zorn	7ae3e96ec9	sparse meta networt	2022-02-09 14:35:55 +01:00
Steffen Illium	594bbaa3dd	parameters for training	2022-02-08 17:24:00 +01:00
Steffen Illium	d4c25872c6	Merge remote-tracking branch 'origin/swarm' into swarm	2022-02-04 21:07:35 +01:00
Steffen Illium	3746c7d26e	StackPlot	2022-02-04 21:07:19 +01:00
Maximilian Zorn	df182d652a	sparse tensor combined train draft notebook	2022-02-04 16:48:19 +01:00
Steffen Illium	21c3e75177	images added	2022-02-03 18:03:43 +01:00
Steffen Illium	382afa8642	Readme Update und Residuals GN Training	2022-02-03 18:03:01 +01:00
Steffen Illium	6b1efd0c49	Readme Update und Residuals GN Training	2022-02-03 12:21:17 +01:00
Steffen Illium	3fe4f49bca	Readme Update und Smaller GN Training	2022-02-02 13:11:35 +01:00
Steffen Illium	eb3b9b8958	Initial Push	2022-02-02 12:03:31 +01:00
Steffen Illium	1b7581e656	MetaNetworks Debugged II	2022-02-01 18:17:11 +01:00
Steffen Illium	246d825bb4	MetaNetworks Debugged	2022-01-31 10:35:11 +01:00
Steffen Illium	49c0d8a621	MetaNetworks	2022-01-26 16:56:05 +01:00
Steffen Illium	5f1f5833d8	Journal TEx Text	2022-01-21 17:28:45 +01:00
Steffen Illium	21dd572969	Merge remote-tracking branch 'origin/journal' into journal # Conflicts: # journal_basins.py	2021-09-13 16:06:03 +02:00
Steffen Illium	5f6c658068	Journal TEx Text	2021-09-13 16:04:48 +02:00
Maximilian Zorn	e51d7ad0b9	Added parent-vs-children plot, changed x-tick labels to base10 notation.	2021-09-07 18:15:06 +02:00
Steffen Illium	6c1a964f31	Second order function	2021-08-24 10:35:29 +02:00
steffen-illium	b22a7ac427	in between plots	2021-06-28 10:51:21 +02:00
steffen-illium	6c2d544f7c	logspace....	2021-06-25 17:46:45 +02:00
steffen-illium	5a7dad2363	Merge remote-tracking branch 'origin/journal' into journal # Conflicts: # journal_soup_basins.py	2021-06-25 10:29:39 +02:00
steffen-illium	14d9a533cb	journal linspace basins	2021-06-25 10:25:25 +02:00
ru43zex	f7a0d360b3	updated journal_soup_basin: all working, only orange lines not showing	2021-06-24 16:48:52 +03:00
Maximilian Zorn	cf6eec639f	Fixed exp pickle save. Added exp results. Fixed soup plot issue.	2021-06-20 17:09:17 +02:00
Maximilian Zorn	1da5bd95d6	Added before-after-plot.	2021-06-17 15:53:48 +02:00
Maximilian Zorn	b40b534d5b	Small fix for PCA plotting in linspace.	2021-06-15 17:18:40 +02:00
steffen-illium	27d763f1fb	journal linspace basins	2021-06-15 14:11:40 +02:00
steffen-illium	0ba109c083	journal linspace basins	2021-06-14 11:55:11 +02:00
steffen-illium	e156540e2c	readme update	2021-06-11 14:41:35 +02:00
steffen-illium	0e2289344a	robustness	2021-06-11 14:38:22 +02:00
ru43zex	987d7b95f3	debugged journal_soup_basin experiment	2021-06-07 11:12:59 +03:00
ru43zex	7e231b5b50	Merge branch 'journal' of gitlab.lrz.de:mobile-ifi/bannana-networks into journal	2021-06-05 17:50:45 +03:00
ru43zex	2077d800ae	fixed soup_basin experiment	2021-06-05 17:44:37 +03:00
steffen-illium	b57d3d32fd	readme updated	2021-06-04 15:01:16 +02:00
steffen-illium	61ae8c2ee5	robustness fixed	2021-06-04 14:13:38 +02:00
steffen-illium	800a2c8f6b	Merge remote-tracking branch 'origin/journal' into journal # Conflicts: # journal_basins.py	2021-06-04 14:03:50 +02:00
steffen-illium	9abde030af	robustness cleaned	2021-06-04 14:03:03 +02:00
ru43zex	0320957b85	implements basin experiments for soup	2021-06-02 02:49:18 +03:00
Maximilian Zorn	32ebb729e8	Added plot variations for basin exp.	2021-05-27 16:02:41 +02:00
steffen-illium	c9efe0a31b	journal_robustness.py redone, now is sensitive to seeds and plots	2021-05-25 17:07:24 +02:00
steffen-illium	5e5511caf8	journal_robustness.py redone, now is sensitive to seeds and plots	2021-05-23 15:49:48 +02:00
steffen-illium	55bdd706b6	journal_robustness.py redone, now is sensitive to seeds and plots	2021-05-23 13:46:21 +02:00
steffen-illium	74d618774a	application losses II	2021-05-23 10:36:22 +02:00
steffen-illium	54590eb147	application losses	2021-05-23 10:33:54 +02:00
Maximilian Zorn	b1dc574f5b	Added distinction time-as-fixpoint and time-to-vergence to be tracked.	2021-05-22 14:43:20 +02:00
Maximilian Zorn	e9f6620b60	Robustness test with synthetic and natural fixpoints. Should now work as intended. Noise gets added to weights instead of input.	2021-05-22 14:27:45 +02:00
Maximilian Zorn	bcfe5807a7	Added pickle save() function for SpawnExperiment, updated README, set plot-pca-all false on default, just True for SpawnExp for now.	2021-05-21 16:37:27 +02:00
Maximilian Zorn	1e8ccd2b8b	PCA now fit and transform over all trajectories. Networks now have start-time properties for that to control where plotting starts. Added distance from parent table/matrix.	2021-05-21 15:28:09 +02:00
Maximilian Zorn	f5ca3d1115	save weights instead of outputs	2021-05-17 09:21:01 +02:00
steffen-illium	c1f58f2675	All Experiments debugged ToDo: - convert strings in pathlib.Path objects - check usage of fixpoint tests	2021-05-16 15:34:43 +02:00
steffen-illium	36377ee27d	functionalities_test.py updated	2021-05-16 14:51:21 +02:00
steffen-illium	b1472479cb	journal_basins.py debugged II Questions for functionalities_test.py corrected some fixes Redo and implementation of everything path related now using pathlib.Path	2021-05-16 13:35:38 +02:00
steffen-illium	042188f15a	journal_basins.py debugged	2021-05-16 11:30:34 +02:00
steffen-illium	5074100b71	Merge remote-tracking branch 'origin/journal' into journal	2021-05-14 17:58:01 +02:00
steffen-illium	4b5c36f6c0	- rearanged experiments.py into single runable files - Reformated net.self_x functions (sa, st) - corrected robustness_exp.py - NO DEBUGGING DONE!!!!!	2021-05-14 17:57:44 +02:00
Cristian Lenta	56ea007f2b	numbering of the points for easy addressing	2021-05-14 14:00:55 +00:00
Maximilian Zorn	22d34d4e75	some work on the new journal experiments with cristions code	2021-05-14 09:13:42 +02:00
Cristian Lenta	9bf37486a0	Update README.md	2021-05-03 06:50:33 +00:00
Cristian Lenta	e176d05cf5	uploaded my code (not yet 100% finished)	2021-05-03 06:43:53 +00:00
Cristian Lenta	9bd65713fe	deleted all filed from the old code	2021-05-03 06:36:23 +00:00
Si11ium	9ce36c2d0e	Merge remote-tracking branch 'origin/master'	2019-06-17 15:20:13 +02:00
Si11ium	4d5ef0e536	TF 113 imports	2019-06-17 15:19:15 +02:00
Thomas Gabor	6c90c3d739	camera ready	2019-05-24 15:38:11 +02:00
Thomas Gabor	0c9fb6efb9	yesterday's final fixes	2019-03-17 20:41:59 +01:00
		`@ -1 +0,0 @@`
			`{'divergent': 0, 'fix_zero': 0, 'fix_other': 13, 'fix_sec': 0, 'other': 7}`
		`@ -1,4 +0,0 @@`
			`TrainingNeuralNetworkDecorator activiation='linear' use_bias=False`
			`{'xs': [0, 10, 20, 30, 40, 50, 60, 70, 80, 90, 100], 'ys': [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], 'zs': [0.0, 1.2, 5.2, 7.4, 8.1, 9.1, 9.6, 9.8, 10.0, 9.9, 9.9]}`
		`@ -1 +0,0 @@`
			`{'divergent': 0, 'fix_zero': 10, 'fix_other': 0, 'fix_sec': 0, 'other': 0}`
		`@ -1 +0,0 @@`
			`{'divergent': 11, 'fix_zero': 9, 'fix_other': 0, 'fix_sec': 0, 'other': 0}`