README.md Update

README.md

@@ -1,6 +1,9 @@
 # Bureaucratic Cohort Swarms
 ### (The Meta-Task Experience)  # Deadline: 28.02.22
 ## Experimente
+
+Data Exchange: [Google Drive Folder](***REMOVED***)
+
 ### Fixpoint Tests:
     
 - [ ] Dropout Test 
@@ -30,19 +33,29 @@
 ---
 
 ### Tasks für Steffen:
+- [x] Sanity Check:
 
-- [x] Training mit kleineren GNs
+  - [x] Neuronen können lernen einen Eingabewert mit x zu multiplizieren?
-  - Accuracy leidet enorm (_0.56_)
+
-    ![image info](./figures/lesser_neurons_training_lineplot.png)
+    | SRNN x*n    3 Neurons            Identity_Func     | SRNN x*n    4 Neurons              Identity_Func  |
-  - Es entstehen mehr SRNN
+    |---------------------------------------------------|----------------------------------------------------|
-  - Der Dropout Effekt wird stärker (diff_ohne_SRNN = _0.0_)
+    | ![](./figures/sanity/sanity_3hidden_xtimesn.png)   | ![](./figures/sanity/sanity_4hidden_xtimesn.png)  |
-    ![image info](./figures/lesser_neurons_dropout_stacked_barplot.png)
+    | SRNN x*n    6 Neurons              Other_Func      | SRNN x*n    10 Neurons      Other_Func            |
-- [X] Weiter Trainieren -> 500 Epochs?
+    | ![](./figures/sanity/sanity_6hidden_xtimesn.png)   | ![](./figures/sanity/sanity_10hidden_xtimesn.png) |
-- [ ] Loss Gewichtung anpassen
+
+- [ ] Training mit kleineren GNs
+  
+    
+- [ ] Weiter Trainieren -> 500 Epochs?
 - [x] Training ohne Residual Skip Connection
-  - Ist kacke
+  - Ist anders:
-  ![image info](./figures/no_res_training_lineplot.png)
+     Self Training wird zunächst priorisiert, dann kommt langsam der eigentliche Task durch:
-  ![image info](./figures/no_res_dropout_stacked_barplot.png)
+      
+      | No Residual Skip connections 8 Neurons in SRNN  Alpha=100                                    | Residual Skip connections 8 Neurons in SRNN    Alpha=100                                 |
+      |------------------------------------------------------------------------------------------|----------------------------------------------------------------------------------------------|
+      | ![LinePlot](./figures/res_no_res/mn_st_200_8_alpha_100_no_res_training_particle_type_lp.png) | ![image info](./figures/res_no_res/mn_st_200_8_alpha_100_training_particle_type_lp.png)  |
+      | ![image info](./figures/res_no_res/mn_st_200_8_alpha_100_no_res_training_lineplot.png)       | ![image info](./figures/res_no_res/mn_st_200_8_alpha_100_training_lineplot.png)          |
+
 - [ ] Test mit Baseline Dense Network 
   - [ ] mit vergleichbaren Neuron Count
   - [ ] mit gesamt Weight Count
@@ -56,4 +69,3 @@
   - (Spaß bei Seite, wäre wichtig für schnellere Forschung)
     <https://pytorch.org/docs/stable/sparse.html>
 
----

experiments/meta_task_exp.py

@@ -198,12 +198,13 @@ if __name__ == '__main__':
     plotting = True
     particle_analysis = True
     as_sparse_network_test = True
-    self_train_alpha = 100
+    self_train_alpha = 1
+    batch_train_beta = 1
 
     data_path = Path('data')
     data_path.mkdir(exist_ok=True, parents=True)
 
-    run_path = Path('output') / 'mn_st_200_8_alpha_100'
+    run_path = Path('output') / 'mn_st_400_2_no_res'
     model_path = run_path / '0000_trained_model.zip'
     df_store_path = run_path / 'train_store.csv'
     weight_store_path = run_path / 'weight_store.csv'
@@ -217,7 +218,7 @@ if __name__ == '__main__':
         d = DataLoader(dataset, batch_size=BATCHSIZE, shuffle=True, drop_last=True, num_workers=WORKER)
 
         interface = np.prod(dataset[0][0].shape)
-        metanet = MetaNet(interface, depth=5, width=6, out=10, residual_skip=True).to(DEVICE)
+        metanet = MetaNet(interface, depth=5, width=6, out=10, residual_skip=False).to(DEVICE)
         meta_weight_count = sum(p.numel() for p in next(metanet.particles).parameters())
 
         loss_fn = nn.CrossEntropyLoss()
@@ -249,7 +250,7 @@ if __name__ == '__main__':
                 batch_x, batch_y = batch_x.to(DEVICE), batch_y.to(DEVICE)
                 y = metanet(batch_x)
                 # loss = loss_fn(y, batch_y.unsqueeze(-1).to(torch.float32))
-                loss = loss_fn(y, batch_y.to(torch.long))
+                loss = loss_fn(y, batch_y.to(torch.long)) * batch_train_beta
                 loss.backward()
 
                 # Adjust learning weights
@@ -312,7 +313,7 @@ if __name__ == '__main__':
         plot_training_result(df_store_path)
         if particle_analysis:
             plot_training_particle_types(df_store_path)
-    exit()
+
     if particle_analysis:
         model_path = next(run_path.glob(f'*e{EPOCH}.tp'))
         latest_model = torch.load(model_path, map_location=DEVICE).eval()

experiments/meta_task_sanity_exp.py

@@ -0,0 +1,103 @@
+import sys
+from collections import defaultdict
+from pathlib import Path
+import platform
+
+import pandas as pd
+import torch.optim
+from matplotlib import pyplot as plt
+from torch import nn
+from torch.utils.data import Dataset, DataLoader
+import numpy as np
+import seaborn as sns
+from tqdm import trange
+from tqdm.contrib import tenumerate
+
+
+if platform.node() == 'CarbonX':
+    debug = True
+    print("@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@")
+    print("@ Warning, Debugging Config@!!!!!! @")
+    print("@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@")
+else:
+    debug = False
+    try:
+        # noinspection PyUnboundLocalVariable
+        if __package__ is None:
+            DIR = Path(__file__).resolve().parent
+            sys.path.insert(0, str(DIR.parent))
+            __package__ = DIR.name
+        else:
+            DIR = None
+    except NameError:
+        DIR = None
+        pass
+
+import functionalities_test
+from network import Net
+
+
+class MultiplyByXTaskDataset(Dataset):
+    def __init__(self, x=0.23, length=int(5e5)):
+        super().__init__()
+        self.length = length
+        self.x = x
+        self.prng = np.random.default_rng()
+
+    def __len__(self):
+        return self.length
+
+    def __getitem__(self, _):
+        ab = self.prng.normal(size=(1,)).astype(np.float32)
+        return ab, ab * self.x
+
+
+if __name__ == '__main__':
+    net = Net(5, 1, 1)
+    multiplication_target = 0.03
+
+    loss_fn = nn.MSELoss()
+    optimizer = torch.optim.SGD(net.parameters(), lr=0.008, momentum=0.9)
+
+    train_frame = pd.DataFrame(columns=['Epoch', 'Batch', 'Metric', 'Score'])
+
+    dataset = MultiplyByXTaskDataset(x=multiplication_target, length=1000000)
+    dataloader = DataLoader(dataset=dataset, batch_size=8000)
+    for epoch in trange(30):
+        mean_batch_loss = []
+        mean_self_tain_loss = []
+        for batch, (batch_x, batch_y) in tenumerate(dataloader):
+            self_train_loss, _ = net.self_train(10, save_history=False)
+            batch_x_emb = torch.zeros(batch_x.shape[0], 5)
+            batch_x_emb[:, -1] = batch_x.squeeze()
+            y = net(batch_x_emb)
+            loss = loss_fn(y, batch_y)
+
+            loss.backward()
+            optimizer.step()
+
+            mean_batch_loss.append(loss.detach())
+            mean_self_tain_loss.append(self_train_loss.detach())
+
+        train_frame.loc[train_frame.shape[0]] = dict(Epoch=epoch, Batch=batch,
+                                                     Metric='Self Train Loss', Score=np.average(mean_self_tain_loss))
+        train_frame.loc[train_frame.shape[0]] = dict(Epoch=epoch, Batch=batch,
+                                                         Metric='Batch Loss', Score=np.average(mean_batch_loss))
+
+    counter = defaultdict(lambda: 0)
+    functionalities_test.test_for_fixpoints(counter, nets=[net])
+    print(dict(counter))
+    sanity = net(torch.Tensor([0,0,0,0,1])).detach()
+    print(sanity)
+    print(abs(sanity - multiplication_target))
+    sns.lineplot(data=train_frame, x='Epoch', y='Score', hue='Metric')
+    outpath = Path('output') / 'sanity' / 'test.png'
+    outpath.parent.mkdir(exist_ok=True, parents=True)
+    plt.savefig(outpath)
+
+
+
+
+
+
+

network.py

@@ -296,7 +296,7 @@ class MetaCell(nn.Module):
         self.name = name
         self.interface = interface
         self.weight_interface = 5
-        self.net_hidden_size = 8
+        self.net_hidden_size = 2
         self.net_ouput_size = 1
         self.meta_weight_list = nn.ModuleList()
         self.meta_weight_list.extend(