steffen/self-replicating-neural-networks
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Readme Update und Smaller GN Training

Browse Source
This commit is contained in:
Steffen Illium 2022-02-02 13:11:35 +01:00
parent eb3b9b8958
commit 3fe4f49bca
2 changed files with 56 additions and 39 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

65
README.md
View File

@ -1,48 +1,51 @@
# Bureaucratic Cohort Swarms # Bureaucratic Cohort Swarms
### (The Meta-Task Experience) # Deadline: 28.02.22 ### (The Meta-Task Experience) # Deadline: 28.02.22
## Experimente ## Experimente
Fixpoint Tests: ### Fixpoint Tests:
-> Dropout Test - [ ] Dropout Test
(Macht das Partikel beim Goal mit oder ist es nur SRN) - (Macht das Partikel beim Goal mit oder ist es nur SRN)
Zero_ident diff = -00.04999637603759766 % - Zero_ident diff = -00.04999637603759766 %
-> gnf(1) -> Aprox. Weight - [ ] gnf(1) -> Aprox. Weight
Übersetung in ein Gewichtsskalar - Übersetung in ein Gewichtsskalar
-> Einbettung in ein Reguläres Netz - Einbettung in ein Reguläres Netz
(-> Übersetung in ein Explainable AI Framework) - [ ] Übersetung in ein Explainable AI Framework
-> Rückschlüsse auf Mikro Netze - Rückschlüsse auf Mikro Netze
-> Visualiserung - [ ] Visualiserung
-> Der Zugehörigkeit - Der Zugehörigkeit
-> Der Vernetzung - Der Vernetzung
-> PCA() - [ ] PCA()
-> Dataframe Epoch, Weight, dim_1, ..., dim_n - Dataframe Epoch, Weight, dim_1, ..., dim_n
-> Visualisierung als Trajectory Cube - Visualisierung als Trajectory Cube
-> Recherche zu Makro Mikro Netze Strukturen - [ ] Recherche zu Makro Mikro Netze Strukturen
Gibts das schon? - gits das schon?
Hypernetwork? - Hypernetwork?
arxiv: 1905.02898 - arxiv: 1905.02898
--- ---
Tasks für Steffen: ### Tasks für Steffen:
- Training mit kleineren GNs
- Weiter Trainieren -> 500 Epochs? - [ ] Training mit kleineren GNs -| Running
- Loss Gewichtung anpassen - [ ] Weiter Trainieren -> 500 Epochs?
- Training ohne Residual Skip Connection - [ ] Loss Gewichtung anpassen
- Test mit Baseline Dense Network - [ ] Training ohne Residual Skip Connection
-> mit vergleichbaren Neuron Count - [ ] Test mit Baseline Dense Network
-> mit gesamt Weight Count - [ ] mit vergleichbaren Neuron Count
- Task/Goal statt SRNN-Task - [ ] mit gesamt Weight Count
- [ ] Task/Goal statt SRNN-Task
--- ---
Für Menschen mit zu viel Zeit: ### Für Menschen mit zu viel Zeit:
-> Sparse Network Training der Self Replication - [ ] Sparse Network Training der Self Replication
(Just for the lulz and speeeeeeed) - Just for the lulz and speeeeeeed)
- (Spaß bei Seite, wäre wichtig für schnellere Forschung)
<https://pytorch.org/docs/stable/sparse.html>
--- ---

30
experiments/meta_task_exp.py
View File

@ -117,8 +117,11 @@ def validate(checkpoint_path, ratio=0.1):
return acc return acc
def new_train_storage_df(): def new_storage_df(identifier, weight_count):
return pd.DataFrame(columns=['Epoch', 'Batch', 'Metric', 'Score']) if identifier == 'train':
return pd.DataFrame(columns=['Epoch', 'Batch', 'Metric', 'Score'])
elif identifier == 'weights':
return pd.DataFrame(columns=['Epoch', 'Weight', *(f'weight_{x}' for x in range(weight_count))])
def checkpoint_and_validate(model, out_path, epoch_n, final_model=False): def checkpoint_and_validate(model, out_path, epoch_n, final_model=False):
@ -163,8 +166,8 @@ def plot_training_result(path_to_dataframe):
if __name__ == '__main__': if __name__ == '__main__':
self_train = False self_train = True
training = False training = True
plotting = True plotting = True
particle_analysis = True particle_analysis = True
as_sparse_network_test = True as_sparse_network_test = True
@ -172,9 +175,10 @@ if __name__ == '__main__':
data_path = Path('data') data_path = Path('data')
data_path.mkdir(exist_ok=True, parents=True) data_path.mkdir(exist_ok=True, parents=True)
run_path = Path('output') / 'mnist_self_train_100_NEW_STYLE' run_path = Path('output') / 'mn_st_smaller'
model_path = run_path / '0000_trained_model.zip' model_path = run_path / '0000_trained_model.zip'
df_store_path = run_path / 'train_store.csv' df_store_path = run_path / 'train_store.csv'
weight_store_path = run_path / 'weight_store.csv'
if training: if training:
utility_transforms = Compose([ToTensor(), ToFloat(), Resize((15, 15)), Flatten(start_dim=0)]) utility_transforms = Compose([ToTensor(), ToFloat(), Resize((15, 15)), Flatten(start_dim=0)])
@ -186,11 +190,13 @@ if __name__ == '__main__':
interface = np.prod(dataset[0][0].shape) interface = np.prod(dataset[0][0].shape)
metanet = MetaNet(interface, depth=5, width=6, out=10).to(DEVICE) metanet = MetaNet(interface, depth=5, width=6, out=10).to(DEVICE)
meta_weight_count = sum(p.numel() for p in next(metanet.particles).parameters())
loss_fn = nn.CrossEntropyLoss() loss_fn = nn.CrossEntropyLoss()
optimizer = torch.optim.SGD(metanet.parameters(), lr=0.008, momentum=0.9) optimizer = torch.optim.SGD(metanet.parameters(), lr=0.008, momentum=0.9)
train_store = new_train_storage_df() train_store = new_storage_df('train', None)
weight_store = new_storage_df('train', meta_weight_count)
for epoch in tqdm(range(EPOCH), desc='MetaNet Train - Epochs'): for epoch in tqdm(range(EPOCH), desc='MetaNet Train - Epochs'):
is_validation_epoch = epoch % VALIDATION_FRQ == 0 if not debug else True is_validation_epoch = epoch % VALIDATION_FRQ == 0 if not debug else True
is_self_train_epoch = epoch % SELF_TRAIN_FRQ == 0 if not debug else True is_self_train_epoch = epoch % SELF_TRAIN_FRQ == 0 if not debug else True
@ -247,16 +253,24 @@ if __name__ == '__main__':
for key, value in dict(counter_dict).items(): for key, value in dict(counter_dict).items():
step_log = dict(Epoch=int(epoch), Batch=BATCHSIZE, Metric=key, Score=value) step_log = dict(Epoch=int(epoch), Batch=BATCHSIZE, Metric=key, Score=value)
train_store.loc[train_store.shape[0]] = step_log train_store.loc[train_store.shape[0]] = step_log
for particle in metanet.particles:
weight_log = (epoch, particle.name, *(x for y in particle.parameters() for x in y))
train_store.to_csv(df_store_path, mode='a', header=not df_store_path.exists()) train_store.to_csv(df_store_path, mode='a', header=not df_store_path.exists())
# train_store = new_train_storage_df() weight_store.to_csv(weight_store_path, mode='a', header=not weight_store_path.exists())
train_store = new_storage_df('train', None)
weight_store = new_storage_df('train', meta_weight_count)
metanet.eval() metanet.eval()
accuracy = checkpoint_and_validate(metanet, run_path, EPOCH, final_model=True) accuracy = checkpoint_and_validate(metanet, run_path, EPOCH, final_model=True)
validation_log = dict(Epoch=EPOCH, Batch=BATCHSIZE, validation_log = dict(Epoch=EPOCH, Batch=BATCHSIZE,
Metric='Test Accuracy', Score=accuracy.item()) Metric='Test Accuracy', Score=accuracy.item())
for particle in metanet.particles:
weight_log = (EPOCH, particle.name, *(x for y in particle.parameters() for x in y))
weight_store.loc[weight_store.shape[0]] = weight_log
train_store.loc[train_store.shape[0]] = validation_log train_store.loc[train_store.shape[0]] = validation_log
train_store.to_csv(df_store_path) train_store.to_csv(df_store_path, mode='a', header=not df_store_path.exists())
weight_store.to_csv(weight_store_path, mode='a', header=not weight_store_path.exists())
if plotting: if plotting:
plot_training_result(df_store_path) plot_training_result(df_store_path)