Final Train Runs

models/transformer_model.py

@@ -21,9 +21,9 @@ class VisualTransformer(CombinedModelMixins,
                         ):
 
     def __init__(self, in_shape, n_classes, weight_init, activation,
-                 embedding_size, heads, attn_depth, patch_size,use_residual,
-                 use_bias, use_norm, dropout, lat_dim, loss,
-                 lr, weight_decay, sto_weight_avg, lr_warm_restart_epochs, opt_reset_interval):
+                 embedding_size, heads, attn_depth, patch_size, use_residual,
+                 use_bias, use_norm, dropout, lat_dim, loss, scheduler, mlp_dim, head_dim,
+                 lr, weight_decay, sto_weight_avg, lr_scheduler_parameter, opt_reset_interval):
 
         # TODO: Move this to parent class, or make it much easieer to access... But How...
         a = dict(locals())
@@ -53,26 +53,26 @@ class VisualTransformer(CombinedModelMixins,
                                                f'attention. Try decreasing your patch size'
 
         # Correct the Embedding Dim
-        if not self.embed_dim % self.params.heads == 0:
-            self.embed_dim = (self.embed_dim // self.params.heads) * self.params.heads
-            message = ('Embedding Dimension was fixed to be devideable by the number' +
-                       f' of attention heads, is now: {self.embed_dim}')
-            for func in print, warnings.warn:
-                func(message)
+        #if not self.embed_dim % self.params.heads == 0:
+        #    self.embed_dim = (self.embed_dim // self.params.heads) * self.params.heads
+        #    message = ('Embedding Dimension was fixed to be devideable by the number' +
+        #               f' of attention heads, is now: {self.embed_dim}')
+        #    for func in print, warnings.warn:
+        #        func(message)
 
         # Utility Modules
         self.autopad = AutoPadToShape((self.image_size, self.image_size))
 
         # Modules with Parameters
-        self.transformer = TransformerModule(in_shape=self.embed_dim, mlp_dim=self.params.lat_dim,
+        self.transformer = TransformerModule(in_shape=self.embed_dim, mlp_dim=self.params.mlp_dim,
+                                             head_dim=self.params.head_dim,
                                              heads=self.params.heads, depth=self.params.attn_depth,
                                              dropout=self.params.dropout, use_norm=self.params.use_norm,
                                              activation=self.params.activation, use_residual=self.params.use_residual
                                              )
 
         self.pos_embedding = nn.Parameter(torch.randn(1, num_patches + 1, self.embed_dim))
-        self.patch_to_embedding = nn.Linear(patch_dim, self.embed_dim) if self.params.embedding_size \
-            else F_x(self.embed_dim)
+        self.patch_to_embedding = nn.Linear(patch_dim, self.embed_dim)
         self.cls_token = nn.Parameter(torch.randn(1, 1, self.embed_dim))
         self.dropout = nn.Dropout(self.params.dropout)
 
@@ -117,4 +117,4 @@ class VisualTransformer(CombinedModelMixins,
         return Namespace(main_out=tensor, attn_weights=attn_weights)
 
     def additional_scores(self, outputs):
-            return MultiClassScores(self)(outputs)
+        return MultiClassScores(self)(outputs)