monitor now returning info objects

environments/factory/base_factory.py

@@ -189,6 +189,10 @@ class BaseFactory(gym.Env):
         if self.steps >= self.max_steps:
             done = True
         self.monitor.set('step_reward', reward)
+        self.monitor.set('step', self.steps)
+
+        if done:
+            info.update(monitor=self.monitor)
         return self.state, reward, done, info
 
     def _is_moving_action(self, action):

environments/factory/simple_factory.py

@@ -145,28 +145,27 @@ class SimpleFactory(BaseFactory):
                     self.print(f'Agent {agent_state.i} did just clean up some dirt at {agent_state.pos}.')
                     self.monitor.set('dirt_cleaned', 1)
                 else:
-                    reward -= 1
+                    reward -= 0.5
                     self.print(f'Agent {agent_state.i} just tried to clean up some dirt '
                                f'at {agent_state.pos}, but was unsucsessfull.')
                     self.monitor.set('failed_cleanup_attempt', 1)
 
             elif self._is_moving_action(agent_state.action):
                 if agent_state.action_valid:
-                    reward -= 0.01
+                    reward -= 0.00
                 else:
                     reward -= 0.5
 
             else:
                 self.monitor.set('no_op', 1)
-                reward -= 0.25
+                reward -= 0.1
 
             for entity in cols:
                 if entity != self.state_slices.by_name("dirt"):
                     self.monitor.set(f'agent_{agent_state.i}_vs_{self.state_slices[entity]}', 1)
 
         self.monitor.set('dirt_amount', current_dirt_amount)
-        self.monitor.set('dirty_tiles', dirty_tiles)
-        self.monitor.set('step', self.steps)
+        self.monitor.set('dirty_tile_count', dirty_tiles)
         self.print(f"reward is {reward}")
         # Potential based rewards ->
         #  track the last reward , minus the current reward = potential

environments/helpers.py

@@ -8,6 +8,8 @@ LEVEL_IDX = 0
 AGENT_START_IDX = 1
 IS_FREE_CELL = 0
 IS_OCCUPIED_CELL = 1
+TO_BE_AVERAGED = ['dirt_amount', 'dirty_tiles']
+IGNORED_DF_COLUMNS = ['Episode', 'Run', 'train_step', 'step', 'index']
 
 
 # Utility functions

environments/logging/monitor.py

@@ -4,7 +4,9 @@ from collections import defaultdict
 
 from stable_baselines3.common.callbacks import BaseCallback
 
+from environments.helpers import IGNORED_DF_COLUMNS
 from environments.logging.plotting import prepare_plot
+import pandas as pd
 
 
 class FactoryMonitor:
@@ -59,16 +61,12 @@ class MonitorCallback(BaseCallback):
     def __init__(self, env, filepath=Path('debug_out/monitor.pick'), plotting=True):
         super(MonitorCallback, self).__init__()
         self.filepath = Path(filepath)
-        self._monitor_list = list()
+        self._monitor_df = pd.DataFrame()
         self.env = env
         self.plotting = plotting
         self.started = False
         self.closed = False
 
-    @property
-    def monitor_as_df_list(self):
-        return [x.to_pd_dataframe() for x in self._monitor_list]
-
     def __enter__(self):
         self._on_training_start()
 
@@ -89,11 +87,10 @@ class MonitorCallback(BaseCallback):
         else:
             # self.out_file.unlink(missing_ok=True)
             with self.filepath.open('wb') as f:
-                pickle.dump(self.monitor_as_df_list, f, protocol=pickle.HIGHEST_PROTOCOL)
+                pickle.dump(self._monitor_df.reset_index(), f, protocol=pickle.HIGHEST_PROTOCOL)
             if self.plotting:
                 print('Monitor files were dumped to disk, now plotting....')
-                # %% Imports
-                import pandas as pd
+
                 # %% Load MonitorList from Disk
                 with self.filepath.open('rb') as f:
                     monitor_list = pickle.load(f)
@@ -111,14 +108,21 @@ class MonitorCallback(BaseCallback):
                     if column != 'episode':
                         df[f'{column}_roll'] = df[column].rolling(window=50).mean()
                 # result.tail()
-                prepare_plot(filepath=self.filepath, results_df=df.filter(regex=(".+_roll")), tag='monitor')
+                prepare_plot(filepath=self.filepath, results_df=df.filter(regex=(".+_roll")))
                 print('Plotting done.')
             self.closed = True
 
     def _on_step(self) -> bool:
-        if self.locals['dones'].item():
-            self._monitor_list.append(self.env.monitor)
-        else:
-            pass
+        for env_idx, done in enumerate(self.locals.get('dones', [])):
+            if done:
+                env_monitor_df = self.locals['infos'][env_idx]['monitor'].to_pd_dataframe()
+                columns = [col for col in env_monitor_df.columns if col not in IGNORED_DF_COLUMNS]
+                env_monitor_df = env_monitor_df.aggregate(
+                    {col: 'mean' if 'amount' in col or 'count' in col else 'sum' for col in columns}
+                )
+                env_monitor_df['episode'] = len(self._monitor_df)
+                self._monitor_df = self._monitor_df.append([env_monitor_df])
+            else:
+                pass
         return True
 

environments/logging/plotting.py

@@ -29,7 +29,7 @@ def plot(filepath, ext='png', **kwargs):
     figure.savefig(str(filepath), format=ext)
 
 
-def prepare_plot(filepath, results_df, ext='png', tag=''):
+def prepare_plot(filepath, results_df, ext='png'):
 
     _ = sns.lineplot(data=results_df, x='Episode', y='Score', hue='Measurement', ci='sd')
 
@@ -50,8 +50,7 @@ def prepare_plot(filepath, results_df, ext='png', tag=''):
     }
 
     try:
-        plot(filepath, ext=ext, tag=tag, **tex_fonts)
+        plot(filepath, ext=ext, **tex_fonts)
     except (FileNotFoundError, RuntimeError):
         tex_fonts['text.usetex'] = False
-        plot(filepath, ext=ext, tag=tag, **tex_fonts)
-    plt.show()
+        plot(filepath, ext=ext, **tex_fonts)

environments/logging/training.py

@@ -32,7 +32,7 @@ class TraningMonitor(BaseCallback):
             df.to_csv(self.filepath, mode='a', header=False)
 
     def _on_step(self) -> bool:
-        for idx, done in np.ndenumerate(self.locals['dones']):
+        for idx, done in np.ndenumerate(self.locals.get('dones', [])):
             idx = idx[0]
             # self.values[self.num_timesteps].update(**{f'reward_env_{idx}': self.locals['rewards'][idx]})
             self.rewards[idx] += self.locals['rewards'][idx]