plotting

environments/factory/base_factory.py

@@ -191,8 +191,7 @@ class BaseFactory(gym.Env):
 
     def calculate_reward(self, agent_states: List[AgentState]) -> (int, dict):
         # Returns: Reward, Info
-        # Set to "raise NotImplementedError"
-        return 0, {}
+        raise NotImplementedError
 
     def render(self):
         raise NotImplementedError

environments/factory/simple_factory.py

@@ -112,7 +112,7 @@ class SimpleFactory(BaseFactory):
     def calculate_reward(self, agent_states: List[AgentState]) -> (int, dict):
         # TODO: What reward to use?
         current_dirt_amount = self.state[DIRT_INDEX].sum()
-        dirty_tiles = len(np.nonzero(self.state[DIRT_INDEX]))
+        dirty_tiles = np.argwhere(self.state[DIRT_INDEX] != h.IS_FREE_CELL).shape[0]
 
         try:
             # penalty = current_dirt_amount
@@ -128,7 +128,7 @@ class SimpleFactory(BaseFactory):
                 if agent_state.action_valid:
                     reward += 2
                     self.print(f'Agent {agent_state.i} did just clean up some dirt at {agent_state.pos}.')
-                    self.monitor.add('dirt_cleaned', self._dirt_properties.clean_amount)
+                    self.monitor.add('dirt_cleaned', 1)
                 else:
                     self.print(f'Agent {agent_state.i} just tried to clean up some dirt '
                                f'at {agent_state.pos}, but was unsucsessfull.')

environments/logging/monitor.py

@@ -4,6 +4,8 @@ from collections import defaultdict
 
 from stable_baselines3.common.callbacks import BaseCallback
 
+from environments.logging.plotting import prepare_plot
+
 
 class FactoryMonitor:
 
@@ -58,11 +60,12 @@ class MonitorCallback(BaseCallback):
 
     ext = 'png'
 
-    def __init__(self, env, filepath=Path('debug_out/monitor.pick')):
+    def __init__(self, env, filepath=Path('debug_out/monitor.pick'), plotting=True):
         super(MonitorCallback, self).__init__()
         self.filepath = Path(filepath)
         self._monitor_list = list()
         self.env = env
+        self.plotting = plotting
         self.started = False
         self.closed = False
 
@@ -91,7 +94,18 @@ class MonitorCallback(BaseCallback):
             # self.out_file.unlink(missing_ok=True)
             with self.filepath.open('wb') as f:
                 pickle.dump(self.monitor_as_df_list, f, protocol=pickle.HIGHEST_PROTOCOL)
-            self.prepare_plot()
+            if self.plotting:
+                print('Monitor files were dumped to disk, now plotting....')
+                # %% Imports
+                import pandas as pd
+                # %% Load MonitorList from Disk
+                with self.filepath.open('rb') as f:
+                    monitor_list = pickle.load(f)
+
+                result = pd.concat(monitor_list, sort=False)
+                # result.tail()
+                prepare_plot(filepath=self.filepath, results_df=result, tag='monitor')
+                print('Plotting done.')
             self.closed = True
 
     def _on_step(self) -> bool:
@@ -99,50 +113,5 @@ class MonitorCallback(BaseCallback):
             self._monitor_list.append(self.env.monitor)
         else:
             pass
+        return True
 
-    def plot(self, **kwargs):
-        from matplotlib import pyplot as plt
-        plt.rcParams.update(kwargs)
-
-        plt.tight_layout()
-        figure = plt.gcf()
-        plt.show()
-        figure.savefig(str(self.filepath.parent / f'{self.filepath.stem}_monitor_measures.{self.ext}'), format=self.ext)
-
-    def prepare_plot(self):
-        # %% Imports
-        import pandas as pd
-        import seaborn as sns
-
-        # %% Load MonitorList from Disk
-        with self.filepath.open('rb') as f:
-            monitor_list = pickle.load(f)
-
-        result = pd.concat(monitor_list, sort=False)
-        # result.tail()
-
-        # %%
-        lineplot = sns.lineplot(data=result)
-        lineplot.title.title = f'Lineplot Summary of {len(monitor_list)} Episodes'
-
-        # %%
-        sns.set_theme(palette='husl', style='whitegrid')
-        font_size = 16
-        tex_fonts = {
-            # Use LaTeX to write all text
-            "text.usetex": True,
-            "font.family": "serif",
-            # Use 10pt font in plots, to match 10pt font in document
-            "axes.labelsize": font_size,
-            "font.size": font_size,
-            # Make the legend/label fonts a little smaller
-            "legend.fontsize": font_size - 2,
-            "xtick.labelsize": font_size - 2,
-            "ytick.labelsize": font_size - 2
-        }
-
-        try:
-            self.plot(**tex_fonts)
-        except FileNotFoundError:
-            tex_fonts['text.usetex'] = False
-            self.plot(**tex_fonts)

environments/logging/plotting.py

@@ -0,0 +1,40 @@
+import seaborn as sns
+
+from matplotlib import pyplot as plt
+
+
+def plot(filepath, ext='png', tag='monitor', **kwargs):
+    plt.rcParams.update(kwargs)
+
+    plt.tight_layout()
+    figure = plt.gcf()
+    plt.show()
+    figure.savefig(str(filepath.parent / f'{filepath.stem}_{tag}_measures.{ext}'), format=ext)
+
+
+def prepare_plot(filepath, results_df, ext='png', tag=''):
+    # %%
+
+    _ = sns.lineplot(data=results_df)
+
+    # %%
+    sns.set_theme(palette='husl', style='whitegrid')
+    font_size = 16
+    tex_fonts = {
+        # Use LaTeX to write all text
+        "text.usetex": False,
+        "font.family": "serif",
+        # Use 10pt font in plots, to match 10pt font in document
+        "axes.labelsize": font_size,
+        "font.size": font_size,
+        # Make the legend/label fonts a little smaller
+        "legend.fontsize": font_size - 2,
+        "xtick.labelsize": font_size - 2,
+        "ytick.labelsize": font_size - 2
+    }
+
+    try:
+        plot(filepath, ext=ext, tag=tag, **tex_fonts)
+    except (FileNotFoundError, RuntimeError):
+        tex_fonts['text.usetex'] = False
+        plot(filepath, ext=ext, tag=tag, **tex_fonts)

environments/logging/training.py

@@ -1,35 +1,54 @@
+from collections import defaultdict
 from pathlib import Path
 
+import numpy as np
 import pandas as pd
 from stable_baselines3.common.callbacks import BaseCallback
 
+from environments.logging.plotting import prepare_plot
+
 
 class TraningMonitor(BaseCallback):
 
     def __init__(self, filepath, flush_interval=None):
         super(TraningMonitor, self).__init__()
-        self.values = dict()
+        self.values = defaultdict(dict)
+        self.rewards = defaultdict(lambda: 0)
+
         self.filepath = Path(filepath)
         self.flush_interval = flush_interval
+        self.next_flush: int
         pass
 
     def _on_training_start(self) -> None:
         self.flush_interval = self.flush_interval or (self.locals['total_timesteps'] * 0.1)
+        self.next_flush = self.flush_interval
 
     def _flush(self):
-        df = pd.DataFrame.from_dict(self.values)
+        df = pd.DataFrame.from_dict(self.values,  orient='index')
         if not self.filepath.exists():
             df.to_csv(self.filepath, mode='wb', header=True)
         else:
             df.to_csv(self.filepath, mode='a', header=False)
-        self.values = dict()
 
     def _on_step(self) -> bool:
-        self.values[self.num_timesteps] = dict(reward=self.locals['rewards'].item())
-        if self.num_timesteps % self.flush_interval == 0:
+        for idx, done in np.ndenumerate(self.locals['dones']):
+            idx = idx[0]
+            # self.values[self.num_timesteps].update(**{f'reward_env_{idx}': self.locals['rewards'][idx]})
+            self.rewards[idx] += self.locals['rewards'][idx]
+            if done:
+                self.values[self.num_timesteps].update(**{f'acc_epispde_r_env_{idx}': self.rewards[idx]})
+                self.rewards[idx] = 0
+
+        if self.num_timesteps >= self.next_flush and self.values:
             self._flush()
+            self.values = defaultdict(dict)
+
+        self.next_flush += self.flush_interval
         return True
 
     def on_training_end(self) -> None:
         self._flush()
+        self.values = defaultdict(dict)
+        # prepare_plot()