New Szenario "Two_Rooms_One_Door"

marl_factory_grid/utils/config_parser.py

@@ -24,7 +24,6 @@ class FactoryConfigParser(object):
         self.config_path = Path(config_path)
         self.custom_modules_path = Path(custom_modules_path) if custom_modules_path is not None else custom_modules_path
         self.config = yaml.safe_load(self.config_path.open())
-        self.do_record = False
 
     def __getattr__(self, item):
         return self['General'][item]

marl_factory_grid/utils/logging/envmonitor.py

@@ -0,0 +1,61 @@
+import pickle
+from os import PathLike
+from pathlib import Path
+from typing import Union
+
+from gymnasium import Wrapper
+
+from marl_factory_grid.utils.helpers import IGNORED_DF_COLUMNS
+
+import pandas as pd
+
+from marl_factory_grid.utils.plotting.compare_runs import plot_single_run
+
+
+class EnvMonitor(Wrapper):
+
+    ext = 'png'
+
+    def __init__(self, env, filepath: Union[str, PathLike] = None):
+        super(EnvMonitor, self).__init__(env)
+        self._filepath = filepath
+        self._monitor_df = pd.DataFrame()
+        self._monitor_dict = dict()
+
+
+    def step(self, action):
+        obs_type, obs, reward, done, info = self.env.step(action)
+        self._read_info(info)
+        self._read_done(done)
+        return obs_type, obs, reward, done, info
+
+    def reset(self):
+        return self.env.reset()
+
+    def _read_info(self, info: dict):
+        self._monitor_dict[len(self._monitor_dict)] = {
+            key: val for key, val in info.items() if
+            key not in ['terminal_observation', 'episode']}
+        return
+
+    def _read_done(self, done):
+        if done:
+            env_monitor_df = pd.DataFrame.from_dict(self._monitor_dict, orient='index')
+            self._monitor_dict = dict()
+            columns = [col for col in env_monitor_df.columns if col not in IGNORED_DF_COLUMNS]
+            env_monitor_df = env_monitor_df.aggregate(
+                {col: 'mean' if col.endswith('ount') else 'sum' for col in columns}
+            )
+            env_monitor_df['episode'] = len(self._monitor_df)
+            self._monitor_df = pd.concat([self._monitor_df, pd.DataFrame([env_monitor_df])], ignore_index=True)
+        else:
+            pass
+        return
+
+    def save_run(self, filepath: Union[Path, str, None] = None, auto_plotting_keys=None):
+        filepath = Path(filepath or self._filepath)
+        filepath.parent.mkdir(exist_ok=True, parents=True)
+        with filepath.open('wb') as f:
+            pickle.dump(self._monitor_df.reset_index(), f, protocol=pickle.HIGHEST_PROTOCOL)
+        if auto_plotting_keys:
+            plot_single_run(filepath, column_keys=auto_plotting_keys)

marl_factory_grid/utils/logging/recorder.py

@@ -0,0 +1,160 @@
+from os import PathLike
+from pathlib import Path
+from typing import Union, List
+
+import yaml
+from gymnasium import Wrapper
+
+import numpy as np
+import pandas as pd
+
+
+class EnvRecorder(Wrapper):
+
+    def __init__(self, env, filepath: Union[str, PathLike] = None,
+                 episodes: Union[List[int], None] = None):
+        super(EnvRecorder, self).__init__(env)
+        self.filepath = filepath
+        self.episodes = episodes
+        self._curr_episode = 0
+        self._curr_ep_recorder = list()
+        self._recorder_out_list = list()
+
+    def reset(self):
+        self._curr_ep_recorder = list()
+        self._recorder_out_list = list()
+        self._curr_episode += 1
+        return self.env.reset()
+
+    def step(self, actions):
+        obs_type, obs, reward, done, info = self.env.step(actions)
+        if not self.episodes or self._curr_episode in self.episodes:
+            summary: dict = self.env.summarize_state()
+            # summary.update(done=done)
+            # summary.update({'episode': self._curr_episode})
+            # TODO Protobuff Adjustments                 ######
+            # summary.update(info)
+            self._curr_ep_recorder.append(summary)
+            if done:
+                self._recorder_out_list.append({'steps': self._curr_ep_recorder,
+                                                'episode_nr': self._curr_episode})
+                self._curr_ep_recorder = list()
+        return obs_type, obs, reward, done, info
+
+    def _finalize(self):
+        if self._curr_ep_recorder:
+            self._recorder_out_list.append({'steps': self._curr_ep_recorder.copy(),
+                                            'episode_nr': len(self._recorder_out_list)})
+
+    def save_records(self, filepath: Union[Path, str, None] = None,
+                     only_deltas=False,
+                     save_occupation_map=False,
+                     save_trajectory_map=False,
+                     ):
+        self._finalize()
+        filepath = Path(filepath or self.filepath)
+        filepath.parent.mkdir(exist_ok=True, parents=True)
+        # cls.out_file.unlink(missing_ok=True)
+        with filepath.open('wb') as f:
+            if only_deltas:
+                from deepdiff import DeepDiff
+                diff_dict = [DeepDiff(t1, t2, ignore_order=True)
+                             for t1, t2 in zip(self._recorder_out_list, self._recorder_out_list[1:])
+                             ]
+                out_dict = {'episodes': diff_dict}
+
+            else:
+                # TODO Protobuff Adjustments Revert
+                dest_prop = dict(
+                    n_dests=0,
+                    dwell_time=0,
+                    spawn_frequency=0,
+                    spawn_in_other_zone=False,
+                    spawn_mode=''
+                )
+                rewards_dest = dict(
+                    WAIT_VALID=0.00,
+                    WAIT_FAIL=0.00,
+                    DEST_REACHED=0.00,
+                )
+                mv_prop = dict(
+                    allow_square_movement=False,
+                    allow_diagonal_movement=False,
+                    allow_no_op=False,
+                )
+                obs_prop = dict(
+                    render_agents='',
+                    omit_agent_self=False,
+                    additional_agent_placeholder=0,
+                    cast_shadows=False,
+                    frames_to_stack=0,
+                    pomdp_r=self.env.params['General']['pomdp_r'],
+                    indicate_door_area=False,
+                    show_global_position_info=False,
+
+                )
+                rewards_base = dict(
+                    MOVEMENTS_VALID=0.00,
+                    MOVEMENTS_FAIL=0.00,
+                    NOOP=0.00,
+                    USE_DOOR_VALID=0.00,
+                    USE_DOOR_FAIL=0.00,
+                    COLLISION=0.00,
+
+                )
+
+                out_dict = {'episodes': self._recorder_out_list}
+            out_dict.update(
+                {'n_episodes': self._curr_episode,
+                 'metadata':dict(
+                     level_name=self.env.params['General']['level_name'],
+                     verbose=False,
+                     n_agents=len(self.env.params['Agents']),
+                     max_steps=100,
+                     done_at_collision=False,
+                     parse_doors=True,
+                     doors_have_area=False,
+                     individual_rewards=True,
+                     class_name='Where does this end up?',
+                     env_seed=69,
+
+                     dest_prop=dest_prop,
+                     rewards_dest=rewards_dest,
+                     mv_prop=mv_prop,
+                     obs_prop=obs_prop,
+                     rewards_base=rewards_base,
+                 ),
+                 # 'env_params': self.env.params,
+                 'header': self.env.summarize_header()
+                 })
+            try:
+                from marl_factory_grid.utils.proto import fiksProto_pb2
+                from google.protobuf import json_format
+
+                bulk = fiksProto_pb2.Bulk()
+                json_format.ParseDict(out_dict, bulk)
+                f.write(bulk.SerializeToString())
+                # yaml.dump(out_dict, f, indent=4)
+            except TypeError:
+                print('Shit')
+        print('done')
+
+        if save_occupation_map:
+            a = np.zeros((15, 15))
+            # noinspection PyTypeChecker
+            for episode in out_dict['episodes']:
+                df = pd.DataFrame([y for x in episode['steps'] for y in x['Agents']])
+
+                b = list(df[['x', 'y']].to_records(index=False))
+
+                np.add.at(a, tuple(zip(*b)), 1)
+
+            # a = np.rot90(a)
+            import seaborn as sns
+            from matplotlib import pyplot as plt
+            hm = sns.heatmap(data=a)
+            hm.set_title('Very Nice Heatmap')
+            plt.show()
+
+        if save_trajectory_map:
+            raise NotImplementedError('This has not yet been implemented.')

marl_factory_grid/utils/plotting/compare_runs.py

@@ -0,0 +1,203 @@
+import pickle
+import re
+from os import PathLike
+from pathlib import Path
+from typing import Union, List
+
+import pandas as pd
+
+from marl_factory_grid.utils.helpers import IGNORED_DF_COLUMNS
+from marl_factory_grid.utils.plotting.plotting import prepare_plot
+
+MODEL_MAP = None
+
+
+def plot_single_run(run_path: Union[str, PathLike], use_tex: bool = False, column_keys=None):
+    run_path = Path(run_path)
+    df_list = list()
+    if run_path.is_dir():
+        monitor_file = next(run_path.glob('*monitor*.pick'))
+    elif run_path.exists() and run_path.is_file():
+        monitor_file = run_path
+    else:
+        raise ValueError
+
+    with monitor_file.open('rb') as f:
+        monitor_df = pickle.load(f)
+
+        monitor_df = monitor_df.fillna(0)
+        df_list.append(monitor_df)
+
+    df = pd.concat(df_list,  ignore_index=True)
+    df = df.fillna(0).rename(columns={'episode': 'Episode'}).sort_values(['Episode'])
+    if column_keys is not None:
+        columns = [col for col in column_keys if col in df.columns]
+    else:
+        columns = [col for col in df.columns if col not in IGNORED_DF_COLUMNS]
+
+    roll_n = 50
+
+    non_overlapp_window = df.groupby(['Episode']).rolling(roll_n, min_periods=1).mean()
+
+    df_melted = df[columns + ['Episode']].reset_index().melt(
+        id_vars=['Episode'], value_vars=columns, var_name="Measurement", value_name="Score"
+    )
+
+    if df_melted['Episode'].max() > 800:
+        skip_n = round(df_melted['Episode'].max() * 0.02)
+        df_melted = df_melted[df_melted['Episode'] % skip_n == 0]
+
+    prepare_plot(run_path.parent / f'{run_path.parent.name}_monitor_lineplot.png', df_melted, use_tex=use_tex)
+    print('Plotting done.')
+
+
+def compare_seed_runs(run_path: Union[str, PathLike], use_tex: bool = False):
+    run_path = Path(run_path)
+    df_list = list()
+    for run, monitor_file in enumerate(run_path.rglob('monitor*.pick')):
+        with monitor_file.open('rb') as f:
+            monitor_df = pickle.load(f)
+
+        monitor_df['run'] = run
+        monitor_df = monitor_df.fillna(0)
+        df_list.append(monitor_df)
+
+    df = pd.concat(df_list,  ignore_index=True)
+    df = df.fillna(0).rename(columns={'episode': 'Episode', 'run': 'Run'}).sort_values(['Run', 'Episode'])
+    columns = [col for col in df.columns if col not in IGNORED_DF_COLUMNS]
+
+    roll_n = 50
+
+    non_overlapp_window = df.groupby(['Run', 'Episode']).rolling(roll_n, min_periods=1).mean()
+
+    df_melted = non_overlapp_window[columns].reset_index().melt(id_vars=['Episode', 'Run'],
+                                                                value_vars=columns, var_name="Measurement",
+                                                                value_name="Score")
+
+    if df_melted['Episode'].max() > 800:
+        skip_n = round(df_melted['Episode'].max() * 0.02)
+        df_melted = df_melted[df_melted['Episode'] % skip_n == 0]
+
+    run_path.mkdir(parents=True, exist_ok=True)
+    if run_path.exists() and run_path.is_file():
+        prepare_plot(run_path.parent / f'{run_path.name}_monitor_lineplot.png', df_melted, use_tex=use_tex)
+    else:
+        prepare_plot(run_path / f'{run_path.name}_monitor_lineplot.png', df_melted, use_tex=use_tex)
+    print('Plotting done.')
+
+
+def compare_model_runs(run_path: Path, run_identifier: Union[str, int], parameter: Union[str, List[str]],
+                       use_tex: bool = False):
+    run_path = Path(run_path)
+    df_list = list()
+    parameter = [parameter] if isinstance(parameter, str) else parameter
+    for path in run_path.iterdir():
+        if path.is_dir() and str(run_identifier) in path.name:
+            for run, monitor_file in enumerate(path.rglob('monitor*.pick')):
+                with monitor_file.open('rb') as f:
+                    monitor_df = pickle.load(f)
+
+                monitor_df['run'] = run
+                monitor_df['model'] = next((x for x in path.name.split('_') if x in MODEL_MAP.keys()))
+                monitor_df = monitor_df.fillna(0)
+                df_list.append(monitor_df)
+
+    df = pd.concat(df_list, ignore_index=True)
+    df = df.fillna(0).rename(columns={'episode': 'Episode', 'run': 'Run', 'model': 'Model'})
+    columns = [col for col in df.columns if col in parameter]
+
+    last_episode_to_report = min(df.groupby(['Model'])['Episode'].max())
+    df = df[df['Episode'] < last_episode_to_report]
+
+    roll_n = 40
+    non_overlapp_window = df.groupby(['Model', 'Run', 'Episode']).rolling(roll_n, min_periods=1).mean()
+
+    df_melted = non_overlapp_window[columns].reset_index().melt(id_vars=['Episode', 'Run', 'Model'],
+                                                                value_vars=columns, var_name="Measurement",
+                                                                value_name="Score")
+
+    if df_melted['Episode'].max() > 80:
+        skip_n = round(df_melted['Episode'].max() * 0.02)
+        df_melted = df_melted[df_melted['Episode'] % skip_n == 0]
+
+    style = 'Measurement' if len(columns) > 1 else None
+    prepare_plot(run_path / f'{run_identifier}_compare_{parameter}.png', df_melted, hue='Model', style=style,
+                 use_tex=use_tex)
+    print('Plotting done.')
+
+
+def compare_all_parameter_runs(run_root_path: Path, parameter: Union[str, List[str]],
+                               param_names: Union[List[str], None] = None, str_to_ignore='', use_tex: bool = False):
+    run_root_path = Path(run_root_path)
+    df_list = list()
+    parameter = [parameter] if isinstance(parameter, str) else parameter
+    for monitor_idx, monitor_file in enumerate(run_root_path.rglob('monitor*.pick')):
+        with monitor_file.open('rb') as f:
+            monitor_df = pickle.load(f)
+
+        params = [x.name for x in monitor_file.parents if x.parent not in run_root_path.parents]
+        if str_to_ignore:
+            params = [re.sub(f'_*({str_to_ignore})', '', param) for param in params]
+
+        if monitor_idx == 0:
+            if param_names is not None:
+                if len(param_names) < len(params):
+                    # FIXME: Missing Seed Detection, see below @111
+                    param_names = [next(param_names) if param not in MODEL_MAP.keys() else 'Model' for param in params]
+                elif len(param_names) == len(params):
+                    pass
+                else:
+                    raise ValueError
+            else:
+                param_names = []
+        for param_idx, param in enumerate(params):
+            dtype = None
+            if param in MODEL_MAP.keys():
+                param_name = 'Model'
+            elif '_' in param:
+                param_split = param.split('_')
+                if len(param_split) == 2 and any(split in MODEL_MAP.keys() for split in param_split):
+                    # Extract the seed
+                    param = int(next(x for x in param_split if x not in MODEL_MAP))
+                    param_name = 'Seed'
+                    dtype = int
+                else:
+                    param_name = f'param_{param_idx}'
+            else:
+                param_name = f'param_{param_idx}'
+            dtype = dtype if dtype is not None else str
+            monitor_df[param_name] = str(param)
+            monitor_df[param_name] = monitor_df[param_name].astype(dtype)
+            if monitor_idx == 0:
+                param_names.append(param_name)
+
+        monitor_df = monitor_df.fillna(0)
+        df_list.append(monitor_df)
+
+    df = pd.concat(df_list, ignore_index=True)
+    df = df.fillna(0).rename(columns={'episode': 'Episode'}).sort_values(['Episode'])
+
+    for param_name in param_names:
+        df[param_name] = df[param_name].astype(str)
+    columns = [col for col in df.columns if col in parameter]
+
+    last_episode_to_report = min(df.groupby(['Model'])['Episode'].max())
+    df = df[df['Episode'] < last_episode_to_report]
+
+    if df['Episode'].max() > 80:
+        skip_n = round(df['Episode'].max() * 0.02)
+        df = df[df['Episode'] % skip_n == 0]
+    combinations = [x for x in param_names if x not in ['Model', 'Seed']]
+    df['Parameter Combination'] = df[combinations].apply(lambda row: '_'.join(row.values.astype(str)), axis=1)
+    df.drop(columns=combinations, inplace=True)
+
+    # non_overlapp_window = df.groupby(param_names).sum()
+
+    df_melted = df.reset_index().melt(id_vars=['Parameter Combination', 'Episode'],
+                                      value_vars=columns, var_name="Measurement",
+                                      value_name="Score")
+
+    style = 'Measurement' if len(columns) > 1 else None
+    prepare_plot(run_root_path / f'compare_{parameter}.png', df_melted, hue='Parameter Combination',
+                 style=style, use_tex=use_tex)
+    print('Plotting done.')

marl_factory_grid/utils/plotting/plotting.py

@@ -0,0 +1,85 @@
+import seaborn as sns
+import matplotlib as mpl
+from matplotlib import pyplot as plt
+
+PALETTE = 10 * (
+    "#377eb8",
+    "#4daf4a",
+    "#984ea3",
+    "#e41a1c",
+    "#ff7f00",
+    "#a65628",
+    "#f781bf",
+    "#888888",
+    "#a6cee3",
+    "#b2df8a",
+    "#cab2d6",
+    "#fb9a99",
+    "#fdbf6f",
+)
+
+
+def plot(filepath, ext='png'):
+    plt.tight_layout()
+    figure = plt.gcf()
+    ax = plt.gca()
+    legends = [c for c in ax.get_children() if isinstance(c, mpl.legend.Legend)]
+
+    if legends:
+        figure.savefig(str(filepath), format=ext,  bbox_extra_artists=(*legends,), bbox_inches='tight')
+    else:
+        figure.savefig(str(filepath), format=ext)
+
+    plt.show()
+    plt.clf()
+
+
+def prepare_tex(df, hue, style, hue_order):
+    sns.set(rc={'text.usetex': True}, style='whitegrid')
+    lineplot = sns.lineplot(data=df, x='Episode', y='Score', ci=95, palette=PALETTE,
+                            hue_order=hue_order, hue=hue, style=style)
+    lineplot.set_title(f'{sorted(list(df["Measurement"].unique()))}')
+    plt.legend(bbox_to_anchor=(1.02, 1), loc='upper left', borderaxespad=0)
+    plt.tight_layout()
+    return lineplot
+
+
+def prepare_plt(df, hue, style, hue_order):
+    print('Struggling to plot Figure using LaTeX - going back to normal.')
+    plt.close('all')
+    sns.set(rc={'text.usetex': False}, style='whitegrid')
+    lineplot = sns.lineplot(data=df, x='Episode', y='Score', hue=hue, style=style,
+                            ci=95, palette=PALETTE, hue_order=hue_order, )
+    plt.legend(bbox_to_anchor=(1.02, 1), loc='upper left', borderaxespad=0)
+    plt.tight_layout()
+    # lineplot.set_title(f'{sorted(list(df["Measurement"].unique()))}')
+    return lineplot
+
+
+def prepare_center_double_column_legend(df, hue, style, hue_order):
+    print('Struggling to plot Figure using LaTeX - going back to normal.')
+    plt.close('all')
+    sns.set(rc={'text.usetex': False}, style='whitegrid')
+    fig = plt.figure(figsize=(10, 11))
+    lineplot = sns.lineplot(data=df, x='Episode', y='Score', hue=hue, style=style,
+                            ci=95, palette=PALETTE, hue_order=hue_order, legend=False)
+    # plt.legend(bbox_to_anchor=(1.02, 1), loc='upper left', borderaxespad=0)
+    lineplot.legend(hue_order, ncol=3, loc='lower center', title='Parameter Combinations', bbox_to_anchor=(0.5, -0.43))
+    plt.tight_layout()
+    return lineplot
+
+
+def prepare_plot(filepath, results_df, ext='png', hue='Measurement', style=None, use_tex=False):
+    df = results_df.copy()
+    df[hue] = df[hue].str.replace('_', '-')
+    hue_order = sorted(list(df[hue].unique()))
+    if use_tex:
+        try:
+            _ = prepare_tex(df, hue, style, hue_order)
+            plot(filepath, ext=ext)  # plot raises errors not lineplot!
+        except (FileNotFoundError, RuntimeError):
+            _ = prepare_plt(df, hue, style, hue_order)
+            plot(filepath, ext=ext)
+    else:
+        _ = prepare_plt(df, hue, style, hue_order)
+        plot(filepath, ext=ext)

marl_factory_grid/utils/tools.py

@@ -120,6 +120,7 @@ class ConfigExplainer:
 
     def _save_to_file(self, data: dict, filepath: PathLike, tag: str = ''):
         filepath = Path(filepath)
+        yaml.Dumper.ignore_aliases = lambda *args: True
         with filepath.open('w') as f:
             yaml.dump(data, f, encoding='utf-8')
         print(f'Example config {"for " + tag + " " if tag else " "}dumped')