Adjustments and Documentation, recording and new environments, refactoring

environments/factory/additional/btry/btry_collections.py

@@ -0,0 +1,50 @@
+from environments.factory.additional.btry.btry_objects import Battery, ChargePod
+from environments.factory.base.registers import EnvObjectCollection, EntityCollection
+
+
+class BatteriesRegister(EnvObjectCollection):
+
+    _accepted_objects = Battery
+
+    def __init__(self, *args, **kwargs):
+        super(BatteriesRegister, self).__init__(*args, individual_slices=True,
+                                                is_blocking_light=False, can_be_shadowed=False, **kwargs)
+        self.is_observable = True
+
+    def spawn_batteries(self, agents, initial_charge_level):
+        batteries = [self._accepted_objects(initial_charge_level, agent, self) for _, agent in enumerate(agents)]
+        self.add_additional_items(batteries)
+
+    def summarize_states(self, n_steps=None):
+        # as dict with additional nesting
+        # return dict(items=super(Inventories, cls).summarize_states())
+        return super(BatteriesRegister, self).summarize_states(n_steps=n_steps)
+
+    # Todo Move this to Mixin!
+    def by_entity(self, entity):
+        try:
+            return next((x for x in self if x.belongs_to_entity(entity)))
+        except StopIteration:
+            return None
+
+    def idx_by_entity(self, entity):
+        try:
+            return next((idx for idx, x in enumerate(self) if x.belongs_to_entity(entity)))
+        except StopIteration:
+            return None
+
+    def as_array_by_entity(self, entity):
+        return self._array[self.idx_by_entity(entity)]
+
+
+class ChargePods(EntityCollection):
+
+    _accepted_objects = ChargePod
+
+    def __repr__(self):
+        super(ChargePods, self).__repr__()
+
+    def summarize_states(self, n_steps=None):
+        # as dict with additional nesting
+        # return dict(items=super(Inventories, cls).summarize_states())
+        return super(ChargePods, self).summarize_states(n_steps=n_steps)

environments/factory/additional/btry/btry_objects.py

@@ -0,0 +1,67 @@
+from environments import helpers as h
+from environments.factory.base.objects import BoundingMixin, EnvObject, Entity
+from environments.factory.additional.btry.btry_util import Constants as c
+
+
+class Battery(BoundingMixin, EnvObject):
+
+    @property
+    def is_discharged(self):
+        return self.charge_level == 0
+
+    def __init__(self, initial_charge_level: float, *args, **kwargs):
+        super(Battery, self).__init__(*args, **kwargs)
+        self.charge_level = initial_charge_level
+
+    def encoding(self):
+        return self.charge_level
+
+    def do_charge_action(self, amount):
+        if self.charge_level < 1:
+            # noinspection PyTypeChecker
+            self.charge_level = min(1, amount + self.charge_level)
+            return c.VALID
+        else:
+            return c.NOT_VALID
+
+    def decharge(self, amount) -> c:
+        if self.charge_level != 0:
+            # noinspection PyTypeChecker
+            self.charge_level = max(0, amount + self.charge_level)
+            self._collection.notify_change_to_value(self)
+            return c.VALID
+        else:
+            return c.NOT_VALID
+
+    def summarize_state(self, **_):
+        attr_dict = {key: str(val) for key, val in self.__dict__.items() if not key.startswith('_') and key != 'data'}
+        attr_dict.update(dict(name=self.name))
+        return attr_dict
+
+
+class ChargePod(Entity):
+
+    @property
+    def encoding(self):
+        return c.CHARGE_POD
+
+    def __init__(self, *args, charge_rate: float = 0.4,
+                 multi_charge: bool = False, **kwargs):
+        super(ChargePod, self).__init__(*args, **kwargs)
+        self.charge_rate = charge_rate
+        self.multi_charge = multi_charge
+
+    def charge_battery(self, battery: Battery):
+        if battery.charge_level == 1.0:
+            return c.NOT_VALID
+        if sum(guest for guest in self.tile.guests if 'agent' in guest.name.lower()) > 1:
+            return c.NOT_VALID
+        valid = battery.do_charge_action(self.charge_rate)
+        return valid
+
+    def summarize_state(self, n_steps=None) -> dict:
+        if n_steps == h.STEPS_START:
+            summary = super().summarize_state(n_steps=n_steps)
+            return summary
+        else:
+            {}

environments/factory/additional/btry/btry_util.py

@@ -0,0 +1,30 @@
+from typing import NamedTuple, Union
+
+from environments.helpers import Constants as BaseConstants, EnvActions as BaseActions
+
+
+class Constants(BaseConstants):
+    # Battery Env
+    CHARGE_PODS          = 'Charge_Pod'
+    BATTERIES            = 'BATTERIES'
+    BATTERY_DISCHARGED   = 'DISCHARGED'
+    CHARGE_POD           = 1
+
+
+class Actions(BaseActions):
+    CHARGE              = 'do_charge_action'
+
+
+class RewardsBtry(NamedTuple):
+    CHARGE_VALID: float        = 0.1
+    CHARGE_FAIL: float         = -0.1
+    BATTERY_DISCHARGED: float  = -1.0
+
+
+class BatteryProperties(NamedTuple):
+    initial_charge: float = 0.8             #
+    charge_rate: float = 0.4                #
+    charge_locations: int = 20               #
+    per_action_costs: Union[dict, float] = 0.02
+    done_when_discharged: bool = False
+    multi_charge: bool = False

environments/factory/additional/btry/factory_battery.py

@@ -0,0 +1,139 @@
+from typing import Dict, List
+
+import numpy as np
+
+from environments.factory.additional.btry.btry_collections import BatteriesRegister, ChargePods
+from environments.factory.additional.btry.btry_util import Constants, Actions, RewardsBtry, BatteryProperties
+from environments.factory.base.base_factory import BaseFactory
+from environments.factory.base.objects import Agent, Action
+from environments.factory.base.renderer import RenderEntity
+
+c = Constants
+a = Actions
+
+
+class BatteryFactory(BaseFactory):
+
+    def __init__(self, *args, btry_prop=BatteryProperties(), rewards_btry: RewardsBtry = RewardsBtry(),
+                 **kwargs):
+        if isinstance(btry_prop, dict):
+            btry_prop = BatteryProperties(**btry_prop)
+        if isinstance(rewards_btry, dict):
+            rewards_btry = RewardsBtry(**rewards_btry)
+        self.btry_prop = btry_prop
+        self.rewards_dest = rewards_btry
+        super().__init__(*args, **kwargs)
+
+    def per_agent_raw_observations_hook(self, agent) -> Dict[str, np.typing.ArrayLike]:
+        additional_raw_observations = super().per_agent_raw_observations_hook(agent)
+        additional_raw_observations.update({c.BATTERIES: self[c.BATTERIES].as_array_by_entity(agent)})
+        return additional_raw_observations
+
+    def observations_hook(self) -> Dict[str, np.typing.ArrayLike]:
+        additional_observations = super().observations_hook()
+        additional_observations.update({c.CHARGE_PODS: self[c.CHARGE_PODS].as_array()})
+        return additional_observations
+
+    @property
+    def entities_hook(self):
+        super_entities = super().entities_hook
+
+        empty_tiles = self[c.FLOOR].empty_tiles[:self.btry_prop.charge_locations]
+        charge_pods = ChargePods.from_tiles(
+            empty_tiles, self._level_shape,
+            entity_kwargs=dict(charge_rate=self.btry_prop.charge_rate,
+                               multi_charge=self.btry_prop.multi_charge)
+        )
+
+        batteries = BatteriesRegister(self._level_shape if not self._pomdp_r else ((self.pomdp_diameter,) * 2),
+                                      )
+        batteries.spawn_batteries(self[c.AGENT], self.btry_prop.initial_charge)
+        super_entities.update({c.BATTERIES: batteries, c.CHARGE_PODS: charge_pods})
+        return super_entities
+
+    def step_hook(self) -> (List[dict], dict):
+        super_reward_info = super(BatteryFactory, self).step_hook()
+
+        # Decharge
+        batteries = self[c.BATTERIES]
+
+        for agent in self[c.AGENT]:
+            if isinstance(self.btry_prop.per_action_costs, dict):
+                energy_consumption = self.btry_prop.per_action_costs[agent.temp_action]
+            else:
+                energy_consumption = self.btry_prop.per_action_costs
+
+            batteries.by_entity(agent).decharge(energy_consumption)
+
+        return super_reward_info
+
+    def do_charge_action(self, agent) -> (dict, dict):
+        if charge_pod := self[c.CHARGE_PODS].by_pos(agent.pos):
+            valid = charge_pod.charge_battery(self[c.BATTERIES].by_entity(agent))
+            if valid:
+                info_dict = {f'{agent.name}_{a.CHARGE}_VALID': 1}
+                self.print(f'{agent.name} just charged batteries at {charge_pod.name}.')
+            else:
+                info_dict = {f'{agent.name}_{a.CHARGE}_FAIL': 1}
+                self.print(f'{agent.name} failed to charged batteries at {charge_pod.name}.')
+        else:
+            valid = c.NOT_VALID
+            info_dict = {f'{agent.name}_{a.CHARGE}_FAIL': 1}
+            # info_dict = {f'{agent.name}_no_charger': 1}
+            self.print(f'{agent.name} failed to charged batteries at {agent.pos}.')
+        reward = dict(value=self.rewards_dest.CHARGE_VALID if valid else self.rewards_dest.CHARGE_FAIL,
+                      reason=a.CHARGE, info=info_dict)
+        return valid, reward
+
+    def do_additional_actions(self, agent: Agent, action: Action) -> (bool, dict):
+        action_result = super().do_additional_actions(agent, action)
+        if action_result is None:
+            if action == a.CHARGE:
+                action_result = self.do_charge_action(agent)
+                return action_result
+            else:
+                return None
+        else:
+            return action_result
+        pass
+
+    def reset_hook(self) -> (List[dict], dict):
+        super_reward_info = super(BatteryFactory, self).reset_hook()
+        # There is Nothing to reset.
+        return super_reward_info
+
+    def check_additional_done(self) -> (bool, dict):
+        super_done, super_dict = super(BatteryFactory, self).check_additional_done()
+        if super_done:
+            return super_done, super_dict
+        else:
+            if self.btry_prop.done_when_discharged:
+                if btry_done := any(battery.is_discharged for battery in self[c.BATTERIES]):
+                    super_dict.update(DISCHARGE_DONE=1)
+                    return btry_done, super_dict
+                else:
+                    pass
+            else:
+                pass
+        return super_done, super_dict
+
+    def per_agent_reward_hook(self, agent: Agent) -> List[dict]:
+        reward_event_list = super(BatteryFactory, self).per_agent_reward_hook(agent)
+        if self[c.BATTERIES].by_entity(agent).is_discharged:
+            self.print(f'{agent.name} Battery is discharged!')
+            info_dict = {f'{agent.name}_{c.BATTERY_DISCHARGED}': 1}
+            reward_event_list.append({'value': self.rewards_dest.BATTERY_DISCHARGED,
+                                      'reason': c.BATTERY_DISCHARGED,
+                                      'info': info_dict}
+                                     )
+        else:
+            # All Fine
+            pass
+        return reward_event_list
+
+    def render_assets_hook(self):
+        # noinspection PyUnresolvedReferences
+        additional_assets = super().render_assets_hook()
+        charge_pods = [RenderEntity(c.CHARGE_PODS, charge_pod.tile.pos) for charge_pod in self[c.CHARGE_PODS]]
+        additional_assets.extend(charge_pods)
+        return additional_assets

environments/factory/additional/combined_factories.py

@@ -0,0 +1,76 @@
+import random
+
+
+# noinspection PyAbstractClass
+from environments.factory.additional.btry.btry_util import BatteryProperties
+from environments.factory.additional.btry.factory_battery import BatteryFactory
+from environments.factory.additional.dest.factory_dest import DestFactory
+from environments.factory.additional.dirt.dirt_util import DirtProperties
+from environments.factory.additional.dirt.factory_dirt import DirtFactory
+from environments.factory.additional.item.factory_item import ItemFactory
+
+
+class DirtItemFactory(ItemFactory, DirtFactory):
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+
+
+# noinspection PyAbstractClass
+class DirtBatteryFactory(DirtFactory, BatteryFactory):
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+
+
+# noinspection PyAbstractClass
+class DirtDestItemFactory(ItemFactory, DirtFactory, DestFactory):
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+
+
+# noinspection PyAbstractClass
+class DestBatteryFactory(BatteryFactory, DestFactory):
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+
+
+if __name__ == '__main__':
+    from environments.utility_classes import AgentRenderOptions as ARO, ObservationProperties
+
+    render = True
+
+    dirt_props = DirtProperties(1, 0.05, 0.1, 3, 1, 20, 0)
+
+    obs_props = ObservationProperties(render_agents=ARO.COMBINED, omit_agent_self=True,
+                                      pomdp_r=2, additional_agent_placeholder=None)
+
+    move_props = {'allow_square_movement': True,
+                  'allow_diagonal_movement': False,
+                  'allow_no_op': False}
+
+    factory = DirtBatteryFactory(n_agents=5, done_at_collision=False,
+                                 level_name='rooms', max_steps=400,
+                                 obs_prop=obs_props, parse_doors=True,
+                                 record_episodes=True, verbose=True,
+                                 btry_prop=BatteryProperties(),
+                                 mv_prop=move_props, dirt_prop=dirt_props
+                                 )
+
+    # noinspection DuplicatedCode
+    n_actions = factory.action_space.n - 1
+    _ = factory.observation_space
+
+    for epoch in range(4):
+        random_actions = [[random.randint(0, n_actions) for _
+                           in range(factory.n_agents)] for _
+                          in range(factory.max_steps + 1)]
+        env_state = factory.reset()
+        r = 0
+        for agent_i_action in random_actions:
+            env_state, step_r, done_bool, info_obj = factory.step(agent_i_action)
+            r += step_r
+            if render:
+                factory.render()
+            if done_bool:
+                break
+        print(f'Factory run {epoch} done, reward is:\n    {r}')
+pass

environments/factory/additional/dest/dest_collections.py

@@ -0,0 +1,44 @@
+from environments.factory.base.registers import EntityCollection
+from environments.factory.additional.dest.dest_util import Constants as c
+from environments.factory.additional.dest.dest_enitites import Destination
+
+
+class Destinations(EntityCollection):
+
+    _accepted_objects = Destination
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.is_blocking_light = False
+        self.can_be_shadowed = False
+
+    def as_array(self):
+        self._array[:] = c.FREE_CELL
+        # ToDo: Switch to new Style Array Put
+        # indices = list(zip(range(len(cls)), *zip(*[x.pos for x in cls])))
+        # np.put(cls._array, [np.ravel_multi_index(x, cls._array.shape) for x in indices], cls.encodings)
+        for item in self:
+            if item.pos != c.NO_POS:
+                self._array[0, item.x, item.y] = item.encoding
+        return self._array
+
+    def __repr__(self):
+        return super(Destinations, self).__repr__()
+
+    def summarize_states(self, n_steps=None):
+        return {}
+
+
+class ReachedDestinations(Destinations):
+    _accepted_objects = Destination
+
+    def __init__(self, *args, **kwargs):
+        super(ReachedDestinations, self).__init__(*args, **kwargs)
+        self.can_be_shadowed = False
+        self.is_blocking_light = False
+
+    def summarize_states(self, n_steps=None):
+        return {}
+
+    def __repr__(self):
+        return super(ReachedDestinations, self).__repr__()

environments/factory/additional/dest/dest_enitites.py

@@ -0,0 +1,44 @@
+from collections import defaultdict
+
+from environments.factory.base.objects import Entity, Agent
+from environments.factory.additional.dest.dest_util import Constants as c
+
+
+class Destination(Entity):
+
+    @property
+    def any_agent_has_dwelled(self):
+        return bool(len(self._per_agent_times))
+
+    @property
+    def currently_dwelling_names(self):
+        return self._per_agent_times.keys()
+
+    @property
+    def encoding(self):
+        return c.DESTINATION
+
+    def __init__(self, *args, dwell_time: int = 0, **kwargs):
+        super(Destination, self).__init__(*args, **kwargs)
+        self.dwell_time = dwell_time
+        self._per_agent_times = defaultdict(lambda: dwell_time)
+
+    def do_wait_action(self, agent: Agent):
+        self._per_agent_times[agent.name] -= 1
+        return c.VALID
+
+    def leave(self, agent: Agent):
+        del self._per_agent_times[agent.name]
+
+    @property
+    def is_considered_reached(self):
+        agent_at_position = any(c.AGENT.lower() in x.name.lower() for x in self.tile.guests_that_can_collide)
+        return (agent_at_position and not self.dwell_time) or any(x == 0 for x in self._per_agent_times.values())
+
+    def agent_is_dwelling(self, agent: Agent):
+        return self._per_agent_times[agent.name] < self.dwell_time
+
+    def summarize_state(self, n_steps=None) -> dict:
+        state_summary = super().summarize_state(n_steps=n_steps)
+        state_summary.update(per_agent_times=self._per_agent_times)
+        return state_summary

environments/factory/additional/dest/dest_util.py

@@ -0,0 +1,41 @@
+from typing import NamedTuple
+
+from environments.helpers import Constants as BaseConstants, EnvActions as BaseActions
+
+
+class Constants(BaseConstants):
+    # Destination Env
+    DEST                    = 'Destination'
+    DESTINATION             = 1
+    DESTINATION_DONE        = 0.5
+    DEST_REACHED            = 'ReachedDestination'
+
+
+class Actions(BaseActions):
+    WAIT_ON_DEST    = 'WAIT'
+
+
+class RewardsDest(NamedTuple):
+
+    WAIT_VALID: float      = 0.1
+    WAIT_FAIL: float       = -0.1
+    DEST_REACHED: float    = 5.0
+
+
+class DestModeOptions(object):
+    DONE        = 'DONE'
+    GROUPED     = 'GROUPED'
+    PER_DEST    = 'PER_DEST'
+
+
+class DestProperties(NamedTuple):
+    n_dests:                                     int = 1     # How many destinations are there
+    dwell_time:                                  int = 0     # How long does the agent need to "wait" on a destination
+    spawn_frequency:                             int = 0
+    spawn_in_other_zone:                        bool = True  #
+    spawn_mode:                                  str = DestModeOptions.DONE
+
+    assert dwell_time >= 0, 'dwell_time cannot be < 0!'
+    assert spawn_frequency >= 0, 'spawn_frequency cannot be < 0!'
+    assert n_dests >= 0, 'n_destinations cannot be < 0!'
+    assert (spawn_mode == DestModeOptions.DONE) != bool(spawn_frequency)

environments/factory/additional/dest/factory_dest.py

@@ -0,0 +1,203 @@
+import time
+from enum import Enum
+from typing import List, Union, Dict
+import numpy as np
+import random
+
+from environments.factory.additional.dest.dest_collections import Destinations, ReachedDestinations
+from environments.factory.additional.dest.dest_enitites import Destination
+from environments.factory.additional.dest.dest_util import Constants, Actions, RewardsDest, DestModeOptions, \
+    DestProperties
+from environments.factory.base.base_factory import BaseFactory
+from environments.factory.base.objects import Agent, Action
+from environments.factory.base.registers import Entities
+
+from environments.factory.base.renderer import RenderEntity
+
+c = Constants
+a = Actions
+
+
+# noinspection PyAttributeOutsideInit, PyAbstractClass
+class DestFactory(BaseFactory):
+    # noinspection PyMissingConstructor
+
+    def __init__(self, *args, dest_prop: DestProperties = DestProperties(), rewards_dest: RewardsDest = RewardsDest(),
+                 env_seed=time.time_ns(), **kwargs):
+        if isinstance(dest_prop, dict):
+            dest_prop = DestProperties(**dest_prop)
+        if isinstance(rewards_dest, dict):
+            rewards_dest = RewardsDest(**rewards_dest)
+        self.dest_prop = dest_prop
+        self.rewards_dest = rewards_dest
+        kwargs.update(env_seed=env_seed)
+        self._dest_rng = np.random.default_rng(env_seed)
+        super().__init__(*args, **kwargs)
+
+    @property
+    def actions_hook(self) -> Union[Action, List[Action]]:
+        # noinspection PyUnresolvedReferences
+        super_actions = super().actions_hook
+        # If targets are considers reached after some time, agents need an action for that.
+        if self.dest_prop.dwell_time:
+            super_actions.append(Action(enum_ident=a.WAIT_ON_DEST))
+        return super_actions
+
+    @property
+    def entities_hook(self) -> Dict[(Enum, Entities)]:
+        # noinspection PyUnresolvedReferences
+        super_entities = super().entities_hook
+
+        empty_tiles = self[c.FLOOR].empty_tiles[:self.dest_prop.n_dests]
+        destinations = Destinations.from_tiles(
+            empty_tiles, self._level_shape,
+            entity_kwargs=dict(
+                dwell_time=self.dest_prop.dwell_time)
+        )
+        reached_destinations = ReachedDestinations(level_shape=self._level_shape)
+
+        super_entities.update({c.DEST: destinations, c.DEST_REACHED: reached_destinations})
+        return super_entities
+
+    def do_wait_action(self, agent: Agent) -> (dict, dict):
+        if destination := self[c.DEST].by_pos(agent.pos):
+            valid = destination.do_wait_action(agent)
+            self.print(f'{agent.name} just waited at {agent.pos}')
+            info_dict = {f'{agent.name}_{a.WAIT_ON_DEST}_VALID': 1}
+        else:
+            valid = c.NOT_VALID
+            self.print(f'{agent.name} just tried to do_wait_action do_wait_action at {agent.pos} but failed')
+            info_dict = {f'{agent.name}_{a.WAIT_ON_DEST}_FAIL': 1}
+        reward = dict(value=self.rewards_dest.WAIT_VALID if valid else self.rewards_dest.WAIT_FAIL,
+                      reason=a.WAIT_ON_DEST, info=info_dict)
+        return valid, reward
+
+    def do_additional_actions(self, agent: Agent, action: Action) -> (dict, dict):
+        # noinspection PyUnresolvedReferences
+        super_action_result = super().do_additional_actions(agent, action)
+        if super_action_result is None:
+            if action == a.WAIT_ON_DEST:
+                action_result = self.do_wait_action(agent)
+                return action_result
+            else:
+                return None
+        else:
+            return super_action_result
+
+    def reset_hook(self) -> None:
+        # noinspection PyUnresolvedReferences
+        super().reset_hook()
+        self._dest_spawn_timer = dict()
+
+    def trigger_destination_spawn(self):
+        destinations_to_spawn = [key for key, val in self._dest_spawn_timer.items()
+                                 if val == self.dest_prop.spawn_frequency]
+        if destinations_to_spawn:
+            n_dest_to_spawn = len(destinations_to_spawn)
+            if self.dest_prop.spawn_mode != DestModeOptions.GROUPED:
+                destinations = [Destination(tile, self[c.DEST]) for tile in self[c.FLOOR].empty_tiles[:n_dest_to_spawn]]
+                self[c.DEST].add_additional_items(destinations)
+                for dest in destinations_to_spawn:
+                    del self._dest_spawn_timer[dest]
+                self.print(f'{n_dest_to_spawn} new destinations have been spawned')
+            elif self.dest_prop.spawn_mode == DestModeOptions.GROUPED and n_dest_to_spawn == self.dest_prop.n_dests:
+                destinations = [Destination(tile, self[c.DEST]) for tile in self[c.FLOOR].empty_tiles[:n_dest_to_spawn]]
+                self[c.DEST].add_additional_items(destinations)
+                for dest in destinations_to_spawn:
+                    del self._dest_spawn_timer[dest]
+                self.print(f'{n_dest_to_spawn} new destinations have been spawned')
+            else:
+                self.print(f'{n_dest_to_spawn} new destinations could be spawned, but waiting for all.')
+                pass
+        else:
+            self.print('No Items are spawning, limit is reached.')
+
+    def step_hook(self) -> (List[dict], dict):
+        # noinspection PyUnresolvedReferences
+        super_reward_info = super().step_hook()
+        for key, val in self._dest_spawn_timer.items():
+            self._dest_spawn_timer[key] = min(self.dest_prop.spawn_frequency, self._dest_spawn_timer[key] + 1)
+
+        for dest in list(self[c.DEST].values()):
+            if dest.is_considered_reached:
+                dest.change_parent_collection(self[c.DEST_REACHED])
+                self._dest_spawn_timer[dest.name] = 0
+                self.print(f'{dest.name} is reached now, removing...')
+            else:
+                for agent_name in dest.currently_dwelling_names:
+                    agent = self[c.AGENT].by_name(agent_name)
+                    if agent.pos == dest.pos:
+                        self.print(f'{agent.name} is still waiting.')
+                        pass
+                    else:
+                        dest.leave(agent)
+                        self.print(f'{agent.name} left the destination early.')
+        self.trigger_destination_spawn()
+        return super_reward_info
+
+    def observations_hook(self) -> Dict[str, np.typing.ArrayLike]:
+        additional_observations = super().observations_hook()
+        additional_observations.update({c.DEST: self[c.DEST].as_array()})
+        return additional_observations
+
+    def per_agent_reward_hook(self, agent: Agent) -> List[dict]:
+        # noinspection PyUnresolvedReferences
+        reward_event_list = super().per_agent_reward_hook(agent)
+        if len(self[c.DEST_REACHED]):
+            for reached_dest in list(self[c.DEST_REACHED]):
+                if agent.pos == reached_dest.pos:
+                    self.print(f'{agent.name} just reached destination at {agent.pos}')
+                    self[c.DEST_REACHED].delete_env_object(reached_dest)
+                    info_dict = {f'{agent.name}_{c.DEST_REACHED}': 1}
+                    reward_event_list.append({'value': self.rewards_dest.DEST_REACHED,
+                                              'reason': c.DEST_REACHED,
+                                              'info': info_dict})
+        return reward_event_list
+
+    def render_assets_hook(self, mode='human'):
+        # noinspection PyUnresolvedReferences
+        additional_assets = super().render_assets_hook()
+        destinations = [RenderEntity(c.DEST, dest.pos) for dest in self[c.DEST]]
+        additional_assets.extend(destinations)
+        return additional_assets
+
+
+if __name__ == '__main__':
+    from environments.utility_classes import AgentRenderOptions as aro, ObservationProperties
+
+    render = True
+
+    dest_probs = DestProperties(n_dests=2, spawn_frequency=5, spawn_mode=DestModeOptions.GROUPED)
+
+    obs_props = ObservationProperties(render_agents=aro.LEVEL, omit_agent_self=True, pomdp_r=2)
+
+    move_props = {'allow_square_movement': True,
+                  'allow_diagonal_movement': False,
+                  'allow_no_op': False}
+
+    factory = DestFactory(n_agents=10, done_at_collision=False,
+                          level_name='rooms', max_steps=400,
+                          obs_prop=obs_props, parse_doors=True,
+                          verbose=True,
+                          mv_prop=move_props, dest_prop=dest_probs
+                          )
+
+    # noinspection DuplicatedCode
+    n_actions = factory.action_space.n - 1
+    _ = factory.observation_space
+
+    for epoch in range(4):
+        random_actions = [[random.randint(0, n_actions) for _
+                           in range(factory.n_agents)] for _
+                          in range(factory.max_steps + 1)]
+        env_state = factory.reset()
+        r = 0
+        for agent_i_action in random_actions:
+            env_state, step_r, done_bool, info_obj = factory.step(agent_i_action)
+            r += step_r
+            if render:
+                factory.render()
+            if done_bool:
+                break
+        print(f'Factory run {epoch} done, reward is:\n    {r}')
+    pass

environments/factory/additional/dirt/dirt_collections.py

@@ -0,0 +1,42 @@
+from environments.factory.additional.dirt.dirt_entity import Dirt
+from environments.factory.additional.dirt.dirt_util import DirtProperties
+from environments.factory.base.objects import Floor
+from environments.factory.base.registers import EntityCollection
+from environments.factory.additional.dirt.dirt_util import Constants as c
+
+
+class DirtRegister(EntityCollection):
+
+    _accepted_objects = Dirt
+
+    @property
+    def amount(self):
+        return sum([dirt.amount for dirt in self])
+
+    @property
+    def dirt_properties(self):
+        return self._dirt_properties
+
+    def __init__(self, dirt_properties, *args):
+        super(DirtRegister, self).__init__(*args)
+        self._dirt_properties: DirtProperties = dirt_properties
+
+    def spawn_dirt(self, then_dirty_tiles) -> bool:
+        if isinstance(then_dirty_tiles, Floor):
+            then_dirty_tiles = [then_dirty_tiles]
+        for tile in then_dirty_tiles:
+            if not self.amount > self.dirt_properties.max_global_amount:
+                dirt = self.by_pos(tile.pos)
+                if dirt is None:
+                    dirt = Dirt(tile, self, amount=self.dirt_properties.max_spawn_amount)
+                    self.add_item(dirt)
+                else:
+                    new_value = dirt.amount + self.dirt_properties.max_spawn_amount
+                    dirt.set_new_amount(min(new_value, self.dirt_properties.max_local_amount))
+            else:
+                return c.NOT_VALID
+        return c.VALID
+
+    def __repr__(self):
+        s = super(DirtRegister, self).__repr__()
+        return f'{s[:-1]}, {self.amount})'

environments/factory/additional/dirt/dirt_entity.py

@@ -0,0 +1,26 @@
+from environments.factory.base.objects import Entity
+
+
+class Dirt(Entity):
+
+    @property
+    def amount(self):
+        return self._amount
+
+    @property
+    def encoding(self):
+        # Edit this if you want items to be drawn in the ops differntly
+        return self._amount
+
+    def __init__(self, *args, amount=None, **kwargs):
+        super(Dirt, self).__init__(*args, **kwargs)
+        self._amount = amount
+
+    def set_new_amount(self, amount):
+        self._amount = amount
+        self._collection.notify_change_to_value(self)
+
+    def summarize_state(self, **kwargs):
+        state_dict = super().summarize_state(**kwargs)
+        state_dict.update(amount=float(self.amount))
+        return state_dict

environments/factory/additional/dirt/dirt_util.py

@@ -0,0 +1,30 @@
+from typing import NamedTuple
+
+from environments.helpers import Constants as BaseConstants, EnvActions as BaseActions
+
+
+class Constants(BaseConstants):
+    DIRT = 'Dirt'
+
+
+class Actions(BaseActions):
+    CLEAN_UP = 'do_cleanup_action'
+
+
+class RewardsDirt(NamedTuple):
+    CLEAN_UP_VALID: float          = 0.5
+    CLEAN_UP_FAIL: float           = -0.1
+    CLEAN_UP_LAST_PIECE: float     = 4.5
+
+
+class DirtProperties(NamedTuple):
+    initial_dirt_ratio: float = 0.3         # On INIT, on max how many tiles does the dirt spawn in percent.
+    initial_dirt_spawn_r_var: float = 0.05  # How much does the dirt spawn amount vary?
+    clean_amount: float = 1                 # How much does the robot clean with one actions.
+    max_spawn_ratio: float = 0.20           # On max how many tiles does the dirt spawn in percent.
+    max_spawn_amount: float = 0.3           # How much dirt does spawn per tile at max.
+    spawn_frequency: int = 0                # Spawn Frequency in Steps.
+    max_local_amount: int = 2               # Max dirt amount per tile.
+    max_global_amount: int = 20             # Max dirt amount in the whole environment.
+    dirt_smear_amount: float = 0.2          # Agents smear dirt, when not cleaning up in place.
+    done_when_clean: bool = True

environments/factory/additional/dirt/factory_dirt.py

@@ -0,0 +1,252 @@
+import time
+from pathlib import Path
+from typing import List, Union, Dict
+import random
+
+import numpy as np
+
+from environments.factory.additional.dirt.dirt_collections import DirtRegister
+from environments.factory.additional.dirt.dirt_entity import Dirt
+from environments.factory.additional.dirt.dirt_util import Constants, Actions, RewardsDirt, DirtProperties
+
+from environments.factory.base.base_factory import BaseFactory
+from environments.factory.base.objects import Agent, Action
+from environments.factory.base.registers import Entities
+
+from environments.factory.base.renderer import RenderEntity
+from environments.utility_classes import ObservationProperties
+
+
+def softmax(x):
+    """Compute softmax values for each sets of scores in x."""
+    e_x = np.exp(x - np.max(x))
+    return e_x / e_x.sum()
+
+
+def entropy(x):
+    return -(x * np.log(x + 1e-8)).sum()
+
+
+c = Constants
+a = Actions
+
+
+# noinspection PyAttributeOutsideInit, PyAbstractClass
+class DirtFactory(BaseFactory):
+
+    @property
+    def actions_hook(self) -> Union[Action, List[Action]]:
+        super_actions = super().actions_hook
+        super_actions.append(Action(str_ident=a.CLEAN_UP))
+        return super_actions
+
+    @property
+    def entities_hook(self) -> Dict[(str, Entities)]:
+        super_entities = super().entities_hook
+        dirt_register = DirtRegister(self.dirt_prop, self._level_shape)
+        super_entities.update(({c.DIRT: dirt_register}))
+        return super_entities
+
+    def __init__(self, *args,
+                 dirt_prop: DirtProperties = DirtProperties(), rewards_dirt: RewardsDirt = RewardsDirt(),
+                 env_seed=time.time_ns(), **kwargs):
+        if isinstance(dirt_prop, dict):
+            dirt_prop = DirtProperties(**dirt_prop)
+        if isinstance(rewards_dirt, dict):
+            rewards_dirt = RewardsDirt(**rewards_dirt)
+        self.dirt_prop = dirt_prop
+        self.rewards_dirt = rewards_dirt
+        self._dirt_rng = np.random.default_rng(env_seed)
+        self._dirt: DirtRegister
+        kwargs.update(env_seed=env_seed)
+        # TODO: Reset ---> document this
+        super().__init__(*args, **kwargs)
+
+    def render_assets_hook(self, mode='human'):
+        additional_assets = super().render_assets_hook()
+        dirt = [RenderEntity('dirt', dirt.tile.pos, min(0.15 + dirt.amount, 1.5), 'scale')
+                for dirt in self[c.DIRT]]
+        additional_assets.extend(dirt)
+        return additional_assets
+
+    def do_cleanup_action(self, agent: Agent) -> (dict, dict):
+        if dirt := self[c.DIRT].by_pos(agent.pos):
+            new_dirt_amount = dirt.amount - self.dirt_prop.clean_amount
+
+            if new_dirt_amount <= 0:
+                self[c.DIRT].delete_env_object(dirt)
+            else:
+                dirt.set_new_amount(max(new_dirt_amount, c.FREE_CELL.value))
+            valid = c.VALID
+            self.print(f'{agent.name} did just clean up some dirt at {agent.pos}.')
+            info_dict = {f'{agent.name}_{a.CLEAN_UP}_VALID': 1, 'cleanup_valid': 1}
+            reward = self.rewards_dirt.CLEAN_UP_VALID
+        else:
+            valid = c.NOT_VALID
+            self.print(f'{agent.name} just tried to clean up some dirt at {agent.pos}, but failed.')
+            info_dict = {f'{agent.name}_{a.CLEAN_UP}_FAIL': 1, 'cleanup_fail': 1}
+            reward = self.rewards_dirt.CLEAN_UP_FAIL
+
+        if valid and self.dirt_prop.done_when_clean and (len(self[c.DIRT]) == 0):
+            reward += self.rewards_dirt.CLEAN_UP_LAST_PIECE
+            self.print(f'{agent.name} picked up the last piece of dirt!')
+            info_dict = {f'{agent.name}_{a.CLEAN_UP}_LAST_PIECE': 1}
+        return valid, dict(value=reward, reason=a.CLEAN_UP, info=info_dict)
+
+    def trigger_dirt_spawn(self, initial_spawn=False):
+        dirt_rng = self._dirt_rng
+        free_for_dirt = [x for x in self[c.FLOOR]
+                         if len(x.guests) == 0 or (len(x.guests) == 1 and isinstance(next(y for y in x.guests), Dirt))
+                         ]
+        self._dirt_rng.shuffle(free_for_dirt)
+        if initial_spawn:
+            var = self.dirt_prop.initial_dirt_spawn_r_var
+            new_spawn = self.dirt_prop.initial_dirt_ratio + dirt_rng.uniform(-var, var)
+        else:
+            new_spawn = dirt_rng.uniform(0, self.dirt_prop.max_spawn_ratio)
+        n_dirt_tiles = max(0, int(new_spawn * len(free_for_dirt)))
+        self[c.DIRT].spawn_dirt(free_for_dirt[:n_dirt_tiles])
+
+    def step_hook(self) -> (List[dict], dict):
+        super_reward_info = super().step_hook()
+        if smear_amount := self.dirt_prop.dirt_smear_amount:
+            for agent in self[c.AGENT]:
+                if agent.step_result['action_valid'] and agent.last_pos != c.NO_POS:
+                    if self._actions.is_moving_action(agent.step_result['action_name']):
+                        if old_pos_dirt := self[c.DIRT].by_pos(agent.last_pos):
+                            if smeared_dirt := round(old_pos_dirt.amount * smear_amount, 2):
+                                old_pos_dirt.set_new_amount(max(0, old_pos_dirt.amount-smeared_dirt))
+                                if new_pos_dirt := self[c.DIRT].by_pos(agent.pos):
+                                    new_pos_dirt.set_new_amount(max(0, new_pos_dirt.amount + smeared_dirt))
+                                else:
+                                    if self[c.DIRT].spawn_dirt(agent.tile):
+                                        new_pos_dirt = self[c.DIRT].by_pos(agent.pos)
+                                        new_pos_dirt.set_new_amount(max(0, new_pos_dirt.amount + smeared_dirt))
+        if self._next_dirt_spawn < 0:
+            pass  # No Dirt Spawn
+        elif not self._next_dirt_spawn:
+            self.trigger_dirt_spawn()
+            self._next_dirt_spawn = self.dirt_prop.spawn_frequency
+        else:
+            self._next_dirt_spawn -= 1
+        return super_reward_info
+
+    def do_additional_actions(self, agent: Agent, action: Action) -> (dict, dict):
+        action_result = super().do_additional_actions(agent, action)
+        if action_result is None:
+            if action == a.CLEAN_UP:
+                return self.do_cleanup_action(agent)
+            else:
+                return None
+        else:
+            return action_result
+
+    def reset_hook(self) -> None:
+        super().reset_hook()
+        self.trigger_dirt_spawn(initial_spawn=True)
+        self._next_dirt_spawn = self.dirt_prop.spawn_frequency if self.dirt_prop.spawn_frequency else -1
+
+    def check_additional_done(self) -> (bool, dict):
+        super_done, super_dict = super().check_additional_done()
+        if self.dirt_prop.done_when_clean:
+            if all_cleaned := len(self[c.DIRT]) == 0:
+                super_dict.update(ALL_CLEAN_DONE=all_cleaned)
+                return all_cleaned, super_dict
+        return super_done, super_dict
+
+    def observations_hook(self) -> Dict[str, np.typing.ArrayLike]:
+        additional_observations = super().observations_hook()
+        additional_observations.update({c.DIRT: self[c.DIRT].as_array()})
+        return additional_observations
+
+    def post_step_hook(self) -> List[Dict[str, int]]:
+        super_post_step = super(DirtFactory, self).post_step_hook()
+        info_dict = dict()
+
+        dirt = [dirt.amount for dirt in self[c.DIRT]]
+        current_dirt_amount = sum(dirt)
+        dirty_tile_count = len(dirt)
+
+        # if dirty_tile_count:
+        #    dirt_distribution_score = entropy(softmax(np.asarray(dirt)) / dirty_tile_count)
+        # else:
+        #    dirt_distribution_score = 0
+
+        info_dict.update(dirt_amount=current_dirt_amount)
+        info_dict.update(dirty_tile_count=dirty_tile_count)
+
+        super_post_step.append(info_dict)
+        return super_post_step
+
+
+if __name__ == '__main__':
+    from environments.utility_classes import AgentRenderOptions as aro
+    render = True
+
+    dirt_props = DirtProperties(
+        initial_dirt_ratio=0.35,
+        initial_dirt_spawn_r_var=0.1,
+        clean_amount=0.34,
+        max_spawn_amount=0.1,
+        max_global_amount=20,
+        max_local_amount=1,
+        spawn_frequency=0,
+        max_spawn_ratio=0.05,
+        dirt_smear_amount=0.0
+    )
+
+    obs_props = ObservationProperties(render_agents=aro.COMBINED, omit_agent_self=True,
+                                      pomdp_r=2, additional_agent_placeholder=None, cast_shadows=True,
+                                      indicate_door_area=False)
+
+    move_props = {'allow_square_movement': True,
+                  'allow_diagonal_movement': False,
+                  'allow_no_op': False}
+    import time
+    global_timings = []
+    for i in range(10):
+
+        factory = DirtFactory(n_agents=10, done_at_collision=False,
+                              level_name='rooms', max_steps=1000,
+                              doors_have_area=False,
+                              obs_prop=obs_props, parse_doors=True,
+                              verbose=True,
+                              mv_prop=move_props, dirt_prop=dirt_props,
+                              # inject_agents=[TSPDirtAgent],
+                              )
+
+
+        # noinspection DuplicatedCode
+        n_actions = factory.action_space.n - 1
+        _ = factory.observation_space
+        obs_space = factory.observation_space
+        obs_space_named = factory.named_observation_space
+        action_space_named = factory.named_action_space
+        times = []
+        for epoch in range(10):
+            start_time = time.time()
+            random_actions = [[random.randint(0, n_actions) for _
+                               in range(factory.n_agents)] for _
+                              in range(factory.max_steps+1)]
+            env_state = factory.reset()
+            if render:
+                factory.render()
+            # tsp_agent = factory.get_injected_agents()[0]
+
+            rwrd = 0
+            for agent_i_action in random_actions:
+                # agent_i_action = tsp_agent.predict()
+                env_state, step_rwrd, done_bool, info_obj = factory.step(agent_i_action)
+                rwrd += step_rwrd
+                if render:
+                    factory.render()
+                if done_bool:
+                    break
+            times.append(time.time() - start_time)
+            # print(f'Factory run {epoch} done, reward is:\n    {r}')
+        print('Mean Time Taken: ', sum(times) / 10)
+        global_timings.extend(times)
+    print('Mean Time Taken: ', sum(global_timings) / len(global_timings))
+    print('Median Time Taken: ', global_timings[len(global_timings)//2])
+
+pass

environments/factory/additional/item/factory_item.py

@@ -0,0 +1,193 @@
+import time
+from typing import List, Union, Dict
+import numpy as np
+import random
+
+from environments.factory.additional.item.item_collections import ItemRegister, Inventories, DropOffLocations
+from environments.factory.additional.item.item_util import Constants, Actions, RewardsItem, ItemProperties
+from environments.factory.base.base_factory import BaseFactory
+from environments.factory.base.objects import Agent, Action
+from environments.factory.base.registers import Entities
+
+from environments.factory.base.renderer import RenderEntity
+
+c = Constants
+a = Actions
+
+
+# noinspection PyAttributeOutsideInit, PyAbstractClass
+class ItemFactory(BaseFactory):
+    # noinspection PyMissingConstructor
+    def __init__(self, *args, item_prop: ItemProperties = ItemProperties(), env_seed=time.time_ns(),
+                 rewards_item: RewardsItem = RewardsItem(), **kwargs):
+        if isinstance(item_prop, dict):
+            item_prop = ItemProperties(**item_prop)
+        if isinstance(rewards_item, dict):
+            rewards_item = RewardsItem(**rewards_item)
+        self.item_prop = item_prop
+        self.rewards_item = rewards_item
+        kwargs.update(env_seed=env_seed)
+        self._item_rng = np.random.default_rng(env_seed)
+        assert (item_prop.n_items <= ((1 + kwargs.get('_pomdp_r', 0) * 2) ** 2)) or not kwargs.get('_pomdp_r', 0)
+        super().__init__(*args, **kwargs)
+
+    @property
+    def actions_hook(self) -> Union[Action, List[Action]]:
+        # noinspection PyUnresolvedReferences
+        super_actions = super().actions_hook
+        super_actions.append(Action(str_ident=a.ITEM_ACTION))
+        return super_actions
+
+    @property
+    def entities_hook(self) -> Dict[(str, Entities)]:
+        # noinspection PyUnresolvedReferences
+        super_entities = super().entities_hook
+
+        empty_tiles = self[c.FLOOR].empty_tiles[:self.item_prop.n_drop_off_locations]
+        drop_offs = DropOffLocations.from_tiles(
+            empty_tiles, self._level_shape,
+            entity_kwargs=dict(
+                storage_size_until_full=self.item_prop.max_dropoff_storage_size)
+        )
+        item_register = ItemRegister(self._level_shape)
+        empty_tiles = self[c.FLOOR].empty_tiles[:self.item_prop.n_items]
+        item_register.spawn_items(empty_tiles)
+
+        inventories = Inventories(self._obs_shape, self._level_shape)
+        inventories.spawn_inventories(self[c.AGENT], self.item_prop.max_agent_inventory_capacity)
+
+        super_entities.update({c.DROP_OFF: drop_offs, c.ITEM: item_register, c.INVENTORY: inventories})
+        return super_entities
+
+    def per_agent_raw_observations_hook(self, agent) -> Dict[str, np.typing.ArrayLike]:
+        additional_raw_observations = super().per_agent_raw_observations_hook(agent)
+        additional_raw_observations.update({c.INVENTORY: self[c.INVENTORY].by_entity(agent).as_array()})
+        return additional_raw_observations
+
+    def observations_hook(self) -> Dict[str, np.typing.ArrayLike]:
+        additional_observations = super().observations_hook()
+        additional_observations.update({c.ITEM: self[c.ITEM].as_array()})
+        additional_observations.update({c.DROP_OFF: self[c.DROP_OFF].as_array()})
+        return additional_observations
+
+    def do_item_action(self, agent: Agent) -> (dict, dict):
+        inventory = self[c.INVENTORY].by_entity(agent)
+        if drop_off := self[c.DROP_OFF].by_pos(agent.pos):
+            if inventory:
+                valid = drop_off.place_item(inventory.pop())
+            else:
+                valid = c.NOT_VALID
+            if valid:
+                self.print(f'{agent.name} just dropped of an item at {drop_off.pos}.')
+                info_dict = {f'{agent.name}_DROPOFF_VALID': 1, 'DROPOFF_VALID': 1}
+            else:
+                self.print(f'{agent.name} just tried to drop off at {agent.pos}, but failed.')
+                info_dict = {f'{agent.name}_DROPOFF_FAIL': 1, 'DROPOFF_FAIL': 1}
+            reward = dict(value=self.rewards_item.DROP_OFF_VALID if valid else self.rewards_item.DROP_OFF_FAIL,
+                          reason=a.ITEM_ACTION, info=info_dict)
+            return valid, reward
+        elif item := self[c.ITEM].by_pos(agent.pos):
+            item.change_parent_collection(inventory)
+            item.set_tile_to(self._NO_POS_TILE)
+            self.print(f'{agent.name} just picked up an item at {agent.pos}')
+            info_dict = {f'{agent.name}_{a.ITEM_ACTION}_VALID': 1, f'{a.ITEM_ACTION}_VALID': 1}
+            return c.VALID, dict(value=self.rewards_item.PICK_UP_VALID, reason=a.ITEM_ACTION, info=info_dict)
+        else:
+            self.print(f'{agent.name} just tried to pick up an item at {agent.pos}, but failed.')
+            info_dict = {f'{agent.name}_{a.ITEM_ACTION}_FAIL': 1, f'{a.ITEM_ACTION}_FAIL': 1}
+            return c.NOT_VALID, dict(value=self.rewards_item.PICK_UP_FAIL, reason=a.ITEM_ACTION, info=info_dict)
+
+    def do_additional_actions(self, agent: Agent, action: Action) -> (dict, dict):
+        # noinspection PyUnresolvedReferences
+        action_result = super().do_additional_actions(agent, action)
+        if action_result is None:
+            if action == a.ITEM_ACTION:
+                action_result = self.do_item_action(agent)
+                return action_result
+            else:
+                return None
+        else:
+            return action_result
+
+    def reset_hook(self) -> None:
+        # noinspection PyUnresolvedReferences
+        super().reset_hook()
+        self._next_item_spawn = self.item_prop.spawn_frequency
+        self.trigger_item_spawn()
+
+    def trigger_item_spawn(self):
+        if item_to_spawns := max(0, (self.item_prop.n_items - len(self[c.ITEM]))):
+            empty_tiles = self[c.FLOOR].empty_tiles[:item_to_spawns]
+            self[c.ITEM].spawn_items(empty_tiles)
+            self._next_item_spawn = self.item_prop.spawn_frequency
+            self.print(f'{item_to_spawns} new items have been spawned; next spawn in {self._next_item_spawn}')
+        else:
+            self.print('No Items are spawning, limit is reached.')
+
+    def step_hook(self) -> (List[dict], dict):
+        # noinspection PyUnresolvedReferences
+        super_reward_info = super().step_hook()
+        for item in list(self[c.ITEM].values()):
+            if item.auto_despawn >= 1:
+                item.set_auto_despawn(item.auto_despawn-1)
+            elif not item.auto_despawn:
+                self[c.ITEM].delete_env_object(item)
+            else:
+                pass
+
+        if not self._next_item_spawn:
+            self.trigger_item_spawn()
+        else:
+            self._next_item_spawn = max(0, self._next_item_spawn-1)
+        return super_reward_info
+
+    def render_assets_hook(self, mode='human'):
+        # noinspection PyUnresolvedReferences
+        additional_assets = super().render_assets_hook()
+        items = [RenderEntity(c.ITEM, item.tile.pos) for item in self[c.ITEM] if item.tile != self._NO_POS_TILE]
+        additional_assets.extend(items)
+        drop_offs = [RenderEntity(c.DROP_OFF, drop_off.tile.pos) for drop_off in self[c.DROP_OFF]]
+        additional_assets.extend(drop_offs)
+        return additional_assets
+
+
+if __name__ == '__main__':
+    from environments.utility_classes import AgentRenderOptions as aro, ObservationProperties
+
+    render = True
+
+    item_probs = ItemProperties(n_items=30, n_drop_off_locations=6)
+
+    obs_props = ObservationProperties(render_agents=aro.SEPERATE, omit_agent_self=True, pomdp_r=2)
+
+    move_props = {'allow_square_movement': True,
+                  'allow_diagonal_movement': True,
+                  'allow_no_op': False}
+
+    factory = ItemFactory(n_agents=6, done_at_collision=False,
+                          level_name='rooms', max_steps=400,
+                          obs_prop=obs_props, parse_doors=True,
+                          record_episodes=True, verbose=True,
+                          mv_prop=move_props, item_prop=item_probs
+                          )
+
+    # noinspection DuplicatedCode
+    n_actions = factory.action_space.n - 1
+    obs_space = factory.observation_space
+    obs_space_named = factory.named_observation_space
+
+    for epoch in range(400):
+        random_actions = [[random.randint(0, n_actions) for _
+                           in range(factory.n_agents)] for _
+                          in range(factory.max_steps + 1)]
+        env_state = factory.reset()
+        rwrd = 0
+        for agent_i_action in random_actions:
+            env_state, step_r, done_bool, info_obj = factory.step(agent_i_action)
+            rwrd += step_r
+            if render:
+                factory.render()
+            if done_bool:
+                break
+        print(f'Factory run {epoch} done, reward is:\n    {rwrd}')
+pass

environments/factory/additional/item/item_collections.py

@@ -0,0 +1,87 @@
+from typing import List
+
+import numpy as np
+
+from environments.factory.base.objects import Floor, Agent
+from environments.factory.base.registers import EntityCollection, BoundEnvObjCollection, ObjectCollection
+from environments.factory.additional.item.item_entities import Item, DropOffLocation
+
+
+class ItemRegister(EntityCollection):
+
+    _accepted_objects = Item
+
+    def spawn_items(self, tiles: List[Floor]):
+        items = [Item(tile, self) for tile in tiles]
+        self.add_additional_items(items)
+
+    def despawn_items(self, items: List[Item]):
+        items = [items] if isinstance(items, Item) else items
+        for item in items:
+            del self[item]
+
+
+class Inventory(BoundEnvObjCollection):
+
+    @property
+    def name(self):
+        return f'{self.__class__.__name__}({self._bound_entity.name})'
+
+    def __init__(self, agent: Agent, capacity: int, *args, **kwargs):
+        super(Inventory, self).__init__(agent, *args,  is_blocking_light=False, can_be_shadowed=False,  **kwargs)
+        self.capacity = capacity
+
+    def as_array(self):
+        if self._array is None:
+            self._array = np.zeros((1, *self._shape))
+        return super(Inventory, self).as_array()
+
+    def summarize_states(self, **kwargs):
+        attr_dict = {key: str(val) for key, val in self.__dict__.items() if not key.startswith('_') and key != 'data'}
+        attr_dict.update(dict(items={key: val.summarize_state(**kwargs) for key, val in self.items()}))
+        attr_dict.update(dict(name=self.name))
+        return attr_dict
+
+    def pop(self):
+        item_to_pop = self[0]
+        self.delete_env_object(item_to_pop)
+        return item_to_pop
+
+
+class Inventories(ObjectCollection):
+
+    _accepted_objects = Inventory
+    is_blocking_light = False
+    can_be_shadowed = False
+
+    def __init__(self, obs_shape, *args, **kwargs):
+        super(Inventories, self).__init__(*args, is_per_agent=True, individual_slices=True, **kwargs)
+        self._obs_shape = obs_shape
+
+    def as_array(self):
+        return np.stack([inventory.as_array() for inv_idx, inventory in enumerate(self)])
+
+    def spawn_inventories(self, agents, capacity):
+        inventories = [self._accepted_objects(agent, capacity, self._obs_shape)
+                       for _, agent in enumerate(agents)]
+        self.add_additional_items(inventories)
+
+    def idx_by_entity(self, entity):
+        try:
+            return next((idx for idx, inv in enumerate(self) if inv.belongs_to_entity(entity)))
+        except StopIteration:
+            return None
+
+    def by_entity(self, entity):
+        try:
+            return next((inv for inv in self if inv.belongs_to_entity(entity)))
+        except StopIteration:
+            return None
+
+    def summarize_states(self, **kwargs):
+        return {key: val.summarize_states(**kwargs) for key, val in self.items()}
+
+
+class DropOffLocations(EntityCollection):
+
+    _accepted_objects = DropOffLocation

environments/factory/additional/item/item_entities.py

@@ -0,0 +1,61 @@
+from collections import deque
+
+from environments import helpers as h
+from environments.factory.additional.item.item_util import Constants
+from environments.factory.base.objects import Entity
+
+
+class Item(Entity):
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self._auto_despawn = -1
+
+    @property
+    def auto_despawn(self):
+        return self._auto_despawn
+
+    @property
+    def encoding(self):
+        # Edit this if you want items to be drawn in the ops differently
+        return 1
+
+    def set_auto_despawn(self, auto_despawn):
+        self._auto_despawn = auto_despawn
+
+    def set_tile_to(self, no_pos_tile):
+        self._tile = no_pos_tile
+
+    def summarize_state(self, **_) -> dict:
+        super_summarization = super(Item, self).summarize_state()
+        super_summarization.update(dict(auto_despawn=self.auto_despawn))
+        return super_summarization
+
+
+class DropOffLocation(Entity):
+
+    @property
+    def encoding(self):
+        return Constants.ITEM_DROP_OFF
+
+    def __init__(self, *args, storage_size_until_full: int = 5, auto_item_despawn_interval: int = 5, **kwargs):
+        super(DropOffLocation, self).__init__(*args, **kwargs)
+        self.auto_item_despawn_interval = auto_item_despawn_interval
+        self.storage = deque(maxlen=storage_size_until_full or None)
+
+    def place_item(self, item: Item):
+        if self.is_full:
+            raise RuntimeWarning("There is currently no way to clear the storage or make it unfull.")
+            return c.NOT_VALID
+        else:
+            self.storage.append(item)
+            item.set_auto_despawn(self.auto_item_despawn_interval)
+            return Constants.VALID
+
+    @property
+    def is_full(self):
+        return False if not self.storage.maxlen else self.storage.maxlen == len(self.storage)
+
+    def summarize_state(self, n_steps=None) -> dict:
+        if n_steps == h.STEPS_START:
+            return super().summarize_state(n_steps=n_steps)

environments/factory/additional/item/item_util.py

@@ -0,0 +1,31 @@
+from typing import NamedTuple
+
+from environments.helpers import Constants as BaseConstants, EnvActions as BaseActions
+
+
+class Constants(BaseConstants):
+    NO_ITEM = 0
+    ITEM_DROP_OFF = 1
+    # Item Env
+    ITEM                = 'Item'
+    INVENTORY           = 'Inventory'
+    DROP_OFF            = 'Drop_Off'
+
+
+class Actions(BaseActions):
+    ITEM_ACTION     = 'ITEMACTION'
+
+
+class RewardsItem(NamedTuple):
+    DROP_OFF_VALID: float = 0.1
+    DROP_OFF_FAIL: float = -0.1
+    PICK_UP_FAIL: float  = -0.1
+    PICK_UP_VALID: float  = 0.1
+
+
+class ItemProperties(NamedTuple):
+    n_items:                         int  = 5     # How many items are there at the same time
+    spawn_frequency:                 int  = 10     # Spawn Frequency in Steps
+    n_drop_off_locations:            int  = 5     # How many DropOff locations are there at the same time
+    max_dropoff_storage_size:        int  = 0     # How many items are needed until the dropoff is full
+    max_agent_inventory_capacity:    int  = 5     # How many items are needed until the agent inventory is full