Agent Trained on Doors

environments/factory/base_factory.py

@@ -22,10 +22,11 @@ class BaseFactory(gym.Env):
     @property
     def observation_space(self):
         agent_slice = self.n_agents if self.omit_agent_slice_in_obs else 0
-        agent_slice = 1 if self.combin_agent_slices_in_obs else agent_slice
+        agent_slice = (self.n_agents - 1) if self.combin_agent_slices_in_obs else agent_slice
         if self.pomdp_radius:
-            return spaces.Box(low=0, high=1, shape=(self._state.shape[0] - agent_slice, self.pomdp_radius * 2 + 1,
-                                                    self.pomdp_radius * 2 + 1), dtype=np.float32)
+            shape = (self._state.shape[0] - agent_slice, self.pomdp_radius * 2 + 1, self.pomdp_radius * 2 + 1)
+            space = spaces.Box(low=0, high=1, shape=shape, dtype=np.float32)
+            return space
         else:
             shape = [x-agent_slice if idx == 0 else x for idx, x in enumerate(self._state.shape)]
             space = spaces.Box(low=0, high=1, shape=shape, dtype=np.float32)
@@ -194,6 +195,14 @@ class BaseFactory(gym.Env):
             if self.done_at_collision and collision_vec.any():
                 done = True
 
+        # Step the door close intervall
+        agents_pos = [agent.pos for agent in self._agent_states]
+        for door_i, door in enumerate(self._door_states):
+            if door.is_open and door.time_to_close and door.pos not in agents_pos:
+                door.time_to_close -= 1
+            elif door.is_open and not door.time_to_close and door.pos not in agents_pos:
+                door.use()
+
         reward, info = self.calculate_reward(self._agent_states)
 
         if self._steps >= self.max_steps:
@@ -256,7 +265,7 @@ class BaseFactory(gym.Env):
         x_new = x + x_diff
         y_new = y + y_diff
 
-        if h.DOORS in self._state_slices.values():
+        if h.DOORS in self._state_slices.values() and self._agent_states[agent_i]._last_pos != (-1, -1):
             door = [door for door in self._door_states if door.pos == (x, y)]
             if door:
                 door = door[0]
@@ -326,7 +335,7 @@ class BaseFactory(gym.Env):
         # Returns: Reward, Info
         raise NotImplementedError
 
-    def render(self):
+    def render(self, mode='human'):
         raise NotImplementedError
 
     def save_params(self, filepath: Path):

environments/factory/levels/rooms.txt

@@ -7,7 +7,7 @@
 ###x#######x###
 #1111##2222222#
 #11111#2222#22#
-#11111D2222222#
+#11111x2222222#
 #11111#2222222#
 #11111#2222222#
 ###############

environments/factory/renderer.py

@@ -42,8 +42,6 @@ class Renderer:
         self.font.set_bold(1.0)
         print('Loading System font with pygame.font.Font took', time.time() - now)
 
-
-
     def fill_bg(self):
         self.screen.fill(Renderer.BG_COLOR)
         if self.grid_lines:
@@ -71,9 +69,9 @@ class Renderer:
 
     def load_asset(self, path, factor=1.0):
         s = int(factor*self.cell_size)
-        wall_img = pygame.image.load(path).convert_alpha()
-        wall_img = pygame.transform.smoothscale(wall_img, (s, s))
-        return wall_img
+        asset = pygame.image.load(path).convert_alpha()
+        asset = pygame.transform.smoothscale(asset, (s, s))
+        return asset
 
     def render(self, entities):
         for event in pygame.event.get():
@@ -82,7 +80,10 @@ class Renderer:
                 sys.exit()
         self.fill_bg()
         blits = deque()
-        for entity in entities:
+        for entity in [x for x in entities if 'door' in x.name]:
+            bp = self.blit_params(entity)
+            blits.append(bp)
+        for entity in [x for x in entities if 'door' not in x.name]:
             bp = self.blit_params(entity)
             blits.append(bp)
             if entity.name.lower() == 'agent':
@@ -106,7 +107,6 @@ class Renderer:
         for blit in blits:
             self.screen.blit(**blit)
 
-
         pygame.display.flip()
         self.clock.tick(self.fps)
 
@@ -114,6 +114,6 @@ class Renderer:
 if __name__ == '__main__':
     renderer = Renderer(fps=2, cell_size=40)
     for i in range(15):
-        entity = Entity('agent', [5, i], 1, 'idle', 'idle')
-        renderer.render([entity])
+        entity_1 = Entity('agent', [5, i], 1, 'idle', 'idle')
+        renderer.render([entity_1])
 

environments/factory/simple_factory.py

@@ -14,7 +14,7 @@ CLEAN_UP_ACTION = 'clean_up'
 
 
 class DirtProperties(NamedTuple):
-    clean_amount: int = 2            # How much does the robot clean with one action.
+    clean_amount: int = 2            # How much does the robot clean with one actions.
     max_spawn_ratio: float = 0.2       # On max how much tiles does the dirt spawn in percent.
     gain_amount: float = 0.5           # How much dirt does spawn per tile
     spawn_frequency: int = 5         # Spawn Frequency in Steps
@@ -41,7 +41,7 @@ class SimpleFactory(BaseFactory):
         self._renderer = None  # expensive - don't use it when not required !
         super(SimpleFactory, self).__init__(*args, additional_slices=['dirt'], **kwargs)
 
-    def render(self):
+    def render(self, mode='human'):
 
         if not self._renderer:  # lazy init
             height, width = self._state.shape[1:]
@@ -67,7 +67,11 @@ class SimpleFactory(BaseFactory):
         for i, agent in enumerate(self._agent_states):
             name, state = asset_str(agent)
             agents.append(Entity(name, agent.pos, 1, 'none', state, i+1))
-        self._renderer.render(dirt+walls+agents)
+        doors = []
+        for i, door in enumerate(self._door_states):
+            name, state = 'door_open' if door.is_open else 'door_closed', 'blank'
+            agents.append(Entity(name, door.pos, 1, 'none', state, i+1))
+        self._renderer.render(dirt+walls+agents+doors)
 
     def spawn_dirt(self) -> None:
         if not np.argwhere(self._state[DIRT_INDEX] != h.IS_FREE_CELL).shape[0] > self.dirt_properties.max_global_amount:
@@ -156,6 +160,7 @@ class SimpleFactory(BaseFactory):
                     self.print(f'Agent {agent_state.i} just tried to clean up some dirt '
                                f'at {agent_state.pos}, but was unsucsessfull.')
                     info_dict.update({f'agent_{agent_state.i}_failed_action': 1})
+                    info_dict.update({f'agent_{agent_state.i}_failed_dirt_cleanup': 1})
 
             elif self._actions.is_moving_action(agent_state.action):
                 if agent_state.action_valid:
@@ -165,6 +170,17 @@ class SimpleFactory(BaseFactory):
                     # self.print('collision')
                     reward -= 0.01
 
+            elif self._actions.is_door_usage(agent_state.action):
+                if agent_state.action_valid:
+                    reward += 0.1
+                    self.print(f'Agent {agent_state.i} did just use the door at {agent_state.pos}.')
+                    info_dict.update(door_used=1)
+                else:
+                    self.print(f'Agent {agent_state.i} just tried to use a door '
+                               f'at {agent_state.pos}, but was unsucsessfull.')
+                    info_dict.update({f'agent_{agent_state.i}_failed_action': 1})
+                    info_dict.update({f'agent_{agent_state.i}_failed_door_open': 1})
+
             else:
                 info_dict.update(no_op=1)
                 reward -= 0.00
@@ -184,7 +200,7 @@ class SimpleFactory(BaseFactory):
 
 
 if __name__ == '__main__':
-    render = True
+    render = False
 
     move_props = MovementProperties(allow_diagonal_movement=True, allow_square_movement=True)
     dirt_props = DirtProperties()
@@ -193,8 +209,9 @@ if __name__ == '__main__':
                             pomdp_radius=3)
 
     n_actions = factory.action_space.n - 1
+    _ = factory.observation_space
 
-    for epoch in range(100):
+    for epoch in range(10000):
         random_actions = [[random.randint(0, n_actions) for _ in range(factory.n_agents)] for _ in range(200)]
         env_state = factory.reset()
         r = 0

environments/oo_factory/_base_factory.py

@@ -0,0 +1,68 @@
+from typing import List, Union
+
+import gym
+
+
+class Entities():
+
+    def __init__(self):
+        pass
+
+
+# noinspection PyAttributeOutsideInit
+class BaseFactory(gym.Env):
+
+    def __enter__(self):
+        return self if self.frames_to_stack == 0 else FrameStack(self, self.frames_to_stack)
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        self.close()
+
+    def __init__(self, level_name='simple', n_agents=1, max_steps=int(5e2), pomdp_radius: Union[None, int] = 0,
+                 movement_properties: MovementProperties = MovementProperties(),
+                 combin_agent_slices_in_obs: bool = False, frames_to_stack=0,
+                 omit_agent_slice_in_obs=False, **kwargs):
+        assert (combin_agent_slices_in_obs != omit_agent_slice_in_obs) or \
+               (not combin_agent_slices_in_obs and not omit_agent_slice_in_obs), \
+            'Both options are exclusive'
+        assert frames_to_stack != 1 and frames_to_stack >= 0, "'frames_to_stack' cannot be negative or 1."
+
+        self.movement_properties = movement_properties
+        self.level_name = level_name
+
+        self.n_agents = n_agents
+        self.max_steps = max_steps
+        self.pomdp_radius = pomdp_radius
+        self.combin_agent_slices_in_obs = combin_agent_slices_in_obs
+        self.omit_agent_slice_in_obs = omit_agent_slice_in_obs
+        self.frames_to_stack = frames_to_stack
+
+        self.done_at_collision = False
+
+        self._state_slices = StateSlices()
+        level_filepath = Path(__file__).parent / h.LEVELS_DIR / f'{self.level_name}.txt'
+        parsed_level = h.parse_level(level_filepath)
+        self._level = h.one_hot_level(parsed_level)
+        parsed_doors = h.one_hot_level(parsed_level, h.DOOR)
+        if parsed_doors.any():
+            self._doors = parsed_doors
+            level_slices = ['level', 'doors']
+            can_use_doors = True
+        else:
+            level_slices = ['level']
+            can_use_doors = False
+        offset = len(level_slices)
+        self._state_slices.register_additional_items([*level_slices,
+                                                      *[f'agent#{i}' for i in range(offset, n_agents + offset)]])
+        if 'additional_slices' in kwargs:
+            self._state_slices.register_additional_items(kwargs.get('additional_slices'))
+        self._zones = Zones(parsed_level)
+
+        self._actions = Actions(self.movement_properties, can_use_doors=can_use_doors)
+        self._actions.register_additional_items(self.additional_actions)
+        self.reset()
+
+
+    def step(self, actions: Union[int, List[int]]):
+        actions = actions if isinstance(actions, list) else [actions]
+        self.entities.step()

environments/utility_classes.py

@@ -108,10 +108,12 @@ class AgentState:
 
 class DoorState:
 
-    def __init__(self, i: int, pos: Tuple[int, int], closed_on_init=True):
+    def __init__(self, i: int, pos: Tuple[int, int], closed_on_init=True, auto_close_interval=10):
         self.i = i
         self.pos = pos
         self._state = self._state = IS_CLOSED if closed_on_init else IS_OPEN
+        self.auto_close_interval = auto_close_interval
+        self.time_to_close = -1
 
     @property
     def is_closed(self):
@@ -126,8 +128,11 @@ class DoorState:
         return self._state
 
     def use(self):
-        self._state: str = IS_CLOSED if self._state == IS_OPEN else IS_OPEN
-
+        if self._state == IS_OPEN:
+            self._state = IS_CLOSED
+        else:
+            self._state = IS_OPEN
+            self.time_to_close = self.auto_close_interval
 
 class Register:
 

main.py

@@ -111,7 +111,7 @@ if __name__ == '__main__':
                     kwargs = dict(ent_coef=0.01)
                 elif modeL_type.__name__ in ["RegDQN", "DQN", "QRDQN"]:
                     kwargs = dict(buffer_size=50000,
-                                  learning_starts=25000,
+                                  learning_starts=64,
                                   batch_size=64,
                                   target_update_interval=5000,
                                   exploration_fraction=0.25,

reload_agent.py

@@ -14,14 +14,14 @@ warnings.filterwarnings('ignore', category=UserWarning)
 
 if __name__ == '__main__':
 
-    model_name = 'PPO_1623052687'
+    model_name = 'A2C_1623923982'
     run_id = 0
     out_path = Path(__file__).parent / 'debug_out'
     model_path = out_path / model_name
 
     with (model_path / f'env_{model_name}.yaml').open('r') as f:
         env_kwargs = yaml.load(f, Loader=yaml.FullLoader)
-    with SimpleFactory(level_name='rooms', **env_kwargs) as env:
+    with SimpleFactory(**env_kwargs) as env:
 
         # Edit THIS:
         model_files = list(natsorted((model_path / f'{run_id}_{model_name}').rglob('model_*.zip')))
@@ -30,5 +30,3 @@ if __name__ == '__main__':
         model = PPO.load(this_model)
         evaluation_result = evaluate_policy(model, env, n_eval_episodes=100, deterministic=False, render=True)
         print(evaluation_result)
-
-