Agent Trained on Doors

environments/factory/base_factory.py

@@ -22,10 +22,11 @@ class BaseFactory(gym.Env):
     @property
     def observation_space(self):
         agent_slice = self.n_agents if self.omit_agent_slice_in_obs else 0
-        agent_slice = 1 if self.combin_agent_slices_in_obs else agent_slice
+        agent_slice = (self.n_agents - 1) if self.combin_agent_slices_in_obs else agent_slice
         if self.pomdp_radius:
-            return spaces.Box(low=0, high=1, shape=(self._state.shape[0] - agent_slice, self.pomdp_radius * 2 + 1,
-                                                    self.pomdp_radius * 2 + 1), dtype=np.float32)
+            shape = (self._state.shape[0] - agent_slice, self.pomdp_radius * 2 + 1, self.pomdp_radius * 2 + 1)
+            space = spaces.Box(low=0, high=1, shape=shape, dtype=np.float32)
+            return space
         else:
             shape = [x-agent_slice if idx == 0 else x for idx, x in enumerate(self._state.shape)]
             space = spaces.Box(low=0, high=1, shape=shape, dtype=np.float32)
@@ -194,6 +195,14 @@ class BaseFactory(gym.Env):
             if self.done_at_collision and collision_vec.any():
                 done = True
 
+        # Step the door close intervall
+        agents_pos = [agent.pos for agent in self._agent_states]
+        for door_i, door in enumerate(self._door_states):
+            if door.is_open and door.time_to_close and door.pos not in agents_pos:
+                door.time_to_close -= 1
+            elif door.is_open and not door.time_to_close and door.pos not in agents_pos:
+                door.use()
+
         reward, info = self.calculate_reward(self._agent_states)
 
         if self._steps >= self.max_steps:
@@ -256,7 +265,7 @@ class BaseFactory(gym.Env):
         x_new = x + x_diff
         y_new = y + y_diff
 
-        if h.DOORS in self._state_slices.values():
+        if h.DOORS in self._state_slices.values() and self._agent_states[agent_i]._last_pos != (-1, -1):
             door = [door for door in self._door_states if door.pos == (x, y)]
             if door:
                 door = door[0]
@@ -326,7 +335,7 @@ class BaseFactory(gym.Env):
         # Returns: Reward, Info
         raise NotImplementedError
 
-    def render(self):
+    def render(self, mode='human'):
         raise NotImplementedError
 
     def save_params(self, filepath: Path):

environments/factory/levels/rooms.txt

@@ -7,7 +7,7 @@
 ###x#######x###
 #1111##2222222#
 #11111#2222#22#
-#11111D2222222#
+#11111x2222222#
 #11111#2222222#
 #11111#2222222#
 ###############

environments/factory/renderer.py

@@ -42,8 +42,6 @@ class Renderer:
         self.font.set_bold(1.0)
         print('Loading System font with pygame.font.Font took', time.time() - now)
 
-
-
     def fill_bg(self):
         self.screen.fill(Renderer.BG_COLOR)
         if self.grid_lines:
@@ -71,9 +69,9 @@ class Renderer:
 
     def load_asset(self, path, factor=1.0):
         s = int(factor*self.cell_size)
-        wall_img = pygame.image.load(path).convert_alpha()
-        wall_img = pygame.transform.smoothscale(wall_img, (s, s))
-        return wall_img
+        asset = pygame.image.load(path).convert_alpha()
+        asset = pygame.transform.smoothscale(asset, (s, s))
+        return asset
 
     def render(self, entities):
         for event in pygame.event.get():
@@ -82,7 +80,10 @@ class Renderer:
                 sys.exit()
         self.fill_bg()
         blits = deque()
-        for entity in entities:
+        for entity in [x for x in entities if 'door' in x.name]:
+            bp = self.blit_params(entity)
+            blits.append(bp)
+        for entity in [x for x in entities if 'door' not in x.name]:
             bp = self.blit_params(entity)
             blits.append(bp)
             if entity.name.lower() == 'agent':
@@ -106,7 +107,6 @@ class Renderer:
         for blit in blits:
             self.screen.blit(**blit)
 
-
         pygame.display.flip()
         self.clock.tick(self.fps)
 
@@ -114,6 +114,6 @@ class Renderer:
 if __name__ == '__main__':
     renderer = Renderer(fps=2, cell_size=40)
     for i in range(15):
-        entity = Entity('agent', [5, i], 1, 'idle', 'idle')
-        renderer.render([entity])
+        entity_1 = Entity('agent', [5, i], 1, 'idle', 'idle')
+        renderer.render([entity_1])
 

environments/factory/simple_factory.py

@@ -14,7 +14,7 @@ CLEAN_UP_ACTION = 'clean_up'
 
 
 class DirtProperties(NamedTuple):
-    clean_amount: int = 2            # How much does the robot clean with one action.
+    clean_amount: int = 2            # How much does the robot clean with one actions.
     max_spawn_ratio: float = 0.2       # On max how much tiles does the dirt spawn in percent.
     gain_amount: float = 0.5           # How much dirt does spawn per tile
     spawn_frequency: int = 5         # Spawn Frequency in Steps
@@ -41,7 +41,7 @@ class SimpleFactory(BaseFactory):
         self._renderer = None  # expensive - don't use it when not required !
         super(SimpleFactory, self).__init__(*args, additional_slices=['dirt'], **kwargs)
 
-    def render(self):
+    def render(self, mode='human'):
 
         if not self._renderer:  # lazy init
             height, width = self._state.shape[1:]
@@ -67,7 +67,11 @@ class SimpleFactory(BaseFactory):
         for i, agent in enumerate(self._agent_states):
             name, state = asset_str(agent)
             agents.append(Entity(name, agent.pos, 1, 'none', state, i+1))
-        self._renderer.render(dirt+walls+agents)
+        doors = []
+        for i, door in enumerate(self._door_states):
+            name, state = 'door_open' if door.is_open else 'door_closed', 'blank'
+            agents.append(Entity(name, door.pos, 1, 'none', state, i+1))
+        self._renderer.render(dirt+walls+agents+doors)
 
     def spawn_dirt(self) -> None:
         if not np.argwhere(self._state[DIRT_INDEX] != h.IS_FREE_CELL).shape[0] > self.dirt_properties.max_global_amount:
@@ -156,6 +160,7 @@ class SimpleFactory(BaseFactory):
                     self.print(f'Agent {agent_state.i} just tried to clean up some dirt '
                                f'at {agent_state.pos}, but was unsucsessfull.')
                     info_dict.update({f'agent_{agent_state.i}_failed_action': 1})
+                    info_dict.update({f'agent_{agent_state.i}_failed_dirt_cleanup': 1})
 
             elif self._actions.is_moving_action(agent_state.action):
                 if agent_state.action_valid:
@@ -165,6 +170,17 @@ class SimpleFactory(BaseFactory):
                     # self.print('collision')
                     reward -= 0.01
 
+            elif self._actions.is_door_usage(agent_state.action):
+                if agent_state.action_valid:
+                    reward += 0.1
+                    self.print(f'Agent {agent_state.i} did just use the door at {agent_state.pos}.')
+                    info_dict.update(door_used=1)
+                else:
+                    self.print(f'Agent {agent_state.i} just tried to use a door '
+                               f'at {agent_state.pos}, but was unsucsessfull.')
+                    info_dict.update({f'agent_{agent_state.i}_failed_action': 1})
+                    info_dict.update({f'agent_{agent_state.i}_failed_door_open': 1})
+
             else:
                 info_dict.update(no_op=1)
                 reward -= 0.00
@@ -184,7 +200,7 @@ class SimpleFactory(BaseFactory):
 
 
 if __name__ == '__main__':
-    render = True
+    render = False
 
     move_props = MovementProperties(allow_diagonal_movement=True, allow_square_movement=True)
     dirt_props = DirtProperties()
@@ -193,8 +209,9 @@ if __name__ == '__main__':
                             pomdp_radius=3)
 
     n_actions = factory.action_space.n - 1
+    _ = factory.observation_space
 
-    for epoch in range(100):
+    for epoch in range(10000):
         random_actions = [[random.randint(0, n_actions) for _ in range(factory.n_agents)] for _ in range(200)]
         env_state = factory.reset()
         r = 0