diff --git a/environments/factory/assets/agent_collision.png b/environments/factory/assets/agent_collision.png deleted file mode 100644 index d7f228c..0000000 Binary files a/environments/factory/assets/agent_collision.png and /dev/null differ diff --git a/environments/factory/base_factory.py b/environments/factory/base_factory.py index 539d60d..7bb131e 100644 --- a/environments/factory/base_factory.py +++ b/environments/factory/base_factory.py @@ -22,10 +22,11 @@ class BaseFactory(gym.Env): @property def observation_space(self): agent_slice = self.n_agents if self.omit_agent_slice_in_obs else 0 - agent_slice = 1 if self.combin_agent_slices_in_obs else agent_slice + agent_slice = (self.n_agents - 1) if self.combin_agent_slices_in_obs else agent_slice if self.pomdp_radius: - return spaces.Box(low=0, high=1, shape=(self._state.shape[0] - agent_slice, self.pomdp_radius * 2 + 1, - self.pomdp_radius * 2 + 1), dtype=np.float32) + shape = (self._state.shape[0] - agent_slice, self.pomdp_radius * 2 + 1, self.pomdp_radius * 2 + 1) + space = spaces.Box(low=0, high=1, shape=shape, dtype=np.float32) + return space else: shape = [x-agent_slice if idx == 0 else x for idx, x in enumerate(self._state.shape)] space = spaces.Box(low=0, high=1, shape=shape, dtype=np.float32) @@ -194,6 +195,14 @@ class BaseFactory(gym.Env): if self.done_at_collision and collision_vec.any(): done = True + # Step the door close intervall + agents_pos = [agent.pos for agent in self._agent_states] + for door_i, door in enumerate(self._door_states): + if door.is_open and door.time_to_close and door.pos not in agents_pos: + door.time_to_close -= 1 + elif door.is_open and not door.time_to_close and door.pos not in agents_pos: + door.use() + reward, info = self.calculate_reward(self._agent_states) if self._steps >= self.max_steps: @@ -256,7 +265,7 @@ class BaseFactory(gym.Env): x_new = x + x_diff y_new = y + y_diff - if h.DOORS in self._state_slices.values(): + if h.DOORS in self._state_slices.values() and self._agent_states[agent_i]._last_pos != (-1, -1): door = [door for door in self._door_states if door.pos == (x, y)] if door: door = door[0] @@ -326,7 +335,7 @@ class BaseFactory(gym.Env): # Returns: Reward, Info raise NotImplementedError - def render(self): + def render(self, mode='human'): raise NotImplementedError def save_params(self, filepath: Path): diff --git a/environments/factory/levels/rooms.txt b/environments/factory/levels/rooms.txt index 781de13..43e8193 100644 --- a/environments/factory/levels/rooms.txt +++ b/environments/factory/levels/rooms.txt @@ -7,7 +7,7 @@ ###x#######x### #1111##2222222# #11111#2222#22# -#11111D2222222# +#11111x2222222# #11111#2222222# #11111#2222222# ############### \ No newline at end of file diff --git a/environments/factory/renderer.py b/environments/factory/renderer.py index b134687..fcc01a6 100644 --- a/environments/factory/renderer.py +++ b/environments/factory/renderer.py @@ -42,8 +42,6 @@ class Renderer: self.font.set_bold(1.0) print('Loading System font with pygame.font.Font took', time.time() - now) - - def fill_bg(self): self.screen.fill(Renderer.BG_COLOR) if self.grid_lines: @@ -71,9 +69,9 @@ class Renderer: def load_asset(self, path, factor=1.0): s = int(factor*self.cell_size) - wall_img = pygame.image.load(path).convert_alpha() - wall_img = pygame.transform.smoothscale(wall_img, (s, s)) - return wall_img + asset = pygame.image.load(path).convert_alpha() + asset = pygame.transform.smoothscale(asset, (s, s)) + return asset def render(self, entities): for event in pygame.event.get(): @@ -82,7 +80,10 @@ class Renderer: sys.exit() self.fill_bg() blits = deque() - for entity in entities: + for entity in [x for x in entities if 'door' in x.name]: + bp = self.blit_params(entity) + blits.append(bp) + for entity in [x for x in entities if 'door' not in x.name]: bp = self.blit_params(entity) blits.append(bp) if entity.name.lower() == 'agent': @@ -106,7 +107,6 @@ class Renderer: for blit in blits: self.screen.blit(**blit) - pygame.display.flip() self.clock.tick(self.fps) @@ -114,6 +114,6 @@ class Renderer: if __name__ == '__main__': renderer = Renderer(fps=2, cell_size=40) for i in range(15): - entity = Entity('agent', [5, i], 1, 'idle', 'idle') - renderer.render([entity]) + entity_1 = Entity('agent', [5, i], 1, 'idle', 'idle') + renderer.render([entity_1]) diff --git a/environments/factory/simple_factory.py b/environments/factory/simple_factory.py index 30de773..206febb 100644 --- a/environments/factory/simple_factory.py +++ b/environments/factory/simple_factory.py @@ -14,7 +14,7 @@ CLEAN_UP_ACTION = 'clean_up' class DirtProperties(NamedTuple): - clean_amount: int = 2 # How much does the robot clean with one action. + clean_amount: int = 2 # How much does the robot clean with one actions. max_spawn_ratio: float = 0.2 # On max how much tiles does the dirt spawn in percent. gain_amount: float = 0.5 # How much dirt does spawn per tile spawn_frequency: int = 5 # Spawn Frequency in Steps @@ -41,7 +41,7 @@ class SimpleFactory(BaseFactory): self._renderer = None # expensive - don't use it when not required ! super(SimpleFactory, self).__init__(*args, additional_slices=['dirt'], **kwargs) - def render(self): + def render(self, mode='human'): if not self._renderer: # lazy init height, width = self._state.shape[1:] @@ -67,7 +67,11 @@ class SimpleFactory(BaseFactory): for i, agent in enumerate(self._agent_states): name, state = asset_str(agent) agents.append(Entity(name, agent.pos, 1, 'none', state, i+1)) - self._renderer.render(dirt+walls+agents) + doors = [] + for i, door in enumerate(self._door_states): + name, state = 'door_open' if door.is_open else 'door_closed', 'blank' + agents.append(Entity(name, door.pos, 1, 'none', state, i+1)) + self._renderer.render(dirt+walls+agents+doors) def spawn_dirt(self) -> None: if not np.argwhere(self._state[DIRT_INDEX] != h.IS_FREE_CELL).shape[0] > self.dirt_properties.max_global_amount: @@ -156,6 +160,7 @@ class SimpleFactory(BaseFactory): self.print(f'Agent {agent_state.i} just tried to clean up some dirt ' f'at {agent_state.pos}, but was unsucsessfull.') info_dict.update({f'agent_{agent_state.i}_failed_action': 1}) + info_dict.update({f'agent_{agent_state.i}_failed_dirt_cleanup': 1}) elif self._actions.is_moving_action(agent_state.action): if agent_state.action_valid: @@ -165,6 +170,17 @@ class SimpleFactory(BaseFactory): # self.print('collision') reward -= 0.01 + elif self._actions.is_door_usage(agent_state.action): + if agent_state.action_valid: + reward += 0.1 + self.print(f'Agent {agent_state.i} did just use the door at {agent_state.pos}.') + info_dict.update(door_used=1) + else: + self.print(f'Agent {agent_state.i} just tried to use a door ' + f'at {agent_state.pos}, but was unsucsessfull.') + info_dict.update({f'agent_{agent_state.i}_failed_action': 1}) + info_dict.update({f'agent_{agent_state.i}_failed_door_open': 1}) + else: info_dict.update(no_op=1) reward -= 0.00 @@ -184,7 +200,7 @@ class SimpleFactory(BaseFactory): if __name__ == '__main__': - render = True + render = False move_props = MovementProperties(allow_diagonal_movement=True, allow_square_movement=True) dirt_props = DirtProperties() @@ -193,8 +209,9 @@ if __name__ == '__main__': pomdp_radius=3) n_actions = factory.action_space.n - 1 + _ = factory.observation_space - for epoch in range(100): + for epoch in range(10000): random_actions = [[random.randint(0, n_actions) for _ in range(factory.n_agents)] for _ in range(200)] env_state = factory.reset() r = 0 diff --git a/environments/oo_factory/__init__.py b/environments/oo_factory/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/environments/oo_factory/_base_factory.py b/environments/oo_factory/_base_factory.py new file mode 100644 index 0000000..5fe2091 --- /dev/null +++ b/environments/oo_factory/_base_factory.py @@ -0,0 +1,68 @@ +from typing import List, Union + +import gym + + +class Entities(): + + def __init__(self): + pass + + +# noinspection PyAttributeOutsideInit +class BaseFactory(gym.Env): + + def __enter__(self): + return self if self.frames_to_stack == 0 else FrameStack(self, self.frames_to_stack) + + def __exit__(self, exc_type, exc_val, exc_tb): + self.close() + + def __init__(self, level_name='simple', n_agents=1, max_steps=int(5e2), pomdp_radius: Union[None, int] = 0, + movement_properties: MovementProperties = MovementProperties(), + combin_agent_slices_in_obs: bool = False, frames_to_stack=0, + omit_agent_slice_in_obs=False, **kwargs): + assert (combin_agent_slices_in_obs != omit_agent_slice_in_obs) or \ + (not combin_agent_slices_in_obs and not omit_agent_slice_in_obs), \ + 'Both options are exclusive' + assert frames_to_stack != 1 and frames_to_stack >= 0, "'frames_to_stack' cannot be negative or 1." + + self.movement_properties = movement_properties + self.level_name = level_name + + self.n_agents = n_agents + self.max_steps = max_steps + self.pomdp_radius = pomdp_radius + self.combin_agent_slices_in_obs = combin_agent_slices_in_obs + self.omit_agent_slice_in_obs = omit_agent_slice_in_obs + self.frames_to_stack = frames_to_stack + + self.done_at_collision = False + + self._state_slices = StateSlices() + level_filepath = Path(__file__).parent / h.LEVELS_DIR / f'{self.level_name}.txt' + parsed_level = h.parse_level(level_filepath) + self._level = h.one_hot_level(parsed_level) + parsed_doors = h.one_hot_level(parsed_level, h.DOOR) + if parsed_doors.any(): + self._doors = parsed_doors + level_slices = ['level', 'doors'] + can_use_doors = True + else: + level_slices = ['level'] + can_use_doors = False + offset = len(level_slices) + self._state_slices.register_additional_items([*level_slices, + *[f'agent#{i}' for i in range(offset, n_agents + offset)]]) + if 'additional_slices' in kwargs: + self._state_slices.register_additional_items(kwargs.get('additional_slices')) + self._zones = Zones(parsed_level) + + self._actions = Actions(self.movement_properties, can_use_doors=can_use_doors) + self._actions.register_additional_items(self.additional_actions) + self.reset() + + + def step(self, actions: Union[int, List[int]]): + actions = actions if isinstance(actions, list) else [actions] + self.entities.step() \ No newline at end of file diff --git a/environments/utility_classes.py b/environments/utility_classes.py index 0449946..2ee27a5 100644 --- a/environments/utility_classes.py +++ b/environments/utility_classes.py @@ -108,10 +108,12 @@ class AgentState: class DoorState: - def __init__(self, i: int, pos: Tuple[int, int], closed_on_init=True): + def __init__(self, i: int, pos: Tuple[int, int], closed_on_init=True, auto_close_interval=10): self.i = i self.pos = pos self._state = self._state = IS_CLOSED if closed_on_init else IS_OPEN + self.auto_close_interval = auto_close_interval + self.time_to_close = -1 @property def is_closed(self): @@ -126,8 +128,11 @@ class DoorState: return self._state def use(self): - self._state: str = IS_CLOSED if self._state == IS_OPEN else IS_OPEN - + if self._state == IS_OPEN: + self._state = IS_CLOSED + else: + self._state = IS_OPEN + self.time_to_close = self.auto_close_interval class Register: diff --git a/main.py b/main.py index ee04369..92fc8a4 100644 --- a/main.py +++ b/main.py @@ -111,7 +111,7 @@ if __name__ == '__main__': kwargs = dict(ent_coef=0.01) elif modeL_type.__name__ in ["RegDQN", "DQN", "QRDQN"]: kwargs = dict(buffer_size=50000, - learning_starts=25000, + learning_starts=64, batch_size=64, target_update_interval=5000, exploration_fraction=0.25, diff --git a/reload_agent.py b/reload_agent.py index 58f3cce..faffb44 100644 --- a/reload_agent.py +++ b/reload_agent.py @@ -14,14 +14,14 @@ warnings.filterwarnings('ignore', category=UserWarning) if __name__ == '__main__': - model_name = 'PPO_1623052687' + model_name = 'A2C_1623923982' run_id = 0 out_path = Path(__file__).parent / 'debug_out' model_path = out_path / model_name with (model_path / f'env_{model_name}.yaml').open('r') as f: env_kwargs = yaml.load(f, Loader=yaml.FullLoader) - with SimpleFactory(level_name='rooms', **env_kwargs) as env: + with SimpleFactory(**env_kwargs) as env: # Edit THIS: model_files = list(natsorted((model_path / f'{run_id}_{model_name}').rglob('model_*.zip'))) @@ -30,5 +30,3 @@ if __name__ == '__main__': model = PPO.load(this_model) evaluation_result = evaluate_policy(model, env, n_eval_episodes=100, deterministic=False, render=True) print(evaluation_result) - -