Files
website/_site/research/mas-emergence-safety/index.html

770 lines
21 KiB
HTML
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
<!doctype html>
<!--
Minimal Mistakes Jekyll Theme 4.27.1 by Michael Rose
Copyright 2013-2025 Michael Rose - mademistakes.com | @mmistakes
Free for personal and commercial use under the MIT license
https://github.com/mmistakes/minimal-mistakes/blob/master/LICENSE
---
Modified by Steffen Illium 2023 - 2024
https://steffenillium.de/about
-->
<html lang="en-US" class="no-js">
<head>
<meta charset="utf-8">
<!-- begin _includes/seo.html --><title>MAS Emergence Safety — Steffen Illium</title>
<meta name="description" content="Formalized MAS emergence misalignment; proposed safety mitigation strategies.">
<meta name="author" content="Steffen Illium">
<meta property="article:author" content="Steffen Illium">
<meta property="og:type" content="article">
<meta property="og:locale" content="en_US">
<meta property="og:site_name" content="Steffen Illium">
<meta property="og:title" content="MAS Emergence Safety">
<meta property="og:url" content="http://localhost:4000/research/mas-emergence-safety/">
<meta property="og:description" content="Formalized MAS emergence misalignment; proposed safety mitigation strategies.">
<meta property="og:image" content="http://localhost:4000/assets/figures/21_coins_teaser.png">
<meta property="article:published_time" content="2024-10-27T00:00:00+02:00">
<link rel="canonical" href="http://localhost:4000/research/mas-emergence-safety/">
<meta name="google-site-verification" content="ZNT7_sXtFdgqBLCPLavTHWcviZRk__BNvClY8I-sFjU" />
<!-- end _includes/seo.html -->
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<script>
document.documentElement.className = document.documentElement.className.replace(/\bno-js\b/g, '') + ' js ';
</script>
<!-- For all browsers -->
<link rel="stylesheet" href="/assets/css/main.css">
<link rel="preload" href="https://cdn.jsdelivr.net/npm/@fortawesome/fontawesome-free@latest/css/all.min.css" as="style" onload="this.onload=null;this.rel='stylesheet'">
<noscript><link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/@fortawesome/fontawesome-free@latest/css/all.min.css"></noscript>
<!-- Academicons -->
<link rel="stylesheet" href="https://cdn.jsdelivr.net/gh/jpswalsh/academicons@1/css/academicons.min.css">
<!-- start custom head snippets -->
<!-- insert favicons. use https://realfavicongenerator.net/ -->
<!-- end custom head snippets -->
</head>
<body class="layout--single" dir="ltr">
<nav class="skip-links">
<ul>
<li><a href="#site-nav" class="screen-reader-shortcut">Skip to primary navigation</a></li>
<li><a href="#main" class="screen-reader-shortcut">Skip to content</a></li>
<li><a href="#footer" class="screen-reader-shortcut">Skip to footer</a></li>
</ul>
</nav>
<div class="masthead">
<div class="masthead__inner-wrap">
<div class="masthead__menu">
<nav id="site-nav" class="greedy-nav">
<a class="site-title" href="/">
portfolio
<span class="site-subtitle"></span>
</a>
<ul class="visible-links"><li class="masthead__menu-item">
<a
href="/publications"
>publications</a>
</li><li class="masthead__menu-item">
<a
href="/research"
>research</a>
</li><li class="masthead__menu-item">
<a
href="/projects"
>projects</a>
</li><li class="masthead__menu-item">
<a
href="/teaching"
>teaching</a>
</li><li class="masthead__menu-item">
<a
href="/about"
>about me</a>
</li></ul>
<button class="greedy-nav__toggle hidden" type="button">
<span class="visually-hidden">Toggle menu</span>
<div class="navicon"></div>
</button>
<ul class="hidden-links hidden"></ul>
</nav>
</div>
</div>
</div>
<div class="initial-content">
<div id="main" role="main">
<div class="sidebar sticky">
<div itemscope itemtype="https://schema.org/Person" class="h-card">
<div class="author__avatar">
<a href="http://localhost:4000/">
<img src="/assets/images/newshot_2.jpg" alt="Steffen Illium" itemprop="image" class="u-photo">
</a>
</div>
<div class="author__content">
<h3 class="author__name p-name" itemprop="name">
<a class="u-url" rel="me" href="http://localhost:4000/" itemprop="url">Steffen Illium</a>
</h3>
<div class="author__bio p-note" itemprop="description">
<p><a href="https://www.mobile.ifi.lmu.de/team/steffen-illium/">PhD. in Comp. Science</a> <br /><a href="/research/">AI Consultant &amp; Researcher</a></p>
</div>
</div>
<div class="author__urls-wrapper">
<button class="btn btn--inverse">Follow</button>
<ul class="author__urls social-icons">
<li itemprop="homeLocation" itemscope itemtype="https://schema.org/Place">
<i class="fas fa-fw fa-map-marker-alt" aria-hidden="true"></i> <span itemprop="name" class="p-locality">Augsburg</span>
</li>
<li><a href="https://www.mobile.ifi.lmu.de/team/steffen-illium/" rel="nofollow noopener noreferrer me" itemprop="sameAs"><i class="fas fa-link" aria-hidden="true"></i><span class="label">LMU-Munich</span></a></li>
<li><a href="https://scholar.google.de/citations?hl=en&pli=1&user=NODAd94AAAAJ" rel="nofollow noopener noreferrer me" itemprop="sameAs"><i class="fab fa-google-scholar" aria-hidden="true"></i><span class="label">Scholar</span></a></li>
<li><a href="https://arxiv.org/a/illium_s_1.html" rel="nofollow noopener noreferrer me" itemprop="sameAs"><i class="ai ai-arxiv" aria-hidden="true"></i><span class="label">Arxive</span></a></li>
<li><a href="https://www.researchgate.net/profile/Steffen-Illium" rel="nofollow noopener noreferrer me" itemprop="sameAs"><i class="fab fa-researchgate" aria-hidden="true"></i><span class="label">Researchgate</span></a></li>
<!--
<li>
<a href="http://link-to-whatever-social-network.com/user/" itemprop="sameAs" rel="nofollow noopener noreferrer me">
<i class="fas fa-fw" aria-hidden="true"></i> Custom Social Profile Link
</a>
</li>
-->
</ul>
</div>
</div>
</div>
<article class="page" itemscope itemtype="https://schema.org/CreativeWork">
<meta itemprop="headline" content="MAS Emergence Safety">
<meta itemprop="description" content="Formalized MAS emergence misalignment; proposed safety mitigation strategies.">
<meta itemprop="datePublished" content="2024-10-27T00:00:00+02:00">
<div class="page__inner-wrap">
<header>
<h1 id="page-title" class="page__title" itemprop="headline">
<a href="http://localhost:4000/research/mas-emergence-safety/" itemprop="url">MAS Emergence Safety
</a>
</h1>
<p class="page__meta">
<span class="page__meta-readtime">
<i class="far fa-clock" aria-hidden="true"></i>
1 minute read
</span>
</p>
</header>
<br>
<section class="page__content" itemprop="text">
<hr>
<header><h4 class="nav__title"><i class="fas fa-book"></i> Reference</h4></header>
<ol class="bibliography"><li><span id="altmann2024emergence"><span style="font-variant: small-caps">Altmann, P., Schönberger, J., Illium, S., et al.</span> 2024. Emergence in Multi-agent Systems: A Safety Perspective. <i>International Symposium on Leveraging Applications of Formal Methods</i>, Springer Nature Switzerland Cham, 104120.</span>
<div class="publication-actions" style="margin-top: 0.5em;">
<a href="/assets/publications/altmann2024emergence.pdf" style="text-decoration: none;">
<button class="btn btnId btnPub--action btnPub--pdf-download"
data-umami-event="PDF Download"
data-umami-event-bibkey="altmann2024emergence"
data-umami-event-slug="mas-emergence-safety"
style="outline:none; position:relative; white-space: normal; margin-right: 5px; vertical-align: middle;">
<i class="fas fa-download"></i> PDF
</button>
</a>
<button type="button" class="btn btnId btnPub--action btnPub--bibtex-download-direct"
data-umami-event="BibTeX File Download"
data-umami-event-bibkey="altmann2024emergence"
data-umami-event-slug="mas-emergence-safety"
data-bibtex-json='"@inproceedings{altmann2024emergence,\n title = {Emergence in Multi-agent Systems: A Safety Perspective},\n author = {Altmann, Philipp and Schönberger, Julian and Illium, Steffen and Zorn, Maximilian and Ritz, Fabian and Haider, Tom and Burton, Simon and Gabor, Thomas},\n booktitle = {International Symposium on Leveraging Applications of Formal Methods},\n pages = {104--120},\n year = {2024},\n organization = {Springer Nature Switzerland Cham}\n}\n"'
data-bibtex-filename='altmann2024emergence.bib'
style="outline:none; position:relative; white-space: normal; margin-right: 5px; vertical-align: middle;">
<i class="fas fa-download"></i> BibTeX
</button>
<button type="button" class="btn btnId btnPub--action btnPub--bibtex-copy-direct"
data-umami-event="BibTeX Copy"
data-umami-event-bibkey="altmann2024emergence"
data-umami-event-slug="mas-emergence-safety"
data-bibtex-json='"@inproceedings{altmann2024emergence,\n title = {Emergence in Multi-agent Systems: A Safety Perspective},\n author = {Altmann, Philipp and Schönberger, Julian and Illium, Steffen and Zorn, Maximilian and Ritz, Fabian and Haider, Tom and Burton, Simon and Gabor, Thomas},\n booktitle = {International Symposium on Leveraging Applications of Formal Methods},\n pages = {104--120},\n year = {2024},\n organization = {Springer Nature Switzerland Cham}\n}\n"'
style="outline:none; position:relative; white-space: normal; vertical-align: middle;">
<i class="fas fa-clipboard"></i> Copy
</button>
</div></li></ol>
<hr>
<p style="display:block; width:40%" class="align-right"><img src="\assets\figures\21_envs.png" alt="Diagrams of the gridworld environments used for evaluation" /></p>
<p>Multi-Agent Systems (MAS), particularly those employing decentralized decision-making based on local information (common in MARL), can exhibit <strong>emergent effects</strong>. These phenomena, arising from complex interactions, range from minor behavioral quirks to potentially catastrophic system failures, posing significant <strong>safety challenges</strong>.</p>
<p>This research provides a framework for understanding and mitigating undesirable emergence from a <strong>safety perspective</strong>. We propose a formal definition: emergent effects arise from <strong>misalignments between the <em>global inherent specification</em></strong> (the intended overall system goal or behavior) <strong>and its <em>local approximation</em></strong> used by individual agents (e.g., distinct reward components, limited observations).</p>
<center>
<img src="/assets/figures/21_coins.png" alt="Visualization showing agents exhibiting emergent coin-collecting behavior" style="display:block; width:70%" />
<figcaption>Example of emergent behavior (e.g., coin hoarding) due to specification misalignment.</figcaption>
</center>
<p><br /></p>
<p>Leveraging established concepts from system safety engineering, we analyze how such misalignments can lead to deviations from intended global behavior. To illustrate the practical implications, we examine two highly configurable gridworld scenarios. These demonstrate how inadequate or independently derived local specifications (rewards/observations) can predictably result in unintended emergent behaviors, such as resource hoarding or inefficient coordination.</p>
<center>
<img src="/assets/figures/21_blocking.png" alt="Visualization showing agents exhibiting emergent blocking behavior" style="display:block; width:60%" />
<figcaption>Example of emergent behavior (e.g., mutual blocking) due to specification misalignment.</figcaption>
</center>
<p><br /></p>
<p>Recognizing that achieving a perfectly aligned global specification might be impractical in complex systems, we propose strategies focused on <strong>adjusting the underlying local parameterizations</strong> (e.g., reward shaping, observation design) to mitigate harmful emergence. By carefully tuning these local components, system alignment can be improved, reducing the risk of emergent failures and enhancing overall safety. <a class="citation" href="/publications#altmann2024emergence">[Altmann et al. 2024]</a></p>
</section>
<footer class="page__meta">
<p class="page__taxonomy">
<strong><i class="fas fa-fw fa-tags" aria-hidden="true"></i> Tags: </strong>
<span itemprop="keywords">
<a href="/tags/emergence" class="page__taxonomy-item p-category" rel="tag">emergence</a><span class="sep">, </span>
<a href="/tags/marl" class="page__taxonomy-item p-category" rel="tag">MARL</a><span class="sep">, </span>
<a href="/tags/multi-agent-systems" class="page__taxonomy-item p-category" rel="tag">multi-agent-systems</a><span class="sep">, </span>
<a href="/tags/safety" class="page__taxonomy-item p-category" rel="tag">safety</a><span class="sep">, </span>
<a href="/tags/system-specification" class="page__taxonomy-item p-category" rel="tag">system-specification</a>
</span>
</p>
<p class="page__date"><strong><i class="fas fa-fw fa-calendar-alt" aria-hidden="true"></i> Updated:</strong> <time class="dt-published" datetime="2024-10-27T00:00:00+02:00">October 27, 2024</time></p>
</footer>
<!--
<nav class="pagination">
<a href="/research/aquarium-marl-environment/" class="pagination--pager" title="Aquarium MARL Environment">Previous</a>
<a href="#" class="pagination--pager disabled">Next</a>
</nav>
-->
</div>
</article>
<div class="page__related">
<h2 class="page__related-title">Related posts</h2>
<div class="grid__wrapper">
<div class="list__item">
<article class="archive__item" itemscope itemtype="https://schema.org/CreativeWork"
style="display: flex; align-items: flex-start;"
>
<div class="archive__item-teaser-list" style="margin-right: 1em; flex-shrink: 0;">
<img src="/assets/figures/20_aquarium.png" alt="" style="display: block;">
</div>
<div style="flex-grow: 1;" >
<h2 class="archive__item-title no_toc" itemprop="headline" style="text-align: left !important;">
<a href="/research/aquarium-marl-environment/" rel="permalink">Aquarium MARL Environment
</a>
</h2>
<p class="page__meta">
<span class="page__meta-readtime">
<i class="far fa-clock" aria-hidden="true"></i>
1 minute read
</span>
</p>
<p class="archive__item-excerpt" itemprop="description" style="text-align: left;">
Aquarium: Open-source MARL environment for predator-prey studies.
</p>
</div>
</article>
</div>
<div class="list__item">
<article class="archive__item" itemscope itemtype="https://schema.org/CreativeWork"
style="display: flex; align-items: flex-start;"
>
<div class="archive__item-teaser-list" style="margin-right: 1em; flex-shrink: 0;">
<img src="/assets/images/projects/arch.png" alt="" style="display: block;">
</div>
<div style="flex-grow: 1;" >
<h2 class="archive__item-title no_toc" itemprop="headline" style="text-align: left !important;">
<a href="/projects/server-administration/" rel="permalink">LMU DevOps Admin
</a>
</h2>
<p class="page__meta">
<span class="page__meta-readtime">
<i class="far fa-clock" aria-hidden="true"></i>
1 minute read
</span>
</p>
<p class="archive__item-excerpt" itemprop="description" style="text-align: left;">
Managed LMU chair IT: Kubernetes, CI/CD, automation (2018-2023).
</p>
</div>
</article>
</div>
<div class="list__item">
<article class="archive__item" itemscope itemtype="https://schema.org/CreativeWork"
style="display: flex; align-items: flex-start;"
>
<div class="archive__item-teaser-list" style="margin-right: 1em; flex-shrink: 0;">
<img src="/assets/figures/19_binary_primates_teaser.jpg" alt="" style="display: block;">
</div>
<div style="flex-grow: 1;" >
<h2 class="archive__item-title no_toc" itemprop="headline" style="text-align: left !important;">
<a href="/research/primate-subsegment-sorting/" rel="permalink">Primate Subsegment Sorting
</a>
</h2>
<p class="page__meta">
<span class="page__meta-readtime">
<i class="far fa-clock" aria-hidden="true"></i>
1 minute read
</span>
</p>
<p class="archive__item-excerpt" itemprop="description" style="text-align: left;">
Binary subsegment presorting improves noisy primate sound classification.
</p>
</div>
</article>
</div>
<div class="list__item">
<article class="archive__item" itemscope itemtype="https://schema.org/CreativeWork"
style="display: flex; align-items: flex-start;"
>
<div class="archive__item-teaser-list" style="margin-right: 1em; flex-shrink: 0;">
<img src="/assets/figures/18_surprised_soup_teaser.jpg" alt="" style="display: block;">
</div>
<div style="flex-grow: 1;" >
<h2 class="archive__item-title no_toc" itemprop="headline" style="text-align: left !important;">
<a href="/research/emergent-social-dynamics/" rel="permalink">Emergent Social Dynamics
</a>
</h2>
<p class="page__meta">
<span class="page__meta-readtime">
<i class="far fa-clock" aria-hidden="true"></i>
1 minute read
</span>
</p>
<p class="archive__item-excerpt" itemprop="description" style="text-align: left;">
Artificial chemistry networks develop predictive models via surprise minimization.
</p>
</div>
</article>
</div>
</div>
</div>
</div>
</div>
<div id="footer" class="page__footer">
<footer>
<!-- start custom footer snippets -->
<!-- end custom footer snippets -->
<div class="page__footer-follow">
<ul class="social-icons">
<li><strong>Contact:</strong></li>
<li><a href="mailto:steffen.illium@ifi.lmu.de" rel="nofollow noopener noreferrer"><i class="fas fa-fw fa-envelope" aria-hidden="true"></i> Email</a></li>
<li><a href="https://www.linkedin.com/in/steffen-illium/" rel="nofollow noopener noreferrer"><i class="fab fa-fw fa-linkedin" aria-hidden="true"></i> LinkedIn</a></li>
<li><a href="https://gitea.steffenillium.de/steffen" rel="nofollow noopener noreferrer"><i class="fab fa-git-alt" aria-hidden="true"></i> Gitea</a></li>
<li><a href="/status/" rel="nofollow noopener noreferrer"><img src="https://uptime.steffenillium.de/api/badge/2/status" alt="System Status"></a></li>
</ul>
</div>
<div class="page__footer-copyright">&copy; 2025 <a href="http://localhost:4000">Steffen Illium</a>. Powered by <a href="https://jekyllrb.com" rel="nofollow">Jekyll</a> &amp; <a href="https://mademistakes.com/work/jekyll-themes/minimal-mistakes/" rel="nofollow">Minimal Mistakes</a>.</div>
</footer>
</div>
<script src="/assets/js/main.min.js"></script>
<script src="/assets/js/custom-bibtex.js"></script>
</body>
</html>