{
  "id": "2405.14573",
  "title": "AndroidWorld: A Dynamic Benchmarking Environment for Autonomous Agents",
  "authors": "Christopher Rawles, Sarah Clinckemaillie, Yifan Chang, Jonathan Waltz, Gabrielle Lau, Marybeth Fair, Alice Li, William Bishop, Wei Li, Folawiyo Campbell-Ajala, Daniel Toyama, Robert Berry, Divya Tyamagundlu, Timothy Lillicrap, Oriana Riva",
  "first_public": "2024-05-23",
  "first_public_source": "https://arxiv.org/abs/2405.14573",
  "venue": "ICLR 2025",
  "arxiv": "2405.14573",
  "paper_url": "https://arxiv.org/abs/2405.14573",
  "lens": [
    "evaluation"
  ],
  "related_terms": [
    "AndroidWorld",
    "mobile agents",
    "GUI evaluation",
    "task success"
  ],
  "summary": "Provides a functional Android environment with parameterized tasks and programmatic checks of device state.",
  "contribution": "Combines task initialization, success checking, and teardown with variable task instances, making mobile-agent evaluation reproducible and less dependent on a fixed test set.",
  "limitations": "The benchmark covers Android apps with parameterized tasks and programmatic success checks.",
  "model_output": "Android observations and task rewards.",
  "action_interface": "Agent actions executed on an Android device environment.",
  "state_memory": "The Android system and apps maintain state; evaluators inspect that state directly.",
  "relationship": "Provides Android tasks and outcome checks for evaluating mobile agents.",
  "relationship_source": "https://arxiv.org/abs/2405.14573",
  "author_names": [
    "Christopher Rawles",
    "Sarah Clinckemaillie",
    "Yifan Chang",
    "Jonathan Waltz",
    "Gabrielle Lau",
    "Marybeth Fair",
    "Alice Li",
    "William Bishop",
    "Wei Li",
    "Folawiyo Campbell-Ajala",
    "Daniel Toyama",
    "Robert Berry",
    "Divya Tyamagundlu",
    "Timothy Lillicrap",
    "Oriana Riva"
  ],
  "record_url": "https://neural-os.com/reading-list/papers/2405.14573/",
  "legacy_record_url": "https://neural-os.com/reading-list/papers/#paper-2405-14573",
  "citation_date": "2024-05-23",
  "json_url": "https://neural-os.com/reading-list/papers/2405.14573/record.json",
  "markdown_url": "https://neural-os.com/reading-list/papers/2405.14573/index.md",
  "bibtex_url": "https://neural-os.com/reading-list/papers/2405.14573/citation.bib",
  "csl_url": "https://neural-os.com/reading-list/papers/2405.14573/citation.csl.json",
  "publication": {
    "title": "AndroidWorld: A Dynamic Benchmarking Environment for Autonomous Agents",
    "author_names": [
      "Chris Rawles",
      "Sarah Clinckemaillie",
      "Yifan Chang",
      "Jonathan Waltz",
      "Gabrielle Lau",
      "Marybeth Fair",
      "Alice Li",
      "William Bishop",
      "Wei Li",
      "Folawiyo Campbell-Ajala",
      "Daniel Toyama",
      "Robert Berry",
      "Divya Tyamagundlu",
      "Timothy Lillicrap",
      "Oriana Riva"
    ],
    "year": 2025,
    "type": "paper-conference",
    "container_title": "International Conference on Learning Representations",
    "url": "https://proceedings.iclr.cc/paper_files/paper/2025/hash/01a83bc2f2732a58e6aa731e659e7101-Abstract-Conference.html",
    "source_url": "https://proceedings.iclr.cc/paper_files/paper/2025/hash/01a83bc2f2732a58e6aa731e659e7101-Abstract-Conference.html",
    "verified_at": "2026-10-19"
  },
  "publication_bibtex_url": "https://neural-os.com/reading-list/papers/2405.14573/publication.bib",
  "publication_csl_url": "https://neural-os.com/reading-list/papers/2405.14573/publication.csl.json"
}