{
  "id": "2404.07972",
  "title": "OSWorld: Benchmarking Multimodal Agents for Open-Ended Tasks in Real Computer Environments",
  "authors": "Tianbao Xie, Danyang Zhang, Jixuan Chen, Xiaochuan Li, Siheng Zhao, Ruisheng Cao, Toh Jing Hua, Zhoujun Cheng, Dongchan Shin, Fangyu Lei, Yitao Liu, Yiheng Xu, Shuyan Zhou, Silvio Savarese, Caiming Xiong, Victor Zhong, Tao Yu",
  "first_public": "2024-04-11",
  "first_public_source": "https://arxiv.org/abs/2404.07972",
  "venue": "NeurIPS 2024",
  "arxiv": "2404.07972",
  "paper_url": "https://arxiv.org/abs/2404.07972",
  "lens": [
    "evaluation"
  ],
  "related_terms": [
    "OSWorld",
    "computer-use agents",
    "execution-based evaluation",
    "desktop benchmark"
  ],
  "summary": "Provides real computer environments and reproducible tasks spanning desktop applications, websites, and file operations.",
  "contribution": "Pairs task setup with execution-based evaluators, allowing agents to complete workflows through actual software rather than matching a fixed action sequence.",
  "limitations": "Evaluates workflows in real desktop applications, websites, and files.",
  "model_output": "Observations and evaluation results from real computer environments.",
  "action_interface": "Computer interaction through the benchmark's supported action interfaces.",
  "state_memory": "State persists in the real operating system and applications used by the environment.",
  "relationship": "Provides desktop tasks and outcome checks for evaluating computer-use agents.",
  "relationship_source": "https://arxiv.org/abs/2404.07972",
  "author_names": [
    "Tianbao Xie",
    "Danyang Zhang",
    "Jixuan Chen",
    "Xiaochuan Li",
    "Siheng Zhao",
    "Ruisheng Cao",
    "Toh Jing Hua",
    "Zhoujun Cheng",
    "Dongchan Shin",
    "Fangyu Lei",
    "Yitao Liu",
    "Yiheng Xu",
    "Shuyan Zhou",
    "Silvio Savarese",
    "Caiming Xiong",
    "Victor Zhong",
    "Tao Yu"
  ],
  "record_url": "https://neural-os.com/reading-list/papers/2404.07972/",
  "legacy_record_url": "https://neural-os.com/reading-list/papers/#paper-2404-07972",
  "citation_date": "2024-04-11",
  "json_url": "https://neural-os.com/reading-list/papers/2404.07972/record.json",
  "markdown_url": "https://neural-os.com/reading-list/papers/2404.07972/index.md",
  "bibtex_url": "https://neural-os.com/reading-list/papers/2404.07972/citation.bib",
  "csl_url": "https://neural-os.com/reading-list/papers/2404.07972/citation.csl.json",
  "publication": {
    "title": "OSWorld: Benchmarking Multimodal Agents for Open-Ended Tasks in Real Computer Environments",
    "author_names": [
      "Tianbao Xie",
      "Danyang Zhang",
      "Jixuan Chen",
      "Xiaochuan Li",
      "Siheng Zhao",
      "Ruisheng Cao",
      "Toh Jing Hua",
      "Zhoujun Cheng",
      "Dongchan Shin",
      "Fangyu Lei",
      "Yitao Liu",
      "Yiheng Xu",
      "Shuyan Zhou",
      "Silvio Savarese",
      "Caiming Xiong",
      "Victor Zhong",
      "Tao Yu"
    ],
    "year": 2024,
    "type": "paper-conference",
    "container_title": "Advances in Neural Information Processing Systems",
    "volume": "37",
    "doi": "10.52202/079017-1650",
    "url": "https://proceedings.neurips.cc/paper_files/paper/2024/hash/5d413e48f84dc61244b6be550f1cd8f5-Abstract-Datasets_and_Benchmarks_Track.html",
    "source_url": "https://proceedings.neurips.cc/paper_files/paper/2024/hash/5d413e48f84dc61244b6be550f1cd8f5-Abstract-Datasets_and_Benchmarks_Track.html",
    "verified_at": "2026-10-19"
  },
  "publication_bibtex_url": "https://neural-os.com/reading-list/papers/2404.07972/publication.bib",
  "publication_csl_url": "https://neural-os.com/reading-list/papers/2404.07972/publication.csl.json"
}