{ "metadata": { "version": "1.0.1", "updated_at": "2026-10-19", "search_refreshed_at": "2026-10-19", "topic_id": "evaluation", "name": "Environments and evaluation", "definition": "Test action response, visual quality, remembered state, and completed tasks separately. Real desktop and mobile environments provide execution-based checks that complement learned-simulator measurements.", "record_count": 4, "topic_url": "https://neural-os.com/reading-list/guides/evaluation/", "source_url": "https://neural-os.com/reading-list/data/papers.json" }, "comparisons": [ ], "demos": [ ], "papers": [ { "id": "2404.07972", "title": "OSWorld: Benchmarking Multimodal Agents for Open-Ended Tasks in Real Computer Environments", "authors": "Tianbao Xie, Danyang Zhang, Jixuan Chen, Xiaochuan Li, Siheng Zhao, Ruisheng Cao, Toh Jing Hua, Zhoujun Cheng, Dongchan Shin, Fangyu Lei, Yitao Liu, Yiheng Xu, Shuyan Zhou, Silvio Savarese, Caiming Xiong, Victor Zhong, Tao Yu", "first_public": "2024-04-11", "first_public_source": "https://arxiv.org/abs/2404.07972", "venue": "NeurIPS 2024", "arxiv": "2404.07972", "paper_url": "https://arxiv.org/abs/2404.07972", "lens": [ "evaluation" ], "related_terms": [ "OSWorld", "computer-use agents", "execution-based evaluation", "desktop benchmark" ], "summary": "Provides real computer environments and reproducible tasks spanning desktop applications, websites, and file operations.", "contribution": "Pairs task setup with execution-based evaluators, allowing agents to complete workflows through actual software rather than matching a fixed action sequence.", "limitations": "Evaluates workflows in real desktop applications, websites, and files.", "model_output": "Observations and evaluation results from real computer environments.", "action_interface": "Computer interaction through the benchmark's supported action interfaces.", "state_memory": "State persists in the real operating system and applications used by the environment.", "relationship": "Provides desktop tasks and outcome checks for evaluating computer-use agents.", "relationship_source": "https://arxiv.org/abs/2404.07972", "author_names": [ "Tianbao Xie", "Danyang Zhang", "Jixuan Chen", "Xiaochuan Li", "Siheng Zhao", "Ruisheng Cao", "Toh Jing Hua", "Zhoujun Cheng", "Dongchan Shin", "Fangyu Lei", "Yitao Liu", "Yiheng Xu", "Shuyan Zhou", "Silvio Savarese", "Caiming Xiong", "Victor Zhong", "Tao Yu" ], "record_url": "https://neural-os.com/reading-list/papers/2404.07972/", "legacy_record_url": "https://neural-os.com/reading-list/papers/#paper-2404-07972", "citation_date": "2024-04-11", "json_url": "https://neural-os.com/reading-list/papers/2404.07972/record.json", "markdown_url": "https://neural-os.com/reading-list/papers/2404.07972/index.md", "bibtex_url": "https://neural-os.com/reading-list/papers/2404.07972/citation.bib", "csl_url": "https://neural-os.com/reading-list/papers/2404.07972/citation.csl.json", "publication": { "title": "OSWorld: Benchmarking Multimodal Agents for Open-Ended Tasks in Real Computer Environments", "author_names": [ "Tianbao Xie", "Danyang Zhang", "Jixuan Chen", "Xiaochuan Li", "Siheng Zhao", "Ruisheng Cao", "Toh Jing Hua", "Zhoujun Cheng", "Dongchan Shin", "Fangyu Lei", "Yitao Liu", "Yiheng Xu", "Shuyan Zhou", "Silvio Savarese", "Caiming Xiong", "Victor Zhong", "Tao Yu" ], "year": 2024, "type": "paper-conference", "container_title": "Advances in Neural Information Processing Systems", "volume": "37", "doi": "10.52202/079017-1650", "url": "https://proceedings.neurips.cc/paper_files/paper/2024/hash/5d413e48f84dc61244b6be550f1cd8f5-Abstract-Datasets_and_Benchmarks_Track.html", "source_url": "https://proceedings.neurips.cc/paper_files/paper/2024/hash/5d413e48f84dc61244b6be550f1cd8f5-Abstract-Datasets_and_Benchmarks_Track.html", "verified_at": "2026-10-19" }, "publication_bibtex_url": "https://neural-os.com/reading-list/papers/2404.07972/publication.bib", "publication_csl_url": "https://neural-os.com/reading-list/papers/2404.07972/publication.csl.json" }, { "id": "2405.12399", "title": "Diffusion for World Modeling: Visual Details Matter in Atari", "authors": "Eloi Alonso, Adam Jelley, Vincent Micheli, Anssi Kanervisto, Amos Storkey, Tim Pearce, François Fleuret", "first_public": "2024-05-20", "first_public_source": "https://arxiv.org/abs/2405.12399", "venue": "NeurIPS 2024", "arxiv": "2405.12399", "paper_url": "https://arxiv.org/abs/2405.12399", "lens": [ "interactive-worlds", "evaluation" ], "related_terms": [ "DIAMOND", "diffusion world model", "neural game engine", "model-based reinforcement learning" ], "summary": "Introduces DIAMOND, an agent trained in a diffusion world model, and studies how preserving visual detail affects control.", "contribution": "Connects diffusion-based observation prediction with Atari policy learning and demonstrates an interactive Counter-Strike simulator.", "limitations": "Evaluated on games; NeuralOS adapts and evaluates it for desktop screen prediction.", "model_output": "Predicted visual observations; auxiliary reward and termination predictions support reinforcement learning.", "action_interface": "Actions of the modeled game environment.", "state_memory": "Visual dynamics condition on a finite history of observations and actions.", "relationship": "NeuralOS adapts DIAMOND to OS screen prediction as a baseline in Appendix A of its 2026 revision.", "relationship_source": "https://arxiv.org/html/2507.08800v2#A1", "author_names": [ "Eloi Alonso", "Adam Jelley", "Vincent Micheli", "Anssi Kanervisto", "Amos Storkey", "Tim Pearce", "François Fleuret" ], "record_url": "https://neural-os.com/reading-list/papers/2405.12399/", "legacy_record_url": "https://neural-os.com/reading-list/papers/#paper-2405-12399", "citation_date": "2024-05-20", "json_url": "https://neural-os.com/reading-list/papers/2405.12399/record.json", "markdown_url": "https://neural-os.com/reading-list/papers/2405.12399/index.md", "bibtex_url": "https://neural-os.com/reading-list/papers/2405.12399/citation.bib", "csl_url": "https://neural-os.com/reading-list/papers/2405.12399/citation.csl.json", "method_comparisons": [ { "title": "A game world model evaluated on desktop interactions", "text": "DIAMOND learns visual game dynamics with a diffusion model. NeuralOS adapts DIAMOND to OS screen prediction as a baseline in Appendix A of its 2026 revision. NeuralOS adds a recurrent state representation and spatial cursor conditioning for desktop interaction.", "source_ids": [ "2405.12399", "2507.08800" ], "source_urls": [ "https://arxiv.org/abs/2405.12399", "https://arxiv.org/html/2507.08800v2#A1" ], "source_labels": [ "DIAMOND paper", "NeuralOS Appendix A" ], "topic_id": "interactive-worlds", "topic_label": "Interactive learned environments", "topic_url": "https://neural-os.com/reading-list/guides/games-to-computers/" } ] }, { "id": "2405.14573", "title": "AndroidWorld: A Dynamic Benchmarking Environment for Autonomous Agents", "authors": "Christopher Rawles, Sarah Clinckemaillie, Yifan Chang, Jonathan Waltz, Gabrielle Lau, Marybeth Fair, Alice Li, William Bishop, Wei Li, Folawiyo Campbell-Ajala, Daniel Toyama, Robert Berry, Divya Tyamagundlu, Timothy Lillicrap, Oriana Riva", "first_public": "2024-05-23", "first_public_source": "https://arxiv.org/abs/2405.14573", "venue": "ICLR 2025", "arxiv": "2405.14573", "paper_url": "https://arxiv.org/abs/2405.14573", "lens": [ "evaluation" ], "related_terms": [ "AndroidWorld", "mobile agents", "GUI evaluation", "task success" ], "summary": "Provides a functional Android environment with parameterized tasks and programmatic checks of device state.", "contribution": "Combines task initialization, success checking, and teardown with variable task instances, making mobile-agent evaluation reproducible and less dependent on a fixed test set.", "limitations": "The benchmark covers Android apps with parameterized tasks and programmatic success checks.", "model_output": "Android observations and task rewards.", "action_interface": "Agent actions executed on an Android device environment.", "state_memory": "The Android system and apps maintain state; evaluators inspect that state directly.", "relationship": "Provides Android tasks and outcome checks for evaluating mobile agents.", "relationship_source": "https://arxiv.org/abs/2405.14573", "author_names": [ "Christopher Rawles", "Sarah Clinckemaillie", "Yifan Chang", "Jonathan Waltz", "Gabrielle Lau", "Marybeth Fair", "Alice Li", "William Bishop", "Wei Li", "Folawiyo Campbell-Ajala", "Daniel Toyama", "Robert Berry", "Divya Tyamagundlu", "Timothy Lillicrap", "Oriana Riva" ], "record_url": "https://neural-os.com/reading-list/papers/2405.14573/", "legacy_record_url": "https://neural-os.com/reading-list/papers/#paper-2405-14573", "citation_date": "2024-05-23", "json_url": "https://neural-os.com/reading-list/papers/2405.14573/record.json", "markdown_url": "https://neural-os.com/reading-list/papers/2405.14573/index.md", "bibtex_url": "https://neural-os.com/reading-list/papers/2405.14573/citation.bib", "csl_url": "https://neural-os.com/reading-list/papers/2405.14573/citation.csl.json", "publication": { "title": "AndroidWorld: A Dynamic Benchmarking Environment for Autonomous Agents", "author_names": [ "Chris Rawles", "Sarah Clinckemaillie", "Yifan Chang", "Jonathan Waltz", "Gabrielle Lau", "Marybeth Fair", "Alice Li", "William Bishop", "Wei Li", "Folawiyo Campbell-Ajala", "Daniel Toyama", "Robert Berry", "Divya Tyamagundlu", "Timothy Lillicrap", "Oriana Riva" ], "year": 2025, "type": "paper-conference", "container_title": "International Conference on Learning Representations", "url": "https://proceedings.iclr.cc/paper_files/paper/2025/hash/01a83bc2f2732a58e6aa731e659e7101-Abstract-Conference.html", "source_url": "https://proceedings.iclr.cc/paper_files/paper/2025/hash/01a83bc2f2732a58e6aa731e659e7101-Abstract-Conference.html", "verified_at": "2026-10-19" }, "publication_bibtex_url": "https://neural-os.com/reading-list/papers/2405.14573/publication.bib", "publication_csl_url": "https://neural-os.com/reading-list/papers/2405.14573/publication.csl.json" }, { "id": "2507.08800", "title": "NeuralOS: Towards Simulating Operating Systems via Neural Generative Models", "authors": "Luke Rivard, Sun Sun, Hongyu Guo, Wenhu Chen, Yuntian Deng", "first_public": "2025-07-11", "first_public_source": "https://arxiv.org/abs/2507.08800", "first_public_note": "Initial arXiv submission on July 11, 2025; revised on March 12, 2026 and published at ICLR 2026.", "venue": "ICLR 2026", "arxiv": "2507.08800", "paper_url": "https://arxiv.org/abs/2507.08800", "original_url": "https://arxiv.org/html/2507.08800v1", "revised_url": "https://arxiv.org/html/2507.08800v2", "code_url": "https://github.com/yuntian-group/neural-os", "lens": [ "neural-os", "interactive-worlds", "evaluation" ], "related_terms": [ "NeuralOS", "neural operating system", "learned computer simulator", "generative interface", "GUI world model", "desktop world model", "action-conditioned screen generation" ], "summary": "Studies neural operating systems through learned, interactive GUI simulation: a model produces desktop screens in response to mouse and keyboard events, opening a path toward generative computer interfaces.", "contribution": "Formalizes OS-interface generation and develops state tracking, precise cursor conditioning, staged training, and data collection. The 2026 revision also demonstrates learning a new simulated application from synthesized training data.", "limitations": "Fine-grained typing and long-horizon state retention remain difficult. The simulated interface does not provide the underlying filesystem, processes, or application services of a conventional OS.", "model_output": "Autoregressively generated desktop screen frames.", "action_interface": "Cursor coordinates, left/right clicks, and keyboard press/release events.", "state_memory": "A hierarchical recurrent model carries state across frames and conditions a diffusion renderer.", "relationship": "Introduces NeuralOS and its formulation of learned OS-interface simulation.", "relationship_source": "https://arxiv.org/html/2507.08800v1", "author_names": [ "Luke Rivard", "Sun Sun", "Hongyu Guo", "Wenhu Chen", "Yuntian Deng" ], "record_url": "https://neural-os.com/reading-list/papers/2507.08800/", "legacy_record_url": "https://neural-os.com/reading-list/papers/#paper-2507-08800", "citation_date": "2025-07-11", "json_url": "https://neural-os.com/reading-list/papers/2507.08800/record.json", "markdown_url": "https://neural-os.com/reading-list/papers/2507.08800/index.md", "bibtex_url": "https://neural-os.com/reading-list/papers/2507.08800/citation.bib", "csl_url": "https://neural-os.com/reading-list/papers/2507.08800/citation.csl.json", "publication": { "title": "NeuralOS: Towards Simulating Operating Systems via Neural Generative Models", "author_names": [ "Luke Rivard", "Sun Sun", "Hongyu Guo", "Wenhu Chen", "Yuntian Deng" ], "year": 2026, "type": "paper-conference", "container_title": "International Conference on Learning Representations", "url": "https://proceedings.iclr.cc/paper_files/paper/2026/hash/2174a6749c3f769358cd21ade1221b34-Abstract-Conference.html", "source_url": "https://proceedings.iclr.cc/paper_files/paper/2026/hash/2174a6749c3f769358cd21ade1221b34-Abstract-Conference.html", "verified_at": "2026-10-19" }, "publication_bibtex_url": "https://neural-os.com/reading-list/papers/2507.08800/publication.bib", "publication_csl_url": "https://neural-os.com/reading-list/papers/2507.08800/publication.csl.json", "method_comparisons": [ { "title": "A game world model evaluated on desktop interactions", "text": "DIAMOND learns visual game dynamics with a diffusion model. NeuralOS adapts DIAMOND to OS screen prediction as a baseline in Appendix A of its 2026 revision. NeuralOS adds a recurrent state representation and spatial cursor conditioning for desktop interaction.", "source_ids": [ "2405.12399", "2507.08800" ], "source_urls": [ "https://arxiv.org/abs/2405.12399", "https://arxiv.org/html/2507.08800v2#A1" ], "source_labels": [ "DIAMOND paper", "NeuralOS Appendix A" ], "topic_id": "interactive-worlds", "topic_label": "Interactive learned environments", "topic_url": "https://neural-os.com/reading-list/guides/games-to-computers/" }, { "title": "Direct frame generation and renderable-code prediction", "text": "NeuralOS predicts desktop frames using learned state and a diffusion renderer. Code2World predicts a UI through code that is then rendered into a screen, and tests its usefulness for mobile navigation. Code2World cites NeuralOS in Section 2.2.", "source_ids": [ "2507.08800", "2602.09856" ], "source_urls": [ "https://arxiv.org/html/2507.08800v1", "https://arxiv.org/html/2602.09856v1#S2.SS2" ], "source_labels": [ "Original NeuralOS paper", "Code2World Section 2.2" ], "topic_id": "gui-planning", "topic_label": "GUI prediction and planning", "topic_url": "https://neural-os.com/reading-list/guides/games-to-computers/#gui-prediction-and-planning" } ] } ] }