{"metadata":{"name":"Neural Operating Systems and GUI World Models","version":"1.3.3","updated_at":"2026-11-14","search_refreshed_at":"2026-10-19","core_record_count":13,"canonical_url":"https://neural-os.com/reading-list/","navigation_url":"https://neural-os.com/reading-list/data/navigation.json","citation_policy":"Preprint exports use initial arXiv submission dates. Publication citations are available separately."},"papers":[{"id":"1507.08750","title":"Action-Conditional Video Prediction using Deep Networks in Atari Games","authors":"Junhyuk Oh, Xiaoxiao Guo, Honglak Lee, Richard Lewis, Satinder Singh","first_public":"2015-07-31","first_public_source":"https://arxiv.org/abs/1507.08750","venue":"NeurIPS 2015","arxiv":"1507.08750","paper_url":"https://arxiv.org/abs/1507.08750","lens":["interactive-worlds"],"related_terms":["action-conditioned video prediction","visual dynamics","Atari","learned simulator"],"summary":"Learns to predict Atari frames from past observations and control inputs, connecting visual generation to an agent's actions.","contribution":"Develops feedforward and recurrent architectures with action-dependent transformations and evaluates long visual rollouts and their usefulness for control.","model_output":"Predicted game frames.","action_interface":"Discrete Atari control actions.","state_memory":"Past observations; the recurrent variant carries a hidden state through time.","relationship":"Earlier action-conditioned visual-prediction work cited in NeuralOS's introduction and references.","relationship_source":"https://arxiv.org/html/2507.08800v1","author_names":["Junhyuk Oh","Xiaoxiao Guo","Honglak Lee","Richard Lewis","Satinder Singh"],"record_url":"https://neural-os.com/reading-list/papers/1507.08750/","legacy_record_url":"https://neural-os.com/reading-list/papers/#paper-1507-08750","citation_date":"2015-07-31","json_url":"https://neural-os.com/reading-list/papers/1507.08750/record.json","markdown_url":"https://neural-os.com/reading-list/papers/1507.08750/index.md","bibtex_url":"https://neural-os.com/reading-list/papers/1507.08750/citation.bib","csl_url":"https://neural-os.com/reading-list/papers/1507.08750/citation.csl.json","translations":{"zh-Hans":{"summary":"根据历史画面和控制输入预测 Atari 游戏的后续帧，将视觉生成与智能体的动作联系起来。","contribution":"提出带有动作相关变换的前馈架构和循环架构，评估连续多步画面预测及其对控制任务的作用。","model_output":"预测的游戏画面。","action_interface":"离散的 Atari 游戏控制动作。","state_memory":"使用历史观测；循环架构通过隐状态在时间上保留信息。","relationship":"NeuralOS 在引言和参考文献中引用的早期动作条件视觉预测工作。","related_terms":["动作条件视频预测","视觉动态","Atari","学习式模拟器"],"record_url":"https://neural-os.com/reading-list/zh/papers/1507.08750/"}}},{"id":"1803.10122","title":"World Models","authors":"David Ha, Jürgen Schmidhuber","first_public":"2018-03-27","first_public_source":"https://arxiv.org/abs/1803.10122","venue":"arXiv","arxiv":"1803.10122","paper_url":"https://arxiv.org/abs/1803.10122","lens":["interactive-worlds"],"related_terms":["world model","recurrent dynamics","latent imagination","model-based reinforcement learning"],"summary":"Combines learned visual compression and recurrent dynamics with a controller, including training an agent inside a generated environment.","contribution":"Demonstrates how a compact learned environment representation can support policy learning and transfer from a simulated dream to the original task.","model_output":"Predicted latent observations, with an image decoder and task-dependent termination predictions.","action_interface":"Actions from the controller in the studied reinforcement-learning environments.","state_memory":"A recurrent model summarizes the history of latent observations and actions.","relationship":"Cited by NeuralOS as background for learned simulation and agent training.","relationship_source":"https://arxiv.org/html/2507.08800v2","author_names":["David Ha","Jürgen Schmidhuber"],"record_url":"https://neural-os.com/reading-list/papers/1803.10122/","legacy_record_url":"https://neural-os.com/reading-list/papers/#paper-1803-10122","citation_date":"2018-03-27","json_url":"https://neural-os.com/reading-list/papers/1803.10122/record.json","markdown_url":"https://neural-os.com/reading-list/papers/1803.10122/index.md","bibtex_url":"https://neural-os.com/reading-list/papers/1803.10122/citation.bib","csl_url":"https://neural-os.com/reading-list/papers/1803.10122/citation.csl.json","translations":{"zh-Hans":{"summary":"将学习得到的视觉压缩表示、循环动态模型与控制器结合，并在生成的环境中训练智能体。","contribution":"展示紧凑的学习式环境表示如何支持策略学习，以及如何将模拟梦境中学到的策略迁移回原始任务。","model_output":"预测的潜在观测表示，配合图像解码器和针对任务的终止预测。","action_interface":"控制器在所研究的强化学习环境中执行的动作。","state_memory":"循环模型汇总潜在观测和动作的历史。","relationship":"NeuralOS 将其作为学习式仿真和智能体训练的背景工作引用。","related_terms":["世界模型","循环动态模型","潜在空间中的想象","基于模型的强化学习"],"record_url":"https://neural-os.com/reading-list/zh/papers/1803.10122/"}}},{"id":"2005.12126","title":"Learning to Simulate Dynamic Environments with GameGAN","authors":"Seung Wook Kim, Yuhao Zhou, Jonah Philion, Antonio Torralba, Sanja Fidler","first_public":"2020-05-25","first_public_source":"https://arxiv.org/abs/2005.12126","venue":"CVPR 2020","arxiv":"2005.12126","paper_url":"https://arxiv.org/abs/2005.12126","lens":["interactive-worlds"],"related_terms":["GameGAN","neural game engine","interactive simulation","visual memory"],"summary":"Learns an interactive visual game simulator from gameplay and keyboard actions using a generative adversarial network.","contribution":"Introduces a memory module for revisited locations and separates static and dynamic image components, enabling visual consistency and component recombination.","model_output":"The next game-screen image.","action_interface":"Game controls represented by keyboard actions.","state_memory":"Recurrent dynamics plus an external memory module that stores an internal environmental map.","relationship":"An earlier interactive neural simulator discussed in NeuralOS's related work.","relationship_source":"https://arxiv.org/html/2507.08800v2","author_names":["Seung Wook Kim","Yuhao Zhou","Jonah Philion","Antonio Torralba","Sanja Fidler"],"record_url":"https://neural-os.com/reading-list/papers/2005.12126/","legacy_record_url":"https://neural-os.com/reading-list/papers/#paper-2005-12126","citation_date":"2020-05-25","json_url":"https://neural-os.com/reading-list/papers/2005.12126/record.json","markdown_url":"https://neural-os.com/reading-list/papers/2005.12126/index.md","bibtex_url":"https://neural-os.com/reading-list/papers/2005.12126/citation.bib","csl_url":"https://neural-os.com/reading-list/papers/2005.12126/citation.csl.json","translations":{"zh-Hans":{"summary":"利用生成对抗网络，从游戏画面和键盘动作中学习可交互的视觉游戏模拟器。","contribution":"引入记忆模块，在重访场景位置时保留信息，并分离图像中的静态与动态成分，以维持视觉一致性和重组场景成分。","model_output":"下一帧游戏画面。","action_interface":"以键盘动作表示的游戏控制输入。","state_memory":"循环动态模型与外部记忆模块结合，后者存储内部环境地图。","relationship":"NeuralOS 在相关工作中讨论的早期交互式神经模拟器。","related_terms":["GameGAN","神经游戏引擎","交互式仿真","视觉记忆"],"record_url":"https://neural-os.com/reading-list/zh/papers/2005.12126/"}}},{"id":"2310.04875","title":"Prompt-to-OS (P2OS): Revolutionizing Operating Systems and Human-Computer Interaction with Integrated AI Generative Models","authors":"Gabriele Tolomei, Cesare Campagnano, Fabrizio Silvestri, Giovanni Trappolini","first_public":"2023-10-07","first_public_source":"https://arxiv.org/abs/2310.04875","venue":"IEEE CogMI 2023","arxiv":"2310.04875","paper_url":"https://arxiv.org/abs/2310.04875","lens":["neural-os"],"related_terms":["Prompt-to-OS","generative interface","neural operating system","human-computer interaction"],"summary":"Proposes an operating-system interaction paradigm in which connected generative models interpret user intentions and mediate or replace conventional application interfaces.","contribution":"Sets out a broader vision for natural-language, personalized computer interaction, including language and diffusion models as interface components.","model_output":"Envisioned context-dependent responses and generated interfaces.","action_interface":"Natural-language user requests, including text and speech.","state_memory":"Proposes personalization and contextual interaction.","relationship":"NeuralOS cites Prompt-to-OS as background for generative computer interaction.","relationship_source":"https://arxiv.org/html/2507.08800v2","author_names":["Gabriele Tolomei","Cesare Campagnano","Fabrizio Silvestri","Giovanni Trappolini"],"record_url":"https://neural-os.com/reading-list/papers/2310.04875/","legacy_record_url":"https://neural-os.com/reading-list/papers/#paper-2310-04875","citation_date":"2023-10-07","json_url":"https://neural-os.com/reading-list/papers/2310.04875/record.json","markdown_url":"https://neural-os.com/reading-list/papers/2310.04875/index.md","bibtex_url":"https://neural-os.com/reading-list/papers/2310.04875/citation.bib","csl_url":"https://neural-os.com/reading-list/papers/2310.04875/citation.csl.json","translations":{"zh-Hans":{"summary":"提出一种操作系统交互范式：由相互连接的生成模型理解用户意图，并承接或替代传统应用界面的交互。","contribution":"提出自然语言驱动、个性化计算机交互的整体构想，将语言模型和扩散模型纳入界面组件。","model_output":"构想中的上下文相关响应和生成式界面。","action_interface":"用户通过文字、语音等形式提出的自然语言请求。","state_memory":"提出个性化和上下文相关交互的设计。","relationship":"NeuralOS 将 Prompt-to-OS 作为生成式计算机交互的背景工作引用。","related_terms":["Prompt-to-OS","生成式界面","神经操作系统","人机交互"],"record_url":"https://neural-os.com/reading-list/zh/papers/2310.04875/"}}},{"id":"2402.15391","title":"Genie: Generative Interactive Environments","authors":"Jake Bruce, Michael Dennis, Ashley Edwards, Jack Parker-Holder, Yuge Shi, Edward Hughes, Matthew Lai, Aditi Mavalankar, Richie Steigerwald, Chris Apps, Yusuf Aytar, Sarah Bechtle, Feryal Behbahani, Stephanie Chan, Nicolas Heess, Lucy Gonzalez, Simon Osindero, Sherjil Ozair, Scott Reed, Jingwei Zhang, Konrad Zolna, Jeff Clune, Nando de Freitas, Satinder Singh, Tim Rocktäschel","first_public":"2024-02-23","first_public_source":"https://arxiv.org/abs/2402.15391","venue":"arXiv","arxiv":"2402.15391","paper_url":"https://arxiv.org/abs/2402.15391","lens":["interactive-worlds"],"related_terms":["Genie","generative interactive environments","latent actions","foundation world model"],"summary":"Learns controllable visual environments from videos without ground-truth action labels, using a learned latent action space.","contribution":"Combines video tokenization, a dynamics model, and latent-action learning so users can interact with worlds initialized from images.","model_output":"Action-conditioned future video frames.","action_interface":"Discrete latent actions inferred during training rather than recorded physical input events.","state_memory":"A spatiotemporal dynamics model conditions predictions on previous visual tokens and actions.","relationship":"Cited in NeuralOS's related work. Genie learns its action space from video; NeuralOS takes mouse and keyboard events as inputs.","relationship_source":"https://arxiv.org/html/2507.08800v2","author_names":["Jake Bruce","Michael Dennis","Ashley Edwards","Jack Parker-Holder","Yuge Shi","Edward Hughes","Matthew Lai","Aditi Mavalankar","Richie Steigerwald","Chris Apps","Yusuf Aytar","Sarah Bechtle","Feryal Behbahani","Stephanie Chan","Nicolas Heess","Lucy Gonzalez","Simon Osindero","Sherjil Ozair","Scott Reed","Jingwei Zhang","Konrad Zolna","Jeff Clune","Nando de Freitas","Satinder Singh","Tim Rocktäschel"],"record_url":"https://neural-os.com/reading-list/papers/2402.15391/","legacy_record_url":"https://neural-os.com/reading-list/papers/#paper-2402-15391","citation_date":"2024-02-23","json_url":"https://neural-os.com/reading-list/papers/2402.15391/record.json","markdown_url":"https://neural-os.com/reading-list/papers/2402.15391/index.md","bibtex_url":"https://neural-os.com/reading-list/papers/2402.15391/citation.bib","csl_url":"https://neural-os.com/reading-list/papers/2402.15391/citation.csl.json","translations":{"zh-Hans":{"summary":"从没有真实动作标签的视频中学习可控制的视觉环境，使用模型学到的潜在动作空间。","contribution":"结合视频词元化、动态模型和潜在动作学习，让用户与由图像初始化的世界交互。","model_output":"以动作为条件的未来视频帧。","action_interface":"训练时从视频中推断的离散潜在动作，而非录制的实际输入事件。","state_memory":"时空动态模型根据先前的视觉词元和动作预测后续画面。","relationship":"NeuralOS 在相关工作中引用 Genie。Genie 从视频中学习动作空间；NeuralOS 接收明确的鼠标和键盘事件。","related_terms":["Genie","生成式交互环境","潜在动作","基础世界模型"],"record_url":"https://neural-os.com/reading-list/zh/papers/2402.15391/"}}},{"id":"2404.07972","title":"OSWorld: Benchmarking Multimodal Agents for Open-Ended Tasks in Real Computer Environments","authors":"Tianbao Xie, Danyang Zhang, Jixuan Chen, Xiaochuan Li, Siheng Zhao, Ruisheng Cao, Toh Jing Hua, Zhoujun Cheng, Dongchan Shin, Fangyu Lei, Yitao Liu, Yiheng Xu, Shuyan Zhou, Silvio Savarese, Caiming Xiong, Victor Zhong, Tao Yu","first_public":"2024-04-11","first_public_source":"https://arxiv.org/abs/2404.07972","venue":"NeurIPS 2024","arxiv":"2404.07972","paper_url":"https://arxiv.org/abs/2404.07972","lens":["evaluation"],"related_terms":["OSWorld","computer-use agents","execution-based evaluation","desktop benchmark"],"summary":"Provides real computer environments and reproducible tasks spanning desktop applications, websites, and file operations.","contribution":"Pairs task setup with execution-based evaluators, allowing agents to complete workflows through actual software rather than matching a fixed action sequence.","model_output":"Observations and evaluation results from real computer environments.","action_interface":"Computer interaction through the benchmark's supported action interfaces.","state_memory":"State persists in the real operating system and applications used by the environment.","relationship":"Provides desktop tasks and outcome checks for evaluating computer-use agents.","relationship_source":"https://arxiv.org/abs/2404.07972","author_names":["Tianbao Xie","Danyang Zhang","Jixuan Chen","Xiaochuan Li","Siheng Zhao","Ruisheng Cao","Toh Jing Hua","Zhoujun Cheng","Dongchan Shin","Fangyu Lei","Yitao Liu","Yiheng Xu","Shuyan Zhou","Silvio Savarese","Caiming Xiong","Victor Zhong","Tao Yu"],"record_url":"https://neural-os.com/reading-list/papers/2404.07972/","legacy_record_url":"https://neural-os.com/reading-list/papers/#paper-2404-07972","citation_date":"2024-04-11","json_url":"https://neural-os.com/reading-list/papers/2404.07972/record.json","markdown_url":"https://neural-os.com/reading-list/papers/2404.07972/index.md","bibtex_url":"https://neural-os.com/reading-list/papers/2404.07972/citation.bib","csl_url":"https://neural-os.com/reading-list/papers/2404.07972/citation.csl.json","publication":{"title":"OSWorld: Benchmarking Multimodal Agents for Open-Ended Tasks in Real Computer Environments","author_names":["Tianbao Xie","Danyang Zhang","Jixuan Chen","Xiaochuan Li","Siheng Zhao","Ruisheng Cao","Toh Jing Hua","Zhoujun Cheng","Dongchan Shin","Fangyu Lei","Yitao Liu","Yiheng Xu","Shuyan Zhou","Silvio Savarese","Caiming Xiong","Victor Zhong","Tao Yu"],"year":2024,"type":"paper-conference","container_title":"Advances in Neural Information Processing Systems","volume":"37","doi":"10.52202/079017-1650","url":"https://proceedings.neurips.cc/paper_files/paper/2024/hash/5d413e48f84dc61244b6be550f1cd8f5-Abstract-Datasets_and_Benchmarks_Track.html","source_url":"https://proceedings.neurips.cc/paper_files/paper/2024/hash/5d413e48f84dc61244b6be550f1cd8f5-Abstract-Datasets_and_Benchmarks_Track.html","verified_at":"2026-10-19"},"publication_bibtex_url":"https://neural-os.com/reading-list/papers/2404.07972/publication.bib","publication_csl_url":"https://neural-os.com/reading-list/papers/2404.07972/publication.csl.json","translations":{"zh-Hans":{"summary":"提供真实计算机环境和可复现任务，覆盖桌面应用、网站和文件操作。","contribution":"将任务初始化与基于执行结果的评估器配套，让智能体通过真实软件完成工作流程，评估实际结果，而非匹配固定的动作序列。","model_output":"来自真实计算机环境的观测和评估结果。","action_interface":"通过基准支持的动作接口与计算机交互。","state_memory":"环境使用的真实操作系统和应用程序负责保存状态。","relationship":"提供用于评估计算机使用智能体的桌面任务和结果检查。","related_terms":["OSWorld","计算机使用智能体","基于执行结果的评估","桌面任务基准"],"record_url":"https://neural-os.com/reading-list/zh/papers/2404.07972/"}}},{"id":"2405.12399","title":"Diffusion for World Modeling: Visual Details Matter in Atari","authors":"Eloi Alonso, Adam Jelley, Vincent Micheli, Anssi Kanervisto, Amos Storkey, Tim Pearce, François Fleuret","first_public":"2024-05-20","first_public_source":"https://arxiv.org/abs/2405.12399","venue":"NeurIPS 2024","arxiv":"2405.12399","paper_url":"https://arxiv.org/abs/2405.12399","lens":["interactive-worlds","evaluation"],"related_terms":["DIAMOND","diffusion world model","neural game engine","model-based reinforcement learning"],"summary":"Introduces DIAMOND, an agent trained in a diffusion world model, and studies how preserving visual detail affects control.","contribution":"Connects diffusion-based observation prediction with Atari policy learning and demonstrates an interactive Counter-Strike simulator.","model_output":"Predicted visual observations; auxiliary reward and termination predictions support reinforcement learning.","action_interface":"Actions of the modeled game environment.","state_memory":"Visual dynamics condition on a finite history of observations and actions.","relationship":"NeuralOS adapts DIAMOND to OS screen prediction as a baseline in Appendix A of its 2026 revision.","relationship_source":"https://arxiv.org/html/2507.08800v2#A1","author_names":["Eloi Alonso","Adam Jelley","Vincent Micheli","Anssi Kanervisto","Amos Storkey","Tim Pearce","François Fleuret"],"record_url":"https://neural-os.com/reading-list/papers/2405.12399/","legacy_record_url":"https://neural-os.com/reading-list/papers/#paper-2405-12399","citation_date":"2024-05-20","json_url":"https://neural-os.com/reading-list/papers/2405.12399/record.json","markdown_url":"https://neural-os.com/reading-list/papers/2405.12399/index.md","bibtex_url":"https://neural-os.com/reading-list/papers/2405.12399/citation.bib","csl_url":"https://neural-os.com/reading-list/papers/2405.12399/citation.csl.json","method_comparisons":[{"title":"A game world model evaluated on desktop interactions","text":"DIAMOND learns visual game dynamics with a diffusion model. NeuralOS adapts DIAMOND to OS screen prediction as a baseline in Appendix A of its 2026 revision. NeuralOS adds a recurrent state representation and spatial cursor conditioning for desktop interaction.","source_ids":["2405.12399","2507.08800"],"source_urls":["https://arxiv.org/abs/2405.12399","https://arxiv.org/html/2507.08800v2#A1"],"source_labels":["DIAMOND paper","NeuralOS Appendix A"],"topic_id":"interactive-worlds","topic_label":"Interactive learned environments","topic_url":"https://neural-os.com/reading-list/guides/games-to-computers/"}],"translations":{"zh-Hans":{"summary":"提出在扩散世界模型中训练的智能体 DIAMOND，研究保留视觉细节对控制任务的影响。","contribution":"将基于扩散模型的观测预测与 Atari 策略学习结合，并展示可交互的 Counter-Strike 模拟器。","model_output":"预测的视觉观测；奖励和终止状态的辅助预测用于支持强化学习。","action_interface":"所建模游戏环境中的动作。","state_memory":"视觉动态以有限长度的观测和动作历史为条件。","relationship":"NeuralOS 在 2026 年修订版的附录 A 中，将 DIAMOND 适配到操作系统画面预测，并作为基线进行比较。","related_terms":["DIAMOND","扩散世界模型","神经游戏引擎","基于模型的强化学习"],"record_url":"https://neural-os.com/reading-list/zh/papers/2405.12399/"}}},{"id":"2405.14573","title":"AndroidWorld: A Dynamic Benchmarking Environment for Autonomous Agents","authors":"Christopher Rawles, Sarah Clinckemaillie, Yifan Chang, Jonathan Waltz, Gabrielle Lau, Marybeth Fair, Alice Li, William Bishop, Wei Li, Folawiyo Campbell-Ajala, Daniel Toyama, Robert Berry, Divya Tyamagundlu, Timothy Lillicrap, Oriana Riva","first_public":"2024-05-23","first_public_source":"https://arxiv.org/abs/2405.14573","venue":"ICLR 2025","arxiv":"2405.14573","paper_url":"https://arxiv.org/abs/2405.14573","lens":["evaluation"],"related_terms":["AndroidWorld","mobile agents","GUI evaluation","task success"],"summary":"Provides a functional Android environment with parameterized tasks and programmatic checks of device state.","contribution":"Combines task initialization, success checking, and teardown with variable task instances, making mobile-agent evaluation reproducible and less dependent on a fixed test set.","model_output":"Android observations and task rewards.","action_interface":"Agent actions executed on an Android device environment.","state_memory":"The Android system and apps maintain state; evaluators inspect that state directly.","relationship":"Provides Android tasks and outcome checks for evaluating mobile agents.","relationship_source":"https://arxiv.org/abs/2405.14573","author_names":["Christopher Rawles","Sarah Clinckemaillie","Yifan Chang","Jonathan Waltz","Gabrielle Lau","Marybeth Fair","Alice Li","William Bishop","Wei Li","Folawiyo Campbell-Ajala","Daniel Toyama","Robert Berry","Divya Tyamagundlu","Timothy Lillicrap","Oriana Riva"],"record_url":"https://neural-os.com/reading-list/papers/2405.14573/","legacy_record_url":"https://neural-os.com/reading-list/papers/#paper-2405-14573","citation_date":"2024-05-23","json_url":"https://neural-os.com/reading-list/papers/2405.14573/record.json","markdown_url":"https://neural-os.com/reading-list/papers/2405.14573/index.md","bibtex_url":"https://neural-os.com/reading-list/papers/2405.14573/citation.bib","csl_url":"https://neural-os.com/reading-list/papers/2405.14573/citation.csl.json","publication":{"title":"AndroidWorld: A Dynamic Benchmarking Environment for Autonomous Agents","author_names":["Chris Rawles","Sarah Clinckemaillie","Yifan Chang","Jonathan Waltz","Gabrielle Lau","Marybeth Fair","Alice Li","William Bishop","Wei Li","Folawiyo Campbell-Ajala","Daniel Toyama","Robert Berry","Divya Tyamagundlu","Timothy Lillicrap","Oriana Riva"],"year":2025,"type":"paper-conference","container_title":"International Conference on Learning Representations","url":"https://proceedings.iclr.cc/paper_files/paper/2025/hash/01a83bc2f2732a58e6aa731e659e7101-Abstract-Conference.html","source_url":"https://proceedings.iclr.cc/paper_files/paper/2025/hash/01a83bc2f2732a58e6aa731e659e7101-Abstract-Conference.html","verified_at":"2026-10-19"},"publication_bibtex_url":"https://neural-os.com/reading-list/papers/2405.14573/publication.bib","publication_csl_url":"https://neural-os.com/reading-list/papers/2405.14573/publication.csl.json","translations":{"zh-Hans":{"summary":"提供功能完整的 Android 环境，包含参数化任务和对设备状态的程序化检查。","contribution":"将任务初始化、成功检查和环境清理与可变任务实例结合，使移动端智能体评估可复现，并减少对固定测试集的依赖。","model_output":"Android 环境中的观测和任务奖励。","action_interface":"在 Android 设备环境中执行的智能体动作。","state_memory":"Android 系统和应用程序维护状态，评估器直接检查这些状态。","relationship":"提供用于评估移动端智能体的 Android 任务和结果检查。","related_terms":["AndroidWorld","移动端智能体","GUI 评估","任务成功率"],"record_url":"https://neural-os.com/reading-list/zh/papers/2405.14573/"}}},{"id":"2408.14837","title":"Diffusion Models Are Real-Time Game Engines","authors":"Dani Valevski, Yaniv Leviathan, Moab Arar, Shlomi Fruchter","first_public":"2024-08-27","first_public_source":"https://arxiv.org/abs/2408.14837","venue":"ICLR 2025","arxiv":"2408.14837","paper_url":"https://arxiv.org/abs/2408.14837","lens":["interactive-worlds"],"related_terms":["GameNGen","neural game engine","real-time simulation","diffusion renderer"],"summary":"Presents GameNGen, a diffusion model that interactively simulates DOOM from previous frames and player actions.","contribution":"Uses agent-collected gameplay, conditioning augmentation, and decoder fine-tuning to sustain autoregressive visual interaction.","model_output":"The next rendered game frame.","action_interface":"DOOM player actions.","state_memory":"Conditioning on a sequence of recent frames and actions; continued play feeds generated frames back into the model.","relationship":"Interactive-simulation background cited by NeuralOS; NeuralOS also refers to its perceptual-evaluation setup.","relationship_source":"https://arxiv.org/html/2507.08800v2","author_names":["Dani Valevski","Yaniv Leviathan","Moab Arar","Shlomi Fruchter"],"record_url":"https://neural-os.com/reading-list/papers/2408.14837/","legacy_record_url":"https://neural-os.com/reading-list/papers/#paper-2408-14837","citation_date":"2024-08-27","json_url":"https://neural-os.com/reading-list/papers/2408.14837/record.json","markdown_url":"https://neural-os.com/reading-list/papers/2408.14837/index.md","bibtex_url":"https://neural-os.com/reading-list/papers/2408.14837/citation.bib","csl_url":"https://neural-os.com/reading-list/papers/2408.14837/citation.csl.json","translations":{"zh-Hans":{"summary":"提出 GameNGen，利用扩散模型根据历史画面和玩家动作交互式地模拟 DOOM。","contribution":"利用智能体采集游戏数据，并通过条件增强和解码器微调，维持自回归的视觉交互。","model_output":"渲染出的下一帧游戏画面。","action_interface":"DOOM 玩家动作。","state_memory":"以近期画面和动作序列为条件；持续交互时，生成的画面被送回模型，成为后续预测的输入。","relationship":"NeuralOS 将其作为交互式仿真的背景工作引用，也参考了其感知评估设置。","related_terms":["GameNGen","神经游戏引擎","实时仿真","扩散渲染器"],"record_url":"https://neural-os.com/reading-list/zh/papers/2408.14837/"}}},{"id":"2410.13232","title":"Web Agents with World Models: Learning and Leveraging Environment Dynamics in Web Navigation","authors":"Hyungjoo Chae, Namyoung Kim, Kai Tzu-iunn Ong, Minju Gwak, Gwanwoo Song, Jihoon Kim, Sunghwan Kim, Dongha Lee, Jinyoung Yeo","first_public":"2024-10-17","first_public_source":"https://arxiv.org/abs/2410.13232","venue":"ICLR 2025","arxiv":"2410.13232","paper_url":"https://arxiv.org/abs/2410.13232","lens":["gui-planning"],"related_terms":["web world model","counterfactual planning","transition prediction","WMA"],"summary":"Predicts the effects of candidate web actions as natural-language descriptions of important changes, then uses those predictions for action selection.","contribution":"Introduces a transition-focused abstraction that reduces the burden of predicting long, repetitive web observations and supports world-model-augmented planning.","model_output":"Natural-language descriptions of changes in the next web observation.","action_interface":"Candidate web-navigation actions.","state_memory":"The planner uses web context and predicted descriptions of state changes.","relationship":"Connects environment prediction to agent decisions in web navigation.","relationship_source":"https://arxiv.org/abs/2410.13232","author_names":["Hyungjoo Chae","Namyoung Kim","Kai Tzu-iunn Ong","Minju Gwak","Gwanwoo Song","Jihoon Kim","Sunghwan Kim","Dongha Lee","Jinyoung Yeo"],"record_url":"https://neural-os.com/reading-list/papers/2410.13232/","legacy_record_url":"https://neural-os.com/reading-list/papers/#paper-2410-13232","citation_date":"2024-10-17","json_url":"https://neural-os.com/reading-list/papers/2410.13232/record.json","markdown_url":"https://neural-os.com/reading-list/papers/2410.13232/index.md","bibtex_url":"https://neural-os.com/reading-list/papers/2410.13232/citation.bib","csl_url":"https://neural-os.com/reading-list/papers/2410.13232/citation.csl.json","translations":{"zh-Hans":{"summary":"用自然语言描述候选网页动作引起的重要变化，再根据这些预测选择动作。","contribution":"提出以状态转移为重点的抽象表示，减轻预测冗长、重复网页观测的负担，并支持借助世界模型进行规划。","model_output":"对下一次网页观测中变化的自然语言描述。","action_interface":"候选网页导航动作。","state_memory":"规划器使用网页上下文和预测的状态变化描述。","relationship":"将环境预测与网页导航中智能体的决策联系起来。","related_terms":["网页世界模型","反事实规划","状态转移预测","WMA"],"record_url":"https://neural-os.com/reading-list/zh/papers/2410.13232/"}}},{"id":"2504.13936","title":"ViMo: A Generative Visual GUI World Model for App Agents","authors":"Dezhao Luo, Bohan Tang, Kang Li, Georgios Papoudakis, Jifei Song, Shaogang Gong, Jianye Hao, Jun Wang, Kun Shao","first_public":"2025-04-15","first_public_source":"https://arxiv.org/abs/2504.13936","venue":"arXiv","arxiv":"2504.13936","paper_url":"https://arxiv.org/abs/2504.13936","lens":["gui-planning"],"related_terms":["ViMo","GUI world model","next-screen prediction","mobile app simulation"],"summary":"Predicts future mobile-app screens to help agents compare possible actions, separating graphic content from textual content.","contribution":"Uses symbolic text placeholders to preserve layout while assigning graphical and text prediction to separate components.","model_output":"Future mobile GUI images assembled from graphics and text predictions.","action_interface":"Candidate mobile-app actions.","state_memory":"Screen-and-action-conditioned prediction; symbolic text representation retains text locations and layout.","relationship":"Visual GUI world-model research released in April 2025, before NeuralOS's July 2025 preprint.","relationship_source":"https://arxiv.org/abs/2504.13936","author_names":["Dezhao Luo","Bohan Tang","Kang Li","Georgios Papoudakis","Jifei Song","Shaogang Gong","Jianye Hao","Jun Wang","Kun Shao"],"record_url":"https://neural-os.com/reading-list/papers/2504.13936/","legacy_record_url":"https://neural-os.com/reading-list/papers/#paper-2504-13936","citation_date":"2025-04-15","json_url":"https://neural-os.com/reading-list/papers/2504.13936/record.json","markdown_url":"https://neural-os.com/reading-list/papers/2504.13936/index.md","bibtex_url":"https://neural-os.com/reading-list/papers/2504.13936/citation.bib","csl_url":"https://neural-os.com/reading-list/papers/2504.13936/citation.csl.json","translations":{"zh-Hans":{"summary":"预测移动应用的后续屏幕，帮助智能体比较候选动作，并将图形内容与文字内容分开处理。","contribution":"用符号化文字占位符保留布局，将图形预测和文字预测交给不同组件完成。","model_output":"由图形预测和文字预测组合而成的未来移动 GUI 图像。","action_interface":"候选移动应用操作。","state_memory":"以屏幕和动作为条件进行预测；符号化文字表示保留文字位置与布局。","relationship":"这项视觉 GUI 世界模型研究于 2025 年 4 月公开，早于 NeuralOS 在 2025 年 7 月发布的预印本。","related_terms":["ViMo","GUI 世界模型","下一屏幕预测","移动应用仿真"],"record_url":"https://neural-os.com/reading-list/zh/papers/2504.13936/"}}},{"id":"2507.08800","title":"NeuralOS: Towards Simulating Operating Systems via Neural Generative Models","authors":"Luke Rivard, Sun Sun, Hongyu Guo, Wenhu Chen, Yuntian Deng","first_public":"2025-07-11","first_public_source":"https://arxiv.org/abs/2507.08800","first_public_note":"Initial arXiv submission on July 11, 2025; revised on March 12, 2026 and published at ICLR 2026.","venue":"ICLR 2026","arxiv":"2507.08800","paper_url":"https://arxiv.org/abs/2507.08800","original_url":"https://arxiv.org/html/2507.08800v1","revised_url":"https://arxiv.org/html/2507.08800v2","code_url":"https://github.com/yuntian-group/neural-os","demo_url":"https://neural-os.com/","models_url":"https://huggingface.co/yuntian-deng/NeuralOS-DiT-Mamba","release_summary":"Download the DiT/Mamba pretrained models and bundled runtime to generate 1024 × 768 desktop frames locally. The release includes the latest v6 checkpoint and the v4 checkpoint used by the online demo.","lens":["neural-os","interactive-worlds","evaluation"],"related_terms":["NeuralOS","neural operating system","learned computer simulator","generative interface","generative UI","on-the-fly interface generation","adaptive user interface","personalized human-computer interaction","applications learned from demonstrations","GUI world model","desktop world model","generative GUI simulation","action-conditioned video prediction","action-conditioned screen generation"],"summary":"Explores neural operating systems and generative interfaces that adapt to users’ inputs, context, and intentions. NeuralOS learns an interactive computer environment, generating desktop screens from mouse and keyboard events and recurrent state.","contribution":"The July 2025 paper articulates a vision of interfaces generated on the fly, personalized interaction, and experiences across application boundaries. It formalizes OS-interface generation and develops state tracking, precise cursor conditioning, staged training, and data collection. The March 2026 revision demonstrates a simulated Doom application learned from synthetic launch, gameplay, and close sequences, without Doom installed in the recorded desktop environment.","model_output":"Autoregressively generated desktop screen frames.","action_interface":"Cursor coordinates, left/right clicks, and keyboard press/release events.","state_memory":"A hierarchical recurrent model carries state across frames and conditions a diffusion renderer.","relationship":"Introduces NeuralOS and its formulation of learned OS-interface simulation.","relationship_source":"https://arxiv.org/html/2507.08800v1","author_names":["Luke Rivard","Sun Sun","Hongyu Guo","Wenhu Chen","Yuntian Deng"],"record_url":"https://neural-os.com/reading-list/papers/2507.08800/","legacy_record_url":"https://neural-os.com/reading-list/papers/#paper-2507-08800","citation_date":"2025-07-11","json_url":"https://neural-os.com/reading-list/papers/2507.08800/record.json","markdown_url":"https://neural-os.com/reading-list/papers/2507.08800/index.md","bibtex_url":"https://neural-os.com/reading-list/papers/2507.08800/citation.bib","csl_url":"https://neural-os.com/reading-list/papers/2507.08800/citation.csl.json","publication":{"title":"NeuralOS: Towards Simulating Operating Systems via Neural Generative Models","author_names":["Luke Rivard","Sun Sun","Hongyu Guo","Wenhu Chen","Yuntian Deng"],"year":2026,"type":"paper-conference","container_title":"International Conference on Learning Representations","url":"https://proceedings.iclr.cc/paper_files/paper/2026/hash/2174a6749c3f769358cd21ade1221b34-Abstract-Conference.html","source_url":"https://proceedings.iclr.cc/paper_files/paper/2026/hash/2174a6749c3f769358cd21ade1221b34-Abstract-Conference.html","verified_at":"2026-10-19"},"publication_bibtex_url":"https://neural-os.com/reading-list/papers/2507.08800/publication.bib","publication_csl_url":"https://neural-os.com/reading-list/papers/2507.08800/publication.csl.json","method_comparisons":[{"title":"A game world model evaluated on desktop interactions","text":"DIAMOND learns visual game dynamics with a diffusion model. NeuralOS adapts DIAMOND to OS screen prediction as a baseline in Appendix A of its 2026 revision. NeuralOS adds a recurrent state representation and spatial cursor conditioning for desktop interaction.","source_ids":["2405.12399","2507.08800"],"source_urls":["https://arxiv.org/abs/2405.12399","https://arxiv.org/html/2507.08800v2#A1"],"source_labels":["DIAMOND paper","NeuralOS Appendix A"],"topic_id":"interactive-worlds","topic_label":"Interactive learned environments","topic_url":"https://neural-os.com/reading-list/guides/games-to-computers/"},{"title":"Direct frame generation and renderable-code prediction","text":"NeuralOS predicts desktop frames using learned state and a diffusion renderer. Code2World predicts a UI through code that is then rendered into a screen, and tests its usefulness for mobile navigation. Code2World cites NeuralOS in Section 2.2.","source_ids":["2507.08800","2602.09856"],"source_urls":["https://arxiv.org/html/2507.08800v1","https://arxiv.org/html/2602.09856v1#S2.SS2"],"source_labels":["Original NeuralOS paper","Code2World Section 2.2"],"topic_id":"gui-planning","topic_label":"GUI prediction and planning","topic_url":"https://neural-os.com/reading-list/guides/games-to-computers/#gui-prediction-and-planning"}],"translations":{"zh-Hans":{"summary":"NeuralOS 提出在使用过程中即时生成计算机界面的研究愿景：根据用户输入、上下文与意图形成个性化交互，并探索跨应用的生成式体验。论文以可学习的桌面环境实现这一方向，结合循环状态跟踪与扩散渲染，根据鼠标和键盘事件生成屏幕。","contribution":"2025 年 7 月 11 日的原始论文提出自适应生成式界面的愿景，将自然语言、手势和跨应用交互列为未来方向，并形式化操作系统界面生成，提出状态跟踪、精确光标条件表示、分阶段训练与数据采集方法。2026 年 3 月 12 日修订版（ICLR 2026）展示了从人工构造的桌面交互与 VizDoom 片段中，学会启动、游玩和关闭原桌面环境中未安装的 Doom 模拟应用。","model_output":"以自回归方式生成的桌面画面序列。","action_interface":"光标坐标、鼠标左键和右键点击，以及键盘按下和释放事件。","state_memory":"层次化循环模型跨帧保留状态，并为扩散渲染器提供条件信息。","relationship":"连接生成式 UI、人机交互与可学习的计算机环境：研究随意图与上下文生成界面的愿景，并通过鼠标和键盘驱动的桌面模型研究持续交互与状态。","release_summary":"下载 DiT/Mamba 预训练模型与配套运行代码，在本地生成 1024 × 768 桌面画面。发布包包含最新的 v6 检查点和在线演示使用的 v4 检查点。","related_terms":["NeuralOS","神经操作系统","AI操作系统","下一代GUI","学习式计算机模拟器","生成式界面","生成式 UI","即时界面生成","个性化界面","上下文感知交互","意图驱动界面","跨应用交互","合成示范","GUI 世界模型","桌面世界模型","生成式 GUI 仿真","状态记忆","动作条件视频预测","动作条件屏幕生成"],"record_url":"https://neural-os.com/reading-list/zh/papers/2507.08800/"}},"media_coverage":[{"publisher":"量子位","author":"一水","date":"2025-07-15","language":"zh-Hans","url":"https://www.qbitai.com/2025/07/309398.html","title":"卡帕西预言成真！华人团队开源全AI操作系统","label":"量子位 (QbitAI) — NeuralOS's neural desktop (Chinese)","zh_label":"量子位 — NeuralOS 的神经网络桌面","summary":"An illustrated introduction to NeuralOS, its recurrent state model, diffusion renderer, and interactive desktop demonstration.","zh_summary":"通过演示介绍 NeuralOS 的桌面交互、循环状态模型与扩散渲染器。"},{"publisher":"MarkTechPost","author":"Sana Hassan","date":"2025-07-16","language":"en","url":"https://www.marktechpost.com/2025/07/16/neuralos-a-generative-framework-for-simulating-interactive-operating-system-interfaces/","title":"NeuralOS: A Generative Framework for Simulating Interactive Operating System Interfaces","label":"MarkTechPost — Generating interactive operating-system interfaces","zh_label":"MarkTechPost — 生成可交互的操作系统界面（英文）","summary":"A walkthrough of NeuralOS's architecture, training data, and screen generation from user inputs.","zh_summary":"介绍 NeuralOS 的架构、训练数据，以及根据用户输入生成屏幕的过程。"},{"publisher":"科技行者 (TechWalker)","author":"科技行者","date":"2025-07-21","language":"zh-Hans","url":"https://www.techwalker.com/2025/0721/3169237.shtml","title":"用神经网络重新定义操作系统：滑铁卢大学的NeuralOS让计算机界面完全由AI生成","label":"科技行者 (TechWalker) — Computer interfaces generated by NeuralOS (Chinese)","zh_label":"科技行者 — 由 NeuralOS 生成计算机界面","summary":"An introduction to NeuralOS's learned computer interfaces and the research behind its interactive simulation.","zh_summary":"介绍 NeuralOS 学习计算机界面的方式及其可交互仿真背后的研究。"}],"author_discussions":[{"publisher":"Reddit · r/MachineLearning","author":"Yuntian Deng (u/yuntiandeng)","date":"2025-07-19","language":"en","url":"https://www.reddit.com/r/MachineLearning/comments/1m3v7ll/r_neuralos_a_generative_os_entirely_powered_by/","title":"[R] NeuralOS: a generative OS entirely powered by neural networks","label":"Reddit — NeuralOS author announcement and discussion","zh_label":"Reddit — NeuralOS 作者发布与讨论（英文）","summary":"Yuntian Deng introduces NeuralOS and discusses customized interfaces, experiences across applications, and generative interactive media.","zh_summary":"Yuntian Deng 介绍 NeuralOS，并讨论定制界面、跨应用体验与生成式交互媒体。"},{"publisher":"Hacker News","author":"Yuntian Deng (yuntian)","date":"2025-07-14","language":"en","url":"https://news.ycombinator.com/item?id=44564531","title":"NeuralOS: An operating system powered by neural networks","label":"Hacker News — NeuralOS author announcement and discussion","zh_label":"Hacker News — NeuralOS 作者发布与讨论（英文）","summary":"The project's launch discussion, with questions from readers and responses from Yuntian Deng.","zh_summary":"项目发布时的社区讨论，包含读者提问与 Yuntian Deng 的回复。"},{"publisher":"Hugging Face","author":"Yuntian Deng","date":"2025-07-14","language":"en","url":"https://huggingface.co/papers/2507.08800","title":"NeuralOS: Towards Simulating Operating Systems via Neural Generative Models","label":"Hugging Face — NeuralOS paper discussion","zh_label":"Hugging Face — NeuralOS 论文讨论（英文）","summary":"The author-submitted paper page, with the research abstract, authors, and community discussion.","zh_summary":"由作者提交的论文页面，包含摘要、完整作者信息与社区讨论。"}]},{"id":"2602.09856","title":"Code2World: A GUI World Model via Renderable Code Generation","authors":"Yuhao Zheng, Li'an Zhong, Yi Wang, Rui Dai, Kaikui Liu, Xiangxiang Chu, Linyuan Lv, Philip Torr, Kevin Qinghong Lin","first_public":"2026-02-10","first_public_source":"https://arxiv.org/abs/2602.09856","venue":"arXiv","arxiv":"2602.09856","paper_url":"https://arxiv.org/abs/2602.09856","lens":["gui-planning"],"related_terms":["Code2World","GUI world model","renderable code","next-screen prediction","AndroidCode"],"summary":"Predicts a GUI's next visual state by generating renderable code, and evaluates whether the predicted outcomes help mobile-agent navigation.","contribution":"Constructs paired screen/action/code data and trains with rendered visual feedback to balance appearance and action consistency.","model_output":"Renderable UI code, converted into the predicted next screen.","action_interface":"A current GUI observation and a proposed interaction, represented for the model with action descriptions and visual cues.","state_memory":"Prediction conditions on the current screen and action; renderable structure represents the output interface.","relationship":"Code2World cites NeuralOS in Section 2.2 and its bibliography.","relationship_source":"https://arxiv.org/html/2602.09856v1#S2.SS2","author_names":["Yuhao Zheng","Li'an Zhong","Yi Wang","Rui Dai","Kaikui Liu","Xiangxiang Chu","Linyuan Lv","Philip Torr","Kevin Qinghong Lin"],"record_url":"https://neural-os.com/reading-list/papers/2602.09856/","legacy_record_url":"https://neural-os.com/reading-list/papers/#paper-2602-09856","citation_date":"2026-02-10","json_url":"https://neural-os.com/reading-list/papers/2602.09856/record.json","markdown_url":"https://neural-os.com/reading-list/papers/2602.09856/index.md","bibtex_url":"https://neural-os.com/reading-list/papers/2602.09856/citation.bib","csl_url":"https://neural-os.com/reading-list/papers/2602.09856/citation.csl.json","method_comparisons":[{"title":"Direct frame generation and renderable-code prediction","text":"NeuralOS predicts desktop frames using learned state and a diffusion renderer. Code2World predicts a UI through code that is then rendered into a screen, and tests its usefulness for mobile navigation. Code2World cites NeuralOS in Section 2.2.","source_ids":["2507.08800","2602.09856"],"source_urls":["https://arxiv.org/html/2507.08800v1","https://arxiv.org/html/2602.09856v1#S2.SS2"],"source_labels":["Original NeuralOS paper","Code2World Section 2.2"],"topic_id":"gui-planning","topic_label":"GUI prediction and planning","topic_url":"https://neural-os.com/reading-list/guides/games-to-computers/#gui-prediction-and-planning"}],"translations":{"zh-Hans":{"summary":"通过生成可渲染代码来预测 GUI 的下一视觉状态，并评估预测结果是否有助于移动端智能体导航。","contribution":"构建配对的屏幕、动作和代码数据，使用渲染后的视觉反馈训练模型，兼顾界面外观和动作一致性。","model_output":"可渲染的界面代码，经渲染得到预测的下一屏幕。","action_interface":"当前 GUI 观测和一个候选交互，通过动作描述与视觉提示输入模型。","state_memory":"预测以当前屏幕和动作为条件；输出界面由可渲染的结构表示。","relationship":"Code2World 在第 2.2 节和参考文献中引用 NeuralOS。","related_terms":["Code2World","GUI 世界模型","可渲染代码","下一屏幕预测","AndroidCode"],"record_url":"https://neural-os.com/reading-list/zh/papers/2602.09856/"}}}],"precursors":[]}
