{"metadata":{"name":"Neural Operating Systems and GUI World Models","version":"1.0.1","updated_at":"2026-10-19","search_refreshed_at":"2026-10-19","core_record_count":13,"canonical_url":"https://neural-os.com/reading-list/","navigation_url":"https://neural-os.com/reading-list/data/navigation.json","citation_policy":"Preprint exports use initial arXiv submission dates. Publication citations are available separately."},"papers":[{"id":"1507.08750","title":"Action-Conditional Video Prediction using Deep Networks in Atari Games","authors":"Junhyuk Oh, Xiaoxiao Guo, Honglak Lee, Richard Lewis, Satinder Singh","first_public":"2015-07-31","first_public_source":"https://arxiv.org/abs/1507.08750","venue":"NeurIPS 2015","arxiv":"1507.08750","paper_url":"https://arxiv.org/abs/1507.08750","lens":["interactive-worlds"],"related_terms":["action-conditioned video prediction","visual dynamics","Atari","learned simulator"],"summary":"Learns to predict Atari frames from past observations and control inputs, connecting visual generation to an agent's actions.","contribution":"Develops feedforward and recurrent architectures with action-dependent transformations and evaluates long visual rollouts and their usefulness for control.","limitations":"The experiments use Atari games.","model_output":"Predicted game frames.","action_interface":"Discrete Atari control actions.","state_memory":"Past observations; the recurrent variant carries a hidden state through time.","relationship":"Earlier action-conditioned visual-prediction work cited in NeuralOS's introduction and references.","relationship_source":"https://arxiv.org/html/2507.08800v1","author_names":["Junhyuk Oh","Xiaoxiao Guo","Honglak Lee","Richard Lewis","Satinder Singh"],"record_url":"https://neural-os.com/reading-list/papers/1507.08750/","legacy_record_url":"https://neural-os.com/reading-list/papers/#paper-1507-08750","citation_date":"2015-07-31","json_url":"https://neural-os.com/reading-list/papers/1507.08750/record.json","markdown_url":"https://neural-os.com/reading-list/papers/1507.08750/index.md","bibtex_url":"https://neural-os.com/reading-list/papers/1507.08750/citation.bib","csl_url":"https://neural-os.com/reading-list/papers/1507.08750/citation.csl.json"},{"id":"1803.10122","title":"World Models","authors":"David Ha, Jürgen Schmidhuber","first_public":"2018-03-27","first_public_source":"https://arxiv.org/abs/1803.10122","venue":"arXiv","arxiv":"1803.10122","paper_url":"https://arxiv.org/abs/1803.10122","lens":["interactive-worlds"],"related_terms":["world model","recurrent dynamics","latent imagination","model-based reinforcement learning"],"summary":"Combines learned visual compression and recurrent dynamics with a controller, including training an agent inside a generated environment.","contribution":"Demonstrates how a compact learned environment representation can support policy learning and transfer from a simulated dream to the original task.","limitations":"The model compresses observations into a latent representation for control.","model_output":"Predicted latent observations, with an image decoder and task-dependent termination predictions.","action_interface":"Actions from the controller in the studied reinforcement-learning environments.","state_memory":"A recurrent model summarizes the history of latent observations and actions.","relationship":"Cited by NeuralOS as background for learned simulation and agent training.","relationship_source":"https://arxiv.org/html/2507.08800v2","author_names":["David Ha","Jürgen Schmidhuber"],"record_url":"https://neural-os.com/reading-list/papers/1803.10122/","legacy_record_url":"https://neural-os.com/reading-list/papers/#paper-1803-10122","citation_date":"2018-03-27","json_url":"https://neural-os.com/reading-list/papers/1803.10122/record.json","markdown_url":"https://neural-os.com/reading-list/papers/1803.10122/index.md","bibtex_url":"https://neural-os.com/reading-list/papers/1803.10122/citation.bib","csl_url":"https://neural-os.com/reading-list/papers/1803.10122/citation.csl.json"},{"id":"2005.12126","title":"Learning to Simulate Dynamic Environments with GameGAN","authors":"Seung Wook Kim, Yuhao Zhou, Jonah Philion, Antonio Torralba, Sanja Fidler","first_public":"2020-05-25","first_public_source":"https://arxiv.org/abs/2005.12126","venue":"CVPR 2020","arxiv":"2005.12126","paper_url":"https://arxiv.org/abs/2005.12126","lens":["interactive-worlds"],"related_terms":["GameGAN","neural game engine","interactive simulation","visual memory"],"summary":"Learns an interactive visual game simulator from gameplay and keyboard actions using a generative adversarial network.","contribution":"Introduces a memory module for revisited locations and separates static and dynamic image components, enabling visual consistency and component recombination.","limitations":"The simulator learns from recorded games, with memory organized as a spatial map.","model_output":"The next game-screen image.","action_interface":"Game controls represented by keyboard actions.","state_memory":"Recurrent dynamics plus an external memory module that stores an internal environmental map.","relationship":"An earlier interactive neural simulator discussed in NeuralOS's related work.","relationship_source":"https://arxiv.org/html/2507.08800v2","author_names":["Seung Wook Kim","Yuhao Zhou","Jonah Philion","Antonio Torralba","Sanja Fidler"],"record_url":"https://neural-os.com/reading-list/papers/2005.12126/","legacy_record_url":"https://neural-os.com/reading-list/papers/#paper-2005-12126","citation_date":"2020-05-25","json_url":"https://neural-os.com/reading-list/papers/2005.12126/record.json","markdown_url":"https://neural-os.com/reading-list/papers/2005.12126/index.md","bibtex_url":"https://neural-os.com/reading-list/papers/2005.12126/citation.bib","csl_url":"https://neural-os.com/reading-list/papers/2005.12126/citation.csl.json"},{"id":"2310.04875","title":"Prompt-to-OS (P2OS): Revolutionizing Operating Systems and Human-Computer Interaction with Integrated AI Generative Models","authors":"Gabriele Tolomei, Cesare Campagnano, Fabrizio Silvestri, Giovanni Trappolini","first_public":"2023-10-07","first_public_source":"https://arxiv.org/abs/2310.04875","venue":"IEEE CogMI 2023","arxiv":"2310.04875","paper_url":"https://arxiv.org/abs/2310.04875","lens":["neural-os"],"related_terms":["Prompt-to-OS","generative interface","neural operating system","human-computer interaction"],"summary":"Proposes an operating-system interaction paradigm in which connected generative models interpret user intentions and mediate or replace conventional application interfaces.","contribution":"Sets out a broader vision for natural-language, personalized computer interaction, including language and diffusion models as interface components.","limitations":"A conceptual proposal for generative computer interfaces.","model_output":"Envisioned context-dependent responses and generated interfaces.","action_interface":"Natural-language user requests, including text and speech.","state_memory":"Proposes personalization and contextual interaction.","relationship":"NeuralOS cites Prompt-to-OS as background for generative computer interaction.","relationship_source":"https://arxiv.org/html/2507.08800v2","author_names":["Gabriele Tolomei","Cesare Campagnano","Fabrizio Silvestri","Giovanni Trappolini"],"record_url":"https://neural-os.com/reading-list/papers/2310.04875/","legacy_record_url":"https://neural-os.com/reading-list/papers/#paper-2310-04875","citation_date":"2023-10-07","json_url":"https://neural-os.com/reading-list/papers/2310.04875/record.json","markdown_url":"https://neural-os.com/reading-list/papers/2310.04875/index.md","bibtex_url":"https://neural-os.com/reading-list/papers/2310.04875/citation.bib","csl_url":"https://neural-os.com/reading-list/papers/2310.04875/citation.csl.json"},{"id":"2402.15391","title":"Genie: Generative Interactive Environments","authors":"Jake Bruce, Michael Dennis, Ashley Edwards, Jack Parker-Holder, Yuge Shi, Edward Hughes, Matthew Lai, Aditi Mavalankar, Richie Steigerwald, Chris Apps, Yusuf Aytar, Sarah Bechtle, Feryal Behbahani, Stephanie Chan, Nicolas Heess, Lucy Gonzalez, Simon Osindero, Sherjil Ozair, Scott Reed, Jingwei Zhang, Konrad Zolna, Jeff Clune, Nando de Freitas, Satinder Singh, Tim Rocktäschel","first_public":"2024-02-23","first_public_source":"https://arxiv.org/abs/2402.15391","venue":"arXiv","arxiv":"2402.15391","paper_url":"https://arxiv.org/abs/2402.15391","lens":["interactive-worlds"],"related_terms":["Genie","generative interactive environments","latent actions","foundation world model"],"summary":"Learns controllable visual environments from videos without ground-truth action labels, using a learned latent action space.","contribution":"Combines video tokenization, a dynamics model, and latent-action learning so users can interact with worlds initialized from images.","limitations":"Controls use learned latent actions inferred from video.","model_output":"Action-conditioned future video frames.","action_interface":"Discrete latent actions inferred during training rather than recorded physical input events.","state_memory":"A spatiotemporal dynamics model conditions predictions on previous visual tokens and actions.","relationship":"Cited in NeuralOS's related work. Genie learns its action space from video; NeuralOS takes mouse and keyboard events as inputs.","relationship_source":"https://arxiv.org/html/2507.08800v2","author_names":["Jake Bruce","Michael Dennis","Ashley Edwards","Jack Parker-Holder","Yuge Shi","Edward Hughes","Matthew Lai","Aditi Mavalankar","Richie Steigerwald","Chris Apps","Yusuf Aytar","Sarah Bechtle","Feryal Behbahani","Stephanie Chan","Nicolas Heess","Lucy Gonzalez","Simon Osindero","Sherjil Ozair","Scott Reed","Jingwei Zhang","Konrad Zolna","Jeff Clune","Nando de Freitas","Satinder Singh","Tim Rocktäschel"],"record_url":"https://neural-os.com/reading-list/papers/2402.15391/","legacy_record_url":"https://neural-os.com/reading-list/papers/#paper-2402-15391","citation_date":"2024-02-23","json_url":"https://neural-os.com/reading-list/papers/2402.15391/record.json","markdown_url":"https://neural-os.com/reading-list/papers/2402.15391/index.md","bibtex_url":"https://neural-os.com/reading-list/papers/2402.15391/citation.bib","csl_url":"https://neural-os.com/reading-list/papers/2402.15391/citation.csl.json"},{"id":"2404.07972","title":"OSWorld: Benchmarking Multimodal Agents for Open-Ended Tasks in Real Computer Environments","authors":"Tianbao Xie, Danyang Zhang, Jixuan Chen, Xiaochuan Li, Siheng Zhao, Ruisheng Cao, Toh Jing Hua, Zhoujun Cheng, Dongchan Shin, Fangyu Lei, Yitao Liu, Yiheng Xu, Shuyan Zhou, Silvio Savarese, Caiming Xiong, Victor Zhong, Tao Yu","first_public":"2024-04-11","first_public_source":"https://arxiv.org/abs/2404.07972","venue":"NeurIPS 2024","arxiv":"2404.07972","paper_url":"https://arxiv.org/abs/2404.07972","lens":["evaluation"],"related_terms":["OSWorld","computer-use agents","execution-based evaluation","desktop benchmark"],"summary":"Provides real computer environments and reproducible tasks spanning desktop applications, websites, and file operations.","contribution":"Pairs task setup with execution-based evaluators, allowing agents to complete workflows through actual software rather than matching a fixed action sequence.","limitations":"Evaluates workflows in real desktop applications, websites, and files.","model_output":"Observations and evaluation results from real computer environments.","action_interface":"Computer interaction through the benchmark's supported action interfaces.","state_memory":"State persists in the real operating system and applications used by the environment.","relationship":"Provides desktop tasks and outcome checks for evaluating computer-use agents.","relationship_source":"https://arxiv.org/abs/2404.07972","author_names":["Tianbao Xie","Danyang Zhang","Jixuan Chen","Xiaochuan Li","Siheng Zhao","Ruisheng Cao","Toh Jing Hua","Zhoujun Cheng","Dongchan Shin","Fangyu Lei","Yitao Liu","Yiheng Xu","Shuyan Zhou","Silvio Savarese","Caiming Xiong","Victor Zhong","Tao Yu"],"record_url":"https://neural-os.com/reading-list/papers/2404.07972/","legacy_record_url":"https://neural-os.com/reading-list/papers/#paper-2404-07972","citation_date":"2024-04-11","json_url":"https://neural-os.com/reading-list/papers/2404.07972/record.json","markdown_url":"https://neural-os.com/reading-list/papers/2404.07972/index.md","bibtex_url":"https://neural-os.com/reading-list/papers/2404.07972/citation.bib","csl_url":"https://neural-os.com/reading-list/papers/2404.07972/citation.csl.json","publication":{"title":"OSWorld: Benchmarking Multimodal Agents for Open-Ended Tasks in Real Computer Environments","author_names":["Tianbao Xie","Danyang Zhang","Jixuan Chen","Xiaochuan Li","Siheng Zhao","Ruisheng Cao","Toh Jing Hua","Zhoujun Cheng","Dongchan Shin","Fangyu Lei","Yitao Liu","Yiheng Xu","Shuyan Zhou","Silvio Savarese","Caiming Xiong","Victor Zhong","Tao Yu"],"year":2024,"type":"paper-conference","container_title":"Advances in Neural Information Processing Systems","volume":"37","doi":"10.52202/079017-1650","url":"https://proceedings.neurips.cc/paper_files/paper/2024/hash/5d413e48f84dc61244b6be550f1cd8f5-Abstract-Datasets_and_Benchmarks_Track.html","source_url":"https://proceedings.neurips.cc/paper_files/paper/2024/hash/5d413e48f84dc61244b6be550f1cd8f5-Abstract-Datasets_and_Benchmarks_Track.html","verified_at":"2026-10-19"},"publication_bibtex_url":"https://neural-os.com/reading-list/papers/2404.07972/publication.bib","publication_csl_url":"https://neural-os.com/reading-list/papers/2404.07972/publication.csl.json"},{"id":"2405.12399","title":"Diffusion for World Modeling: Visual Details Matter in Atari","authors":"Eloi Alonso, Adam Jelley, Vincent Micheli, Anssi Kanervisto, Amos Storkey, Tim Pearce, François Fleuret","first_public":"2024-05-20","first_public_source":"https://arxiv.org/abs/2405.12399","venue":"NeurIPS 2024","arxiv":"2405.12399","paper_url":"https://arxiv.org/abs/2405.12399","lens":["interactive-worlds","evaluation"],"related_terms":["DIAMOND","diffusion world model","neural game engine","model-based reinforcement learning"],"summary":"Introduces DIAMOND, an agent trained in a diffusion world model, and studies how preserving visual detail affects control.","contribution":"Connects diffusion-based observation prediction with Atari policy learning and demonstrates an interactive Counter-Strike simulator.","limitations":"Evaluated on games; NeuralOS adapts and evaluates it for desktop screen prediction.","model_output":"Predicted visual observations; auxiliary reward and termination predictions support reinforcement learning.","action_interface":"Actions of the modeled game environment.","state_memory":"Visual dynamics condition on a finite history of observations and actions.","relationship":"NeuralOS adapts DIAMOND to OS screen prediction as a baseline in Appendix A of its 2026 revision.","relationship_source":"https://arxiv.org/html/2507.08800v2#A1","author_names":["Eloi Alonso","Adam Jelley","Vincent Micheli","Anssi Kanervisto","Amos Storkey","Tim Pearce","François Fleuret"],"record_url":"https://neural-os.com/reading-list/papers/2405.12399/","legacy_record_url":"https://neural-os.com/reading-list/papers/#paper-2405-12399","citation_date":"2024-05-20","json_url":"https://neural-os.com/reading-list/papers/2405.12399/record.json","markdown_url":"https://neural-os.com/reading-list/papers/2405.12399/index.md","bibtex_url":"https://neural-os.com/reading-list/papers/2405.12399/citation.bib","csl_url":"https://neural-os.com/reading-list/papers/2405.12399/citation.csl.json","method_comparisons":[{"title":"A game world model evaluated on desktop interactions","text":"DIAMOND learns visual game dynamics with a diffusion model. NeuralOS adapts DIAMOND to OS screen prediction as a baseline in Appendix A of its 2026 revision. NeuralOS adds a recurrent state representation and spatial cursor conditioning for desktop interaction.","source_ids":["2405.12399","2507.08800"],"source_urls":["https://arxiv.org/abs/2405.12399","https://arxiv.org/html/2507.08800v2#A1"],"source_labels":["DIAMOND paper","NeuralOS Appendix A"],"topic_id":"interactive-worlds","topic_label":"Interactive learned environments","topic_url":"https://neural-os.com/reading-list/guides/games-to-computers/"}]},{"id":"2405.14573","title":"AndroidWorld: A Dynamic Benchmarking Environment for Autonomous Agents","authors":"Christopher Rawles, Sarah Clinckemaillie, Yifan Chang, Jonathan Waltz, Gabrielle Lau, Marybeth Fair, Alice Li, William Bishop, Wei Li, Folawiyo Campbell-Ajala, Daniel Toyama, Robert Berry, Divya Tyamagundlu, Timothy Lillicrap, Oriana Riva","first_public":"2024-05-23","first_public_source":"https://arxiv.org/abs/2405.14573","venue":"ICLR 2025","arxiv":"2405.14573","paper_url":"https://arxiv.org/abs/2405.14573","lens":["evaluation"],"related_terms":["AndroidWorld","mobile agents","GUI evaluation","task success"],"summary":"Provides a functional Android environment with parameterized tasks and programmatic checks of device state.","contribution":"Combines task initialization, success checking, and teardown with variable task instances, making mobile-agent evaluation reproducible and less dependent on a fixed test set.","limitations":"The benchmark covers Android apps with parameterized tasks and programmatic success checks.","model_output":"Android observations and task rewards.","action_interface":"Agent actions executed on an Android device environment.","state_memory":"The Android system and apps maintain state; evaluators inspect that state directly.","relationship":"Provides Android tasks and outcome checks for evaluating mobile agents.","relationship_source":"https://arxiv.org/abs/2405.14573","author_names":["Christopher Rawles","Sarah Clinckemaillie","Yifan Chang","Jonathan Waltz","Gabrielle Lau","Marybeth Fair","Alice Li","William Bishop","Wei Li","Folawiyo Campbell-Ajala","Daniel Toyama","Robert Berry","Divya Tyamagundlu","Timothy Lillicrap","Oriana Riva"],"record_url":"https://neural-os.com/reading-list/papers/2405.14573/","legacy_record_url":"https://neural-os.com/reading-list/papers/#paper-2405-14573","citation_date":"2024-05-23","json_url":"https://neural-os.com/reading-list/papers/2405.14573/record.json","markdown_url":"https://neural-os.com/reading-list/papers/2405.14573/index.md","bibtex_url":"https://neural-os.com/reading-list/papers/2405.14573/citation.bib","csl_url":"https://neural-os.com/reading-list/papers/2405.14573/citation.csl.json","publication":{"title":"AndroidWorld: A Dynamic Benchmarking Environment for Autonomous Agents","author_names":["Chris Rawles","Sarah Clinckemaillie","Yifan Chang","Jonathan Waltz","Gabrielle Lau","Marybeth Fair","Alice Li","William Bishop","Wei Li","Folawiyo Campbell-Ajala","Daniel Toyama","Robert Berry","Divya Tyamagundlu","Timothy Lillicrap","Oriana Riva"],"year":2025,"type":"paper-conference","container_title":"International Conference on Learning Representations","url":"https://proceedings.iclr.cc/paper_files/paper/2025/hash/01a83bc2f2732a58e6aa731e659e7101-Abstract-Conference.html","source_url":"https://proceedings.iclr.cc/paper_files/paper/2025/hash/01a83bc2f2732a58e6aa731e659e7101-Abstract-Conference.html","verified_at":"2026-10-19"},"publication_bibtex_url":"https://neural-os.com/reading-list/papers/2405.14573/publication.bib","publication_csl_url":"https://neural-os.com/reading-list/papers/2405.14573/publication.csl.json"},{"id":"2408.14837","title":"Diffusion Models Are Real-Time Game Engines","authors":"Dani Valevski, Yaniv Leviathan, Moab Arar, Shlomi Fruchter","first_public":"2024-08-27","first_public_source":"https://arxiv.org/abs/2408.14837","venue":"ICLR 2025","arxiv":"2408.14837","paper_url":"https://arxiv.org/abs/2408.14837","lens":["interactive-worlds"],"related_terms":["GameNGen","neural game engine","real-time simulation","diffusion renderer"],"summary":"Presents GameNGen, a diffusion model that interactively simulates DOOM from previous frames and player actions.","contribution":"Uses agent-collected gameplay, conditioning augmentation, and decoder fine-tuning to sustain autoregressive visual interaction.","limitations":"The simulator is trained and evaluated on DOOM.","model_output":"The next rendered game frame.","action_interface":"DOOM player actions.","state_memory":"Conditioning on a sequence of recent frames and actions; continued play feeds generated frames back into the model.","relationship":"Interactive-simulation background cited by NeuralOS; NeuralOS also refers to its perceptual-evaluation setup.","relationship_source":"https://arxiv.org/html/2507.08800v2","author_names":["Dani Valevski","Yaniv Leviathan","Moab Arar","Shlomi Fruchter"],"record_url":"https://neural-os.com/reading-list/papers/2408.14837/","legacy_record_url":"https://neural-os.com/reading-list/papers/#paper-2408-14837","citation_date":"2024-08-27","json_url":"https://neural-os.com/reading-list/papers/2408.14837/record.json","markdown_url":"https://neural-os.com/reading-list/papers/2408.14837/index.md","bibtex_url":"https://neural-os.com/reading-list/papers/2408.14837/citation.bib","csl_url":"https://neural-os.com/reading-list/papers/2408.14837/citation.csl.json"},{"id":"2410.13232","title":"Web Agents with World Models: Learning and Leveraging Environment Dynamics in Web Navigation","authors":"Hyungjoo Chae, Namyoung Kim, Kai Tzu-iunn Ong, Minju Gwak, Gwanwoo Song, Jihoon Kim, Sunghwan Kim, Dongha Lee, Jinyoung Yeo","first_public":"2024-10-17","first_public_source":"https://arxiv.org/abs/2410.13232","venue":"ICLR 2025","arxiv":"2410.13232","paper_url":"https://arxiv.org/abs/2410.13232","lens":["gui-planning"],"related_terms":["web world model","counterfactual planning","transition prediction","WMA"],"summary":"Predicts the effects of candidate web actions as natural-language descriptions of important changes, then uses those predictions for action selection.","contribution":"Introduces a transition-focused abstraction that reduces the burden of predicting long, repetitive web observations and supports world-model-augmented planning.","limitations":"The paper evaluates web navigation using textual descriptions of state changes.","model_output":"Natural-language descriptions of changes in the next web observation.","action_interface":"Candidate web-navigation actions.","state_memory":"The planner uses web context and predicted descriptions of state changes.","relationship":"Connects environment prediction to agent decisions in web navigation.","relationship_source":"https://arxiv.org/abs/2410.13232","author_names":["Hyungjoo Chae","Namyoung Kim","Kai Tzu-iunn Ong","Minju Gwak","Gwanwoo Song","Jihoon Kim","Sunghwan Kim","Dongha Lee","Jinyoung Yeo"],"record_url":"https://neural-os.com/reading-list/papers/2410.13232/","legacy_record_url":"https://neural-os.com/reading-list/papers/#paper-2410-13232","citation_date":"2024-10-17","json_url":"https://neural-os.com/reading-list/papers/2410.13232/record.json","markdown_url":"https://neural-os.com/reading-list/papers/2410.13232/index.md","bibtex_url":"https://neural-os.com/reading-list/papers/2410.13232/citation.bib","csl_url":"https://neural-os.com/reading-list/papers/2410.13232/citation.csl.json"},{"id":"2504.13936","title":"ViMo: A Generative Visual GUI World Model for App Agents","authors":"Dezhao Luo, Bohan Tang, Kang Li, Georgios Papoudakis, Jifei Song, Shaogang Gong, Jianye Hao, Jun Wang, Kun Shao","first_public":"2025-04-15","first_public_source":"https://arxiv.org/abs/2504.13936","venue":"arXiv","arxiv":"2504.13936","paper_url":"https://arxiv.org/abs/2504.13936","lens":["gui-planning"],"related_terms":["ViMo","GUI world model","next-screen prediction","mobile app simulation"],"summary":"Predicts future mobile-app screens to help agents compare possible actions, separating graphic content from textual content.","contribution":"Uses symbolic text placeholders to preserve layout while assigning graphical and text prediction to separate components.","limitations":"The experiments predict mobile-app screens for agent planning.","model_output":"Future mobile GUI images assembled from graphics and text predictions.","action_interface":"Candidate mobile-app actions.","state_memory":"Screen-and-action-conditioned prediction; symbolic text representation retains text locations and layout.","relationship":"Visual GUI world-model research released in April 2025, before NeuralOS's July 2025 preprint.","relationship_source":"https://arxiv.org/abs/2504.13936","author_names":["Dezhao Luo","Bohan Tang","Kang Li","Georgios Papoudakis","Jifei Song","Shaogang Gong","Jianye Hao","Jun Wang","Kun Shao"],"record_url":"https://neural-os.com/reading-list/papers/2504.13936/","legacy_record_url":"https://neural-os.com/reading-list/papers/#paper-2504-13936","citation_date":"2025-04-15","json_url":"https://neural-os.com/reading-list/papers/2504.13936/record.json","markdown_url":"https://neural-os.com/reading-list/papers/2504.13936/index.md","bibtex_url":"https://neural-os.com/reading-list/papers/2504.13936/citation.bib","csl_url":"https://neural-os.com/reading-list/papers/2504.13936/citation.csl.json"},{"id":"2507.08800","title":"NeuralOS: Towards Simulating Operating Systems via Neural Generative Models","authors":"Luke Rivard, Sun Sun, Hongyu Guo, Wenhu Chen, Yuntian Deng","first_public":"2025-07-11","first_public_source":"https://arxiv.org/abs/2507.08800","first_public_note":"Initial arXiv submission on July 11, 2025; revised on March 12, 2026 and published at ICLR 2026.","venue":"ICLR 2026","arxiv":"2507.08800","paper_url":"https://arxiv.org/abs/2507.08800","original_url":"https://arxiv.org/html/2507.08800v1","revised_url":"https://arxiv.org/html/2507.08800v2","code_url":"https://github.com/yuntian-group/neural-os","lens":["neural-os","interactive-worlds","evaluation"],"related_terms":["NeuralOS","neural operating system","learned computer simulator","generative interface","GUI world model","desktop world model","action-conditioned screen generation"],"summary":"Studies neural operating systems through learned, interactive GUI simulation: a model produces desktop screens in response to mouse and keyboard events, opening a path toward generative computer interfaces.","contribution":"Formalizes OS-interface generation and develops state tracking, precise cursor conditioning, staged training, and data collection. The 2026 revision also demonstrates learning a new simulated application from synthesized training data.","limitations":"Fine-grained typing and long-horizon state retention remain difficult. The simulated interface does not provide the underlying filesystem, processes, or application services of a conventional OS.","model_output":"Autoregressively generated desktop screen frames.","action_interface":"Cursor coordinates, left/right clicks, and keyboard press/release events.","state_memory":"A hierarchical recurrent model carries state across frames and conditions a diffusion renderer.","relationship":"Introduces NeuralOS and its formulation of learned OS-interface simulation.","relationship_source":"https://arxiv.org/html/2507.08800v1","author_names":["Luke Rivard","Sun Sun","Hongyu Guo","Wenhu Chen","Yuntian Deng"],"record_url":"https://neural-os.com/reading-list/papers/2507.08800/","legacy_record_url":"https://neural-os.com/reading-list/papers/#paper-2507-08800","citation_date":"2025-07-11","json_url":"https://neural-os.com/reading-list/papers/2507.08800/record.json","markdown_url":"https://neural-os.com/reading-list/papers/2507.08800/index.md","bibtex_url":"https://neural-os.com/reading-list/papers/2507.08800/citation.bib","csl_url":"https://neural-os.com/reading-list/papers/2507.08800/citation.csl.json","publication":{"title":"NeuralOS: Towards Simulating Operating Systems via Neural Generative Models","author_names":["Luke Rivard","Sun Sun","Hongyu Guo","Wenhu Chen","Yuntian Deng"],"year":2026,"type":"paper-conference","container_title":"International Conference on Learning Representations","url":"https://proceedings.iclr.cc/paper_files/paper/2026/hash/2174a6749c3f769358cd21ade1221b34-Abstract-Conference.html","source_url":"https://proceedings.iclr.cc/paper_files/paper/2026/hash/2174a6749c3f769358cd21ade1221b34-Abstract-Conference.html","verified_at":"2026-10-19"},"publication_bibtex_url":"https://neural-os.com/reading-list/papers/2507.08800/publication.bib","publication_csl_url":"https://neural-os.com/reading-list/papers/2507.08800/publication.csl.json","method_comparisons":[{"title":"A game world model evaluated on desktop interactions","text":"DIAMOND learns visual game dynamics with a diffusion model. NeuralOS adapts DIAMOND to OS screen prediction as a baseline in Appendix A of its 2026 revision. NeuralOS adds a recurrent state representation and spatial cursor conditioning for desktop interaction.","source_ids":["2405.12399","2507.08800"],"source_urls":["https://arxiv.org/abs/2405.12399","https://arxiv.org/html/2507.08800v2#A1"],"source_labels":["DIAMOND paper","NeuralOS Appendix A"],"topic_id":"interactive-worlds","topic_label":"Interactive learned environments","topic_url":"https://neural-os.com/reading-list/guides/games-to-computers/"},{"title":"Direct frame generation and renderable-code prediction","text":"NeuralOS predicts desktop frames using learned state and a diffusion renderer. Code2World predicts a UI through code that is then rendered into a screen, and tests its usefulness for mobile navigation. Code2World cites NeuralOS in Section 2.2.","source_ids":["2507.08800","2602.09856"],"source_urls":["https://arxiv.org/html/2507.08800v1","https://arxiv.org/html/2602.09856v1#S2.SS2"],"source_labels":["Original NeuralOS paper","Code2World Section 2.2"],"topic_id":"gui-planning","topic_label":"GUI prediction and planning","topic_url":"https://neural-os.com/reading-list/guides/games-to-computers/#gui-prediction-and-planning"}]},{"id":"2602.09856","title":"Code2World: A GUI World Model via Renderable Code Generation","authors":"Yuhao Zheng, Li'an Zhong, Yi Wang, Rui Dai, Kaikui Liu, Xiangxiang Chu, Linyuan Lv, Philip Torr, Kevin Qinghong Lin","first_public":"2026-02-10","first_public_source":"https://arxiv.org/abs/2602.09856","venue":"arXiv","arxiv":"2602.09856","paper_url":"https://arxiv.org/abs/2602.09856","lens":["gui-planning"],"related_terms":["Code2World","GUI world model","renderable code","next-screen prediction","AndroidCode"],"summary":"Predicts a GUI's next visual state by generating renderable code, and evaluates whether the predicted outcomes help mobile-agent navigation.","contribution":"Constructs paired screen/action/code data and trains with rendered visual feedback to balance appearance and action consistency.","limitations":"Uses generated code and a browser to predict screens for Android navigation.","model_output":"Renderable UI code, converted into the predicted next screen.","action_interface":"A current GUI observation and a proposed interaction, represented for the model with action descriptions and visual cues.","state_memory":"Prediction conditions on the current screen and action; renderable structure represents the output interface.","relationship":"Code2World cites NeuralOS in Section 2.2 and its bibliography.","relationship_source":"https://arxiv.org/html/2602.09856v1#S2.SS2","author_names":["Yuhao Zheng","Li'an Zhong","Yi Wang","Rui Dai","Kaikui Liu","Xiangxiang Chu","Linyuan Lv","Philip Torr","Kevin Qinghong Lin"],"record_url":"https://neural-os.com/reading-list/papers/2602.09856/","legacy_record_url":"https://neural-os.com/reading-list/papers/#paper-2602-09856","citation_date":"2026-02-10","json_url":"https://neural-os.com/reading-list/papers/2602.09856/record.json","markdown_url":"https://neural-os.com/reading-list/papers/2602.09856/index.md","bibtex_url":"https://neural-os.com/reading-list/papers/2602.09856/citation.bib","csl_url":"https://neural-os.com/reading-list/papers/2602.09856/citation.csl.json","method_comparisons":[{"title":"Direct frame generation and renderable-code prediction","text":"NeuralOS predicts desktop frames using learned state and a diffusion renderer. Code2World predicts a UI through code that is then rendered into a screen, and tests its usefulness for mobile navigation. Code2World cites NeuralOS in Section 2.2.","source_ids":["2507.08800","2602.09856"],"source_urls":["https://arxiv.org/html/2507.08800v1","https://arxiv.org/html/2602.09856v1#S2.SS2"],"source_labels":["Original NeuralOS paper","Code2World Section 2.2"],"topic_id":"gui-planning","topic_label":"GUI prediction and planning","topic_url":"https://neural-os.com/reading-list/guides/games-to-computers/#gui-prediction-and-planning"}]}],"precursors":[]}