{ "metadata": { "version": "1.0.1", "updated_at": "2026-10-19", "search_refreshed_at": "2026-10-19", "topic_id": "gui-planning", "name": "GUI prediction and planning", "definition": "Predict the consequences of interface actions so an agent can compare alternatives before execution. Prediction targets include textual changes, screen images, and renderable interface code.", "record_count": 3, "topic_url": "https://neural-os.com/reading-list/guides/games-to-computers/#gui-prediction-and-planning", "source_url": "https://neural-os.com/reading-list/data/papers.json" }, "comparisons": [ { "title": "Direct frame generation and renderable-code prediction", "text": "NeuralOS predicts desktop frames using learned state and a diffusion renderer. Code2World predicts a UI through code that is then rendered into a screen, and tests its usefulness for mobile navigation. Code2World cites NeuralOS in Section 2.2.", "source_ids": [ "2507.08800", "2602.09856" ], "source_urls": [ "https://arxiv.org/html/2507.08800v1", "https://arxiv.org/html/2602.09856v1#S2.SS2" ], "source_labels": [ "Original NeuralOS paper", "Code2World Section 2.2" ] } ], "demos": [ ], "papers": [ { "id": "2410.13232", "title": "Web Agents with World Models: Learning and Leveraging Environment Dynamics in Web Navigation", "authors": "Hyungjoo Chae, Namyoung Kim, Kai Tzu-iunn Ong, Minju Gwak, Gwanwoo Song, Jihoon Kim, Sunghwan Kim, Dongha Lee, Jinyoung Yeo", "first_public": "2024-10-17", "first_public_source": "https://arxiv.org/abs/2410.13232", "venue": "ICLR 2025", "arxiv": "2410.13232", "paper_url": "https://arxiv.org/abs/2410.13232", "lens": [ "gui-planning" ], "related_terms": [ "web world model", "counterfactual planning", "transition prediction", "WMA" ], "summary": "Predicts the effects of candidate web actions as natural-language descriptions of important changes, then uses those predictions for action selection.", "contribution": "Introduces a transition-focused abstraction that reduces the burden of predicting long, repetitive web observations and supports world-model-augmented planning.", "limitations": "The paper evaluates web navigation using textual descriptions of state changes.", "model_output": "Natural-language descriptions of changes in the next web observation.", "action_interface": "Candidate web-navigation actions.", "state_memory": "The planner uses web context and predicted descriptions of state changes.", "relationship": "Connects environment prediction to agent decisions in web navigation.", "relationship_source": "https://arxiv.org/abs/2410.13232", "author_names": [ "Hyungjoo Chae", "Namyoung Kim", "Kai Tzu-iunn Ong", "Minju Gwak", "Gwanwoo Song", "Jihoon Kim", "Sunghwan Kim", "Dongha Lee", "Jinyoung Yeo" ], "record_url": "https://neural-os.com/reading-list/papers/2410.13232/", "legacy_record_url": "https://neural-os.com/reading-list/papers/#paper-2410-13232", "citation_date": "2024-10-17", "json_url": "https://neural-os.com/reading-list/papers/2410.13232/record.json", "markdown_url": "https://neural-os.com/reading-list/papers/2410.13232/index.md", "bibtex_url": "https://neural-os.com/reading-list/papers/2410.13232/citation.bib", "csl_url": "https://neural-os.com/reading-list/papers/2410.13232/citation.csl.json" }, { "id": "2504.13936", "title": "ViMo: A Generative Visual GUI World Model for App Agents", "authors": "Dezhao Luo, Bohan Tang, Kang Li, Georgios Papoudakis, Jifei Song, Shaogang Gong, Jianye Hao, Jun Wang, Kun Shao", "first_public": "2025-04-15", "first_public_source": "https://arxiv.org/abs/2504.13936", "venue": "arXiv", "arxiv": "2504.13936", "paper_url": "https://arxiv.org/abs/2504.13936", "lens": [ "gui-planning" ], "related_terms": [ "ViMo", "GUI world model", "next-screen prediction", "mobile app simulation" ], "summary": "Predicts future mobile-app screens to help agents compare possible actions, separating graphic content from textual content.", "contribution": "Uses symbolic text placeholders to preserve layout while assigning graphical and text prediction to separate components.", "limitations": "The experiments predict mobile-app screens for agent planning.", "model_output": "Future mobile GUI images assembled from graphics and text predictions.", "action_interface": "Candidate mobile-app actions.", "state_memory": "Screen-and-action-conditioned prediction; symbolic text representation retains text locations and layout.", "relationship": "Visual GUI world-model research released in April 2025, before NeuralOS's July 2025 preprint.", "relationship_source": "https://arxiv.org/abs/2504.13936", "author_names": [ "Dezhao Luo", "Bohan Tang", "Kang Li", "Georgios Papoudakis", "Jifei Song", "Shaogang Gong", "Jianye Hao", "Jun Wang", "Kun Shao" ], "record_url": "https://neural-os.com/reading-list/papers/2504.13936/", "legacy_record_url": "https://neural-os.com/reading-list/papers/#paper-2504-13936", "citation_date": "2025-04-15", "json_url": "https://neural-os.com/reading-list/papers/2504.13936/record.json", "markdown_url": "https://neural-os.com/reading-list/papers/2504.13936/index.md", "bibtex_url": "https://neural-os.com/reading-list/papers/2504.13936/citation.bib", "csl_url": "https://neural-os.com/reading-list/papers/2504.13936/citation.csl.json" }, { "id": "2602.09856", "title": "Code2World: A GUI World Model via Renderable Code Generation", "authors": "Yuhao Zheng, Li'an Zhong, Yi Wang, Rui Dai, Kaikui Liu, Xiangxiang Chu, Linyuan Lv, Philip Torr, Kevin Qinghong Lin", "first_public": "2026-02-10", "first_public_source": "https://arxiv.org/abs/2602.09856", "venue": "arXiv", "arxiv": "2602.09856", "paper_url": "https://arxiv.org/abs/2602.09856", "lens": [ "gui-planning" ], "related_terms": [ "Code2World", "GUI world model", "renderable code", "next-screen prediction", "AndroidCode" ], "summary": "Predicts a GUI's next visual state by generating renderable code, and evaluates whether the predicted outcomes help mobile-agent navigation.", "contribution": "Constructs paired screen/action/code data and trains with rendered visual feedback to balance appearance and action consistency.", "limitations": "Uses generated code and a browser to predict screens for Android navigation.", "model_output": "Renderable UI code, converted into the predicted next screen.", "action_interface": "A current GUI observation and a proposed interaction, represented for the model with action descriptions and visual cues.", "state_memory": "Prediction conditions on the current screen and action; renderable structure represents the output interface.", "relationship": "Code2World cites NeuralOS in Section 2.2 and its bibliography.", "relationship_source": "https://arxiv.org/html/2602.09856v1#S2.SS2", "author_names": [ "Yuhao Zheng", "Li'an Zhong", "Yi Wang", "Rui Dai", "Kaikui Liu", "Xiangxiang Chu", "Linyuan Lv", "Philip Torr", "Kevin Qinghong Lin" ], "record_url": "https://neural-os.com/reading-list/papers/2602.09856/", "legacy_record_url": "https://neural-os.com/reading-list/papers/#paper-2602-09856", "citation_date": "2026-02-10", "json_url": "https://neural-os.com/reading-list/papers/2602.09856/record.json", "markdown_url": "https://neural-os.com/reading-list/papers/2602.09856/index.md", "bibtex_url": "https://neural-os.com/reading-list/papers/2602.09856/citation.bib", "csl_url": "https://neural-os.com/reading-list/papers/2602.09856/citation.csl.json", "method_comparisons": [ { "title": "Direct frame generation and renderable-code prediction", "text": "NeuralOS predicts desktop frames using learned state and a diffusion renderer. Code2World predicts a UI through code that is then rendered into a screen, and tests its usefulness for mobile navigation. Code2World cites NeuralOS in Section 2.2.", "source_ids": [ "2507.08800", "2602.09856" ], "source_urls": [ "https://arxiv.org/html/2507.08800v1", "https://arxiv.org/html/2602.09856v1#S2.SS2" ], "source_labels": [ "Original NeuralOS paper", "Code2World Section 2.2" ], "topic_id": "gui-planning", "topic_label": "GUI prediction and planning", "topic_url": "https://neural-os.com/reading-list/guides/games-to-computers/#gui-prediction-and-planning" } ] } ] }