{
  "metadata": {
    "version": "1.3.3",
    "updated_at": "2026-11-14",
    "search_refreshed_at": "2026-10-19",
    "topic_id": "gui-planning",
    "name": "GUI prediction and planning",
    "definition": "Predict the consequences of interface actions so an agent can compare alternatives before execution. Prediction targets include textual changes, screen images, and renderable interface code.",
    "record_count": 3,
    "topic_url": "https://neural-os.com/reading-list/guides/games-to-computers/#gui-prediction-and-planning",
    "source_url": "https://neural-os.com/reading-list/data/papers.json"
  },
  "comparisons": [
    {
      "title": "Direct frame generation and renderable-code prediction",
      "text": "NeuralOS predicts desktop frames using learned state and a diffusion renderer. Code2World predicts a UI through code that is then rendered into a screen, and tests its usefulness for mobile navigation. Code2World cites NeuralOS in Section 2.2.",
      "source_ids": [
        "2507.08800",
        "2602.09856"
      ],
      "source_urls": [
        "https://arxiv.org/html/2507.08800v1",
        "https://arxiv.org/html/2602.09856v1#S2.SS2"
      ],
      "source_labels": [
        "Original NeuralOS paper",
        "Code2World Section 2.2"
      ]
    }
  ],
  "demos": [

  ],
  "papers": [
    {
      "id": "2410.13232",
      "title": "Web Agents with World Models: Learning and Leveraging Environment Dynamics in Web Navigation",
      "authors": "Hyungjoo Chae, Namyoung Kim, Kai Tzu-iunn Ong, Minju Gwak, Gwanwoo Song, Jihoon Kim, Sunghwan Kim, Dongha Lee, Jinyoung Yeo",
      "first_public": "2024-10-17",
      "first_public_source": "https://arxiv.org/abs/2410.13232",
      "venue": "ICLR 2025",
      "arxiv": "2410.13232",
      "paper_url": "https://arxiv.org/abs/2410.13232",
      "lens": [
        "gui-planning"
      ],
      "related_terms": [
        "web world model",
        "counterfactual planning",
        "transition prediction",
        "WMA"
      ],
      "summary": "Predicts the effects of candidate web actions as natural-language descriptions of important changes, then uses those predictions for action selection.",
      "contribution": "Introduces a transition-focused abstraction that reduces the burden of predicting long, repetitive web observations and supports world-model-augmented planning.",
      "model_output": "Natural-language descriptions of changes in the next web observation.",
      "action_interface": "Candidate web-navigation actions.",
      "state_memory": "The planner uses web context and predicted descriptions of state changes.",
      "relationship": "Connects environment prediction to agent decisions in web navigation.",
      "relationship_source": "https://arxiv.org/abs/2410.13232",
      "author_names": [
        "Hyungjoo Chae",
        "Namyoung Kim",
        "Kai Tzu-iunn Ong",
        "Minju Gwak",
        "Gwanwoo Song",
        "Jihoon Kim",
        "Sunghwan Kim",
        "Dongha Lee",
        "Jinyoung Yeo"
      ],
      "record_url": "https://neural-os.com/reading-list/papers/2410.13232/",
      "legacy_record_url": "https://neural-os.com/reading-list/papers/#paper-2410-13232",
      "citation_date": "2024-10-17",
      "json_url": "https://neural-os.com/reading-list/papers/2410.13232/record.json",
      "markdown_url": "https://neural-os.com/reading-list/papers/2410.13232/index.md",
      "bibtex_url": "https://neural-os.com/reading-list/papers/2410.13232/citation.bib",
      "csl_url": "https://neural-os.com/reading-list/papers/2410.13232/citation.csl.json",
      "translations": {
        "zh-Hans": {
          "summary": "用自然语言描述候选网页动作引起的重要变化，再根据这些预测选择动作。",
          "contribution": "提出以状态转移为重点的抽象表示，减轻预测冗长、重复网页观测的负担，并支持借助世界模型进行规划。",
          "model_output": "对下一次网页观测中变化的自然语言描述。",
          "action_interface": "候选网页导航动作。",
          "state_memory": "规划器使用网页上下文和预测的状态变化描述。",
          "relationship": "将环境预测与网页导航中智能体的决策联系起来。",
          "related_terms": [
            "网页世界模型",
            "反事实规划",
            "状态转移预测",
            "WMA"
          ],
          "record_url": "https://neural-os.com/reading-list/zh/papers/2410.13232/"
        }
      }
    },
    {
      "id": "2504.13936",
      "title": "ViMo: A Generative Visual GUI World Model for App Agents",
      "authors": "Dezhao Luo, Bohan Tang, Kang Li, Georgios Papoudakis, Jifei Song, Shaogang Gong, Jianye Hao, Jun Wang, Kun Shao",
      "first_public": "2025-04-15",
      "first_public_source": "https://arxiv.org/abs/2504.13936",
      "venue": "arXiv",
      "arxiv": "2504.13936",
      "paper_url": "https://arxiv.org/abs/2504.13936",
      "lens": [
        "gui-planning"
      ],
      "related_terms": [
        "ViMo",
        "GUI world model",
        "next-screen prediction",
        "mobile app simulation"
      ],
      "summary": "Predicts future mobile-app screens to help agents compare possible actions, separating graphic content from textual content.",
      "contribution": "Uses symbolic text placeholders to preserve layout while assigning graphical and text prediction to separate components.",
      "model_output": "Future mobile GUI images assembled from graphics and text predictions.",
      "action_interface": "Candidate mobile-app actions.",
      "state_memory": "Screen-and-action-conditioned prediction; symbolic text representation retains text locations and layout.",
      "relationship": "Visual GUI world-model research released in April 2025, before NeuralOS's July 2025 preprint.",
      "relationship_source": "https://arxiv.org/abs/2504.13936",
      "author_names": [
        "Dezhao Luo",
        "Bohan Tang",
        "Kang Li",
        "Georgios Papoudakis",
        "Jifei Song",
        "Shaogang Gong",
        "Jianye Hao",
        "Jun Wang",
        "Kun Shao"
      ],
      "record_url": "https://neural-os.com/reading-list/papers/2504.13936/",
      "legacy_record_url": "https://neural-os.com/reading-list/papers/#paper-2504-13936",
      "citation_date": "2025-04-15",
      "json_url": "https://neural-os.com/reading-list/papers/2504.13936/record.json",
      "markdown_url": "https://neural-os.com/reading-list/papers/2504.13936/index.md",
      "bibtex_url": "https://neural-os.com/reading-list/papers/2504.13936/citation.bib",
      "csl_url": "https://neural-os.com/reading-list/papers/2504.13936/citation.csl.json",
      "translations": {
        "zh-Hans": {
          "summary": "预测移动应用的后续屏幕，帮助智能体比较候选动作，并将图形内容与文字内容分开处理。",
          "contribution": "用符号化文字占位符保留布局，将图形预测和文字预测交给不同组件完成。",
          "model_output": "由图形预测和文字预测组合而成的未来移动 GUI 图像。",
          "action_interface": "候选移动应用操作。",
          "state_memory": "以屏幕和动作为条件进行预测；符号化文字表示保留文字位置与布局。",
          "relationship": "这项视觉 GUI 世界模型研究于 2025 年 4 月公开，早于 NeuralOS 在 2025 年 7 月发布的预印本。",
          "related_terms": [
            "ViMo",
            "GUI 世界模型",
            "下一屏幕预测",
            "移动应用仿真"
          ],
          "record_url": "https://neural-os.com/reading-list/zh/papers/2504.13936/"
        }
      }
    },
    {
      "id": "2602.09856",
      "title": "Code2World: A GUI World Model via Renderable Code Generation",
      "authors": "Yuhao Zheng, Li'an Zhong, Yi Wang, Rui Dai, Kaikui Liu, Xiangxiang Chu, Linyuan Lv, Philip Torr, Kevin Qinghong Lin",
      "first_public": "2026-02-10",
      "first_public_source": "https://arxiv.org/abs/2602.09856",
      "venue": "arXiv",
      "arxiv": "2602.09856",
      "paper_url": "https://arxiv.org/abs/2602.09856",
      "lens": [
        "gui-planning"
      ],
      "related_terms": [
        "Code2World",
        "GUI world model",
        "renderable code",
        "next-screen prediction",
        "AndroidCode"
      ],
      "summary": "Predicts a GUI's next visual state by generating renderable code, and evaluates whether the predicted outcomes help mobile-agent navigation.",
      "contribution": "Constructs paired screen/action/code data and trains with rendered visual feedback to balance appearance and action consistency.",
      "model_output": "Renderable UI code, converted into the predicted next screen.",
      "action_interface": "A current GUI observation and a proposed interaction, represented for the model with action descriptions and visual cues.",
      "state_memory": "Prediction conditions on the current screen and action; renderable structure represents the output interface.",
      "relationship": "Code2World cites NeuralOS in Section 2.2 and its bibliography.",
      "relationship_source": "https://arxiv.org/html/2602.09856v1#S2.SS2",
      "author_names": [
        "Yuhao Zheng",
        "Li'an Zhong",
        "Yi Wang",
        "Rui Dai",
        "Kaikui Liu",
        "Xiangxiang Chu",
        "Linyuan Lv",
        "Philip Torr",
        "Kevin Qinghong Lin"
      ],
      "record_url": "https://neural-os.com/reading-list/papers/2602.09856/",
      "legacy_record_url": "https://neural-os.com/reading-list/papers/#paper-2602-09856",
      "citation_date": "2026-02-10",
      "json_url": "https://neural-os.com/reading-list/papers/2602.09856/record.json",
      "markdown_url": "https://neural-os.com/reading-list/papers/2602.09856/index.md",
      "bibtex_url": "https://neural-os.com/reading-list/papers/2602.09856/citation.bib",
      "csl_url": "https://neural-os.com/reading-list/papers/2602.09856/citation.csl.json",
      "method_comparisons": [
        {
          "title": "Direct frame generation and renderable-code prediction",
          "text": "NeuralOS predicts desktop frames using learned state and a diffusion renderer. Code2World predicts a UI through code that is then rendered into a screen, and tests its usefulness for mobile navigation. Code2World cites NeuralOS in Section 2.2.",
          "source_ids": [
            "2507.08800",
            "2602.09856"
          ],
          "source_urls": [
            "https://arxiv.org/html/2507.08800v1",
            "https://arxiv.org/html/2602.09856v1#S2.SS2"
          ],
          "source_labels": [
            "Original NeuralOS paper",
            "Code2World Section 2.2"
          ],
          "topic_id": "gui-planning",
          "topic_label": "GUI prediction and planning",
          "topic_url": "https://neural-os.com/reading-list/guides/games-to-computers/#gui-prediction-and-planning"
        }
      ],
      "translations": {
        "zh-Hans": {
          "summary": "通过生成可渲染代码来预测 GUI 的下一视觉状态，并评估预测结果是否有助于移动端智能体导航。",
          "contribution": "构建配对的屏幕、动作和代码数据，使用渲染后的视觉反馈训练模型，兼顾界面外观和动作一致性。",
          "model_output": "可渲染的界面代码，经渲染得到预测的下一屏幕。",
          "action_interface": "当前 GUI 观测和一个候选交互，通过动作描述与视觉提示输入模型。",
          "state_memory": "预测以当前屏幕和动作为条件；输出界面由可渲染的结构表示。",
          "relationship": "Code2World 在第 2.2 节和参考文献中引用 NeuralOS。",
          "related_terms": [
            "Code2World",
            "GUI 世界模型",
            "可渲染代码",
            "下一屏幕预测",
            "AndroidCode"
          ],
          "record_url": "https://neural-os.com/reading-list/zh/papers/2602.09856/"
        }
      }
    }
  ]
}
