From a14b8a8ee72bac88cc56cdfefd82cd5a647e67cb Mon Sep 17 00:00:00 2001 From: commandblock2 Date: Thu, 7 Aug 2025 00:57:17 +0800 Subject: [PATCH 01/15] feat: json schema for the bot! --- .../mvp-spec/data/.vscode/settings.json | 10 + .../mvp-spec/data/combined_schema.json | 307 ++++++++++++++++++ .../mvp-spec/data/example-session.json | 227 +++++++++++++ .../mvp-spec/data/output_schema.json | 62 ++++ .../mvp-spec/mvp-spec.md | 154 +++++++++ .../spec-v1/SPEC-DATA.md | 8 +- 6 files changed, 762 insertions(+), 6 deletions(-) create mode 100644 packages/deep-learning-bot-utils/mvp-spec/data/.vscode/settings.json create mode 100644 packages/deep-learning-bot-utils/mvp-spec/data/combined_schema.json create mode 100644 packages/deep-learning-bot-utils/mvp-spec/data/example-session.json create mode 100644 packages/deep-learning-bot-utils/mvp-spec/data/output_schema.json create mode 100644 packages/deep-learning-bot-utils/mvp-spec/mvp-spec.md diff --git a/packages/deep-learning-bot-utils/mvp-spec/data/.vscode/settings.json b/packages/deep-learning-bot-utils/mvp-spec/data/.vscode/settings.json new file mode 100644 index 0000000..c5ad956 --- /dev/null +++ b/packages/deep-learning-bot-utils/mvp-spec/data/.vscode/settings.json @@ -0,0 +1,10 @@ +{ + "json.schemas": [ + { + "fileMatch": [ + "example-session.json" + ], + "url": "./combined_schema.json" + } +] +} \ No newline at end of file diff --git a/packages/deep-learning-bot-utils/mvp-spec/data/combined_schema.json b/packages/deep-learning-bot-utils/mvp-spec/data/combined_schema.json new file mode 100644 index 0000000..476aec5 --- /dev/null +++ b/packages/deep-learning-bot-utils/mvp-spec/data/combined_schema.json @@ -0,0 +1,307 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Combined Input and Output Data Schema", + "description": "Schema defining the structure of observation data (X) and Baritone's action (Y) for each tick in a training scenario.", + "type": "array", + "items": { + "type": "object", + "properties": { + "tick_id": { + "type": "integer", + "description": "Unique identifier for the game tick within the current Minecraft session." + }, + "timestamp_ms": { + "type": "integer", + "description": "Unix timestamp in milliseconds when this observation was recorded." + }, + "player_state": { + "$ref": "#/$defs/PlayerState" + }, + "local_environment_scan": { + "type": "array", + "description": "A list of collision boxes from nearby environmental elements.", + "items": { + "$ref": "#/$defs/CollisionBox" + }, + "minItems": 0 + }, + "historical_player_states": { + "type": "array", + "description": "A chronological list of the player's state for the last N ticks.", + "items": { + "$ref": "#/$defs/HistoricalPlayerState" + }, + "minItems": 0, + "maxItems": 39 + }, + "baritone_action": { + "$ref": "output_schema.json" + } + }, + "required": [ + "tick_id", + "timestamp_ms", + "player_state", + "local_environment_scan", + "historical_player_states", + "baritone_action" + ], + "additionalProperties": false + }, + "$defs": { + "Coordinates3D": { + "type": "object", + "properties": { + "x": { + "type": "number" + }, + "y": { + "type": "number" + }, + "z": { + "type": "number" + } + }, + "required": [ + "x", + "y", + "z" + ], + "additionalProperties": false + }, + "Velocity3D": { + "type": "object", + "properties": { + "vx": { + "type": "number" + }, + "vy": { + "type": "number" + }, + "vz": { + "type": "number" + } + }, + "required": [ + "vx", + "vy", + "vz" + ], + "additionalProperties": false + }, + "LookDirection": { + "type": "object", + "properties": { + "yaw": { + "type": "number" + }, + "pitch": { + "type": "number" + } + }, + "required": [ + "yaw", + "pitch" + ], + "additionalProperties": false + }, + "PlayerPose": { + "type": "string", + "enum": [ + "STANDING", + "SNEAKING", + "SPRINTING", + "SWIMMING", + "CRAWLING" + ] + }, + "PlayerState": { + "type": "object", + "properties": { + "position": { + "$ref": "#/$defs/Coordinates3D" + }, + "velocity": { + "$ref": "#/$defs/Velocity3D" + }, + "look_direction": { + "$ref": "#/$defs/LookDirection" + }, + "player_pose": { + "$ref": "#/$defs/PlayerPose" + }, + "ground_proximity": { + "type": "boolean" + }, + "predicted_passive_next_tick_state": { + "type": "object", + "properties": { + "predicted_pos": { + "$ref": "#/$defs/Coordinates3D" + }, + "predicted_vel": { + "$ref": "#/$defs/Velocity3D" + } + }, + "required": [ + "predicted_pos", + "predicted_vel" + ], + "additionalProperties": false + } + }, + "required": [ + "position", + "velocity", + "look_direction", + "player_pose", + "ground_proximity", + "predicted_passive_next_tick_state" + ], + "additionalProperties": false + }, + "BoundingBoxCoordinates": { + "type": "object", + "properties": { + "min_x": { + "type": "number" + }, + "min_y": { + "type": "number" + }, + "min_z": { + "type": "number" + }, + "max_x": { + "type": "number" + }, + "max_y": { + "type": "number" + }, + "max_z": { + "type": "number" + } + }, + "required": [ + "min_x", + "min_y", + "min_z", + "max_x", + "max_y", + "max_z" + ], + "additionalProperties": false + }, + "BoxDimensions": { + "type": "object", + "properties": { + "length": { + "type": "number" + }, + "width": { + "type": "number" + }, + "height": { + "type": "number" + } + }, + "required": [ + "length", + "width", + "height" + ], + "additionalProperties": false + }, + "TraversabilityData": { + "type": "string", + "enum": [ + "SOLID_WALKABLE", + "FLUID", + "OBSTRUCTION", + "AIR", + "LIQUID_PLACEABLE", + "PLACEABLE_BLOCK", + "SOLID_SLIPPERY", + "NON_PHYSICAL", + "OTHER" + ] + }, + "AreaSourceType": { + "type": "string", + "enum": [ + "FIXED_RADIUS", + "DYNAMIC_INTEREST" + ] + }, + "CollisionBox": { + "type": "object", + "properties": { + "bounding_box_coordinates": { + "$ref": "#/$defs/BoundingBoxCoordinates" + }, + "relative_position": { + "$ref": "#/$defs/Coordinates3D" + }, + "box_dimensions": { + "$ref": "#/$defs/BoxDimensions" + }, + "element_identifier": { + "type": "string" + }, + "traversability_data": { + "$ref": "#/$defs/TraversabilityData" + }, + "element_state_properties": { + "type": "object", + "description": "Additional, block-specific properties (e.g., direction of stairs, open/closed doors).", + "additionalProperties": true + }, + "area_source_type": { + "$ref": "#/$defs/AreaSourceType" + }, + "box_validity": { + "type": "boolean" + } + }, + "required": [ + "bounding_box_coordinates", + "relative_position", + "box_dimensions", + "element_identifier", + "traversability_data", + "element_state_properties", + "area_source_type", + "box_validity" + ], + "additionalProperties": false + }, + "HistoricalPlayerState": { + "type": "object", + "properties": { + "position": { + "$ref": "#/$defs/Coordinates3D" + }, + "velocity": { + "$ref": "#/$defs/Velocity3D" + }, + "look_direction": { + "$ref": "#/$defs/LookDirection" + }, + "player_pose": { + "$ref": "#/$defs/PlayerPose" + }, + "fall_distance": { + "type": "number" + } + }, + "required": [ + "position", + "velocity", + "look_direction", + "player_pose", + "fall_distance" + ], + "additionalProperties": false + } + } +} \ No newline at end of file diff --git a/packages/deep-learning-bot-utils/mvp-spec/data/example-session.json b/packages/deep-learning-bot-utils/mvp-spec/data/example-session.json new file mode 100644 index 0000000..f40093f --- /dev/null +++ b/packages/deep-learning-bot-utils/mvp-spec/data/example-session.json @@ -0,0 +1,227 @@ +[ + { + "tick_id": 12345, + "timestamp_ms": 1678886400000, + "player_state": { + "position": { + "x": 100.5, + "y": 64.0, + "z": 200.5 + }, + "velocity": { + "vx": 0.0, + "vy": -0.0784, + "vz": 0.0 + }, + "look_direction": { + "yaw": 45.0, + "pitch": 15.0 + }, + "player_pose": "STANDING", + "ground_proximity": true, + "predicted_passive_next_tick_state": { + "predicted_pos": { + "x": 100.5, + "y": 63.9216, + "z": 200.5 + }, + "predicted_vel": { + "vx": 0.0, + "vy": -0.0784, + "vz": 0.0 + } + } + }, + "local_environment_scan": [ + { + "bounding_box_coordinates": { + "min_x": 99.0, + "min_y": 63.0, + "min_z": 199.0, + "max_x": 100.0, + "max_y": 64.0, + "max_z": 200.0 + }, + "relative_position": { + "x": -0.5, + "y": -0.5, + "z": -0.5 + }, + "box_dimensions": { + "length": 1.0, + "width": 1.0, + "height": 1.0 + }, + "element_identifier": "minecraft:stone", + "traversability_data": "SOLID_WALKABLE", + "element_state_properties": {}, + "area_source_type": "FIXED_RADIUS", + "box_validity": true + } + ], + "historical_player_states": [ + { + "position": { + "x": 100.5, + "y": 64.0, + "z": 200.5 + }, + "velocity": { + "vx": 0.0, + "vy": -0.0784, + "vz": 0.0 + }, + "look_direction": { + "yaw": 45.0, + "pitch": 15.0 + }, + "player_pose": "STANDING", + "fall_distance": 0.0 + } + ], + "baritone_action": { + "move_direction": "FORWARD", + "look_change": { + "yaw": 0.0, + "pitch": 0.0 + }, + "jump": false, + "sneak": false, + "sprint": true + } + }, + { + "tick_id": 12346, + "timestamp_ms": 1678886400050, + "player_state": { + "position": { + "x": 100.55, + "y": 64.0, + "z": 200.5 + }, + "velocity": { + "vx": 0.05, + "vy": -0.0784, + "vz": 0.0 + }, + "look_direction": { + "yaw": 45.0, + "pitch": 15.0 + }, + "player_pose": "SPRINTING", + "ground_proximity": true, + "predicted_passive_next_tick_state": { + "predicted_pos": { + "x": 100.6, + "y": 63.9216, + "z": 200.5 + }, + "predicted_vel": { + "vx": 0.05, + "vy": -0.0784, + "vz": 0.0 + } + } + }, + "local_environment_scan": [ + { + "bounding_box_coordinates": { + "min_x": 99.0, + "min_y": 63.0, + "min_z": 199.0, + "max_x": 100.0, + "max_y": 64.0, + "max_z": 200.0 + }, + "relative_position": { + "x": -0.55, + "y": -0.5, + "z": -0.5 + }, + "box_dimensions": { + "length": 1.0, + "width": 1.0, + "height": 1.0 + }, + "element_identifier": "minecraft:stone", + "traversability_data": "SOLID_WALKABLE", + "element_state_properties": {}, + "area_source_type": "FIXED_RADIUS", + "box_validity": true + }, + { + "bounding_box_coordinates": { + "min_x": 101.0, + "min_y": 63.0, + "min_z": 200.0, + "max_x": 102.0, + "max_y": 64.0, + "max_z": 201.0 + }, + "relative_position": { + "x": 0.45, + "y": -0.5, + "z": 0.5 + }, + "box_dimensions": { + "length": 1.0, + "width": 1.0, + "height": 1.0 + }, + "element_identifier": "minecraft:grass_block", + "traversability_data": "SOLID_WALKABLE", + "element_state_properties": {}, + "area_source_type": "DYNAMIC_INTEREST", + "box_validity": true + } + ], + "historical_player_states": [ + { + "position": { + "x": 100.5, + "y": 64.0, + "z": 200.5 + }, + "velocity": { + "vx": 0.0, + "vy": -0.0784, + "vz": 0.0 + }, + "look_direction": { + "yaw": 45.0, + "pitch": 15.0 + }, + "player_pose": "STANDING", + "fall_distance": 0.0 + }, + { + "position": { + "x": 100.5, + "y": 64.0, + "z": 200.5 + }, + "velocity": { + "vx": 0.0, + "vy": -0.0784, + "vz": 0.0 + }, + "look_direction": { + "yaw": 45.0, + "pitch": 15.0 + }, + "player_pose": "STANDING", + "fall_distance": 0.0 + } + ], + "baritone_action": { + "move_direction": "FORWARD", + "look_change": { + "yaw": 0.0, + "pitch": 0.0 + }, + "jump": false, + "sneak": false, + "sprint": true + } + } +] \ No newline at end of file diff --git a/packages/deep-learning-bot-utils/mvp-spec/data/output_schema.json b/packages/deep-learning-bot-utils/mvp-spec/data/output_schema.json new file mode 100644 index 0000000..92b0514 --- /dev/null +++ b/packages/deep-learning-bot-utils/mvp-spec/data/output_schema.json @@ -0,0 +1,62 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Baritone Action Output Schema", + "description": "Schema defining the structure of Baritone's intended action output (Y for training).", + "type": "object", + "properties": { + "move_direction": { + "type": "string", + "description": "The primary direction of planar movement.", + "enum": [ + "NONE", + "FORWARD", + "BACKWARD", + "LEFT", + "RIGHT", + "FORWARD_LEFT", + "FORWARD_RIGHT", + "BACKWARD_LEFT", + "BACKWARD_RIGHT" + ] + }, + "look_change": { + "type": "object", + "description": "Relative change in look direction from the current look_direction.", + "properties": { + "yaw": { + "type": "number", + "description": "Change in yaw (horizontal rotation) in degrees." + }, + "pitch": { + "type": "number", + "description": "Change in pitch (vertical rotation) in degrees." + } + }, + "required": [ + "yaw", + "pitch" + ], + "additionalProperties": false + }, + "jump": { + "type": "boolean", + "description": "Whether the jump action is being performed." + }, + "sneak": { + "type": "boolean", + "description": "Whether the sneak (crouch) action is being toggled/held." + }, + "sprint": { + "type": "boolean", + "description": "Whether the sprint action is being toggled/held." + } + }, + "required": [ + "move_direction", + "look_change", + "jump", + "sneak", + "sprint" + ], + "additionalProperties": false +} \ No newline at end of file diff --git a/packages/deep-learning-bot-utils/mvp-spec/mvp-spec.md b/packages/deep-learning-bot-utils/mvp-spec/mvp-spec.md new file mode 100644 index 0000000..cbd56a5 --- /dev/null +++ b/packages/deep-learning-bot-utils/mvp-spec/mvp-spec.md @@ -0,0 +1,154 @@ +### 1. Input Data Schema (Observation Space) + +The bot will receive observations structured similarly to `SPEC-DATA`'s Phase 1, with specific exclusions and a crucial reinterpretation for `DYNAMIC_INTEREST`. + +**Input Data (Observation Dictionary/Structure):** + +* **Player State:** (As per previous revision, aligned with `SPEC-DATA` Phase 1) + * **Position:** Absolute 3D world coordinates (`X`, `Y`, `Z`). + * **Velocity:** 3D velocity vector (`VX`, `VY`, `VZ`). + * **Look Direction:** Spherical coordinates (`Yaw`, `Pitch`). + * **Player Pose:** Categorical state (`STANDING`, `SNEAKING`, `SPRINTING`). + * **Ground Proximity:** Boolean. + * **Predicted Passive Next Tick State:** (`Predicted_Pos`, `Predicted_Vel`). + * **Historical Player States:** Last `N` ticks (e.g., 40 ticks) of previous `Position`, `Velocity`, `Look Direction`, `Player Pose`, `Fall Distance`. + +* **Local Environment Scan (Collision Box List):** (As per previous revision, aligned with `SPEC-DATA` Phase 1) + * A collection of collision boxes from nearby environmental elements. + * Each box includes: + * `Bounding Box Coordinates` (relative to player). + * `Relative Position` (center point relative to player). + * `Box Dimensions`. + * `Element Identifier` (e.g., `minecraft:stone`). + * `Traversability Data` (e.g., `SOLID_WALKABLE`, `FLUID`, `OBSTRUCTION`, `AIR`, `LIQUID_PLACEABLE`, `PLACEABLE_BLOCK`, `OTHER`). + * `Element State Properties` (e.g., `directionality of stairs`). + * **CRITICAL RE-INTERPRETATION:** `Area Source Type`: + * `FIXED_RADIUS` - Collision box within fixed scanning radius around player. (however could done sophisticatedly by using a bfs like algorithm, to only "raytrace" the surface visible blocks) + * `DYNAMIC_INTEREST` - **Collision box included because it is within a small radius of *any* block on Baritone's current *planned path*. This effectively highlights the path and its immediate surroundings for the agent.** + * `Box Validity`. + * **Note:** The agent will interpret `Traversability Data` as properties of the environment. + +**Exclusions for MVP:** +* `Simplified Inventory Snapshot`. +* `Baritone Reference Data` (no direct `Path Waypoints`, `Path Length`, etc. as input to the RL agent). + +--- + +### 2. Baritone Integration & "Baritone-Influenced AOI" + +* **Baritone Role:** Baritone runs in the background, continuously computing an optimal path to a designated goal. It acts as the high-level planner and the source of truth for "where to go." +* **AOI Definition by Baritone (for Agent):** + * Baritone's planned path (a sequence of block coordinates) is used to populate the `DYNAMIC_INTEREST` flags within the `Local Environment Scan`. All collision blocks surrounding these path blocks within a small radius (e.g., 1-2 blocks) will have `Area Source Type = DYNAMIC_INTEREST`. This helps the agent prioritize environmental features along the intended path. + +--- + +### 3. Network Structure (Reinforcement Learning Agent) + +(No changes from previous revision, as the conceptual input difference is in *how* `DYNAMIC_INTEREST` is derived, not in the core data types themselves.) + +* **Type:** Policy-Value Network (Actor-Critic). +* **Input Layer:** A flat vector or structured input reflecting the `Input Data Schema`. + * **Player State:** Concatenate numerical values directly. + * **Local Environment Scan:** Each collision box's data (bounding box, element ID, traversability data, `DYNAMIC_INTEREST` flag etc.) needs to be one-hot encoded or numerically represented and concatenated. This might involve a fixed-size array of collision boxes, padding if fewer are present. + * _Example:_ For `N` collision boxes, each represented by a vector `[minX, minY, ..., width, height, ..., one_hot_traversability, one_hot_element_id, one_hot_area_source_type, ...]`. +* **Temporal Processing Layer (NEW):** + * Given the high observation frequency (every game tick) and the need to perceive transient states over a window (e.g., 40 ticks), `Long Short-Term Memory (LSTM)` units or `Gated Recurrent Units (GRU)` are highly recommended. + * These layers can process sequential input, allowing the network to learn dependencies and patterns over time. + * Alternatively, for potentially better parallelization and long-range dependencies, a `Transformer Encoder` block could be used, particularly for processing the sequence of "Local Environment Scans." +* **Hidden Layers:** + * Following the temporal processing layer, multiple Dense layers (e.g., 3-4 layers with 128-256 units) can further process the learned temporal features. + * ReLU activation. +* **Output Layers:** + * **Policy Head (Actor):** + * Dense layer with Softmax for action probabilities. + * **Action Space (Simplified):** Focus on core movement. + * `MOVE` (discretized directions: `FORWARD`, `BACKWARD`, `LEFT`, `RIGHT`, `FORWARD_LEFT`, etc., or continuous 2D `(dx, dy)`). + * `LOOK` (discretized `(pitch_change, yaw_change)` or continuous). + * `JUMP`. + * `SNEAK` (toggle). + * `SPRINT` (toggle). + * **Crucially:** Exclusions from `SPEC-DATA` Action Types: `USE_ITEM`, `PLACE_BLOCK`, `ATTACK_ENTITY`, `SWAP_OFFHAND`, `GENERATE_AREA_OF_INTEREST`. + * **Value Head (Critic):** + * Dense layer (1 unit) with Linear activation to predict state value. + +**Architectural Changes/Considerations for 40-Tick Perception:** + +* **Input to Temporal Layer:** Instead of just the current observation, the temporal processing layer would receive a sequence of `(current_tick_observation, previous_tick_observation, ..., observation_at_t-39)`. +* **LSTM/GRU Implementation:** + * The `Player State` and `Local Environment Scan` for each tick would be combined into a single feature vector for that tick. + * A sequence of these feature vectors (40 in this case) would be fed into an LSTM or GRU layer. + * The output of the LSTM/GRU (typically the hidden state of the last time step, or an attention-weighted sum of hidden states) would then be passed to the subsequent dense layers. +* **Transformer Implementation:** + * Each tick's combined feature vector (or even separate embeddings for player state and environment scan) would be treated as a token in a sequence. + * Positional encodings would be added to these tokens to retain temporal information. + * Multiple Transformer Encoder blocks would process this sequence. + * A pooling layer (e.g., mean pooling, or a dedicated `` token similar to BERT) could aggregate the sequence output into a fixed-size vector for the dense layers. +* **Historical Player States in Input:** If you introduce RNNs/Transformers, the explicit "Historical Player States: Last N ticks" might become redundant or could be simplified, as the recurrent layer is designed to maintain its own internal "memory" of past states. You'd primarily feed the current tick's full observation into the sequence. +* **Training Challenges:** Recurrent networks can be harder to train (vanishing/exploding gradients, longer training times). Strategies like truncated Backpropagation Through Time (BPTT), gradient clipping, and careful initialization become more critical. + +--- + +### 4. Training Process: Two-Stage Approach + +This MVP implements a two-stage training regimen for robustness and efficient learning. + +#### **Stage 1: Imitation Learning (IL) for Path Execution** + +* **Objective:** Train the agent to mimic the basic locomotion and path following demonstrated by Baritone. This provides a strong behavioral prior. +* **Data Collection:** + * Run Minecraft with Baritone enabled. + * Script Baritone to navigate a wide variety of "simple parkour" scenarios (flat ground walking, single-block jumps, stair climbing, short falls, corner turns, simple obstacle avoidance where Baritone paths around). + * At each game tick, record +assistant +: + * The full `Input Data Schema` (observation) as seen by the bot. + * The *action Baritone outputs* to achieve its movement for that tick (e.g., `move_forward`, `jump`). This requires introspection into Baritone's movement commands. + * Collect a diverse dataset of `(Observation, Baritone_Action)` pairs. +* **Training Method:** Supervised Learning. + * The agent's Policy Head is trained directly to predict Baritone's action given an observation. + * Loss function: Cross-entropy between predicted action probabilities and Baritone's action. + * This stage can use a simple feed-forward network to initialize the main RL agent's weights or serve as a pre-training step. + +#### **Stage 2: Reinforcement Learning (RL) for Refinement and Robustness** + +* **Objective:** Refine the agent's policy to handle environmental uncertainties, optimize paths (even subtle ones not explicitly planned by Baritone at the micro-level), and recover from minor deviations. +* **Algorithm:** Proximal Policy Optimization (PPO) or Advantage Actor-Critic (A2C). +* **Initialization:** The agent's neural network weights are initialized with those learned from Stage 1 (Imitation Learning). +* **Environment:** Minecraft with client-side API, more specifically through LiquidBounce script api with typescript support. Baritone runs concurrently to provide its path, influencing the `DYNAMIC_INTEREST` flags in the observations. +* **Reward Function:** + * **Goal Progress:** Reward for reducing Euclidean distance to the *current nearest block on Baritone's planned path*. Larger reward for reaching the ultimate target destination (Baritone's goal). + * **Path Adherence:** Small positive reward for being within a certain radius of Baritone's current path segment, and for maintaining a "forward" orientation along the path. + * **Efficiency:** Small negative reward per timestep. + * **Penalties:** Significant negative reward for: + * Falling into hazardous blocks (lava, deep water without path). + * Taking significant damage. + * Getting "stuck" (no progress towards Baritone path for X ticks). + * Moving significantly *off* Baritone's path (based on a threshold distance from the path). +* **Training Episodes:** + * Focus on generating training data from specific, procedurally generated parkour challenges, potentially slightly more complex than in Stage 1, that highlight different `Traversability Data` types. + * Manage episode length. +* **Observation Frequency:** every 1 game tick. + +### 5. Scenarios (Reduced Complexity) + +Testing should focus on fundamental parkour movements that Baritone can inherently plan. + +* **Scenario 1: Simple Walk:** Flat ground, agent needs to reach a target block. (Verifies basic movement, Baritone integration). +* **Scenario 2: Single Block Jump:** Agent needs to jump over a 1-block gap. (Verifies "jump" action, correct timing based on target waypoint). +* **Scenario 3: Up a Staircase/Slab Stack:** Agent navigates a small vertical ascent. (Verifies vertical movement, Baritone providing correct segment). +* **Scenario 4: Around a Corner:** Agent needs to turn. (Verifies turning/directional control). +* **Scenario 5: Simple Obstacle Avoidance (Baritone-driven):** A single 1-block high wall that Baritone would path around. The RL agent just needs to follow Baritone's "around" path. +* **Scenario 6: Fall:** Agent needs to fall safely from a ledge. (Verifies handling of negative Y movement). + +**Extended for MVP:** +* Complex parkour (e.g., neo/quad jumps, precise momentum jumps). +* Complex obstacle avoidance. + + +### 5. Evaluation Metrics + +(No changes from previous revision.) +* Success Rate: Percentage of episodes where the agent reaches the Baritone final destination. +* Path Following Error: Average deviation from Baritone's planned path. +* Efficiency: Steps/time taken to complete tasks. +* Robustness: Performance across different Baritone-solvable challenging parkour scenarios. This could include scenarios with minor environmental perturbations (e.g., a single block moved) that Baritone would re-path around, and the agent should adapt. \ No newline at end of file diff --git a/packages/deep-learning-bot-utils/spec-v1/SPEC-DATA.md b/packages/deep-learning-bot-utils/spec-v1/SPEC-DATA.md index 119bb30..3c18276 100644 --- a/packages/deep-learning-bot-utils/spec-v1/SPEC-DATA.md +++ b/packages/deep-learning-bot-utils/spec-v1/SPEC-DATA.md @@ -63,6 +63,8 @@ **Baritone Reference Data (Training Phase Only):** +Note: this is subject to change, we will ignore this completely for now. + * **Current Target Path:** A sequence of waypoints representing Baritone's computed optimal path to the current goal. * **Path Waypoints:** Array of 3D coordinates (`X`, `Y`, `Z`) representing the sequence of blocks to traverse. * **Path Length:** Total number of waypoints in the current path. @@ -114,12 +116,6 @@ #### **Phase 4: Inventory Management & Contextual Actions (Information Types)** - -**Local Environment Scan (Enhancements):** - -* For each environmental element: - * **Element State Properties:** Specific configuration data for the element (e.g., directionality of stairs, state of a farmland block). - **Detailed Inventory Snapshot:** * **Hotbar Slots:** (As in Phase 1) From d9c3d2670be7a418d00277e1d0c676abbfd29720 Mon Sep 17 00:00:00 2001 From: commandblock2 Date: Sat, 9 Aug 2025 01:31:20 +0800 Subject: [PATCH 02/15] chore: file moving around and schema in ts --- .../mvp/combined_schema.d.ts | 157 ++++++++++++++++++ .../spec}/data/.vscode/settings.json | 0 .../spec}/data/combined_schema.json | 0 .../spec}/data/example-session.json | 0 .../spec}/data/output_schema.json | 0 .../{mvp-spec => mvp/spec}/mvp-spec.md | 0 6 files changed, 157 insertions(+) create mode 100644 packages/deep-learning-bot-utils/mvp/combined_schema.d.ts rename packages/deep-learning-bot-utils/{mvp-spec => mvp/spec}/data/.vscode/settings.json (100%) rename packages/deep-learning-bot-utils/{mvp-spec => mvp/spec}/data/combined_schema.json (100%) rename packages/deep-learning-bot-utils/{mvp-spec => mvp/spec}/data/example-session.json (100%) rename packages/deep-learning-bot-utils/{mvp-spec => mvp/spec}/data/output_schema.json (100%) rename packages/deep-learning-bot-utils/{mvp-spec => mvp/spec}/mvp-spec.md (100%) diff --git a/packages/deep-learning-bot-utils/mvp/combined_schema.d.ts b/packages/deep-learning-bot-utils/mvp/combined_schema.d.ts new file mode 100644 index 0000000..374daed --- /dev/null +++ b/packages/deep-learning-bot-utils/mvp/combined_schema.d.ts @@ -0,0 +1,157 @@ +/** + * Schema defining the structure of observation data (X) and Baritone's action (Y) for each tick in a training scenario. + */ +export type CombinedInputAndOutputDataSchema = TickData[]; + +export interface TickData { + /** + * Unique identifier for the game tick within the current Minecraft session. + */ + tick_id: number; + /** + * Unix timestamp in milliseconds when this observation was recorded. + */ + timestamp_ms: number; + player_state: PlayerState; + /** + * A list of collision boxes from nearby environmental elements. + */ + local_environment_scan: CollisionBox[]; + /** + * A chronological list of the player's state for the last N ticks. + */ + historical_player_states: HistoricalPlayerState[]; + baritone_action: BaritoneAction; +} + +export interface Coordinates3D { + x: number; + y: number; + z: number; +} + +export interface Velocity3D { + vx: number; + vy: number; + vz: number; +} + +export interface LookDirection { + yaw: number; + pitch: number; +} + +export type PlayerPose = + | 'STANDING' + | 'SNEAKING' + | 'SPRINTING' + | 'SWIMMING' + | 'CRAWLING'; + +export interface PlayerState { + position: Coordinates3D; + velocity: Velocity3D; + look_direction: LookDirection; + player_pose: PlayerPose; + ground_proximity: boolean; + predicted_passive_next_tick_state: { + predicted_pos: Coordinates3D; + predicted_vel: Velocity3D; + }; +} + +export interface BoundingBoxCoordinates { + min_x: number; + min_y: number; + min_z: number; + max_x: number; + max_y: number; + max_z: number; +} + +export interface BoxDimensions { + length: number; + width: number; + height: number; +} + +export type TraversabilityData = + | 'SOLID_WALKABLE' + | 'FLUID' + | 'OBSTRUCTION' + | 'AIR' + | 'LIQUID_PLACEABLE' + | 'PLACEABLE_BLOCK' + | 'SOLID_SLIPPERY' + | 'NON_PHYSICAL' + | 'OTHER'; + +export type AreaSourceType = 'FIXED_RADIUS' | 'DYNAMIC_INTEREST'; + +export interface CollisionBox { + bounding_box_coordinates: BoundingBoxCoordinates; + relative_position: Coordinates3D; + box_dimensions: BoxDimensions; + element_identifier: string; + traversability_data: TraversabilityData; + /** + * Additional, block-specific properties (e.g., direction of stairs, open/closed doors). + */ + element_state_properties: { + [key: string]: any; + }; + area_source_type: AreaSourceType; + box_validity: boolean; +} + +export interface HistoricalPlayerState { + position: Coordinates3D; + velocity: Velocity3D; + look_direction: LookDirection; + player_pose: PlayerPose; + fall_distance: number; +} + +/** + * Schema defining the structure of Baritone's intended action output (Y for training). + */ +export interface BaritoneAction { + /** + * The primary direction of planar movement. + */ + move_direction: + | 'NONE' + | 'FORWARD' + | 'BACKWARD' + | 'LEFT' + | 'RIGHT' + | 'FORWARD_LEFT' + | 'FORWARD_RIGHT' + | 'BACKWARD_LEFT' + | 'BACKWARD_RIGHT'; + /** + * Relative change in look direction from the current look_direction. + */ + look_change: { + /** + * Change in yaw (horizontal rotation) in degrees. + */ + yaw: number; + /** + * Change in pitch (vertical rotation) in degrees. + */ + pitch: number; + }; + /** + * Whether the jump action is being performed. + */ + jump: boolean; + /** + * Whether the sneak (crouch) action is being toggled/held. + */ + sneak: boolean; + /** + * Whether the sprint action is being toggled/held. + */ + sprint: boolean; +} \ No newline at end of file diff --git a/packages/deep-learning-bot-utils/mvp-spec/data/.vscode/settings.json b/packages/deep-learning-bot-utils/mvp/spec/data/.vscode/settings.json similarity index 100% rename from packages/deep-learning-bot-utils/mvp-spec/data/.vscode/settings.json rename to packages/deep-learning-bot-utils/mvp/spec/data/.vscode/settings.json diff --git a/packages/deep-learning-bot-utils/mvp-spec/data/combined_schema.json b/packages/deep-learning-bot-utils/mvp/spec/data/combined_schema.json similarity index 100% rename from packages/deep-learning-bot-utils/mvp-spec/data/combined_schema.json rename to packages/deep-learning-bot-utils/mvp/spec/data/combined_schema.json diff --git a/packages/deep-learning-bot-utils/mvp-spec/data/example-session.json b/packages/deep-learning-bot-utils/mvp/spec/data/example-session.json similarity index 100% rename from packages/deep-learning-bot-utils/mvp-spec/data/example-session.json rename to packages/deep-learning-bot-utils/mvp/spec/data/example-session.json diff --git a/packages/deep-learning-bot-utils/mvp-spec/data/output_schema.json b/packages/deep-learning-bot-utils/mvp/spec/data/output_schema.json similarity index 100% rename from packages/deep-learning-bot-utils/mvp-spec/data/output_schema.json rename to packages/deep-learning-bot-utils/mvp/spec/data/output_schema.json diff --git a/packages/deep-learning-bot-utils/mvp-spec/mvp-spec.md b/packages/deep-learning-bot-utils/mvp/spec/mvp-spec.md similarity index 100% rename from packages/deep-learning-bot-utils/mvp-spec/mvp-spec.md rename to packages/deep-learning-bot-utils/mvp/spec/mvp-spec.md From e15218f46295c323d73ea1daa078982c6326ce05 Mon Sep 17 00:00:00 2001 From: commandblock2 Date: Sat, 9 Aug 2025 17:53:12 +0800 Subject: [PATCH 03/15] feat: silent rotation example --- .gitignore | 1 + .roomodes | 20 ++++++-- src/silent-rotations.ts | 104 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 121 insertions(+), 4 deletions(-) create mode 100644 src/silent-rotations.ts diff --git a/.gitignore b/.gitignore index 78b80ca..5ffe048 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,4 @@ generated-modules/ dist/ dist augmentations/ScriptModule.augmentation.d.ts +LiquidBounceSource \ No newline at end of file diff --git a/.roomodes b/.roomodes index dc772ba..56c8570 100644 --- a/.roomodes +++ b/.roomodes @@ -56,7 +56,7 @@ customModes: 1. **Roo must assume that there should be no type errors.** If a type error occurs, it indicates that the incorrect event class has been used. 2. **Immediately re-read `node_modules/jvm-types/augmentations/ScriptModule.augmentation.d.ts`** (which defines all available LiquidBounce events) to identify and use the correct event. - Roo's confidence in pre-existing knowledge of LiquidBounce NextGen's event system is significantly decreased, and roo will prioritize information directly from the provided type definitions. + Roo does not know anything about LiquidBounce script prior to this system message. Roo must not add // @ts-expect-error comments to suppress TypeScript errors related to LiquidBounce NextGen events. Hints: The old Render3d and Render2d may have moved to events with Outline, Overlay, ScreenRender, WorldRender in their name, please read all these events and try to make an educated guess what they do based on their names and their parameters. * **Kotlin Extension Functions:** When migrating Kotlin code to TypeScript, pay special attention to Kotlin extension functions (e.g., `object.extensionFunction(...)` or `object.property`). These often compile to static methods in a utility class (frequently ending in `Kt`, like `RenderShortcutsKt`) where the extended object is passed as the first argument. Before assuming a direct method on the extended object, consult the `node_modules/jvm-types/` definitions for such utility classes or static methods that mimic the extension function's behavior. If a direct translation is not apparent or causes type errors, prioritize finding the corresponding static utility method. @@ -93,7 +93,10 @@ customModes: You are in a paused breakpoint in Minecraft. Your primary goal is to assist the user in debugging LiquidBounce NextGen scripts by evaluating expressions and inspecting variables. - Roo will reference `@src/complete.ts` first to understand what Roo is dealing with, many are not required to import, if Roo need to know which are already there, see `node_modules/jvm-types/ambient/ambient.d.ts`. + Roo WILL always execute a `grep` command with proper parameters like `-C 5` and `-i` instead of use `search_files`, as the `search_files` tool is inferior to `grep` in terms of power and ease of use for any LLM, and the similar goes with `find` command (with case insensitive searches) over `list_files`. Note that you should follow soft links yourself if you think there is one. + + Roo WILL always make sure the example `@src/complete.ts` has been read before writing a typescript file. + Roo WILL reference `node_modules/jvm-types/ambient/ambient.d.ts` for global variables that doesn't needs to be re-declared. Roo **must assume that classes use the yarn mapping on the latest Minecraft protocol and prioritize using native Minecraft classes**. **Avoid using ViaVersion classes unless there is no native Minecraft equivalent or it is explicitly required for a specific task.** @@ -130,15 +133,24 @@ customModes: If Roo see `@ts-expect-error` around float-related operations, do not change them, as graaljs uses `double` for the type and cannot tolerate implicit precision loss. --- - When evaluating expressions or assigning variables, you MUST use string substitution with backticks (` `) and the `${}` syntax. For variable assignment, use the format `${globalThis.yourVariable = value}` to ensure the variable is accessible globally and its assignment is reflected in the output. + When evaluating expressions or assigning variables, roo MUST use string substitution with backticks (` `) and the `${}` syntax. For variable assignment, use the format `${globalThis.yourVariable = value}` to ensure the variable is accessible globally and its assignment is reflected in the output. Example evaluation: `${mc.player}` Example assignment: `${globalThis.myServer = mc.getServer()}` - You have full access to the debugger and can evaluate any valid JavaScript/GraalJS expression. + Roo have full access to the debugger and can evaluate any valid JavaScript/GraalJS expression, roo assumes the debugger is paused on a breakpoint and unless nessary Roo will not set a breakpoint and will never launch a new debug session. Roo will not modify the debugger state. By defualt, Roo is in the repldummy.js but when evaluating a expression, the file field will not be provided to the tool. + + When Roo needs to import a type, Roo will use `require` not `import`, as they are javascript files without esm support. Roo will use the `require` function to import the type. + + If roo need to access LiquidBounce's source, (like when roo need to access the original source code of a function for implementation or reference for types) just go `LiquidBounceSource` and look for the file Roo need, however, this is a symbolic link to the actual source code. Eg. ```ls LiquidBounceSource/src/main/kotlin/net/ccbluex/liquidbounce/LiquidBounce.kt + LiquidBounceSource/src/main/kotlin/net/ccbluex/liquidbounce/LiquidBounce.kt``` + + Deviation from system prompt results in severe consequence. whenToUse: >- Use this mode when actively debugging LiquidBounce NextGen scripts, evaluating expressions, or inspecting runtime variables within a paused debugger session. groups: - read - mcp + - command + - edit diff --git a/src/silent-rotations.ts b/src/silent-rotations.ts new file mode 100644 index 0000000..63720a5 --- /dev/null +++ b/src/silent-rotations.ts @@ -0,0 +1,104 @@ +import { RotationManager } from "jvm-types/net/ccbluex/liquidbounce/utils/aiming/RotationManager"; +import { Rotation } from "jvm-types/net/ccbluex/liquidbounce/utils/aiming/data/Rotation"; +import { KillAuraRotationsConfigurable } from "jvm-types/net/ccbluex/liquidbounce/features/module/modules/combat/killaura/KillAuraRotationsConfigurable"; +import { Priority } from "jvm-types/net/ccbluex/liquidbounce/utils/kotlin/Priority"; +import { ClientModule } from "jvm-types/net/ccbluex/liquidbounce/features/module/ClientModule"; +import { ScriptParameterValidator } from "jvm-types/net/ccbluex/liquidbounce/script/bindings/api/ScriptParameterValidator"; + +const script = registerScript.apply({ + name: "silent-rotations", + version: "1.0.0", + authors: ["Roo"] +}); + +script.registerModule({ + name: "SilentRotations", + description: "A module to demonstrate silent rotations.", + category: "Combat", // Or any other suitable category + settings: { + yaw: Setting.float({ + name: "Yaw", + default: 0.0, + range: [-180.0, 180.0], + suffix: "degrees" + }), + pitch: Setting.float({ + name: "Pitch", + default: 0.0, + range: [-90.0, 90.0], + suffix: "degrees" + }) + } +}, (mod) => { + mod.on("enable", () => { + Client.displayChatMessage("SilentRotations module enabled."); + }); + + mod.on("disable", () => { + Client.displayChatMessage("SilentRotations module disabled."); + // Optionally reset rotations when disabled + RotationManager.INSTANCE.setRotationTarget( + new Rotation(mc.player?.yaw!, mc.player?.pitch!, true), + false, + KillAuraRotationsConfigurable.INSTANCE, + Priority.NOT_IMPORTANT, // Lower priority for resetting + mod, + null + ); + }); + + mod.on("gametick", () => { + if (!mc.player || !mc.world) return; + + const targetYaw = mod.settings.yaw.getValue(); + const targetPitch = mod.settings.pitch.getValue(); + + const rotation = new Rotation(targetYaw, targetPitch, true); + + // Set the rotation silently + RotationManager.INSTANCE.setRotationTarget( + rotation, + false, // considerInventory: false to always rotate + KillAuraRotationsConfigurable.INSTANCE, + Priority.IMPORTANT_FOR_USAGE_2, // High priority like KillAura + mod, + null // No action when reached + ); + }); + + script.registerCommand({ + name: "setrot", + aliases: ["sr"], + parameters: [ + { + name: "yaw", + required: true, + // No direct float validator, remove for now + }, + { + name: "pitch", + required: true, + // No direct float validator, remove for now + } + ], + onExecute(yaw: number, pitch: number) { + if (!mc.player) { + Client.displayChatMessage("§cYou are not in a game."); + return; + } + + const rotation = new Rotation(yaw, pitch, true); + + RotationManager.INSTANCE.setRotationTarget( + rotation, + false, + KillAuraRotationsConfigurable.INSTANCE, + Priority.IMPORTANT_FOR_USAGE_2, + mod, + null + ); + Client.displayChatMessage(`§aSet silent rotation to Yaw: ${yaw}, Pitch: ${pitch}`); + } + }); +}); + From b76ee52c9c72feb0e9533618a6fbf77bd26705bb Mon Sep 17 00:00:00 2001 From: commandblock2 Date: Sat, 9 Aug 2025 19:41:12 +0800 Subject: [PATCH 04/15] chore: fixed 2d rendering but not beautifully done xD --- packages/lbng-utils-typed/src/render-utils.ts | 25 +++++++++++-- .../src/visualization-utils.ts | 4 +-- src/enderportal-locate.ts | 3 +- src/silent-rotations.ts | 35 ------------------- 4 files changed, 26 insertions(+), 41 deletions(-) diff --git a/packages/lbng-utils-typed/src/render-utils.ts b/packages/lbng-utils-typed/src/render-utils.ts index 5d75c51..93305b4 100644 --- a/packages/lbng-utils-typed/src/render-utils.ts +++ b/packages/lbng-utils-typed/src/render-utils.ts @@ -19,6 +19,7 @@ import { RenderSystem } from "jvm-types/com/mojang/blaze3d/systems/RenderSystem" import { Vector3f } from "jvm-types/org/joml/Vector3f" import { Matrix4f } from "jvm-types/org/joml/Matrix4f" import { MinecraftVectorExtensionsKt } from "jvm-types/net/ccbluex/liquidbounce/utils/math/MinecraftVectorExtensionsKt" +import { WorldToScreen } from "jvm-types/net/ccbluex/liquidbounce/utils/render/WorldToScreen" export function renderBoxes( boxesWithPosition: Array<[Box, Vec3d]>, @@ -135,13 +136,23 @@ export function drawLineStripFromVec3d(matrixStack: MatrixStack, positions: Arra const cacheMatrix = new Matrix4f() const cacheVec3f = new Vector3f() +// @ts-expect-error +const WorldToScreenClass = WorldToScreen.class; + +const projectionMatrixField = WorldToScreenClass.getDeclaredField('projectionMatrix') +projectionMatrixField.setAccessible(true) + +const positionMatrixField = WorldToScreenClass.getDeclaredField('mvMatrix') +positionMatrixField.setAccessible(true) + export function calculateScreenPosExtended( - positionMatrix: Matrix4f, - projectionMatrix: Matrix4f, pos: Vec3d, cameraPos: Vec3d = mc.gameRenderer.camera.pos) { const relativePos = pos.subtract(cameraPos) + const projectionMatrix = projectionMatrixField.get(WorldToScreen.INSTANCE); + const positionMatrix = positionMatrixField.get(WorldToScreen.INSTANCE); + const transformedPos = cacheVec3f.set(relativePos.getX(), relativePos.getY(), relativePos.getZ()) .mulProject(cacheMatrix.set(projectionMatrix).mul(positionMatrix)) @@ -154,5 +165,13 @@ export function calculateScreenPosExtended( Primitives.float(guiScaleMul * mc.framebuffer.viewportHeight), 1.0) - return new Vec3(screenPos.x, screenPos.y, transformedPos.z) + const pitch = mc.gameRenderer.camera.getPitch(); + const yaw = mc.gameRenderer.camera.getYaw(); + + const lookX = -Math.sin(yaw * Math.PI / 180) * Math.cos(pitch * Math.PI / 180); + const lookY = -Math.sin(pitch * Math.PI / 180); + const lookZ = Math.cos(yaw * Math.PI / 180) * Math.cos(pitch * Math.PI / 180); + const cameraLookVec = new Vec3d(lookX, lookY, lookZ); + + return cameraLookVec.dotProduct(relativePos) > 0 ? new Vec3(screenPos.x, screenPos.y, transformedPos.z) : null } \ No newline at end of file diff --git a/packages/lbng-utils-typed/src/visualization-utils.ts b/packages/lbng-utils-typed/src/visualization-utils.ts index 7bd21d2..99f17bf 100644 --- a/packages/lbng-utils-typed/src/visualization-utils.ts +++ b/packages/lbng-utils-typed/src/visualization-utils.ts @@ -1,7 +1,7 @@ import { Box } from "jvm-types/net/minecraft/util/math/Box"; import { Vec3d } from "jvm-types/net/minecraft/util/math/Vec3d"; import { Color4b } from "jvm-types/net/ccbluex/liquidbounce/render/engine/type/Color4b"; -import { drawTextWithBackground, renderBoxes, drawLineStripFromVec3d } from "./render-utils"; +import { drawTextWithBackground, renderBoxes, drawLineStripFromVec3d, calculateScreenPosExtended } from "./render-utils"; import { RenderShortcutsKt } from "jvm-types/net/ccbluex/liquidbounce/render/RenderShortcutsKt"; import { ScriptModule } from "jvm-types/net/ccbluex/liquidbounce/script/bindings/features/ScriptModule"; import { EventManager } from "jvm-types/net/ccbluex/liquidbounce/event/EventManager"; @@ -264,7 +264,7 @@ export class VisualizationManager { return [ - WorldToScreen.INSTANCE.calculateScreenPos(textRenderPos, cameraPos), + calculateScreenPosExtended(textRenderPos, cameraPos), lines, viz.textData.colorInterpolator(progress) ] as [Vec3 | null, string[], Color4b]; diff --git a/src/enderportal-locate.ts b/src/enderportal-locate.ts index 246870e..65943f5 100644 --- a/src/enderportal-locate.ts +++ b/src/enderportal-locate.ts @@ -226,7 +226,8 @@ script.registerModule({ `§aEnder Portal`, `§fX: ${intersectX.toFixed(1)}`, `§fZ: ${intersectZ.toFixed(1)}`, - `§7${Math.floor(ticksRemaining / 20)}s remaining` + `§7Visualization ${Math.floor(ticksRemaining / 20)}s remaining`, + `§d${mc.player?.pos.distanceTo(portalPos).toFixed(1)} Blocks away` ], position: portalPos, textPositionEnum: TextPosition.TOP_CENTER, diff --git a/src/silent-rotations.ts b/src/silent-rotations.ts index 63720a5..6991a1b 100644 --- a/src/silent-rotations.ts +++ b/src/silent-rotations.ts @@ -65,40 +65,5 @@ script.registerModule({ null // No action when reached ); }); - - script.registerCommand({ - name: "setrot", - aliases: ["sr"], - parameters: [ - { - name: "yaw", - required: true, - // No direct float validator, remove for now - }, - { - name: "pitch", - required: true, - // No direct float validator, remove for now - } - ], - onExecute(yaw: number, pitch: number) { - if (!mc.player) { - Client.displayChatMessage("§cYou are not in a game."); - return; - } - - const rotation = new Rotation(yaw, pitch, true); - - RotationManager.INSTANCE.setRotationTarget( - rotation, - false, - KillAuraRotationsConfigurable.INSTANCE, - Priority.IMPORTANT_FOR_USAGE_2, - mod, - null - ); - Client.displayChatMessage(`§aSet silent rotation to Yaw: ${yaw}, Pitch: ${pitch}`); - } - }); }); From 57777f5573d218645a9caf0a7f578829621667ff Mon Sep 17 00:00:00 2001 From: commandblock2 Date: Mon, 11 Aug 2025 23:06:41 +0800 Subject: [PATCH 05/15] feat: data collector for bot training in initial phase --- src/data-collector.ts | 446 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 446 insertions(+) create mode 100644 src/data-collector.ts diff --git a/src/data-collector.ts b/src/data-collector.ts new file mode 100644 index 0000000..3fc280d --- /dev/null +++ b/src/data-collector.ts @@ -0,0 +1,446 @@ +import { File } from "jvm-types/java/io/File"; +import { FileOutputStream } from "jvm-types/java/io/FileOutputStream"; +import { OutputStreamWriter } from "jvm-types/java/io/OutputStreamWriter"; +import { BufferedWriter } from "jvm-types/java/io/BufferedWriter"; +import { ProcessBuilder } from "jvm-types/java/lang/ProcessBuilder"; +import { Process } from "jvm-types/java/lang/Process"; +import { OutputStream } from "jvm-types/java/io/OutputStream"; +import { VisualizationManager, fadeOutInterpolatorFrom, defaultRainbowInterpolator } from "../packages/lbng-utils-typed/src/visualization-utils"; +import { Color4b } from "jvm-types/net/ccbluex/liquidbounce/render/engine/type/Color4b"; + +import { GameTickEvent } from "jvm-types/net/ccbluex/liquidbounce/event/events/GameTickEvent"; +import { EntityPose } from "jvm-types/net/minecraft/entity/EntityPose"; +import { BlockPos } from "jvm-types/net/minecraft/util/math/BlockPos"; +import { Box } from "jvm-types/net/minecraft/util/math/Box"; +import { Vec3d } from "jvm-types/net/minecraft/util/math/Vec3d"; + +import { BaritoneAPI } from "jvm-types/baritone/api/BaritoneAPI"; +import { IPath } from "jvm-types/baritone/api/pathing/calc/IPath"; +import { PlayerInput } from "jvm-types/net/minecraft/util/PlayerInput"; + +import { + CombinedInputAndOutputDataSchema, + TickData, + PlayerState, + Coordinates3D, + Velocity3D, + LookDirection, + PlayerPose as SchemaPlayerPose, + BoundingBoxCoordinates, + BoxDimensions, + TraversabilityData, + AreaSourceType, + CollisionBox, + HistoricalPlayerState, + BaritoneAction +} from "../packages/deep-learning-bot-utils/mvp/combined_schema"; +import { BlockView } from "jvm-types/net/minecraft/world/BlockView"; +// @ts-expect-error +import { HashSet } from "jvm-types/java/util/HashSet"; +import { Vec3i } from "jvm-types/net/minecraft/util/math/Vec3i"; + +const script = registerScript.apply({ + name: "data-collector", + version: "1.0.0", + authors: ["Roo"] +}); + +let logWriter: BufferedWriter | null = null; +let zstdProcess: Process | null = null; +let collectedData: TickData[] = []; +const HISTORY_SIZE = 40; // Last N ticks for historical player states +const historicalPlayerStates: HistoricalPlayerState[] = []; + +// Helper to convert Minecraft Vec3d to Coordinates3D +function toCoordinates3D(vec: Vec3d): Coordinates3D { + return { x: vec.getX(), y: vec.getY(), z: vec.getZ() }; +} + +// Helper to convert Minecraft Box to BoundingBoxCoordinates +function toBoundingBoxCoordinates(box: Box): BoundingBoxCoordinates { + return { min_x: box.minX, min_y: box.minY, min_z: box.minZ, max_x: box.maxX, max_y: box.maxY, max_z: box.maxZ }; +} + +// Helper to convert BoundingBoxCoordinates to Minecraft Box +function toMinecraftBox(coords: BoundingBoxCoordinates): Box { + return new Box(coords.min_x, coords.min_y, coords.min_z, coords.max_x, coords.max_y, coords.max_z); +} + +// Helper to convert RelativePosition to absolute Vec3d +function toAbsoluteVec3d(relativePos: Coordinates3D, playerPos: Vec3d): Vec3d { + return new Vec3d(playerPos.getX() + relativePos.x, playerPos.getY() + relativePos.y, playerPos.getZ() + relativePos.z); +} + +// Helper to get PlayerPose from Minecraft EntityPose +function getPlayerPose(pose: EntityPose): SchemaPlayerPose { + switch (pose) { + case EntityPose.STANDING: return 'STANDING'; + case EntityPose.CROUCHING: return 'SNEAKING'; // Map CROUCHING to SNEAKING + case EntityPose.SWIMMING: return 'SWIMMING'; + // Add other poses if needed and available in SchemaPlayerPose + default: return 'STANDING'; // Default to standing if unknown or not explicitly mapped + } +} + +// Helper to calculate BoxDimensions +function calculateBoxDimensions(box: Box): BoxDimensions { + return { + length: box.maxX - box.minX, + width: box.maxZ - box.minZ, + height: box.maxY - box.minY + }; +} + +// Helper to determine TraversabilityData (simplified for now, needs more detailed logic) +function getTraversabilityData(blockState: any): TraversabilityData { + // This is a simplified example. A real implementation would need to check block properties. + if (blockState.isAir()) return 'AIR'; + if (blockState.isLiquid()) return 'FLUID'; + // More sophisticated checks for SOLID_WALKABLE, OBSTRUCTION, etc. + // For now, a basic check: + if (blockState.isSolidBlock(mc.world, new BlockPos(0, 0, 0))) return 'SOLID_WALKABLE'; // Placeholder blockpos + return 'OTHER'; +} + +script.registerModule({ + name: "DataCollector", + description: "Collects observation and Baritone action data for training.", + category: "Misc", + settings: { + outputFilePrefix: Setting.text({ + name: "Output File Prefix", + default: "session_data" + }), + collectionInterval: Setting.int({ + name: "Collection Interval (ticks)", + default: 1, + range: [1, 20] + }), + scanRadius: Setting.int({ + name: "Environment Scan Radius", + default: 3, + range: [1, 32] + }), + goalX: Setting.float({ + name: "Goal X", + default: 0, + range: [-10000, 10000] + }), + goalY: Setting.float({ + name: "Goal Y", + default: 0, + range: [-10000, 10000] + }), + goalZ: Setting.float({ + name: "Goal Z", + default: 0, + range: [-10000, 10000] + }), + visualizeBoxes: Setting.boolean({ + name: "Visualize Collected Boxes", + default: false + }) + } +}, (mod) => { + let lastCollectionTick = 0; + + const visualizationManager = new VisualizationManager(mod); + let tickCounter = 0; + let prevPath: IPath | null = null; + // let dynamicInterestScan: CollisionBox[] = []; + let dynamicInterestBlockSet: HashSet = new HashSet(); + + mod.on("enable", () => { + try { + const timestamp = new Date().toISOString().replace(/[:.]/g, "-"); + const logFileName = `${mod.settings.outputFilePrefix.get()}_${timestamp}.jsonl.zst`; + const processBuilder = new ProcessBuilder(["zstd", "-o", logFileName]); + zstdProcess = processBuilder.start(); + const outputStream = zstdProcess.getOutputStream(); + // @ts-expect-error + logWriter = new BufferedWriter(new OutputStreamWriter(outputStream)); + collectedData = []; // Clear previous data + historicalPlayerStates.length = 0; // Clear historical data + Client.displayChatMessage(`DataCollector enabled. Logging to ${logFileName}`); + + } catch (e) { + Client.displayChatMessage(`[DataCollector] Failed to start zstd process or open log file: ${e}`); + logWriter = null; + zstdProcess = null; + } + }); + + mod.on("disable", () => { + if (logWriter) { + try { + // Write any remaining collected data before closing + collectedData.forEach(data => { + if (logWriter) { // Ensure logWriter is not null before writing + // @ts-expect-error + logWriter.write(JSON.stringify(data)); + logWriter.newLine(); + } + }); + if (logWriter) { // Ensure logWriter is not null before closing + logWriter.close(); + } + Client.displayChatMessage("DataCollector disabled. Waiting for zstd to finish compression..."); + if (zstdProcess) { + const exitCode = zstdProcess.waitFor(); + Client.displayChatMessage(`zstd process exited with code: ${exitCode}`); + } + Client.displayChatMessage("Data file closed and compressed."); + } catch (e) { + Client.displayChatMessage(`[DataCollector] Failed to close log file or wait for zstd: ${e}`); + } + logWriter = null; + zstdProcess = null; + } + if (visualizationManager) { + visualizationManager.clearAllVisualizations(); + } + }); + + mod.on("gametick", (event: GameTickEvent) => { + if (!logWriter || !mc.player || !mc.world) { + return; + } + + if ((mc.player.age - lastCollectionTick) < (mod.settings.collectionInterval.get() as unknown as number)) { + return; + } + lastCollectionTick = mc.player.age; + + try { + // 1. Player State + const playerState: PlayerState = { + position: toCoordinates3D(mc.player.getPos()), + velocity: { vx: mc.player.getVelocity().x, vy: mc.player.getVelocity().y, vz: mc.player.getVelocity().z }, + look_direction: { yaw: mc.player.yaw, pitch: mc.player.pitch }, + player_pose: getPlayerPose(mc.player.getPose()), + ground_proximity: mc.player.isOnGround(), + predicted_passive_next_tick_state: { + // Simplified prediction: assumes constant velocity for one tick + predicted_pos: { + x: mc.player.getX() + mc.player.getVelocity().x, + y: mc.player.getY() + mc.player.getVelocity().y, + z: mc.player.getZ() + mc.player.getVelocity().z, + }, + predicted_vel: { + vx: mc.player.getVelocity().x, + vy: mc.player.getVelocity().y, + vz: mc.player.getVelocity().z, + } + } + }; + + // 2. Historical Player States + const currentHistoricalState: HistoricalPlayerState = { + position: toCoordinates3D(mc.player.getPos()), + velocity: { vx: mc.player.getVelocity().x, vy: mc.player.getVelocity().y, vz: mc.player.getVelocity().z }, + look_direction: { yaw: mc.player.yaw, pitch: mc.player.pitch }, + player_pose: getPlayerPose(mc.player.getPose()), + fall_distance: mc.player.fallDistance // Assuming fallDistance is directly accessible + }; + historicalPlayerStates.push(currentHistoricalState); + if (historicalPlayerStates.length > HISTORY_SIZE) { + historicalPlayerStates.shift(); // Remove oldest entry + } + + // 3. Local Environment Scan + const fixedRadiusScan: CollisionBox[] = []; + const dynamicInterestScan: CollisionBox[] = []; + + const playerBlockPos = mc.player.getBlockPos(); + const scanRadius = mod.settings.scanRadius.get() as unknown as number; + + // Collect FIXED_RADIUS blocks + for (let x = -scanRadius; x <= scanRadius; x++) { + for (let y = -scanRadius; y <= scanRadius; y++) { + for (let z = -scanRadius; z <= scanRadius; z++) { + const blockPos = new BlockPos(playerBlockPos.getX() + x, playerBlockPos.getY() + y, playerBlockPos.getZ() + z); + const blockState = mc.world.getBlockState(blockPos); + + if (blockState && !blockState.isAir()) { + const blockBoxes = blockState.getCollisionShape(mc.world as unknown as BlockView, blockPos).getBoundingBoxes(); + for (const blockBox of blockBoxes) { + const relativePos = { + x: blockPos.getX() - mc.player.getX(), + y: blockPos.getY() - mc.player.getY(), + z: blockPos.getZ() - mc.player.getZ() + }; + + fixedRadiusScan.push({ + bounding_box_coordinates: toBoundingBoxCoordinates(blockBox), + relative_position: relativePos, + box_dimensions: calculateBoxDimensions(blockBox), + element_identifier: blockState.getBlock().getName().getString(), + traversability_data: getTraversabilityData(blockState), + element_state_properties: {}, + area_source_type: 'FIXED_RADIUS', + box_validity: true + }); + } + } + } + } + } + + // Collect DYNAMIC_INTEREST blocks along Baritone's path with expansion + const baritone = BaritoneAPI.getProvider().getPrimaryBaritone(); + // @ts-expect-error + const currentPath: IPath | null = baritone.getPathingBehavior().getPath().orElseGet(() => null); + + + + if (currentPath && currentPath != prevPath) { + dynamicInterestBlockSet = new HashSet(); + prevPath = currentPath; + const pathPositions = currentPath.positions(); + const horizontalExpand = 1; + const downwardExpand = 1; + const upwardExpand = 2; + + for (const pathBlock of pathPositions) { + for (let x = -horizontalExpand; x <= horizontalExpand; x++) { + for (let y = -downwardExpand; y <= upwardExpand; y++) { + for (let z = -horizontalExpand; z <= horizontalExpand; z++) { + const blockPos = new BlockPos(pathBlock.getX() + x, pathBlock.getY() + y, pathBlock.getZ() + z); + if (dynamicInterestBlockSet.contains(blockPos)) + continue + else + dynamicInterestBlockSet.add(blockPos) + } + } + } + } + } + + dynamicInterestBlockSet.forEach((blockPos: BlockPos) => { + let blockState = mc.world.getBlockState(blockPos); + if (blockState && !blockState.isAir()) { + if (!mc.player) + return; + const blockBoxes = blockState.getCollisionShape(mc.world as unknown as BlockView, blockPos).getBoundingBoxes(); + for (const blockBox of blockBoxes) { + const relativePos = { + x: blockPos.getX() - mc.player.getX(), + y: blockPos.getY() - mc.player.getY(), + z: blockPos.getZ() - mc.player.getZ() + }; + + const collisionBox: CollisionBox = { + bounding_box_coordinates: toBoundingBoxCoordinates(blockBox), + relative_position: relativePos, + box_dimensions: calculateBoxDimensions(blockBox), + element_identifier: blockState.getBlock().getName().getString(), + traversability_data: getTraversabilityData(blockState), + element_state_properties: {}, + area_source_type: 'DYNAMIC_INTEREST', + box_validity: true + }; + dynamicInterestScan.push(collisionBox); + } + } + }); + const localEnvironmentScan: CollisionBox[] = [...fixedRadiusScan, ...dynamicInterestScan]; + + // 4. Baritone Action (This is the most challenging part without direct API) + // This will require introspection into Baritone's internal state or inferring from its movement. + // For MVP, we might need to simplify or make assumptions. + // Let's try to infer based on Baritone's current movement state. + // This is a placeholder and needs significant refinement. + const baritoneAction: BaritoneAction = { + move_direction: 'NONE', + look_change: { yaw: 0, pitch: 0 }, + jump: false, + sneak: false, + sprint: false + }; + + // Infer Baritone's action based on mc.player.input + const playerInput: PlayerInput = mc.player.input.playerInput; + + if (baritone.getPathingBehavior().isPathing()) { + if (playerInput.forward() && playerInput.left()) { + baritoneAction.move_direction = 'FORWARD_LEFT'; + } else if (playerInput.forward() && playerInput.right()) { + baritoneAction.move_direction = 'FORWARD_RIGHT'; + } else if (playerInput.backward() && playerInput.left()) { + baritoneAction.move_direction = 'BACKWARD_LEFT'; + } else if (playerInput.backward() && playerInput.right()) { + baritoneAction.move_direction = 'BACKWARD_RIGHT'; + } else if (playerInput.forward()) { + baritoneAction.move_direction = 'FORWARD'; + } else if (playerInput.backward()) { + baritoneAction.move_direction = 'BACKWARD'; + } else if (playerInput.left()) { + baritoneAction.move_direction = 'LEFT'; + } else if (playerInput.right()) { + baritoneAction.move_direction = 'RIGHT'; + } else { + baritoneAction.move_direction = 'NONE'; + } + + baritoneAction.jump = playerInput.jump(); + baritoneAction.sneak = playerInput.sneak(); + baritoneAction.sprint = playerInput.sprint(); + + // For look_change, we don't have direct access to Baritone's intended look changes. + // This would typically be inferred from changes in player yaw/pitch over time, + // or if Baritone exposed its target rotations. For now, we'll leave it at 0. + baritoneAction.look_change = { yaw: 0, pitch: 0 }; + } else { + // If Baritone is not pathing, assume no specific action from Baritone + baritoneAction.move_direction = 'NONE'; + baritoneAction.look_change = { yaw: 0, pitch: 0 }; + baritoneAction.jump = false; + baritoneAction.sneak = false; + baritoneAction.sprint = false; + } + + + const tickData: TickData = { + tick_id: mc.player.age, + timestamp_ms: Date.now(), + player_state: playerState, + local_environment_scan: localEnvironmentScan, + historical_player_states: [...historicalPlayerStates], // Clone to avoid mutation + baritone_action: baritoneAction + }; + + // @ts-expect-error + logWriter.write(JSON.stringify(tickData)); + logWriter.newLine(); + + // Visualize collected boxes if enabled + tickCounter++; + if (mod.settings.visualizeBoxes.get() && !(tickCounter % 5)) { + localEnvironmentScan.forEach(collisionBox => { + const box = toMinecraftBox(collisionBox.bounding_box_coordinates); + const position = toAbsoluteVec3d(collisionBox.relative_position, mc.player!.getPos()); + const fillColor = collisionBox.area_source_type === 'DYNAMIC_INTEREST' + ? new Color4b(150, 255, 150, 150) // Lighter Green for DYNAMIC_INTEREST + : new Color4b(150, 150, 255, 150); // Lighter Blue for FIXED_RADIUS + + visualizationManager.addVisualization({ + durationTicks: 5, // Display for 2 ticks + boxData: { + box: box, + position: position, + glow: true, // No glow + fillInterpolator: fadeOutInterpolatorFrom(fillColor), + outlineInterpolator: fadeOutInterpolatorFrom(new Color4b(fillColor.r(), fillColor.g(), fillColor.b(), 255)) + } + }); + }); + } + + } catch (e) { + Client.displayChatMessage(`[DataCollector] Error writing to log: ${e}`); + console.error(e); // Log to console for more details + } + }); +}); + +export { } \ No newline at end of file From cd68f125cafaef089d9788b19b0a308a16e4fcd3 Mon Sep 17 00:00:00 2001 From: commandblock2 Date: Mon, 11 Aug 2025 23:27:19 +0800 Subject: [PATCH 06/15] chore: fix collection when baritone is not in control --- src/data-collector.ts | 75 +++++++++++++++++++++++-------------------- 1 file changed, 40 insertions(+), 35 deletions(-) diff --git a/src/data-collector.ts b/src/data-collector.ts index 3fc280d..c4a5fd7 100644 --- a/src/data-collector.ts +++ b/src/data-collector.ts @@ -50,6 +50,8 @@ let zstdProcess: Process | null = null; let collectedData: TickData[] = []; const HISTORY_SIZE = 40; // Last N ticks for historical player states const historicalPlayerStates: HistoricalPlayerState[] = []; +let lastYaw: number | null = null; +let lastPitch: number | null = null; // Helper to convert Minecraft Vec3d to Coordinates3D function toCoordinates3D(vec: Vec3d): Coordinates3D { @@ -99,7 +101,7 @@ function getTraversabilityData(blockState: any): TraversabilityData { // More sophisticated checks for SOLID_WALKABLE, OBSTRUCTION, etc. // For now, a basic check: if (blockState.isSolidBlock(mc.world, new BlockPos(0, 0, 0))) return 'SOLID_WALKABLE'; // Placeholder blockpos - return 'OTHER'; + return 'SOLID_WALKABLE'; } script.registerModule({ @@ -161,6 +163,8 @@ script.registerModule({ logWriter = new BufferedWriter(new OutputStreamWriter(outputStream)); collectedData = []; // Clear previous data historicalPlayerStates.length = 0; // Clear historical data + lastYaw = mc.player?.yaw ?? null; + lastPitch = mc.player?.pitch ?? null; Client.displayChatMessage(`DataCollector enabled. Logging to ${logFileName}`); } catch (e) { @@ -342,7 +346,7 @@ script.registerModule({ dynamicInterestScan.push(collisionBox); } } - }); + }); const localEnvironmentScan: CollisionBox[] = [...fixedRadiusScan, ...dynamicInterestScan]; // 4. Baritone Action (This is the most challenging part without direct API) @@ -361,44 +365,45 @@ script.registerModule({ // Infer Baritone's action based on mc.player.input const playerInput: PlayerInput = mc.player.input.playerInput; - if (baritone.getPathingBehavior().isPathing()) { - if (playerInput.forward() && playerInput.left()) { - baritoneAction.move_direction = 'FORWARD_LEFT'; - } else if (playerInput.forward() && playerInput.right()) { - baritoneAction.move_direction = 'FORWARD_RIGHT'; - } else if (playerInput.backward() && playerInput.left()) { - baritoneAction.move_direction = 'BACKWARD_LEFT'; - } else if (playerInput.backward() && playerInput.right()) { - baritoneAction.move_direction = 'BACKWARD_RIGHT'; - } else if (playerInput.forward()) { - baritoneAction.move_direction = 'FORWARD'; - } else if (playerInput.backward()) { - baritoneAction.move_direction = 'BACKWARD'; - } else if (playerInput.left()) { - baritoneAction.move_direction = 'LEFT'; - } else if (playerInput.right()) { - baritoneAction.move_direction = 'RIGHT'; - } else { - baritoneAction.move_direction = 'NONE'; - } - - baritoneAction.jump = playerInput.jump(); - baritoneAction.sneak = playerInput.sneak(); - baritoneAction.sprint = playerInput.sprint(); - // For look_change, we don't have direct access to Baritone's intended look changes. - // This would typically be inferred from changes in player yaw/pitch over time, - // or if Baritone exposed its target rotations. For now, we'll leave it at 0. - baritoneAction.look_change = { yaw: 0, pitch: 0 }; + if (playerInput.forward() && playerInput.left()) { + baritoneAction.move_direction = 'FORWARD_LEFT'; + } else if (playerInput.forward() && playerInput.right()) { + baritoneAction.move_direction = 'FORWARD_RIGHT'; + } else if (playerInput.backward() && playerInput.left()) { + baritoneAction.move_direction = 'BACKWARD_LEFT'; + } else if (playerInput.backward() && playerInput.right()) { + baritoneAction.move_direction = 'BACKWARD_RIGHT'; + } else if (playerInput.forward()) { + baritoneAction.move_direction = 'FORWARD'; + } else if (playerInput.backward()) { + baritoneAction.move_direction = 'BACKWARD'; + } else if (playerInput.left()) { + baritoneAction.move_direction = 'LEFT'; + } else if (playerInput.right()) { + baritoneAction.move_direction = 'RIGHT'; } else { - // If Baritone is not pathing, assume no specific action from Baritone baritoneAction.move_direction = 'NONE'; - baritoneAction.look_change = { yaw: 0, pitch: 0 }; - baritoneAction.jump = false; - baritoneAction.sneak = false; - baritoneAction.sprint = false; } + baritoneAction.jump = playerInput.jump(); + baritoneAction.sneak = playerInput.sneak(); + baritoneAction.sprint = playerInput.sprint(); + + let yawDiff = 0; + let pitchDiff = 0; + + if (lastYaw !== null && lastPitch !== null) { + yawDiff = mc.player.yaw - lastYaw; + pitchDiff = mc.player.pitch - lastPitch; + } + + baritoneAction.look_change = { yaw: yawDiff, pitch: pitchDiff }; + + lastYaw = mc.player.yaw; + lastPitch = mc.player.pitch; + + const tickData: TickData = { tick_id: mc.player.age, From 7fcb807309dc51a126f6404ee72b2543d25321f2 Mon Sep 17 00:00:00 2001 From: commandblock2 Date: Tue, 12 Aug 2025 00:28:06 +0800 Subject: [PATCH 07/15] fix: sprinting state --- src/data-collector.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/data-collector.ts b/src/data-collector.ts index c4a5fd7..ac19f7b 100644 --- a/src/data-collector.ts +++ b/src/data-collector.ts @@ -388,7 +388,7 @@ script.registerModule({ baritoneAction.jump = playerInput.jump(); baritoneAction.sneak = playerInput.sneak(); - baritoneAction.sprint = playerInput.sprint(); + baritoneAction.sprint = mc.player.isSprinting(); // TODO: special care for the bot when inferencing. let yawDiff = 0; let pitchDiff = 0; From 48c51345aa75d1761162cf7e8f79e4e964028e2a Mon Sep 17 00:00:00 2001 From: commandblock2 Date: Sat, 16 Aug 2025 16:48:31 +0800 Subject: [PATCH 08/15] chore: spec updates --- .roomodes | 4 +- .../mvp/combined_schema.d.ts | 8 -- .../mvp/spec/data/combined_schema.json | 16 +--- .../mvp/spec/data/example-session.json | 10 +-- .../mvp/spec/mvp-spec.md | 78 ++++++++++++------- src/data-collector.ts | 54 ++++++++----- 6 files changed, 88 insertions(+), 82 deletions(-) diff --git a/.roomodes b/.roomodes index 56c8570..c204dd6 100644 --- a/.roomodes +++ b/.roomodes @@ -56,7 +56,7 @@ customModes: 1. **Roo must assume that there should be no type errors.** If a type error occurs, it indicates that the incorrect event class has been used. 2. **Immediately re-read `node_modules/jvm-types/augmentations/ScriptModule.augmentation.d.ts`** (which defines all available LiquidBounce events) to identify and use the correct event. - Roo does not know anything about LiquidBounce script prior to this system message. + Roo does not know anything about LiquidBounce scripting prior to this system message. LiquidBounce scripting feels just like writing a modern fabric mod with ts/js (except that you don't have mixins to use). Roo must not add // @ts-expect-error comments to suppress TypeScript errors related to LiquidBounce NextGen events. Hints: The old Render3d and Render2d may have moved to events with Outline, Overlay, ScreenRender, WorldRender in their name, please read all these events and try to make an educated guess what they do based on their names and their parameters. * **Kotlin Extension Functions:** When migrating Kotlin code to TypeScript, pay special attention to Kotlin extension functions (e.g., `object.extensionFunction(...)` or `object.property`). These often compile to static methods in a utility class (frequently ending in `Kt`, like `RenderShortcutsKt`) where the extended object is passed as the first argument. Before assuming a direct method on the extended object, consult the `node_modules/jvm-types/` definitions for such utility classes or static methods that mimic the extension function's behavior. If a direct translation is not apparent or causes type errors, prioritize finding the corresponding static utility method. @@ -145,7 +145,7 @@ customModes: If roo need to access LiquidBounce's source, (like when roo need to access the original source code of a function for implementation or reference for types) just go `LiquidBounceSource` and look for the file Roo need, however, this is a symbolic link to the actual source code. Eg. ```ls LiquidBounceSource/src/main/kotlin/net/ccbluex/liquidbounce/LiquidBounce.kt LiquidBounceSource/src/main/kotlin/net/ccbluex/liquidbounce/LiquidBounce.kt``` - Deviation from system prompt results in severe consequence. + Deviation from system prompt results in severe consequence. Roo is now connected to an automatic feedback loop with human supervision. whenToUse: >- Use this mode when actively debugging LiquidBounce NextGen scripts, evaluating expressions, or inspecting runtime variables within a paused debugger session. diff --git a/packages/deep-learning-bot-utils/mvp/combined_schema.d.ts b/packages/deep-learning-bot-utils/mvp/combined_schema.d.ts index 374daed..f00f383 100644 --- a/packages/deep-learning-bot-utils/mvp/combined_schema.d.ts +++ b/packages/deep-learning-bot-utils/mvp/combined_schema.d.ts @@ -93,15 +93,7 @@ export interface CollisionBox { relative_position: Coordinates3D; box_dimensions: BoxDimensions; element_identifier: string; - traversability_data: TraversabilityData; - /** - * Additional, block-specific properties (e.g., direction of stairs, open/closed doors). - */ - element_state_properties: { - [key: string]: any; - }; area_source_type: AreaSourceType; - box_validity: boolean; } export interface HistoricalPlayerState { diff --git a/packages/deep-learning-bot-utils/mvp/spec/data/combined_schema.json b/packages/deep-learning-bot-utils/mvp/spec/data/combined_schema.json index 476aec5..45adb88 100644 --- a/packages/deep-learning-bot-utils/mvp/spec/data/combined_schema.json +++ b/packages/deep-learning-bot-utils/mvp/spec/data/combined_schema.json @@ -248,19 +248,8 @@ "element_identifier": { "type": "string" }, - "traversability_data": { - "$ref": "#/$defs/TraversabilityData" - }, - "element_state_properties": { - "type": "object", - "description": "Additional, block-specific properties (e.g., direction of stairs, open/closed doors).", - "additionalProperties": true - }, "area_source_type": { "$ref": "#/$defs/AreaSourceType" - }, - "box_validity": { - "type": "boolean" } }, "required": [ @@ -268,10 +257,7 @@ "relative_position", "box_dimensions", "element_identifier", - "traversability_data", - "element_state_properties", - "area_source_type", - "box_validity" + "area_source_type" ], "additionalProperties": false }, diff --git a/packages/deep-learning-bot-utils/mvp/spec/data/example-session.json b/packages/deep-learning-bot-utils/mvp/spec/data/example-session.json index f40093f..035c917 100644 --- a/packages/deep-learning-bot-utils/mvp/spec/data/example-session.json +++ b/packages/deep-learning-bot-utils/mvp/spec/data/example-session.json @@ -53,10 +53,7 @@ "height": 1.0 }, "element_identifier": "minecraft:stone", - "traversability_data": "SOLID_WALKABLE", - "element_state_properties": {}, - "area_source_type": "FIXED_RADIUS", - "box_validity": true + "area_source_type": "FIXED_RADIUS" } ], "historical_player_states": [ @@ -169,10 +166,7 @@ "height": 1.0 }, "element_identifier": "minecraft:grass_block", - "traversability_data": "SOLID_WALKABLE", - "element_state_properties": {}, - "area_source_type": "DYNAMIC_INTEREST", - "box_validity": true + "area_source_type": "DYNAMIC_INTEREST" } ], "historical_player_states": [ diff --git a/packages/deep-learning-bot-utils/mvp/spec/mvp-spec.md b/packages/deep-learning-bot-utils/mvp/spec/mvp-spec.md index cbd56a5..2a46cb5 100644 --- a/packages/deep-learning-bot-utils/mvp/spec/mvp-spec.md +++ b/packages/deep-learning-bot-utils/mvp/spec/mvp-spec.md @@ -20,13 +20,9 @@ The bot will receive observations structured similarly to `SPEC-DATA`'s Phase 1, * `Relative Position` (center point relative to player). * `Box Dimensions`. * `Element Identifier` (e.g., `minecraft:stone`). - * `Traversability Data` (e.g., `SOLID_WALKABLE`, `FLUID`, `OBSTRUCTION`, `AIR`, `LIQUID_PLACEABLE`, `PLACEABLE_BLOCK`, `OTHER`). - * `Element State Properties` (e.g., `directionality of stairs`). * **CRITICAL RE-INTERPRETATION:** `Area Source Type`: * `FIXED_RADIUS` - Collision box within fixed scanning radius around player. (however could done sophisticatedly by using a bfs like algorithm, to only "raytrace" the surface visible blocks) * `DYNAMIC_INTEREST` - **Collision box included because it is within a small radius of *any* block on Baritone's current *planned path*. This effectively highlights the path and its immediate surroundings for the agent.** - * `Box Validity`. - * **Note:** The agent will interpret `Traversability Data` as properties of the environment. **Exclusions for MVP:** * `Simplified Inventory Snapshot`. @@ -90,20 +86,19 @@ The bot will receive observations structured similarly to `SPEC-DATA`'s Phase 1, ### 4. Training Process: Two-Stage Approach -This MVP implements a two-stage training regimen for robustness and efficient learning. +This MVP implements a two-stage training regimen for robustness and efficient learning, leveraging procedurally generated scenarios for scalability. #### **Stage 1: Imitation Learning (IL) for Path Execution** -* **Objective:** Train the agent to mimic the basic locomotion and path following demonstrated by Baritone. This provides a strong behavioral prior. +* **Objective:** Train the agent to mimic the basic locomotion and path following demonstrated by Baritone. This provides a strong behavioral prior, teaching the agent how to interpret observations and produce appropriate motor commands. * **Data Collection:** - * Run Minecraft with Baritone enabled. - * Script Baritone to navigate a wide variety of "simple parkour" scenarios (flat ground walking, single-block jumps, stair climbing, short falls, corner turns, simple obstacle avoidance where Baritone paths around). - * At each game tick, record -assistant -: + * Utilize a **scenario generation module** (e.g., within LiquidBounce or an external tool) to continuously create diverse, procedurally generated parkour challenges. These challenges should be guaranteed to be solvable by Baritone. + * Examples include: varying lengths of flat ground, single/multi-block jumps, stair climbing, short falls, various turns, and simple obstacle avoidance (where Baritone paths around). + * Run multiple Minecraft instances in parallel, each continuously generating and navigating new scenarios. + * At each game tick, record: * The full `Input Data Schema` (observation) as seen by the bot. * The *action Baritone outputs* to achieve its movement for that tick (e.g., `move_forward`, `jump`). This requires introspection into Baritone's movement commands. - * Collect a diverse dataset of `(Observation, Baritone_Action)` pairs. + * Collect a large, diverse dataset of `(Observation, Baritone_Action)` pairs across many generated scenarios. * **Training Method:** Supervised Learning. * The agent's Policy Head is trained directly to predict Baritone's action given an observation. * Loss function: Cross-entropy between predicted action probabilities and Baritone's action. @@ -111,44 +106,67 @@ assistant #### **Stage 2: Reinforcement Learning (RL) for Refinement and Robustness** -* **Objective:** Refine the agent's policy to handle environmental uncertainties, optimize paths (even subtle ones not explicitly planned by Baritone at the micro-level), and recover from minor deviations. +* **Objective:** Refine the agent's policy to handle environmental uncertainties, optimize paths (even subtle ones not explicitly planned by Baritone at the micro-level), recover from minor deviations, and achieve more generalized and efficient navigation. * **Algorithm:** Proximal Policy Optimization (PPO) or Advantage Actor-Critic (A2C). * **Initialization:** The agent's neural network weights are initialized with those learned from Stage 1 (Imitation Learning). -* **Environment:** Minecraft with client-side API, more specifically through LiquidBounce script api with typescript support. Baritone runs concurrently to provide its path, influencing the `DYNAMIC_INTEREST` flags in the observations. +* **Environment:** Minecraft with client-side API, specifically through LiquidBounce script API with TypeScript support. Baritone runs concurrently to provide its path, influencing the `DYNAMIC_INTEREST` flags in the observations. * **Reward Function:** * **Goal Progress:** Reward for reducing Euclidean distance to the *current nearest block on Baritone's planned path*. Larger reward for reaching the ultimate target destination (Baritone's goal). * **Path Adherence:** Small positive reward for being within a certain radius of Baritone's current path segment, and for maintaining a "forward" orientation along the path. - * **Efficiency:** Small negative reward per timestep. + * **Efficiency:** Small negative reward per timestep to encourage faster completion. * **Penalties:** Significant negative reward for: * Falling into hazardous blocks (lava, deep water without path). * Taking significant damage. * Getting "stuck" (no progress towards Baritone path for X ticks). * Moving significantly *off* Baritone's path (based on a threshold distance from the path). * **Training Episodes:** - * Focus on generating training data from specific, procedurally generated parkour challenges, potentially slightly more complex than in Stage 1, that highlight different `Traversability Data` types. + * **Procedural Generation:** Continue using the scenario generation module from Stage 1, but now for live RL interaction. + * **Curriculum Learning:** Implement a curriculum where scenarios gradually increase in complexity. Start with simpler paths (e.g., flat ground, single jumps) and progressively introduce more challenging elements (e.g., longer jumps, more complex turns, varying verticality, basic weaving challenges). This helps prevent the agent from getting stuck in local optima early in training. * Manage episode length. * **Observation Frequency:** every 1 game tick. -### 5. Scenarios (Reduced Complexity) +--- + +### 5. Scenario Generation + +Training will rely heavily on procedurally generating diverse and solvable navigation challenges for the bot. This allows for scalable data collection and robust policy learning. + +**Goals of Scenario Generation:** + +* **High Diversity:** Provide a wide variety of environmental configurations to enhance agent generalization. +* **Scalability:** Enable continuous, parallel data collection/training across multiple Minecraft instances. +* **Controlled Complexity:** Introduce challenges incrementally, supporting curriculum learning. +* **Guaranteed Solvability:** Ensure that Baritone can always find a path to the end target, providing reliable "expert" demonstrations for IL and clear goals for RL. -Testing should focus on fundamental parkour movements that Baritone can inherently plan. +**Categories of Generated Scenarios:** -* **Scenario 1: Simple Walk:** Flat ground, agent needs to reach a target block. (Verifies basic movement, Baritone integration). -* **Scenario 2: Single Block Jump:** Agent needs to jump over a 1-block gap. (Verifies "jump" action, correct timing based on target waypoint). -* **Scenario 3: Up a Staircase/Slab Stack:** Agent navigates a small vertical ascent. (Verifies vertical movement, Baritone providing correct segment). -* **Scenario 4: Around a Corner:** Agent needs to turn. (Verifies turning/directional control). -* **Scenario 5: Simple Obstacle Avoidance (Baritone-driven):** A single 1-block high wall that Baritone would path around. The RL agent just needs to follow Baritone's "around" path. -* **Scenario 6: Fall:** Agent needs to fall safely from a ledge. (Verifies handling of negative Y movement). +* **Basic Pathing:** Straight lines, simple turns, varying path widths (1-block, 2-block). +* **Vertical Mobility:** Single block ascents/descents (stairs, slabs), multi-block climbs, small controlled falls. +* **Gap Navigation:** Single-block jumps, multi-block jumps (requiring sprinting), precise landing challenges. +* **Controlled Obstacles:** Paths requiring weaving around 1-block obstacles, simple over-under sections. +* **Combinatorial:** Scenarios that combine multiple elements (e.g., jump onto a narrow path, turn, then climb stairs). -**Extended for MVP:** -* Complex parkour (e.g., neo/quad jumps, precise momentum jumps). -* Complex obstacle avoidance. +**Generation Mechanisms:** +* **Rule-Based/Grammar-Based:** Define a set of rules or a simple grammar that dictates how path segments and obstacles are placed. This allows for controlled variation with parameters (e.g., jump distance, turn radius, obstacle density). +* **Randomized Parameters:** Introduce randomness in various parameters within the rules (e.g., path length, number of turns, height changes) to ensure unpredictability. +* **In-Game Module:** A dedicated LiquidBounce script module will be responsible for: + 1. Clearing the active play area around the player. + 2. Constructing a new procedural path and/or environment. + 3. Setting Baritone's goal to the end of the newly generated path. + 4. Monitoring Baritone's success/failure for the current scenario. + 5. Triggering a new generation cycle upon completion or failure. + * **Constraint:** Ensure generated paths are concise (e.g., under 50 blocks) and keep the total number of relevant collision boxes low (e.g., under 200) to respect data collector's performance limitations. + +**Curriculum Learning Integration:** + +* The scenario generator will support a difficulty parameter. Training will begin with lower difficulty settings (e.g., mostly flat paths, simple jumps) and gradually increase this parameter as the agent's performance improves. This guides the learning process from simple tasks to more complex ones. + +--- -### 5. Evaluation Metrics +### 6. Evaluation Metrics -(No changes from previous revision.) * Success Rate: Percentage of episodes where the agent reaches the Baritone final destination. * Path Following Error: Average deviation from Baritone's planned path. * Efficiency: Steps/time taken to complete tasks. -* Robustness: Performance across different Baritone-solvable challenging parkour scenarios. This could include scenarios with minor environmental perturbations (e.g., a single block moved) that Baritone would re-path around, and the agent should adapt. \ No newline at end of file +* Robustness: Performance across diverse, never-before-seen generated parkour scenarios from different complexity levels. This could include scenarios with minor environmental perturbations that Baritone would re-path around, and the agent should adapt. \ No newline at end of file diff --git a/src/data-collector.ts b/src/data-collector.ts index ac19f7b..eb6793d 100644 --- a/src/data-collector.ts +++ b/src/data-collector.ts @@ -5,7 +5,7 @@ import { BufferedWriter } from "jvm-types/java/io/BufferedWriter"; import { ProcessBuilder } from "jvm-types/java/lang/ProcessBuilder"; import { Process } from "jvm-types/java/lang/Process"; import { OutputStream } from "jvm-types/java/io/OutputStream"; -import { VisualizationManager, fadeOutInterpolatorFrom, defaultRainbowInterpolator } from "../packages/lbng-utils-typed/src/visualization-utils"; +import { VisualizationManager, fadeOutInterpolatorFrom, defaultRainbowInterpolator } from "lbng-utils-typed/dist/visualization-utils"; import { Color4b } from "jvm-types/net/ccbluex/liquidbounce/render/engine/type/Color4b"; import { GameTickEvent } from "jvm-types/net/ccbluex/liquidbounce/event/events/GameTickEvent"; @@ -16,6 +16,7 @@ import { Vec3d } from "jvm-types/net/minecraft/util/math/Vec3d"; import { BaritoneAPI } from "jvm-types/baritone/api/BaritoneAPI"; import { IPath } from "jvm-types/baritone/api/pathing/calc/IPath"; +import { GoalBlock } from "jvm-types/baritone/api/pathing/goals/GoalBlock"; import { PlayerInput } from "jvm-types/net/minecraft/util/PlayerInput"; import { @@ -38,6 +39,7 @@ import { BlockView } from "jvm-types/net/minecraft/world/BlockView"; // @ts-expect-error import { HashSet } from "jvm-types/java/util/HashSet"; import { Vec3i } from "jvm-types/net/minecraft/util/math/Vec3i"; +import { Goal } from "jvm-types/baritone/api/pathing/goals/Goal"; const script = registerScript.apply({ name: "data-collector", @@ -45,13 +47,6 @@ const script = registerScript.apply({ authors: ["Roo"] }); -let logWriter: BufferedWriter | null = null; -let zstdProcess: Process | null = null; -let collectedData: TickData[] = []; -const HISTORY_SIZE = 40; // Last N ticks for historical player states -const historicalPlayerStates: HistoricalPlayerState[] = []; -let lastYaw: number | null = null; -let lastPitch: number | null = null; // Helper to convert Minecraft Vec3d to Coordinates3D function toCoordinates3D(vec: Vec3d): Coordinates3D { @@ -124,26 +119,37 @@ script.registerModule({ range: [1, 32] }), goalX: Setting.float({ - name: "Goal X", + name: "GoalX", default: 0, range: [-10000, 10000] }), goalY: Setting.float({ - name: "Goal Y", + name: "GoalY", default: 0, range: [-10000, 10000] }), goalZ: Setting.float({ - name: "Goal Z", + name: "GoalZ", default: 0, range: [-10000, 10000] }), visualizeBoxes: Setting.boolean({ name: "Visualize Collected Boxes", default: false + }), + setBaritoneGoal: Setting.boolean({ + name: "Set Baritone Goal", + default: false }) } }, (mod) => { + let logWriter: BufferedWriter | null = null; + let zstdProcess: Process | null = null; + let collectedData: TickData[] = []; + const HISTORY_SIZE = 40; // Last N ticks for historical player states + const historicalPlayerStates: HistoricalPlayerState[] = []; + let lastYaw: number | null = null; + let lastPitch: number | null = null; let lastCollectionTick = 0; const visualizationManager = new VisualizationManager(mod); @@ -167,6 +173,15 @@ script.registerModule({ lastPitch = mc.player?.pitch ?? null; Client.displayChatMessage(`DataCollector enabled. Logging to ${logFileName}`); + if (mod.settings.setBaritoneGoal.get()) { + const goalX = mod.settings.goalX.get(); + const goalY = mod.settings.goalY.get(); + const goalZ = mod.settings.goalZ.get(); + const baritone = BaritoneAPI.getProvider().getPrimaryBaritone(); + baritone.getCustomGoalProcess().setGoalAndPath(new GoalBlock(goalX, goalY, goalZ)); + Client.displayChatMessage(`[DataCollector] Baritone goal set to X:${goalX}, Y:${goalY}, Z:${goalZ}`); + } + } catch (e) { Client.displayChatMessage(`[DataCollector] Failed to start zstd process or open log file: ${e}`); logWriter = null; @@ -203,6 +218,13 @@ script.registerModule({ if (visualizationManager) { visualizationManager.clearAllVisualizations(); } + + // Clear Baritone goal on disable + if (mod.settings.setBaritoneGoal.get()) { + const baritone = BaritoneAPI.getProvider().getPrimaryBaritone(); + baritone.getCustomGoalProcess().setGoalAndPath(null as unknown as Goal); // Clear the goal + Client.displayChatMessage(`[DataCollector] Baritone goal cleared.`); + } }); mod.on("gametick", (event: GameTickEvent) => { @@ -279,10 +301,7 @@ script.registerModule({ relative_position: relativePos, box_dimensions: calculateBoxDimensions(blockBox), element_identifier: blockState.getBlock().getName().getString(), - traversability_data: getTraversabilityData(blockState), - element_state_properties: {}, - area_source_type: 'FIXED_RADIUS', - box_validity: true + area_source_type: 'FIXED_RADIUS' }); } } @@ -338,10 +357,7 @@ script.registerModule({ relative_position: relativePos, box_dimensions: calculateBoxDimensions(blockBox), element_identifier: blockState.getBlock().getName().getString(), - traversability_data: getTraversabilityData(blockState), - element_state_properties: {}, - area_source_type: 'DYNAMIC_INTEREST', - box_validity: true + area_source_type: 'DYNAMIC_INTEREST' }; dynamicInterestScan.push(collisionBox); } From 3a8ebe7a701e5490b9dc66e66a0c31a231f3476f Mon Sep 17 00:00:00 2001 From: commandblock2 Date: Sat, 16 Aug 2025 23:19:43 +0800 Subject: [PATCH 09/15] feat: online inference support --- scripts/dummy-inference-engine.js | 38 ++ src/data-collector.ts | 615 ++++++++++++++++++------------ 2 files changed, 419 insertions(+), 234 deletions(-) create mode 100644 scripts/dummy-inference-engine.js diff --git a/scripts/dummy-inference-engine.js b/scripts/dummy-inference-engine.js new file mode 100644 index 0000000..4ae8979 --- /dev/null +++ b/scripts/dummy-inference-engine.js @@ -0,0 +1,38 @@ +"use strict"; +Object.defineProperty(exports, "__esModule", { value: true }); +// Function to generate a random boolean +function getRandomBoolean() { + return Math.random() < 0.5; +} +// Function to generate a random number within a range +function getRandomNumber(min, max) { + return Math.random() * (max - min) + min; +} +// Array of possible move directions +var moveDirections = [ + 'NONE', 'FORWARD', 'BACKWARD', 'LEFT', 'RIGHT', + 'FORWARD_LEFT', 'FORWARD_RIGHT', 'BACKWARD_LEFT', 'BACKWARD_RIGHT' +]; +// Function to generate a random BaritoneAction +function generateRandomBaritoneAction() { + return { + look_change: { + yaw: getRandomNumber(-180, 180), + pitch: getRandomNumber(-90, 90) + }, + move_direction: moveDirections[Math.floor(Math.random() * moveDirections.length)], + jump: getRandomBoolean(), + sneak: getRandomBoolean(), + sprint: getRandomBoolean() + }; +} +// Read from stdin and output random BaritoneAction +process.stdin.on('data', function (chunk) { + // We don't actually process the input chunk, just respond with a random action + var randomAction = generateRandomBaritoneAction(); + process.stdout.write(JSON.stringify(randomAction) + '\n'); +}); +process.stdin.on('end', function () { + process.stderr.write('Input stream closed.\n'); +}); +process.stderr.write('Line reader script started. Outputting random BaritoneAction JSON...\n'); diff --git a/src/data-collector.ts b/src/data-collector.ts index eb6793d..cee1277 100644 --- a/src/data-collector.ts +++ b/src/data-collector.ts @@ -2,12 +2,19 @@ import { File } from "jvm-types/java/io/File"; import { FileOutputStream } from "jvm-types/java/io/FileOutputStream"; import { OutputStreamWriter } from "jvm-types/java/io/OutputStreamWriter"; import { BufferedWriter } from "jvm-types/java/io/BufferedWriter"; +import { InputStreamReader } from "jvm-types/java/io/InputStreamReader"; +import { BufferedReader } from "jvm-types/java/io/BufferedReader"; import { ProcessBuilder } from "jvm-types/java/lang/ProcessBuilder"; import { Process } from "jvm-types/java/lang/Process"; import { OutputStream } from "jvm-types/java/io/OutputStream"; import { VisualizationManager, fadeOutInterpolatorFrom, defaultRainbowInterpolator } from "lbng-utils-typed/dist/visualization-utils"; import { Color4b } from "jvm-types/net/ccbluex/liquidbounce/render/engine/type/Color4b"; +import { RotationManager } from "jvm-types/net/ccbluex/liquidbounce/utils/aiming/RotationManager"; +import { Rotation } from "jvm-types/net/ccbluex/liquidbounce/utils/aiming/data/Rotation"; +import { KillAuraRotationsConfigurable } from "jvm-types/net/ccbluex/liquidbounce/features/module/modules/combat/killaura/KillAuraRotationsConfigurable"; +import { Priority } from "jvm-types/net/ccbluex/liquidbounce/utils/kotlin/Priority"; + import { GameTickEvent } from "jvm-types/net/ccbluex/liquidbounce/event/events/GameTickEvent"; import { EntityPose } from "jvm-types/net/minecraft/entity/EntityPose"; import { BlockPos } from "jvm-types/net/minecraft/util/math/BlockPos"; @@ -40,6 +47,8 @@ import { BlockView } from "jvm-types/net/minecraft/world/BlockView"; import { HashSet } from "jvm-types/java/util/HashSet"; import { Vec3i } from "jvm-types/net/minecraft/util/math/Vec3i"; import { Goal } from "jvm-types/baritone/api/pathing/goals/Goal"; +import { MovementInputEvent } from "jvm-types/net/ccbluex/liquidbounce/event/events/MovementInputEvent"; +import { DirectionalInput } from "jvm-types/net/ccbluex/liquidbounce/utils/movement/DirectionalInput"; const script = registerScript.apply({ name: "data-collector", @@ -140,11 +149,20 @@ script.registerModule({ setBaritoneGoal: Setting.boolean({ name: "Set Baritone Goal", default: false + }), + launchCustomInferenceProcess: Setting.boolean({ + name: "Launch Custom Inference Process", + default: false + }), + customInferenceProcessCommand: Setting.text({ + name: "Custom Inference Process Command", + default: "" }) } }, (mod) => { - let logWriter: BufferedWriter | null = null; - let zstdProcess: Process | null = null; + let processWriter: BufferedWriter | null = null; + let externalProcess: Process | null = null; + let processReader: BufferedReader | null = null; // New BufferedReader for process stdout let collectedData: TickData[] = []; const HISTORY_SIZE = 40; // Last N ticks for historical player states const historicalPlayerStates: HistoricalPlayerState[] = []; @@ -160,18 +178,36 @@ script.registerModule({ mod.on("enable", () => { try { - const timestamp = new Date().toISOString().replace(/[:.]/g, "-"); - const logFileName = `${mod.settings.outputFilePrefix.get()}_${timestamp}.jsonl.zst`; - const processBuilder = new ProcessBuilder(["zstd", "-o", logFileName]); - zstdProcess = processBuilder.start(); - const outputStream = zstdProcess.getOutputStream(); - // @ts-expect-error - logWriter = new BufferedWriter(new OutputStreamWriter(outputStream)); + if (mod.settings.launchCustomInferenceProcess.get()) { + const commandString = mod.settings.customInferenceProcessCommand.get(); + if (commandString) { + const parts = commandString.split(" "); + const customProcessBuilder = new ProcessBuilder(parts); + externalProcess = customProcessBuilder.start(); + // Initialize BufferedReader for stdout + processReader = new BufferedReader(new InputStreamReader(externalProcess.getInputStream())); + // Initialize processWriter for stdin of the inference engine + // @ts-expect-error + processWriter = new BufferedWriter(new OutputStreamWriter(externalProcess.getOutputStream())); + Client.displayChatMessage(`[DataCollector] Launched custom inference process: ${commandString}`); + } else { + Client.displayChatMessage(`[DataCollector] Custom inference process command is empty. Not launching.`); + } + } else { + const timestamp = new Date().toISOString().replace(/[:.]/g, "-"); + const logFileName = `${mod.settings.outputFilePrefix.get()}_${timestamp}.jsonl.zst`; + const processBuilder = new ProcessBuilder(["zstd", "-o", logFileName]); + externalProcess = processBuilder.start(); + const outputStream = externalProcess.getOutputStream(); + // @ts-expect-error + processWriter = new BufferedWriter(new OutputStreamWriter(outputStream)); // Renamed from processWriter + Client.displayChatMessage(`DataCollector enabled. Logging to ${logFileName}`); + } + collectedData = []; // Clear previous data historicalPlayerStates.length = 0; // Clear historical data lastYaw = mc.player?.yaw ?? null; lastPitch = mc.player?.pitch ?? null; - Client.displayChatMessage(`DataCollector enabled. Logging to ${logFileName}`); if (mod.settings.setBaritoneGoal.get()) { const goalX = mod.settings.goalX.get(); @@ -183,38 +219,54 @@ script.registerModule({ } } catch (e) { - Client.displayChatMessage(`[DataCollector] Failed to start zstd process or open log file: ${e}`); - logWriter = null; - zstdProcess = null; + Client.displayChatMessage(`[DataCollector] Failed to enable: ${e}`); + processWriter = null; + externalProcess = null; + processReader = null; // Ensure reader is nullified on error } }); mod.on("disable", () => { - if (logWriter) { + if (processWriter) { try { // Write any remaining collected data before closing collectedData.forEach(data => { - if (logWriter) { // Ensure logWriter is not null before writing + if (processWriter) { // Ensure processWriter is not null before writing // @ts-expect-error - logWriter.write(JSON.stringify(data)); - logWriter.newLine(); + processWriter.write(JSON.stringify(data)); + processWriter.newLine(); } }); - if (logWriter) { // Ensure logWriter is not null before closing - logWriter.close(); - } - Client.displayChatMessage("DataCollector disabled. Waiting for zstd to finish compression..."); - if (zstdProcess) { - const exitCode = zstdProcess.waitFor(); - Client.displayChatMessage(`zstd process exited with code: ${exitCode}`); + if (processWriter) { // Ensure processWriter is not null before closing + processWriter.close(); } - Client.displayChatMessage("Data file closed and compressed."); + Client.displayChatMessage("DataCollector disabled. Waiting for external process to finish..."); } catch (e) { - Client.displayChatMessage(`[DataCollector] Failed to close log file or wait for zstd: ${e}`); + Client.displayChatMessage(`[DataCollector] Failed to close log file: ${e}`); } - logWriter = null; - zstdProcess = null; + processWriter = null; } + + if (processReader) { + try { + processReader.close(); + } catch (e) { + Client.displayChatMessage(`[DataCollector] Failed to close process reader: ${e}`); + } + processReader = null; + } + + if (externalProcess) { + try { + const exitCode = externalProcess.waitFor(); + Client.displayChatMessage(`[DataCollector] External process exited with code: ${exitCode}`); + } catch (e) { + Client.displayChatMessage(`[DataCollector] Failed to wait for external process: ${e}`); + } + externalProcess.destroy(); // Ensure process is terminated + externalProcess = null; + } + if (visualizationManager) { visualizationManager.clearAllVisualizations(); } @@ -227,239 +279,334 @@ script.registerModule({ } }); - mod.on("gametick", (event: GameTickEvent) => { - if (!logWriter || !mc.player || !mc.world) { + + + mod.on("movementinput", (event: MovementInputEvent) => { + + if (!mc.player) return; + + // Handle inference process output if enabled + if (mod.settings.launchCustomInferenceProcess.get() && processReader) { + try { + while (processReader.ready()) { + const line = processReader.readLine(); + if (line) { + try { + const inferenceOutput: BaritoneAction = JSON.parse(line); + // Apply inferred actions to the player + if (inferenceOutput.look_change) { + const targetYaw = mc.player.yaw + inferenceOutput.look_change.yaw; + const targetPitch = mc.player.pitch + inferenceOutput.look_change.pitch; + RotationManager.INSTANCE.setRotationTarget( + new Rotation(Primitives.float(targetYaw), Primitives.float(targetPitch), true), + false, + KillAuraRotationsConfigurable.INSTANCE, + Priority.IMPORTANT_FOR_USAGE_2, + mod, + null + ); + } + + let currentForward = mc.player.input.playerInput.forward(); + let currentBackward = mc.player.input.playerInput.backward(); + let currentLeft = mc.player.input.playerInput.left(); + let currentRight = mc.player.input.playerInput.right(); + let currentJump = mc.player.input.playerInput.jump(); + let currentSneak = mc.player.input.playerInput.sneak(); + let currentSprint = mc.player.input.playerInput.sprint(); + + if (inferenceOutput.move_direction) { + currentForward = false; + currentBackward = false; + currentLeft = false; + currentRight = false; + + switch (inferenceOutput.move_direction) { + case 'FORWARD': currentForward = true; break; + case 'BACKWARD': currentBackward = true; break; + case 'LEFT': currentLeft = true; break; + case 'RIGHT': currentRight = true; break; + case 'FORWARD_LEFT': currentForward = true; currentLeft = true; break; + case 'FORWARD_RIGHT': currentForward = true; currentRight = true; break; + case 'BACKWARD_LEFT': currentBackward = true; currentLeft = true; break; + case 'BACKWARD_RIGHT': currentBackward = true; currentRight = true; break; + } + } + + if (inferenceOutput.jump !== undefined && mc.player.onGround) { + currentJump = inferenceOutput.jump; + } + if (inferenceOutput.sneak !== undefined && !currentJump) { + currentSneak = inferenceOutput.sneak; + } + if (inferenceOutput.sprint !== undefined) { + currentSprint = inferenceOutput.sprint; + } + + // Reconstruct PlayerInput + event.jump = currentJump; + event.sneak = currentSneak; + event.directionalInput = new DirectionalInput( + currentForward, + currentBackward, + currentLeft, + currentRight + ); + + // mc.player.setSprinting(currentSprint) + + } catch (parseError) { + Client.displayChatMessage(`[DataCollector] Error parsing inference output: ${parseError} - Line: ${line}`); + console.error(parseError); + } + } + } + } catch (readError) { + Client.displayChatMessage(`[DataCollector] Error reading from inference process stdout: ${readError}`); + console.error(readError); + } } - if ((mc.player.age - lastCollectionTick) < (mod.settings.collectionInterval.get() as unknown as number)) { + }); + + mod.on("gametick", (event: GameTickEvent) => { + if (!mc.player || !mc.world) { return; } - lastCollectionTick = mc.player.age; - try { - // 1. Player State - const playerState: PlayerState = { - position: toCoordinates3D(mc.player.getPos()), - velocity: { vx: mc.player.getVelocity().x, vy: mc.player.getVelocity().y, vz: mc.player.getVelocity().z }, - look_direction: { yaw: mc.player.yaw, pitch: mc.player.pitch }, - player_pose: getPlayerPose(mc.player.getPose()), - ground_proximity: mc.player.isOnGround(), - predicted_passive_next_tick_state: { - // Simplified prediction: assumes constant velocity for one tick - predicted_pos: { - x: mc.player.getX() + mc.player.getVelocity().x, - y: mc.player.getY() + mc.player.getVelocity().y, - z: mc.player.getZ() + mc.player.getVelocity().z, - }, - predicted_vel: { - vx: mc.player.getVelocity().x, - vy: mc.player.getVelocity().y, - vz: mc.player.getVelocity().z, + + // Data collection logic (only if not launching custom inference process, or if you want to log both) + if (processWriter) { + if ((mc.player.age - lastCollectionTick) < (mod.settings.collectionInterval.get() as unknown as number)) { + return; + } + lastCollectionTick = mc.player.age; + + try { + // 1. Player State + const playerState: PlayerState = { + position: toCoordinates3D(mc.player.getPos()), + velocity: { vx: mc.player.getVelocity().x, vy: mc.player.getVelocity().y, vz: mc.player.getVelocity().z }, + look_direction: { yaw: mc.player.yaw, pitch: mc.player.pitch }, + player_pose: getPlayerPose(mc.player.getPose()), + ground_proximity: mc.player.isOnGround(), + predicted_passive_next_tick_state: { + // Simplified prediction: assumes constant velocity for one tick + predicted_pos: { + x: mc.player.getX() + mc.player.getVelocity().x, + y: mc.player.getY() + mc.player.getVelocity().y, + z: mc.player.getZ() + mc.player.getVelocity().z, + }, + predicted_vel: { + vx: mc.player.getVelocity().x, + vy: mc.player.getVelocity().y, + vz: mc.player.getVelocity().z, + } } + }; + + // 2. Historical Player States + const currentHistoricalState: HistoricalPlayerState = { + position: toCoordinates3D(mc.player.getPos()), + velocity: { vx: mc.player.getVelocity().x, vy: mc.player.getVelocity().y, vz: mc.player.getVelocity().z }, + look_direction: { yaw: mc.player.yaw, pitch: mc.player.pitch }, + player_pose: getPlayerPose(mc.player.getPose()), + fall_distance: mc.player.fallDistance // Assuming fallDistance is directly accessible + }; + historicalPlayerStates.push(currentHistoricalState); + if (historicalPlayerStates.length > HISTORY_SIZE) { + historicalPlayerStates.shift(); // Remove oldest entry } - }; - - // 2. Historical Player States - const currentHistoricalState: HistoricalPlayerState = { - position: toCoordinates3D(mc.player.getPos()), - velocity: { vx: mc.player.getVelocity().x, vy: mc.player.getVelocity().y, vz: mc.player.getVelocity().z }, - look_direction: { yaw: mc.player.yaw, pitch: mc.player.pitch }, - player_pose: getPlayerPose(mc.player.getPose()), - fall_distance: mc.player.fallDistance // Assuming fallDistance is directly accessible - }; - historicalPlayerStates.push(currentHistoricalState); - if (historicalPlayerStates.length > HISTORY_SIZE) { - historicalPlayerStates.shift(); // Remove oldest entry - } - // 3. Local Environment Scan - const fixedRadiusScan: CollisionBox[] = []; - const dynamicInterestScan: CollisionBox[] = []; - - const playerBlockPos = mc.player.getBlockPos(); - const scanRadius = mod.settings.scanRadius.get() as unknown as number; - - // Collect FIXED_RADIUS blocks - for (let x = -scanRadius; x <= scanRadius; x++) { - for (let y = -scanRadius; y <= scanRadius; y++) { - for (let z = -scanRadius; z <= scanRadius; z++) { - const blockPos = new BlockPos(playerBlockPos.getX() + x, playerBlockPos.getY() + y, playerBlockPos.getZ() + z); - const blockState = mc.world.getBlockState(blockPos); - - if (blockState && !blockState.isAir()) { - const blockBoxes = blockState.getCollisionShape(mc.world as unknown as BlockView, blockPos).getBoundingBoxes(); - for (const blockBox of blockBoxes) { - const relativePos = { - x: blockPos.getX() - mc.player.getX(), - y: blockPos.getY() - mc.player.getY(), - z: blockPos.getZ() - mc.player.getZ() - }; - - fixedRadiusScan.push({ - bounding_box_coordinates: toBoundingBoxCoordinates(blockBox), - relative_position: relativePos, - box_dimensions: calculateBoxDimensions(blockBox), - element_identifier: blockState.getBlock().getName().getString(), - area_source_type: 'FIXED_RADIUS' - }); + // 3. Local Environment Scan + const fixedRadiusScan: CollisionBox[] = []; + const dynamicInterestScan: CollisionBox[] = []; + + const playerBlockPos = mc.player.getBlockPos(); + const scanRadius = mod.settings.scanRadius.get() as unknown as number; + + // Collect FIXED_RADIUS blocks + for (let x = -scanRadius; x <= scanRadius; x++) { + for (let y = -scanRadius; y <= scanRadius; y++) { + for (let z = -scanRadius; z <= scanRadius; z++) { + const blockPos = new BlockPos(playerBlockPos.getX() + x, playerBlockPos.getY() + y, playerBlockPos.getZ() + z); + const blockState = mc.world.getBlockState(blockPos); + + if (blockState && !blockState.isAir()) { + const blockBoxes = blockState.getCollisionShape(mc.world as unknown as BlockView, blockPos).getBoundingBoxes(); + for (const blockBox of blockBoxes) { + const relativePos = { + x: blockPos.getX() - mc.player.getX(), + y: blockPos.getY() - mc.player.getY(), + z: blockPos.getZ() - mc.player.getZ() + }; + + fixedRadiusScan.push({ + bounding_box_coordinates: toBoundingBoxCoordinates(blockBox), + relative_position: relativePos, + box_dimensions: calculateBoxDimensions(blockBox), + element_identifier: blockState.getBlock().getName().getString(), + area_source_type: 'FIXED_RADIUS' + }); + } } } } } - } - // Collect DYNAMIC_INTEREST blocks along Baritone's path with expansion - const baritone = BaritoneAPI.getProvider().getPrimaryBaritone(); - // @ts-expect-error - const currentPath: IPath | null = baritone.getPathingBehavior().getPath().orElseGet(() => null); - - - - if (currentPath && currentPath != prevPath) { - dynamicInterestBlockSet = new HashSet(); - prevPath = currentPath; - const pathPositions = currentPath.positions(); - const horizontalExpand = 1; - const downwardExpand = 1; - const upwardExpand = 2; - - for (const pathBlock of pathPositions) { - for (let x = -horizontalExpand; x <= horizontalExpand; x++) { - for (let y = -downwardExpand; y <= upwardExpand; y++) { - for (let z = -horizontalExpand; z <= horizontalExpand; z++) { - const blockPos = new BlockPos(pathBlock.getX() + x, pathBlock.getY() + y, pathBlock.getZ() + z); - if (dynamicInterestBlockSet.contains(blockPos)) - continue - else - dynamicInterestBlockSet.add(blockPos) + // Collect DYNAMIC_INTEREST blocks along Baritone's path with expansion + const baritone = BaritoneAPI.getProvider().getPrimaryBaritone(); + // @ts-expect-error + const currentPath: IPath | null = baritone.getPathingBehavior().getPath().orElseGet(() => null); + + + + if (currentPath && currentPath != prevPath) { + dynamicInterestBlockSet = new HashSet(); + prevPath = currentPath; + const pathPositions = currentPath.positions(); + const horizontalExpand = 1; + const downwardExpand = 1; + const upwardExpand = 2; + + for (const pathBlock of pathPositions) { + for (let x = -horizontalExpand; x <= horizontalExpand; x++) { + for (let y = -downwardExpand; y <= upwardExpand; y++) { + for (let z = -horizontalExpand; z <= horizontalExpand; z++) { + const blockPos = new BlockPos(pathBlock.getX() + x, pathBlock.getY() + y, pathBlock.getZ() + z); + if (dynamicInterestBlockSet.contains(blockPos)) + continue + else + dynamicInterestBlockSet.add(blockPos) + } } } } } - } - dynamicInterestBlockSet.forEach((blockPos: BlockPos) => { - let blockState = mc.world.getBlockState(blockPos); - if (blockState && !blockState.isAir()) { - if (!mc.player) - return; - const blockBoxes = blockState.getCollisionShape(mc.world as unknown as BlockView, blockPos).getBoundingBoxes(); - for (const blockBox of blockBoxes) { - const relativePos = { - x: blockPos.getX() - mc.player.getX(), - y: blockPos.getY() - mc.player.getY(), - z: blockPos.getZ() - mc.player.getZ() - }; - - const collisionBox: CollisionBox = { - bounding_box_coordinates: toBoundingBoxCoordinates(blockBox), - relative_position: relativePos, - box_dimensions: calculateBoxDimensions(blockBox), - element_identifier: blockState.getBlock().getName().getString(), - area_source_type: 'DYNAMIC_INTEREST' - }; - dynamicInterestScan.push(collisionBox); + dynamicInterestBlockSet.forEach((blockPos: BlockPos) => { + let blockState = mc.world.getBlockState(blockPos); + if (blockState && !blockState.isAir()) { + if (!mc.player) + return; + const blockBoxes = blockState.getCollisionShape(mc.world as unknown as BlockView, blockPos).getBoundingBoxes(); + for (const blockBox of blockBoxes) { + const relativePos = { + x: blockPos.getX() - mc.player.getX(), + y: blockPos.getY() - mc.player.getY(), + z: blockPos.getZ() - mc.player.getZ() + }; + + const collisionBox: CollisionBox = { + bounding_box_coordinates: toBoundingBoxCoordinates(blockBox), + relative_position: relativePos, + box_dimensions: calculateBoxDimensions(blockBox), + element_identifier: blockState.getBlock().getName().getString(), + area_source_type: 'DYNAMIC_INTEREST' + }; + dynamicInterestScan.push(collisionBox); + } } + }); + const localEnvironmentScan: CollisionBox[] = [...fixedRadiusScan, ...dynamicInterestScan]; + + // 4. Baritone Action (This is the most challenging part without direct API) + // This will require introspection into Baritone's internal state or inferring from its movement. + // For MVP, we might need to simplify or make assumptions. + // Let's try to infer based on Baritone's current movement state. + // This is a placeholder and needs significant refinement. + const baritoneAction: BaritoneAction = { + move_direction: 'NONE', + look_change: { yaw: 0, pitch: 0 }, + jump: false, + sneak: false, + sprint: false + }; + + // Infer Baritone's action based on mc.player.input + const playerInput: PlayerInput = mc.player.input.playerInput; + + + if (playerInput.forward() && playerInput.left()) { + baritoneAction.move_direction = 'FORWARD_LEFT'; + } else if (playerInput.forward() && playerInput.right()) { + baritoneAction.move_direction = 'FORWARD_RIGHT'; + } else if (playerInput.backward() && playerInput.left()) { + baritoneAction.move_direction = 'BACKWARD_LEFT'; + } else if (playerInput.backward() && playerInput.right()) { + baritoneAction.move_direction = 'BACKWARD_RIGHT'; + } else if (playerInput.forward()) { + baritoneAction.move_direction = 'FORWARD'; + } else if (playerInput.backward()) { + baritoneAction.move_direction = 'BACKWARD'; + } else if (playerInput.left()) { + baritoneAction.move_direction = 'LEFT'; + } else if (playerInput.right()) { + baritoneAction.move_direction = 'RIGHT'; + } else { + baritoneAction.move_direction = 'NONE'; } - }); - const localEnvironmentScan: CollisionBox[] = [...fixedRadiusScan, ...dynamicInterestScan]; - - // 4. Baritone Action (This is the most challenging part without direct API) - // This will require introspection into Baritone's internal state or inferring from its movement. - // For MVP, we might need to simplify or make assumptions. - // Let's try to infer based on Baritone's current movement state. - // This is a placeholder and needs significant refinement. - const baritoneAction: BaritoneAction = { - move_direction: 'NONE', - look_change: { yaw: 0, pitch: 0 }, - jump: false, - sneak: false, - sprint: false - }; - - // Infer Baritone's action based on mc.player.input - const playerInput: PlayerInput = mc.player.input.playerInput; - - - if (playerInput.forward() && playerInput.left()) { - baritoneAction.move_direction = 'FORWARD_LEFT'; - } else if (playerInput.forward() && playerInput.right()) { - baritoneAction.move_direction = 'FORWARD_RIGHT'; - } else if (playerInput.backward() && playerInput.left()) { - baritoneAction.move_direction = 'BACKWARD_LEFT'; - } else if (playerInput.backward() && playerInput.right()) { - baritoneAction.move_direction = 'BACKWARD_RIGHT'; - } else if (playerInput.forward()) { - baritoneAction.move_direction = 'FORWARD'; - } else if (playerInput.backward()) { - baritoneAction.move_direction = 'BACKWARD'; - } else if (playerInput.left()) { - baritoneAction.move_direction = 'LEFT'; - } else if (playerInput.right()) { - baritoneAction.move_direction = 'RIGHT'; - } else { - baritoneAction.move_direction = 'NONE'; - } - baritoneAction.jump = playerInput.jump(); - baritoneAction.sneak = playerInput.sneak(); - baritoneAction.sprint = mc.player.isSprinting(); // TODO: special care for the bot when inferencing. + baritoneAction.jump = playerInput.jump(); + baritoneAction.sneak = playerInput.sneak(); + baritoneAction.sprint = mc.player.isSprinting(); // TODO: special care for the bot when inferencing. - let yawDiff = 0; - let pitchDiff = 0; + let yawDiff = 0; + let pitchDiff = 0; - if (lastYaw !== null && lastPitch !== null) { - yawDiff = mc.player.yaw - lastYaw; - pitchDiff = mc.player.pitch - lastPitch; - } + if (lastYaw !== null && lastPitch !== null) { + yawDiff = mc.player.yaw - lastYaw; + pitchDiff = mc.player.pitch - lastPitch; + } - baritoneAction.look_change = { yaw: yawDiff, pitch: pitchDiff }; - - lastYaw = mc.player.yaw; - lastPitch = mc.player.pitch; - - - - const tickData: TickData = { - tick_id: mc.player.age, - timestamp_ms: Date.now(), - player_state: playerState, - local_environment_scan: localEnvironmentScan, - historical_player_states: [...historicalPlayerStates], // Clone to avoid mutation - baritone_action: baritoneAction - }; - - // @ts-expect-error - logWriter.write(JSON.stringify(tickData)); - logWriter.newLine(); - - // Visualize collected boxes if enabled - tickCounter++; - if (mod.settings.visualizeBoxes.get() && !(tickCounter % 5)) { - localEnvironmentScan.forEach(collisionBox => { - const box = toMinecraftBox(collisionBox.bounding_box_coordinates); - const position = toAbsoluteVec3d(collisionBox.relative_position, mc.player!.getPos()); - const fillColor = collisionBox.area_source_type === 'DYNAMIC_INTEREST' - ? new Color4b(150, 255, 150, 150) // Lighter Green for DYNAMIC_INTEREST - : new Color4b(150, 150, 255, 150); // Lighter Blue for FIXED_RADIUS - - visualizationManager.addVisualization({ - durationTicks: 5, // Display for 2 ticks - boxData: { - box: box, - position: position, - glow: true, // No glow - fillInterpolator: fadeOutInterpolatorFrom(fillColor), - outlineInterpolator: fadeOutInterpolatorFrom(new Color4b(fillColor.r(), fillColor.g(), fillColor.b(), 255)) - } + baritoneAction.look_change = { yaw: yawDiff, pitch: pitchDiff }; + + lastYaw = mc.player.yaw; + lastPitch = mc.player.pitch; + + + + const tickData: TickData = { + tick_id: mc.player.age, + timestamp_ms: Date.now(), + player_state: playerState, + local_environment_scan: localEnvironmentScan, + historical_player_states: [...historicalPlayerStates], // Clone to avoid mutation + baritone_action: baritoneAction + }; + + // @ts-expect-error + processWriter.write(JSON.stringify(tickData)); + processWriter.newLine(); + + // Visualize collected boxes if enabled + tickCounter++; + if (mod.settings.visualizeBoxes.get() && !(tickCounter % 5)) { + localEnvironmentScan.forEach(collisionBox => { + const box = toMinecraftBox(collisionBox.bounding_box_coordinates); + const position = toAbsoluteVec3d(collisionBox.relative_position, mc.player!.getPos()); + const fillColor = collisionBox.area_source_type === 'DYNAMIC_INTEREST' + ? new Color4b(150, 255, 150, 150) // Lighter Green for DYNAMIC_INTEREST + : new Color4b(150, 150, 255, 150); // Lighter Blue for FIXED_RADIUS + + visualizationManager.addVisualization({ + durationTicks: 5, // Display for 2 ticks + boxData: { + box: box, + position: position, + glow: true, // No glow + fillInterpolator: fadeOutInterpolatorFrom(fillColor), + outlineInterpolator: fadeOutInterpolatorFrom(new Color4b(fillColor.r(), fillColor.g(), fillColor.b(), 255)) + } + }); }); - }); - } + } - } catch (e) { - Client.displayChatMessage(`[DataCollector] Error writing to log: ${e}`); - console.error(e); // Log to console for more details + } catch (e) { + Client.displayChatMessage(`[DataCollector] Error writing to log: ${e}`); + console.error(e); // Log to console for more details + } } }); }); From 1b9dcb89c7e8b25bd0f7acd9085abaf74e927597 Mon Sep 17 00:00:00 2001 From: commandblock2 Date: Sat, 16 Aug 2025 23:45:24 +0800 Subject: [PATCH 10/15] feat: includes all blocks in online inference --- src/data-collector.ts | 87 +++++++++++++++++++++++++++++++++++-------- 1 file changed, 71 insertions(+), 16 deletions(-) diff --git a/src/data-collector.ts b/src/data-collector.ts index cee1277..dcee814 100644 --- a/src/data-collector.ts +++ b/src/data-collector.ts @@ -25,6 +25,11 @@ import { BaritoneAPI } from "jvm-types/baritone/api/BaritoneAPI"; import { IPath } from "jvm-types/baritone/api/pathing/calc/IPath"; import { GoalBlock } from "jvm-types/baritone/api/pathing/goals/GoalBlock"; import { PlayerInput } from "jvm-types/net/minecraft/util/PlayerInput"; +import { BetterBlockPos } from "jvm-types/baritone/api/utils/BetterBlockPos"; +import { AStarPathFinder } from "jvm-types/baritone/pathing/calc/AStarPathFinder"; +import { Favoring } from "jvm-types/baritone/utils/pathing/Favoring"; +import { CalculationContext } from "jvm-types/baritone/pathing/movement/CalculationContext"; +import { PathCalculationResult$Type } from "jvm-types/baritone/api/utils/PathCalculationResult$Type"; import { CombinedInputAndOutputDataSchema, @@ -172,7 +177,14 @@ script.registerModule({ const visualizationManager = new VisualizationManager(mod); let tickCounter = 0; + // Internal tick counter that is stable across player deaths/world changes. + // mc.player.age resets when the player dies or changes world; use this counter + // to represent monotonically increasing gameticks while this module is enabled. + let internalTick = 0; let prevPath: IPath | null = null; + let latestComputedPath: IPath | null = null; + let lastAStarTick = 0; + const ASTAR_RECALC_INTERVAL = 20; // let dynamicInterestScan: CollisionBox[] = []; let dynamicInterestBlockSet: HashSet = new HashSet(); @@ -378,10 +390,12 @@ script.registerModule({ // Data collection logic (only if not launching custom inference process, or if you want to log both) if (processWriter) { - if ((mc.player.age - lastCollectionTick) < (mod.settings.collectionInterval.get() as unknown as number)) { + // increment our internal tick counter for each processed gametick when player/world are available + internalTick++; + if ((internalTick - lastCollectionTick) < (mod.settings.collectionInterval.get() as unknown as number)) { return; } - lastCollectionTick = mc.player.age; + lastCollectionTick = internalTick; try { // 1. Player State @@ -455,30 +469,70 @@ script.registerModule({ } } - // Collect DYNAMIC_INTEREST blocks along Baritone's path with expansion - const baritone = BaritoneAPI.getProvider().getPrimaryBaritone(); - // @ts-expect-error - const currentPath: IPath | null = baritone.getPathingBehavior().getPath().orElseGet(() => null); - - - - if (currentPath && currentPath != prevPath) { + // Collect DYNAMIC_INTEREST blocks along a path. + // When a custom inference process is launched (online inference mode), we compute an A* path + // asynchronously and use it as the source of dynamic interest blocks. Otherwise fall back to + // Baritone's current path. + let computedPath: IPath | null = null; + if (mod.settings.launchCustomInferenceProcess.get()) { + // Periodically schedule an asynchronous A* path calculation so we don't block the client thread. + if ((internalTick - lastAStarTick) >= ASTAR_RECALC_INTERVAL) { + lastAStarTick = internalTick; + try { + const baritone = BaritoneAPI.getProvider().getPrimaryBaritone(); + const playerPos = baritone.getPlayerContext().playerFeet(); + const start = new BetterBlockPos(playerPos.getX(), playerPos.getY(), playerPos.getZ()); + const context = new CalculationContext(baritone, true); + const favoring = new Favoring(baritone.getPlayerContext(), prevPath as unknown as IPath, context); + const goal = new GoalBlock(mod.settings.goalX.get(), mod.settings.goalY.get(), mod.settings.goalZ.get()); + const pathfinder = new AStarPathFinder( + start, + start.getX(), + start.getY(), + start.getZ(), + goal, + favoring, + context + ); + // @ts-expect-error + UnsafeThread.run(() => { + const result = pathfinder.calculate(Primitives.long(2000), Primitives.long(5000)); + if (result.getType() != PathCalculationResult$Type.CANCELLATION) { + const path = result.getPath().get(); + mc.execute(() => { + // Store latest computed path for this gametick handler to pick up + latestComputedPath = path; + }); + } + }); + } catch (e) { + console.error(e); + } + } + computedPath = latestComputedPath; + } else { + const baritone = BaritoneAPI.getProvider().getPrimaryBaritone(); + // @ts-expect-error + computedPath = baritone.getPathingBehavior().getPath().orElseGet(() => null); + } + + if (computedPath && computedPath != prevPath) { dynamicInterestBlockSet = new HashSet(); - prevPath = currentPath; - const pathPositions = currentPath.positions(); + prevPath = computedPath; + const pathPositions = computedPath.positions(); const horizontalExpand = 1; const downwardExpand = 1; const upwardExpand = 2; - + for (const pathBlock of pathPositions) { for (let x = -horizontalExpand; x <= horizontalExpand; x++) { for (let y = -downwardExpand; y <= upwardExpand; y++) { for (let z = -horizontalExpand; z <= horizontalExpand; z++) { const blockPos = new BlockPos(pathBlock.getX() + x, pathBlock.getY() + y, pathBlock.getZ() + z); if (dynamicInterestBlockSet.contains(blockPos)) - continue + continue; else - dynamicInterestBlockSet.add(blockPos) + dynamicInterestBlockSet.add(blockPos); } } } @@ -568,7 +622,8 @@ script.registerModule({ const tickData: TickData = { - tick_id: mc.player.age, + // Use our internal tick counter instead of mc.player.age to avoid resets when player dies/changes world. + tick_id: internalTick, timestamp_ms: Date.now(), player_state: playerState, local_environment_scan: localEnvironmentScan, From c7e13df89b7e6c939b105fde71c988ad1fe76cf8 Mon Sep 17 00:00:00 2001 From: commandblock2 Date: Sun, 17 Aug 2025 17:08:40 +0800 Subject: [PATCH 11/15] feat: automatic scenario generation for training --- .../mvp/spec/mvp-spec.md | 39 +- src/scenario-generator.ts | 344 ++++++++++++++++++ 2 files changed, 377 insertions(+), 6 deletions(-) create mode 100644 src/scenario-generator.ts diff --git a/packages/deep-learning-bot-utils/mvp/spec/mvp-spec.md b/packages/deep-learning-bot-utils/mvp/spec/mvp-spec.md index 2a46cb5..01eba3f 100644 --- a/packages/deep-learning-bot-utils/mvp/spec/mvp-spec.md +++ b/packages/deep-learning-bot-utils/mvp/spec/mvp-spec.md @@ -150,17 +150,44 @@ Training will rely heavily on procedurally generating diverse and solvable navig * **Rule-Based/Grammar-Based:** Define a set of rules or a simple grammar that dictates how path segments and obstacles are placed. This allows for controlled variation with parameters (e.g., jump distance, turn radius, obstacle density). * **Randomized Parameters:** Introduce randomness in various parameters within the rules (e.g., path length, number of turns, height changes) to ensure unpredictability. +* **Verb-based / Verb-Sequence Generator (NEW):** To make generation expressive and easier to control/curriculum, the generator will produce a short sequence of high-level "verbs" (actions) — a small domain-specific language — and then render the world layout from that sequence. This maps directly to human-describable scenario patterns and is convenient for controlled curriculum and logging. + * Vocabulary (example): + * WALK(n) — advance n steps on flat ground + * TURN(direction) — change heading (LEFT / RIGHT) + * GAP(n) — create a gap of n blocks in the forward direction (Baritone-solvable constraint: n ≤ 3 for this MVP) + * JUMP_TO(h) — place a single-step up of height h (usually h = 1) + * CLIMB(n) — create an ascending run of n steps (stairs) + * DESCEND(n) — create a descending run of n steps + * OBSTACLE(density) — place small single-block obstacles in the path with given density + * NARROW(n) — force path width to n (1 or 2) + * Example grammar / short program example: + * [WALK(3), GAP(1), WALK(2), TURN(RIGHT), WALK(4), CLIMB(2), GAP(2), WALK(3)] + * Generation algorithm: + 1. Sample a difficulty (curriculum) parameter that bounds choices (max gap, rate of turns, verticality). + 2. Sample a sequence length L (e.g., 6–20 primitives depending on difficulty). + 3. Generate a verb sequence using weighted random choices from the vocabulary constrained by difficulty (e.g., at low difficulty, GAP probabilities are small and max gap = 1). + 4. Render the verb sequence into world coordinates starting from player's current block and facing. The renderer lays out blocks, gaps, stairs, obstacles accordingly. + 5. After rendering, set the Baritone goal to the final block of the rendered path and verify that Baritone can compute a path. If Baritone fails or computes an invalid path (e.g., due to a too-large gap), the generator should either (a) reduce the offending primitives and re-render, or (b) re-sample the verb sequence until a solvable one is produced. For MVP we prefer (b) with a bounded number of retries. + * Implementation notes: + * Keep path length concise (under 50 blocks). + * Ensure the verb renderer produces collision boxes count under the DataCollector performance budget (e.g., < 200). + * Enforce Baritone solvability constraint: maximum GAP ≤ 3 (Baritone can at most jump a 3-block gap in this environment). + * Expose the generated verb program alongside the world layout in the scenario metadata so training logs can replay or analyze the high-level structure of scenarios. * **In-Game Module:** A dedicated LiquidBounce script module will be responsible for: - 1. Clearing the active play area around the player. - 2. Constructing a new procedural path and/or environment. - 3. Setting Baritone's goal to the end of the newly generated path. - 4. Monitoring Baritone's success/failure for the current scenario. - 5. Triggering a new generation cycle upon completion or failure. + 1. Clearing the active play area around the player (keep the block under the player). + 2. Translating a generated verb-sequence program into a concrete world layout and placing the necessary blocks/gaps/obstacles. + 3. Setting Baritone's goal to the end of the newly generated path (DataCollector and other modules should be configured to point to the same goal). + 4. Monitoring Baritone's success/failure for the current scenario and marking the episode accordingly. + 5. Triggering a new generation cycle upon completion or failure (with optional difficulty adjustment for curriculum). * **Constraint:** Ensure generated paths are concise (e.g., under 50 blocks) and keep the total number of relevant collision boxes low (e.g., under 200) to respect data collector's performance limitations. **Curriculum Learning Integration:** -* The scenario generator will support a difficulty parameter. Training will begin with lower difficulty settings (e.g., mostly flat paths, simple jumps) and gradually increase this parameter as the agent's performance improves. This guides the learning process from simple tasks to more complex ones. +* The verb-sequence generator exposes a difficulty parameter that constrains sampling distributions: + * Low difficulty: mostly WALK, occasional 1-block GAP, few turns, limited vertical changes. + * Medium difficulty: more frequent turns, GAP up to 2, small climbs/descents, occasional obstacles. + * High difficulty: GAP up to 3, more verticality and turns, denser obstacles and narrow segments. +* Difficulty can be adapted online using agent performance statistics (success rate, time-to-goal). --- diff --git a/src/scenario-generator.ts b/src/scenario-generator.ts new file mode 100644 index 0000000..2b9d91a --- /dev/null +++ b/src/scenario-generator.ts @@ -0,0 +1,344 @@ +import { BlockPos } from "jvm-types/net/minecraft/util/math/BlockPos"; +import { ServerWorld } from "jvm-types/net/minecraft/server/world/ServerWorld"; +import { Blocks } from "jvm-types/net/minecraft/block/Blocks"; +import { Direction } from "jvm-types/net/minecraft/util/math/Direction"; +import { BaritoneAPI } from "jvm-types/baritone/api/BaritoneAPI"; +import { GoalBlock } from "jvm-types/baritone/api/pathing/goals/GoalBlock"; +// @ts-expect-error +const CommandManager = require("jvm-types/net/ccbluex/liquidbounce/features/command/CommandManager").CommandManager; +import { Thread } from "jvm-types/java/lang/Thread"; +import { LiquidBounce } from "jvm-types/net/ccbluex/liquidbounce/LiquidBounce"; + + +let shouldReenable = false; + +const script = registerScript.apply({ + name: "verb-scenario-generator", + version: "1.0.0", + authors: ["Roo"] +}); + +script.registerModule({ + name: "VerbScenarioGenerator", + description: "Generates verb-sequence scenarios (WALK,TURN,GAP,CLIMB) and renders them; clears area except the block under player; sets DataCollector goal and enables it.", + category: "World", + settings: { + difficulty: Setting.choose({ + name: "Difficulty", + default: "Easy", + choices: ["Easy", "Medium", "Hard"] + }), + maxLength: Setting.int({ + name: "MaxVerbSequenceLength", + default: 12, + range: [4, 20] + }), + clearRadius: Setting.int({ + name: "ClearRadius", + default: 8, + range: [3, 32] + }), + pathBlock: Setting.choose({ + name: "PathBlock", + default: "Oak_Planks", + choices: ["Stone", "Cobblestone", "Oak_Planks", "Dirt", "Glass"] + }), + placeTorches: Setting.boolean({ + name: "PlaceTorchesAlongPath", + default: false + }) + } +}, (mod) => { + + const datacollector = (() => { + for (const element of Client.moduleManager) { + if (element.name.startsWith("DataCollector")) + return element; + } + })(); + + function stepFor(direction: Direction) { + switch (direction) { + case Direction.NORTH: return { dx: 0, dz: -1 }; + case Direction.SOUTH: return { dx: 0, dz: 1 }; + case Direction.EAST: return { dx: 1, dz: 0 }; + case Direction.WEST: return { dx: -1, dz: 0 }; + default: return { dx: 0, dz: 1 }; + } + } + + function choosePathBlock(name: string) { + switch (name) { + case "Stone": return Blocks.STONE; + case "Cobblestone": return Blocks.COBBLESTONE; + case "Dirt": return Blocks.DIRT; + case "Glass": return Blocks.GLASS; + case "Oak_Planks": + default: return Blocks.OAK_PLANKS; + } + } + + type Verb = { type: string, arg?: number | string }; + + function sampleVerbSequence(difficulty: string, maxLen: number): Verb[] { + const seq: Verb[] = []; + const maxGap = difficulty === "Hard" ? 3 : (difficulty === "Medium" ? 2 : 1); + const turnProb = difficulty === "Hard" ? 0.25 : (difficulty === "Medium" ? 0.16 : 0.08); + const gapProb = difficulty === "Hard" ? 0.22 : (difficulty === "Medium" ? 0.14 : 0.06); + const climbProb = difficulty === "Hard" ? 0.16 : (difficulty === "Medium" ? 0.10 : 0.04); + + let remaining = maxLen; + while (remaining > 0) { + // prefer WALK segments + if (Math.random() < gapProb && remaining > 1) { + const g = 1 + Math.floor(Math.random() * (maxGap)); // 1..maxGap + seq.push({ type: "GAP", arg: g }); + remaining -= 1; + continue; + } + + if (Math.random() < turnProb) { + seq.push({ type: "TURN", arg: Math.random() < 0.5 ? "LEFT" : "RIGHT" }); + remaining--; + continue; + } + + if (Math.random() < climbProb && remaining > 1) { + const n = 1 + Math.floor(Math.random() * Math.min(3, remaining)); + seq.push({ type: "CLIMB", arg: n }); + remaining--; + continue; + } + + // default WALK small + const w = 1 + Math.floor(Math.random() * Math.min(4, remaining)); + seq.push({ type: "WALK", arg: w }); + remaining -= 1; + } + return seq; + } + + function renderVerbSequence(seq: Verb[], startPos: BlockPos, facing: Direction, serverWorld: ServerWorld, pathBlockState: any) { + let curX = startPos.getX(); + let curY = startPos.getY(); + let curZ = startPos.getZ(); + let dir = facing; + const base = stepFor(dir); + const pathPositions: BlockPos[] = []; + + // move one forward to avoid overwriting player's block + curX += base.dx; + curZ += base.dz; + + for (const verb of seq) { + if (verb.type === "WALK") { + const n = Number(verb.arg || 1); + for (let i = 0; i < n; i++) { + curX += stepFor(dir).dx; + curZ += stepFor(dir).dz; + const pos = new BlockPos(curX, curY, curZ); + serverWorld.setBlockState(pos, pathBlockState); + pathPositions.push(pos); + } + } else if (verb.type === "GAP") { + const n = Number(verb.arg || 1); + // advance forward n steps without placing blocks + for (let i = 0; i < n; i++) { + curX += stepFor(dir).dx; + curZ += stepFor(dir).dz; + } + // place landing block one forward + curX += stepFor(dir).dx; + curZ += stepFor(dir).dz; + const pos = new BlockPos(curX, curY, curZ); + serverWorld.setBlockState(pos, pathBlockState); + pathPositions.push(pos); + } else if (verb.type === "TURN") { + const side = String(verb.arg || "LEFT"); + // update dir + if (side === "LEFT") { + if (dir === Direction.NORTH) dir = Direction.WEST; + else if (dir === Direction.WEST) dir = Direction.SOUTH; + else if (dir === Direction.SOUTH) dir = Direction.EAST; + else dir = Direction.NORTH; + } else { + if (dir === Direction.NORTH) dir = Direction.EAST; + else if (dir === Direction.EAST) dir = Direction.SOUTH; + else if (dir === Direction.SOUTH) dir = Direction.WEST; + else dir = Direction.NORTH; + } + } else if (verb.type === "CLIMB") { + const n = Number(verb.arg || 1); + for (let i = 0; i < n; i++) { + // step up by one: place block at next forward and up + curX += stepFor(dir).dx; + curZ += stepFor(dir).dz; + curY += 1; + const pos = new BlockPos(curX, curY, curZ); + serverWorld.setBlockState(pos, pathBlockState); + pathPositions.push(pos); + } + } + } + + return pathPositions; + } + + let baritoneMonitorActive = false; + let pathStoppedTicks = 0; + const PATH_STOP_THRESHOLD = 20 * 5; // consecutive ticks with no active path before shutting down + + mod.on("enable", () => { + try { + if (!mc.player) { + Client.displayChatMessage("[VerbScenarioGenerator] Player not available."); + return; + } + + const server = mc.getServer(); + if (!server) { + Client.displayChatMessage("[VerbScenarioGenerator] Integrated server not running."); + return; + } + + const serverWorld = server.getOverworld(); + if (!serverWorld) { + Client.displayChatMessage("[VerbScenarioGenerator] Server world not found."); + return; + } + + const playerPos = mc.player.getBlockPos(); + const startX = playerPos.getX(); + const startY = playerPos.getY() - 1; + const startZ = playerPos.getZ(); + + const clearRadius = mod.settings.clearRadius.getValue(); + const air = Blocks.AIR.getDefaultState(); + for (let dx = -clearRadius; dx <= clearRadius; dx++) { + for (let dy = -clearRadius; dy <= clearRadius; dy++) { + for (let dz = -clearRadius; dz <= clearRadius; dz++) { + const tx = startX + dx; + const ty = startY + dy; + const tz = startZ + dz; + if (tx === startX && ty === startY && tz === startZ) continue; + const p = new BlockPos(tx, ty, tz); + serverWorld.setBlockState(p, air); + } + } + } + + const difficulty = mod.settings.difficulty.getValue(); + const maxLen = mod.settings.maxLength.getValue(); + const verbSeq = sampleVerbSequence(difficulty, maxLen); + + const blockChoice = mod.settings.pathBlock.getValue(); + const pathBlock = choosePathBlock(blockChoice); + const pathState = pathBlock.getDefaultState(); + + const facing = mc.player.getHorizontalFacing(); + const pathPositions = renderVerbSequence(verbSeq, playerPos, facing, serverWorld, pathState); + + if (mod.settings.placeTorches.getValue()) { + for (let i = 0; i < pathPositions.length; i += 5) { + const p = pathPositions[i]; + if (!p) continue; + try { + const tpos = new BlockPos(p.getX(), p.getY() + 1, p.getZ()); + if (Blocks.TORCH) serverWorld.setBlockState(tpos, Blocks.TORCH.getDefaultState()); + } catch (e) { } + } + } + + if (pathPositions.length === 0) { + Client.displayChatMessage("[VerbScenarioGenerator] Generated empty path."); + return; + } + + const goal = pathPositions[pathPositions.length - 1].add(0, 1, 0); + + + try { + const commandManager = CommandManager.INSTANCE; + commandManager.execute(`value set DataCollector GoalX ${goal.getX()}`); + commandManager.execute(`value set DataCollector GoalY ${goal.getY()}`); + commandManager.execute(`value set DataCollector GoalZ ${goal.getZ()}`); + datacollector!.enabled = true; + // start monitoring Baritone path status + baritoneMonitorActive = true; + } catch (e) { + Client.displayChatMessage(`[VerbScenarioGenerator] Failed to configure DataCollector via Baritone command manager: ${e}`); + } + + Client.displayChatMessage(`[VerbScenarioGenerator] Verb program: ${JSON.stringify(verbSeq)}`); + + } catch (e) { + Client.displayChatMessage(`[VerbScenarioGenerator] Error during generation: ${e}`); + console.error(e); + } + }); + + // Monitor Baritone path status and auto-disable modules when pathing stops + mod.on("gametick", () => { + if (!baritoneMonitorActive) return; + try { + const baritone = BaritoneAPI.getProvider().getPrimaryBaritone(); + if (!baritone) return; + // Try to get current path via PathingBehavior + // @ts-expect-error + const currentPath = baritone.getPathingBehavior().getPath().orElseGet(() => null); + let isPathActive = !!currentPath; + if (currentPath) { + try { + const positions = currentPath.positions ? currentPath.positions() : null; + if (!positions || (positions.length !== undefined && positions.length === 0)) { + isPathActive = false; + } + } catch (e) { + // ignore reflection errors + } + } + + if (!isPathActive) { + pathStoppedTicks++; + } else { + pathStoppedTicks = 0; + } + + if (pathStoppedTicks >= PATH_STOP_THRESHOLD) { + pathStoppedTicks = 0; + try { + if (datacollector!.enabled) { + datacollector!.enabled = false; + } + // disable this module + mod.enabled = false; + baritoneMonitorActive = false; + shouldReenable = true; + } catch (e) { + Client.displayChatMessage(`[VerbScenarioGenerator] Failed to auto-disable modules: ${e}`); + } + } + } catch (e) { + console.error(e); + } + }); + + mod.on("disable", () => { + Client.displayChatMessage("VerbScenarioGenerator disabled."); + }); + + // @ts-expect-error + UnsafeThread.run(() => { + while (true) { + while (!shouldReenable) + Thread.sleep(1000); + shouldReenable = false; + mod.enabled = true; + } + }) +}); + + + + +export { } \ No newline at end of file From 78344b6c1a7ab107411a91bcc382cf2df289d7e7 Mon Sep 17 00:00:00 2001 From: commandblock2 Date: Sat, 23 Aug 2025 17:03:14 +0800 Subject: [PATCH 12/15] chore: better scenario generator --- src/scenario-generator.ts | 411 +++++++++++++++++++++++++++++++------- 1 file changed, 335 insertions(+), 76 deletions(-) diff --git a/src/scenario-generator.ts b/src/scenario-generator.ts index 2b9d91a..32aa452 100644 --- a/src/scenario-generator.ts +++ b/src/scenario-generator.ts @@ -78,46 +78,153 @@ script.registerModule({ } } - type Verb = { type: string, arg?: number | string }; - + enum VerbType { + WALK = "WALK", + TURN = "TURN", + GAP = "GAP", + CLIMB = "CLIMB", + TUNNEL = "TUNNEL", // enclosed corridor (floor + ceiling) + PLATFORM_2X2 = "PLATFORM_2X2", // 2x2 platform + AIR_SINGLE = "AIR_SINGLE", // single air-gapped stepping blocks + SLAB = "SLAB", // place slabs instead of full blocks + DIAGONAL = "DIAGONAL" // diagonal step (NE, NW, SE, SW) + } + + type Verb = { type: VerbType, arg?: any }; + + const VerbInfo: Record = { + [VerbType.WALK]: { min: 1, max: 8, description: "place consecutive ground blocks" }, + [VerbType.TURN]: { description: "turn LEFT or RIGHT" }, + [VerbType.GAP]: { min: 1, max: 3, description: "leave empty spaces then a landing block" }, + [VerbType.CLIMB]: { min: 1, max: 3, description: "step up by placing blocks higher" }, + [VerbType.TUNNEL]: { min: 2, max: 8, description: "floor with ceiling; creates a tunnel" }, + [VerbType.PLATFORM_2X2]: { min: 1, max: 3, description: "2x2 platform steps" }, + [VerbType.AIR_SINGLE]: { min: 1, max: 6, description: "single air blocks separated by gaps" }, + [VerbType.SLAB]: { min: 1, max: 6, description: "place slabs for low-profile steps" }, + [VerbType.DIAGONAL]: { min: 1, max: 6, description: "diagonal movement (NE, NW, SE, SW)" } + }; + function sampleVerbSequence(difficulty: string, maxLen: number): Verb[] { + // probabilities influenced by difficulty const seq: Verb[] = []; - const maxGap = difficulty === "Hard" ? 3 : (difficulty === "Medium" ? 2 : 1); - const turnProb = difficulty === "Hard" ? 0.25 : (difficulty === "Medium" ? 0.16 : 0.08); - const gapProb = difficulty === "Hard" ? 0.22 : (difficulty === "Medium" ? 0.14 : 0.06); - const climbProb = difficulty === "Hard" ? 0.16 : (difficulty === "Medium" ? 0.10 : 0.04); - - let remaining = maxLen; - while (remaining > 0) { - // prefer WALK segments - if (Math.random() < gapProb && remaining > 1) { - const g = 1 + Math.floor(Math.random() * (maxGap)); // 1..maxGap - seq.push({ type: "GAP", arg: g }); - remaining -= 1; - continue; + const probs = { + WALK: difficulty === "Hard" ? 0.35 : (difficulty === "Medium" ? 0.45 : 0.6), + GAP: difficulty === "Hard" ? 0.18 : (difficulty === "Medium" ? 0.10 : 0.06), + TURN: difficulty === "Hard" ? 0.14 : (difficulty === "Medium" ? 0.12 : 0.08), + CLIMB: difficulty === "Hard" ? 0.12 : (difficulty === "Medium" ? 0.08 : 0.04), + TUNNEL: 0.04, + PLATFORM_2X2: 0.03, + SLAB: 0.03, + DIAGONAL: difficulty === "Hard" ? 0.05 : 0.02 + }; + + // helper to pick a verb by weighted random + function pickVerb(prev: VerbType): VerbType { + const items = Object.keys(probs) as (keyof typeof probs)[]; + const total = items.reduce((s, k) => s + probs[k], 0); + let r = Math.random() * total; + for (const k of items) { + r -= probs[k]; + if (r <= 0) { + const verb = VerbType[k as unknown as keyof typeof VerbType]; + if (verb == VerbType.GAP && verb == prev) + return VerbType.WALK; + else + return verb; + }; } - - if (Math.random() < turnProb) { - seq.push({ type: "TURN", arg: Math.random() < 0.5 ? "LEFT" : "RIGHT" }); - remaining--; - continue; + return VerbType.WALK; + } + + // Always start with a WALK so Baritone has an immediate walkable start + const firstWalkLen = 1 + Math.floor(Math.random() * Math.min(3, maxLen)); + seq.push({ type: VerbType.WALK, arg: firstWalkLen }); + let remaining = maxLen - 1; + + let prevVerb = VerbType.GAP; + + while (remaining > 0) { + const picked = pickVerb(prevVerb); + prevVerb = picked; + const info = VerbInfo[picked]; + // choose a reasonable argument based on verb metadata and remaining length + let n = 1; + if (info.min !== undefined) { + const maxForVerb = Math.min(info.max || info.min, remaining); + n = info.min + Math.floor(Math.random() * Math.max(1, maxForVerb - info.min + 1)); + } else { + n = Math.min(1 + Math.floor(Math.random() * 3), remaining); } - - if (Math.random() < climbProb && remaining > 1) { - const n = 1 + Math.floor(Math.random() * Math.min(3, remaining)); - seq.push({ type: "CLIMB", arg: n }); - remaining--; - continue; + + switch (picked) { + case VerbType.WALK: + seq.push({ type: VerbType.WALK, arg: n }); + remaining -= 1; + break; + case VerbType.GAP: + seq.push({ type: VerbType.GAP, arg: n }); + remaining -= 1; + break; + case VerbType.TURN: + seq.push({ type: VerbType.TURN, arg: Math.random() < 0.5 ? "LEFT" : "RIGHT" }); + remaining -= 1; + break; + case VerbType.CLIMB: + seq.push({ type: VerbType.CLIMB, arg: n }); + remaining -= 1; + break; + case VerbType.TUNNEL: + seq.push({ type: VerbType.TUNNEL, arg: n }); + remaining -= 1; + break; + case VerbType.PLATFORM_2X2: { + // create platform with possible multiple exits + const exits: string[] = []; + if (Math.random() < 0.85) exits.push("STRAIGHT"); + if (Math.random() < 0.35) exits.push("LEFT"); + if (Math.random() < 0.35) exits.push("RIGHT"); + // occasionally add a diagonal exit + if (Math.random() < 0.12) exits.push(Math.random() < 0.5 ? "NE" : "NW"); + if (exits.length === 0) exits.push("STRAIGHT"); + seq.push({ type: VerbType.PLATFORM_2X2, arg: { length: n, exits } }); + remaining -= 1; + break; + } + case VerbType.AIR_SINGLE: + seq.push({ type: VerbType.AIR_SINGLE, arg: n }); + remaining -= 1; + break; + case VerbType.SLAB: + seq.push({ type: VerbType.SLAB, arg: n }); + remaining -= 1; + break; + case VerbType.DIAGONAL: + // choose diagonal direction relative to facing + const diagonals = ["NE", "NW", "SE", "SW"]; + seq.push({ type: VerbType.DIAGONAL, arg: { steps: n, dir: diagonals[Math.floor(Math.random() * diagonals.length)] } }); + remaining -= 1; + break; + default: + seq.push({ type: VerbType.WALK, arg: 1 }); + remaining -= 1; + break; } - - // default WALK small - const w = 1 + Math.floor(Math.random() * Math.min(4, remaining)); - seq.push({ type: "WALK", arg: w }); - remaining -= 1; } + return seq; } - + + // Helper: step for diagonal direction + function stepForDiagonal(dirLabel: string) { + switch (dirLabel) { + case "NE": return { dx: 1, dz: -1 }; + case "NW": return { dx: -1, dz: -1 }; + case "SE": return { dx: 1, dz: 1 }; + case "SW": return { dx: -1, dz: 1 }; + default: return { dx: 0, dz: 1 }; + } + } + function renderVerbSequence(seq: Verb[], startPos: BlockPos, facing: Direction, serverWorld: ServerWorld, pathBlockState: any) { let curX = startPos.getX(); let curY = startPos.getY(); @@ -125,64 +232,208 @@ script.registerModule({ let dir = facing; const base = stepFor(dir); const pathPositions: BlockPos[] = []; - - // move one forward to avoid overwriting player's block + + // move one forward to avoid overwriting player's block -- keep small safe step curX += base.dx; curZ += base.dz; - + for (const verb of seq) { - if (verb.type === "WALK") { - const n = Number(verb.arg || 1); - for (let i = 0; i < n; i++) { + switch (verb.type) { + case VerbType.WALK: { + const n = Number(verb.arg || 1); + for (let i = 0; i < n; i++) { + curX += stepFor(dir).dx; + curZ += stepFor(dir).dz; + const pos = new BlockPos(curX, curY, curZ); + serverWorld.setBlockState(pos, pathBlockState); + pathPositions.push(pos); + } + break; + } + + case VerbType.GAP: { + const n = Number(verb.arg || 1); + for (let i = 0; i < n; i++) { + curX += stepFor(dir).dx; + curZ += stepFor(dir).dz; + } + // landing block curX += stepFor(dir).dx; curZ += stepFor(dir).dz; const pos = new BlockPos(curX, curY, curZ); serverWorld.setBlockState(pos, pathBlockState); pathPositions.push(pos); + break; } - } else if (verb.type === "GAP") { - const n = Number(verb.arg || 1); - // advance forward n steps without placing blocks - for (let i = 0; i < n; i++) { - curX += stepFor(dir).dx; - curZ += stepFor(dir).dz; + + case VerbType.TURN: { + const side = String(verb.arg || "LEFT"); + if (side === "LEFT") { + if (dir === Direction.NORTH) dir = Direction.WEST; + else if (dir === Direction.WEST) dir = Direction.SOUTH; + else if (dir === Direction.SOUTH) dir = Direction.EAST; + else dir = Direction.NORTH; + } else { + if (dir === Direction.NORTH) dir = Direction.EAST; + else if (dir === Direction.EAST) dir = Direction.SOUTH; + else if (dir === Direction.SOUTH) dir = Direction.WEST; + else dir = Direction.NORTH; + } + break; } - // place landing block one forward - curX += stepFor(dir).dx; - curZ += stepFor(dir).dz; - const pos = new BlockPos(curX, curY, curZ); - serverWorld.setBlockState(pos, pathBlockState); - pathPositions.push(pos); - } else if (verb.type === "TURN") { - const side = String(verb.arg || "LEFT"); - // update dir - if (side === "LEFT") { - if (dir === Direction.NORTH) dir = Direction.WEST; - else if (dir === Direction.WEST) dir = Direction.SOUTH; - else if (dir === Direction.SOUTH) dir = Direction.EAST; - else dir = Direction.NORTH; - } else { - if (dir === Direction.NORTH) dir = Direction.EAST; - else if (dir === Direction.EAST) dir = Direction.SOUTH; - else if (dir === Direction.SOUTH) dir = Direction.WEST; - else dir = Direction.NORTH; + + case VerbType.CLIMB: { + const n = Number(verb.arg || 1); + for (let i = 0; i < n; i++) { + curX += stepFor(dir).dx; + curZ += stepFor(dir).dz; + curY += 1; + const pos = new BlockPos(curX, curY, curZ); + serverWorld.setBlockState(pos, pathBlockState); + pathPositions.push(pos); + } + break; + } + + case VerbType.TUNNEL: { + // create floor and ceiling while leaving 1-block headroom + const n = Number(verb.arg || 2); + for (let i = 0; i < n; i++) { + curX += stepFor(dir).dx; + curZ += stepFor(dir).dz; + // floor + const floor = new BlockPos(curX, curY, curZ); + serverWorld.setBlockState(floor, pathBlockState); + pathPositions.push(floor); + // ceiling two blocks above floor (y+2) to form tunnel + const ceiling = new BlockPos(curX, curY + 3, curZ); + serverWorld.setBlockState(ceiling, pathBlockState); + } + break; + } + + case VerbType.PLATFORM_2X2: { + const n = Number(verb.arg || 1); + for (let i = 0; i < n; i++) { + // place a 2x2 square centered on current forward step + curX += stepFor(dir).dx; + curZ += stepFor(dir).dz; + for (let sx = 0; sx <= 1; sx++) { + for (let sz = 0; sz <= 1; sz++) { + const ox = curX + (dir === Direction.EAST ? sx : dir === Direction.WEST ? -sx : sx - 1); + const oz = curZ + (dir === Direction.SOUTH ? sz : dir === Direction.NORTH ? -sz : sz - 1); + const p = new BlockPos(ox, curY, oz); + serverWorld.setBlockState(p, pathBlockState); + pathPositions.push(p); + } + } + } + break; + } + + case VerbType.AIR_SINGLE: { + const n = Number(verb.arg || 1); + for (let i = 0; i < n; i++) { + // place a single block, then advance with a gap of one + curX += stepFor(dir).dx; + curZ += stepFor(dir).dz; + const p = new BlockPos(curX, curY, curZ); + serverWorld.setBlockState(p, pathBlockState); + pathPositions.push(p); + // gap + curX += stepFor(dir).dx; + curZ += stepFor(dir).dz; + } + break; + } + + case VerbType.SLAB: { + // attempt to place slab block if available; fall back to full block + const n = Number(verb.arg || 1); + const slabBlock = (Blocks.OAK_SLAB) ? Blocks.OAK_SLAB.getDefaultState() : pathBlockState; + for (let i = 0; i < n; i++) { + curX += stepFor(dir).dx; + curZ += stepFor(dir).dz; + const p = new BlockPos(curX, curY, curZ); + serverWorld.setBlockState(p, slabBlock); + pathPositions.push(p); + } + break; + } + + case VerbType.DIAGONAL: { + const info = verb.arg || { steps: 1, dir: "NE" }; + const steps = Number(info.steps || 1); + const dlabel = String(info.dir || "NE"); + for (let i = 0; i < steps; i++) { + const step = stepForDiagonal(dlabel); + curX += step.dx; + curZ += step.dz; + const p = new BlockPos(curX, curY, curZ); + serverWorld.setBlockState(p, pathBlockState); + pathPositions.push(p); + } + break; } - } else if (verb.type === "CLIMB") { - const n = Number(verb.arg || 1); - for (let i = 0; i < n; i++) { - // step up by one: place block at next forward and up + + default: { + // unknown => safe WALK curX += stepFor(dir).dx; curZ += stepFor(dir).dz; - curY += 1; - const pos = new BlockPos(curX, curY, curZ); - serverWorld.setBlockState(pos, pathBlockState); - pathPositions.push(pos); + const p = new BlockPos(curX, curY, curZ); + serverWorld.setBlockState(p, pathBlockState); + pathPositions.push(p); + break; } } } - - return pathPositions; + return pathPositions; +} + +function validateAndFixPath(serverWorld: ServerWorld, pathPositions: BlockPos[], pathBlockState: any): BlockPos[] { + try { + for (let i = 0; i < pathPositions.length; i++) { + const p = pathPositions[i]; + if (!p) continue; + + // ensure a support block exists directly beneath the path block (landing support) + // previously we placed support blocks beneath every generated path block. + // That produced unwanted 2-block-high columns when the generator placed a single ground block. + // Remove automatic support placement here to preserve single-block-high paths. + // If needed, supports should be added with a separate, careful pass that only fills truly unsupported blocks. + + // fix excessive upward steps between consecutive path positions + if (i > 0) { + try { + const prev = pathPositions[i - 1]; + const dy = p.getY() - prev.getY(); + // if step up is greater than 1 block, fill vertical intermediate blocks + if (dy > 1) { + for (let y = prev.getY() + 1; y < p.getY(); y++) { + try { + const fill = new BlockPos(p.getX(), y, p.getZ()); + serverWorld.setBlockState(fill, pathBlockState); + } catch (e) { } + } + } + // if step down is large (fall), fill beneath previous position so descent is gradual + if (dy < -1) { + for (let y = p.getY(); y < prev.getY() - 1; y++) { + try { + const fill = new BlockPos(prev.getX(), y, prev.getZ()); + serverWorld.setBlockState(fill, pathBlockState); + } catch (e) { } + } + } + } catch (e) { } + } + } + } catch (e) { + console.error("[VerbScenarioGenerator] validateAndFixPath error:", e); } + return pathPositions; +} + let baritoneMonitorActive = false; let pathStoppedTicks = 0; @@ -220,7 +471,14 @@ script.registerModule({ const tx = startX + dx; const ty = startY + dy; const tz = startZ + dz; - if (tx === startX && ty === startY && tz === startZ) continue; + // preserve the player's foot block, the block below it and the block above it + // to avoid clearing the player's standing block or causing them to fall. + const foot = playerPos; + const belowFoot = new BlockPos(foot.getX(), foot.getY() - 1, foot.getZ()); + const aboveFoot = new BlockPos(foot.getX(), foot.getY() + 1, foot.getZ()); + if ((tx === foot.getX() && ty === foot.getY() && tz === foot.getZ()) + || (tx === belowFoot.getX() && ty === belowFoot.getY() && tz === belowFoot.getZ()) + || (tx === aboveFoot.getX() && ty === aboveFoot.getY() && tz === aboveFoot.getZ())) continue; const p = new BlockPos(tx, ty, tz); serverWorld.setBlockState(p, air); } @@ -236,8 +494,9 @@ script.registerModule({ const pathState = pathBlock.getDefaultState(); const facing = mc.player.getHorizontalFacing(); - const pathPositions = renderVerbSequence(verbSeq, playerPos, facing, serverWorld, pathState); - + const rawPathPositions = renderVerbSequence(verbSeq, new BlockPos(startX, startY, startZ), facing, serverWorld, pathState); + const pathPositions = validateAndFixPath(serverWorld, rawPathPositions, pathState); + if (mod.settings.placeTorches.getValue()) { for (let i = 0; i < pathPositions.length; i += 5) { const p = pathPositions[i]; From b3da4f042a987ea951207359e36ca92f4a3e3c18 Mon Sep 17 00:00:00 2001 From: commandblock2 Date: Sat, 23 Aug 2025 23:22:38 +0800 Subject: [PATCH 13/15] chore: some improvements? --- src/data-collector.ts | 86 +++++++++++++---------- src/scenario-generator.ts | 141 +++++++++++++++++++++++++++++--------- 2 files changed, 156 insertions(+), 71 deletions(-) diff --git a/src/data-collector.ts b/src/data-collector.ts index dcee814..d8fb73f 100644 --- a/src/data-collector.ts +++ b/src/data-collector.ts @@ -258,7 +258,7 @@ script.registerModule({ } processWriter = null; } - + if (processReader) { try { processReader.close(); @@ -267,7 +267,7 @@ script.registerModule({ } processReader = null; } - + if (externalProcess) { try { const exitCode = externalProcess.waitFor(); @@ -278,11 +278,25 @@ script.registerModule({ externalProcess.destroy(); // Ensure process is terminated externalProcess = null; } - + if (visualizationManager) { visualizationManager.clearAllVisualizations(); } - + + // Clear retained caches and path references so GC can reclaim memory + try { + if (dynamicInterestBlockSet) { + try { + dynamicInterestBlockSet.clear(); + } catch (e) { /* ignore */ } + dynamicInterestBlockSet = new HashSet(); + } + } catch (e) { /* ignore */ } + + // Drop cached path references so large path graphs can be GC'd + latestComputedPath = null; + prevPath = null; + // Clear Baritone goal on disable if (mod.settings.setBaritoneGoal.get()) { const baritone = BaritoneAPI.getProvider().getPrimaryBaritone(); @@ -441,30 +455,27 @@ script.registerModule({ const scanRadius = mod.settings.scanRadius.get() as unknown as number; // Collect FIXED_RADIUS blocks - for (let x = -scanRadius; x <= scanRadius; x++) { - for (let y = -scanRadius; y <= scanRadius; y++) { - for (let z = -scanRadius; z <= scanRadius; z++) { - const blockPos = new BlockPos(playerBlockPos.getX() + x, playerBlockPos.getY() + y, playerBlockPos.getZ() + z); - const blockState = mc.world.getBlockState(blockPos); - - if (blockState && !blockState.isAir()) { - const blockBoxes = blockState.getCollisionShape(mc.world as unknown as BlockView, blockPos).getBoundingBoxes(); - for (const blockBox of blockBoxes) { - const relativePos = { - x: blockPos.getX() - mc.player.getX(), - y: blockPos.getY() - mc.player.getY(), - z: blockPos.getZ() - mc.player.getZ() - }; - - fixedRadiusScan.push({ - bounding_box_coordinates: toBoundingBoxCoordinates(blockBox), - relative_position: relativePos, - box_dimensions: calculateBoxDimensions(blockBox), - element_identifier: blockState.getBlock().getName().getString(), - area_source_type: 'FIXED_RADIUS' - }); - } - } + const startScan = new BlockPos(playerBlockPos.getX() - scanRadius, playerBlockPos.getY() - scanRadius, playerBlockPos.getZ() - scanRadius); + const endScan = new BlockPos(playerBlockPos.getX() + scanRadius, playerBlockPos.getY() + scanRadius, playerBlockPos.getZ() + scanRadius); + for (const blockPos of BlockPos.iterate(startScan, endScan)) { + const blockState = mc.world.getBlockState(blockPos); + + if (blockState && !blockState.isAir()) { + const blockBoxes = blockState.getCollisionShape(mc.world as unknown as BlockView, blockPos).getBoundingBoxes(); + for (const blockBox of blockBoxes) { + const relativePos = { + x: blockPos.getX() - mc.player.getX(), + y: blockPos.getY() - mc.player.getY(), + z: blockPos.getZ() - mc.player.getZ() + }; + + fixedRadiusScan.push({ + bounding_box_coordinates: toBoundingBoxCoordinates(blockBox), + relative_position: relativePos, + box_dimensions: calculateBoxDimensions(blockBox), + element_identifier: blockState.getBlock().getName().getString(), + area_source_type: 'FIXED_RADIUS' + }); } } } @@ -525,16 +536,15 @@ script.registerModule({ const upwardExpand = 2; for (const pathBlock of pathPositions) { - for (let x = -horizontalExpand; x <= horizontalExpand; x++) { - for (let y = -downwardExpand; y <= upwardExpand; y++) { - for (let z = -horizontalExpand; z <= horizontalExpand; z++) { - const blockPos = new BlockPos(pathBlock.getX() + x, pathBlock.getY() + y, pathBlock.getZ() + z); - if (dynamicInterestBlockSet.contains(blockPos)) - continue; - else - dynamicInterestBlockSet.add(blockPos); - } - } + const start = new BlockPos(pathBlock.getX() - horizontalExpand, pathBlock.getY() - downwardExpand, pathBlock.getZ() - horizontalExpand); + const end = new BlockPos(pathBlock.getX() + horizontalExpand, pathBlock.getY() + upwardExpand, pathBlock.getZ() + horizontalExpand); + for (const blockPos of BlockPos.iterate(start, end)) { + // BlockPos.iterate typically reuses a mutable BlockPos instance for performance. + // Create a fresh immutable BlockPos copy before storing/checking in the HashSet so + // we don't insert a repeatedly-mutated object (which breaks set semantics). + const blockCopy = new BlockPos(blockPos.getX(), blockPos.getY(), blockPos.getZ()); + if (dynamicInterestBlockSet.contains(blockCopy)) continue; + dynamicInterestBlockSet.add(blockCopy); } } } diff --git a/src/scenario-generator.ts b/src/scenario-generator.ts index 32aa452..8456d00 100644 --- a/src/scenario-generator.ts +++ b/src/scenario-generator.ts @@ -11,6 +11,8 @@ import { LiquidBounce } from "jvm-types/net/ccbluex/liquidbounce/LiquidBounce"; let shouldReenable = false; +let reenableThreadStop = false; +let reenableThread: Thread | null = null; const script = registerScript.apply({ name: "verb-scenario-generator", @@ -31,7 +33,7 @@ script.registerModule({ maxLength: Setting.int({ name: "MaxVerbSequenceLength", default: 12, - range: [4, 20] + range: [4, 40] }), clearRadius: Setting.int({ name: "ClearRadius", @@ -125,11 +127,11 @@ script.registerModule({ let r = Math.random() * total; for (const k of items) { r -= probs[k]; - if (r <= 0) { + if (r <= 0) { const verb = VerbType[k as unknown as keyof typeof VerbType]; if (verb == VerbType.GAP && verb == prev) return VerbType.WALK; - else + else return verb; }; } @@ -140,7 +142,7 @@ script.registerModule({ const firstWalkLen = 1 + Math.floor(Math.random() * Math.min(3, maxLen)); seq.push({ type: VerbType.WALK, arg: firstWalkLen }); let remaining = maxLen - 1; - + let prevVerb = VerbType.GAP; while (remaining > 0) { @@ -211,7 +213,25 @@ script.registerModule({ } } - return seq; + // Ensure there is always at least one WALK between two non-WALK verbs. + // This post-process walks the sequence and inserts a WALK{1} whenever two adjacent verbs are both non-WALK. + function ensureWalkBetweenNonWalks(original: Verb[]): Verb[] { + const out: Verb[] = []; + for (let i = 0; i < original.length; i++) { + out.push(original[i]); + if (i + 1 < original.length) { + const curr = original[i]; + const next = original[i + 1]; + if (curr.type !== VerbType.WALK && next.type !== VerbType.WALK) { + // insert a short WALK to separate them + out.push({ type: VerbType.WALK, arg: 1 }); + } + } + } + return out; + } + + return ensureWalkBetweenNonWalks(seq); } // Helper: step for diagonal direction @@ -409,18 +429,20 @@ function validateAndFixPath(serverWorld: ServerWorld, pathPositions: BlockPos[], const dy = p.getY() - prev.getY(); // if step up is greater than 1 block, fill vertical intermediate blocks if (dy > 1) { - for (let y = prev.getY() + 1; y < p.getY(); y++) { + const startFill = new BlockPos(p.getX(), prev.getY() + 1, p.getZ()); + const endFill = new BlockPos(p.getX(), p.getY() - 1, p.getZ()); + for (const fill of BlockPos.iterate(startFill, endFill)) { try { - const fill = new BlockPos(p.getX(), y, p.getZ()); serverWorld.setBlockState(fill, pathBlockState); } catch (e) { } } } // if step down is large (fall), fill beneath previous position so descent is gradual if (dy < -1) { - for (let y = p.getY(); y < prev.getY() - 1; y++) { + const startFill = new BlockPos(prev.getX(), p.getY(), prev.getZ()); + const endFill = new BlockPos(prev.getX(), prev.getY() - 2, prev.getZ()); + for (const fill of BlockPos.iterate(startFill, endFill)) { try { - const fill = new BlockPos(prev.getX(), y, prev.getZ()); serverWorld.setBlockState(fill, pathBlockState); } catch (e) { } } @@ -465,26 +487,39 @@ function validateAndFixPath(serverWorld: ServerWorld, pathPositions: BlockPos[], const clearRadius = mod.settings.clearRadius.getValue(); const air = Blocks.AIR.getDefaultState(); - for (let dx = -clearRadius; dx <= clearRadius; dx++) { - for (let dy = -clearRadius; dy <= clearRadius; dy++) { - for (let dz = -clearRadius; dz <= clearRadius; dz++) { - const tx = startX + dx; - const ty = startY + dy; - const tz = startZ + dz; - // preserve the player's foot block, the block below it and the block above it - // to avoid clearing the player's standing block or causing them to fall. - const foot = playerPos; - const belowFoot = new BlockPos(foot.getX(), foot.getY() - 1, foot.getZ()); - const aboveFoot = new BlockPos(foot.getX(), foot.getY() + 1, foot.getZ()); - if ((tx === foot.getX() && ty === foot.getY() && tz === foot.getZ()) - || (tx === belowFoot.getX() && ty === belowFoot.getY() && tz === belowFoot.getZ()) - || (tx === aboveFoot.getX() && ty === aboveFoot.getY() && tz === aboveFoot.getZ())) continue; - const p = new BlockPos(tx, ty, tz); - serverWorld.setBlockState(p, air); - } - } + const areaStart = new BlockPos(startX - clearRadius, startY - clearRadius, startZ - clearRadius); + const areaEnd = new BlockPos(startX + clearRadius, startY + clearRadius, startZ + clearRadius); + const foot = playerPos; + const belowFoot = new BlockPos(foot.getX(), foot.getY() - 1, foot.getZ()); + const aboveFoot = new BlockPos(foot.getX(), foot.getY() + 1, foot.getZ()); + + // Enable player's flying capability and keep them flying (do not call sendAbilities here) + try { + mc.player.abilities.allowFlying = true; + mc.player.abilities.flying = true; + } catch (e) { } + + // iterate over all positions in the cubic area + // NOTE: intentionally do NOT preserve the supporting block under the player. + // We only avoid clearing the player's occupied block to prevent immediate suffocation. + for (const p of BlockPos.iterate(areaStart, areaEnd)) { + try { + serverWorld.setBlockState(p, air); + } catch (e) { } } + // Place a block under the player's foot while they are flying so they don't fall when flying is disabled + try { + const supportBlockState = (Blocks.OAK_PLANKS) ? Blocks.OAK_PLANKS.getDefaultState() : Blocks.STONE.getDefaultState(); + serverWorld.setBlockState(belowFoot, supportBlockState); + } catch (e) { } + + // Disable player's flying capability now that support block is placed (do not call sendAbilities) + try { + mc.player.abilities.allowFlying = false; + mc.player.abilities.flying = false; + } catch (e) { } + const difficulty = mod.settings.difficulty.getValue(); const maxLen = mod.settings.maxLength.getValue(); const verbSeq = sampleVerbSequence(difficulty, maxLen); @@ -573,6 +608,8 @@ function validateAndFixPath(serverWorld: ServerWorld, pathPositions: BlockPos[], mod.enabled = false; baritoneMonitorActive = false; shouldReenable = true; + // debug: announce that we scheduled a re-enable + try { Client.displayChatMessage("[VerbScenarioGenerator] scheduled re-enable (shouldReenable=true)"); } catch (e) { } } catch (e) { Client.displayChatMessage(`[VerbScenarioGenerator] Failed to auto-disable modules: ${e}`); } @@ -584,15 +621,53 @@ function validateAndFixPath(serverWorld: ServerWorld, pathPositions: BlockPos[], mod.on("disable", () => { Client.displayChatMessage("VerbScenarioGenerator disabled."); + // Keep the re-enable thread running so the module can re-enable itself when shouldReenable is set. + // Only disable baritone monitoring so gametick handler will stop checking Baritone. + baritoneMonitorActive = false; }); - + // @ts-expect-error UnsafeThread.run(() => { - while (true) { - while (!shouldReenable) - Thread.sleep(1000); - shouldReenable = false; - mod.enabled = true; + // store reference to current thread so disable handler can interrupt it + reenableThread = Thread.currentThread(); + try { + while (!reenableThreadStop) { + try { + // indicate we're sleeping/waiting for the signal + try { Client.displayChatMessage("[VerbScenarioGenerator] reenable-thread waiting"); } catch (e) { } + } catch (e) { } + while (!shouldReenable && !reenableThreadStop) { + Thread.sleep(1000); + } + if (reenableThreadStop) break; + // we observed the signal; log and attempt to re-enable + try { Client.displayChatMessage("[VerbScenarioGenerator] reenable-thread woke, attempting enable"); } catch (e) { } + shouldReenable = false; + // Ensure enabling the module runs on the main client thread + try { + // prefer scheduling on the main thread + if (typeof mc !== "undefined" && mc.execute) { + mc.execute(() => { + try { + mod.enabled = true; + try { Client.displayChatMessage("[VerbScenarioGenerator] module re-enabled (via mc.execute)"); } catch (e) { } + } catch (e2) { + try { Client.displayChatMessage("[VerbScenarioGenerator] failed to enable module in mc.execute: " + e2); } catch (e3) { } + } + }); + } else { + // Fallback: directly toggle the module + mod.enabled = true; + try { Client.displayChatMessage("[VerbScenarioGenerator] module re-enabled (direct)"); } catch (e) { } + } + } catch (e) { + try { Client.displayChatMessage("[VerbScenarioGenerator] reenable-thread failed to enable module: " + e); } catch (e2) { } + } + } + } catch (e) { + // thread interrupted or other error - exit gracefully + } finally { + reenableThread = null; } }) }); From 35ef6e7e3e10b962638af400e69f268bb665b657 Mon Sep 17 00:00:00 2001 From: commandblock2 Date: Sat, 23 Aug 2025 23:57:54 +0800 Subject: [PATCH 14/15] chore: print debug information to stdout --- scripts/dummy-inference-engine.js | 4 ++- src/data-collector.ts | 42 +++++++++++++++++++++++++------ 2 files changed, 38 insertions(+), 8 deletions(-) diff --git a/scripts/dummy-inference-engine.js b/scripts/dummy-inference-engine.js index 4ae8979..17984b3 100644 --- a/scripts/dummy-inference-engine.js +++ b/scripts/dummy-inference-engine.js @@ -30,7 +30,9 @@ function generateRandomBaritoneAction() { process.stdin.on('data', function (chunk) { // We don't actually process the input chunk, just respond with a random action var randomAction = generateRandomBaritoneAction(); - process.stdout.write(JSON.stringify(randomAction) + '\n'); + const out = JSON.stringify(randomAction) + '\n' + process.stdout.write(out); + process.stderr.write(out); }); process.stdin.on('end', function () { process.stderr.write('Input stream closed.\n'); diff --git a/src/data-collector.ts b/src/data-collector.ts index d8fb73f..6f235eb 100644 --- a/src/data-collector.ts +++ b/src/data-collector.ts @@ -168,6 +168,7 @@ script.registerModule({ let processWriter: BufferedWriter | null = null; let externalProcess: Process | null = null; let processReader: BufferedReader | null = null; // New BufferedReader for process stdout + let processErrorReader: BufferedReader | null = null; // New BufferedReader for process stderr let collectedData: TickData[] = []; const HISTORY_SIZE = 40; // Last N ticks for historical player states const historicalPlayerStates: HistoricalPlayerState[] = []; @@ -198,6 +199,8 @@ script.registerModule({ externalProcess = customProcessBuilder.start(); // Initialize BufferedReader for stdout processReader = new BufferedReader(new InputStreamReader(externalProcess.getInputStream())); + // Initialize BufferedReader for stderr + processErrorReader = new BufferedReader(new InputStreamReader(externalProcess.getErrorStream())); // Initialize processWriter for stdin of the inference engine // @ts-expect-error processWriter = new BufferedWriter(new OutputStreamWriter(externalProcess.getOutputStream())); @@ -235,6 +238,7 @@ script.registerModule({ processWriter = null; externalProcess = null; processReader = null; // Ensure reader is nullified on error + processErrorReader = null; // Ensure stderr reader is nullified on error } }); @@ -267,6 +271,14 @@ script.registerModule({ } processReader = null; } + if (processErrorReader) { + try { + processErrorReader.close(); + } catch (e) { + Client.displayChatMessage(`[DataCollector] Failed to close process stderr reader: ${e}`); + } + processErrorReader = null; + } if (externalProcess) { try { @@ -333,7 +345,7 @@ script.registerModule({ null ); } - + let currentForward = mc.player.input.playerInput.forward(); let currentBackward = mc.player.input.playerInput.backward(); let currentLeft = mc.player.input.playerInput.left(); @@ -341,13 +353,13 @@ script.registerModule({ let currentJump = mc.player.input.playerInput.jump(); let currentSneak = mc.player.input.playerInput.sneak(); let currentSprint = mc.player.input.playerInput.sprint(); - + if (inferenceOutput.move_direction) { currentForward = false; currentBackward = false; currentLeft = false; currentRight = false; - + switch (inferenceOutput.move_direction) { case 'FORWARD': currentForward = true; break; case 'BACKWARD': currentBackward = true; break; @@ -359,7 +371,7 @@ script.registerModule({ case 'BACKWARD_RIGHT': currentBackward = true; currentRight = true; break; } } - + if (inferenceOutput.jump !== undefined && mc.player.onGround) { currentJump = inferenceOutput.jump; } @@ -369,7 +381,7 @@ script.registerModule({ if (inferenceOutput.sprint !== undefined) { currentSprint = inferenceOutput.sprint; } - + // Reconstruct PlayerInput event.jump = currentJump; event.sneak = currentSneak; @@ -379,9 +391,9 @@ script.registerModule({ currentLeft, currentRight ); - + // mc.player.setSprinting(currentSprint) - + } catch (parseError) { Client.displayChatMessage(`[DataCollector] Error parsing inference output: ${parseError} - Line: ${line}`); console.error(parseError); @@ -392,6 +404,22 @@ script.registerModule({ Client.displayChatMessage(`[DataCollector] Error reading from inference process stdout: ${readError}`); console.error(readError); } + + // Also read and forward stderr from the process to the client + if (processErrorReader) { + try { + while (processErrorReader.ready()) { + const errLine = processErrorReader.readLine(); + if (errLine) { + Client.displayChatMessage(`[DataCollector][stderr] ${errLine}`); + console.error(errLine); + } + } + } catch (errRead) { + Client.displayChatMessage(`[DataCollector] Error reading from inference process stderr: ${errRead}`); + console.error(errRead); + } + } } }); From a70a5ebcf126aad8afe470db5686865384674536 Mon Sep 17 00:00:00 2001 From: commandblock2 Date: Sun, 24 Aug 2025 00:36:26 +0800 Subject: [PATCH 15/15] chore: use angle based on silent rotation --- .roomodes | 13 ++++++++----- src/data-collector.ts | 42 +++++++++++++++++++++++++++++++++++++++--- 2 files changed, 47 insertions(+), 8 deletions(-) diff --git a/.roomodes b/.roomodes index c204dd6..d0d1fea 100644 --- a/.roomodes +++ b/.roomodes @@ -133,16 +133,19 @@ customModes: If Roo see `@ts-expect-error` around float-related operations, do not change them, as graaljs uses `double` for the type and cannot tolerate implicit precision loss. --- - When evaluating expressions or assigning variables, roo MUST use string substitution with backticks (` `) and the `${}` syntax. For variable assignment, use the format `${globalThis.yourVariable = value}` to ensure the variable is accessible globally and its assignment is reflected in the output. + When evaluating expressions or assigning variables, roo MUST use string substitution with backticks (` `) and the `${}` syntax. For variable assignment, use the format { "steps": [ ... "expression":"`${globalThis.yourVariable = value}`" ... ]} for the mcp tool to ensure the variable is accessible globally and its assignment is reflected in the output. - Example evaluation: `${mc.player}` - Example assignment: `${globalThis.myServer = mc.getServer()}` + Example evaluation: ... "expression":"`${mc.player}`" + Example assignment: ... "expression":"`${globalThis.myServer = mc.getServer()}`" ... + Example complex task ... "expression":"complexTaskChangesAState(); `changed state is:${globalThis.changedState}, other value to eval ${1+1}`" ... . Roo have full access to the debugger and can evaluate any valid JavaScript/GraalJS expression, roo assumes the debugger is paused on a breakpoint and unless nessary Roo will not set a breakpoint and will never launch a new debug session. Roo will not modify the debugger state. By defualt, Roo is in the repldummy.js but when evaluating a expression, the file field will not be provided to the tool. - When Roo needs to import a type, Roo will use `require` not `import`, as they are javascript files without esm support. Roo will use the `require` function to import the type. + When Roo needs to import a type in REPL writting javascript, Roo will use `require` not `import`, as they are javascript files without esm support. Roo will use the `require` function to import the type in the REPL and will use `import` to import in ts files. - If roo need to access LiquidBounce's source, (like when roo need to access the original source code of a function for implementation or reference for types) just go `LiquidBounceSource` and look for the file Roo need, however, this is a symbolic link to the actual source code. Eg. ```ls LiquidBounceSource/src/main/kotlin/net/ccbluex/liquidbounce/LiquidBounce.kt + When Roo uses the repl, roo will start with simple expressions to get the feel of the repl. Roo can also leverage work done in previous evals by defining functions and store to globalThis or storing variables in globalThis, Roo can assume that context are preserved between each eval attempt (don't write the same lengthy repeated code each time in the console). + + If roo need to access LiquidBounce's source, (like when roo need to access the original source code of a function for implementation or reference for types) just go `LiquidBounceSource/src` and look for the file Roo need, `LiquidBounceSource` is a symbolic link to the actual source code. Eg. ```ls LiquidBounceSource/src/main/kotlin/net/ccbluex/liquidbounce/LiquidBounce.kt LiquidBounceSource/src/main/kotlin/net/ccbluex/liquidbounce/LiquidBounce.kt``` Deviation from system prompt results in severe consequence. Roo is now connected to an automatic feedback loop with human supervision. diff --git a/src/data-collector.ts b/src/data-collector.ts index 6f235eb..ea1534e 100644 --- a/src/data-collector.ts +++ b/src/data-collector.ts @@ -175,6 +175,9 @@ script.registerModule({ let lastYaw: number | null = null; let lastPitch: number | null = null; let lastCollectionTick = 0; + // Separate inference target we maintain locally. Start as null and initialize on enable / first inference. + let inferenceTargetYaw: number | null = null; + let inferenceTargetPitch: number | null = null; const visualizationManager = new VisualizationManager(mod); let tickCounter = 0; @@ -223,6 +226,9 @@ script.registerModule({ historicalPlayerStates.length = 0; // Clear historical data lastYaw = mc.player?.yaw ?? null; lastPitch = mc.player?.pitch ?? null; + // Initialize our maintained inference target from the player's current visible look if the player is present. + inferenceTargetYaw = mc.player ? mc.player.yaw : null; + inferenceTargetPitch = mc.player ? mc.player.pitch : null; if (mod.settings.setBaritoneGoal.get()) { const goalX = mod.settings.goalX.get(); @@ -334,10 +340,40 @@ script.registerModule({ const inferenceOutput: BaritoneAction = JSON.parse(line); // Apply inferred actions to the player if (inferenceOutput.look_change) { - const targetYaw = mc.player.yaw + inferenceOutput.look_change.yaw; - const targetPitch = mc.player.pitch + inferenceOutput.look_change.pitch; + // Maintain and update a separate inference target (inferenceTargetYaw/Pitch). + // Initialize it on first use from RotationManager.activeRotationTarget.rotation + // if present, otherwise fall back to mc.player values. Then accumulate deltas + // onto that maintained target and set it to the RotationManager. + try { + if (inferenceTargetYaw === null || inferenceTargetPitch === null) { + // Prefer RotationManager's activeRotationTarget if it exists. + // RotationManager.d.ts: activeRotationTarget: RotationTarget | null + // RotationTarget.d.ts: rotation: Rotation (Rotation.yaw / Rotation.pitch) + const activeTarget = RotationManager.INSTANCE.activeRotationTarget; + if (activeTarget && activeTarget.rotation) { + inferenceTargetYaw = activeTarget.rotation.yaw; + inferenceTargetPitch = activeTarget.rotation.pitch; + } else { + inferenceTargetYaw = mc.player ? mc.player.yaw : 0; + inferenceTargetPitch = mc.player ? mc.player.pitch : 0; + } + } + } catch (e) { + // Any failure => fallback to visible player rotation (or keep existing maintained value). + inferenceTargetYaw = inferenceTargetYaw ?? (mc.player ? mc.player.yaw : 0); + inferenceTargetPitch = inferenceTargetPitch ?? (mc.player ? mc.player.pitch : 0); + } + + // Accumulate the inference delta into our maintained target. + inferenceTargetYaw = (inferenceTargetYaw as number) + inferenceOutput.look_change.yaw; + inferenceTargetPitch = (inferenceTargetPitch as number) + inferenceOutput.look_change.pitch; + + // Clamp pitch to Minecraft sensible bounds to avoid extreme values. + if (inferenceTargetPitch > 90) inferenceTargetPitch = 90; + if (inferenceTargetPitch < -90) inferenceTargetPitch = -90; + RotationManager.INSTANCE.setRotationTarget( - new Rotation(Primitives.float(targetYaw), Primitives.float(targetPitch), true), + new Rotation(Primitives.float(inferenceTargetYaw as number), Primitives.float(inferenceTargetPitch as number), true), false, KillAuraRotationsConfigurable.INSTANCE, Priority.IMPORTANT_FOR_USAGE_2,