From 4c4fe1175d6a9ab1d194dd2019133cf4d290fa58 Mon Sep 17 00:00:00 2001 From: Gyuheon Oh Date: Fri, 5 Dec 2025 21:05:31 +0000 Subject: [PATCH] First pass --- Cargo.lock | 2 + libdd-crashtracker/Cargo.toml | 2 + libdd-crashtracker/src/crash_info/builder.rs | 42 +++++++++++++------ libdd-crashtracker/src/crash_info/mod.rs | 34 ++++++++++++++- .../src/receiver/receive_report.rs | 27 +++++++----- 5 files changed, 84 insertions(+), 23 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e1216c6d9a..30da990663 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2774,6 +2774,7 @@ dependencies = [ "rand 0.8.5", "schemars 0.8.22", "serde", + "serde_bytes", "serde_json", "symbolic-common", "symbolic-demangle", @@ -2782,6 +2783,7 @@ dependencies = [ "tokio", "uuid", "windows 0.59.0", + "zstd", ] [[package]] diff --git a/libdd-crashtracker/Cargo.toml b/libdd-crashtracker/Cargo.toml index 9e7942c3e8..080f292779 100644 --- a/libdd-crashtracker/Cargo.toml +++ b/libdd-crashtracker/Cargo.toml @@ -60,11 +60,13 @@ rand = "0.8.5" schemars = "0.8.21" serde = {version = "1.0", features = ["derive"]} serde_json = {version = "1.0"} +serde_bytes = "0.11" symbolic-demangle = { version = "12.8.0", default-features = false, features = ["rust", "cpp", "msvc"] } symbolic-common = { version = "12.8.0", default-features = false } tokio = { version = "1.23", features = ["rt", "macros", "io-std", "io-util"] } uuid = { version = "1.4.1", features = ["v4", "serde"] } thiserror = "1.0" +zstd = { version = "0.13", default-features = false } [target.'cfg(windows)'.dependencies] windows = { version = "0.59.0", features = ["Win32_System_Diagnostics_Debug", "Win32_System_Diagnostics_ToolHelp", "Win32_System_ErrorReporting", "Win32_System_Kernel", "Win32_System_ProcessStatus", "Win32_System_Registry", "Win32_System_SystemInformation", "Win32_System_SystemServices", "Win32_System_Threading", "Win32_Security"] } diff --git a/libdd-crashtracker/src/crash_info/builder.rs b/libdd-crashtracker/src/crash_info/builder.rs index fdff37f213..e1a74d588f 100644 --- a/libdd-crashtracker/src/crash_info/builder.rs +++ b/libdd-crashtracker/src/crash_info/builder.rs @@ -6,12 +6,16 @@ use crate::runtime_callback::RuntimeStack; use chrono::{DateTime, Utc}; use error_data::ThreadData; use stacktrace::StackTrace; -use std::io::{BufRead, BufReader}; +use std::fs; use unknown_value::UnknownValue; use uuid::Uuid; +use zstd::bulk; use super::*; +const PROC_SELF_MAPS_PATH: &str = "/proc/self/maps"; +const PROC_SELF_MAPS_ZSTD_LEVEL: i32 = 3; + #[derive(Debug, Default, PartialEq)] pub struct ErrorDataBuilder { pub kind: Option, @@ -96,7 +100,7 @@ pub struct CrashInfoBuilder { pub counters: Option>, pub error: ErrorDataBuilder, pub experimental: Option, - pub files: Option>>, + pub files: Option>, pub fingerprint: Option, pub incomplete: Option, pub log_messages: Option>, @@ -232,10 +236,8 @@ impl CrashInfoBuilder { } pub fn with_file(&mut self, filename: String) -> anyhow::Result<()> { - let file = File::open(&filename).with_context(|| format!("filename: {filename}"))?; - let lines: std::io::Result> = BufReader::new(file).lines().collect(); - self.with_file_and_contents(filename, lines?)?; - Ok(()) + let data = fs::read(&filename).with_context(|| format!("filename: {filename}"))?; + self.with_file_bytes(filename, data) } /// Appends the given file to the current set of files in the builder. @@ -244,20 +246,36 @@ impl CrashInfoBuilder { filename: String, contents: Vec, ) -> anyhow::Result<()> { - if let Some(ref mut files) = &mut self.files { - files.insert(filename, contents); - } else { - self.files = Some(HashMap::from([(filename, contents)])); + let mut data = Vec::new(); + for line in contents { + data.extend_from_slice(line.as_bytes()); + data.push(b'\n'); } - Ok(()) + self.with_file_bytes(filename, data) } /// Sets the current set of files in the builder. - pub fn with_files(&mut self, files: HashMap>) -> anyhow::Result<()> { + pub fn with_files(&mut self, files: HashMap) -> anyhow::Result<()> { self.files = Some(files); Ok(()) } + pub fn with_file_bytes(&mut self, filename: String, contents: Vec) -> anyhow::Result<()> { + let file = if filename == PROC_SELF_MAPS_PATH { + let compressed = bulk::compress(&contents, PROC_SELF_MAPS_ZSTD_LEVEL) + .context("unable to zstd encode /proc/self/maps")?; + CrashFile::with_encoding(compressed, FileEncoding::Zstd) + } else { + CrashFile::new(contents) + }; + if let Some(ref mut files) = &mut self.files { + files.insert(filename, file); + } else { + self.files = Some(HashMap::from([(filename, file)])); + } + Ok(()) + } + pub fn with_fingerprint(&mut self, fingerprint: String) -> anyhow::Result<()> { anyhow::ensure!(!fingerprint.is_empty(), "Expect non-empty fingerprint"); self.fingerprint = Some(fingerprint); diff --git a/libdd-crashtracker/src/crash_info/mod.rs b/libdd-crashtracker/src/crash_info/mod.rs index ffe3cf4b1d..36cac2d234 100644 --- a/libdd-crashtracker/src/crash_info/mod.rs +++ b/libdd-crashtracker/src/crash_info/mod.rs @@ -42,6 +42,36 @@ pub fn build_crash_ping_message(sig_info: &SigInfo) -> String { ) } +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "lowercase")] +pub enum FileEncoding { + Zstd, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +pub struct CrashFile { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub encoding: Option, + #[serde(default)] + pub data: Vec, +} + +impl CrashFile { + pub fn new(data: Vec) -> Self { + Self { + encoding: None, + data, + } + } + + pub fn with_encoding(data: Vec, encoding: FileEncoding) -> Self { + Self { + encoding: Some(encoding), + data, + } + } +} + #[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] pub struct CrashInfo { #[serde(default, skip_serializing_if = "HashMap::is_empty")] @@ -51,7 +81,9 @@ pub struct CrashInfo { #[serde(default, skip_serializing_if = "Option::is_none")] pub experimental: Option, #[serde(default, skip_serializing_if = "HashMap::is_empty")] - pub files: HashMap>, + /// Supplemental files attached to the crash report. Each entry stores the raw + /// bytes plus optional encoding metadata (e.g. `/proc/self/maps` is zstd-compressed). + pub files: HashMap, #[serde(default, skip_serializing_if = "Option::is_none")] pub fingerprint: Option, pub incomplete: bool, diff --git a/libdd-crashtracker/src/receiver/receive_report.rs b/libdd-crashtracker/src/receiver/receive_report.rs index 5f44cb0e59..3b2f4d1c45 100644 --- a/libdd-crashtracker/src/receiver/receive_report.rs +++ b/libdd-crashtracker/src/receiver/receive_report.rs @@ -130,6 +130,20 @@ fn process_line( telemetry_logger: &Option>, ) -> anyhow::Result { let next = match state { + StdinState::File(filename, contents) if line.starts_with(DD_CRASHTRACK_END_FILE) => { + let mut data = Vec::new(); + for entry in contents { + data.extend_from_slice(entry.as_bytes()); + data.push(b'\n'); + } + builder.with_file_bytes(filename, data)?; + StdinState::Waiting + } + StdinState::File(name, mut contents) => { + contents.push(line.to_string()); + StdinState::File(name, contents) + } + StdinState::AdditionalTags if line.starts_with(DD_CRASHTRACK_END_ADDITIONAL_TAGS) => { StdinState::Waiting } @@ -171,15 +185,6 @@ fn process_line( StdinState::Done } - StdinState::File(filename, lines) if line.starts_with(DD_CRASHTRACK_END_FILE) => { - builder.with_file_and_contents(filename, lines)?; - StdinState::Waiting - } - StdinState::File(name, mut contents) => { - contents.push(line.to_string()); - StdinState::File(name, contents) - } - StdinState::Metadata if line.starts_with(DD_CRASHTRACK_END_METADATA) => StdinState::Waiting, StdinState::Metadata => { let metadata = serde_json::from_str(line)?; @@ -406,7 +411,9 @@ pub(crate) async fn receive_report_from_stream( ); break; }; - let Some(next_line) = next_line else { break }; + let Some(next_line) = next_line else { + break; + }; match process_line( &mut builder,