From 0ea6545caf2c233f0e22dd2e76cd1883e2067a03 Mon Sep 17 00:00:00 2001 From: Kaise Cheng Date: Tue, 18 Nov 2025 15:26:21 +0000 Subject: [PATCH 1/6] add trace log to track event size expansion --- CHANGELOG.md | 3 +++ lib/logstash/filters/split.rb | 12 ++++++++++++ logstash-filter-split.gemspec | 2 +- 3 files changed, 16 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bfdc4b1..533fe55 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,6 @@ +## 3.1.10 + - Added trace log to track event size expansion + ## 3.1.9 - [DOC] Added introductory statement to clarify purpose of the plugin [#43](https://github.com/logstash-plugins/logstash-filter-split/pull/43) diff --git a/lib/logstash/filters/split.rb b/lib/logstash/filters/split.rb index 07741da..10be767 100644 --- a/lib/logstash/filters/split.rb +++ b/lib/logstash/filters/split.rb @@ -81,6 +81,7 @@ def filter(event) # set event_target to @field if not configured event_target = @target.nil? ? @field : @target + split_bytes = 0 splits.each do |value| next if value.nil? || (value.is_a?(String) && value.empty?) @@ -90,10 +91,21 @@ def filter(event) event_split.set(event_target, value) filter_matched(event_split) + logger.trace? && split_bytes += event_split.to_json.size + # Push this new event onto the stack at the LogStash::FilterWorker yield event_split end + + if logger.trace? + original_bytes = event.to_json.size + logger.trace("Event is split into #{splits.size}", + :split_bytes => split_bytes, + :original_bytes => original_bytes, + :ratio => (split_bytes.to_f / original_bytes).round(2)) + end + # Cancel this event, we'll use the newly generated ones above. event.cancel end diff --git a/logstash-filter-split.gemspec b/logstash-filter-split.gemspec index 66997cf..85dbf5b 100644 --- a/logstash-filter-split.gemspec +++ b/logstash-filter-split.gemspec @@ -1,7 +1,7 @@ Gem::Specification.new do |s| s.name = 'logstash-filter-split' - s.version = '3.1.9' + s.version = '3.1.10' s.licenses = ['Apache License (2.0)'] s.summary = "Splits multi-line messages into distinct events" s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program" From 41849d162a7d371cbc8ecf31c733f40bac517df6 Mon Sep 17 00:00:00 2001 From: Kaise Cheng Date: Tue, 18 Nov 2025 15:31:32 +0000 Subject: [PATCH 2/6] changelog --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 533fe55..65656c8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,5 @@ ## 3.1.10 - - Added trace log to track event size expansion + - Added trace log to track event size expansion [#49](https://github.com/logstash-plugins/logstash-filter-split/pull/49) ## 3.1.9 - [DOC] Added introductory statement to clarify purpose of the plugin [#43](https://github.com/logstash-plugins/logstash-filter-split/pull/43) From 5aa2b961335ac447769a8842310be43eabd2a7bb Mon Sep 17 00:00:00 2001 From: kaisecheng <69120390+kaisecheng@users.noreply.github.com> Date: Tue, 18 Nov 2025 18:06:38 +0000 Subject: [PATCH 3/6] Update lib/logstash/filters/split.rb Co-authored-by: Cas Donoghue --- lib/logstash/filters/split.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/logstash/filters/split.rb b/lib/logstash/filters/split.rb index 10be767..d023c07 100644 --- a/lib/logstash/filters/split.rb +++ b/lib/logstash/filters/split.rb @@ -82,7 +82,7 @@ def filter(event) # set event_target to @field if not configured event_target = @target.nil? ? @field : @target split_bytes = 0 - + logger.trace("Event being split into #{splits.size} events") splits.each do |value| next if value.nil? || (value.is_a?(String) && value.empty?) @logger.debug? && @logger.debug("Split event", :value => value, :field => @field) From 7a645a64b2bb7017fda28f25cbff82d10f11f08a Mon Sep 17 00:00:00 2001 From: kaisecheng <69120390+kaisecheng@users.noreply.github.com> Date: Tue, 18 Nov 2025 18:07:06 +0000 Subject: [PATCH 4/6] Update lib/logstash/filters/split.rb --- lib/logstash/filters/split.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/logstash/filters/split.rb b/lib/logstash/filters/split.rb index d023c07..c1c9745 100644 --- a/lib/logstash/filters/split.rb +++ b/lib/logstash/filters/split.rb @@ -82,7 +82,8 @@ def filter(event) # set event_target to @field if not configured event_target = @target.nil? ? @field : @target split_bytes = 0 - logger.trace("Event being split into #{splits.size} events") + logger.trace? && logger.trace("Event being split into #{splits.size} events") + splits.each do |value| next if value.nil? || (value.is_a?(String) && value.empty?) @logger.debug? && @logger.debug("Split event", :value => value, :field => @field) From 41f12d8d1930f4acef0919c7442879b6dc2e7050 Mon Sep 17 00:00:00 2001 From: Kaise Cheng Date: Tue, 18 Nov 2025 18:13:15 +0000 Subject: [PATCH 5/6] remove verbose logging --- lib/logstash/filters/split.rb | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/logstash/filters/split.rb b/lib/logstash/filters/split.rb index c1c9745..9eaa456 100644 --- a/lib/logstash/filters/split.rb +++ b/lib/logstash/filters/split.rb @@ -86,7 +86,6 @@ def filter(event) splits.each do |value| next if value.nil? || (value.is_a?(String) && value.empty?) - @logger.debug? && @logger.debug("Split event", :value => value, :field => @field) event_split = event.clone event_split.set(event_target, value) From 0cae22c244d02bea403cfeda6698990d39804287 Mon Sep 17 00:00:00 2001 From: Kaise Cheng Date: Tue, 18 Nov 2025 18:17:02 +0000 Subject: [PATCH 6/6] improve trace log msg --- lib/logstash/filters/split.rb | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/logstash/filters/split.rb b/lib/logstash/filters/split.rb index 9eaa456..ce2e395 100644 --- a/lib/logstash/filters/split.rb +++ b/lib/logstash/filters/split.rb @@ -100,10 +100,10 @@ def filter(event) if logger.trace? original_bytes = event.to_json.size - logger.trace("Event is split into #{splits.size}", - :split_bytes => split_bytes, - :original_bytes => original_bytes, - :ratio => (split_bytes.to_f / original_bytes).round(2)) + logger.trace("Estimated event size growth after split", + :original_bytes => original_bytes, + :split_bytes => split_bytes, + :growth_ratio => (split_bytes.to_f / original_bytes).round(2)) end # Cancel this event, we'll use the newly generated ones above.