From ec71d5ac7dd241154f2123beb90b617ada64bf3c Mon Sep 17 00:00:00 2001 From: Philipp Gortan Date: Thu, 16 Feb 2017 11:18:12 +0100 Subject: [PATCH 1/4] Silence "invalid dirhandle" warning if process has died in the meantime --- lib/InfluxDB/Writer/FileTailer.pm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/InfluxDB/Writer/FileTailer.pm b/lib/InfluxDB/Writer/FileTailer.pm index 50b0881..9f453a1 100644 --- a/lib/InfluxDB/Writer/FileTailer.pm +++ b/lib/InfluxDB/Writer/FileTailer.pm @@ -111,7 +111,7 @@ sub is_running { my $abs_file = File::Spec->rel2abs( $file ); my $found = 0; - opendir(my $dh, $fd_dir); + opendir(my $dh, $fd_dir) or return; # process died in the meantime while ( my $f = readdir($dh) ) { $f = catfile($fd_dir, $f); From 2ab1c8afb3a4f699aa7e7ef413750b6856f1c0f7 Mon Sep 17 00:00:00 2001 From: Stefan Hrdlicka Date: Thu, 25 Jan 2018 15:24:00 +0100 Subject: [PATCH 2/4] removed on_inital because it read over content in files --- lib/InfluxDB/Writer/FileTailer.pm | 5 --- t/filetailer.t | 70 +++++++++++++++++++++++++++++++ 2 files changed, 70 insertions(+), 5 deletions(-) create mode 100644 t/filetailer.t diff --git a/lib/InfluxDB/Writer/FileTailer.pm b/lib/InfluxDB/Writer/FileTailer.pm index 9f453a1..8c36927 100644 --- a/lib/InfluxDB/Writer/FileTailer.pm +++ b/lib/InfluxDB/Writer/FileTailer.pm @@ -171,11 +171,6 @@ sub setup_file_watcher { if ( open( my $fh, "<", $file ) ) { my $filestream = IO::Async::FileStream->new( read_handle => $fh, - on_initial => sub { - my ($stream) = @_; - $stream->seek_to_last("\n"); # TODO remember last position? - }, - on_read => sub { my ( $stream, $buffref ) = @_; diff --git a/t/filetailer.t b/t/filetailer.t new file mode 100644 index 0000000..c0fd2ea --- /dev/null +++ b/t/filetailer.t @@ -0,0 +1,70 @@ +#! /usr/bin/env perl + +use Test::More; + +use FindBin; +use File::Spec::Functions qw(catfile catdir); +use File::Slurper qw/write_text/; +use IO::Async::Loop; + +use InfluxDB::Writer::FileTailer; + +use Test::MockModule; +use Test::TempDir::Tiny; + +my $datadir = catdir( $FindBin::Bin, 'testdata', 'something' ); + +my %basic_args = ( + influx_host => 'computer1', + influx_port => '27015', + influx_db => 'test' +); + +subtest 'pid_dead' => sub { + my $tmpdir = tempdir('pid_dead'); + + my $stats_file = catfile( $tmpdir, 'pid_dead_123.stats' ); + write_text( $stats_file, "test pid dead" ); + + my $mock = Test::MockModule->new('InfluxDB::Writer::FileTailer'); + $mock->mock( 'is_running', sub {0} ); + + my $ft = InfluxDB::Writer::FileTailer->new( dir => $tmpdir, %basic_args ); + my $filestream = $ft->setup_file_watcher($stats_file); + ok( !$filestream, 'no pid should not return a filestream' ); +}; + +subtest 'pid_alive_already_content_in_file' => sub { + my $tmpdir = tempdir('pid_dead'); + + my $text = "test pid dead\n"; + my $stats_file = catfile( $tmpdir, 'pid_dead_123.stats' ); + my @expected_result = (); + open my $fd, ">", $stats_file + or die "Cannot open file $stats_file for writing"; + $fd->autoflush(1); + $fd->syswrite("1 $text"); + push @expected_result, "1 $text"; + + my $mock = Test::MockModule->new('InfluxDB::Writer::FileTailer'); + $mock->mock( 'is_running', sub {1} ); + $mock->mock( 'send', sub { } ); + + my $ft = InfluxDB::Writer::FileTailer->new( dir => $tmpdir, %basic_args ); + my $filestream = $ft->setup_file_watcher($stats_file); + ok( $filestream, 'got a filestream for stats file' ); + is( @{ $ft->buffer }, 0, 'no new lines, nothing should be read' ); + + my $loop = IO::Async::Loop->new_builtin(); + $loop->add($filestream); + + $fd->syswrite("2 $text"); + push @expected_result, "2 $text"; + + $loop->loop_once(); + + is( @{ $ft->buffer }, 2, 'buffer should contain two lines' ); + is_deeply( $ft->buffer, \@expected_result, 'compare buffer content' ); +}; + +done_testing(); From b40ee51d8fc1f1fa10a694ded63e180170a20cf8 Mon Sep 17 00:00:00 2001 From: Philipp Gortan Date: Wed, 7 Feb 2018 11:25:22 +0100 Subject: [PATCH 3/4] SecCRS: don't log each request in debug mode --- lib/InfluxDB/Writer/FileTailer.pm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/InfluxDB/Writer/FileTailer.pm b/lib/InfluxDB/Writer/FileTailer.pm index 8c36927..8de21aa 100644 --- a/lib/InfluxDB/Writer/FileTailer.pm +++ b/lib/InfluxDB/Writer/FileTailer.pm @@ -225,7 +225,7 @@ sub send { body => $body, %args, }; - $log->debugf("The Hijk::Request: %s", $request_data); + $log->tracef("The Hijk::Request: %s", $request_data); my $res = Hijk::request($request_data); if (my $current_error = $res->{error}) { From 99baf5612cb970bcae780a499c8a0016eba38ef8 Mon Sep 17 00:00:00 2001 From: Philipp Gortan Date: Wed, 7 Feb 2018 15:26:18 +0100 Subject: [PATCH 4/4] SecCRS: When the writing process died, read to the EOF before archiving --- lib/InfluxDB/Writer/FileTailer.pm | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lib/InfluxDB/Writer/FileTailer.pm b/lib/InfluxDB/Writer/FileTailer.pm index 8de21aa..9e28af4 100644 --- a/lib/InfluxDB/Writer/FileTailer.pm +++ b/lib/InfluxDB/Writer/FileTailer.pm @@ -144,6 +144,10 @@ sub setup_file_watcher { if (!$is_running) { if ( my $w = $self->_files->{$file} ) { + until ($w->is_read_eof()) { + $log->debugf("Reading to the end of %s", $file); + $w->read_more(); + } $self->_loop->remove($w); undef $w; delete $self->_files->{$file};