From a6db0e55b504882c59472e6cd5eb5a6d9a6403a0 Mon Sep 17 00:00:00 2001 From: Daniel Lamberger Date: Thu, 13 Jun 2019 13:43:29 +0300 Subject: [PATCH] #95891 Microdot: grainReq: optionally log Orleans system grains --- .../GigyaSiloHost.cs | 117 +++++++++++------- .../Configurations/LoadShedding.cs | 4 + .../WarmupTestService/GigyaSiloHostFake.cs | 6 +- 3 files changed, 80 insertions(+), 47 deletions(-) diff --git a/Gigya.Microdot.Orleans.Hosting/GigyaSiloHost.cs b/Gigya.Microdot.Orleans.Hosting/GigyaSiloHost.cs index c91e004e..8779d5ae 100644 --- a/Gigya.Microdot.Orleans.Hosting/GigyaSiloHost.cs +++ b/Gigya.Microdot.Orleans.Hosting/GigyaSiloHost.cs @@ -21,6 +21,7 @@ #endregion using System; +using System.Collections.Concurrent; using System.Collections.Generic; using System.Linq; using System.Reflection; @@ -28,6 +29,7 @@ using System.Threading.Tasks.Dataflow; using Gigya.Common.Contracts.Exceptions; using Gigya.Microdot.Hosting.HttpService; +using Gigya.Microdot.Interfaces.Configuration; using Gigya.Microdot.Interfaces.Events; using Gigya.Microdot.Interfaces.Logging; using Gigya.Microdot.Orleans.Hosting.Events; @@ -58,18 +60,29 @@ public class GigyaSiloHost private ISourceBlock OrleansConfigSourceBlock { get; } public static OrleansConfig PreviousOrleansConfig { get; private set; } private Func LoadSheddingConfig { get; } + private Func GrainLoggingConfig { get; } + + [ConfigurationRoot("Microdot.GrainLogging", RootStrategy.ReplaceClassNameWithPath)] + public class GrainLogging : IConfigObject + { + public bool LogServiceGrains; + public bool LogMicrodotGrains; + public bool LogOrleansGrains; + } + public GigyaSiloHost(ILog log, OrleansConfigurationBuilder configBuilder, HttpServiceListener httpServiceListener, IEventPublisher eventPublisher, Func loadSheddingConfig, - ISourceBlock orleansConfigSourceBlock, OrleansConfig orleansConfig) + ISourceBlock orleansConfigSourceBlock, OrleansConfig orleansConfig, Func grainLoggingConfig) { Log = log; ConfigBuilder = configBuilder; HttpServiceListener = httpServiceListener; EventPublisher = eventPublisher; LoadSheddingConfig = loadSheddingConfig; + GrainLoggingConfig = grainLoggingConfig; OrleansConfigSourceBlock = orleansConfigSourceBlock; @@ -178,47 +191,6 @@ private async Task BootstrapInit(IProviderRuntime providerRuntime) public TaskScheduler GrainTaskScheduler { get; set; } - private void OutgoingCallInterceptor(InvokeMethodRequest request, IGrain target) - { - TracingContext.SetUpStorage(); - TracingContext.SpanStartTime = DateTimeOffset.UtcNow; - } - - - private async Task IncomingCallInterceptor(MethodInfo targetMethod, InvokeMethodRequest request, IGrain target, IGrainMethodInvoker invoker) - { - if (targetMethod == null) - throw new ArgumentNullException(nameof(targetMethod)); - - var declaringNameSpace = targetMethod.DeclaringType?.Namespace; - - // Do not intercept Orleans grains or other grains which should not be included in statistics. - if (targetMethod.DeclaringType.GetCustomAttribute() != null || - declaringNameSpace?.StartsWith("Orleans") == true) - return await invoker.Invoke(target, request); - - RequestTimings.GetOrCreate(); // Ensure request timings is created here and not in the grain call. - - RequestTimings.Current.Request.Start(); - Exception ex = null; - - try - { - RejectRequestIfLateOrOverloaded(); - return await invoker.Invoke(target, request); - } - catch (Exception e) - { - ex = e; - throw; - } - finally - { - RequestTimings.Current.Request.Stop(); - PublishEvent(targetMethod, target, ex); - } - } - public void UpdateOrleansAgeLimitChange(OrleansConfig orleanConfig) { try @@ -256,13 +228,68 @@ public void UpdateOrleansAgeLimitChange(OrleansConfig orleanConfig) } } - private void RejectRequestIfLateOrOverloaded() + + private void OutgoingCallInterceptor(InvokeMethodRequest request, IGrain target) + { + TracingContext.SetUpStorage(); + TracingContext.SpanStartTime = DateTimeOffset.UtcNow; + } + + + static ConcurrentDictionary MICRODOT_GRAINS = new ConcurrentDictionary(); + + private async Task IncomingCallInterceptor(MethodInfo targetMethod, InvokeMethodRequest request, IGrain target, IGrainMethodInvoker invoker) + { + if (targetMethod == null) + throw new ArgumentNullException(nameof(targetMethod)); + + // Identify the grain type + bool isOrleansGrain = targetMethod.DeclaringType?.Namespace?.StartsWith("Orleans") == true; + bool isMicrodotGrain = MICRODOT_GRAINS.GetOrAdd(targetMethod.DeclaringType, type => type.GetCustomAttribute() != null); + bool isServiceGrain = !isOrleansGrain && !isMicrodotGrain; + + // Drop the request if we're overloaded + var loadSheddingConfig = LoadSheddingConfig(); + if ( (loadSheddingConfig.ApplyToOrleansGrains && isOrleansGrain) + || (loadSheddingConfig.ApplyToMicrodotGrains && isMicrodotGrain) + || (loadSheddingConfig.ApplyToServiceGrains && isServiceGrain)) + { + RejectRequestIfLateOrOverloaded(loadSheddingConfig); + } + + RequestTimings.GetOrCreate(); // Ensure request timings is created here and not in the grain call. + RequestTimings.Current.Request.Start(); + Exception ex = null; + + try + { + return await invoker.Invoke(target, request); + } + catch (Exception e) + { + ex = e; + throw; + } + finally + { + RequestTimings.Current.Request.Stop(); + var loggingConfig = GrainLoggingConfig(); + if ( (loggingConfig.LogOrleansGrains && isOrleansGrain) + || (loggingConfig.LogMicrodotGrains && isMicrodotGrain) + || (loggingConfig.LogServiceGrains && isServiceGrain)) + { + PublishEvent(targetMethod, target, ex); + } + } + } + + + private void RejectRequestIfLateOrOverloaded(LoadShedding config) { - var config = LoadSheddingConfig(); var now = DateTimeOffset.UtcNow; // Too much time passed since our direct caller made the request to us; something's causing a delay. Log or reject the request, if needed. - if (config.DropOrleansRequestsBySpanTime != LoadShedding.Toggle.Disabled + if ( config.DropOrleansRequestsBySpanTime != LoadShedding.Toggle.Disabled && TracingContext.SpanStartTime != null && TracingContext.SpanStartTime.Value + config.DropOrleansRequestsOlderThanSpanTimeBy < now) { diff --git a/Gigya.Microdot.SharedLogic/Configurations/LoadShedding.cs b/Gigya.Microdot.SharedLogic/Configurations/LoadShedding.cs index 983f8f7f..ad7c0492 100644 --- a/Gigya.Microdot.SharedLogic/Configurations/LoadShedding.cs +++ b/Gigya.Microdot.SharedLogic/Configurations/LoadShedding.cs @@ -24,6 +24,10 @@ public enum Toggle public Toggle DropOrleansRequestsBySpanTime { get; set; } = Toggle.Disabled; public TimeSpan DropOrleansRequestsOlderThanSpanTimeBy { get; set; } = TimeSpan.FromSeconds(5); + + public bool ApplyToServiceGrains { get; set; } = true; + public bool ApplyToMicrodotGrains { get; set; } = false; + public bool ApplyToOrleansGrains { get; set; } = false; } } diff --git a/tests/Gigya.Microdot.Orleans.Hosting.UnitTests/Microservice/WarmupTestService/GigyaSiloHostFake.cs b/tests/Gigya.Microdot.Orleans.Hosting.UnitTests/Microservice/WarmupTestService/GigyaSiloHostFake.cs index bd853828..a411dffa 100644 --- a/tests/Gigya.Microdot.Orleans.Hosting.UnitTests/Microservice/WarmupTestService/GigyaSiloHostFake.cs +++ b/tests/Gigya.Microdot.Orleans.Hosting.UnitTests/Microservice/WarmupTestService/GigyaSiloHostFake.cs @@ -12,8 +12,10 @@ namespace Gigya.Microdot.Orleans.Hosting.UnitTests.Microservice.WarmupTestServic { public class GigyaSiloHostFake : GigyaSiloHost { - public GigyaSiloHostFake(WarmupTestServiceHostWithSiloHostFake host, ILog log, OrleansConfigurationBuilder configBuilder, HttpServiceListener httpServiceListener, IEventPublisher eventPublisher, Func loadSheddingConfig, ISourceBlock orleansConfigSourceBlock, OrleansConfig orleansConfig) : - base(log, configBuilder, httpServiceListener, eventPublisher, loadSheddingConfig, orleansConfigSourceBlock, orleansConfig) + public GigyaSiloHostFake(WarmupTestServiceHostWithSiloHostFake host, ILog log, OrleansConfigurationBuilder configBuilder, + HttpServiceListener httpServiceListener, IEventPublisher eventPublisher, Func loadSheddingConfig, + ISourceBlock orleansConfigSourceBlock, OrleansConfig orleansConfig, Func grainLoggingConfig) : + base(log, configBuilder, httpServiceListener, eventPublisher, loadSheddingConfig, orleansConfigSourceBlock, orleansConfig, grainLoggingConfig) { try {