diff --git a/build.gradle b/build.gradle index 027c5a6bc..98c28e5eb 100644 --- a/build.gradle +++ b/build.gradle @@ -61,7 +61,7 @@ dependencies { implementation 'com.fasterxml.jackson.dataformat:jackson-dataformat-yaml' implementation 'com.squareup.moshi:moshi:1.15.1' implementation 'com.squareup.moshi:moshi-adapters:1.15.1' - implementation 'redis.clients:jedis:5.1.3' + implementation 'redis.clients:jedis:5.2.0' implementation 'io.github.resilience4j:resilience4j-ratelimiter:0.17.0' implementation 'io.micronaut:micronaut-retry' // caching deps diff --git a/src/main/groovy/io/seqera/wave/service/counter/AbstractCounterStore.groovy b/src/main/groovy/io/seqera/wave/service/counter/AbstractCounterStore.groovy index 3e51f771c..232fb5990 100644 --- a/src/main/groovy/io/seqera/wave/service/counter/AbstractCounterStore.groovy +++ b/src/main/groovy/io/seqera/wave/service/counter/AbstractCounterStore.groovy @@ -55,4 +55,9 @@ abstract class AbstractCounterStore implements CounterStore { Map getAllMatchingEntries(String pattern) { provider.getAllMatchingEntries(getPrefix(), pattern) } + + @Override + void deleteAllMatchingEntries(String pattern) { + provider.getAllMatchingEntries(getPrefix(), pattern) + } } diff --git a/src/main/groovy/io/seqera/wave/service/counter/CounterStore.groovy b/src/main/groovy/io/seqera/wave/service/counter/CounterStore.groovy index 41e64c4e0..c91b46067 100644 --- a/src/main/groovy/io/seqera/wave/service/counter/CounterStore.groovy +++ b/src/main/groovy/io/seqera/wave/service/counter/CounterStore.groovy @@ -34,4 +34,10 @@ interface CounterStore { * @return all the entries whose field matches 'pattern' */ Map getAllMatchingEntries(String pattern) + + /** + * @param pattern + * @return void + */ + void deleteAllMatchingEntries(String pattern) } diff --git a/src/main/groovy/io/seqera/wave/service/counter/impl/CounterProvider.groovy b/src/main/groovy/io/seqera/wave/service/counter/impl/CounterProvider.groovy index c13f8f4e2..9fc868ab4 100644 --- a/src/main/groovy/io/seqera/wave/service/counter/impl/CounterProvider.groovy +++ b/src/main/groovy/io/seqera/wave/service/counter/impl/CounterProvider.groovy @@ -35,4 +35,5 @@ interface CounterProvider { * @return all the entries whose field matches 'pattern' */ Map getAllMatchingEntries(String key, String pattern) + } diff --git a/src/main/groovy/io/seqera/wave/service/counter/impl/LocalCounterProvider.groovy b/src/main/groovy/io/seqera/wave/service/counter/impl/LocalCounterProvider.groovy index c7ea99ac6..8234e7dba 100644 --- a/src/main/groovy/io/seqera/wave/service/counter/impl/LocalCounterProvider.groovy +++ b/src/main/groovy/io/seqera/wave/service/counter/impl/LocalCounterProvider.groovy @@ -61,4 +61,5 @@ class LocalCounterProvider implements CounterProvider { } return result } + } diff --git a/src/main/groovy/io/seqera/wave/service/counter/impl/RedisCounterProvider.groovy b/src/main/groovy/io/seqera/wave/service/counter/impl/RedisCounterProvider.groovy index 5e28467bf..e540acfb8 100644 --- a/src/main/groovy/io/seqera/wave/service/counter/impl/RedisCounterProvider.groovy +++ b/src/main/groovy/io/seqera/wave/service/counter/impl/RedisCounterProvider.groovy @@ -18,6 +18,8 @@ package io.seqera.wave.service.counter.impl +import java.time.Duration + import groovy.transform.CompileStatic import io.micronaut.context.annotation.Requires import io.micronaut.context.annotation.Value @@ -42,10 +44,21 @@ class RedisCounterProvider implements CounterProvider { @Value('${redis.hscan.count:10000}') private Integer hscanCount + @Value('${redis.key.expiry:1s}') + private Duration keyExpiry + + final private static String DATE_PATTERN = /\b\d{4}-\d{2}-\d{2}\b/ + + static boolean isMetricPerDate(String field) { + return field =~ DATE_PATTERN + } @Override long inc(String key, String field, long value) { try(Jedis conn=pool.getResource() ) { - return conn.hincrBy(key, field, value) + long count = conn.hincrBy(key, field, value) + if(isMetricPerDate(field)) + conn.hexpire(key, keyExpiry.toSeconds(), field) + return count } } @@ -70,4 +83,5 @@ class RedisCounterProvider implements CounterProvider { return result } } + } diff --git a/src/test/groovy/io/seqera/wave/service/counter/impl/RedisCounterProviderTest.groovy b/src/test/groovy/io/seqera/wave/service/counter/impl/RedisCounterProviderTest.groovy index 2364ddc9f..0dc6f0351 100644 --- a/src/test/groovy/io/seqera/wave/service/counter/impl/RedisCounterProviderTest.groovy +++ b/src/test/groovy/io/seqera/wave/service/counter/impl/RedisCounterProviderTest.groovy @@ -36,7 +36,11 @@ class RedisCounterProviderTest extends Specification implements RedisTestContain RedisCounterProvider redisCounterProvider def setup() { - applicationContext = ApplicationContext.run('test', 'redis') + applicationContext = ApplicationContext.run([ + REDIS_HOST : redisHostName, + REDIS_PORT : redisPort, + 'redis.key.expiry': '1s' + ], 'test', 'redis') redisCounterProvider = applicationContext.getBean(RedisCounterProvider) sleep(500) // workaround to wait for Redis connection } @@ -82,9 +86,22 @@ class RedisCounterProviderTest extends Specification implements RedisTestContain then: redisCounterProvider.getAllMatchingEntries('metrics/v1', 'pulls/o/*') == ['pulls/o/abc.in':3, 'pulls/o/bar.es':2, 'pulls/o/foo.it':1, 'pulls/o/abc.com.au/d/2024-05-30':1, 'pulls/o/abc.com.au/d/2024-05-31':1] + } + + def 'should expire the hash'(){ + when: + redisCounterProvider.inc('metrics/v1', 'pulls/o/abc.com.au', 1) + redisCounterProvider.inc('metrics/v1', 'pulls/o/abc.com.au/d/2024-07-14', 1) + sleep(500) + redisCounterProvider.inc('metrics/v1', 'pulls/o/abc.com.au/d/2024-07-15', 1) + sleep(500) + then:'this value should be one, because foo should be expired' + redisCounterProvider.get('metrics/v1', 'pulls/o/abc.com.au/d/2024-07-14') == null + sleep(500) and: - redisCounterProvider.getAllMatchingEntries('metrics/v1', 'pulls/o/*/d/2024-05-30') == - ['pulls/o/abc.com.au/d/2024-05-30':1] + redisCounterProvider.get('metrics/v1', 'pulls/o/abc.com.au/d/2024-07-15') == null + and: 'this value should be one, because org metric should not expire' + redisCounterProvider.get('metrics/v1', 'pulls/o/abc.com.au') == 1 } def 'should get correct org count for mirror and scan' () { diff --git a/src/test/groovy/io/seqera/wave/test/RedisTestContainer.groovy b/src/test/groovy/io/seqera/wave/test/RedisTestContainer.groovy index a9780d144..c1fe12669 100644 --- a/src/test/groovy/io/seqera/wave/test/RedisTestContainer.groovy +++ b/src/test/groovy/io/seqera/wave/test/RedisTestContainer.groovy @@ -46,7 +46,8 @@ trait RedisTestContainer { } def setupSpec() { - redisContainer = new GenericContainer(DockerImageName.parse("redis:7.0.4-alpine")) + log.debug "Starting Redis test container" + redisContainer = new GenericContainer(DockerImageName.parse("redis:7.4.0-alpine")) .withExposedPorts(6379) .waitingFor(Wait.forLogMessage(".*Ready to accept connections.*\\n", 1)) // starting redis