[F] Implement cache warmer + Disable Layout Invalidation on Deploy

Dockerfile

@@ -51,6 +51,13 @@ ENV GEOIPUPDATE_EDITION_IDS="GeoLite2-ASN GeoLite2-City GeoLite2-Country"
 ENV GEOIPUPDATE_PRESERVE_FILE_TIMES=1
 ENV GEOIPUPDATE_VERBOSE=1
 ENV REMEMBERED_WB_UNPROTECTED_OBJECTS_LIMIT_RATIO=0.01
+ENV RUBY_GC_OLDMALLOC_LIMIT=1000000000000
+ENV RUBY_GC_OLDMALLOC_LIMIT_MAX=1000000000000
+ENV RUBY_GC_HEAP_0_INIT_SLOTS=1819132
+ENV RUBY_GC_HEAP_1_INIT_SLOTS=409209
+ENV RUBY_GC_HEAP_2_INIT_SLOTS=455808
+ENV RUBY_GC_HEAP_3_INIT_SLOTS=19968
+ENV RUBY_GC_HEAP_4_INIT_SLOTS=9265
 
 WORKDIR /srv/app
 COPY Gemfile /srv/app/Gemfile

app/jobs/cache_warmers/enqueue_enabled_job.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module CacheWarmers
+  # Enqueue each enabled {CacheWarmer} with a {CacheWarmers::RunJob}.
+  class EnqueueEnabledJob < ApplicationJob
+    include JobIteration::Iteration
+
+    good_job_control_concurrency_with(
+      total_limit: 1,
+      key: "CacheWarmers::EnqueueEnabledJob"
+    )
+
+    queue_as :maintenance
+
+    # @param [String] cursor
+    # @return [void]
+    def build_enumerator(cursor:)
+      enumerator_builder.active_record_on_records(
+        CacheWarmer.enabled,
+        cursor:
+      )
+    end
+
+    # @see CacheWarmers::RunJob
+    # @param [CacheWarmer] cache_warmer
+    # @return [void]
+    def each_iteration(cache_warmer)
+      cache_warmer.run_asynchronously!
+    end
+  end
+end

app/jobs/cache_warmers/run_job.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module CacheWarmers
+  # @see CacheWarmer
+  # @see CacheWarmers::Run
+  # @see CacheWarmers::Runner
+  class RunJob < ApplicationJob
+    queue_as :cache_warming
+
+    unique_job! by: :first_arg
+
+    # @return [void]
+    def perform(cache_warmer)
+      call_operation!("cache_warmers.run", cache_warmer)
+    end
+  end
+end

app/models/cache_warmer.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+# @see CacheWarmable
+class CacheWarmer < ApplicationRecord
+  include HasEphemeralSystemSlug
+  include TimestampScopes
+
+  belongs_to :warmable, polymorphic: true
+
+  has_many :cache_warmings, dependent: :delete_all, inverse_of: :cache_warmer
+
+  # @see CacheWarmers::Run
+  # @see CacheWarmers::Runner
+  monadic_operation! def run
+    call_operation("cache_warmers.run", self)
+  end
+
+  # @see CacheWarmers::EnqueueEnabledJob
+  # @see CacheWarmers::RunJob
+  # @return [void]
+  def run_asynchronously!
+    CacheWarmers::RunJob.perform_later self
+  end
+
+  class << self
+    # @return [ActiveRecord::Relation<CacheWarmer>]
+    def enabled(global: CachingConfig.warming_enabled?)
+      return none unless global
+
+      where(arel_enabled_by_warmable)
+    end
+
+    # @api private
+    # @return [Arel::Nodes::Case]
+    def arel_enabled_by_warmable
+      comm_query = Community.cache_warming_enabled.select(:id)
+      coll_query = Collection.cache_warming_enabled.select(:id)
+      item_query = Item.cache_warming_enabled.select(:id)
+
+      arel_case(arel_table[:warmable_type]) do |c|
+        c.when("Community").then(arel_attr_in_query(:warmable_id, comm_query.to_sql))
+        c.when("Collection").then(arel_attr_in_query(:warmable_id, coll_query.to_sql))
+        c.when("Item").then(arel_attr_in_query(:warmable_id, item_query.to_sql))
+        c.else(true)
+      end
+    end
+  end
+end

app/models/cache_warming.rb

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+class CacheWarming < ApplicationRecord
+  include HasEphemeralSystemSlug
+  include TimestampScopes
+
+  belongs_to :cache_warmer, inverse_of: :cache_warmings
+
+  validates :url, presence: true
+end

app/models/concerns/cache_warmable.rb

@@ -0,0 +1,87 @@
+# frozen_string_literal: true
+
+# A concern for models that should warm URLs on the frontend
+# via {CacheWarmer}.
+module CacheWarmable
+  extend ActiveSupport::Concern
+  extend DefinesMonadicOperation
+
+  included do
+    pg_enum! :cache_warming_status, as: :cache_warming_status, default: "default", allow_blank: false, prefix: :cache_warming
+
+    has_one :cache_warmer, as: :warmable, dependent: :destroy, inverse_of: :warmable
+
+    scope :cache_warming_enabled, -> { where(cache_warming_enabled: true) }
+
+    scope :cache_warming_disabled, -> { where(cache_warming_enabled: false) }
+
+    before_validation :determine_cache_warming_default_enabled!
+    before_validation :determine_cache_warming_enabled!
+
+    after_save :maybe_create_cache_warmer!
+  end
+
+  # @api private
+  # @return [Boolean]
+  def determine_cache_warmability!
+    determine_cache_warming_default_enabled!
+    determine_cache_warming_enabled!
+
+    update_columns(
+      cache_warming_default_enabled: cache_warming_default_enabled,
+      cache_warming_enabled: cache_warming_enabled
+    ) if changed?
+
+    maybe_create_cache_warmer!
+  end
+
+  # @return [CacheWarmer]
+  def maybe_create_cache_warmer!
+    return unless cache_warming_enabled?
+
+    cache_warmer || create_cache_warmer!
+  end
+
+  # @see CacheWarmers::RunWarmable
+  # @return [Dry::Monads::Result]
+  monadic_operation! def run_cache_warmer
+    call_operation("cache_warmers.run_warmable", self)
+  end
+
+  private
+
+  # @return [Boolean]
+  def derive_cache_warming_default_enabled
+    case self
+    when ::Community then true
+    when ::Collection then depth < CachingConfig.collection_depth
+    when ::Item then depth < CachingConfig.item_depth
+    else
+      # :nocov:
+      false
+      # :nocov:
+    end
+  end
+
+  # @return [Boolean]
+  def derive_cache_warming_enabled
+    case cache_warming_status
+    when "on"
+      true
+    when "off"
+      false
+    else
+      cache_warming_default_enabled
+    end
+  end
+
+  # @return [void]
+  def determine_cache_warming_default_enabled!
+    self.cache_warming_default_enabled = derive_cache_warming_default_enabled
+  end
+
+  # @return [void]
+  def determine_cache_warming_enabled!
+    self.cache_warming_enabled = derive_cache_warming_enabled
+  end
+end

app/models/concerns/hierarchical_entity.rb

@@ -5,6 +5,7 @@ module HierarchicalEntity
   extend DefinesMonadicOperation
 
   include AssociationHelpers
+  include CacheWarmable
   include ChecksContextualPermissions
   include ConfiguresContributionRole
   include EntityTemplating

app/operations/cache_warmers/run.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+module CacheWarmers
+  # @see CacheWarmers::Runner
+  class Run < Support::SimpleServiceOperation
+    service_klass CacheWarmers::Runner
+  end
+end

app/operations/cache_warmers/run_warmable.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module CacheWarmers
+  # An operation to run the {CacheWarmer} for a given {CacheWarmable} record.
+  #
+  # @see CacheWarmable
+  # @see CacheWarmer
+  # @see CacheWarmers::Run
+  # @see CacheWarmers::Runner
+  class RunWarmable
+    include Dry::Monads[:result]
+
+    # @param [CacheWarmable] warmable
+    # @return [Dry::Monads::Success(Integer)]
+    def call(warmable)
+      cache_warmer = warmable.cache_warmer
+
+      return Success(0) unless cache_warmer.present?
+
+      cache_warmer.run
+    end
+  end
+end

app/operations/schemas/static/load_version.rb

@@ -37,7 +37,9 @@ def call(definition, static_version)
       def populate_root_layouts_for(version)
         defs = yield version.populate_root_layouts(skip_layout_invalidation:)
 
-        layout_definitions.concat defs
+        if LayoutsConfig.invalidate_on_deploy?
+          layout_definitions.concat defs
+        end
 
         Success()
       end

app/services/cache_warmers/runner.rb

@@ -0,0 +1,109 @@
+# frozen_string_literal: true
+
+module CacheWarmers
+  # @see CacheWarmers::Run
+  class Runner < Support::HookBased::Actor
+    include Dry::Core::Constants
+    include Dry::Initializer[undefined: false].define -> do
+      param :cache_warmer, Types::CacheWarmer
+    end
+
+    standard_execution!
+
+    define_model_callbacks :make_request
+
+    delegate :warmable, to: :cache_warmer
+
+    delegate :cache_warming_enabled?, to: :warmable
+
+    # @return [<String>]
+    attr_reader :urls
+
+    # @return [Integer]
+    attr_reader :warming_count
+
+    # @return [Dry::Monads::Success(Integer)]
+    def call
+      run_callbacks :execute do
+        yield prepare!
+
+        yield make_requests!
+      end
+
+      Success warming_count
+    end
+
+    wrapped_hook! def prepare
+      @cache_warming = nil
+
+      @frontend_url = LocationsConfig.frontend_request
+
+      @http = yield Support::System["networking.http.build_client"].(@frontend_url, allow_insecure: true)
+
+      @urls = build_urls
+
+      @warming_count = 0
+
+      super
+    end
+
+    wrapped_hook! def make_requests
+      urls.each do |url|
+        make_request!(url)
+      end
+
+      super
+    end
+
+    around_make_request :time_request!
+
+    private
+
+    # @param [String] url
+    # @return [void]
+    def make_request!(url)
+      @cache_warming = cache_warmer.cache_warmings.build(url:)
+
+      run_callbacks :make_request do
+        response = @http.get(url)
+
+        @cache_warming.status = response.status
+      rescue Faraday::ConnectionFailed, Faraday::TimeoutError, Faraday::Error => e
+        @cache_warming.error_klass = e.class.name
+        @cache_warming.error_message = e.message
+        @cache_warming.status = -1
+      end
+
+      @cache_warming.save!
+
+      @warming_count += 1
+    ensure
+      @cache_warming = nil
+    end
+
+    # @return [void]
+    def time_request!
+      @cache_warming.duration = AbsoluteTime.realtime do
+        yield
+      end
+    end
+
+    # @return [<String>]
+    def build_urls
+      return EMPTY_ARRAY unless cache_warming_enabled?
+
+      [].tap do |u|
+        u.concat(frontend_entity_urls)
+      end
+    end
+
+    # @return [<String>]
+    def frontend_entity_urls
+      [].tap do |u|
+        url = FrontendConfig.entity_url_for(warmable)
+
+        u << url
+      end
+    end
+  end
+end

app/services/cache_warmers/types.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+module CacheWarmers
+  module Types
+    include Dry.Types
+
+    extend Support::EnhancedTypes
+
+    CacheWarmer = ModelInstance("CacheWarmer")
+  end
+end