[E] Cache entity lists and template hidden checks

app/graphql/types/template_entity_list_type.rb

@@ -74,5 +74,9 @@ class TemplateEntityListType < Types::BaseObject
       The minimum hierarchical depth for the entities in the list (if known).
       TEXT
     end
+
+    load_association! :entities
+
+    load_association! :list_item_layout_instances, as: :list_item_layouts
   end
 end

app/graphql/types/template_has_entity_list_type.rb

@@ -17,5 +17,13 @@ module TemplateHasEntityListType
       The list of entities to render as part of this template's content.
       TEXT
     end
+
+    load_association! :cached_entity_list
+
+    def entity_list
+      cached_entity_list.then do |cel|
+        cel.presence || ::Templates::EntityList.new
+      end
+    end
   end
 end

app/jobs/entities/reprocess_layouts_job.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module Entities
+  class ReprocessLayoutsJob < ApplicationJob
+    queue_as :layouts
+
+    queue_with_priority do
+      case arguments.first
+      when ::Item then 500
+      when ::Collection then 600
+      when ::Community then 700
+      else
+        # :nocov:
+        999
+        # :nocov:
+      end
+    end
+
+    # @param [HierarchicalEntity] entity
+    # @return [void]
+    def perform(entity)
+      call_operation!("entities.reprocess_layouts", entity)
+    end
+  end
+end

app/jobs/rendering/abstract_reprocess_entities_job.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+module Rendering
+  # @abstract
+  # Enqueues reprocessing jobs for all entities of a given class.
+  # @see Entities::ReprocessLayoutsJob
+  class AbstractReprocessEntitiesJob < ApplicationJob
+    extend Dry::Core::ClassAttributes
+
+    include JobIteration::Iteration
+
+    defines :entity_klass, type: Support::Types::Class
+
+    entity_klass ApplicationRecord
+
+    good_job_control_concurrency_with(
+      total_limit: 1,
+      key: -> { "#{self.class.name}-#{queue_name}" }
+    )
+
+    queue_as :default
+
+    # @param [String] cursor
+    # @return [void]
+    def build_enumerator(cursor:)
+      enumerator_builder.active_record_on_records(
+        self.class.entity_klass.all,
+        cursor:
+      )
+    end
+
+    # @see Entities::ReprocessLayoutsJob
+    # @param [HierarchicalEntity] entity
+    # @return [void]
+    def each_iteration(entity)
+      Entities::ReprocessLayoutsJob.perform_later(entity)
+    end
+  end
+end

app/jobs/rendering/reprocess_all_collections_job.rb

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+module Rendering
+  # Enqueues reprocessing jobs for all {Collection} entities.
+  class ReprocessAllCollectionsJob < AbstractReprocessEntitiesJob
+    entity_klass Collection
+
+    queue_with_priority 200
+  end
+end

app/jobs/rendering/reprocess_all_communities_job.rb

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+module Rendering
+  # Enqueues reprocessing jobs for all {Community} entities.
+  class ReprocessAllCommunitiesJob < AbstractReprocessEntitiesJob
+    entity_klass Community
+
+    queue_with_priority 300
+  end
+end

app/jobs/rendering/reprocess_all_entities_job.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module Rendering
+  # Enqueues reprocessing jobs for all entities.
+  class ReprocessAllEntitiesJob < ApplicationJob
+    queue_as :default
+
+    # @return [void]
+    def perform
+      Rendering::ReprocessAllItemsJob.perform_later
+      Rendering::ReprocessAllCollectionsJob.perform_later
+      Rendering::ReprocessAllCommunitiesJob.perform_later
+    end
+  end
+end

app/jobs/rendering/reprocess_all_items_job.rb

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+module Rendering
+  # Enqueues reprocessing jobs for all {Item} entities.
+  class ReprocessAllItemsJob < AbstractReprocessEntitiesJob
+    entity_klass Item
+
+    queue_with_priority 100
+  end
+end

app/jobs/system/shutdown_worker_job.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+module System
+  class ShutdownWorkerJob < ApplicationJob
+    queue_as :default
+
+    queue_with_priority(-10_000)
+
+    # @return [void]
+    def perform
+      GOOD_JOB_KEEP_RUNNING.make_false
+    end
+  end
+end

app/models/concerns/entity_templating_support.rb

@@ -6,10 +6,12 @@ module EntityTemplatingSupport
   extend ActiveSupport::Concern
   extend DefinesMonadicOperation
 
-  include Dry::Effects.Reader(:layout_invalidation_disabled, default: false)
   include ModelMutationSupport
 
   included do
+    has_many :cached_entity_lists, as: :entity, class_name: "Templates::CachedEntityList", inverse_of: :entity, dependent: :delete_all
+    has_many :cached_entity_list_items, as: :entity, class_name: "Templates::CachedEntityListItem", inverse_of: :entity, dependent: :delete_all
+
     has_many :entity_derived_layout_definitions, as: :entity, inverse_of: :entity, dependent: :delete_all
 
     has_many_readonly :missing_layout_instances, as: :entity, inverse_of: :entity, class_name: "EntityMissingLayoutInstance"
@@ -82,9 +84,16 @@ def has_no_layout_definitions_derived?
     call_operation("entities.invalidate_related_layouts", self)
   end
 
+  # @see Layouts::Disabled
+  # @see Layouts::Disabler
+  # @return [void]
+  def layout_invalidation_currently_disabled?
+    Layouts::Disabled.currently?
+  end
+
   # @see ModelMutationSupport#in_graphql_mutation?
   def layout_invalidation_disabled?
-    in_graphql_mutation? || layout_invalidation_disabled || force_disable_layout_invalidation
+    in_graphql_mutation? || layout_invalidation_currently_disabled? || force_disable_layout_invalidation
   end
 
   # @return [String]
@@ -107,25 +116,23 @@ def render_lock_key
     call_operation("entities.render_layouts", self)
   end
 
-  def stale?
-    has_layout_invalidations? || has_missing_layouts? || has_missing_templates? || has_no_layout_definitions_derived?
+  # @see Entities::ReprocessLayout
+  # @see Entities::LayoutReprocessor
+  # @param [Layouts::Types::Kind] layout_kind
+  # @return [Dry::Monads::Success(HierarchicalEntity)]
+  def reprocess_layout(layout_kind)
+    call_operation("entities.reprocess_layout", self, layout_kind:)
   end
 
-  # @return [HierarchicalEntity]
-  def validated_layout_source
-    if has_invalid_layouts?
-      invalidation = layout_invalidations.latest
-
-      invalidation.process!
-
-      reload
-    elsif has_missing_layouts?
-      render_layouts!
-
-      reload
-    end
+  # @see Entities::ReprocessLayouts
+  # @see Entities::LayoutsReprocessor
+  # @return [Dry::Monads::Success(HierarchicalEntity)]
+  monadic_matcher! def reprocess_layouts
+    call_operation("entities.reprocess_layouts", self)
+  end
 
-    return self
+  def stale?
+    has_layout_invalidations? || has_missing_layouts? || has_missing_templates? || has_no_layout_definitions_derived?
   end
 
   module ClassMethods

app/models/concerns/layout_instance.rb

@@ -21,6 +21,14 @@ module LayoutInstance
 
     template_instance_names [].freeze
 
+    if model_name == "Layouts::ListItemInstance"
+      has_many :cached_entity_list_items,
+        class_name: "Templates::CachedEntityListItem",
+        foreign_key: :list_item_layout_instance_id,
+        inverse_of: :list_item_layout_instance,
+        dependent: :delete_all
+    end
+
     scope :root, -> { joins(:layout_definition).merge(layout_record.definition_klass.root) }
 
     scope :with_template_instances, -> { includes(*template_instance_names) }
@@ -60,6 +68,9 @@ def build_digest_attributes
       layout_kind:,
       generation:,
       config:,
+      post_processed_at:,
+      all_hidden:,
+      all_slots_empty:,
     }
   end
 
@@ -92,6 +103,18 @@ def template_instances
     @template_instances ||= fetch_template_instances
   end
 
+  monadic_operation! def post_process(**options)
+    call_operation("layouts.instances.post_process", self, **options)
+  end
+
+  monadic_operation! def process(**options)
+    call_operation("layouts.instances.process", self, **options)
+  end
+
+  monadic_operation! def reprocess(**options)
+    call_operation("layouts.instances.reprocess", self, **options)
+  end
+
   monadic_operation! def upsert_layout_instance_digest
     call_operation("layouts.digests.instances.upsert", self)
   end

app/models/concerns/template_instance.rb

@@ -7,7 +7,6 @@ module TemplateInstance
   include HasLayoutKind
   include HasTemplateKind
   include Renderable
-  include Support::Caching::Usage
 
   included do
     attribute :config, Templates::Instances::Config.to_type
@@ -24,22 +23,6 @@ module TemplateInstance
     before_validation :infer_config!
   end
 
-  # @!attribute [r] all_slots_empty
-  # @see Templates::SlotMappings::AbstractInstanceSlots#all_empty?
-  # @return [Boolean]
-  def all_slots_empty
-    slots.all_empty?
-  end
-
-  alias all_slots_empty? all_slots_empty
-
-  # Boolean complement of {#force_show?}.
-  #
-  # Used in {#hidden} to bypass the hide logic.
-  def allow_hide?
-    !force_show?
-  end
-
   # @see Templates::Instances::BuildConfig
   # @see Templates::Instances::ConfigBuilder
   monadic_operation! def build_config
@@ -52,16 +35,11 @@ def allow_hide?
     call_operation("templates.instances.build_digest_attributes", self)
   end
 
-  # @api private
-  # @abstract
-  # @return [Boolean]
-  def force_show
-    false
-  end
-
-  # @see #force_show
-  def force_show?
-    force_show
+  # Boolean complement of {#force_show?}.
+  #
+  # Used when calculating {#hidden} to bypass the hide logic.
+  def calculate_allow_hide?
+    !force_show?
   end
 
   # For most templates, it is just derived from from {#hidden_by_empty_slots}.
@@ -74,27 +52,37 @@ def calculate_hidden
     hidden_by_empty_slots?
   end
 
-  # @!attribute [r] hidden
-  # Whether or not the template should be hidden in the frontend, derived from {#calculate_hidden}.
-  #
-  # Because it can be expensive to calculate at runtime, and we need to expose
-  # it on siblings, it uses {Support::Caching::Cache} to safely store the value across requests.
+  # @api private
+  # @abstract
   # @return [Boolean]
-  def hidden
-    vog_cache cache_key, :hidden do
-      allow_hide? && calculate_hidden
-    end
+  def force_show
+    false
   end
 
-  alias hidden? hidden
+  # @see #force_show
+  def force_show?
+    force_show
+  end
 
-  # @!attribute [r] hidden_by_empty_slots
-  # @return [Boolean]
-  def hidden_by_empty_slots
-    slots.hides_template?
+  # @see Templates::Instances::PostProcessor
+  # @return [Dry::Monads::Success(TemplateInstance)]
+  monadic_operation! def post_process
+    call_operation("templates.instances.post_process", self)
+  end
+
+  # @see Templates::Instances::Processor
+  # @return [Dry::Monads::Success(TemplateInstance)]
+  monadic_operation! def process
+    call_operation("templates.instances.process", self)
   end
 
-  alias hidden_by_empty_slots? hidden_by_empty_slots
+  # @see Templates::Instances::Reprocessor
+  # @param [Hash] options
+  # @option options [Boolean] :update_digest (false)
+  # @return [Dry::Monads::Success(TemplateInstance)]
+  monadic_operation! def reprocess(**options)
+    call_operation("templates.instances.reprocess", self, **options)
+  end
 
   # @see Templates::Digests::Instances::TemplateUpserter
   monadic_operation! def upsert_instance_digests

app/models/schema_version.rb

@@ -25,6 +25,8 @@ class SchemaVersion < ApplicationRecord
 
   belongs_to :schema_definition, inverse_of: :schema_versions
 
+  has_many :cached_entity_list_items, inverse_of: :schema_version, dependent: :delete_all, class_name: "Templates::CachedEntityListItem"
+
   has_many :communities, dependent: :restrict_with_error, inverse_of: :schema_version
   has_many :collections, dependent: :restrict_with_error, inverse_of: :schema_version
   has_many :items, dependent: :restrict_with_error, inverse_of: :schema_version