diff --git a/Dockerfile b/Dockerfile index 601a7c9b..fde93015 100644 --- a/Dockerfile +++ b/Dockerfile @@ -51,6 +51,13 @@ ENV GEOIPUPDATE_EDITION_IDS="GeoLite2-ASN GeoLite2-City GeoLite2-Country" ENV GEOIPUPDATE_PRESERVE_FILE_TIMES=1 ENV GEOIPUPDATE_VERBOSE=1 ENV REMEMBERED_WB_UNPROTECTED_OBJECTS_LIMIT_RATIO=0.01 +ENV RUBY_GC_OLDMALLOC_LIMIT=1000000000000 +ENV RUBY_GC_OLDMALLOC_LIMIT_MAX=1000000000000 +ENV RUBY_GC_HEAP_0_INIT_SLOTS=1819132 +ENV RUBY_GC_HEAP_1_INIT_SLOTS=409209 +ENV RUBY_GC_HEAP_2_INIT_SLOTS=455808 +ENV RUBY_GC_HEAP_3_INIT_SLOTS=19968 +ENV RUBY_GC_HEAP_4_INIT_SLOTS=9265 WORKDIR /srv/app COPY Gemfile /srv/app/Gemfile diff --git a/app/jobs/cache_warmers/enqueue_enabled_job.rb b/app/jobs/cache_warmers/enqueue_enabled_job.rb new file mode 100644 index 00000000..faf1dc50 --- /dev/null +++ b/app/jobs/cache_warmers/enqueue_enabled_job.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +module CacheWarmers + # Enqueue each enabled {CacheWarmer} with a {CacheWarmers::RunJob}. + class EnqueueEnabledJob < ApplicationJob + include JobIteration::Iteration + + good_job_control_concurrency_with( + total_limit: 1, + key: "CacheWarmers::EnqueueEnabledJob" + ) + + queue_as :maintenance + + # @param [String] cursor + # @return [void] + def build_enumerator(cursor:) + enumerator_builder.active_record_on_records( + CacheWarmer.enabled, + cursor: + ) + end + + # @see CacheWarmers::RunJob + # @param [CacheWarmer] cache_warmer + # @return [void] + def each_iteration(cache_warmer) + cache_warmer.run_asynchronously! + end + end +end diff --git a/app/jobs/cache_warmers/run_job.rb b/app/jobs/cache_warmers/run_job.rb new file mode 100644 index 00000000..e882d892 --- /dev/null +++ b/app/jobs/cache_warmers/run_job.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +module CacheWarmers + # @see CacheWarmer + # @see CacheWarmers::Run + # @see CacheWarmers::Runner + class RunJob < ApplicationJob + queue_as :cache_warming + + unique_job! by: :first_arg + + # @return [void] + def perform(cache_warmer) + call_operation!("cache_warmers.run", cache_warmer) + end + end +end diff --git a/app/models/cache_warmer.rb b/app/models/cache_warmer.rb new file mode 100644 index 00000000..a6874c08 --- /dev/null +++ b/app/models/cache_warmer.rb @@ -0,0 +1,48 @@ +# frozen_string_literal: true + +# @see CacheWarmable +class CacheWarmer < ApplicationRecord + include HasEphemeralSystemSlug + include TimestampScopes + + belongs_to :warmable, polymorphic: true + + has_many :cache_warmings, dependent: :delete_all, inverse_of: :cache_warmer + + # @see CacheWarmers::Run + # @see CacheWarmers::Runner + monadic_operation! def run + call_operation("cache_warmers.run", self) + end + + # @see CacheWarmers::EnqueueEnabledJob + # @see CacheWarmers::RunJob + # @return [void] + def run_asynchronously! + CacheWarmers::RunJob.perform_later self + end + + class << self + # @return [ActiveRecord::Relation] + def enabled(global: CachingConfig.warming_enabled?) + return none unless global + + where(arel_enabled_by_warmable) + end + + # @api private + # @return [Arel::Nodes::Case] + def arel_enabled_by_warmable + comm_query = Community.cache_warming_enabled.select(:id) + coll_query = Collection.cache_warming_enabled.select(:id) + item_query = Item.cache_warming_enabled.select(:id) + + arel_case(arel_table[:warmable_type]) do |c| + c.when("Community").then(arel_attr_in_query(:warmable_id, comm_query.to_sql)) + c.when("Collection").then(arel_attr_in_query(:warmable_id, coll_query.to_sql)) + c.when("Item").then(arel_attr_in_query(:warmable_id, item_query.to_sql)) + c.else(true) + end + end + end +end diff --git a/app/models/cache_warming.rb b/app/models/cache_warming.rb new file mode 100644 index 00000000..b771db4a --- /dev/null +++ b/app/models/cache_warming.rb @@ -0,0 +1,10 @@ +# frozen_string_literal: true + +class CacheWarming < ApplicationRecord + include HasEphemeralSystemSlug + include TimestampScopes + + belongs_to :cache_warmer, inverse_of: :cache_warmings + + validates :url, presence: true +end diff --git a/app/models/concerns/cache_warmable.rb b/app/models/concerns/cache_warmable.rb new file mode 100644 index 00000000..95058945 --- /dev/null +++ b/app/models/concerns/cache_warmable.rb @@ -0,0 +1,87 @@ +# frozen_string_literal: true + +# A concern for models that should warm URLs on the frontend +# via {CacheWarmer}. +module CacheWarmable + extend ActiveSupport::Concern + extend DefinesMonadicOperation + + included do + pg_enum! :cache_warming_status, as: :cache_warming_status, default: "default", allow_blank: false, prefix: :cache_warming + + has_one :cache_warmer, as: :warmable, dependent: :destroy, inverse_of: :warmable + + scope :cache_warming_enabled, -> { where(cache_warming_enabled: true) } + + scope :cache_warming_disabled, -> { where(cache_warming_enabled: false) } + + before_validation :determine_cache_warming_default_enabled! + before_validation :determine_cache_warming_enabled! + + after_save :maybe_create_cache_warmer! + end + + # @api private + # @return [Boolean] + def determine_cache_warmability! + determine_cache_warming_default_enabled! + determine_cache_warming_enabled! + + update_columns( + cache_warming_default_enabled: cache_warming_default_enabled, + cache_warming_enabled: cache_warming_enabled + ) if changed? + + maybe_create_cache_warmer! + end + + # @return [CacheWarmer] + def maybe_create_cache_warmer! + return unless cache_warming_enabled? + + cache_warmer || create_cache_warmer! + end + + # @see CacheWarmers::RunWarmable + # @return [Dry::Monads::Result] + monadic_operation! def run_cache_warmer + call_operation("cache_warmers.run_warmable", self) + end + + private + + # @return [Boolean] + def derive_cache_warming_default_enabled + case self + when ::Community then true + when ::Collection then depth < CachingConfig.collection_depth + when ::Item then depth < CachingConfig.item_depth + else + # :nocov: + false + # :nocov: + end + end + + # @return [Boolean] + def derive_cache_warming_enabled + case cache_warming_status + when "on" + true + when "off" + false + else + cache_warming_default_enabled + end + end + + # @return [void] + def determine_cache_warming_default_enabled! + self.cache_warming_default_enabled = derive_cache_warming_default_enabled + end + + # @return [void] + def determine_cache_warming_enabled! + self.cache_warming_enabled = derive_cache_warming_enabled + end +end diff --git a/app/models/concerns/hierarchical_entity.rb b/app/models/concerns/hierarchical_entity.rb index cec7b48d..04a5868c 100644 --- a/app/models/concerns/hierarchical_entity.rb +++ b/app/models/concerns/hierarchical_entity.rb @@ -5,6 +5,7 @@ module HierarchicalEntity extend DefinesMonadicOperation include AssociationHelpers + include CacheWarmable include ChecksContextualPermissions include ConfiguresContributionRole include EntityTemplating diff --git a/app/operations/cache_warmers/run.rb b/app/operations/cache_warmers/run.rb new file mode 100644 index 00000000..80e8fee2 --- /dev/null +++ b/app/operations/cache_warmers/run.rb @@ -0,0 +1,8 @@ +# frozen_string_literal: true + +module CacheWarmers + # @see CacheWarmers::Runner + class Run < Support::SimpleServiceOperation + service_klass CacheWarmers::Runner + end +end diff --git a/app/operations/cache_warmers/run_warmable.rb b/app/operations/cache_warmers/run_warmable.rb new file mode 100644 index 00000000..96a4943a --- /dev/null +++ b/app/operations/cache_warmers/run_warmable.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +module CacheWarmers + # An operation to run the {CacheWarmer} for a given {CacheWarmable} record. + # + # @see CacheWarmable + # @see CacheWarmer + # @see CacheWarmers::Run + # @see CacheWarmers::Runner + class RunWarmable + include Dry::Monads[:result] + + # @param [CacheWarmable] warmable + # @return [Dry::Monads::Success(Integer)] + def call(warmable) + cache_warmer = warmable.cache_warmer + + return Success(0) unless cache_warmer.present? + + cache_warmer.run + end + end +end diff --git a/app/operations/schemas/static/load_version.rb b/app/operations/schemas/static/load_version.rb index d064ce05..33e0ef72 100644 --- a/app/operations/schemas/static/load_version.rb +++ b/app/operations/schemas/static/load_version.rb @@ -37,7 +37,9 @@ def call(definition, static_version) def populate_root_layouts_for(version) defs = yield version.populate_root_layouts(skip_layout_invalidation:) - layout_definitions.concat defs + if LayoutsConfig.invalidate_on_deploy? + layout_definitions.concat defs + end Success() end diff --git a/app/services/cache_warmers/runner.rb b/app/services/cache_warmers/runner.rb new file mode 100644 index 00000000..6c364c63 --- /dev/null +++ b/app/services/cache_warmers/runner.rb @@ -0,0 +1,109 @@ +# frozen_string_literal: true + +module CacheWarmers + # @see CacheWarmers::Run + class Runner < Support::HookBased::Actor + include Dry::Core::Constants + include Dry::Initializer[undefined: false].define -> do + param :cache_warmer, Types::CacheWarmer + end + + standard_execution! + + define_model_callbacks :make_request + + delegate :warmable, to: :cache_warmer + + delegate :cache_warming_enabled?, to: :warmable + + # @return [] + attr_reader :urls + + # @return [Integer] + attr_reader :warming_count + + # @return [Dry::Monads::Success(Integer)] + def call + run_callbacks :execute do + yield prepare! + + yield make_requests! + end + + Success warming_count + end + + wrapped_hook! def prepare + @cache_warming = nil + + @frontend_url = LocationsConfig.frontend_request + + @http = yield Support::System["networking.http.build_client"].(@frontend_url, allow_insecure: true) + + @urls = build_urls + + @warming_count = 0 + + super + end + + wrapped_hook! def make_requests + urls.each do |url| + make_request!(url) + end + + super + end + + around_make_request :time_request! + + private + + # @param [String] url + # @return [void] + def make_request!(url) + @cache_warming = cache_warmer.cache_warmings.build(url:) + + run_callbacks :make_request do + response = @http.get(url) + + @cache_warming.status = response.status + rescue Faraday::ConnectionFailed, Faraday::TimeoutError, Faraday::Error => e + @cache_warming.error_klass = e.class.name + @cache_warming.error_message = e.message + @cache_warming.status = -1 + end + + @cache_warming.save! + + @warming_count += 1 + ensure + @cache_warming = nil + end + + # @return [void] + def time_request! + @cache_warming.duration = AbsoluteTime.realtime do + yield + end + end + + # @return [] + def build_urls + return EMPTY_ARRAY unless cache_warming_enabled? + + [].tap do |u| + u.concat(frontend_entity_urls) + end + end + + # @return [] + def frontend_entity_urls + [].tap do |u| + url = FrontendConfig.entity_url_for(warmable) + + u << url + end + end + end +end diff --git a/app/services/cache_warmers/types.rb b/app/services/cache_warmers/types.rb new file mode 100644 index 00000000..5477a9b3 --- /dev/null +++ b/app/services/cache_warmers/types.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +module CacheWarmers + module Types + include Dry.Types + + extend Support::EnhancedTypes + + CacheWarmer = ModelInstance("CacheWarmer") + end +end diff --git a/app/services/schemas/static/tracks_layout_definitions.rb b/app/services/schemas/static/tracks_layout_definitions.rb index f2915904..ac4a47bf 100644 --- a/app/services/schemas/static/tracks_layout_definitions.rb +++ b/app/services/schemas/static/tracks_layout_definitions.rb @@ -18,8 +18,12 @@ def capture_layout_definitions_to_invalidate! end end - # Not an operation that should/can fail. - MeruAPI::Container["layouts.invalidate_batch"].(layout_definitions).value! + # :nocov: + if LayoutsConfig.invalidate_on_deploy? + # Not an operation that should/can fail. + MeruAPI::Container["layouts.invalidate_batch"].(layout_definitions).value! + end + # :nocov: return result end diff --git a/config/caching.yml b/config/caching.yml new file mode 100644 index 00000000..4a24f977 --- /dev/null +++ b/config/caching.yml @@ -0,0 +1,14 @@ +default: &default + warming_enabled: true + +development: + <<: *default + warming_enabled: false + +test: + <<: *default + warming_enabled: true + +production: + <<: *default + warming_enabled: true diff --git a/config/configs/caching_config.rb b/config/configs/caching_config.rb new file mode 100644 index 00000000..97399c94 --- /dev/null +++ b/config/configs/caching_config.rb @@ -0,0 +1,7 @@ +# frozen_string_literal: true + +class CachingConfig < ApplicationConfig + attr_config warming_enabled: true, collection_depth: 2, item_depth: 0 + + coerce_types warming_enabled: :boolean, collection_depth: :integer, item_depth: :integer +end diff --git a/config/configs/frontend_config.rb b/config/configs/frontend_config.rb index 0f47dacb..4be20734 100644 --- a/config/configs/frontend_config.rb +++ b/config/configs/frontend_config.rb @@ -2,4 +2,35 @@ class FrontendConfig < ApplicationConfig attr_config :revalidate_secret + + COMMUNITY_PATH_TEMPLATE = "/communities/%s" + COLLECTION_PATH_TEMPLATE = "/collections/%s" + ITEM_PATH_TEMPLATE = "/items/%s" + + # @param [HierarchicalEntity] entity + # @return [String] + def entity_path_for(entity) + params = { system_slug: entity.system_slug } + + case entity + when Community + COMMUNITY_PATH_TEMPLATE % params + when Collection + COLLECTION_PATH_TEMPLATE % params + when Item + ITEM_PATH_TEMPLATE % params + else + # :nocov: + raise "Unsupported entity type: #{entity.class.name}" + # :nocov: + end + end + + # @param [HierarchicalEntity] entity + # @return [String] + def entity_url_for(entity) + base_url = LocationsConfig.frontend_request + + URI.join(base_url, entity_path_for(entity)).to_s + end end diff --git a/config/configs/layouts_config.rb b/config/configs/layouts_config.rb new file mode 100644 index 00000000..8c02393d --- /dev/null +++ b/config/configs/layouts_config.rb @@ -0,0 +1,8 @@ +# frozen_string_literal: true + +# Anyway config class for managing layouts and rendering +class LayoutsConfig < ApplicationConfig + attr_config invalidate_on_deploy: false, track_slot_compilation_time: false + + coerce_types invalidate_on_deploy: :boolean, track_slot_compilation_time: :boolean +end diff --git a/config/configs/locations_config.rb b/config/configs/locations_config.rb index e3a6a27f..16e50fc4 100644 --- a/config/configs/locations_config.rb +++ b/config/configs/locations_config.rb @@ -3,7 +3,15 @@ class LocationsConfig < ApplicationConfig INCLUDED_PORT = Dry::Types["integer"].constrained(excluded_from: [80, 443]) - attr_config frontend: "http://localhost:14700", admin: "http://localhost:3000", api: "http://localhost:6222", debug: "http://localhost:3400" + attr_config :frontend_internal, :admin_internal, frontend: "http://localhost:14700", admin: "http://localhost:3000", api: "http://localhost:6222", debug: "http://localhost:3400" + + def admin_request + admin_internal.presence || admin + end + + def frontend_request + frontend_internal.presence || frontend + end # @return [Hash] memoize def api_url_options diff --git a/config/initializers/900_good_job.rb b/config/initializers/900_good_job.rb index 50a1a196..893144e3 100644 --- a/config/initializers/900_good_job.rb +++ b/config/initializers/900_good_job.rb @@ -7,10 +7,10 @@ queues = [ "maintenance:1", "rendering:1", + "+revalidations,cache_warming:1", "+purging,hierarchies,entities,orderings,invalidations,layouts:2", "+harvest_pruning,extraction,harvesting,asset_fetching:2", "permissions,processing,default,mailers,ahoy:2", - "revalidations:1", ].join(?;) config.good_job.preserve_job_records = :on_unhandled_error @@ -46,6 +46,11 @@ class: "Attributions::Items::ManageJob", description: "Ensure item attributions are up to date", }, + "cache_warmers.enqueue_enabled": { + cron: "25 */2 * * *", + class: "CacheWarmers::EnqueueEnabledJob", + description: "Enqueue enabled cache warmers", + }, "contributors.audit_contribution_counts": { cron: "*/5 * * * *", class: "Contributors::AuditContributionCountsJob", diff --git a/config/layouts.yml b/config/layouts.yml new file mode 100644 index 00000000..6e2a1914 --- /dev/null +++ b/config/layouts.yml @@ -0,0 +1,15 @@ +default: &default + invalidate_on_deploy: true + +development: + <<: *default + invalidate_on_deploy: false + track_slot_compilation_time: true + +test: + <<: *default + invalidate_on_deploy: true + +production: + <<: *default + invalidate_on_deploy: false diff --git a/config/locations.yml b/config/locations.yml new file mode 100644 index 00000000..75e2c1a6 --- /dev/null +++ b/config/locations.yml @@ -0,0 +1,11 @@ +default: &default + +development: + <<: *default + admin_internal: "http://host.docker.internal:3000" + frontend_internal: "http://host.docker.internal:14700" + +test: + <<: *default + admin_internal: "http://admin.meru.test" + frontend_internal: "http://frontend.meru.test" \ No newline at end of file diff --git a/db/migrate/20251009053741_create_cache_warmers.rb b/db/migrate/20251009053741_create_cache_warmers.rb new file mode 100644 index 00000000..d6b9221c --- /dev/null +++ b/db/migrate/20251009053741_create_cache_warmers.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +class CreateCacheWarmers < ActiveRecord::Migration[7.2] + def change + create_table :cache_warmers, id: :uuid do |t| + t.references :warmable, polymorphic: true, null: false, type: :uuid, index: { unique: true } + + t.timestamps null: false, default: -> { "CURRENT_TIMESTAMP" } + end + end +end diff --git a/db/migrate/20251009053907_create_cache_warmings.rb b/db/migrate/20251009053907_create_cache_warmings.rb new file mode 100644 index 00000000..a98eeea5 --- /dev/null +++ b/db/migrate/20251009053907_create_cache_warmings.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +class CreateCacheWarmings < ActiveRecord::Migration[7.2] + def change + create_table :cache_warmings, id: :uuid do |t| + t.references :cache_warmer, null: false, foreign_key: { on_delete: :cascade }, type: :uuid + t.integer :status + t.decimal :duration + + t.text :url, null: false + t.text :error_klass + t.text :error_message + + t.timestamps null: false, default: -> { "CURRENT_TIMESTAMP" } + end + end +end diff --git a/db/migrate/20251009055637_add_cache_warming_to_entities.rb b/db/migrate/20251009055637_add_cache_warming_to_entities.rb new file mode 100644 index 00000000..030bc035 --- /dev/null +++ b/db/migrate/20251009055637_add_cache_warming_to_entities.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +class AddCacheWarmingToEntities < ActiveRecord::Migration[7.2] + TABLES = %i[communities collections items].freeze + + create_enum :cache_warming_status, %w[default on off] + + def change + TABLES.each do |table| + change_table table do |t| + t.boolean :cache_warming_default_enabled, null: false, default: table == :communities + + t.boolean :cache_warming_enabled, null: false, default: false + + t.enum :cache_warming_status, enum_type: :cache_warming_status, null: false, default: "default" + end + end + end +end diff --git a/db/structure.sql b/db/structure.sql index a9c54335..1072ee7a 100644 --- a/db/structure.sql +++ b/db/structure.sql @@ -182,6 +182,17 @@ CREATE TYPE public.blurb_background AS ENUM ( ); +-- +-- Name: cache_warming_status; Type: TYPE; Schema: public; Owner: - +-- + +CREATE TYPE public.cache_warming_status AS ENUM ( + 'default', + 'on', + 'off' +); + + -- -- Name: child_entity_kind; Type: TYPE; Schema: public; Owner: - -- @@ -4008,7 +4019,10 @@ CREATE TABLE public.collections ( marked_for_purge boolean DEFAULT false NOT NULL, generation uuid, render_duration numeric, - last_rendered_at timestamp without time zone + last_rendered_at timestamp without time zone, + cache_warming_default_enabled boolean DEFAULT false NOT NULL, + cache_warming_enabled boolean DEFAULT false NOT NULL, + cache_warming_status public.cache_warming_status DEFAULT 'default'::public.cache_warming_status NOT NULL ); @@ -4055,7 +4069,10 @@ CREATE TABLE public.communities ( marked_for_purge boolean DEFAULT false NOT NULL, generation uuid, render_duration numeric, - last_rendered_at timestamp without time zone + last_rendered_at timestamp without time zone, + cache_warming_default_enabled boolean DEFAULT true NOT NULL, + cache_warming_enabled boolean DEFAULT false NOT NULL, + cache_warming_status public.cache_warming_status DEFAULT 'default'::public.cache_warming_status NOT NULL ); @@ -4118,7 +4135,10 @@ CREATE TABLE public.items ( marked_for_purge boolean DEFAULT false NOT NULL, generation uuid, render_duration numeric, - last_rendered_at timestamp without time zone + last_rendered_at timestamp without time zone, + cache_warming_default_enabled boolean DEFAULT false NOT NULL, + cache_warming_enabled boolean DEFAULT false NOT NULL, + cache_warming_status public.cache_warming_status DEFAULT 'default'::public.cache_warming_status NOT NULL ); @@ -4182,6 +4202,36 @@ CREATE VIEW public.audits_mismatched_item_parents AS ORDER BY c.id, hier.generations DESC; +-- +-- Name: cache_warmers; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.cache_warmers ( + id uuid DEFAULT gen_random_uuid() NOT NULL, + warmable_type character varying NOT NULL, + warmable_id uuid NOT NULL, + created_at timestamp(6) without time zone DEFAULT CURRENT_TIMESTAMP NOT NULL, + updated_at timestamp(6) without time zone DEFAULT CURRENT_TIMESTAMP NOT NULL +); + + +-- +-- Name: cache_warmings; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.cache_warmings ( + id uuid DEFAULT gen_random_uuid() NOT NULL, + cache_warmer_id uuid NOT NULL, + status integer, + duration numeric, + url text NOT NULL, + error_klass text, + error_message text, + created_at timestamp(6) without time zone DEFAULT CURRENT_TIMESTAMP NOT NULL, + updated_at timestamp(6) without time zone DEFAULT CURRENT_TIMESTAMP NOT NULL +); + + -- -- Name: collection_attributions; Type: TABLE; Schema: public; Owner: - -- @@ -7972,6 +8022,22 @@ ALTER TABLE ONLY public.assets ADD CONSTRAINT assets_pkey PRIMARY KEY (id); +-- +-- Name: cache_warmers cache_warmers_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.cache_warmers + ADD CONSTRAINT cache_warmers_pkey PRIMARY KEY (id); + + +-- +-- Name: cache_warmings cache_warmings_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.cache_warmings + ADD CONSTRAINT cache_warmings_pkey PRIMARY KEY (id); + + -- -- Name: collection_attributions collection_attributions_pkey; Type: CONSTRAINT; Schema: public; Owner: - -- @@ -9823,6 +9889,20 @@ CREATE INDEX index_authorizing_entities_on_hierarchical ON public.authorizing_en CREATE INDEX index_authorizing_entities_single_user ON public.authorizing_entities USING btree (auth_path, scope) INCLUDE (hierarchical_id, hierarchical_type); +-- +-- Name: index_cache_warmers_on_warmable; Type: INDEX; Schema: public; Owner: - +-- + +CREATE UNIQUE INDEX index_cache_warmers_on_warmable ON public.cache_warmers USING btree (warmable_type, warmable_id); + + +-- +-- Name: index_cache_warmings_on_cache_warmer_id; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX index_cache_warmings_on_cache_warmer_id ON public.cache_warmings USING btree (cache_warmer_id); + + -- -- Name: index_collection_attributions_ordering; Type: INDEX; Schema: public; Owner: - -- @@ -13795,6 +13875,14 @@ ALTER TABLE ONLY public.granted_permissions ADD CONSTRAINT fk_rails_461469ccc7 FOREIGN KEY (access_grant_id) REFERENCES public.access_grants(id) ON DELETE CASCADE; +-- +-- Name: cache_warmings fk_rails_490e4452b3; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.cache_warmings + ADD CONSTRAINT fk_rails_490e4452b3 FOREIGN KEY (cache_warmer_id) REFERENCES public.cache_warmers(id) ON DELETE CASCADE; + + -- -- Name: access_grants fk_rails_4cf2824701; Type: FK CONSTRAINT; Schema: public; Owner: - -- @@ -14922,6 +15010,9 @@ ALTER TABLE ONLY public.templates_ordering_instances SET search_path TO "$user", public; INSERT INTO "schema_migrations" (version) VALUES +('20251009055637'), +('20251009053907'), +('20251009053741'), ('20251008224630'), ('20251008224043'), ('20251008180200'), diff --git a/docker/production/Dockerfile b/docker/production/Dockerfile index 61f4d5fe..43f79467 100644 --- a/docker/production/Dockerfile +++ b/docker/production/Dockerfile @@ -49,6 +49,14 @@ ENV BUNDLE_PATH=/bundle \ ENV PATH="${BUNDLE_BIN}:${PATH}" ENV LD_PRELOAD=libjemalloc.so.2 ENV REMEMBERED_WB_UNPROTECTED_OBJECTS_LIMIT_RATIO=0.01 +ENV RUBY_GC_OLDMALLOC_LIMIT=1000000000000 +ENV RUBY_GC_OLDMALLOC_LIMIT_MAX=1000000000000 +ENV RUBY_GC_HEAP_0_INIT_SLOTS=1819132 +ENV RUBY_GC_HEAP_1_INIT_SLOTS=409209 +ENV RUBY_GC_HEAP_2_INIT_SLOTS=455808 +ENV RUBY_GC_HEAP_3_INIT_SLOTS=19968 +ENV RUBY_GC_HEAP_4_INIT_SLOTS=9265 + RUN gem update --system && gem install bundler:2.7.2 diff --git a/lib/frozen_record/static_cached_columns.yml b/lib/frozen_record/static_cached_columns.yml index 9806b737..09547bcb 100644 --- a/lib/frozen_record/static_cached_columns.yml +++ b/lib/frozen_record/static_cached_columns.yml @@ -1493,6 +1493,193 @@ default_function: CURRENT_TIMESTAMP has_default: true virtual: false +- id: CacheWarmer#id + model_name: CacheWarmer + table_name: cache_warmers + name: id + type: uuid + 'null': false + sql_type_metadata: + sql_type: uuid + type: uuid + default: + default_function: gen_random_uuid() + has_default: true + virtual: false +- id: CacheWarmer#warmable_type + model_name: CacheWarmer + table_name: cache_warmers + name: warmable_type + type: string + 'null': false + sql_type_metadata: + sql_type: character varying + type: string + default: + default_function: + has_default: false + virtual: false +- id: CacheWarmer#warmable_id + model_name: CacheWarmer + table_name: cache_warmers + name: warmable_id + type: uuid + 'null': false + sql_type_metadata: + sql_type: uuid + type: uuid + default: + default_function: + has_default: false + virtual: false +- id: CacheWarmer#created_at + model_name: CacheWarmer + table_name: cache_warmers + name: created_at + type: datetime + 'null': false + sql_type_metadata: + sql_type: timestamp(6) without time zone + type: datetime + precision: 6 + default: + default_function: CURRENT_TIMESTAMP + has_default: true + virtual: false +- id: CacheWarmer#updated_at + model_name: CacheWarmer + table_name: cache_warmers + name: updated_at + type: datetime + 'null': false + sql_type_metadata: + sql_type: timestamp(6) without time zone + type: datetime + precision: 6 + default: + default_function: CURRENT_TIMESTAMP + has_default: true + virtual: false +- id: CacheWarming#id + model_name: CacheWarming + table_name: cache_warmings + name: id + type: uuid + 'null': false + sql_type_metadata: + sql_type: uuid + type: uuid + default: + default_function: gen_random_uuid() + has_default: true + virtual: false +- id: CacheWarming#cache_warmer_id + model_name: CacheWarming + table_name: cache_warmings + name: cache_warmer_id + type: uuid + 'null': false + sql_type_metadata: + sql_type: uuid + type: uuid + default: + default_function: + has_default: false + virtual: false +- id: CacheWarming#status + model_name: CacheWarming + table_name: cache_warmings + name: status + type: integer + 'null': true + sql_type_metadata: + sql_type: integer + type: integer + limit: 4 + default: + default_function: + has_default: false + virtual: false +- id: CacheWarming#duration + model_name: CacheWarming + table_name: cache_warmings + name: duration + type: decimal + 'null': true + sql_type_metadata: + sql_type: numeric + type: decimal + default: + default_function: + has_default: false + virtual: false +- id: CacheWarming#url + model_name: CacheWarming + table_name: cache_warmings + name: url + type: text + 'null': false + sql_type_metadata: + sql_type: text + type: text + default: + default_function: + has_default: false + virtual: false +- id: CacheWarming#error_klass + model_name: CacheWarming + table_name: cache_warmings + name: error_klass + type: text + 'null': true + sql_type_metadata: + sql_type: text + type: text + default: + default_function: + has_default: false + virtual: false +- id: CacheWarming#error_message + model_name: CacheWarming + table_name: cache_warmings + name: error_message + type: text + 'null': true + sql_type_metadata: + sql_type: text + type: text + default: + default_function: + has_default: false + virtual: false +- id: CacheWarming#created_at + model_name: CacheWarming + table_name: cache_warmings + name: created_at + type: datetime + 'null': false + sql_type_metadata: + sql_type: timestamp(6) without time zone + type: datetime + precision: 6 + default: + default_function: CURRENT_TIMESTAMP + has_default: true + virtual: false +- id: CacheWarming#updated_at + model_name: CacheWarming + table_name: cache_warmings + name: updated_at + type: datetime + 'null': false + sql_type_metadata: + sql_type: timestamp(6) without time zone + type: datetime + precision: 6 + default: + default_function: CURRENT_TIMESTAMP + has_default: true + virtual: false - id: Collection#id model_name: Collection table_name: collections @@ -1847,6 +2034,45 @@ default_function: has_default: false virtual: false +- id: Collection#cache_warming_default_enabled + model_name: Collection + table_name: collections + name: cache_warming_default_enabled + type: boolean + 'null': false + sql_type_metadata: + sql_type: boolean + type: boolean + default: 'false' + default_function: + has_default: true + virtual: false +- id: Collection#cache_warming_enabled + model_name: Collection + table_name: collections + name: cache_warming_enabled + type: boolean + 'null': false + sql_type_metadata: + sql_type: boolean + type: boolean + default: 'false' + default_function: + has_default: true + virtual: false +- id: Collection#cache_warming_status + model_name: Collection + table_name: collections + name: cache_warming_status + type: enum + 'null': false + sql_type_metadata: + sql_type: cache_warming_status + type: enum + default: default + default_function: + has_default: true + virtual: false - id: CollectionAttribution#id model_name: CollectionAttribution table_name: collection_attributions @@ -2457,6 +2683,45 @@ default_function: has_default: false virtual: false +- id: Community#cache_warming_default_enabled + model_name: Community + table_name: communities + name: cache_warming_default_enabled + type: boolean + 'null': false + sql_type_metadata: + sql_type: boolean + type: boolean + default: 'true' + default_function: + has_default: true + virtual: false +- id: Community#cache_warming_enabled + model_name: Community + table_name: communities + name: cache_warming_enabled + type: boolean + 'null': false + sql_type_metadata: + sql_type: boolean + type: boolean + default: 'false' + default_function: + has_default: true + virtual: false +- id: Community#cache_warming_status + model_name: Community + table_name: communities + name: cache_warming_status + type: enum + 'null': false + sql_type_metadata: + sql_type: cache_warming_status + type: enum + default: default + default_function: + has_default: true + virtual: false - id: CommunityMembership#id model_name: CommunityMembership table_name: community_memberships @@ -11246,6 +11511,45 @@ default_function: has_default: false virtual: false +- id: Item#cache_warming_default_enabled + model_name: Item + table_name: items + name: cache_warming_default_enabled + type: boolean + 'null': false + sql_type_metadata: + sql_type: boolean + type: boolean + default: 'false' + default_function: + has_default: true + virtual: false +- id: Item#cache_warming_enabled + model_name: Item + table_name: items + name: cache_warming_enabled + type: boolean + 'null': false + sql_type_metadata: + sql_type: boolean + type: boolean + default: 'false' + default_function: + has_default: true + virtual: false +- id: Item#cache_warming_status + model_name: Item + table_name: items + name: cache_warming_status + type: enum + 'null': false + sql_type_metadata: + sql_type: cache_warming_status + type: enum + default: default + default_function: + has_default: true + virtual: false - id: ItemAttribution#id model_name: ItemAttribution table_name: item_attributions diff --git a/spec/jobs/cache_warmers/enqueue_enabled_job_spec.rb b/spec/jobs/cache_warmers/enqueue_enabled_job_spec.rb new file mode 100644 index 00000000..6ffb86b9 --- /dev/null +++ b/spec/jobs/cache_warmers/enqueue_enabled_job_spec.rb @@ -0,0 +1,39 @@ +# frozen_string_literal: true + +RSpec.describe CacheWarmers::EnqueueEnabledJob, type: :job do + let_it_be(:community, refind: true) { FactoryBot.create(:community) } + + let(:cache_warmer) do + community.cache_warmer + end + + it "enqueues the job if the cache warmer is enabled" do + expect do + described_class.perform_now + end.to have_enqueued_job(CacheWarmers::RunJob).with(cache_warmer) + end + + context "when the cache warmer is disabled" do + before do + community.cache_warming_off! + end + + it "does not enqueue the job" do + expect do + described_class.perform_now + end.not_to have_enqueued_job(CacheWarmers::RunJob).with(cache_warmer) + end + end + + context "when cache warming is globally disabled" do + before do + allow(CachingConfig).to receive(:warming_enabled?).and_return(false) + end + + it "does not enqueue the job" do + expect do + described_class.perform_now + end.not_to have_enqueued_job(CacheWarmers::RunJob) + end + end +end diff --git a/spec/jobs/cache_warmers/run_job_spec.rb b/spec/jobs/cache_warmers/run_job_spec.rb new file mode 100644 index 00000000..26888f2f --- /dev/null +++ b/spec/jobs/cache_warmers/run_job_spec.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +RSpec.describe CacheWarmers::RunJob, type: :job do + let_it_be(:community, refind: true) { FactoryBot.create(:community) } + + let(:cache_warmer) do + community.cache_warmer + end + + it_behaves_like "a pass-through operation job", "cache_warmers.run" do + let(:job_arg) { cache_warmer } + end +end diff --git a/spec/models/cache_warmer_spec.rb b/spec/models/cache_warmer_spec.rb new file mode 100644 index 00000000..cb6846e1 --- /dev/null +++ b/spec/models/cache_warmer_spec.rb @@ -0,0 +1,5 @@ +# frozen_string_literal: true + +RSpec.describe CacheWarmer, type: :model do + pending "add some examples to (or delete) #{__FILE__}" +end diff --git a/spec/models/cache_warming_spec.rb b/spec/models/cache_warming_spec.rb new file mode 100644 index 00000000..47e608de --- /dev/null +++ b/spec/models/cache_warming_spec.rb @@ -0,0 +1,5 @@ +# frozen_string_literal: true + +RSpec.describe CacheWarming, type: :model do + pending "add some examples to (or delete) #{__FILE__}" +end diff --git a/spec/operations/cache_warmers/run_warmable_spec.rb b/spec/operations/cache_warmers/run_warmable_spec.rb new file mode 100644 index 00000000..4062755a --- /dev/null +++ b/spec/operations/cache_warmers/run_warmable_spec.rb @@ -0,0 +1,101 @@ +# frozen_string_literal: true + +RSpec.describe CacheWarmers::RunWarmable, type: :operation do + let_it_be(:community, refind: true) { FactoryBot.create(:community) } + let_it_be(:collection, refind: true) { FactoryBot.create(:collection, community:) } + let_it_be(:item, refind: true) { FactoryBot.create(:item, collection:) } + + shared_examples_for "a cache warmable entity" do + let(:entity) { raise "must be defined" } + let(:enabled_by_default) { true } + let(:disabled_result) { 0 } + let(:entity_path) { FrontendConfig.entity_url_for(entity) } + + context "when the entity is turned on" do + before do + entity.cache_warming_on! + end + + context "when the frontend is responding appropriately" do + before do + stub_request(:get, entity_path).to_return(status: 200, body: "", headers: {}) + end + + it "warms the cache" do + expect do + expect_calling_with(entity).to succeed.with(1) + end.to change(CacheWarming.where(status: 200), :count).by(1) + end + end + + context "when the frontend times out" do + before do + stub_request(:get, entity_path).to_timeout + end + + it "records the failure" do + expect do + expect_calling_with(entity).to succeed.with(1) + end.to change(CacheWarming.where(status: -1), :count).by(1) + end + end + + context "when the frontend is not found" do + before do + stub_request(:get, entity_path).to_return(status: 404, body: "", headers: {}) + end + + it "records the failure" do + expect do + expect_calling_with(entity).to succeed.with(1) + end.to change(CacheWarming.where(status: 404), :count).by(1) + end + end + + context "when the frontend is down" do + before do + stub_request(:get, entity_path).to_raise(Faraday::ConnectionFailed.new("Connection failed")) + end + + it "records the failure" do + expect do + expect_calling_with(entity).to succeed.with(1) + end.to change(CacheWarming.where(status: -1), :count).by(1) + end + end + end + + context "when the entity is manually disabled" do + before do + entity.cache_warming_off! + end + + it "skips the warming" do + expect do + expect_calling_with(entity).to succeed.with(disabled_result) + end.to keep_the_same(CacheWarming, :count) + end + end + end + + context "with a Community" do + it_behaves_like "a cache warmable entity" do + let(:entity) { community } + let(:enabled_by_default) { true } + end + end + + context "with a Collection" do + it_behaves_like "a cache warmable entity" do + let(:entity) { collection } + let(:enabled_by_default) { true } + end + end + + context "with an Item" do + it_behaves_like "a cache warmable entity" do + let(:entity) { item } + let(:enabled_by_default) { false } + end + end +end