Fix: Dashboard timeout handling and critical architecture improvements

config/dev.exs

@@ -6,7 +6,12 @@ config :reencodarr, Reencodarr.Repo,
   stacktrace: true,
   show_sensitive_data_on_connection_error: true,
   # Increase pool size for better concurrency with Broadway pipelines
-  pool_size: 20
+  pool_size: 20,
+  # Increase checkout timeout for slow queries (especially JSON fragment queries in SQLite)
+  timeout: 30_000,
+  # DBConnection queue configuration for better handling under load
+  queue_target: 5_000,
+  queue_interval: 2_000
 
 # For development, we disable any cache and enable
 # debugging and code reloading.

flake.nix

@@ -14,8 +14,16 @@
       system: let
         pkgs = import nixpkgs {inherit system;};
         lib = pkgs.lib;
-        # Use latest stable OTP 28 with Elixir 1.19
-        erlang = pkgs.erlang_28;
+        # Build Erlang 28.1.1 from source
+        erlang = pkgs.beam.interpreters.erlang_28.overrideAttrs (oldAttrs: rec {
+          version = "28.1.1";
+          src = pkgs.fetchFromGitHub {
+            owner = "erlang";
+            repo = "otp";
+            rev = "OTP-${version}";
+            hash = "sha256-2Yop9zpx3dY549NFsjBRghb5vw+SnUSEmv6VIA0m5yQ=";
+          };
+        });
         beamPackages = pkgs.beam.packagesWith erlang;
         elixir = beamPackages.elixir_1_19;
       in {

lib/reencodarr/ab_av1.ex

@@ -68,6 +68,35 @@ defmodule Reencodarr.AbAv1 do
   """
   @spec encode(Media.Vmaf.t()) :: :ok | {:error, atom()}
   def encode(vmaf) do
+    # Skip MP4 files - compatibility issues to be resolved later
+    video_id = Map.get(vmaf, :video_id) || Map.get(vmaf, "video_id")
+
+    if is_integer(video_id) do
+      try do
+        video = Media.get_video!(video_id)
+
+        if is_binary(video.path) and String.ends_with?(video.path, ".mp4") do
+          # Skip MP4 files - compatibility issues
+          Logger.info("Skipping encode for MP4 file (compatibility issues): #{video.path}")
+          # Mark as failed to skip future encoding attempts
+          case Media.mark_as_failed(video) do
+            {:ok, _updated} -> :ok
+            error -> error
+          end
+        else
+          do_queue_encode(vmaf)
+        end
+      rescue
+        Ecto.NoResultsError ->
+          # Video doesn't exist - fall back to normal validation/queuing
+          do_queue_encode(vmaf)
+      end
+    else
+      do_queue_encode(vmaf)
+    end
+  end
+
+  defp do_queue_encode(vmaf) do
     case QueueManager.validate_encode_request(vmaf) do
       {:ok, validated_vmaf} ->
         message = QueueManager.build_encode_message(validated_vmaf)

lib/reencodarr/ab_av1/crf_search.ex

@@ -13,12 +13,12 @@ defmodule Reencodarr.AbAv1.CrfSearch do
   alias Reencodarr.AbAv1.Helper
   alias Reencodarr.AbAv1.OutputParser
   alias Reencodarr.Core.Parsers
+  alias Reencodarr.Core.Retry
   alias Reencodarr.Core.Time
   alias Reencodarr.Dashboard.Events
   alias Reencodarr.ErrorHelpers
   alias Reencodarr.Formatters
   alias Reencodarr.Media
-  alias Reencodarr.Media.VideoStateMachine
   alias Reencodarr.Repo
 
   require Logger
@@ -151,7 +151,7 @@ defmodule Reencodarr.AbAv1.CrfSearch do
   @impl true
   def handle_cast({:crf_search, video, vmaf_percent}, %{port: :none} = state) do
     # Mark video as crf_searching NOW that we're actually starting
-    case VideoStateMachine.transition_to_crf_searching(video) do
+    case Media.mark_as_crf_searching(video) do
       {:ok, _updated_video} ->
         :ok
 
@@ -241,32 +241,26 @@ defmodule Reencodarr.AbAv1.CrfSearch do
     full_line = buffer <> line
 
     try do
-      process_line(full_line, video, args, target_vmaf)
+      # Use retry logic for database operations
+      Retry.retry_on_db_busy(fn ->
+        process_line(full_line, video, args, target_vmaf)
+      end)
 
       # Add the line to our output buffer for failure tracking
       new_output_buffer = [full_line | output_buffer]
 
       {:noreply, %{state | partial_line_buffer: "", output_buffer: new_output_buffer}}
     rescue
+      # If retries are exhausted, log and continue
       e in Exqlite.Error ->
-        # Database is busy — don't crash. Requeue the same port message with a short delay.
-        Logger.warning(
-          "CrfSearch: Database busy while upserting VMAF, requeuing line: #{inspect(e)}"
-        )
-
-        # Re-send the exact same message after a short backoff to retry
-        Process.send_after(self(), {port, {:data, {:eol, line}}}, 200)
+        Logger.error("CrfSearch: Database error after retries, skipping line: #{inspect(e)}")
 
-        # Keep state intact — we'll retry later
-        {:noreply, state}
+        {:noreply, %{state | partial_line_buffer: ""}}
 
       e in DBConnection.ConnectionError ->
-        Logger.warning(
-          "CrfSearch: DB connection error while upserting VMAF, requeuing line: #{inspect(e)}"
-        )
+        Logger.error("CrfSearch: DB connection error after retries, skipping line: #{inspect(e)}")
 
-        Process.send_after(self(), {port, {:data, {:eol, line}}}, 200)
-        {:noreply, state}
+        {:noreply, %{state | partial_line_buffer: ""}}
     end
   end
 

lib/reencodarr/ab_av1/encode.ex

@@ -273,28 +273,39 @@ defmodule Reencodarr.AbAv1.Encode do
 
   # Private Helper Functions
   defp prepare_encode_state(vmaf, state) do
-    args = build_encode_args(vmaf)
-    output_file = Path.join(Helper.temp_dir(), "#{vmaf.video.id}.mkv")
+    # Mark video as encoding BEFORE starting the port to prevent duplicate dispatches
+    case Media.mark_as_encoding(vmaf.video) do
+      {:ok, _updated_video} ->
+        args = build_encode_args(vmaf)
+        output_file = Path.join(Helper.temp_dir(), "#{vmaf.video.id}.mkv")
 
-    port = Helper.open_port(args)
+        port = Helper.open_port(args)
 
-    # Broadcast encoding started to Dashboard Events
-    Events.broadcast_event(:encoding_started, %{
-      video_id: vmaf.video.id,
-      filename: Path.basename(vmaf.video.path)
-    })
+        # Broadcast encoding started to Dashboard Events
+        Events.broadcast_event(:encoding_started, %{
+          video_id: vmaf.video.id,
+          filename: Path.basename(vmaf.video.path)
+        })
 
-    # Set up a periodic timer to check if we're still alive and potentially emit progress
-    # Check every 10 seconds
-    Process.send_after(self(), :periodic_check, 10_000)
+        # Set up a periodic timer to check if we're still alive and potentially emit progress
+        # Check every 10 seconds
+        Process.send_after(self(), :periodic_check, 10_000)
 
-    %{
-      state
-      | port: port,
-        video: vmaf.video,
-        vmaf: vmaf,
-        output_file: output_file
-    }
+        %{
+          state
+          | port: port,
+            video: vmaf.video,
+            vmaf: vmaf,
+            output_file: output_file
+        }
+
+      {:error, reason} ->
+        Logger.error(
+          "Failed to mark video #{vmaf.video.id} as encoding: #{inspect(reason)}. Skipping encode."
+        )
+
+        state
+    end
   end
 
   defp build_encode_args(vmaf) do