Stop all a job's retries from quickly draining out through one node in under a minute

contrib/admin/mypy-with-ignore.py

@@ -29,7 +29,6 @@ def main():
         'docs/conf.py',
         'docs/vendor/sphinxcontrib/fulltoc.py',
         'docs/vendor/sphinxcontrib/__init__.py',
-        'src/toil/job.py',
         'src/toil/leader.py',
         'src/toil/__init__.py',
         'src/toil/deferred.py',

docs/running/cliOptions.rst

@@ -475,8 +475,14 @@ systems have issues!).
   --retryCount INT
                         Number of times to retry a failing job before giving
                         up and labeling job failed. default=1
+  --retryBackoffSeconds FLOAT
+                        Number of seconds to wait when first retrying a job.
+                        default=2
+  --retryBackoffFactor FLOAT
+                        Factor to increase retry backof time by for each
+                        additional retry. default=3
   --stopOnFirstFailure BOOL
-                        Stop the workflow at the first complete job failure. 
+                        Stop the workflow at the first complete job failure.
   --enableUnlimitedPreemptibleRetries
                         If set, preemptible failures (or any failure due to an
                         instance getting unexpectedly terminated) will not count

src/toil/common.py

@@ -218,6 +218,8 @@ class Config:
 
     # Retrying/rescuing jobs
     retryCount: int
+    retry_backoff_seconds: float
+    retry_backoff_factor: float
     stop_on_first_failure: bool
     enableUnlimitedPreemptibleRetries: bool
     doubleMem: bool
@@ -397,6 +399,8 @@ def set_option(option_name: str, old_names: list[str] | None = None) -> None:
 
         # Retrying/rescuing jobs
         set_option("retryCount")
+        set_option("retry_backoff_seconds")
+        set_option("retry_backoff_factor")
         set_option("stop_on_first_failure")
         set_option("enableUnlimitedPreemptibleRetries")
         set_option("doubleMem")

src/toil/cwl/cwltoil.py

@@ -2939,14 +2939,12 @@ def makeRootJob(
         # Get metadata for non-tool input files
         input_metadata = get_file_sizes(
             input_filenames,
-            toil._jobStore,
             include_remote_files=options.reference_inputs,
         )
 
         # Also get metadata for tool input files, so we can resilve them to candidate URIs
         tool_metadata = get_file_sizes(
             input_filenames,
-            toil._jobStore,
             include_remote_files=options.reference_inputs,
         )
 

src/toil/deferred.py

@@ -17,6 +17,8 @@
 import tempfile
 from collections import namedtuple
 from contextlib import contextmanager
+from typing import Any, Callable
+from typing_extensions import ParamSpec
 
 import dill
 
@@ -40,8 +42,10 @@ class DeferredFunction(
     True
     """
 
+    P = ParamSpec("P")
+
     @classmethod
-    def create(cls, function, *args, **kwargs):
+    def create(cls, function: Callable[P, Any], *args: P.args, **kwargs: P.kwargs):
         """
         Capture the given callable and arguments as an instance of this class.