diff --git a/test/General/test_errors.jl b/test/General/test_errors.jl
index 0c1b85b5c8..2f3da89a08 100644
--- a/test/General/test_errors.jl
+++ b/test/General/test_errors.jl
@@ -427,6 +427,31 @@ function test_GetAttributeNotAllowed_showerror()
     return
 end
 
+function _test_method_redefinition(filename)
+    contents = read(filename, String)
+    functions = Set{String}()
+    for regex in (r"^function (test\_.+?)\(.*?\)"m, r"^(test\_.+?)\(.*?\) \= "m)
+        for m in eachmatch(regex, contents)
+            fn_name = String(m[1])
+            if fn_name in functions
+                error("In $filename: overwritten method: $fn_name")
+            end
+            push!(functions, fn_name)
+        end
+    end
+    return
+end
+
+# This function tests that all files in `/test` do not have redefined methods.
+function test_method_redefinition()
+    for (root, dirs, files) in walkdir(dirname(@__DIR__))
+        for file in files
+            _test_method_redefinition(joinpath(root, file))
+        end
+    end
+    return
+end
+
 end  # module
 
 TestErrors.runtests()
diff --git a/test/runtests.jl b/test/runtests.jl
index 31ebfd8c3a..0bfe64cf5e 100644
--- a/test/runtests.jl
+++ b/test/runtests.jl
@@ -31,51 +31,23 @@ MODULES_TO_TEST = get(
     "General;Benchmarks;Bridges/General;Bridges/Constraint;Bridges/Objective;Bridges/Variable;FileFormats;Nonlinear;Test;Utilities",
 )
 
-"""
-    include_with_method_redefinition_check(jl_filename)
-
-This function runs `include(jl_filename)` with an additional check that there
-are no `WARNING: Method definition foo in module Foo overwritten` warnings.
-
-It does this by piping `stderr` to a file, and then parsing the file.
-
-One consequence is that `stderr` is not printed until the _end_ of this
-function. Thus, some warnings may appear in the wrong place.
+is_test_file(f) = startswith(f, "test_") && endswith(f, ".jl")
 
-This function requires Julia to be started with `--warn-overwrite=true`.
-"""
-const init_code = quote
-    function include_with_method_redefinition_check(jl_filename)
-        stderr_filename = tempname()
-        open(stderr_filename, "w") do io
-            return redirect_stderr(() -> include(jl_filename), io)
-        end
-        contents = read(stderr_filename, String)
-        print(stderr, contents)
-        regex =
-            r"WARNING: Method definition (.+?) in module (.+?) at (.+?) overwritten at (.+?)\n"
-        if match(regex, contents) !== nothing
-            error("Found overwritten method in $jl_filename")
+testsuite = Dict{String,Expr}()
+for submodule in split(MODULES_TO_TEST, ";")
+    for (root, dirs, files) in walkdir(submodule)
+        for file in joinpath.(root, filter(is_test_file, files))
+            testsuite[file] = :(include($file))
         end
-        return
     end
 end
 
-is_test_file(f) = startswith(f, "test_") && endswith(f, ".jl")
-
-testsuite = Dict{String,Expr}(
-    file => :(include_with_method_redefinition_check($file)) for
-    submodule in split(MODULES_TO_TEST, ";") for
-    (root, dirs, files) in walkdir(submodule) for
-    file in joinpath.(root, filter(is_test_file, files))
-)
-
 import MathOptInterface
 import ParallelTestRunner
 import Test
 
 if Sys.WORD_SIZE == 64
-    ParallelTestRunner.runtests(MathOptInterface, ARGS; testsuite, init_code)
+    ParallelTestRunner.runtests(MathOptInterface, ARGS; testsuite)
 else
     Test.@testset "$filename" for filename in keys(testsuite)
         include(filename)