Add support for JULIA_REFERENCETESTS_UPDATE_AND_ERROR flag

Project.toml

@@ -1,6 +1,6 @@
 name = "ReferenceTests"
 uuid = "324d217c-45ce-50fc-942e-d289b448e8cf"
-version = "0.10.6"
+version = "0.10.7"
 authors = ["Christof Stocker <[email protected]>", "Frames White <[email protected]>", "Johnny Chen <[email protected]>"]
 
 [deps]

README.md

@@ -75,7 +75,7 @@ using ReferenceTests, TestImages
 
 Proper image formats such as `.png` are also supported for full-res image testing. If your terminal 
 [supports Sixel](https://www.arewesixelyet.com/), then ReferenceTests will display the full image
-in your terminal using (Sixel.jl)[https://github.com/JuliaIO/Sixel.jl]:
+in your terminal using [Sixel.jl](https://github.com/JuliaIO/Sixel.jl):
 
 ![sixel_demo](.github/images/sixel_demo.png)
 
@@ -97,11 +97,15 @@ test suite. These tests are easy to run via `pkg> test` but
 the child process used within `pkg> test` is non-interactive, so the
 update prompt will not show if there are mismatches.
 
-To update references within a package test suite, there are three options:
+To update references within a package test suite, there are four options:
 
 - Set the environment variable `JULIA_REFERENCETESTS_UPDATE` to `"true"`
   and run `pkg> test`, which will force update any non-matches. You can then
   check changes to any git-tracked reference images before commit.
+- Set the environment variable `JULIA_REFERENCETESTS_UPDATE_AND_ERROR` to 
+  `"true"` and run `pkg> test`, which will also force update any non-matches,
+  but each non-matching file will result in a test failure. You can then
+  check changes to any git-tracked reference images before commit.
 - Delete any reference images you wish to update and run `pkg> test`, given
   that missing references are created automatically.
 - Run the `test/runtests.jl` interactively. This may be easier using

src/test_reference.jl

@@ -171,14 +171,26 @@ function test_reference(
 
         if force_update() || input_bool("Replace reference with actual result?")
             mv(actual_path, reference_path; force=true)  # overwrite old file it
-            @info "Please run the tests again for any changes to take effect"
+            if error_on_force_update()
+                error("""
+                The reference file has been updated, but an error was thrown because the environment variable
+                `JULIA_REFERENCETEST_UPDATE_AND_ERROR` was set to `true`.
+                """)
+            else
+                @info "Please run the tests again for any changes to take effect"
+            end
         else
             @test false
         end
     end
 end
 
-force_update() = tryparse(Bool, get(ENV, "JULIA_REFERENCETESTS_UPDATE", "false")) === true
+function force_update()
+    update = tryparse(Bool, get(ENV, "JULIA_REFERENCETESTS_UPDATE", "false")) === true
+    update_and_err = tryparse(Bool, get(ENV, "JULIA_REFERENCETESTS_UPDATE_AND_ERROR", "false")) === true
+    update || update_and_err
+end
+error_on_force_update() = tryparse(Bool, get(ENV, "JULIA_REFERENCETESTS_UPDATE_AND_ERROR", "false")) === true
 
 """
     mismatch_staging_dir()

test/runtests.jl

@@ -205,4 +205,19 @@ end
     @test_reference file Dict(:ar=>arr_float) by=comp
 end
 
+@testset "force update and error" begin
+    withenv("JULIA_REFERENCETESTS_UPDATE" => "true") do
+        @test begin
+            @test_reference "references/random_string.txt" rand(10)
+            @test_reference "references/random_string.txt" rand(10)
+            true
+        end
+    end
+    withenv("JULIA_REFERENCETESTS_UPDATE_AND_ERROR" => "true") do
+        @test_throws Exception begin
+            @test_reference "references/random_string.txt" rand(10)
+        end
+    end
+end
+
 end  # top level testset