Improve testing (#182)

* Bump Distances bound

* Fix up the tests

* Bump patch

* Relax bound on exponentiated

* Improve exponential kernel robustness

* Revert changes

* Produce some output during basekernel tests

Author: willtebbutt

Project.toml

@@ -1,6 +1,6 @@
 name = "KernelFunctions"
 uuid = "ec8451be-7e33-11e9-00cf-bbf324bd1392"
-version = "0.8.4"
+version = "0.8.5"
 
 [deps]
 Compat = "34da2185-b29b-5c13-b0c7-acf172513d20"
@@ -18,7 +18,7 @@ ZygoteRules = "700de1a5-db45-46bc-99cf-38207098b444"
 
 [compat]
 Compat = "2.2, 3"
-Distances = "0.9"
+Distances = "0.9.1"
 Functors = "0.1"
 Requires = "1.0.1"
 SpecialFunctions = "0.8, 0.9, 0.10"

src/test_utils.jl

@@ -1,7 +1,9 @@
 module TestUtils
 
 const __ATOL = 1e-9
+const __RTOL = 1e-9
 
+using Distances
 using LinearAlgebra
 using KernelFunctions
 using Random
@@ -33,6 +35,7 @@ function test_interface(
     x1::AbstractVector,
     x2::AbstractVector;
     atol=__ATOL,
+    rtol=__RTOL,
 )
     # Ensure that we have the required inputs.
     @assert length(x0) == length(x1)
@@ -69,8 +72,8 @@ function test_interface(
     @test eigmin(Matrix(kernelmatrix(k, x0))) > -atol
 
     # Check that unary elementwise / pairwise are consistent with the binary versions.
-    @test kerneldiagmatrix(k, x0) ≈ kerneldiagmatrix(k, x0, x0) atol=atol
-    @test kernelmatrix(k, x0) ≈ kernelmatrix(k, x0, x0) atol=atol
+    @test kerneldiagmatrix(k, x0) ≈ kerneldiagmatrix(k, x0, x0) atol=atol rtol=rtol
+    @test kernelmatrix(k, x0) ≈ kernelmatrix(k, x0, x0) atol=atol rtol=rtol
 
     # Check that basic kernel evaluation succeeds and is consistent with `kernelmatrix`.
     @test k(first(x0), first(x1)) isa Real
@@ -89,17 +92,17 @@ end
 function test_interface(
     rng::AbstractRNG, k::Kernel, ::Type{Vector{T}}; kwargs...
 ) where {T<:Real}
-    test_interface(k, randn(rng, T, 3), randn(rng, T, 3), randn(rng, T, 2); kwargs...)
+    test_interface(k, randn(rng, T, 1001), randn(rng, T, 1001), randn(rng, T, 1000); kwargs...)
 end
 
 function test_interface(
     rng::AbstractRNG, k::Kernel, ::Type{<:ColVecs{T}}; dim_in=2, kwargs...,
 ) where {T<:Real}
     test_interface(
         k,
-        ColVecs(randn(rng, T, dim_in, 3)),
-        ColVecs(randn(rng, T, dim_in, 3)),
-        ColVecs(randn(rng, T, dim_in, 2));
+        ColVecs(randn(rng, T, dim_in, 1001)),
+        ColVecs(randn(rng, T, dim_in, 1001)),
+        ColVecs(randn(rng, T, dim_in, 1000));
         kwargs...,
     )
 end
@@ -109,9 +112,9 @@ function test_interface(
 ) where {T<:Real}
     test_interface(
         k,
-        RowVecs(randn(rng, T, 3, dim_in)),
-        RowVecs(randn(rng, T, 3, dim_in)),
-        RowVecs(randn(rng, T, 2, dim_in));
+        RowVecs(randn(rng, T, 1001, dim_in)),
+        RowVecs(randn(rng, T, 1001, dim_in)),
+        RowVecs(randn(rng, T, 1000, dim_in));
         kwargs...,
     )
 end
@@ -121,9 +124,15 @@ function test_interface(k::Kernel, T::Type{<:AbstractVector}; kwargs...)
 end
 
 function test_interface(rng::AbstractRNG, k::Kernel, T::Type{<:Real}; kwargs...)
-    test_interface(rng, k, Vector{T}; kwargs...)
-    test_interface(rng, k, ColVecs{T}; kwargs...)
-    test_interface(rng, k, RowVecs{T}; kwargs...)
+    @testset "Vector{$T}" begin
+        test_interface(rng, k, Vector{T}; kwargs...)
+    end
+    @testset "ColVecs{$T}" begin
+        test_interface(rng, k, ColVecs{T}; kwargs...)
+    end
+    @testset "RowVecs{$T}" begin
+        test_interface(rng, k, RowVecs{T}; kwargs...)
+    end
 end
 
 function test_interface(k::Kernel, T::Type{<:Real}=Float64; kwargs...)

test/basekernels/exponential.jl

@@ -16,7 +16,7 @@
         @test KernelFunctions.iskroncompatible(k) == true
 
         # Standardised tests.
-        TestUtils.test_interface(k, Float64)
+        TestUtils.test_interface(k)
         test_ADs(SEKernel)
     end
     @testset "ExponentialKernel" begin
@@ -30,7 +30,7 @@
         @test KernelFunctions.iskroncompatible(k) == true
 
         # Standardised tests.
-        TestUtils.test_interface(k, Float64)
+        TestUtils.test_interface(k)
         test_ADs(ExponentialKernel)
     end
     @testset "GammaExponentialKernel" begin
@@ -46,6 +46,8 @@
 
         test_ADs(γ -> GammaExponentialKernel(gamma=first(γ)), [γ])
         test_params(k, ([γ],))
+        TestUtils.test_interface(GammaExponentialKernel(γ=1.36))
+
         #Coherence :
         @test isapprox(
             GammaExponentialKernel(γ=2.0)(sqrt(0.5) * v1, sqrt(0.5) * v2),

test/basekernels/exponentiated.jl

@@ -11,7 +11,7 @@
     @test metric(ExponentiatedKernel()) == KernelFunctions.DotProduct()
     @test repr(k) == "Exponentiated Kernel"
 
-    # Standardised tests.
-    TestUtils.test_interface(k, Float64)
+    # Standardised tests. This kernel appears to be fairly numerically unstable.
+    TestUtils.test_interface(k; atol=1e-3)
     test_ADs(ExponentiatedKernel)
 end

test/basekernels/gabor.jl

@@ -19,7 +19,7 @@
     @test k.p ≈ 1.0 atol=1e-5
     @test repr(k) == "Gabor Kernel (ell = 1.0, p = 1.0)"
 
-    test_interface(k)
+    test_interface(k, Vector{Float64})
 
     test_ADs(x -> GaborKernel(ell = x[1], p = x[2]), [ell, p], ADs = [:Zygote])
 

test/runtests.jl

@@ -82,20 +82,35 @@ include("test_utils.jl")
 
     @testset "basekernels" begin
         include(joinpath("basekernels", "constant.jl"))
+        print(" ")
         include(joinpath("basekernels", "cosine.jl"))
+        print(" ")
         include(joinpath("basekernels", "exponential.jl"))
+        print(" ")
         include(joinpath("basekernels", "exponentiated.jl"))
+        print(" ")
         include(joinpath("basekernels", "fbm.jl"))
+        print(" ")
         include(joinpath("basekernels", "gabor.jl"))
+        print(" ")
         include(joinpath("basekernels", "maha.jl"))
+        print(" ")
         include(joinpath("basekernels", "matern.jl"))
+        print(" ")
         include(joinpath("basekernels", "nn.jl"))
+        print(" ")
         include(joinpath("basekernels", "periodic.jl"))
+        print(" ")
         include(joinpath("basekernels", "piecewisepolynomial.jl"))
+        print(" ")
         include(joinpath("basekernels", "polynomial.jl"))
+        print(" ")
         include(joinpath("basekernels", "rationalquad.jl"))
+        print(" ")
         include(joinpath("basekernels", "sm.jl"))
+        print(" ")
         include(joinpath("basekernels", "wiener.jl"))
+        print(" ")
     end
     @info "Ran tests on BaseKernel"