JuliaFEM.jl/src/test.jl

# This file is a part of JuliaFEM.
# License is MIT: see https://github.com/JuliaFEM/JuliaFEM.jl/blob/master/LICENSE.md

using BaseTestNext

abstract TestResult

type NormalTestResult <: TestResult
    test_function :: Function
    result
end

type CriticalTestResult <: TestResult
    filename :: ASCIIString
    message
end

function TestResult(test_function::Function, result)
    NormalTestResult(test_function, result)
end

function TestResult(filename::ASCIIString, message)
    CriticalTestResult(filename, message)
end

global test_results = []

    
function get_test_functions(func)
    return Function[]
end

""" Return all functions from module with name starting test """
function get_test_functions(mod::Module)
    test_function_names = filter((k) -> startswith(string(k), "test_"), names(mod, true))
    if haskey(ENV, "JULIAFEM_TEST_SLOW")
        info("JULIAFEM_TEST_SLOW set, testing also tests that are taking a long time")
        slow_test_functions = filter((k) -> startswith(string(k), "slow_test_"), names(mod, true))
        append!(test_function_names, slow_test_functions)
    end
    test_function_expressions = map((k) -> :($mod.$k), test_function_names)
    test_functions = map(eval, test_function_expressions)
    return test_functions
end

""" Run tests from some file. """
function run_test(filename::ASCIIString, test_function=nothing)
    info("running tests from $filename")
    test_module = nothing
    try
        test_module = include(filename)
    catch error
        warn("Unable to include file $filename for testing.")
        err = Base.showerror(Base.STDOUT, error)
        push!(test_results, TestResult(filename, "Unable to include file: $error"))
        return
    end
    if isa(test_function, Void)
        test_functions = get_test_functions(test_module)
    else
        test_functions = [eval( :($test_module.$test_function) )]
    end
    if length(test_functions) == 0
        warn("Unable to get test functions for file $filename. Define test functions inside module, look for test_heat.jl for concrete example how to do that.")
        push!(test_results, TestResult(filename, "Unable to find test functions"))
        return
    end
    for test_function in test_functions
        run_test(test_function)
    end
end

""" Run single test set. """
function run_test(test_function::Function)

    allok = true

    function test_handler(r::Base.Test.Success)
        print(".")
        result = TestResult(test_function, r)
        push!(test_results, result)
    end 

    function test_handler(r::Base.Test.Failure)
        allok = false
        print(" [\x1b[31mFAIL\x1b[0m]")
        push!(test_results, TestResult(test_function, r))
        println()
        println("Test failed: $(r.expr)")
        #warn("partially evaluated expression: $(r.resultexpr)")
    end 

    function test_handler(r::Base.Test.Error)
        allok = false
        println(" [\x1b[31mERROR\x1b[0m]")
        push!(test_results, TestResult(test_function, r))
        println("Error when testing: $(r.expr)")
    end 

    Base.Test.with_handler(test_handler) do
        print("TEST: $test_function() ")
        # TODO: print docstring of test function if defined.
        #@doc($test_function)
        try
            test_function()
        catch error
            allok = false
            println("[\x1b[31mCRITICAL\x1b[0m]")
            println("Testing $test_function() stopped for critical error:\n$error")
            Base.showerror(Base.STDOUT, error)
            println()
            println("Cannot continue to test function $test_function()")
            push!(test_results, TestResult("$test_function", error))
        end

        if allok
            println(" [\x1b[32mPASS\x1b[0m]")
        else
            filename, linenum = Base.functionloc(test_function)
            println("Test $test_function() failed: file $filename, line $linenum.")
            println()
        end
    end

end

function print_test_statistics()
    info("################# TEST RESULTS #####################")
    passed = 0
    failed = 0
    errors = 0
    critical = 0
    for result in test_results
        if isa(result, NormalTestResult)
            if isa(result.result, Base.Test.Success)
                passed += 1
                continue
            elseif isa(result.result, Base.Test.Failure)
                failed += 1
                filename, linenum = Base.functionloc(result.test_function)
                functionname = Base.function_name(result.test_function)
                warn("test failed: $(result.result.expr), partially evaluated expression: $(result.result.expr)")
                warn("in function $functionname, file $filename, line $linenum")
                continue
            elseif isa(result.result, Base.Test.Error)
                errors += 1
                filename, linenum = Base.functionloc(result.test_function)
                functionname = Base.function_name(result.test_function)
                warn("error in function: $(result.result.expr)")
                warn("in function $functionname, file $filename, line $linenum")
                continue
            end
        elseif isa(result, CriticalTestResult)
            critical += 1
            warn("Critical error on $(result.filename): $(result.message)")
        end
    end
    info("$passed test passed, $failed test failed, $errors errors, $critical critical failures")
    return passed, failed, errors, critical
end

export @test, @testset, @test_throws, run_test, print_test_statistics