functional testing ...

This commit is contained in:
Jukka Aho
2015-07-05 17:02:17 +03:00
parent 2c662b1ec4
commit 69797247ee
4 changed files with 207 additions and 73 deletions
+53 -7
View File
@@ -1,5 +1,40 @@
# Automatically run notebooks and generate rst table for results
""" rst file parser to find title, author and factcheck status.
"""
function parse_rst(filename)
res = Dict("author" => "unknown", "status" => 0, "title" => "", "abstract" => "")
title_found = false
open(filename) do fid
data = readlines(fid)
for line in data
line = strip(line)
if line == ""
continue
end
if !title_found
res["title"] = line
title_found = true
end
if startswith(line, "Author(s):")
auth = split(line[12:end], ' ')
auth = filter(s -> !('@' in s), auth)
auth = join(auth, ' ')
res["author"] = auth
end
if startswith(line, "Abstract:") # not working, todo, fixme, ...
res["abstract"] = line[11:end]
end
if startswith(line, "Failed:")
res["status"] = 1
end
m = matchall(r"[-0-9.]+", line)
end
end
return res
end
function run_notebooks()
k = 0
results = Dict[]
@@ -21,10 +56,19 @@ function run_notebooks()
println("did not work")
end
runtime = toc()
run(`ipython nbconvert tutorials/$ipynb --to rst --output=tutorials/$(ipynb[1:end-6])`)
#println("took $runtime seconds")
bn = "tutorials/$(ipynb[1:end-6])"
run(`ipython nbconvert tutorials/$ipynb --to rst --output=$bn`)
run(`ipython nbconvert tutorials/$ipynb --to pdf --output=$bn`)
data = Dict("author" => "unknown", "status" => status, "runtime" => runtime,
"filename" => ipynb, "last_run" => time(), "description"=>"")
res = parse_rst("$bn.rst")
data["description"] = res["title"]
data["author"] = res["author"]
# there is two possibilities, either notbook does not run for syntax error (status = 1)
# or notebook is fine but factcheck returns failed. This checks the latter one.
if status == 0
data["status"] = res["status"]
end
push!(results, data)
end
end
@@ -35,7 +79,7 @@ end
Make rows from results ready to tabular form
"""
function makerows(results)
rows = ["id" "author" "description" "last run" "runtime (s)" "status" "html" "pdf"]
rows = ["id" "author" "description" "last run" "runtime (s)" "status" "ipynb" "pdf"]
for (i, data) in enumerate(results)
row = ["na" "unknown" "unknown" "unknown" "unknown" "unknown" "na" "na"]
println(i)
@@ -44,7 +88,9 @@ function makerows(results)
if desc == ""
desc = data["filename"]
end
row[3] = desc
row[2] = data["author"]
fn = data["filename"][1:end-6]
row[3] = ":doc:`$desc <$fn>`"
row[4] = Libc.strftime("%Y-%m-%d %H:%M:%S", data["last_run"])
row[5] = string(round(data["runtime"], 2))
if data["status"] == 0
@@ -52,10 +98,10 @@ function makerows(results)
else
row[6] = "FAIL"
end
fn = data["filename"][1:end-6]*".rst"
row[7] = ":doc:`html <$fn>`"
fn = data["filename"]
row[7] = ":download:`ipynb <$fn>`"
fn = data["filename"][1:end-6]*".pdf"
row[8] = ":download:`html <$fn>`"
row[8] = ":download:`pdf <$fn>`"
rows = vcat(rows, row)
end
return rows
@@ -0,0 +1,91 @@
{
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# First steps toward functional testing, part 1\n",
"\n",
"Author(s): Jukka Aho <jukka.aho@kapsi.fi>\n",
"\n",
"Abstract: This is demonstration document describing how to combine functional testing, continous integration and documentation in to the single document.\n",
"\n",
"These multi-purposal documents test several things at once. While they test that documentation is up-to-date, calculation results are what is expected and code base is working, they also serve as a tutorials and examples how to use JuliaFEM."
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Let's start. **Hello world**. *Local* time is here"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"Libc.strftime(time())"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Using FactCheck to validate your results"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"f(x) = 2*x\n",
"f(1)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"using FactCheck"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"facts(\"test that f(1) equals to 3, this is failing on purpose\") do\n",
" @fact f(1) => 3\n",
"end"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Julia 0.4.0-dev",
"language": "julia",
"name": "julia-0.4"
},
"language_info": {
"name": "julia",
"version": "0.4.0"
}
},
"nbformat": 4,
"nbformat_minor": 0
}
@@ -4,7 +4,20 @@
"cell_type": "markdown",
"metadata": {},
"source": [
"Hello world. Time is here"
"# First steps toward functional testing, part 2\n",
"\n",
"Author(s): Jukka Aho <jukka.aho@kapsi.fi>\n",
"\n",
"Abstract: This is demonstration document describing how to combine functional testing, continous integration and documentation in to the single document.\n",
"\n",
"These multi-purposal documents test several things at once. While they test that documentation is up-to-date, calculation results are what is expected and code base is working, they also serve as a tutorials and examples how to use JuliaFEM."
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Let's start. **Hello world**. *Local* time is here"
]
},
{
@@ -48,6 +61,55 @@
"@assert f(1) == 2"
]
},
{
"cell_type": "markdown",
"metadata": {
"collapsed": true
},
"source": [
"\\begin{equation}\n",
"A_{ij}=\\int_{0}^{L}\\phi_{i}\\phi_{j}\\,\\mathrm{d}x\n",
"\\end{equation}"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Use FactCheck to validate your results"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"We actually use FactCheck to test our results"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"using FactCheck"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"facts(\"test that f(1) equals to 2\") do\n",
" @fact f(1) => 2\n",
"end"
]
},
{
"cell_type": "code",
"execution_count": null,
@@ -1,65 +0,0 @@
{
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Hello world. Time is here"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"Libc.strftime(time())"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Basic testing"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"f(x) = 2*x\n",
"f(1)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"@assert f(1) == 3"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Julia 0.4.0-dev",
"language": "julia",
"name": "julia-0.4"
},
"language_info": {
"name": "julia",
"version": "0.4.0"
}
},
"nbformat": 4,
"nbformat_minor": 0
}