regression -report_stats
This commit is contained in:
James Cherry
parent
7f3b34c492
commit
2763f8873d
|
|
@ -25,8 +25,10 @@ proc regression_main {} {
|
|||
proc setup {} {
|
||||
global result_dir diff_file failure_file errors
|
||||
global use_valgrind valgrind_shared_lib_failure
|
||||
global report_stats
|
||||
|
||||
set use_valgrind 0
|
||||
set report_stats 0
|
||||
|
||||
if { !([file exists $result_dir] && [file isdirectory $result_dir]) } {
|
||||
file mkdir $result_dir
|
||||
|
|
@ -37,11 +39,6 @@ proc setup {} {
|
|||
set errors(error) 0
|
||||
set errors(memory) 0
|
||||
set errors(leak) 0
|
||||
set errors(race) 0
|
||||
set errors(slow) 0
|
||||
set errors(fast) 0
|
||||
set errors(big) 0
|
||||
set errors(small) 0
|
||||
set errors(fail) 0
|
||||
set errors(no_cmd) 0
|
||||
set errors(no_ok) 0
|
||||
|
|
@ -52,13 +49,15 @@ proc parse_args {} {
|
|||
global argv app_options tests test_groups cmd_paths
|
||||
global use_valgrind
|
||||
global result_dir tests
|
||||
global report_stats
|
||||
|
||||
while { $argv != {} } {
|
||||
set arg [lindex $argv 0]
|
||||
if { $arg == "help" || $arg == "-help" } {
|
||||
puts {Usage: regression [-help] [-threads threads] [-valgrind] tests...}
|
||||
puts {Usage: regression [-help] [-threads threads] [-valgrind] [-report_stats] tests...}
|
||||
puts " -threads max|integer - number of threads to use"
|
||||
puts " -valgrind - run valgrind (linux memory checker)"
|
||||
puts " -report_stats - report run time and memory"
|
||||
puts " Wildcarding for test names is supported (enclose in \"'s)"
|
||||
puts " Tests are: all, fast, med, slow, or a test group or test name"
|
||||
puts ""
|
||||
|
|
@ -76,6 +75,9 @@ proc parse_args {} {
|
|||
} elseif { $arg == "-valgrind" } {
|
||||
set use_valgrind 1
|
||||
set argv [lrange $argv 1 end]
|
||||
} elseif { $arg == "-report_stats" } {
|
||||
set report_stats 1
|
||||
set argv [lrange $argv 1 end]
|
||||
} elseif { [llength $argv] == 0 } {
|
||||
# Default is to run dist tests.
|
||||
set tests [group_tests dist]
|
||||
|
|
@ -121,7 +123,7 @@ proc run_tests {} {
|
|||
}
|
||||
|
||||
proc run_test { test } {
|
||||
global result_dir diff_file errors diff_options
|
||||
global result_dir diff_file errors diff_options report_stats
|
||||
|
||||
set cmd_file [test_cmd_file $test]
|
||||
if [file exists $cmd_file] {
|
||||
|
|
@ -162,26 +164,8 @@ proc run_test { test } {
|
|||
append_failure $test
|
||||
incr errors(leak)
|
||||
}
|
||||
if { [lsearch $test_errors "RACE"] != -1 } {
|
||||
append error_msg " *RACE*"
|
||||
append_failure $test
|
||||
incr errors(race)
|
||||
}
|
||||
if { [lsearch $test_errors "SLOW"] != -1 } {
|
||||
append error_msg " *SLOW*"
|
||||
incr errors(slow)
|
||||
}
|
||||
if { [lsearch $test_errors "FAST"] != -1 } {
|
||||
append error_msg " *FAST*"
|
||||
incr errors(fast)
|
||||
}
|
||||
if { [lsearch $test_errors "BIG"] != -1 } {
|
||||
append error_msg " *BIG*"
|
||||
incr errors(big)
|
||||
}
|
||||
if { [lsearch $test_errors "SMALL"] != -1 } {
|
||||
append error_msg " *SMALL*"
|
||||
incr errors(small)
|
||||
if { $report_stats } {
|
||||
append error_msg " [test_stats_summary $test]"
|
||||
}
|
||||
|
||||
if [file exists $ok_file] {
|
||||
|
|
@ -209,6 +193,39 @@ proc run_test { test } {
|
|||
}
|
||||
}
|
||||
|
||||
proc test_stats { test } {
|
||||
if { ![catch {open [test_stats_file $test] r} stream] } {
|
||||
gets $stream line1
|
||||
close $stream
|
||||
return $line1
|
||||
} else {
|
||||
return {}
|
||||
}
|
||||
}
|
||||
|
||||
proc test_stats_summary { test } {
|
||||
set stats [test_stats $test]
|
||||
set elapsed_time [lindex $stats 0]
|
||||
set user_time [lindex $stats 1]
|
||||
set memory [lindex $stats 2]
|
||||
if { [string is double $elapsed_time] } {
|
||||
set elapsed [format "%.1fe" $elapsed_time]
|
||||
} else {
|
||||
set elapsed "?"
|
||||
}
|
||||
if { [string is double $user_time] } {
|
||||
set user [format "%.1fu" $user_time]
|
||||
} else {
|
||||
set user "?"
|
||||
}
|
||||
if { [string is double $memory] } {
|
||||
set mem [format "%.0fmb" [expr $memory * 1e-6]]
|
||||
} else {
|
||||
set mem "?"
|
||||
}
|
||||
return "$elapsed $user $mem"
|
||||
}
|
||||
|
||||
proc append_failure { test } {
|
||||
global failure_file
|
||||
set fail_ch [open $failure_file "a"]
|
||||
|
|
@ -228,20 +245,27 @@ proc run_test_app { test cmd_file log_file } {
|
|||
|
||||
proc run_test_plain { test cmd_file log_file } {
|
||||
global app_path app_options result_dir errorCode
|
||||
|
||||
global report_stats
|
||||
|
||||
if { ![file exists $app_path] } {
|
||||
return "ERROR $app_path not found."
|
||||
} elseif { ![file executable $app_path] } {
|
||||
return "ERROR $app_path is not executable."
|
||||
} else {
|
||||
set save_dir [pwd]
|
||||
cd [file dirname $cmd_file]
|
||||
if { [catch [concat exec $app_path $app_options \
|
||||
[file tail $cmd_file] >& $log_file]] } {
|
||||
cd $save_dir
|
||||
set run_file [test_run_file $test]
|
||||
set run_stream [open $run_file "w"]
|
||||
puts $run_stream "cd [file dirname $cmd_file]"
|
||||
puts $run_stream "source [file tail $cmd_file]"
|
||||
if { $report_stats } {
|
||||
set stat_file [file normalize [test_stats_file $test]]
|
||||
puts $run_stream "sta::write_stats $stat_file"
|
||||
}
|
||||
close $run_stream
|
||||
|
||||
if { [catch [concat exec $app_path $app_options $run_file >& $log_file]] } {
|
||||
set signal [lindex $errorCode 2]
|
||||
set error [lindex $errorCode 3]
|
||||
# Errors strings are not consistent across platforms but signal
|
||||
# Error strings are not consistent across platforms but signal
|
||||
# names are.
|
||||
if { $signal == "SIGSEGV" } {
|
||||
# Save corefiles to regression results directory.
|
||||
|
|
@ -254,7 +278,6 @@ proc run_test_plain { test cmd_file log_file } {
|
|||
cleanse_logfile $test $log_file
|
||||
return "ERROR $error"
|
||||
}
|
||||
cd $save_dir
|
||||
cleanse_logfile $test $log_file
|
||||
return ""
|
||||
}
|
||||
|
|
@ -376,9 +399,6 @@ proc show_summary {} {
|
|||
if { $errors(memory) != 0 } {
|
||||
puts "Memory corruption in $errors(memory)/$test_count"
|
||||
}
|
||||
if { $errors(race) != 0 } {
|
||||
puts "Race errors in $errors(race)/$test_count"
|
||||
}
|
||||
if { $errors(no_ok) != 0 } {
|
||||
puts "No ok file for $errors(no_ok)/$test_count"
|
||||
}
|
||||
|
|
@ -402,8 +422,7 @@ proc found_errors {} {
|
|||
|
||||
return [expr $errors(error) != 0 || $errors(fail) != 0 \
|
||||
|| $errors(no_cmd) != 0 || $errors(no_ok) != 0 \
|
||||
|| $errors(memory) != 0 || $errors(leak) != 0 \
|
||||
|| $errors(race) != 0]
|
||||
|| $errors(memory) != 0 || $errors(leak) != 0]
|
||||
}
|
||||
|
||||
################################################################
|
||||
|
|
@ -471,6 +490,11 @@ proc test_log_file { test } {
|
|||
return [file join $result_dir "$test.log"]
|
||||
}
|
||||
|
||||
proc test_run_file { test } {
|
||||
global result_dir
|
||||
return [file join $result_dir $test.run]
|
||||
}
|
||||
|
||||
proc test_tmp_file { test } {
|
||||
global result_dir
|
||||
return [file join $result_dir $test.tmp]
|
||||
|
|
@ -486,6 +510,11 @@ proc test_valgrind_file { test } {
|
|||
return [file join $result_dir $test.valgrind]
|
||||
}
|
||||
|
||||
proc test_stats_file { test } {
|
||||
global result_dir
|
||||
return [file join $result_dir "$test.stats"]
|
||||
}
|
||||
|
||||
proc test_core_file { test } {
|
||||
global result_dir
|
||||
return [file join $result_dir $test.core]
|
||||
|
|
|
|||
Loading…
Reference in New Issue