regression -j

Signed-off-by: James Cherry <cherry@parallaxsw.com>
This commit is contained in:
James Cherry 2026-03-21 10:18:55 -07:00
parent d90bf7d93b
commit bacb61feaa
1 changed files with 298 additions and 187 deletions

View File

@ -30,7 +30,8 @@
#
# This notice may not be removed or altered from any source distribution.
# regression -help | [-threads threads] [-valgrind] [-report_stats] test1 [test2...]
# Usage: regression -help | [-threads threads] [-j jobs] [-valgrind] [-report_stats]
# test1 [test2...]
proc regression_main {} {
setup
@ -41,12 +42,13 @@ proc regression_main {} {
}
proc setup {} {
global result_dir diff_file failure_file errors
global result_dir diff_file failure_file errors failed_tests
global use_valgrind valgrind_shared_lib_failure
global report_stats
global report_stats max_jobs app_path
set use_valgrind 0
set report_stats 0
set max_jobs 1
if { !([file exists $result_dir] && [file isdirectory $result_dir]) } {
file mkdir $result_dir
@ -60,20 +62,28 @@ proc setup {} {
set errors(fail) 0
set errors(no_cmd) 0
set errors(no_ok) 0
set failed_tests {}
set valgrind_shared_lib_failure 0
if { ![file exists $app_path] } {
error "$app_path not found."
} elseif { ![file executable $app_path] } {
error "$app_path is not executable."
}
}
proc parse_args {} {
global argv app_options tests test_groups cmd_paths
global use_valgrind
global result_dir tests
global report_stats
global report_stats max_jobs
while { $argv != {} } {
set arg [lindex $argv 0]
if { $arg == "help" || $arg == "-help" } {
puts {Usage: regression [-help] [-threads threads] [-valgrind] [-report_stats] tests...}
puts {Usage: regression [-help] [-threads threads] [-j jobs] [-valgrind] [-report_stats] tests...}
puts " -threads max|integer - number of threads to use"
puts " -j jobs - number of parallel jobs (processes) to run"
puts " -valgrind - run valgrind (linux memory checker)"
puts " -report_stats - report run time and memory"
puts " Wildcarding for test names is supported (enclose in \"'s)"
@ -84,12 +94,20 @@ proc parse_args {} {
} elseif { $arg == "-threads" } {
set threads [lindex $argv 1]
if { !([string is integer $threads] || $threads == "max") } {
puts "Error: -threads arg $threads is not an integer or max."
exit 0
puts "Error: -threads arg $threads is not an integer or max."
exit 0
}
lappend app_options "-threads"
lappend app_options $threads
set argv [lrange $argv 2 end]
} elseif { $arg == "-j" } {
set jobs [lindex $argv 1]
if { ![string is integer $jobs] || $jobs < 1 } {
puts "Error: -j arg $jobs must be a positive integer."
exit 0
}
set max_jobs $jobs
set argv [lrange $argv 2 end]
} elseif { $arg == "-valgrind" } {
if { ![find_valgrind] } {
error "valgrind not found."
@ -131,7 +149,7 @@ proc expand_tests { argv } {
if { [info exists test_groups($arg)] } {
set tests [concat $tests $test_groups($arg)]
} elseif { [string first "*" $arg] != -1 \
|| [string first "?" $arg] != -1 } {
|| [string first "?" $arg] != -1 } {
# Find wildcard matches.
foreach test [group_tests "all"] {
if [string match $arg $test] {
@ -143,200 +161,296 @@ proc expand_tests { argv } {
} else {
puts "Error: test $arg not found."
incr errors(no_cmd)
}
}
}
return $tests
}
proc run_tests {} {
global tests errors app_path
global tests errors app_path max_jobs
foreach test $tests {
run_test $test
if { $max_jobs > 1 } {
run_tests_parallel
} else {
foreach test $tests {
run_test $test
}
}
# Macos debug info generated by valgrind.
file delete -force "$app_path.dSYM"
write_failure_file
write_diff_file
}
proc run_test { test } {
global result_dir diff_file errors diff_options report_stats
global result_dir diff_file errors diff_options
set cmd_file [test_cmd_file $test]
if [file exists $cmd_file] {
set ok_file [test_ok_file $test]
puts -nonewline $test
flush stdout
set exit_code 0
if { [test_cmd_file_exists $test] } {
set cmd [make_cmd_file $test]
set log_file [test_log_file $test]
foreach file [glob -nocomplain [file join $result_dir $test.*]] {
file delete -force $file
}
puts -nonewline $test
flush stdout
set test_errors [run_test_app $test $cmd_file $log_file]
if { [lindex $test_errors 0] == "ERROR" } {
puts " *ERROR* [lrange $test_errors 1 end]"
append_failure $test
incr errors(error)
# For some reason seg faults aren't echoed in the log - add them.
if { [llength $test_errors] > 1 && [file exists $log_file] } {
set log_ch [open $log_file "a"]
puts $log_ch $test_errors
close $log_ch
}
# Report partial log diff anyway.
if [file exists $ok_file] {
catch [concat exec diff $diff_options $ok_file $log_file \
>> $diff_file]
}
} else {
set error_msg ""
if { [lsearch $test_errors "MEMORY"] != -1 } {
append error_msg " *MEMORY*"
append_failure $test
incr errors(memory)
}
if { [lsearch $test_errors "LEAK"] != -1 } {
append error_msg " *LEAK*"
append_failure $test
incr errors(leak)
}
if { $report_stats } {
append error_msg " [test_stats_summary $test]"
}
if [file exists $ok_file] {
# Filter dos '/r's from log file.
set tmp_file [file join $result_dir $test.tmp]
exec tr -d "\r" < $log_file > $tmp_file
file rename -force $tmp_file $log_file
if [catch [concat exec diff $diff_options $ok_file $log_file \
>> $diff_file]] {
puts " *FAIL*$error_msg"
append_failure $test
incr errors(fail)
} else {
puts " pass$error_msg"
}
if { [catch [concat "exec" "$cmd >& $log_file"] result result_options] } {
set details [dict get $result_options -errorcode]
set exit_signal [lindex $details 2]
if { $exit_signal == "SIGSEGV" } {
set exit_code 139
} else {
puts " *NO OK FILE*"
append_failure $test
incr errors(no_ok)
set exit_code 128
}
}
} else {
puts "$test *NO CMD FILE*"
incr errors(no_cmd)
}
puts " [test_status $test $exit_code]"
}
################################################################
# Parallel runs use one pipeline per test; close() yields the real exit status.
# (Non-blocking channels must be switched to blocking before close - see Tcl manual.)
proc regression_parallel_close_pipe { fh } {
fconfigure $fh -blocking 1
if { [catch {close $fh} err opts] } {
set ec [dict get $opts -errorcode]
if { [lindex $ec 0] == "CHILDSTATUS" } {
return [lindex $ec 2]
}
return 128
}
return 0
}
proc regression_pipe_readable { fh test } {
global reg_parallel_active reg_parallel_job_done
read $fh
if { [eof $fh] } {
fileevent $fh readable {}
set exit_code [regression_parallel_close_pipe $fh]
puts "$test [test_status $test $exit_code]"
incr reg_parallel_active -1
incr reg_parallel_job_done
}
}
proc test_stats { test } {
if { ![catch {open [test_stats_file $test] r} stream] } {
gets $stream line1
close $stream
return $line1
} else {
return {}
proc open_test_pipeline { test } {
set cmd [make_cmd_file $test]
set log [test_log_file $test]
set inner [format {%s > %s 2>&1} $cmd [file nativename $log]]
set fh [open [format {|/bin/sh -c %s} [list $inner]] r]
fconfigure $fh -blocking 0
return $fh
}
proc run_tests_parallel {} {
global tests max_jobs reg_parallel_active reg_parallel_job_done
set reg_parallel_active 0
set reg_parallel_job_done 0
set test_idx 0
set test_count [llength $tests]
while { $test_idx < $test_count || $reg_parallel_active > 0 } {
while { $reg_parallel_active < $max_jobs && $test_idx < $test_count } {
set test [lindex $tests $test_idx]
incr test_idx
if { ![test_cmd_file_exists $test] } {
puts -nonewline $test
flush stdout
puts " [test_status $test 0]"
continue
}
set fh [open_test_pipeline $test]
fileevent $fh readable [list regression_pipe_readable $fh $test]
incr reg_parallel_active
}
if { $reg_parallel_active > 0 } {
set before $reg_parallel_job_done
while { $reg_parallel_job_done == $before } {
vwait reg_parallel_job_done
}
}
}
}
proc make_cmd_file { test } {
global app_path app_options result_dir use_valgrind report_stats
foreach file [glob -nocomplain [file join $result_dir $test.*]] {
file delete -force $file
}
set cmd_file [test_cmd_file $test]
set ok_file [test_ok_file $test]
set log_file [test_log_file $test]
set run_file [test_run_file $test]
set run_stream [open $run_file "w"]
puts $run_stream "cd [file dirname $cmd_file]"
puts $run_stream "include [file tail $cmd_file]"
if { $use_valgrind } {
puts $run_stream "sta::delete_all_memory"
}
if { $report_stats } {
puts $run_stream "sta::write_stats [test_stats_file $test]"
}
close $run_stream
if { $use_valgrind } {
global valgrind_options
set cmd "valgrind $valgrind_options $app_path $app_options $run_file"
} else {
set cmd "$app_path $app_options $run_file"
}
return $cmd
}
proc test_cmd_file_exists { test } {
set cmd_file [test_cmd_file $test]
return [file exists $cmd_file]
}
proc test_status { test exit_code } {
global result_dir diff_options errors
global use_valgrind report_stats test_status
set test_status {}
if { ![test_cmd_file_exists $test] } {
test_failed $test "no_cmd"
} else {
set log_file [test_log_file $test]
if { [file exists $log_file] } {
# Check log file for error patterns
set log_ch [open $log_file "r"]
set log_content [read $log_ch]
close $log_ch
# Check if exit code indicates a segfault or signal termination
# Exit codes >= 128 typically indicate termination by a signal
# 139 = 128 + 11 (SIGSEGV), 134 = 128 + 6 (SIGABRT), etc.
if { $exit_code >= 128
|| [string match "*Segmentation fault*" $log_content] \
|| [string match "*DEADLYSIGNAL*" $log_content] \
|| [string match "*Abort*" $log_content] \
|| [string match "*Fatal*" $log_content] } {
test_failed $test "error"
} elseif { [string match "*heap-use-after-free*" $log_content] } {
# ASAN error
test_failed $test "memory"
}
set ok_file [test_ok_file $test]
if { [file exists $ok_file] } {
if { $use_valgrind } {
cleanse_valgrind_logfile $test
}
if { [catch [concat exec diff $diff_options $ok_file $log_file]] } {
if { $test_status == "" } {
test_failed $test "fail"
}
}
} else {
if { $test_status == "" } {
test_failed $test "no_ok"
}
}
} else {
# Log file doesn't exist, likely an error
test_failed $test "error" "*ERROR* no log file"
}
}
if { $test_status == {} } {
append test_status "pass"
}
if { $report_stats } {
append test_status " [test_stats_summary $test]"
}
return $test_status
}
proc test_exit_code { test } {
# Read exit code
set test_error ""
set exit_code_file [test_exit_code_file $test]
if { [file exists $exit_code_file] } {
set exit_code_ch [open $exit_code_file "r"]
set exit_code [string trim [read $exit_code_ch]]
close $exit_code_ch
if { [string is integer $exit_code] } {
return $exit_code
}
}
return 0
}
proc test_stats_summary { test } {
set stats [test_stats $test]
set elapsed_time [lindex $stats 0]
set user_time [lindex $stats 1]
set memory [lindex $stats 2]
if { [string is double $elapsed_time] } {
set elapsed [format "%.1fe" $elapsed_time]
} else {
set elapsed "?"
}
if { [string is double $user_time] } {
set user [format "%.1fu" $user_time]
} else {
set user "?"
}
if { [string is double $memory] } {
set mem [format "%.0fmb" [expr $memory * 1e-6]]
} else {
set mem "?"
}
return "$elapsed $user $mem"
}
if { ![catch {open [test_stats_file $test] r} stream] } {
gets $stream stats
close $stream
proc append_failure { test } {
global failure_file
set fail_ch [open $failure_file "a"]
puts $fail_ch $test
close $fail_ch
}
# Return error.
proc run_test_app { test cmd_file log_file } {
global app_path errorCode use_valgrind
if { $use_valgrind } {
return [run_test_valgrind $test $cmd_file $log_file]
} else {
return [run_test_plain $test $cmd_file $log_file]
}
}
proc run_test_plain { test cmd_file log_file } {
global app_path app_options result_dir errorCode
global report_stats
if { ![file exists $app_path] } {
return "ERROR $app_path not found."
} elseif { ![file executable $app_path] } {
return "ERROR $app_path is not executable."
} else {
set run_file [test_run_file $test]
set run_stream [open $run_file "w"]
puts $run_stream "cd [file dirname $cmd_file]"
puts $run_stream "include [file tail $cmd_file]"
if { $report_stats } {
set stat_file [file normalize [test_stats_file $test]]
puts $run_stream "sta::write_stats $stat_file"
set elapsed_time [lindex $stats 0]
set user_time [lindex $stats 1]
set memory [lindex $stats 2]
if { [string is double $elapsed_time] } {
set elapsed [format "%.1fe" $elapsed_time]
} else {
set elapsed "?"
}
close $run_stream
if { [catch [concat exec $app_path $app_options $run_file >& $log_file]] } {
set signal [lindex $errorCode 2]
set error [lindex $errorCode 3]
# Error strings are not consistent across platforms but signal
# names are.
if { $signal == "SIGSEGV" } {
# Save corefiles to regression results directory.
set pid [lindex $errorCode 1]
set sys_corefile [test_sys_core_file $test $pid]
if { [file exists $sys_corefile] } {
file copy $sys_corefile [test_core_file $test]
}
}
return "ERROR $error"
if { [string is double $user_time] } {
set user [format "%.1fu" $user_time]
} else {
set user "?"
}
file delete $run_file
if { [string is double $memory] } {
set mem [format "%.0fmb" [expr $memory * 1e-6]]
} else {
set mem "?"
}
return "$elapsed $user $mem"
} else {
return ""
}
}
proc run_test_valgrind { test cmd_file log_file } {
global app_path app_options valgrind_options result_dir errorCode
proc test_failed { test reason } {
global errors test_status failed_tests
set vg_cmd_file [test_valgrind_cmd_file $test]
set vg_stream [open $vg_cmd_file "w"]
puts $vg_stream "cd [file dirname $cmd_file]"
puts $vg_stream "include [file tail $cmd_file]"
puts $vg_stream "sta::delete_all_memory"
close $vg_stream
set cmd [concat exec valgrind $valgrind_options \
$app_path $app_options $vg_cmd_file >& $log_file]
set error_msg ""
if { [catch $cmd] } {
set error_msg "ERROR [lindex $errorCode 3]"
if { $reason == "error" } {
set test_status "*ERROR*"
} elseif { $reason == "no_cmd" } {
set test_status "*NO CMD FILE*"
} elseif { $reason == "memory" } {
set test_status "*MEMORY*"
} elseif { $reason == "leak" } {
set test_status "*LEAK*"
} elseif { $reason == "fail" } {
set test_status "*FAIL*"
} elseif { $reason == "no_ok" } {
set test_status "*NO OK FILE*"
} else {
error "unknown test failure reason $reason"
}
lappend failed_tests $test
incr errors($reason)
}
proc write_failure_file {} {
global failure_file failed_tests
set ch [open $failure_file "w"]
foreach test $failed_tests {
puts $ch $test
}
close $ch
}
proc write_diff_file {} {
global diff_file diff_options failed_tests
foreach test $failed_tests {
set log_file [test_log_file $test]
set ok_file [test_ok_file $test]
catch [concat exec diff $diff_options $ok_file $log_file >> $diff_file]
}
file delete $vg_cmd_file
set error_msg [concat $error_msg [cleanse_valgrind_logfile $test $log_file]]
return $error_msg
}
# Error messages can be found in "valgrind/memcheck/mc_errcontext.c".
@ -363,25 +477,26 @@ set valgrind_shared_lib_failure_regexp "No malloc'd blocks -- no leaks are possi
# Scan the log file to separate valgrind notifications and check for
# valgrind errors.
proc cleanse_valgrind_logfile { test log_file } {
proc cleanse_valgrind_logfile { test } {
global valgrind_mem_regexp valgrind_leak_regexp
global valgrind_shared_lib_failure_regexp
global valgrind_shared_lib_failure
global valgrind_shared_lib_failure error
set log_file [test_log_file $test]
set tmp_file [test_tmp_file $test]
set valgrind_log_file [test_valgrind_file $test]
file copy -force $log_file $tmp_file
set tmp [open $tmp_file "r"]
set log [open $log_file "w"]
set valgrind [open $valgrind_log_file "w"]
set leaks 0
set leak 0
set mem_errors 0
gets $tmp line
while { ![eof $tmp] } {
if {[regexp "^==" $line]} {
puts $valgrind $line
if {[regexp $valgrind_leak_regexp $line]} {
set leaks 1
set leak 1
}
if {[regexp $valgrind_mem_regexp $line]} {
set mem_errors 1
@ -399,16 +514,12 @@ proc cleanse_valgrind_logfile { test log_file } {
close $log
close $tmp
close $valgrind
file delete $tmp_file
set errors {}
if { $mem_errors } {
lappend errors "MEMORY"
test_failed $test "memory"
} elseif { $leak } {
test_failed $test "leak"
}
if { $leaks } {
lappend errors "LEAK"
}
return $errors
}
################################################################
@ -533,11 +644,6 @@ proc test_tmp_file { test } {
return [file join $result_dir $test.tmp]
}
proc test_valgrind_cmd_file { test } {
global result_dir
return [file join $result_dir $test.vg_cmd]
}
proc test_valgrind_file { test } {
global result_dir
return [file join $result_dir $test.valgrind]
@ -563,6 +669,11 @@ proc test_sys_core_file { test pid } {
return [file join [test_cmd_dir $test] "core"]
}
proc test_exit_code_file { test } {
global result_dir
return [file join $result_dir "$test.exitcode"]
}
################################################################
# Local Variables: