From d38b1794d2c39d58e8e18fec3e693d0e26eb4bf0 Mon Sep 17 00:00:00 2001 From: Matthias Koefferlein Date: Sun, 12 Feb 2017 15:27:12 +0100 Subject: [PATCH] Tests with missing input data are skipped now --- src/unit_tests/dbBoxScanner.cc | 4 +- src/unit_tests/dbCIFReader.cc | 8 +- src/unit_tests/dbDXFReader.cc | 6 +- src/unit_tests/extGerberImport.cc | 6 +- src/unit_tests/extLEFDEFImport.cc | 14 ++- src/ut/utHead.h | 42 ++------ src/ut/utMain.cc | 167 +++++++++++++++++++++--------- 7 files changed, 142 insertions(+), 105 deletions(-) diff --git a/src/unit_tests/dbBoxScanner.cc b/src/unit_tests/dbBoxScanner.cc index 0753c47d2..66f607c96 100644 --- a/src/unit_tests/dbBoxScanner.cc +++ b/src/unit_tests/dbBoxScanner.cc @@ -678,8 +678,8 @@ struct BooleanAndInteractionClusterCollector TEST(100) { - std::string fn (ut::testsrc ()); - fn += "/testdata_private/other/"; + std::string fn (ut::testsrc_private ()); + fn += "/testdata/other/"; fn += "bs100.oas.gz"; db::Layout layout; diff --git a/src/unit_tests/dbCIFReader.cc b/src/unit_tests/dbCIFReader.cc index c372e9591..0f1caff5c 100644 --- a/src/unit_tests/dbCIFReader.cc +++ b/src/unit_tests/dbCIFReader.cc @@ -60,8 +60,8 @@ static void run_test (ut::TestBase *_this, const char *file, const char *file_au db::Layout layout (&m), layout2 (&m), layout2_cif (&m), layout_au (&m); { - std::string fn (ut::testsrc ()); - fn += "/testdata_private/cif/"; + std::string fn (ut::testsrc_private ()); + fn += "/testdata/cif/"; fn += file; tl::InputStream stream (fn); db::Reader reader (stream); @@ -120,8 +120,8 @@ static void run_test (ut::TestBase *_this, const char *file, const char *file_au } { - std::string fn (ut::testsrc ()); - fn += "/testdata_private/cif/"; + std::string fn (ut::testsrc_private ()); + fn += "/testdata/cif/"; fn += file_au; tl::InputStream stream (fn); db::Reader reader (stream); diff --git a/src/unit_tests/dbDXFReader.cc b/src/unit_tests/dbDXFReader.cc index 76a8a8ba9..17b3486f5 100644 --- a/src/unit_tests/dbDXFReader.cc +++ b/src/unit_tests/dbDXFReader.cc @@ -59,15 +59,15 @@ static void run_test (ut::TestBase *_this, const char *file, const char *file_au db::Layout layout; { - std::string fn (ut::testsrc ()); - fn += "/testdata_private/dxf/"; + std::string fn (ut::testsrc_private ()); + fn += "/testdata/dxf/"; fn += file; tl::InputStream stream (fn); db::Reader reader (stream); reader.read (layout, options); } - _this->compare_layouts (layout, ut::testsrc () + std::string ("/testdata_private/dxf/") + file_au); + _this->compare_layouts (layout, ut::testsrc_private () + std::string ("/testdata/dxf/") + file_au); } TEST(1a) diff --git a/src/unit_tests/extGerberImport.cc b/src/unit_tests/extGerberImport.cc index 119a597fa..7c4a03aef 100644 --- a/src/unit_tests/extGerberImport.cc +++ b/src/unit_tests/extGerberImport.cc @@ -38,8 +38,8 @@ static void run_test (ut::TestBase *_this, const char *dir) db::Layout layout; { - std::string fn (ut::testsrc ()); - fn += "/testdata_private/pcb/"; + std::string fn (ut::testsrc_private ()); + fn += "/testdata/pcb/"; fn += dir; fn += "/import.pcb"; tl::InputStream stream (fn); @@ -47,7 +47,7 @@ static void run_test (ut::TestBase *_this, const char *dir) reader.read (layout, options); } - _this->compare_layouts (layout, ut::testsrc () + "/testdata_private/pcb/" + dir + "/au.oas.gz", ut::WriteOAS); + _this->compare_layouts (layout, ut::testsrc_private () + "/testdata/pcb/" + dir + "/au.oas.gz", ut::WriteOAS); } TEST(1) diff --git a/src/unit_tests/extLEFDEFImport.cc b/src/unit_tests/extLEFDEFImport.cc index b940ee137..c4a41e60f 100644 --- a/src/unit_tests/extLEFDEFImport.cc +++ b/src/unit_tests/extLEFDEFImport.cc @@ -63,8 +63,8 @@ static void run_test (ut::TestBase *_this, const char *lef_dir, const char *file if (ex.test ("def:")) { - std::string fn (ut::testsrc ()); - fn += "/testdata_private/lefdef/"; + std::string fn (ut::testsrc_private ()); + fn += "/testdata/lefdef/"; fn += lef_dir; fn += "/"; std::string f; @@ -76,8 +76,8 @@ static void run_test (ut::TestBase *_this, const char *lef_dir, const char *file } else if (ex.test ("lef:")) { - std::string fn (ut::testsrc ()); - fn += "/testdata_private/lefdef/"; + std::string fn (ut::testsrc_private ()); + fn += "/testdata/lefdef/"; fn += lef_dir; fn += "/"; std::string f; @@ -120,8 +120,8 @@ static void run_test (ut::TestBase *_this, const char *lef_dir, const char *file if (au) { - std::string fn (ut::testsrc ()); - fn += "/testdata_private/lefdef/"; + std::string fn (ut::testsrc_private ()); + fn += "/testdata/lefdef/"; fn += lef_dir; fn += "/"; fn += au; @@ -213,5 +213,3 @@ TEST(16) { run_test (_this, "def7", "lef:cells.lef+lef:tech.lef+def:in.def.gz", "au.oas.gz"); } - - diff --git a/src/ut/utHead.h b/src/ut/utHead.h index 3752b8873..7e549b8f4 100644 --- a/src/ut/utHead.h +++ b/src/ut/utHead.h @@ -113,6 +113,14 @@ enum NormalizationMode */ UT_PUBLIC std::string testsrc (); +/** + * @brief Gets the path of the private test data + * This path is specified through the environment variable $TESTSRC and the + * private testdata directory. If no private test data is available, this + * method will throw a CancelException which makes the test skipped. + */ +UT_PUBLIC std::string testsrc_private (); + /** * @brief A basic exception for the unit test framework */ @@ -360,44 +368,11 @@ struct Registrar ms_instance->m_tests.push_back (t); } - static int do_tests (const std::vector *selected, const std::string &mode) - { - if (ms_instance) { - return ms_instance->run_tests (selected, mode); - } else { - return 0; - } - } - - int run_tests (const std::vector *selected, const std::string &mode) - { - int failed = 0; - m_failed.clear (); - if (! selected) { - selected = &m_tests; - } - for (std::vector ::const_iterator t = selected->begin (); t != selected->end (); ++t) { - (*t)->remove_tmp_folder (); - } - for (std::vector ::const_iterator t = selected->begin (); t != selected->end (); ++t) { - if (! (*t)->do_test (mode)) { - m_failed.push_back (*t); - ++failed; - } - } - return failed; - } - static Registrar *instance () { return ms_instance; } - const std::vector &failed_tests () const - { - return m_failed; - } - const std::vector &tests () const { return m_tests; @@ -409,7 +384,6 @@ private: Registrar () : m_tests () { } std::vector m_tests; - std::vector m_failed; }; /** diff --git a/src/ut/utMain.cc b/src/ut/utMain.cc index 40a583914..07116c744 100644 --- a/src/ut/utMain.cc +++ b/src/ut/utMain.cc @@ -96,6 +96,16 @@ std::string testsrc () return ts; } +std::string testsrc_private () +{ + QDir d (QDir (tl::to_qstring (ut::testsrc ())).filePath (QString::fromUtf8 ("private"))); + if (! d.exists ()) { + throw tl::CancelException (); + } + return tl::to_string (d.path ()); +} + + bool equals (double a, double b) { double m = fabs (0.5 * (a + b)); @@ -504,7 +514,7 @@ TestBase::TestBase (const std::string &file, const std::string &name) ut::Registrar::reg (this); } -bool TestBase::do_test (const std::string &mode) +bool TestBase::do_test (const std::string & /*mode*/) { // Ensures the test temp directory is present QDir dir (testtmp ()); @@ -526,57 +536,21 @@ bool TestBase::do_test (const std::string &mode) testtmp_value = std::string ("TESTTMP_WITH_NAME=") + m_testtmp.toUtf8().constData(); putenv (const_cast (testtmp_value.c_str ())); - ut::ctrl << ""; + reset_checkpoint (); - ut::noctrl << replicate ("-", TestConsole::instance ()->real_columns ()); - ut::noctrl << "Running " << m_test; + tl::Timer timer; + timer.start(); - try { + execute (this); - reset_checkpoint (); + timer.stop(); - tl::Timer timer; - timer.start(); + m_testtmp.clear (); - execute (this); + ut::noctrl << "Time: " << timer.sec_wall () << "s (wall) " << timer.sec_user () << "s (user) " << timer.sec_sys () << "s (sys)"; + ut::ctrl << ""; - timer.stop(); - - if (m_any_failed) { - - ut::ctrl << ""; - tl::error << "Test " << m_test << " failed (continued mode - see previous messages)"; - ut::ctrl << ""; - ut::ctrl << ""; - - m_testtmp.clear (); - return false; - - } else { - - ut::noctrl << "Time: " << timer.sec_wall () << "s (wall) " << timer.sec_user () << "s (user) " << timer.sec_sys () << "s (sys)"; - ut::ctrl << ""; - ut::ctrl << ""; - ut::ctrl << ""; - - m_testtmp.clear (); - return true; - - } - - } catch (tl::Exception &ex) { - - ut::ctrl << ""; - tl::error << "Test " << m_test << " failed:"; - tl::info << ex.msg (); - ut::ctrl << ""; - - ut::ctrl << ""; - - m_testtmp.clear (); - return false; - - } + return (!m_any_failed); } std::string TestBase::tmp_file (const std::string &fn) const @@ -603,7 +577,6 @@ static void empty_dir (QDir dir) } } - void TestBase::remove_tmp_folder () { // Ensures the test temp directory is present @@ -932,7 +905,7 @@ main_cont (int argc, char **argv) tl::info << "TESTSRC=" << ut::testsrc (); tl::info << "TESTTMP=" << tl::to_string (ut::testtmp ().absolutePath ()); - std::vector *selected_tests = 0; + const std::vector *selected_tests = 0; std::vector subset; if (! test_list.empty ()) { @@ -961,11 +934,15 @@ main_cont (int argc, char **argv) } + } else { + selected_tests = &ut::Registrar::instance()->tests (); } ut::s_verbose_flag = false; int failed_ne = 0, failed_e = 0; std::vector failed_tests_e, failed_tests_ne; + int skipped_ne = 0, skipped_e = 0; + std::vector skipped_tests_e, skipped_tests_ne; for (int e = 0; e < 2; ++e) { @@ -978,13 +955,72 @@ main_cont (int argc, char **argv) app.set_editable (e != 0); int failed = 0; + std::vector failed_tests; + int skipped = 0; + std::vector skipped_tests; tl::Timer timer; timer.start (); try { - failed = ut::Registrar::do_tests (selected_tests, mode); + + failed = 0; + failed_tests.clear (); + skipped = 0; + skipped_tests.clear (); + + for (std::vector ::const_iterator t = selected_tests->begin (); t != selected_tests->end (); ++t) { + (*t)->remove_tmp_folder (); + } + + for (std::vector ::const_iterator t = selected_tests->begin (); t != selected_tests->end (); ++t) { + + ut::ctrl << "name () << "\" mode=\"" << mode << "\">"; + + ut::noctrl << replicate ("-", TestConsole::instance ()->real_columns ()); + ut::noctrl << "Running " << (*t)->name (); + + try { + + if (! (*t)->do_test (mode)) { + + ut::ctrl << ""; + tl::error << "Test " << (*t)->name () << " failed (continued mode - see previous messages)"; + ut::ctrl << ""; + + failed_tests.push_back (*t); + ++failed; + + } else { + ut::ctrl << ""; + ut::ctrl << ""; + } + + } catch (tl::CancelException &) { + + ut::ctrl << ""; + tl::error << "Test " << (*t)->name () << " skipped"; + + skipped_tests.push_back (*t); + ++skipped; + + } catch (tl::Exception &ex) { + + ut::ctrl << ""; + tl::error << "Test " << (*t)->name () << " failed:"; + tl::info << ex.msg (); + ut::ctrl << ""; + + failed_tests.push_back (*t); + ++failed; + + } + + ut::ctrl << ""; + + } + } catch (tl::Exception &ex) { tl::error << "Caught tl::exception: " << ex.msg (); failed = 1; @@ -1003,12 +1039,23 @@ main_cont (int argc, char **argv) ut::noctrl << replicate ("=", console.real_columns ()); ut::noctrl << "Summary"; + if (skipped > 0) { + if (e == 0) { + skipped_tests_ne = skipped_tests; + skipped_ne = skipped; + } else { + skipped_tests_e = skipped_tests; + skipped_e = skipped; + } + tl::warn << skipped << " test(s) skipped"; + } + if (failed > 0) { if (e == 0) { - failed_tests_ne = ut::Registrar::instance ()->failed_tests (); + failed_tests_ne = failed_tests; failed_ne = failed; } else { - failed_tests_e = ut::Registrar::instance ()->failed_tests (); + failed_tests_e = failed_tests; failed_e = failed; } tl::warn << failed << " test(s) failed"; @@ -1074,6 +1121,22 @@ main_cont (int argc, char **argv) ut::ctrl << ""; + if (skipped_e + skipped_ne > 0) { + if (non_editable) { + tl::warn << "Skipped in non-editable mode"; + for (std::vector ::const_iterator f = skipped_tests_ne.begin (); f != skipped_tests_ne.end (); ++f) { + tl::warn << replicate (" ", console.indent ()) << (*f)->name (); + } + } + if (editable) { + tl::warn << "Skipped in editable mode"; + for (std::vector ::const_iterator f = skipped_tests_e.begin (); f != skipped_tests_e.end (); ++f) { + tl::warn << replicate (" ", console.indent ()) << (*f)->name (); + } + } + tl::warn << tl::to_string (result) << " test(s) skipped"; + } + result = failed_e + failed_ne; if (result > 0) { if (non_editable) { @@ -1121,5 +1184,7 @@ main_cont (int argc, char **argv) return result; } + + } // namespace ut