From 083fb7e9c23772a8135c212b99f6fdf8ae954b61 Mon Sep 17 00:00:00 2001 From: Andrew Nolte Date: Fri, 6 Sep 2024 18:15:18 -0400 Subject: [PATCH] Add partial coverage symbol and branch data in lcov info files (#5388) --- docs/guide/exe_verilator_coverage.rst | 51 +++++++++++----------- src/VlcOptions.h | 1 + src/VlcSource.h | 26 +++++------ src/VlcTop.cpp | 41 +++++++++++++---- test_regress/t/t_cover_line.out | 20 ++++----- test_regress/t/t_cover_line_cc.info.out | 44 +++++++++++++++++++ test_regress/t/t_cover_line_cc.pl | 1 + test_regress/t/t_cover_toggle.out | 4 +- test_regress/t/t_cover_toggle_min.info.out | 8 ++++ test_regress/t/t_cover_toggle_points.out | 4 +- test_regress/t/t_vlcov_info.info.out | 9 ++++ 11 files changed, 148 insertions(+), 61 deletions(-) diff --git a/docs/guide/exe_verilator_coverage.rst b/docs/guide/exe_verilator_coverage.rst index 37dbc977a..142531cd7 100644 --- a/docs/guide/exe_verilator_coverage.rst +++ b/docs/guide/exe_verilator_coverage.rst @@ -58,32 +58,36 @@ to read multiple inputs. If no data file is specified, by default, Specifies the directory name to which source files with annotated coverage data should be written. -Converting from the Verilator coverage data format to the info format is -lossy; the info will have all forms of coverage merged line coverage, and -if there are multiple coverage points on a single line they will merge. -The minimum coverage across all merged points will be used to report -coverage of the line. +Points are children of each line coverage- branches or toggle points. +When point counts are aggregated into a line, the minimum and maximum counts +are used to determine the status of the line (complete, partial, failing) +The count is equal to the minimum of the points. Coverage data is annotated at the beginning of the line and is formatted as a special character followed by the number of coverage hits. The special -characters " ,%,+,-" indicate summary of the coverage, and allow use of grep +characters " ,%,~,+,-" indicate summary of the coverage, and allow use of grep to filter the report. -* " " (whitespace) indicates that all points on the line are above the coverage limit. -* "%" indicates at least one point on the line was below the coverage limit. -* "+" coverage point was at or above the limit. Only used with :option:`--annotate-points`. -* "-" coverage point was below the limit. Only used with :option:`--annotate-points`. +* " " (whitespace) indicates that all points on the line are above the coverage min. +* "%" indicates that all points on the line are below the coverage min. +* "~" indicates that some points on the line are above the coverage min and some are below. +* "+" coverage point was at or above the min. Only used with :option:`--annotate-points`. +* "-" coverage point was below the min. Only used with :option:`--annotate-points`. .. code-block:: - 100000 input logic a; // Begins with whitespace, because - // number of hits (100000) is above the limit. - +100000 point: comment=a // Begins with +, because - // number of hits (100000) is above the limit. - %000000 input logic b; // Begins with %, because - // number of hits (0) is below the limit. - -000000 point: comment=b // Begins with -, because - // number of hits (0) is below the limit. + 100000 input logic a; // Begins with whitespace, because + // number of hits (100000) is above the min. + +100000 point: comment=a // Begins with +, because + // number of hits (100000) is above the min. + %000000 input logic b; // Begins with %, because + // number of hits (0) is below the min. + -000000 point: comment=b // Begins with -, because + // number of hits (0) is below the min. + ~000000 if (cyc!=0) begin // Begins with ~, because + // branches are below and above the min. + +000010 point: comment=if // The if branch is above the min. + -000000 point: comment=else // The else branch is below the min. .. option:: --annotate-all @@ -154,10 +158,7 @@ generated from random test runs) into one master coverage file. Specifies the aggregate coverage results, summed across all the files, should be written to the given filename in :command:`lcov` .info format. This may be used to feed into :command:`lcov` to aggregate or generate -reports. - -Converting from the Verilator coverage data format to the info format is -lossy; the info will have all forms of coverage merged line coverage, and -if there are multiple coverage points on a single line they will merge. -The minimum coverage across all merged points will be used to report -coverage of the line. +reports. This format lacks the comments for cover points that the +verilator_coverage format has. It can be used with :command:`genhtml` +to generate an HTML report. :command:`genhtml --branch-coverage` will +also display the branch coverage, analogous to :option:`--annotate-points` diff --git a/src/VlcOptions.h b/src/VlcOptions.h index 85cc3d61a..02671951d 100644 --- a/src/VlcOptions.h +++ b/src/VlcOptions.h @@ -63,6 +63,7 @@ public: string annotateOut() const { return m_annotateOut; } bool annotateAll() const { return m_annotateAll; } int annotateMin() const { return m_annotateMin; } + bool countOk(uint64_t count) const { return count >= static_cast(m_annotateMin); } bool annotatePoints() const { return m_annotatePoints; } bool rank() const { return m_rank; } bool unlink() const { return m_unlink; } diff --git a/src/VlcSource.h b/src/VlcSource.h index 3d6b55e9b..35a99a065 100644 --- a/src/VlcSource.h +++ b/src/VlcSource.h @@ -20,6 +20,8 @@ #include "config_build.h" #include "verilatedos.h" +#include "VlcPoint.h" + #include #include #include @@ -37,8 +39,8 @@ class VlcSourceCount final { // MEMBERS const int m_lineno; ///< Line number - uint64_t m_count = std::numeric_limits::max(); ///< Count - bool m_ok = false; ///< Coverage is above threshold + uint64_t m_maxCount = 0; ///< Max count + uint64_t m_minCount = std::numeric_limits::max(); ///< Min count PointsSet m_points; // Points on this line public: @@ -49,19 +51,16 @@ public: // ACCESSORS int lineno() const { return m_lineno; } - uint64_t count() const { return m_count; } - bool ok() const { return m_ok; } + uint64_t maxCount() const { return m_maxCount; } + uint64_t minCount() const { return m_minCount; } // METHODS - void incCount(uint64_t count, bool ok) { - if (m_count == std::numeric_limits::max()) - m_ok = ok; - else - m_ok = m_ok && ok; - m_count = std::min(m_count, count); + void insertPoint(const VlcPoint* pointp) { + m_maxCount = std::max(m_maxCount, pointp->count()); + m_minCount = std::min(m_minCount, pointp->count()); + m_points.emplace(pointp); } - void insertPoint(const VlcPoint* pointp) { m_points.emplace(pointp); } - PointsSet& points() { return m_points; } + const PointsSet& points() { return m_points; } }; //******************************************************************** @@ -91,9 +90,8 @@ public: LinenoMap& lines() { return m_lines; } // METHODS - void lineIncCount(int lineno, uint64_t count, bool ok, const VlcPoint* pointp) { + void insertPoint(int lineno, const VlcPoint* pointp) { VlcSourceCount& sc = m_lines.emplace(lineno, lineno).first->second; - sc.incCount(count, ok); sc.insertPoint(pointp); } }; diff --git a/src/VlcTop.cpp b/src/VlcTop.cpp index d936e5cd5..a7927a332 100644 --- a/src/VlcTop.cpp +++ b/src/VlcTop.cpp @@ -116,10 +116,28 @@ void VlcTop::writeInfo(const string& filename) { VlcSource& source = si.second; os << "SF:" << source.name() << '\n'; VlcSource::LinenoMap& lines = source.lines(); + int branchesFound = 0; + int branchesHit = 0; for (auto& li : lines) { - const VlcSourceCount& sc = li.second; - os << "DA:" << sc.lineno() << "," << sc.count() << "\n"; + VlcSourceCount& sc = li.second; + os << "DA:" << sc.lineno() << "," << sc.minCount() << "\n"; + int num_branches = sc.points().size(); + if (num_branches == 1) continue; + branchesFound += num_branches; + int point_num = 0; + for (const auto& point : sc.points()) { + os << "BRDA:" << sc.lineno() << ","; + os << "0,"; + os << point_num << ","; + os << point->count() << "\n"; + + branchesHit += opt.countOk(point->count()); + ++point_num; + } } + os << "BRF:" << branchesFound << '\n'; + os << "BRH:" << branchesHit << '\n'; + os << "end_of_record\n"; } } @@ -195,11 +213,10 @@ void VlcTop::annotateCalc() { const int lineno = point.lineno(); if (!filename.empty() && lineno != 0) { VlcSource& source = sources().findNewSource(filename); - const bool ok = point.ok(opt.annotateMin()); UINFO(9, "AnnoCalc count " << filename << ":" << lineno << ":" << point.column() << " " << point.count() << " " << point.linescov() << '\n'); // Base coverage - source.lineIncCount(lineno, point.count(), ok, &point); + source.insertPoint(lineno, &point); // Additional lines covered by this statement bool range = false; int start = 0; @@ -208,7 +225,7 @@ void VlcTop::annotateCalc() { for (const char* covp = linescov.c_str(); true; ++covp) { if (!*covp || *covp == ',') { // Ending for (int lni = start; start && lni <= end; ++lni) { - source.lineIncCount(lni, point.count(), ok, &point); + source.insertPoint(lni, &point); } if (!*covp) break; start = 0; // Prep for next @@ -242,7 +259,7 @@ void VlcTop::annotateCalcNeeded() { VlcSourceCount& sc = li.second; // UINFO(0, "Source "<second; // UINFO(0,"Source // "<dumpAnnotate(os, opt.annotateMin()); diff --git a/test_regress/t/t_cover_line.out b/test_regress/t/t_cover_line.out index 4efcf9b6a..21c0e9b24 100644 --- a/test_regress/t/t_cover_line.out +++ b/test_regress/t/t_cover_line.out @@ -49,7 +49,7 @@ 000010 always @ (posedge clk) begin +000010 point: comment=block -%000000 if (cyc!=0) begin +~000000 if (cyc!=0) begin +000010 point: comment=if -000000 point: comment=else 000010 cyc <= cyc + 1; @@ -138,10 +138,10 @@ end %000000 do ; while (0); -000000 point: comment=block -%000000 do begin +~000000 do begin +000010 point: comment=if -000000 point: comment=block -%000000 $write(""); +~000000 $write(""); +000010 point: comment=if -000000 point: comment=block %000000 end while (0); @@ -190,7 +190,7 @@ input toggle; 000020 always @ (posedge clk) begin +000020 point: comment=block -%000002 if (toggle) begin // CHECK_COVER(0,"top.t.a*",2) +~000002 if (toggle) begin // CHECK_COVER(0,"top.t.a*",2) -000002 point: comment=if +000018 point: comment=else %000002 $write(""); @@ -221,14 +221,14 @@ +000020 point: comment=block 000020 $write(""); // Always covered +000020 point: comment=block -%000000 if (0) begin // CHECK_COVER(0,"top.t.b*",0) +~000000 if (0) begin // CHECK_COVER(0,"top.t.b*",0) -000000 point: comment=if +000020 point: comment=else // Make sure that we don't optimize away zero buckets %000000 $write(""); -000000 point: comment=if end -%000002 if (toggle) begin // CHECK_COVER(0,"top.t.b*",2) +~000002 if (toggle) begin // CHECK_COVER(0,"top.t.b*",2) -000002 point: comment=if +000018 point: comment=else // t.b1 and t.b2 collapse to a count of 2 @@ -263,7 +263,7 @@ endfunction 000011 static function void fstatic(bit toggle); +000011 point: comment=block -%000000 if (1) begin // CHECK_COVER(0,"top.$unit::Cls",1) +~000000 if (1) begin // CHECK_COVER(0,"top.$unit::Cls",1) +000011 point: comment=if -000000 point: comment=else 000011 $write(""); @@ -272,7 +272,7 @@ endfunction 000011 function void fauto(); +000011 point: comment=block -%000000 if (m_toggle) begin // CHECK_COVER(0,"top.$unit::Cls",1) +~000000 if (m_toggle) begin // CHECK_COVER(0,"top.$unit::Cls",1) +000011 point: comment=if -000000 point: comment=else 000011 $write(""); @@ -301,13 +301,13 @@ input external; 000011 begin +000011 point: comment=block -%000001 if (toggle) begin // CHECK_COVER(0,"top.t.t1",1) +~000001 if (toggle) begin // CHECK_COVER(0,"top.t.t1",1) -000001 point: comment=if +000010 point: comment=else %000001 $write(""); -000001 point: comment=if end -%000001 if (external) begin // CHECK_COVER(0,"top.t.t1",1) +~000001 if (external) begin // CHECK_COVER(0,"top.t.t1",1) -000001 point: comment=if +000010 point: comment=else %000001 $write("[%0t] Got external pulse\n", $time); diff --git a/test_regress/t/t_cover_line_cc.info.out b/test_regress/t/t_cover_line_cc.info.out index d02a0b274..23d6cad0f 100644 --- a/test_regress/t/t_cover_line_cc.info.out +++ b/test_regress/t/t_cover_line_cc.info.out @@ -4,18 +4,32 @@ DA:15,1 DA:18,1 DA:47,10 DA:48,0 +BRDA:48,0,0,10 +BRDA:48,0,1,0 DA:49,10 DA:50,10 DA:52,1 +BRDA:52,0,0,1 +BRDA:52,0,1,9 DA:53,1 +BRDA:53,0,0,1 +BRDA:53,0,1,9 DA:54,1 DA:55,1 DA:58,1 +BRDA:58,0,0,1 +BRDA:58,0,1,9 DA:59,1 +BRDA:59,0,0,1 +BRDA:59,0,1,9 DA:61,9 DA:62,9 DA:65,1 +BRDA:65,0,0,1 +BRDA:65,0,1,9 DA:66,1 +BRDA:66,0,0,1 +BRDA:66,0,1,9 DA:67,1 DA:68,1 DA:71,9 @@ -27,6 +41,8 @@ DA:79,1 DA:80,1 DA:81,1 DA:83,1 +BRDA:83,0,0,1 +BRDA:83,0,1,7 DA:84,1 DA:85,1 DA:88,7 @@ -36,45 +52,67 @@ DA:93,0 DA:94,0 DA:96,0 DA:97,0 +BRDA:97,0,0,10 +BRDA:97,0,1,0 DA:98,0 +BRDA:98,0,0,10 +BRDA:98,0,1,0 DA:99,0 DA:102,1 DA:103,1 DA:105,1 DA:107,1 DA:112,1 +BRDA:112,0,0,1 +BRDA:112,0,1,7 DA:113,1 DA:114,1 DA:119,1 DA:121,1 DA:132,20 DA:133,2 +BRDA:133,0,0,2 +BRDA:133,0,1,18 DA:134,2 DA:137,18 DA:156,20 DA:157,20 DA:158,0 +BRDA:158,0,0,0 +BRDA:158,0,1,20 DA:160,0 DA:162,2 +BRDA:162,0,0,2 +BRDA:162,0,1,18 DA:164,2 DA:166,18 DA:180,1 DA:181,1 DA:182,0 +BRDA:182,0,0,1 +BRDA:182,0,1,0 DA:183,1 DA:186,11 DA:187,0 +BRDA:187,0,0,11 +BRDA:187,0,1,0 DA:188,11 DA:191,11 DA:192,0 +BRDA:192,0,0,11 +BRDA:192,0,1,0 DA:193,11 DA:207,10 DA:208,10 DA:211,11 DA:213,11 DA:214,1 +BRDA:214,0,0,1 +BRDA:214,0,1,10 DA:215,1 DA:217,1 +BRDA:217,0,0,1 +BRDA:217,0,1,10 DA:218,1 DA:221,11 DA:222,1 @@ -82,6 +120,12 @@ DA:223,11 DA:224,11 DA:245,10 DA:246,1 +BRDA:246,0,0,1 +BRDA:246,0,1,9 DA:248,1 DA:249,0 +BRDA:249,0,0,0 +BRDA:249,0,1,1 +BRF:42 +BRH:10 end_of_record diff --git a/test_regress/t/t_cover_line_cc.pl b/test_regress/t/t_cover_line_cc.pl index 089793bb7..c287e550c 100755 --- a/test_regress/t/t_cover_line_cc.pl +++ b/test_regress/t/t_cover_line_cc.pl @@ -50,6 +50,7 @@ if (`lcov --version` !~ /version/i } else { run(cmd => ["genhtml", "$Self->{obj_dir}/coverage.info", + "--branch-coverage", "--output-directory $Self->{obj_dir}/html", ]); } diff --git a/test_regress/t/t_cover_toggle.out b/test_regress/t/t_cover_toggle.out index c5ede53c4..a6c3482ed 100644 --- a/test_regress/t/t_cover_toggle.out +++ b/test_regress/t/t_cover_toggle.out @@ -37,7 +37,7 @@ %000000 reg [1:0][1:0] ptoggle; initial ptoggle=0; integer cyc; initial cyc=1; -%000000 wire [7:0] cyc_copy = cyc[7:0]; +~000000 wire [7:0] cyc_copy = cyc[7:0]; %000002 wire toggle_up; typedef struct { @@ -116,7 +116,7 @@ // CHECK_COVER(-1,"top.t.a*",4) // 2 edges * (t.a1 and t.a2) -%000000 input [7:0] cyc_copy; +~000000 input [7:0] cyc_copy; // CHECK_COVER(-1,"top.t.a*","cyc_copy[0]",22) // CHECK_COVER(-2,"top.t.a*","cyc_copy[1]",10) // CHECK_COVER(-3,"top.t.a*","cyc_copy[2]",4) diff --git a/test_regress/t/t_cover_toggle_min.info.out b/test_regress/t/t_cover_toggle_min.info.out index 7608c5284..5b96fc438 100644 --- a/test_regress/t/t_cover_toggle_min.info.out +++ b/test_regress/t/t_cover_toggle_min.info.out @@ -1,6 +1,14 @@ TN:verilator_coverage SF:t/t_cover_toggle_min.v DA:10,0 +BRDA:10,0,0,1 +BRDA:10,0,1,0 DA:11,0 +BRDA:11,0,0,0 +BRDA:11,0,1,1 DA:12,1 +BRDA:12,0,0,2 +BRDA:12,0,1,1 +BRF:6 +BRH:0 end_of_record diff --git a/test_regress/t/t_cover_toggle_points.out b/test_regress/t/t_cover_toggle_points.out index 6593bf99b..5267c11d9 100644 --- a/test_regress/t/t_cover_toggle_points.out +++ b/test_regress/t/t_cover_toggle_points.out @@ -47,7 +47,7 @@ -000000 point: comment=ptoggle[1][1] integer cyc; initial cyc=1; -%000000 wire [7:0] cyc_copy = cyc[7:0]; +~000000 wire [7:0] cyc_copy = cyc[7:0]; +000011 point: comment=cyc_copy[0] -000005 point: comment=cyc_copy[1] -000002 point: comment=cyc_copy[2] @@ -161,7 +161,7 @@ // CHECK_COVER(-1,"top.t.a*",4) // 2 edges * (t.a1 and t.a2) -%000000 input [7:0] cyc_copy; +~000000 input [7:0] cyc_copy; +000022 point: comment=cyc_copy[0] +000010 point: comment=cyc_copy[1] -000004 point: comment=cyc_copy[2] diff --git a/test_regress/t/t_vlcov_info.info.out b/test_regress/t/t_vlcov_info.info.out index f34103ee6..198c520fe 100644 --- a/test_regress/t/t_vlcov_info.info.out +++ b/test_regress/t/t_vlcov_info.info.out @@ -1,4 +1,13 @@ TN:verilator_coverage SF:file1.sp DA:159,0 +BRDA:159,0,0,0 +BRDA:159,0,1,1 +BRDA:159,0,2,20 +BRDA:159,0,3,0 +BRDA:159,0,4,1 +BRDA:159,0,5,9 +BRDA:159,0,6,22 +BRF:7 +BRH:2 end_of_record