From 9853e0814578a7d859723767394d4b5511478672 Mon Sep 17 00:00:00 2001 From: Ejub Sabic Date: Thu, 4 Jun 2026 15:13:18 +0200 Subject: [PATCH] test: Added xpath coverage report generation (make test) This commit introduces xpath coverage test report. It plugs into transport layer of test system and generates .md report. Commit also introduces "make xpath-coverage-report" used to generate pdf out of md. Resolves: #581 Signed-off-by: Ejub Sabic --- .github/workflows/test.yml | 12 ++ test/.env | 2 +- test/docker/Dockerfile | 5 +- test/docker/pip-requirements.txt | 1 + test/infamy/coverage.py | 95 +++++++++++++++ test/infamy/netconf.py | 11 +- test/infamy/restconf.py | 11 +- test/test.mk | 30 ++++- test/utils/coverage_report.css | 137 +++++++++++++++++++++ test/utils/coverage_report.py | 191 ++++++++++++++++++++++++++++++ test/utils/extract_xpaths.py | 110 +++++++++++++++++ test/utils/render_coverage_pdf.sh | 43 +++++++ 12 files changed, 642 insertions(+), 6 deletions(-) create mode 100644 test/infamy/coverage.py create mode 100644 test/utils/coverage_report.css create mode 100644 test/utils/coverage_report.py create mode 100644 test/utils/extract_xpaths.py create mode 100755 test/utils/render_coverage_pdf.sh diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 707d5dfc4..be21f3725 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -143,3 +143,15 @@ jobs: with: name: test-report path: output/images/test-report.pdf + + - name: Generate XPath Coverage Report for ${{ env.TARGET }} + if: always() + run: | + make test-dir="$(pwd)/$TEST_PATH" xpath-coverage-report + + - name: Upload XPath Coverage Report as Artifact + if: always() + uses: actions/upload-artifact@v7 + with: + name: xpath-coverage-report + path: output/images/xpath-coverage-report.pdf diff --git a/test/.env b/test/.env index b804a029f..857d9e128 100644 --- a/test/.env +++ b/test/.env @@ -2,7 +2,7 @@ # shellcheck disable=SC2034,SC2154 # Current container image -INFIX_TEST=ghcr.io/kernelkit/infix-test:2.9 +INFIX_TEST=ghcr.io/kernelkit/infix-test:2.10 ixdir=$(readlink -f "$testdir/..") logdir=$(readlink -f "$testdir/.log") diff --git a/test/docker/Dockerfile b/test/docker/Dockerfile index 7678ec10a..c1750270c 100644 --- a/test/docker/Dockerfile +++ b/test/docker/Dockerfile @@ -11,6 +11,7 @@ RUN apk add --no-cache \ ethtool \ fakeroot \ file \ + font-dejavu \ gcc \ git \ graphviz \ @@ -24,6 +25,7 @@ RUN apk add --no-cache \ nmap \ openssh-client \ openssl \ + pandoc-cli \ python3-dev \ qemu-img \ qemu-system-x86_64 \ @@ -33,7 +35,8 @@ RUN apk add --no-cache \ squashfs-tools \ sshpass \ tcpdump \ - tshark + tshark \ + weasyprint ARG MTOOL_VERSION="3.0" RUN wget https://github.com/troglobit/mtools/releases/download/v3.0/mtools-$MTOOL_VERSION.tar.gz -O /tmp/mtools-$MTOOL_VERSION.tar.gz diff --git a/test/docker/pip-requirements.txt b/test/docker/pip-requirements.txt index 83793011d..06bb9ebaa 100644 --- a/test/docker/pip-requirements.txt +++ b/test/docker/pip-requirements.txt @@ -1,4 +1,5 @@ libyang==2.7.1 +pyang==2.7.1 netconf_client==3.1.1 netifaces==0.11.0 networkx==3.1 diff --git a/test/infamy/coverage.py b/test/infamy/coverage.py new file mode 100644 index 000000000..8561d3f75 --- /dev/null +++ b/test/infamy/coverage.py @@ -0,0 +1,95 @@ +"""XPath coverage tracker for infamy test runs. + +Writes covered xpaths to $NINEPM_LOG_PATH/xpath_coverage.log, one per line. +Call track_xpath(), track_module(), or track_dict() from transport methods. +""" + +import os +import threading + +_lock = threading.Lock() +_tracked: set[str] = set() + + +def _log_path() -> str | None: + log_dir = os.environ.get("NINEPM_LOG_PATH") + if not log_dir: + return None + return os.path.join(log_dir, "xpath_coverage.log") + + +def _flush(xpaths: list[str]) -> None: + path = _log_path() + if not path or not xpaths: + return + try: + with open(path, "a", encoding="utf-8") as f: + for x in xpaths: + f.write(x + "\n") + except OSError: + pass + + +def _record(xpaths: list[str]) -> None: + new: list[str] = [] + with _lock: + for x in xpaths: + if x not in _tracked: + _tracked.add(x) + new.append(x) + _flush(new) + + +def _normalize(xpath: str) -> str: + """Strip module prefix from every segment except the root. + + pyang and the CSV always use /mod:top/child (no mid-path prefixes). + Tests sometimes pass /mod:top/other-mod:child; we canonicalize here. + """ + if not xpath or ":" not in xpath: + return xpath + segs = xpath.lstrip("/").split("/") + out = [] + for i, seg in enumerate(segs): + out.append(seg if i == 0 else seg.split(":", 1)[-1]) + return "/" + "/".join(out) + + +def track_xpath(xpath: str | None) -> None: + """Record a direct xpath access (get_data, delete_xpath, call_action, ...).""" + if xpath: + _record([_normalize(xpath)]) + + +def track_module(modname: str | None) -> None: + """Record a module-level access (call_dict with no specific path).""" + if modname: + _record([f"/{modname}:*"]) + + +def track_dict(modname: str | None, data: object) -> None: + """Record xpaths implied by a config/rpc dict (put_config_dicts, ...).""" + if not modname or not isinstance(data, dict): + return + _record(list(_dict_to_xpaths(modname, data))) + + +def _dict_to_xpaths(modname: str, data: object, prefix: str = "") -> set[str]: + xpaths: set[str] = set() + + if isinstance(data, dict): + for key, value in data.items(): + node = key.split(":", 1)[-1] + + if not prefix: + xpath = f"/{modname}:{node}" + else: + xpath = f"{prefix}/{node}" + + xpaths.add(xpath) + xpaths.update(_dict_to_xpaths(modname, value, xpath)) + elif isinstance(data, list): + for item in data: + xpaths.update(_dict_to_xpaths(modname, item, prefix)) + + return xpaths diff --git a/test/infamy/netconf.py b/test/infamy/netconf.py index c82fc33ae..dbcf2282a 100644 --- a/test/infamy/netconf.py +++ b/test/infamy/netconf.py @@ -17,7 +17,7 @@ import netconf_client.ncclient from infamy.transport import Transport,infer_put_dict from netconf_client.error import RpcError -from . import env, netutil +from . import env, netutil, coverage def netconf_syn(addr): @@ -249,6 +249,7 @@ def reboot(self): }) def get(self, xpath, parse=True): + coverage.track_xpath(xpath) xpath_filter = self._build_xpath_filter(xpath) response = self.ncc.get(filter=xpath_filter) return self._parse_response(response, parse) @@ -263,6 +264,7 @@ def get_dict(self, xpath): return data.print_dict() def get_data(self, xpath=None, parse=True): + coverage.track_xpath(xpath) xpath_filter = self._build_xpath_filter(xpath, get_data_xpath=True) response = self.ncc.get_data(datastore="ds:operational", filter=xpath_filter) parsed_data = self._parse_response(response, parse) @@ -273,6 +275,7 @@ def get_data(self, xpath=None, parse=True): return parsed_data def get_config(self, xpath): + coverage.track_xpath(xpath) xpath_filter = self._build_xpath_filter(xpath) response = self.ncc.get_config(source="running", filter=xpath_filter) return self._parse_response(response, True) @@ -317,6 +320,8 @@ def put_config_dicts(self, models, retries=3): """PUT full configuration of all models to running-config""" config = "" infer_put_dict(self.name, models) + for mod, data in models.items(): + coverage.track_dict(mod, data) for model in models.keys(): try: @@ -349,6 +354,7 @@ def patch_config(self, modname, edit, retries=3): f"Available models can be checked with get_schema_list()") from None lyd = mod.parse_data_dict(edit, no_state=True, validate=False) config = lyd.print_mem("xml", with_siblings=True, pretty=False) + coverage.track_dict(modname, edit) # print(f"Send new XML config: {config}") return self.put_config(config, retries=retries) @@ -358,6 +364,7 @@ def call(self, call): def call_dict(self, modname, call): """Call RPC, Python dictionary version""" + coverage.track_dict(modname, call) try: mod = self.ly.get_module(modname) except libyang.util.LibyangError: @@ -374,6 +381,7 @@ def call_action(self, xpath, input_data=None): the action's input leaves. Defaults to an empty input for actions that take no parameters. """ + coverage.track_xpath(xpath) action={} pattern = r"^/(?P[^:]+):(?P[^/]+)" match = re.search(pattern, xpath) @@ -416,6 +424,7 @@ def get_schema(self, schema, outdir): f.write(data.schema) def delete_xpath(self, xpath): + coverage.track_xpath(xpath) # Split out the model and the container from xpath' pattern = r"^/(?P[^:]+):(?P[^/]+)" match = re.search(pattern, xpath) diff --git a/test/infamy/restconf.py b/test/infamy/restconf.py index ed7ec0f96..3cb0a5bed 100644 --- a/test/infamy/restconf.py +++ b/test/infamy/restconf.py @@ -11,7 +11,7 @@ from urllib3.exceptions import InsecureRequestWarning from dataclasses import dataclass from infamy.transport import Transport, infer_put_dict -from . import env +from . import env, coverage # We know we have a self-signed certificate, silence warning about it warnings.simplefilter('ignore', InsecureRequestWarning) @@ -258,6 +258,7 @@ def put_datastore(self, datastore, data): def get_config_dict(self, xpath): """Get all configuration matching @xpath as dictionary""" + coverage.track_xpath(xpath) # Strip leading slash if present (for compatibility with NETCONF xpath style) xpath = xpath.lstrip('/') @@ -295,6 +296,8 @@ def put_config_dicts(self, models, retries=3): retries: Number of retry attempts on failure (default 3) """ infer_put_dict(self.name, models) + for mod, data in models.items(): + coverage.track_dict(mod, data) # Copy running to candidate first (to preserve existing config) self.copy("running", "candidate") @@ -343,6 +346,7 @@ def put_config_dicts(self, models, retries=3): # Copy candidate to running (acts as "commit", triggers sysrepo callbacks) self.copy("candidate", "running") + def patch_config(self, xpath, edit, retries=3): """PATCH configuration directly to running datastore @@ -356,6 +360,7 @@ def patch_config(self, xpath, edit, retries=3): edit: Configuration dictionary retries: Number of retry attempts on failure (default 3) """ + coverage.track_dict(xpath, edit) try: mod = self.lyctx.get_module(xpath) except libyang.util.LibyangError: @@ -406,6 +411,7 @@ def patch_config(self, xpath, edit, retries=3): raise last_error def call_dict(self, model, call): + coverage.track_dict(model, call) pass # Need implementation def call_rpc(self, rpc): @@ -426,6 +432,7 @@ def get_dict(self, xpath=None, parse=True): def get_data(self, xpath=None, parse=True): """Get operational data""" + coverage.track_xpath(xpath) uri = xpath_to_uri(xpath) if xpath is not None else None data = self.get_operational(uri, parse) @@ -453,6 +460,7 @@ def reboot(self): self.call_rpc("ietf-system:system-restart") def call_action(self, xpath, input_data=None): + coverage.track_xpath(xpath) path = xpath_to_uri(xpath) url = f"{self.restconf_url}/data{path}" @@ -484,6 +492,7 @@ def call_action(self, xpath, input_data=None): def delete_xpath(self, xpath): """Delete XPath from running config""" + coverage.track_xpath(xpath) path = f"/ds/ietf-datastores:running{xpath_to_uri(xpath)}" url = f"{self.restconf_url}{path}" response = requests_workaround_delete(url, headers=self.headers, diff --git a/test/test.mk b/test/test.mk index 9ff8d4c54..df3894bba 100644 --- a/test/test.mk +++ b/test/test.mk @@ -10,6 +10,11 @@ LOGO ?= $(test-dir)/../doc/logo.png[top=40%, align=right, pdfwidth UNIT_TESTS ?= $(test-dir)/case/all-repo.yaml $(test-dir)/case/all-unit.yaml TESTS ?= $(test-dir)/case/all.yaml +yang_extractor := $(test-dir)/utils/extract_xpaths.py +coverage_reporter := $(test-dir)/utils/coverage_report.py +YANG_DIR ?= $(BR2_EXTERNAL_INFIX_PATH)/src/confd/yang/confd +xpaths_all_csv := $(test-dir)/.log/xpaths_all.csv + base := -b $(base-dir) TEST_MODE ?= qeneth @@ -30,9 +35,15 @@ endif test: $(test-dir)/env -r $(base) $(mode) $(binaries) $(pkg-$(ARCH)) \ - sh -c '$(ninepm) -v $(TESTS); rc=$$?; \ + sh -c 'test -f $(xpaths_all_csv) || python3 $(yang_extractor) $(YANG_DIR) $(xpaths_all_csv) || true; \ + $(ninepm) -v $(TESTS); rc=$$?; \ $(ninepm_report) github $(test-dir)/.log/last/result.json; \ $(ninepm_report) asciidoc $(test-dir)/.log/last/result.json; \ + python3 $(coverage_reporter) \ + $(xpaths_all_csv) \ + $(test-dir)/.log/last/xpath_coverage.log \ + $(test-dir)/.log/last/xpath_coverage_report.md \ + 2>/dev/null || true; \ chmod -R 777 $(test-dir)/.log; \ exit $$rc' @@ -62,9 +73,24 @@ test-report: -a pdf-fontsdir=$(spec-dir)/fonts \ -o $(test-report) $(test-dir)/.log/last/report.adoc +xpath_coverage_report_md := $(test-dir)/.log/last/xpath_coverage_report.md +xpath_coverage_report_css := $(test-dir)/utils/coverage_report.css +xpath_coverage_report_pdf := $(BINARIES_DIR)/xpath-coverage-report.pdf +xpath_coverage_renderer := $(test-dir)/utils/render_coverage_pdf.sh +xpath_coverage_logo := $(abspath $(test-dir)/../doc/logo.png) +# Rendered inside the infix-test container, which carries pandoc, weasyprint +# and the fonts; make variables are expanded on the host and passed as args. +xpath-coverage-report: + $(test-dir)/env $(base) $(xpath_coverage_renderer) \ + $(xpath_coverage_report_md) \ + $(xpath_coverage_report_css) \ + $(xpath_coverage_logo) \ + "$(subst ",,$(INFIX_NAME)) $(INFIX_VERSION)" \ + $(xpath_coverage_report_pdf) + # Unit tests run with random (-r) hostname and container name to # prevent race conditions when running in CI environments. test-unit: $(test-dir)/env -r $(base) $(ninepm) -v $(UNIT_TESTS) -.PHONY: test test-sh test-unit test-spec +.PHONY: test test-sh test-unit test-spec xpath-coverage-report diff --git a/test/utils/coverage_report.css b/test/utils/coverage_report.css new file mode 100644 index 000000000..c4b4a53ed --- /dev/null +++ b/test/utils/coverage_report.css @@ -0,0 +1,137 @@ +/* + * Stylesheet for the XPath coverage report PDF (pandoc --pdf-engine=weasyprint). + * + * The Markdown report uses emoji (🟢🟠🔴✅❌) for status. WeasyPrint cannot + * embed the bitmap glyphs of "Noto Color Emoji" into a PDF, so those render as + * empty boxes. The Makefile therefore rewrites the emoji into marks + * (● ✓ ✗) which are plain text glyphs present in every font; the colours below + * restore the green/orange/red meaning. + */ + +@page { + size: A4; + margin: 18mm 14mm; + + /* Footer: a full-width rule with the page number on it. The number sits + * in the outside corner (alternated below); the border-top draws the line + * above the footer, like the test specification/report. */ + @bottom-center { + content: counter(page); + width: 100%; + border-top: 0.75pt solid #999999; + padding-top: 6pt; + font-family: "DejaVu Sans", "Noto Sans", sans-serif; + font-size: 8pt; + color: #777777; + } +} + +/* Page number in the outside corner: left on left-hand (even) pages, + * right on right-hand (odd) pages. */ +@page :left { @bottom-center { text-align: left; } } +@page :right { @bottom-center { text-align: right; } } + +/* The cover page carries no footer (no rule, no number). */ +@page cover { + @bottom-center { content: none; } +} + +body { + font-family: "DejaVu Sans", "Noto Sans", sans-serif; + font-size: 9pt; + line-height: 1.4; + color: #1b1b1b; +} + +h1 { font-size: 22pt; margin: 0 0 4pt; } + +/* + * Cover page — logo, title, software version and generation date, on a page of + * its own (page-break-after). Mirrors the asciidoctor-pdf title page used by + * the test specification/report; the logo is doc/logo.png (set by the Makefile). + * + * NOTE: WeasyPrint does not support viewport units (vh/vw), so the vertical + * offset is a fixed length (~40% down an A4 page, like the test report logo). + */ +.cover { + page: cover; + page-break-after: always; + text-align: right; + margin-left: auto; + padding-top: 80mm; + /* Cover is page 0 (number suppressed below), so the first body page is 1. */ + counter-reset: page 0; +} +.cover-logo { + width: 90mm; + margin-bottom: 18pt; +} +.cover-title { + color: gray; + font-size: 26pt; + font-weight: 400; + margin: 0 0 8pt; + border: none; +} +.cover-version { + font-size: 13pt; + font-weight: 400; + margin: 0 0 2pt; +} +.cover-date { + font-size: 11pt; + color: #777777; + margin: 0; +} +h2 { + font-size: 15pt; + margin: 18pt 0 6pt; + border-bottom: 1px solid #dddddd; + padding-bottom: 3pt; +} +h3 { font-size: 11pt; margin: 14pt 0 4pt; } + +code, pre { + font-family: "DejaVu Sans Mono", "Noto Sans Mono", monospace; + font-size: 8pt; +} + +table { + border-collapse: collapse; + width: 100%; + margin: 6pt 0 12pt; +} + +th, td { + border: 1px solid #d0d0d0; + padding: 3pt 6pt; + vertical-align: top; + /* Long XPaths have no spaces; allow them to wrap anywhere. */ + word-break: break-all; + overflow-wrap: anywhere; +} + +thead th { + background: #f2f2f2; + font-weight: 600; + border-bottom: 2px solid #bbbbbb; +} + +tbody tr:nth-child(even) { background: #fafafa; } + +/* Status dots (module summary + section headings). */ +.dot { + display: inline-block; + width: 9pt; + height: 9pt; + border-radius: 50%; + vertical-align: middle; +} +.dot.ok { background: #2e9e44; } /* 🟢 >= 80% */ +.dot.warn { background: #f0883e; } /* 🟠 1-79% */ +.dot.err { background: #e5534b; } /* 🔴 0% */ + +/* Per-row covered marks. */ +.mark { font-weight: 700; } +.mark.ok { color: #2e9e44; } /* ✅ */ +.mark.err { color: #e5534b; } /* ❌ */ diff --git a/test/utils/coverage_report.py b/test/utils/coverage_report.py new file mode 100644 index 000000000..54e315e06 --- /dev/null +++ b/test/utils/coverage_report.py @@ -0,0 +1,191 @@ +#!/usr/bin/env python3 +"""Generate an XPath coverage report after a test run. + +Usage: + coverage_report.py + +xpaths_all.csv - produced by extract_xpaths.py; columns: module,keyword,xpath +xpath_coverage.log - produced by infamy/coverage.py; one xpath per line +output.md - destination for the markdown report +""" + +import csv +import os +import sys +from collections import defaultdict +from datetime import datetime + + +# Thresholds for colour coding +GREEN_THRESHOLD = 80 # % +ORANGE_THRESHOLD = 1 # % (anything >= 1 and < 80 is orange) + +# Modules excluded from the report +SKIP_MODULES: set[str] = { + "infix-if-base", # submodule of infix-interfaces; only a structural choice node +} + + +def load_all_xpaths(csv_path: str) -> list[dict]: + rows = [] + with open(csv_path, newline="", encoding="utf-8") as f: + for row in csv.DictReader(f): + if row["module"] not in SKIP_MODULES: + rows.append(row) + return rows + + +def load_tracked_xpaths(log_path: str) -> set[str]: + tracked: set[str] = set() + if not os.path.exists(log_path): + return tracked + with open(log_path, encoding="utf-8") as f: + for line in f: + x = line.strip() + if x: + tracked.add(x) + return tracked + + +def is_covered(yang_xpath: str, tracked: set[str], module: str, + xpath_module: dict[str, str]) -> bool: + """ Return True if yang_xpath was exercised by a test. """ + if f"/{module}:*" in tracked: + return True + if yang_xpath in tracked: + return True + + prefix = yang_xpath + "/" + for t in tracked: + if not prefix.startswith(t + "/"): + continue + # Determine the defining module of the tracked ancestor + t_module = xpath_module.get(t) + if t_module is None: + # Not in the CSV; infer from the root path segment (e.g. /mod:node → mod) + seg = t.lstrip("/").split("/")[0] + t_module = seg.split(":")[0] if ":" in seg else None + if t_module == module: + return True + + return False + + +def build_coverage(all_xpaths: list[dict], tracked: set[str]) -> dict: + """Return per-module coverage data. + + Returns: + {module: {"total": int, "covered": int, "rows": [{"xpath", "keyword", "covered"}]}} + """ + xpath_module: dict[str, str] = {row["xpath"]: row["module"] for row in all_xpaths} + modules: dict[str, dict] = defaultdict(lambda: {"total": 0, "covered": 0, "rows": []}) + + for row in all_xpaths: + mod = row["module"] + xpath = row["xpath"] + kw = row["keyword"] + hit = is_covered(xpath, tracked, mod, xpath_module) + + modules[mod]["total"] += 1 + modules[mod]["covered"] += int(hit) + modules[mod]["rows"].append({"xpath": xpath, "keyword": kw, "covered": hit}) + + # Sort rows within each module by xpath + for mod_data in modules.values(): + mod_data["rows"].sort(key=lambda r: r["xpath"]) + + return dict(sorted(modules.items())) + + +def status_emoji(pct: float) -> str: + if pct >= GREEN_THRESHOLD: + return "🟢" + if pct >= ORANGE_THRESHOLD: + return "🟠" + return "🔴" + + +def pct(covered: int, total: int) -> float: + return 100.0 * covered / total if total else 0.0 + + +def write_markdown(coverage: dict, output_path: str) -> None: + total_all = sum(d["total"] for d in coverage.values()) + covered_all = sum(d["covered"] for d in coverage.values()) + overall_pct = pct(covered_all, total_all) + + lines: list[str] = [] + lines.append("# XPath Coverage Report\n") + lines.append(f"Generated: {datetime.now().strftime('%Y-%m-%d')}\n") + lines.append("") + + # ── Overall summary ────────────────────────────────────────────────────── + lines.append("## Summary\n") + lines.append( + f"{status_emoji(overall_pct)} **Overall: " + f"{covered_all}/{total_all} XPaths covered " + f"({overall_pct:.1f}%)**\n" + ) + lines.append("") + + lines.append("| Status | Module | Covered | Total | % |") + lines.append("|--------|--------|--------:|------:|--:|") + for mod, d in sorted(coverage.items()): + p = pct(d["covered"], d["total"]) + lines.append( + f"| {status_emoji(p)} | {mod} " + f"| {d['covered']} | {d['total']} | {p:.1f}% |" + ) + lines.append("") + + # ── Per-module detail ───────────────────────────────────────────────────── + lines.append("## Details\n") + + for mod, d in coverage.items(): + p = pct(d["covered"], d["total"]) + lines.append( + f"### {status_emoji(p)} {mod} " + f"({d['covered']}/{d['total']} — {p:.1f}%)\n" + ) + lines.append("| XPath | Keyword | Covered |") + lines.append("|-------|---------|:-------:|") + for row in d["rows"]: + tick = "✅" if row["covered"] else "❌" + lines.append(f"| `{row['xpath']}` | {row['keyword']} | {tick} |") + lines.append("") + + with open(output_path, "w", encoding="utf-8") as f: + f.write("\n".join(lines)) + + print(f"Coverage report written to {output_path}") + + +def main() -> None: + if len(sys.argv) < 4: + print(f"Usage: {sys.argv[0]} ") + sys.exit(1) + + all_csv = sys.argv[1] + cov_log = sys.argv[2] + out_md = sys.argv[3] + + if not os.path.exists(all_csv): + print(f"Error: XPath list not found: {all_csv}", file=sys.stderr) + sys.exit(1) + + all_xpaths = load_all_xpaths(all_csv) + tracked = load_tracked_xpaths(cov_log) + coverage = build_coverage(all_xpaths, tracked) + + write_markdown(coverage, out_md) + + total_all = sum(d["total"] for d in coverage.values()) + covered_all = sum(d["covered"] for d in coverage.values()) + print( + f"Overall: {covered_all}/{total_all} XPaths covered " + f"({pct(covered_all, total_all):.1f}%)" + ) + + +if __name__ == "__main__": + main() diff --git a/test/utils/extract_xpaths.py b/test/utils/extract_xpaths.py new file mode 100644 index 000000000..61daf4c7e --- /dev/null +++ b/test/utils/extract_xpaths.py @@ -0,0 +1,110 @@ +#!/usr/bin/env python3 +"""Extract all XPaths from YANG modules in a directory. + +Outputs a CSV with columns: module, keyword, xpath +XPaths use NETCONF format: /module:top-container/sub/... +""" + +import os +import sys +import csv +from pyang import context, repository + + +def build_xpath(node): + """Build a NETCONF-style xpath for a schema node.""" + parts = [] + module_name = None + + while node is not None: + if hasattr(node, "arg") and node.arg: + parts.append(node.arg) + parent = getattr(node, "parent", None) + if parent is None or parent.keyword == "module": + if parent is not None: + module_name = parent.arg + break + node = parent + + reversed_parts = list(reversed(parts)) + if module_name and reversed_parts: + reversed_parts[0] = f"{module_name}:{reversed_parts[0]}" + + return "/" + "/".join(reversed_parts) + + +def collect_xpaths(stmt, results): + schema_keywords = { + "container", + "list", + "leaf", + "leaf-list", + "choice", + "case", + "rpc", + "action", + "notification", + "anyxml", + "anydata", + } + + # config false (operational state) is inherited; pyang resolves the + # effective status into i_config during validation. Only skip when it is + # explicitly False so rpc/action/notification (i_config is None) are kept. + if stmt.keyword in schema_keywords and getattr(stmt, "i_config", None) is not False: + results.append({ + "module": stmt.i_module.arg, + "keyword": stmt.keyword, + "xpath": build_xpath(stmt), + }) + + for child in getattr(stmt, "i_children", []): + collect_xpaths(child, results) + + +def load_yang_modules(yang_dir): + repo = repository.FileRepository(yang_dir) + ctx = context.Context(repo) + + for root, _, files in os.walk(yang_dir): + for file in sorted(files): + if file.endswith(".yang") and "@" not in file: + filepath = os.path.join(root, file) + with open(filepath, "r", encoding="utf-8") as f: + text = f.read() + module = ctx.add_module(file, text) + if module is None: + print(f"Warning: failed to load {file}", file=sys.stderr) + + ctx.validate() + return ctx + + +def export_xpaths(yang_dir, output_csv): + ctx = load_yang_modules(yang_dir) + + results = [] + for module in ctx.modules.values(): + for child in getattr(module, "i_children", []): + collect_xpaths(child, results) + + out_dir = os.path.dirname(output_csv) + if out_dir: + os.makedirs(out_dir, exist_ok=True) + + with open(output_csv, "w", newline="", encoding="utf-8") as csvfile: + writer = csv.DictWriter(csvfile, fieldnames=["module", "keyword", "xpath"]) + writer.writeheader() + for row in sorted(results, key=lambda x: (x["module"], x["xpath"])): + writer.writerow(row) + + print(f"Extracted {len(results)} XPaths to {output_csv}") + return len(results) + + +if __name__ == "__main__": + if len(sys.argv) < 3: + print(f"Usage: {sys.argv[0]} ") + sys.exit(1) + + export_xpaths(sys.argv[1], sys.argv[2]) diff --git a/test/utils/render_coverage_pdf.sh b/test/utils/render_coverage_pdf.sh new file mode 100755 index 000000000..9ef1ec2bf --- /dev/null +++ b/test/utils/render_coverage_pdf.sh @@ -0,0 +1,43 @@ +#!/bin/sh +# Render the XPath coverage markdown report to PDF using pandoc + weasyprint. +# +# Prepends an HTML cover page and replaces the emoji status markers with +# CSS-styled spans, then hands the result to pandoc. Runs inside the +# infix-test container, where pandoc, weasyprint and the fonts live. +# +# Usage: +# render_coverage_pdf.sh +set -e + +md="$1" +css="$2" +logo="$3" +ver="$4" +pdf="$5" + +gen=$(sed -n 's/^Generated: //p' "$md" | head -1 | cut -d' ' -f1) + +{ + printf '
\n' + printf '\n' "$logo" + printf '

XPath Coverage Report

\n' + printf '

%s

\n' "$ver" + printf '

%s

\n' "$gen" + printf '
\n\n' + sed -e 's,🟢,,g' \ + -e 's,🟠,,g' \ + -e 's,🔴,,g' \ + -e 's,✅,✓,g' \ + -e 's,❌,✗,g' \ + -e '/^# XPath Coverage Report$/d' \ + -e '/^Generated: /d' \ + "$md" +} > "$md.pdf.md" + +pandoc "$md.pdf.md" \ + -f gfm \ + --pdf-engine=weasyprint \ + -c "$css" \ + -o "$pdf" + +rm -f "$md.pdf.md"