test: Added xpath coverage report generation (make test)

Resolves: #581
Signed-off-by: Ejub Sabic <ejub1946@outlook.com>

Author: saba8814

.github/workflows/test.yml

@@ -116,6 +116,15 @@ jobs:
           tar xf "$tarfile"
           ln -s "${tarfile%.tar.gz}" images
 
+      - name: Extract YANG XPaths for coverage
+        run: |
+          python3 -m venv $RUNNER_TEMP/pyang-env
+          $RUNNER_TEMP/pyang-env/bin/pip install pyang weasyprint -q
+          mkdir -p $TEST_PATH/.log
+          $RUNNER_TEMP/pyang-env/bin/python3 $TEST_PATH/utils/extract_xpaths.py \
+            src/confd/yang/confd \
+            $TEST_PATH/.log/xpaths_all.csv || true
+
       - name: Regression Test ${{ env.TARGET }}
         env:
           INFIX_RELEASE: ${{ inputs.release }}
@@ -143,3 +152,16 @@ jobs:
         with:
           name: test-report
           path: output/images/test-report.pdf
+
+      - name: Generate XPath Coverage Report for ${{ env.TARGET }}
+        if: always()
+        run: |
+          export PATH="$RUNNER_TEMP/pyang-env/bin:$PATH"
+          make test-dir="$(pwd)/$TEST_PATH" xpath-coverage-report
+
+      - name: Upload XPath Coverage Report as Artifact
+        if: always()
+        uses: actions/upload-artifact@v7
+        with:
+          name: xpath-coverage-report
+          path: output/images/xpath-coverage-report.pdf

test/infamy/coverage.py

@@ -0,0 +1,95 @@
+"""XPath coverage tracker for infamy test runs.
+
+Writes covered xpaths to $NINEPM_LOG_PATH/xpath_coverage.log, one per line.
+Call track_xpath(), track_module(), or track_dict() from transport methods.
+"""
+
+import os
+import threading
+
+_lock = threading.Lock()
+_tracked: set[str] = set()
+
+
+def _log_path() -> str | None:
+    log_dir = os.environ.get("NINEPM_LOG_PATH")
+    if not log_dir:
+        return None
+    return os.path.join(log_dir, "xpath_coverage.log")
+
+
+def _flush(xpaths: list[str]) -> None:
+    path = _log_path()
+    if not path or not xpaths:
+        return
+    try:
+        with open(path, "a", encoding="utf-8") as f:
+            for x in xpaths:
+                f.write(x + "\n")
+    except OSError:
+        pass
+
+
+def _record(xpaths: list[str]) -> None:
+    new: list[str] = []
+    with _lock:
+        for x in xpaths:
+            if x not in _tracked:
+                _tracked.add(x)
+                new.append(x)
+    _flush(new)
+
+
+def _normalize(xpath: str) -> str:
+    """Strip module prefix from every segment except the root.
+
+    pyang and the CSV always use /mod:top/child (no mid-path prefixes).
+    Tests sometimes pass /mod:top/other-mod:child; we canonicalize here.
+    """
+    if not xpath or ":" not in xpath:
+        return xpath
+    segs = xpath.lstrip("/").split("/")
+    out = []
+    for i, seg in enumerate(segs):
+        out.append(seg if i == 0 else seg.split(":", 1)[-1])
+    return "/" + "/".join(out)
+
+
+def track_xpath(xpath: str | None) -> None:
+    """Record a direct xpath access (get_data, delete_xpath, call_action, ...)."""
+    if xpath:
+        _record([_normalize(xpath)])
+
+
+def track_module(modname: str | None) -> None:
+    """Record a module-level access (call_dict with no specific path)."""
+    if modname:
+        _record([f"/{modname}:*"])
+
+
+def track_dict(modname: str | None, data: object) -> None:
+    """Record xpaths implied by a config/rpc dict (put_config_dicts, ...)."""
+    if not modname or not isinstance(data, dict):
+        return
+    _record(list(_dict_to_xpaths(modname, data)))
+
+
+def _dict_to_xpaths(modname: str, data: object, prefix: str = "") -> set[str]:
+    xpaths: set[str] = set()
+
+    if isinstance(data, dict):
+        for key, value in data.items():
+            node = key.split(":", 1)[-1]
+
+            if not prefix:
+                xpath = f"/{modname}:{node}"
+            else:
+                xpath = f"{prefix}/{node}"
+
+            xpaths.add(xpath)
+            xpaths.update(_dict_to_xpaths(modname, value, xpath))
+    elif isinstance(data, list):
+        for item in data:
+            xpaths.update(_dict_to_xpaths(modname, item, prefix))
+            
+    return xpaths

test/infamy/netconf.py

@@ -17,7 +17,7 @@
 import netconf_client.ncclient
 from infamy.transport import Transport,infer_put_dict
 from netconf_client.error import RpcError
-from . import env, netutil
+from . import env, netutil, coverage
 
 
 def netconf_syn(addr):
@@ -249,6 +249,7 @@ def reboot(self):
         })
 
     def get(self, xpath, parse=True):
+        coverage.track_xpath(xpath)
         xpath_filter = self._build_xpath_filter(xpath)
         response = self.ncc.get(filter=xpath_filter)
         return self._parse_response(response, parse)
@@ -263,6 +264,7 @@ def get_dict(self, xpath):
         return data.print_dict()
 
     def get_data(self, xpath=None, parse=True):
+        coverage.track_xpath(xpath)
         xpath_filter = self._build_xpath_filter(xpath, get_data_xpath=True)
         response = self.ncc.get_data(datastore="ds:operational", filter=xpath_filter)
         parsed_data = self._parse_response(response, parse)
@@ -273,6 +275,7 @@ def get_data(self, xpath=None, parse=True):
         return parsed_data
 
     def get_config(self, xpath):
+        coverage.track_xpath(xpath)
         xpath_filter = self._build_xpath_filter(xpath)
         response = self.ncc.get_config(source="running", filter=xpath_filter)
         return self._parse_response(response, True)
@@ -317,6 +320,8 @@ def put_config_dicts(self, models, retries=3):
         """PUT full configuration of all models to running-config"""
         config = ""
         infer_put_dict(self.name, models)
+        for mod, data in models.items():
+            coverage.track_dict(mod, data)
 
         for model in models.keys():
             try:
@@ -349,6 +354,7 @@ def patch_config(self, modname, edit, retries=3):
                            f"Available models can be checked with get_schema_list()") from None
         lyd = mod.parse_data_dict(edit, no_state=True, validate=False)
         config = lyd.print_mem("xml", with_siblings=True, pretty=False)
+        coverage.track_dict(modname, edit)
         # print(f"Send new XML config: {config}")
         return self.put_config(config, retries=retries)
 
@@ -358,6 +364,7 @@ def call(self, call):
 
     def call_dict(self, modname, call):
         """Call RPC, Python dictionary version"""
+        coverage.track_dict(modname, call)
         try:
             mod = self.ly.get_module(modname)
         except libyang.util.LibyangError:
@@ -374,6 +381,7 @@ def call_action(self, xpath, input_data=None):
         the action's input leaves.  Defaults to an empty input for
         actions that take no parameters.
         """
+        coverage.track_xpath(xpath)
         action={}
         pattern = r"^/(?P<module>[^:]+):(?P<path>[^/]+)"
         match = re.search(pattern, xpath)
@@ -416,6 +424,7 @@ def get_schema(self, schema, outdir):
             f.write(data.schema)
 
     def delete_xpath(self, xpath):
+        coverage.track_xpath(xpath)
         # Split out the model and the container from xpath'
         pattern = r"^/(?P<module>[^:]+):(?P<path>[^/]+)"
         match = re.search(pattern, xpath)

test/infamy/restconf.py

@@ -11,7 +11,7 @@
 from urllib3.exceptions import InsecureRequestWarning
 from dataclasses import dataclass
 from infamy.transport import Transport, infer_put_dict
-from . import env
+from . import env, coverage
 
 # We know we have a self-signed certificate, silence warning about it
 warnings.simplefilter('ignore', InsecureRequestWarning)
@@ -258,6 +258,7 @@ def put_datastore(self, datastore, data):
 
     def get_config_dict(self, xpath):
         """Get all configuration matching @xpath as dictionary"""
+        coverage.track_xpath(xpath)
         # Strip leading slash if present (for compatibility with NETCONF xpath style)
         xpath = xpath.lstrip('/')
 
@@ -295,6 +296,8 @@ def put_config_dicts(self, models, retries=3):
             retries: Number of retry attempts on failure (default 3)
         """
         infer_put_dict(self.name, models)
+        for mod, data in models.items():
+            coverage.track_dict(mod, data)
 
         # Copy running to candidate first (to preserve existing config)
         self.copy("running", "candidate")
@@ -343,6 +346,7 @@ def put_config_dicts(self, models, retries=3):
         # Copy candidate to running (acts as "commit", triggers sysrepo callbacks)
         self.copy("candidate", "running")
 
+
     def patch_config(self, xpath, edit, retries=3):
         """PATCH configuration directly to running datastore
 
@@ -356,6 +360,7 @@ def patch_config(self, xpath, edit, retries=3):
             edit:    Configuration dictionary
             retries: Number of retry attempts on failure (default 3)
         """
+        coverage.track_dict(xpath, edit)
         try:
             mod = self.lyctx.get_module(xpath)
         except libyang.util.LibyangError:
@@ -406,6 +411,7 @@ def patch_config(self, xpath, edit, retries=3):
             raise last_error
 
     def call_dict(self, model, call):
+        coverage.track_dict(model, call)
         pass # Need implementation
 
     def call_rpc(self, rpc):
@@ -426,6 +432,7 @@ def get_dict(self, xpath=None, parse=True):
 
     def get_data(self, xpath=None, parse=True):
         """Get operational data"""
+        coverage.track_xpath(xpath)
         uri = xpath_to_uri(xpath) if xpath is not None else None
         data = self.get_operational(uri, parse)
 
@@ -453,6 +460,7 @@ def reboot(self):
         self.call_rpc("ietf-system:system-restart")
 
     def call_action(self, xpath, input_data=None):
+        coverage.track_xpath(xpath)
         path = xpath_to_uri(xpath)
         url = f"{self.restconf_url}/data{path}"
 
@@ -484,6 +492,7 @@ def call_action(self, xpath, input_data=None):
 
     def delete_xpath(self, xpath):
         """Delete XPath from running config"""
+        coverage.track_xpath(xpath)
         path = f"/ds/ietf-datastores:running{xpath_to_uri(xpath)}"
         url = f"{self.restconf_url}{path}"
         response = requests_workaround_delete(url, headers=self.headers,

test/test.mk

@@ -10,6 +10,11 @@ LOGO               ?= $(test-dir)/../doc/logo.png[top=40%, align=right, pdfwidth
 UNIT_TESTS         ?= $(test-dir)/case/all-repo.yaml $(test-dir)/case/all-unit.yaml
 TESTS              ?= $(test-dir)/case/all.yaml
 
+yang_extractor     := $(test-dir)/utils/extract_xpaths.py
+coverage_reporter  := $(test-dir)/utils/coverage_report.py
+YANG_DIR           ?= $(BR2_EXTERNAL_INFIX_PATH)/src/confd/yang/confd
+xpaths_all_csv     := $(test-dir)/.log/xpaths_all.csv
+
 base := -b $(base-dir)
 
 TEST_MODE ?= qeneth
@@ -30,9 +35,15 @@ endif
 
 test:
 	$(test-dir)/env -r $(base) $(mode) $(binaries) $(pkg-$(ARCH)) \
-		sh -c '$(ninepm) -v $(TESTS); rc=$$?; \
+		sh -c 'test -f $(xpaths_all_csv) || python3 $(yang_extractor) $(YANG_DIR) $(xpaths_all_csv) || true; \
+		       $(ninepm) -v $(TESTS); rc=$$?; \
 		       $(ninepm_report) github   $(test-dir)/.log/last/result.json; \
 		       $(ninepm_report) asciidoc $(test-dir)/.log/last/result.json; \
+		       python3 $(coverage_reporter) \
+		           $(xpaths_all_csv) \
+		           $(test-dir)/.log/last/xpath_coverage.log \
+		           $(test-dir)/.log/last/xpath_coverage_report.md \
+		           2>/dev/null || true; \
 		       chmod -R 777 $(test-dir)/.log; \
 		       exit $$rc'
 
@@ -62,9 +73,38 @@ test-report:
 		-a pdf-fontsdir=$(spec-dir)/fonts \
 		-o $(test-report) $(test-dir)/.log/last/report.adoc
 
+xpath_coverage_report_md   := $(test-dir)/.log/last/xpath_coverage_report.md
+xpath_coverage_report_css  := $(test-dir)/utils/coverage_report.css
+xpath_coverage_report_pdf  := $(BINARIES_DIR)/xpath-coverage-report.pdf
+xpath-coverage-report:
+	@ver="$(subst ",,$(INFIX_NAME)) $(INFIX_VERSION)"; \
+	 gen=$$(sed -n 's/^Generated: //p' $(xpath_coverage_report_md) | head -1 | cut -d' ' -f1); \
+	 { \
+	   printf '<div class="cover">\n'; \
+	   printf '<img class="cover-logo" src="%s">\n' "$(abspath $(test-dir)/../doc/logo.png)"; \
+	   printf '<h1 class="cover-title">XPath Coverage Report</h1>\n'; \
+	   printf '<p class="cover-version">%s</p>\n' "$$ver"; \
+	   printf '<p class="cover-date">%s</p>\n' "$$gen"; \
+	   printf '</div>\n\n'; \
+	   sed -e 's,🟢,<span class="dot ok"></span>,g'   \
+	       -e 's,🟠,<span class="dot warn"></span>,g' \
+	       -e 's,🔴,<span class="dot err"></span>,g'  \
+	       -e 's,✅,<span class="mark ok">✓</span>,g'  \
+	       -e 's,❌,<span class="mark err">✗</span>,g' \
+	       -e '/^# XPath Coverage Report$$/d'          \
+	       -e '/^Generated: /d'                        \
+	       $(xpath_coverage_report_md); \
+	 } > $(xpath_coverage_report_md).pdf.md
+	pandoc $(xpath_coverage_report_md).pdf.md \
+		-f gfm \
+		--pdf-engine=weasyprint \
+		-c $(xpath_coverage_report_css) \
+		-o $(xpath_coverage_report_pdf)
+	@rm -f $(xpath_coverage_report_md).pdf.md
+
 # Unit tests run with random (-r) hostname and container name to
 # prevent race conditions when running in CI environments.
 test-unit:
 	$(test-dir)/env -r $(base) $(ninepm) -v $(UNIT_TESTS)
 
-.PHONY: test test-sh test-unit test-spec
+.PHONY: test test-sh test-unit test-spec xpath-coverage-report