data-meta-model/scripts/run_shacl_validation.py

#!/usr/bin/env python3
"""
PR Data Meta Model - SHACL validation runner
=============================================
The automated gate behind rule DQ-011. Validates an instance graph against the
executable rulebook (SHACL shapes), with the T-Box supplying subclass
entailment.

USAGE
    python3 scripts/run_shacl_validation.py                    # all instances
    python3 scripts/run_shacl_validation.py instances/sodh.ttl # one file
    python3 scripts/run_shacl_validation.py --quiet            # CI mode

Paths resolve against the repository root (the parent of scripts/), so the
script runs correctly from anywhere.

EXIT CODES
    0  conforms, no blocking violation   -> eligible for PUBLISHED
    1  violations found                  -> stays DRAFT / UNDER_REVIEW
    2  the run itself is not trustworthy -> a green result would be meaningless

Exit code 2 is the important one. A validation gate that passes because no
shape matched anything is worse than one that fails: it reports success while
checking nothing. The pre-flight below refuses to let that happen.

DEPENDENCIES
    pyshacl, rdflib. On GrosseBertha these need a venv built with the STABLE
    python (never the DSM system python, which moves between DSM releases):

        /usr/local/bin/python3.9 -m venv venv
        . venv/bin/activate
        pip install pyshacl rdflib
"""
import glob
import os
import sys
from collections import defaultdict

REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
ONTOLOGY = os.path.join(REPO, "ontology", "pr_metamodel.ttl")
SHAPES = os.path.join(REPO, "shapes", "pr_metamodel_shapes.ttl")
INSTANCES = os.path.join(REPO, "instances")

W = 78
BAR = "=" * W


def rel(path):
    """Path relative to the repo root, for readable output."""
    try:
        return os.path.relpath(path, REPO)
    except ValueError:
        return path


def die(code, title, lines):
    print(BAR)
    print("PR META MODEL - SHACL VALIDATION")
    print(BAR)
    print(title)
    print("-" * W)
    for line in lines:
        print("  " + line)
    print(BAR)
    sys.exit(code)


def load_deps():
    """
    Import the validation stack.

    Catches every exception, not just ImportError: a dependency that installs
    but cannot be imported (pyshacl built for a newer Python than the venv,
    an rdflib major-version clash) raises TypeError or AttributeError at import
    time. Letting those escape would end the process with an exit code CI reads
    as "the model has violations", when in fact validation never started.
    Anything wrong here is exit 2 -- the run is not trustworthy.
    """
    try:
        from pyshacl import validate
        from rdflib import Graph
        from rdflib.namespace import RDF, SH
        return validate, Graph, RDF, SH
    except Exception as exc:
        hint = []
        if "unsupported operand type" in str(exc) or "|" in str(exc):
            hint = [
                "This is PEP 604 syntax (str | X) evaluated at import time:",
                "the installed pyshacl needs Python 3.10+, this venv is %d.%d."
                % (sys.version_info[0], sys.version_info[1]),
                "pyshacl advertises 3.9 support but its code does not honour it,",
                "so pip installs a version that cannot be imported.",
                "",
                "Pin a release that really supports this interpreter:",
                '  pip install "pyshacl==0.26.0" "rdflib>=6.3.2,<7.1"',
                "or rebuild the venv on a newer Python if one exists.",
            ]
        else:
            hint = [
                "Build the venv with the stable python, never the DSM system one:",
                "  /usr/local/bin/python3.9 -m venv venv",
                "  . venv/bin/activate && pip install pyshacl rdflib",
            ]
        die(2, "DEPENDENCY UNUSABLE -- validation did not run",
            ["%s: %s" % (type(exc).__name__, exc), ""] + hint)


def term_namespaces(graph, predicate=None):
    """
    Namespaces actually used by pr: terms in a graph, with usage counts.

    Only real IRIs count. Literals are skipped: SPARQL constraint bodies
    (sh:select) embed their own PREFIX declarations, and counting those would
    put raw query text in the diagnostic table.
    """
    counts = defaultdict(int)
    for s, p, o in graph:
        for term in (s, p, o):
            text = str(term)
            if "ontology.pernod-ricard.com/metamodel" not in text:
                continue
            if not text.startswith("http") or any(c in text for c in " \t\n\r<>\""):
                continue                      # literal, not an IRI
            counts[text.rsplit("/", 1)[0] + "/"] += 1
    return counts


def preflight(data, shapes, onto, data_files):
    """
    Refuse to produce a green result that means nothing.

    Two ways a SHACL run passes while checking nothing:
      1. the shapes target classes in a namespace the instances never use;
      2. the ontology declares its classes in a third namespace, so subclass
         entailment cannot reach the instances and every shape targeting an
         abstract parent (MetaModelObject) finds zero focus nodes.
    Both look identical to a caller reading only "CONFORMS: True".
    """
    problems = []
    ns_problem = False

    if not len(data):
        problems.append("The instance graph is empty -- nothing to validate.")

    ns_data = term_namespaces(data)
    ns_shapes = term_namespaces(shapes)
    ns_onto = term_namespaces(onto) if onto is not None else {}

    if ns_data and ns_shapes and not (set(ns_data) & set(ns_shapes)):
        ns_problem = True
        problems.append(
            "Shapes and instances use disjoint term namespaces -- no shape can "
            "match any node.")

    if ns_onto and ns_data and not (set(ns_onto) & set(ns_data)):
        ns_problem = True
        problems.append(
            "The T-Box declares its terms in a namespace the instances never "
            "use. Subclass entailment cannot reach the data, so every shape "
            "targeting an abstract parent finds zero focus nodes.")

    if problems:
        lines = list(problems)
        lines.append("")
        lines.append("Namespaces in use:")
        for label, counts in (("instances", ns_data), ("shapes", ns_shapes),
                              ("ontology", ns_onto)):
            if counts:
                for ns, n in sorted(counts.items(), key=lambda kv: -kv[1]):
                    lines.append("  %-10s %-52s %6d refs" % (label, ns, n))
            else:
                lines.append("  %-10s (none)" % label)
        if ns_problem:
            lines.append("")
            lines.append("Fix: align the namespaces before trusting any result.")
            lines.append("  sed -i 's|/metamodel/0.9/|/metamodel/|g' "
                         "instances/*.ttl shapes/*.ttl")
        die(2, "PRE-FLIGHT FAILED -- a result here would be meaningless", lines)


def main():
    args = [a for a in sys.argv[1:] if not a.startswith("-")]
    quiet = "--quiet" in sys.argv or "-q" in sys.argv

    validate, Graph, RDF, SH = load_deps()

    # ---- resolve inputs ---------------------------------------------------
    if args:
        data_files = [a if os.path.isabs(a) else os.path.join(REPO, a) for a in args]
    else:
        data_files = sorted(glob.glob(os.path.join(INSTANCES, "*.ttl")))

    missing = [f for f in data_files if not os.path.exists(f)]
    if missing or not data_files:
        die(2, "NO INSTANCE DATA -- validation did not run",
            [rel(f) + " not found" for f in missing] or
            ["instances/ contains no .ttl file"])

    if not os.path.exists(SHAPES):
        die(2, "NO SHAPES -- validation did not run",
            [rel(SHAPES) + " not found"])

    # ---- parse ------------------------------------------------------------
    data = Graph()
    for f in data_files:
        try:
            data.parse(f, format="turtle")
        except Exception as exc:
            die(2, "PARSE ERROR -- validation did not run",
                [rel(f), str(exc)])

    try:
        shapes = Graph().parse(SHAPES, format="turtle")
    except Exception as exc:
        die(2, "PARSE ERROR in shapes -- validation did not run",
            [rel(SHAPES), str(exc)])

    onto = None
    if os.path.exists(ONTOLOGY):
        try:
            onto = Graph().parse(ONTOLOGY, format="turtle")
        except Exception as exc:
            die(2, "PARSE ERROR in ontology -- validation did not run",
                [rel(ONTOLOGY), str(exc)])

    preflight(data, shapes, onto, data_files)

    # ---- validate ---------------------------------------------------------
    conforms, results, _ = validate(
        data_graph=data,
        shacl_graph=shapes,
        ont_graph=onto,
        inference="rdfs",      # a Metric is also a MetaModelObject
        advanced=True,         # SPARQL constraints (NR-002, OW-007, DQ-006)
        abort_on_first=False,
        meta_shacl=False,
    )

    def shape_label(g, res, SH):
        """
        A readable name for the shape that fired.

        sh:sourceShape on a property constraint is a blank node, which prints as
        'ne775aabb...' and tells the reader nothing. Fall back, in order, to the
        constrained path, then to the named NodeShape that owns the blank node,
        then to the constraint component.
        """
        shape = g.value(res, SH.sourceShape)
        if shape is not None and not str(shape).startswith("n"):
            name = str(shape).rsplit("/", 1)[-1]
            if name and not name.startswith("N"):
                return name
        path = g.value(res, SH.resultPath)
        if path is not None:
            owner = None
            for s_, p_, o_ in g.triples((None, SH.property, shape)):
                owner = s_
                break
            base = str(path).rsplit("/", 1)[-1].rsplit("#", 1)[-1]
            if owner is not None and not str(owner).startswith("n"):
                return "%s / %s" % (str(owner).rsplit("/", 1)[-1], base)
            return "path %s" % base
        comp = g.value(res, SH.sourceConstraintComponent)
        if comp is not None:
            return str(comp).rsplit("#", 1)[-1]
        return "(unnamed shape)"

    rows = []
    for res in results.subjects(RDF.type, SH.ValidationResult):
        sev = str(results.value(res, SH.resultSeverity)).rsplit("#", 1)[-1]
        focus = str(results.value(res, SH.focusNode)).rsplit("/", 1)[-1]
        msg = str(results.value(res, SH.resultMessage) or "")
        rows.append((sev, shape_label(results, res, SH), focus, msg))

    violations = sum(1 for r in rows if r[0] == "Violation")
    warnings = len(rows) - violations

    # ---- report -----------------------------------------------------------
    print(BAR)
    print("PR META MODEL - SHACL VALIDATION")
    print("  data     : " + ", ".join(rel(f) for f in data_files))
    print("  shapes   : " + rel(SHAPES))
    print("  ontology : " + (rel(ONTOLOGY) if onto is not None else "(absent - no entailment)"))
    print("  triples  : %d data / %d shapes" % (len(data), len(shapes)))
    print(BAR)
    print("CONFORMS : %s" % conforms)
    print("RESULTS  : %d total  |  %d BLOCKING  |  %d MAJOR (warning)"
          % (len(rows), violations, warnings))

    if rows and not quiet:
        print("-" * W)
        by_shape = defaultdict(list)
        for sev, src, focus, msg in rows:
            by_shape[(sev, src)].append((focus, msg))
        # blocking first, then by volume: the rules to fix first come first
        order = sorted(by_shape.items(),
                       key=lambda kv: (kv[0][0] != "Violation", -len(kv[1])))
        for (sev, src), items in order:
            tag = "BLOCK" if sev == "Violation" else "WARN "
            print("[%s] %-42s %3d node(s)" % (tag, src or "(unnamed shape)", len(items)))
            print("        %s" % (items[0][1][:66] or "(no message)"))
            for focus, _ in items[:6]:
                print("          - %s" % focus)
            if len(items) > 6:
                print("          ... and %d more" % (len(items) - 6))

    print(BAR)
    sys.exit(0 if violations == 0 else 1)


if __name__ == "__main__":
    main()