Bastien Gourdon
285 lines
11 KiB
Python
285 lines
11 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
PR Data Meta Model - SHACL validation runner
|
|
=============================================
|
|
The automated gate behind rule DQ-011. Validates an instance graph against the
|
|
executable rulebook (SHACL shapes), with the T-Box supplying subclass
|
|
entailment.
|
|
|
|
USAGE
|
|
python3 scripts/run_shacl_validation.py # all instances
|
|
python3 scripts/run_shacl_validation.py instances/sodh.ttl # one file
|
|
python3 scripts/run_shacl_validation.py --quiet # CI mode
|
|
|
|
Paths resolve against the repository root (the parent of scripts/), so the
|
|
script runs correctly from anywhere.
|
|
|
|
EXIT CODES
|
|
0 conforms, no blocking violation -> eligible for PUBLISHED
|
|
1 violations found -> stays DRAFT / UNDER_REVIEW
|
|
2 the run itself is not trustworthy -> a green result would be meaningless
|
|
|
|
Exit code 2 is the important one. A validation gate that passes because no
|
|
shape matched anything is worse than one that fails: it reports success while
|
|
checking nothing. The pre-flight below refuses to let that happen.
|
|
|
|
DEPENDENCIES
|
|
pyshacl, rdflib. On GrosseBertha these need a venv built with the STABLE
|
|
python (never the DSM system python, which moves between DSM releases):
|
|
|
|
/usr/local/bin/python3.9 -m venv venv
|
|
. venv/bin/activate
|
|
pip install pyshacl rdflib
|
|
"""
|
|
import glob
|
|
import os
|
|
import sys
|
|
from collections import defaultdict
|
|
|
|
REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
|
ONTOLOGY = os.path.join(REPO, "ontology", "pr_metamodel.ttl")
|
|
SHAPES = os.path.join(REPO, "shapes", "pr_metamodel_shapes.ttl")
|
|
INSTANCES = os.path.join(REPO, "instances")
|
|
|
|
W = 78
|
|
BAR = "=" * W
|
|
|
|
|
|
def rel(path):
|
|
"""Path relative to the repo root, for readable output."""
|
|
try:
|
|
return os.path.relpath(path, REPO)
|
|
except ValueError:
|
|
return path
|
|
|
|
|
|
def die(code, title, lines):
|
|
print(BAR)
|
|
print("PR META MODEL - SHACL VALIDATION")
|
|
print(BAR)
|
|
print(title)
|
|
print("-" * W)
|
|
for line in lines:
|
|
print(" " + line)
|
|
print(BAR)
|
|
sys.exit(code)
|
|
|
|
|
|
def load_deps():
|
|
"""
|
|
Import the validation stack.
|
|
|
|
Catches every exception, not just ImportError: a dependency that installs
|
|
but cannot be imported (pyshacl built for a newer Python than the venv,
|
|
an rdflib major-version clash) raises TypeError or AttributeError at import
|
|
time. Letting those escape would end the process with an exit code CI reads
|
|
as "the model has violations", when in fact validation never started.
|
|
Anything wrong here is exit 2 -- the run is not trustworthy.
|
|
"""
|
|
try:
|
|
from pyshacl import validate
|
|
from rdflib import Graph
|
|
from rdflib.namespace import RDF, SH
|
|
return validate, Graph, RDF, SH
|
|
except Exception as exc:
|
|
hint = []
|
|
if "unsupported operand type" in str(exc) or "|" in str(exc):
|
|
hint = [
|
|
"This is PEP 604 syntax (str | X) evaluated at import time:",
|
|
"the installed pyshacl needs Python 3.10+, this venv is %d.%d."
|
|
% (sys.version_info[0], sys.version_info[1]),
|
|
"pyshacl advertises 3.9 support but its code does not honour it,",
|
|
"so pip installs a version that cannot be imported.",
|
|
"",
|
|
"Pin a release that really supports this interpreter:",
|
|
' pip install "pyshacl==0.26.0" "rdflib>=6.3.2,<7.1"',
|
|
"or rebuild the venv on a newer Python if one exists.",
|
|
]
|
|
else:
|
|
hint = [
|
|
"Build the venv with the stable python, never the DSM system one:",
|
|
" /usr/local/bin/python3.9 -m venv venv",
|
|
" . venv/bin/activate && pip install pyshacl rdflib",
|
|
]
|
|
die(2, "DEPENDENCY UNUSABLE -- validation did not run",
|
|
["%s: %s" % (type(exc).__name__, exc), ""] + hint)
|
|
|
|
|
|
def term_namespaces(graph, predicate=None):
|
|
"""
|
|
Namespaces actually used by pr: terms in a graph, with usage counts.
|
|
|
|
Only real IRIs count. Literals are skipped: SPARQL constraint bodies
|
|
(sh:select) embed their own PREFIX declarations, and counting those would
|
|
put raw query text in the diagnostic table.
|
|
"""
|
|
counts = defaultdict(int)
|
|
for s, p, o in graph:
|
|
for term in (s, p, o):
|
|
text = str(term)
|
|
if "ontology.pernod-ricard.com/metamodel" not in text:
|
|
continue
|
|
if not text.startswith("http") or any(c in text for c in " \t\n\r<>\""):
|
|
continue # literal, not an IRI
|
|
counts[text.rsplit("/", 1)[0] + "/"] += 1
|
|
return counts
|
|
|
|
|
|
def preflight(data, shapes, onto, data_files):
|
|
"""
|
|
Refuse to produce a green result that means nothing.
|
|
|
|
Two ways a SHACL run passes while checking nothing:
|
|
1. the shapes target classes in a namespace the instances never use;
|
|
2. the ontology declares its classes in a third namespace, so subclass
|
|
entailment cannot reach the instances and every shape targeting an
|
|
abstract parent (MetaModelObject) finds zero focus nodes.
|
|
Both look identical to a caller reading only "CONFORMS: True".
|
|
"""
|
|
problems = []
|
|
ns_problem = False
|
|
|
|
if not len(data):
|
|
problems.append("The instance graph is empty -- nothing to validate.")
|
|
|
|
ns_data = term_namespaces(data)
|
|
ns_shapes = term_namespaces(shapes)
|
|
ns_onto = term_namespaces(onto) if onto is not None else {}
|
|
|
|
if ns_data and ns_shapes and not (set(ns_data) & set(ns_shapes)):
|
|
ns_problem = True
|
|
problems.append(
|
|
"Shapes and instances use disjoint term namespaces -- no shape can "
|
|
"match any node.")
|
|
|
|
if ns_onto and ns_data and not (set(ns_onto) & set(ns_data)):
|
|
ns_problem = True
|
|
problems.append(
|
|
"The T-Box declares its terms in a namespace the instances never "
|
|
"use. Subclass entailment cannot reach the data, so every shape "
|
|
"targeting an abstract parent finds zero focus nodes.")
|
|
|
|
if problems:
|
|
lines = list(problems)
|
|
lines.append("")
|
|
lines.append("Namespaces in use:")
|
|
for label, counts in (("instances", ns_data), ("shapes", ns_shapes),
|
|
("ontology", ns_onto)):
|
|
if counts:
|
|
for ns, n in sorted(counts.items(), key=lambda kv: -kv[1]):
|
|
lines.append(" %-10s %-52s %6d refs" % (label, ns, n))
|
|
else:
|
|
lines.append(" %-10s (none)" % label)
|
|
if ns_problem:
|
|
lines.append("")
|
|
lines.append("Fix: align the namespaces before trusting any result.")
|
|
lines.append(" sed -i 's|/metamodel/0.9/|/metamodel/|g' "
|
|
"instances/*.ttl shapes/*.ttl")
|
|
die(2, "PRE-FLIGHT FAILED -- a result here would be meaningless", lines)
|
|
|
|
|
|
def main():
|
|
args = [a for a in sys.argv[1:] if not a.startswith("-")]
|
|
quiet = "--quiet" in sys.argv or "-q" in sys.argv
|
|
|
|
validate, Graph, RDF, SH = load_deps()
|
|
|
|
# ---- resolve inputs ---------------------------------------------------
|
|
if args:
|
|
data_files = [a if os.path.isabs(a) else os.path.join(REPO, a) for a in args]
|
|
else:
|
|
data_files = sorted(glob.glob(os.path.join(INSTANCES, "*.ttl")))
|
|
|
|
missing = [f for f in data_files if not os.path.exists(f)]
|
|
if missing or not data_files:
|
|
die(2, "NO INSTANCE DATA -- validation did not run",
|
|
[rel(f) + " not found" for f in missing] or
|
|
["instances/ contains no .ttl file"])
|
|
|
|
if not os.path.exists(SHAPES):
|
|
die(2, "NO SHAPES -- validation did not run",
|
|
[rel(SHAPES) + " not found"])
|
|
|
|
# ---- parse ------------------------------------------------------------
|
|
data = Graph()
|
|
for f in data_files:
|
|
try:
|
|
data.parse(f, format="turtle")
|
|
except Exception as exc:
|
|
die(2, "PARSE ERROR -- validation did not run",
|
|
[rel(f), str(exc)])
|
|
|
|
try:
|
|
shapes = Graph().parse(SHAPES, format="turtle")
|
|
except Exception as exc:
|
|
die(2, "PARSE ERROR in shapes -- validation did not run",
|
|
[rel(SHAPES), str(exc)])
|
|
|
|
onto = None
|
|
if os.path.exists(ONTOLOGY):
|
|
try:
|
|
onto = Graph().parse(ONTOLOGY, format="turtle")
|
|
except Exception as exc:
|
|
die(2, "PARSE ERROR in ontology -- validation did not run",
|
|
[rel(ONTOLOGY), str(exc)])
|
|
|
|
preflight(data, shapes, onto, data_files)
|
|
|
|
# ---- validate ---------------------------------------------------------
|
|
conforms, results, _ = validate(
|
|
data_graph=data,
|
|
shacl_graph=shapes,
|
|
ont_graph=onto,
|
|
inference="rdfs", # a Metric is also a MetaModelObject
|
|
advanced=True, # SPARQL constraints (NR-002, OW-007, DQ-006)
|
|
abort_on_first=False,
|
|
meta_shacl=False,
|
|
)
|
|
|
|
rows = []
|
|
for res in results.subjects(RDF.type, SH.ValidationResult):
|
|
sev = str(results.value(res, SH.resultSeverity)).rsplit("#", 1)[-1]
|
|
focus = str(results.value(res, SH.focusNode)).rsplit("/", 1)[-1]
|
|
msg = str(results.value(res, SH.resultMessage) or "")
|
|
src = str(results.value(res, SH.sourceShape) or "").rsplit("/", 1)[-1]
|
|
rows.append((sev, src, focus, msg))
|
|
|
|
violations = sum(1 for r in rows if r[0] == "Violation")
|
|
warnings = len(rows) - violations
|
|
|
|
# ---- report -----------------------------------------------------------
|
|
print(BAR)
|
|
print("PR META MODEL - SHACL VALIDATION")
|
|
print(" data : " + ", ".join(rel(f) for f in data_files))
|
|
print(" shapes : " + rel(SHAPES))
|
|
print(" ontology : " + (rel(ONTOLOGY) if onto is not None else "(absent - no entailment)"))
|
|
print(" triples : %d data / %d shapes" % (len(data), len(shapes)))
|
|
print(BAR)
|
|
print("CONFORMS : %s" % conforms)
|
|
print("RESULTS : %d total | %d BLOCKING | %d MAJOR (warning)"
|
|
% (len(rows), violations, warnings))
|
|
|
|
if rows and not quiet:
|
|
print("-" * W)
|
|
by_shape = defaultdict(list)
|
|
for sev, src, focus, msg in rows:
|
|
by_shape[(sev, src)].append((focus, msg))
|
|
# blocking first, then by volume: the rules to fix first come first
|
|
order = sorted(by_shape.items(),
|
|
key=lambda kv: (kv[0][0] != "Violation", -len(kv[1])))
|
|
for (sev, src), items in order:
|
|
tag = "BLOCK" if sev == "Violation" else "WARN "
|
|
print("[%s] %-42s %3d node(s)" % (tag, src or "(unnamed shape)", len(items)))
|
|
print(" %s" % (items[0][1][:66] or "(no message)"))
|
|
for focus, _ in items[:6]:
|
|
print(" - %s" % focus)
|
|
if len(items) > 6:
|
|
print(" ... and %d more" % (len(items) - 6))
|
|
|
|
print(BAR)
|
|
sys.exit(0 if violations == 0 else 1)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|