tbox: v1.1 - belongsTo unifie, implements devient represents, aboutConcept/usesConcept remplacent composedOf, monitoredBy sur BO seul, grain deplace du Metric au DataObject, couche Delivery et DataInterface. shapes v0.2 scindees curated/observed avec XOR de sens sur DataElement. instances SODH migrees et arbitrees : 12 BO, 12 BC, 6 concepts empruntes en TO_ARBITRATE. Validation : 52 bloquants, dont 52 leves par la migration BR-013 restante
This commit is contained in:
Bastien Gourdon
@@ -0,0 +1,284 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
PR Data Meta Model - SHACL validation runner
|
||||
=============================================
|
||||
The automated gate behind rule DQ-011. Validates an instance graph against the
|
||||
executable rulebook (SHACL shapes), with the T-Box supplying subclass
|
||||
entailment.
|
||||
|
||||
USAGE
|
||||
python3 scripts/run_shacl_validation.py # all instances
|
||||
python3 scripts/run_shacl_validation.py instances/sodh.ttl # one file
|
||||
python3 scripts/run_shacl_validation.py --quiet # CI mode
|
||||
|
||||
Paths resolve against the repository root (the parent of scripts/), so the
|
||||
script runs correctly from anywhere.
|
||||
|
||||
EXIT CODES
|
||||
0 conforms, no blocking violation -> eligible for PUBLISHED
|
||||
1 violations found -> stays DRAFT / UNDER_REVIEW
|
||||
2 the run itself is not trustworthy -> a green result would be meaningless
|
||||
|
||||
Exit code 2 is the important one. A validation gate that passes because no
|
||||
shape matched anything is worse than one that fails: it reports success while
|
||||
checking nothing. The pre-flight below refuses to let that happen.
|
||||
|
||||
DEPENDENCIES
|
||||
pyshacl, rdflib. On GrosseBertha these need a venv built with the STABLE
|
||||
python (never the DSM system python, which moves between DSM releases):
|
||||
|
||||
/usr/local/bin/python3.9 -m venv venv
|
||||
. venv/bin/activate
|
||||
pip install pyshacl rdflib
|
||||
"""
|
||||
import glob
|
||||
import os
|
||||
import sys
|
||||
from collections import defaultdict
|
||||
|
||||
REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
ONTOLOGY = os.path.join(REPO, "ontology", "pr_metamodel.ttl")
|
||||
SHAPES = os.path.join(REPO, "shapes", "pr_metamodel_shapes.ttl")
|
||||
INSTANCES = os.path.join(REPO, "instances")
|
||||
|
||||
W = 78
|
||||
BAR = "=" * W
|
||||
|
||||
|
||||
def rel(path):
|
||||
"""Path relative to the repo root, for readable output."""
|
||||
try:
|
||||
return os.path.relpath(path, REPO)
|
||||
except ValueError:
|
||||
return path
|
||||
|
||||
|
||||
def die(code, title, lines):
|
||||
print(BAR)
|
||||
print("PR META MODEL - SHACL VALIDATION")
|
||||
print(BAR)
|
||||
print(title)
|
||||
print("-" * W)
|
||||
for line in lines:
|
||||
print(" " + line)
|
||||
print(BAR)
|
||||
sys.exit(code)
|
||||
|
||||
|
||||
def load_deps():
|
||||
"""
|
||||
Import the validation stack.
|
||||
|
||||
Catches every exception, not just ImportError: a dependency that installs
|
||||
but cannot be imported (pyshacl built for a newer Python than the venv,
|
||||
an rdflib major-version clash) raises TypeError or AttributeError at import
|
||||
time. Letting those escape would end the process with an exit code CI reads
|
||||
as "the model has violations", when in fact validation never started.
|
||||
Anything wrong here is exit 2 -- the run is not trustworthy.
|
||||
"""
|
||||
try:
|
||||
from pyshacl import validate
|
||||
from rdflib import Graph
|
||||
from rdflib.namespace import RDF, SH
|
||||
return validate, Graph, RDF, SH
|
||||
except Exception as exc:
|
||||
hint = []
|
||||
if "unsupported operand type" in str(exc) or "|" in str(exc):
|
||||
hint = [
|
||||
"This is PEP 604 syntax (str | X) evaluated at import time:",
|
||||
"the installed pyshacl needs Python 3.10+, this venv is %d.%d."
|
||||
% (sys.version_info[0], sys.version_info[1]),
|
||||
"pyshacl advertises 3.9 support but its code does not honour it,",
|
||||
"so pip installs a version that cannot be imported.",
|
||||
"",
|
||||
"Pin a release that really supports this interpreter:",
|
||||
' pip install "pyshacl==0.26.0" "rdflib>=6.3.2,<7.1"',
|
||||
"or rebuild the venv on a newer Python if one exists.",
|
||||
]
|
||||
else:
|
||||
hint = [
|
||||
"Build the venv with the stable python, never the DSM system one:",
|
||||
" /usr/local/bin/python3.9 -m venv venv",
|
||||
" . venv/bin/activate && pip install pyshacl rdflib",
|
||||
]
|
||||
die(2, "DEPENDENCY UNUSABLE -- validation did not run",
|
||||
["%s: %s" % (type(exc).__name__, exc), ""] + hint)
|
||||
|
||||
|
||||
def term_namespaces(graph, predicate=None):
|
||||
"""
|
||||
Namespaces actually used by pr: terms in a graph, with usage counts.
|
||||
|
||||
Only real IRIs count. Literals are skipped: SPARQL constraint bodies
|
||||
(sh:select) embed their own PREFIX declarations, and counting those would
|
||||
put raw query text in the diagnostic table.
|
||||
"""
|
||||
counts = defaultdict(int)
|
||||
for s, p, o in graph:
|
||||
for term in (s, p, o):
|
||||
text = str(term)
|
||||
if "ontology.pernod-ricard.com/metamodel" not in text:
|
||||
continue
|
||||
if not text.startswith("http") or any(c in text for c in " \t\n\r<>\""):
|
||||
continue # literal, not an IRI
|
||||
counts[text.rsplit("/", 1)[0] + "/"] += 1
|
||||
return counts
|
||||
|
||||
|
||||
def preflight(data, shapes, onto, data_files):
|
||||
"""
|
||||
Refuse to produce a green result that means nothing.
|
||||
|
||||
Two ways a SHACL run passes while checking nothing:
|
||||
1. the shapes target classes in a namespace the instances never use;
|
||||
2. the ontology declares its classes in a third namespace, so subclass
|
||||
entailment cannot reach the instances and every shape targeting an
|
||||
abstract parent (MetaModelObject) finds zero focus nodes.
|
||||
Both look identical to a caller reading only "CONFORMS: True".
|
||||
"""
|
||||
problems = []
|
||||
ns_problem = False
|
||||
|
||||
if not len(data):
|
||||
problems.append("The instance graph is empty -- nothing to validate.")
|
||||
|
||||
ns_data = term_namespaces(data)
|
||||
ns_shapes = term_namespaces(shapes)
|
||||
ns_onto = term_namespaces(onto) if onto is not None else {}
|
||||
|
||||
if ns_data and ns_shapes and not (set(ns_data) & set(ns_shapes)):
|
||||
ns_problem = True
|
||||
problems.append(
|
||||
"Shapes and instances use disjoint term namespaces -- no shape can "
|
||||
"match any node.")
|
||||
|
||||
if ns_onto and ns_data and not (set(ns_onto) & set(ns_data)):
|
||||
ns_problem = True
|
||||
problems.append(
|
||||
"The T-Box declares its terms in a namespace the instances never "
|
||||
"use. Subclass entailment cannot reach the data, so every shape "
|
||||
"targeting an abstract parent finds zero focus nodes.")
|
||||
|
||||
if problems:
|
||||
lines = list(problems)
|
||||
lines.append("")
|
||||
lines.append("Namespaces in use:")
|
||||
for label, counts in (("instances", ns_data), ("shapes", ns_shapes),
|
||||
("ontology", ns_onto)):
|
||||
if counts:
|
||||
for ns, n in sorted(counts.items(), key=lambda kv: -kv[1]):
|
||||
lines.append(" %-10s %-52s %6d refs" % (label, ns, n))
|
||||
else:
|
||||
lines.append(" %-10s (none)" % label)
|
||||
if ns_problem:
|
||||
lines.append("")
|
||||
lines.append("Fix: align the namespaces before trusting any result.")
|
||||
lines.append(" sed -i 's|/metamodel/0.9/|/metamodel/|g' "
|
||||
"instances/*.ttl shapes/*.ttl")
|
||||
die(2, "PRE-FLIGHT FAILED -- a result here would be meaningless", lines)
|
||||
|
||||
|
||||
def main():
|
||||
args = [a for a in sys.argv[1:] if not a.startswith("-")]
|
||||
quiet = "--quiet" in sys.argv or "-q" in sys.argv
|
||||
|
||||
validate, Graph, RDF, SH = load_deps()
|
||||
|
||||
# ---- resolve inputs ---------------------------------------------------
|
||||
if args:
|
||||
data_files = [a if os.path.isabs(a) else os.path.join(REPO, a) for a in args]
|
||||
else:
|
||||
data_files = sorted(glob.glob(os.path.join(INSTANCES, "*.ttl")))
|
||||
|
||||
missing = [f for f in data_files if not os.path.exists(f)]
|
||||
if missing or not data_files:
|
||||
die(2, "NO INSTANCE DATA -- validation did not run",
|
||||
[rel(f) + " not found" for f in missing] or
|
||||
["instances/ contains no .ttl file"])
|
||||
|
||||
if not os.path.exists(SHAPES):
|
||||
die(2, "NO SHAPES -- validation did not run",
|
||||
[rel(SHAPES) + " not found"])
|
||||
|
||||
# ---- parse ------------------------------------------------------------
|
||||
data = Graph()
|
||||
for f in data_files:
|
||||
try:
|
||||
data.parse(f, format="turtle")
|
||||
except Exception as exc:
|
||||
die(2, "PARSE ERROR -- validation did not run",
|
||||
[rel(f), str(exc)])
|
||||
|
||||
try:
|
||||
shapes = Graph().parse(SHAPES, format="turtle")
|
||||
except Exception as exc:
|
||||
die(2, "PARSE ERROR in shapes -- validation did not run",
|
||||
[rel(SHAPES), str(exc)])
|
||||
|
||||
onto = None
|
||||
if os.path.exists(ONTOLOGY):
|
||||
try:
|
||||
onto = Graph().parse(ONTOLOGY, format="turtle")
|
||||
except Exception as exc:
|
||||
die(2, "PARSE ERROR in ontology -- validation did not run",
|
||||
[rel(ONTOLOGY), str(exc)])
|
||||
|
||||
preflight(data, shapes, onto, data_files)
|
||||
|
||||
# ---- validate ---------------------------------------------------------
|
||||
conforms, results, _ = validate(
|
||||
data_graph=data,
|
||||
shacl_graph=shapes,
|
||||
ont_graph=onto,
|
||||
inference="rdfs", # a Metric is also a MetaModelObject
|
||||
advanced=True, # SPARQL constraints (NR-002, OW-007, DQ-006)
|
||||
abort_on_first=False,
|
||||
meta_shacl=False,
|
||||
)
|
||||
|
||||
rows = []
|
||||
for res in results.subjects(RDF.type, SH.ValidationResult):
|
||||
sev = str(results.value(res, SH.resultSeverity)).rsplit("#", 1)[-1]
|
||||
focus = str(results.value(res, SH.focusNode)).rsplit("/", 1)[-1]
|
||||
msg = str(results.value(res, SH.resultMessage) or "")
|
||||
src = str(results.value(res, SH.sourceShape) or "").rsplit("/", 1)[-1]
|
||||
rows.append((sev, src, focus, msg))
|
||||
|
||||
violations = sum(1 for r in rows if r[0] == "Violation")
|
||||
warnings = len(rows) - violations
|
||||
|
||||
# ---- report -----------------------------------------------------------
|
||||
print(BAR)
|
||||
print("PR META MODEL - SHACL VALIDATION")
|
||||
print(" data : " + ", ".join(rel(f) for f in data_files))
|
||||
print(" shapes : " + rel(SHAPES))
|
||||
print(" ontology : " + (rel(ONTOLOGY) if onto is not None else "(absent - no entailment)"))
|
||||
print(" triples : %d data / %d shapes" % (len(data), len(shapes)))
|
||||
print(BAR)
|
||||
print("CONFORMS : %s" % conforms)
|
||||
print("RESULTS : %d total | %d BLOCKING | %d MAJOR (warning)"
|
||||
% (len(rows), violations, warnings))
|
||||
|
||||
if rows and not quiet:
|
||||
print("-" * W)
|
||||
by_shape = defaultdict(list)
|
||||
for sev, src, focus, msg in rows:
|
||||
by_shape[(sev, src)].append((focus, msg))
|
||||
# blocking first, then by volume: the rules to fix first come first
|
||||
order = sorted(by_shape.items(),
|
||||
key=lambda kv: (kv[0][0] != "Violation", -len(kv[1])))
|
||||
for (sev, src), items in order:
|
||||
tag = "BLOCK" if sev == "Violation" else "WARN "
|
||||
print("[%s] %-42s %3d node(s)" % (tag, src or "(unnamed shape)", len(items)))
|
||||
print(" %s" % (items[0][1][:66] or "(no message)"))
|
||||
for focus, _ in items[:6]:
|
||||
print(" - %s" % focus)
|
||||
if len(items) > 6:
|
||||
print(" ... and %d more" % (len(items) - 6))
|
||||
|
||||
print(BAR)
|
||||
sys.exit(0 if violations == 0 else 1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user