tbox: v2.2 - polymorphisme declare sur 19 proprietes, 17 domaines poses, provenance portee par les classes concretes, GovernanceLayerObject rattachee, affectations de gouvernance completes (12 completees, 10 SDO, 1 PO, 16 stewards par objet metier), Panel Coverage gouverne. Validation SHACL sans violation bloquante
This commit is contained in:
Bastien Gourdon
@@ -0,0 +1,296 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Migrate the Pernod Ricard Data MetaModel from v2.0 to v2.1.
|
||||
|
||||
USAGE
|
||||
python3 migrate_tbox_v2_1.py # dry run, writes nothing
|
||||
python3 migrate_tbox_v2_1.py --apply
|
||||
|
||||
v2.1 renames nothing. It declares what v2.0 left silent: which properties are
|
||||
polymorphic on purpose, what domain the others take, and how the instances of
|
||||
each concrete class are produced.
|
||||
|
||||
DESIGN
|
||||
EV-004 dry run is the default
|
||||
EV-005 every edit goes through rdflib
|
||||
EV-006 guards test the target state, so a replay reports zero change
|
||||
EV-015 an execution log is written for every attempt
|
||||
"""
|
||||
import argparse
|
||||
import datetime
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
from collections import OrderedDict
|
||||
|
||||
import yaml
|
||||
from rdflib import Graph, Literal, Namespace, RDF, RDFS, OWL, URIRef
|
||||
|
||||
HERE = os.path.dirname(os.path.abspath(__file__))
|
||||
SPEC = os.path.join(HERE, "declarations_v2_1.yaml")
|
||||
|
||||
|
||||
class Report(object):
|
||||
def __init__(self):
|
||||
self.steps = OrderedDict()
|
||||
self.notes = []
|
||||
|
||||
def add(self, step, n=1, detail=None):
|
||||
self.steps[step] = self.steps.get(step, 0) + n
|
||||
if detail:
|
||||
self.notes.append("%s: %s" % (step, detail))
|
||||
|
||||
@property
|
||||
def total(self):
|
||||
return sum(self.steps.values())
|
||||
|
||||
|
||||
def md5(path):
|
||||
h = hashlib.md5()
|
||||
with open(path, "rb") as fh:
|
||||
for chunk in iter(lambda: fh.read(65536), b""):
|
||||
h.update(chunk)
|
||||
return h.hexdigest()
|
||||
|
||||
|
||||
def derive_label(name):
|
||||
"""TN-018 and TN-023: a label is derived from the local name, never typed."""
|
||||
spaced = re.sub(r"(?<=[a-z0-9])(?=[A-Z])|(?<=[A-Z])(?=[A-Z][a-z])", " ", name)
|
||||
return spaced[0].lower() + spaced[1:]
|
||||
|
||||
|
||||
def local(uri, ns):
|
||||
s = str(uri)
|
||||
return s[len(ns):] if s.startswith(ns) else None
|
||||
|
||||
|
||||
def ancestors(g, term):
|
||||
seen, stack = set(), [term]
|
||||
while stack:
|
||||
node = stack.pop()
|
||||
for parent in g.objects(node, RDFS.subClassOf):
|
||||
if isinstance(parent, URIRef) and parent not in seen:
|
||||
seen.add(parent)
|
||||
stack.append(parent)
|
||||
return seen
|
||||
|
||||
|
||||
def check_version(g, spec, path):
|
||||
"""EV-010 applied to a migration script: refuse to run against the wrong input.
|
||||
|
||||
A script announcing 2.1 -> 2.2 will happily work on a file still in 2.0 and
|
||||
report success, leaving a state that is neither version. The same silent
|
||||
failure the shapes are guarded against, one layer down.
|
||||
"""
|
||||
ns = spec["meta"]["namespace"]
|
||||
expected = URIRef(ns.rstrip("/") + "/" + spec["meta"]["from_version"])
|
||||
target = URIRef(ns.rstrip("/") + "/" + spec["meta"]["to_version"])
|
||||
found = set(g.objects(None, OWL.versionIRI))
|
||||
if target in found:
|
||||
print("ALREADY AT %s — nothing to do." % spec["meta"]["to_version"])
|
||||
sys.exit(0)
|
||||
if expected not in found:
|
||||
print("ABORT — %s declares %s; this script migrates from %s."
|
||||
% (os.path.basename(path),
|
||||
", ".join(sorted(str(f) for f in found)) or "no version",
|
||||
expected))
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
def migrate(g, instances, spec, report):
|
||||
ns = spec["meta"]["namespace"]
|
||||
pr = Namespace(ns)
|
||||
|
||||
# 1 — declare the annotation property itself
|
||||
for a in spec.get("declare_annotations") or []:
|
||||
term = pr[a["term"]]
|
||||
if (term, RDF.type, OWL.AnnotationProperty) not in g:
|
||||
g.add((term, RDF.type, OWL.AnnotationProperty))
|
||||
g.add((term, RDFS.label, Literal(a["label"])))
|
||||
g.add((term, RDFS.comment, Literal(" ".join(a["comment"].split()))))
|
||||
report.add("annotations_declared", 1, a["term"])
|
||||
|
||||
# 2 — declare the new properties (procedure C1)
|
||||
for d in spec.get("declare_properties") or []:
|
||||
term = pr[d["term"]]
|
||||
kind = getattr(OWL, d["kind"])
|
||||
if (term, RDF.type, kind) not in g:
|
||||
g.add((term, RDF.type, kind))
|
||||
g.add((term, RDFS.label, Literal(derive_label(d["term"]))))
|
||||
g.add((term, RDFS.domain, pr[d["domain"]]))
|
||||
g.add((term, RDFS.range, URIRef(d["range"])))
|
||||
g.add((term, RDFS.comment, Literal(" ".join(d["comment"].split()))))
|
||||
report.add("properties_declared", 1, d["term"])
|
||||
|
||||
# 3 — mark the deliberately polymorphic properties (TN-025)
|
||||
for p in spec.get("polymorphic") or []:
|
||||
term = pr[p["term"]]
|
||||
if (term, None, None) not in g:
|
||||
report.add("polymorphic_missing", 1, p["term"])
|
||||
continue
|
||||
if (term, pr.polymorphic, Literal(True)) not in g:
|
||||
g.add((term, pr.polymorphic, Literal(True)))
|
||||
report.add("polymorphic_declared", 1)
|
||||
if list(g.objects(term, RDFS.domain)):
|
||||
report.add("polymorphic_has_domain", 1,
|
||||
"%s is marked polymorphic and still declares a domain" % p["term"])
|
||||
|
||||
# 4 — declare the domains that were merely missing (TN-025)
|
||||
for d in spec.get("domains") or []:
|
||||
term, target = pr[d["term"]], pr[d["domain"]]
|
||||
if (term, None, None) not in g:
|
||||
report.add("domain_missing_term", 1, d["term"])
|
||||
continue
|
||||
if (target, RDF.type, OWL.Class) not in g:
|
||||
report.add("domain_missing_class", 1, "%s -> %s" % (d["term"], d["domain"]))
|
||||
continue
|
||||
if (term, RDFS.domain, target) not in g:
|
||||
g.remove((term, RDFS.domain, None))
|
||||
g.add((term, RDFS.domain, target))
|
||||
report.add("domains_declared", 1)
|
||||
|
||||
# 5 — ranges, where a polymorphic property still points at one class
|
||||
for r in spec.get("ranges") or []:
|
||||
term, target = pr[r["term"]], pr[r["range"]]
|
||||
if (term, RDFS.range, target) not in g:
|
||||
g.remove((term, RDFS.range, None))
|
||||
g.add((term, RDFS.range, target))
|
||||
report.add("ranges_declared", 1)
|
||||
|
||||
# 6 — attach a layer root to its provenance axis (TN-027)
|
||||
for a in spec.get("attach") or []:
|
||||
term, parent = pr[a["term"]], pr[a["parent"]]
|
||||
if (term, RDFS.subClassOf, parent) not in g:
|
||||
g.add((term, RDFS.subClassOf, parent))
|
||||
report.add("attached", 1, "%s -> %s" % (a["term"], a["parent"]))
|
||||
|
||||
# 7 — provenance belongs to the concrete classes, and to nothing else (TN-026)
|
||||
prov = spec.get("provenance") or {}
|
||||
concrete = {c for c in g.subjects(RDF.type, OWL.Class)
|
||||
if (c, pr.isAbstract, Literal(False)) in g}
|
||||
|
||||
if prov.get("strip_from_non_concrete"):
|
||||
for s in set(g.subjects(pr.authoringMode, None)):
|
||||
if s in concrete:
|
||||
continue
|
||||
g.remove((s, pr.authoringMode, None))
|
||||
g.remove((s, pr.harvestSource, None))
|
||||
report.add("provenance_stripped", 1, local(s, ns))
|
||||
|
||||
if prov.get("derive_from_axis"):
|
||||
for cls in sorted(concrete, key=str):
|
||||
up = ancestors(g, cls)
|
||||
if pr.CapturedObject in up:
|
||||
want = "HARVESTED"
|
||||
elif pr.DefinedObject in up:
|
||||
want = "ASSERTED"
|
||||
else:
|
||||
report.add("provenance_no_axis", 1, local(cls, ns))
|
||||
continue
|
||||
if (cls, pr.authoringMode, Literal(want)) in g:
|
||||
continue
|
||||
g.remove((cls, pr.authoringMode, None))
|
||||
g.add((cls, pr.authoringMode, Literal(want)))
|
||||
report.add("provenance_declared", 1)
|
||||
|
||||
# 8 — instance corrections, each one surfaced by declaring a domain
|
||||
ex = Namespace("https://data.pernod-ricard.com/sodh/")
|
||||
for fix in spec.get("instance_fixes") or []:
|
||||
prop = pr[fix["property"]]
|
||||
dest = pr[fix["to"]] if fix.get("to") else None
|
||||
for subject in fix["subjects"]:
|
||||
node = ex[subject]
|
||||
for gi in instances:
|
||||
for _, _, value in list(gi.triples((node, prop, None))):
|
||||
gi.remove((node, prop, value))
|
||||
if dest is not None:
|
||||
gi.add((node, dest, value))
|
||||
report.add("instances_%sd" % fix["action"], 1,
|
||||
"%s %s" % (subject, fix["property"]))
|
||||
|
||||
# 9 — bump the version (EV-014: the namespace itself never moves)
|
||||
target = URIRef(ns.rstrip("/") + "/" + spec["meta"]["to_version"])
|
||||
for onto in set(g.subjects(RDF.type, OWL.Ontology)):
|
||||
if (onto, OWL.versionIRI, target) not in g:
|
||||
g.remove((onto, OWL.versionIRI, None))
|
||||
g.add((onto, OWL.versionIRI, target))
|
||||
report.add("version_bumped", 1, str(target))
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--apply", action="store_true")
|
||||
ap.add_argument("--ontology", default=os.path.join(HERE, "..", "..", "ontology",
|
||||
"pr_metamodel.ttl"))
|
||||
ap.add_argument("--instances", action="append", default=[])
|
||||
ap.add_argument("--log-dir", default=os.path.join(HERE, "logs"))
|
||||
args = ap.parse_args()
|
||||
|
||||
if not os.path.exists(args.ontology):
|
||||
print("MISSING INPUT: %s" % args.ontology)
|
||||
sys.exit(1)
|
||||
|
||||
spec = yaml.safe_load(open(SPEC, encoding="utf-8"))
|
||||
checksums = OrderedDict((p, md5(p)) for p in [args.ontology] + args.instances)
|
||||
|
||||
g = Graph()
|
||||
g.parse(args.ontology, format="turtle")
|
||||
instances = []
|
||||
for path in args.instances:
|
||||
gi = Graph()
|
||||
gi.parse(path, format="turtle")
|
||||
instances.append(gi)
|
||||
check_version(g, spec, args.ontology)
|
||||
|
||||
before = [len(g)] + [len(x) for x in instances]
|
||||
|
||||
report = Report()
|
||||
migrate(g, instances, spec, report)
|
||||
after = [len(g)] + [len(x) for x in instances]
|
||||
|
||||
print("MIGRATION %s -> %s %s"
|
||||
% (spec["meta"]["from_version"], spec["meta"]["to_version"],
|
||||
"APPLY" if args.apply else "DRY RUN"))
|
||||
print()
|
||||
for step, n in report.steps.items():
|
||||
print(" %-28s %6d" % (step, n))
|
||||
print(" %-28s %6d" % ("total changes", report.total))
|
||||
print()
|
||||
for path, b, a in zip([args.ontology] + args.instances, before, after):
|
||||
print(" %-46s %6d -> %6d triples" % (os.path.basename(path), b, a))
|
||||
if report.notes:
|
||||
print()
|
||||
for n in report.notes:
|
||||
print(" note: " + n)
|
||||
|
||||
os.makedirs(args.log_dir, exist_ok=True)
|
||||
stamp = datetime.datetime.now().strftime("%Y%m%dT%H%M%S")
|
||||
log_path = os.path.join(args.log_dir, "migration_v2_1_%s.json" % stamp)
|
||||
json.dump({"attempt_timestamp": datetime.datetime.now().isoformat(timespec="seconds"),
|
||||
"mode": "apply" if args.apply else "dry-run",
|
||||
"from_version": spec["meta"]["from_version"],
|
||||
"to_version": spec["meta"]["to_version"],
|
||||
"input_checksums": checksums,
|
||||
"steps": report.steps, "total_changes": report.total,
|
||||
"triples_before": dict(zip([args.ontology] + args.instances, before)),
|
||||
"triples_after": dict(zip([args.ontology] + args.instances, after)),
|
||||
"notes": report.notes},
|
||||
open(log_path, "w", encoding="utf-8"), indent=2)
|
||||
print("\n log: %s" % log_path)
|
||||
|
||||
if not args.apply:
|
||||
print("\n DRY RUN — nothing written. Re-run with --apply when the counts "
|
||||
"above are what you expect.")
|
||||
return
|
||||
|
||||
g.serialize(destination=args.ontology, format="turtle")
|
||||
for path, gi in zip(args.instances, instances):
|
||||
gi.serialize(destination=path, format="turtle")
|
||||
print("\n written. Replay this script now: a second run must report zero "
|
||||
"changes (EV-006).")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user