297 lines
11 KiB
Python
297 lines
11 KiB
Python
|
#!/usr/bin/env python3
|
||
|
|
"""
|
||
|
|
Migrate the Pernod Ricard Data MetaModel from v2.0 to v2.1.
|
||
|
|
|
||
|
|
USAGE
|
||
|
|
python3 migrate_tbox_v2_1.py # dry run, writes nothing
|
||
|
|
python3 migrate_tbox_v2_1.py --apply
|
||
|
|
|
||
|
|
v2.1 renames nothing. It declares what v2.0 left silent: which properties are
|
||
|
|
polymorphic on purpose, what domain the others take, and how the instances of
|
||
|
|
each concrete class are produced.
|
||
|
|
|
||
|
|
DESIGN
|
||
|
|
EV-004 dry run is the default
|
||
|
|
EV-005 every edit goes through rdflib
|
||
|
|
EV-006 guards test the target state, so a replay reports zero change
|
||
|
|
EV-015 an execution log is written for every attempt
|
||
|
|
"""
|
||
|
|
import argparse
|
||
|
|
import datetime
|
||
|
|
import hashlib
|
||
|
|
import json
|
||
|
|
import os
|
||
|
|
import re
|
||
|
|
import sys
|
||
|
|
from collections import OrderedDict
|
||
|
|
|
||
|
|
import yaml
|
||
|
|
from rdflib import Graph, Literal, Namespace, RDF, RDFS, OWL, URIRef
|
||
|
|
|
||
|
|
HERE = os.path.dirname(os.path.abspath(__file__))
|
||
|
|
SPEC = os.path.join(HERE, "declarations_v2_1.yaml")
|
||
|
|
|
||
|
|
|
||
|
|
class Report(object):
|
||
|
|
def __init__(self):
|
||
|
|
self.steps = OrderedDict()
|
||
|
|
self.notes = []
|
||
|
|
|
||
|
|
def add(self, step, n=1, detail=None):
|
||
|
|
self.steps[step] = self.steps.get(step, 0) + n
|
||
|
|
if detail:
|
||
|
|
self.notes.append("%s: %s" % (step, detail))
|
||
|
|
|
||
|
|
@property
|
||
|
|
def total(self):
|
||
|
|
return sum(self.steps.values())
|
||
|
|
|
||
|
|
|
||
|
|
def md5(path):
|
||
|
|
h = hashlib.md5()
|
||
|
|
with open(path, "rb") as fh:
|
||
|
|
for chunk in iter(lambda: fh.read(65536), b""):
|
||
|
|
h.update(chunk)
|
||
|
|
return h.hexdigest()
|
||
|
|
|
||
|
|
|
||
|
|
def derive_label(name):
|
||
|
|
"""TN-018 and TN-023: a label is derived from the local name, never typed."""
|
||
|
|
spaced = re.sub(r"(?<=[a-z0-9])(?=[A-Z])|(?<=[A-Z])(?=[A-Z][a-z])", " ", name)
|
||
|
|
return spaced[0].lower() + spaced[1:]
|
||
|
|
|
||
|
|
|
||
|
|
def local(uri, ns):
|
||
|
|
s = str(uri)
|
||
|
|
return s[len(ns):] if s.startswith(ns) else None
|
||
|
|
|
||
|
|
|
||
|
|
def ancestors(g, term):
|
||
|
|
seen, stack = set(), [term]
|
||
|
|
while stack:
|
||
|
|
node = stack.pop()
|
||
|
|
for parent in g.objects(node, RDFS.subClassOf):
|
||
|
|
if isinstance(parent, URIRef) and parent not in seen:
|
||
|
|
seen.add(parent)
|
||
|
|
stack.append(parent)
|
||
|
|
return seen
|
||
|
|
|
||
|
|
|
||
|
|
def check_version(g, spec, path):
|
||
|
|
"""EV-010 applied to a migration script: refuse to run against the wrong input.
|
||
|
|
|
||
|
|
A script announcing 2.1 -> 2.2 will happily work on a file still in 2.0 and
|
||
|
|
report success, leaving a state that is neither version. The same silent
|
||
|
|
failure the shapes are guarded against, one layer down.
|
||
|
|
"""
|
||
|
|
ns = spec["meta"]["namespace"]
|
||
|
|
expected = URIRef(ns.rstrip("/") + "/" + spec["meta"]["from_version"])
|
||
|
|
target = URIRef(ns.rstrip("/") + "/" + spec["meta"]["to_version"])
|
||
|
|
found = set(g.objects(None, OWL.versionIRI))
|
||
|
|
if target in found:
|
||
|
|
print("ALREADY AT %s — nothing to do." % spec["meta"]["to_version"])
|
||
|
|
sys.exit(0)
|
||
|
|
if expected not in found:
|
||
|
|
print("ABORT — %s declares %s; this script migrates from %s."
|
||
|
|
% (os.path.basename(path),
|
||
|
|
", ".join(sorted(str(f) for f in found)) or "no version",
|
||
|
|
expected))
|
||
|
|
sys.exit(1)
|
||
|
|
|
||
|
|
|
||
|
|
def migrate(g, instances, spec, report):
|
||
|
|
ns = spec["meta"]["namespace"]
|
||
|
|
pr = Namespace(ns)
|
||
|
|
|
||
|
|
# 1 — declare the annotation property itself
|
||
|
|
for a in spec.get("declare_annotations") or []:
|
||
|
|
term = pr[a["term"]]
|
||
|
|
if (term, RDF.type, OWL.AnnotationProperty) not in g:
|
||
|
|
g.add((term, RDF.type, OWL.AnnotationProperty))
|
||
|
|
g.add((term, RDFS.label, Literal(a["label"])))
|
||
|
|
g.add((term, RDFS.comment, Literal(" ".join(a["comment"].split()))))
|
||
|
|
report.add("annotations_declared", 1, a["term"])
|
||
|
|
|
||
|
|
# 2 — declare the new properties (procedure C1)
|
||
|
|
for d in spec.get("declare_properties") or []:
|
||
|
|
term = pr[d["term"]]
|
||
|
|
kind = getattr(OWL, d["kind"])
|
||
|
|
if (term, RDF.type, kind) not in g:
|
||
|
|
g.add((term, RDF.type, kind))
|
||
|
|
g.add((term, RDFS.label, Literal(derive_label(d["term"]))))
|
||
|
|
g.add((term, RDFS.domain, pr[d["domain"]]))
|
||
|
|
g.add((term, RDFS.range, URIRef(d["range"])))
|
||
|
|
g.add((term, RDFS.comment, Literal(" ".join(d["comment"].split()))))
|
||
|
|
report.add("properties_declared", 1, d["term"])
|
||
|
|
|
||
|
|
# 3 — mark the deliberately polymorphic properties (TN-025)
|
||
|
|
for p in spec.get("polymorphic") or []:
|
||
|
|
term = pr[p["term"]]
|
||
|
|
if (term, None, None) not in g:
|
||
|
|
report.add("polymorphic_missing", 1, p["term"])
|
||
|
|
continue
|
||
|
|
if (term, pr.polymorphic, Literal(True)) not in g:
|
||
|
|
g.add((term, pr.polymorphic, Literal(True)))
|
||
|
|
report.add("polymorphic_declared", 1)
|
||
|
|
if list(g.objects(term, RDFS.domain)):
|
||
|
|
report.add("polymorphic_has_domain", 1,
|
||
|
|
"%s is marked polymorphic and still declares a domain" % p["term"])
|
||
|
|
|
||
|
|
# 4 — declare the domains that were merely missing (TN-025)
|
||
|
|
for d in spec.get("domains") or []:
|
||
|
|
term, target = pr[d["term"]], pr[d["domain"]]
|
||
|
|
if (term, None, None) not in g:
|
||
|
|
report.add("domain_missing_term", 1, d["term"])
|
||
|
|
continue
|
||
|
|
if (target, RDF.type, OWL.Class) not in g:
|
||
|
|
report.add("domain_missing_class", 1, "%s -> %s" % (d["term"], d["domain"]))
|
||
|
|
continue
|
||
|
|
if (term, RDFS.domain, target) not in g:
|
||
|
|
g.remove((term, RDFS.domain, None))
|
||
|
|
g.add((term, RDFS.domain, target))
|
||
|
|
report.add("domains_declared", 1)
|
||
|
|
|
||
|
|
# 5 — ranges, where a polymorphic property still points at one class
|
||
|
|
for r in spec.get("ranges") or []:
|
||
|
|
term, target = pr[r["term"]], pr[r["range"]]
|
||
|
|
if (term, RDFS.range, target) not in g:
|
||
|
|
g.remove((term, RDFS.range, None))
|
||
|
|
g.add((term, RDFS.range, target))
|
||
|
|
report.add("ranges_declared", 1)
|
||
|
|
|
||
|
|
# 6 — attach a layer root to its provenance axis (TN-027)
|
||
|
|
for a in spec.get("attach") or []:
|
||
|
|
term, parent = pr[a["term"]], pr[a["parent"]]
|
||
|
|
if (term, RDFS.subClassOf, parent) not in g:
|
||
|
|
g.add((term, RDFS.subClassOf, parent))
|
||
|
|
report.add("attached", 1, "%s -> %s" % (a["term"], a["parent"]))
|
||
|
|
|
||
|
|
# 7 — provenance belongs to the concrete classes, and to nothing else (TN-026)
|
||
|
|
prov = spec.get("provenance") or {}
|
||
|
|
concrete = {c for c in g.subjects(RDF.type, OWL.Class)
|
||
|
|
if (c, pr.isAbstract, Literal(False)) in g}
|
||
|
|
|
||
|
|
if prov.get("strip_from_non_concrete"):
|
||
|
|
for s in set(g.subjects(pr.authoringMode, None)):
|
||
|
|
if s in concrete:
|
||
|
|
continue
|
||
|
|
g.remove((s, pr.authoringMode, None))
|
||
|
|
g.remove((s, pr.harvestSource, None))
|
||
|
|
report.add("provenance_stripped", 1, local(s, ns))
|
||
|
|
|
||
|
|
if prov.get("derive_from_axis"):
|
||
|
|
for cls in sorted(concrete, key=str):
|
||
|
|
up = ancestors(g, cls)
|
||
|
|
if pr.CapturedObject in up:
|
||
|
|
want = "HARVESTED"
|
||
|
|
elif pr.DefinedObject in up:
|
||
|
|
want = "ASSERTED"
|
||
|
|
else:
|
||
|
|
report.add("provenance_no_axis", 1, local(cls, ns))
|
||
|
|
continue
|
||
|
|
if (cls, pr.authoringMode, Literal(want)) in g:
|
||
|
|
continue
|
||
|
|
g.remove((cls, pr.authoringMode, None))
|
||
|
|
g.add((cls, pr.authoringMode, Literal(want)))
|
||
|
|
report.add("provenance_declared", 1)
|
||
|
|
|
||
|
|
# 8 — instance corrections, each one surfaced by declaring a domain
|
||
|
|
ex = Namespace("https://data.pernod-ricard.com/sodh/")
|
||
|
|
for fix in spec.get("instance_fixes") or []:
|
||
|
|
prop = pr[fix["property"]]
|
||
|
|
dest = pr[fix["to"]] if fix.get("to") else None
|
||
|
|
for subject in fix["subjects"]:
|
||
|
|
node = ex[subject]
|
||
|
|
for gi in instances:
|
||
|
|
for _, _, value in list(gi.triples((node, prop, None))):
|
||
|
|
gi.remove((node, prop, value))
|
||
|
|
if dest is not None:
|
||
|
|
gi.add((node, dest, value))
|
||
|
|
report.add("instances_%sd" % fix["action"], 1,
|
||
|
|
"%s %s" % (subject, fix["property"]))
|
||
|
|
|
||
|
|
# 9 — bump the version (EV-014: the namespace itself never moves)
|
||
|
|
target = URIRef(ns.rstrip("/") + "/" + spec["meta"]["to_version"])
|
||
|
|
for onto in set(g.subjects(RDF.type, OWL.Ontology)):
|
||
|
|
if (onto, OWL.versionIRI, target) not in g:
|
||
|
|
g.remove((onto, OWL.versionIRI, None))
|
||
|
|
g.add((onto, OWL.versionIRI, target))
|
||
|
|
report.add("version_bumped", 1, str(target))
|
||
|
|
|
||
|
|
|
||
|
|
def main():
|
||
|
|
ap = argparse.ArgumentParser()
|
||
|
|
ap.add_argument("--apply", action="store_true")
|
||
|
|
ap.add_argument("--ontology", default=os.path.join(HERE, "..", "..", "ontology",
|
||
|
|
"pr_metamodel.ttl"))
|
||
|
|
ap.add_argument("--instances", action="append", default=[])
|
||
|
|
ap.add_argument("--log-dir", default=os.path.join(HERE, "logs"))
|
||
|
|
args = ap.parse_args()
|
||
|
|
|
||
|
|
if not os.path.exists(args.ontology):
|
||
|
|
print("MISSING INPUT: %s" % args.ontology)
|
||
|
|
sys.exit(1)
|
||
|
|
|
||
|
|
spec = yaml.safe_load(open(SPEC, encoding="utf-8"))
|
||
|
|
checksums = OrderedDict((p, md5(p)) for p in [args.ontology] + args.instances)
|
||
|
|
|
||
|
|
g = Graph()
|
||
|
|
g.parse(args.ontology, format="turtle")
|
||
|
|
instances = []
|
||
|
|
for path in args.instances:
|
||
|
|
gi = Graph()
|
||
|
|
gi.parse(path, format="turtle")
|
||
|
|
instances.append(gi)
|
||
|
|
check_version(g, spec, args.ontology)
|
||
|
|
|
||
|
|
before = [len(g)] + [len(x) for x in instances]
|
||
|
|
|
||
|
|
report = Report()
|
||
|
|
migrate(g, instances, spec, report)
|
||
|
|
after = [len(g)] + [len(x) for x in instances]
|
||
|
|
|
||
|
|
print("MIGRATION %s -> %s %s"
|
||
|
|
% (spec["meta"]["from_version"], spec["meta"]["to_version"],
|
||
|
|
"APPLY" if args.apply else "DRY RUN"))
|
||
|
|
print()
|
||
|
|
for step, n in report.steps.items():
|
||
|
|
print(" %-28s %6d" % (step, n))
|
||
|
|
print(" %-28s %6d" % ("total changes", report.total))
|
||
|
|
print()
|
||
|
|
for path, b, a in zip([args.ontology] + args.instances, before, after):
|
||
|
|
print(" %-46s %6d -> %6d triples" % (os.path.basename(path), b, a))
|
||
|
|
if report.notes:
|
||
|
|
print()
|
||
|
|
for n in report.notes:
|
||
|
|
print(" note: " + n)
|
||
|
|
|
||
|
|
os.makedirs(args.log_dir, exist_ok=True)
|
||
|
|
stamp = datetime.datetime.now().strftime("%Y%m%dT%H%M%S")
|
||
|
|
log_path = os.path.join(args.log_dir, "migration_v2_1_%s.json" % stamp)
|
||
|
|
json.dump({"attempt_timestamp": datetime.datetime.now().isoformat(timespec="seconds"),
|
||
|
|
"mode": "apply" if args.apply else "dry-run",
|
||
|
|
"from_version": spec["meta"]["from_version"],
|
||
|
|
"to_version": spec["meta"]["to_version"],
|
||
|
|
"input_checksums": checksums,
|
||
|
|
"steps": report.steps, "total_changes": report.total,
|
||
|
|
"triples_before": dict(zip([args.ontology] + args.instances, before)),
|
||
|
|
"triples_after": dict(zip([args.ontology] + args.instances, after)),
|
||
|
|
"notes": report.notes},
|
||
|
|
open(log_path, "w", encoding="utf-8"), indent=2)
|
||
|
|
print("\n log: %s" % log_path)
|
||
|
|
|
||
|
|
if not args.apply:
|
||
|
|
print("\n DRY RUN — nothing written. Re-run with --apply when the counts "
|
||
|
|
"above are what you expect.")
|
||
|
|
return
|
||
|
|
|
||
|
|
g.serialize(destination=args.ontology, format="turtle")
|
||
|
|
for path, gi in zip(args.instances, instances):
|
||
|
|
gi.serialize(destination=path, format="turtle")
|
||
|
|
print("\n written. Replay this script now: a second run must report zero "
|
||
|
|
"changes (EV-006).")
|
||
|
|
|
||
|
|
|
||
|
|
if __name__ == "__main__":
|
||
|
|
main()
|