#!/usr/bin/env python3 """ Migrate the Pernod Ricard Data MetaModel from v2.0 to v2.1. USAGE python3 migrate_tbox_v2_1.py # dry run, writes nothing python3 migrate_tbox_v2_1.py --apply v2.1 renames nothing. It declares what v2.0 left silent: which properties are polymorphic on purpose, what domain the others take, and how the instances of each concrete class are produced. DESIGN EV-004 dry run is the default EV-005 every edit goes through rdflib EV-006 guards test the target state, so a replay reports zero change EV-015 an execution log is written for every attempt """ import argparse import datetime import hashlib import json import os import re import sys from collections import OrderedDict import yaml from rdflib import Graph, Literal, Namespace, RDF, RDFS, OWL, URIRef HERE = os.path.dirname(os.path.abspath(__file__)) SPEC = os.path.join(HERE, "declarations_v2_1.yaml") class Report(object): def __init__(self): self.steps = OrderedDict() self.notes = [] def add(self, step, n=1, detail=None): self.steps[step] = self.steps.get(step, 0) + n if detail: self.notes.append("%s: %s" % (step, detail)) @property def total(self): return sum(self.steps.values()) def md5(path): h = hashlib.md5() with open(path, "rb") as fh: for chunk in iter(lambda: fh.read(65536), b""): h.update(chunk) return h.hexdigest() def derive_label(name): """TN-018 and TN-023: a label is derived from the local name, never typed.""" spaced = re.sub(r"(?<=[a-z0-9])(?=[A-Z])|(?<=[A-Z])(?=[A-Z][a-z])", " ", name) return spaced[0].lower() + spaced[1:] def local(uri, ns): s = str(uri) return s[len(ns):] if s.startswith(ns) else None def ancestors(g, term): seen, stack = set(), [term] while stack: node = stack.pop() for parent in g.objects(node, RDFS.subClassOf): if isinstance(parent, URIRef) and parent not in seen: seen.add(parent) stack.append(parent) return seen def check_version(g, spec, path): """EV-010 applied to a migration script: refuse to run against the wrong input. A script announcing 2.1 -> 2.2 will happily work on a file still in 2.0 and report success, leaving a state that is neither version. The same silent failure the shapes are guarded against, one layer down. """ ns = spec["meta"]["namespace"] expected = URIRef(ns.rstrip("/") + "/" + spec["meta"]["from_version"]) target = URIRef(ns.rstrip("/") + "/" + spec["meta"]["to_version"]) found = set(g.objects(None, OWL.versionIRI)) if target in found: print("ALREADY AT %s — nothing to do." % spec["meta"]["to_version"]) sys.exit(0) if expected not in found: print("ABORT — %s declares %s; this script migrates from %s." % (os.path.basename(path), ", ".join(sorted(str(f) for f in found)) or "no version", expected)) sys.exit(1) def migrate(g, instances, spec, report): ns = spec["meta"]["namespace"] pr = Namespace(ns) # 1 — declare the annotation property itself for a in spec.get("declare_annotations") or []: term = pr[a["term"]] if (term, RDF.type, OWL.AnnotationProperty) not in g: g.add((term, RDF.type, OWL.AnnotationProperty)) g.add((term, RDFS.label, Literal(a["label"]))) g.add((term, RDFS.comment, Literal(" ".join(a["comment"].split())))) report.add("annotations_declared", 1, a["term"]) # 2 — declare the new properties (procedure C1) for d in spec.get("declare_properties") or []: term = pr[d["term"]] kind = getattr(OWL, d["kind"]) if (term, RDF.type, kind) not in g: g.add((term, RDF.type, kind)) g.add((term, RDFS.label, Literal(derive_label(d["term"])))) g.add((term, RDFS.domain, pr[d["domain"]])) g.add((term, RDFS.range, URIRef(d["range"]))) g.add((term, RDFS.comment, Literal(" ".join(d["comment"].split())))) report.add("properties_declared", 1, d["term"]) # 3 — mark the deliberately polymorphic properties (TN-025) for p in spec.get("polymorphic") or []: term = pr[p["term"]] if (term, None, None) not in g: report.add("polymorphic_missing", 1, p["term"]) continue if (term, pr.polymorphic, Literal(True)) not in g: g.add((term, pr.polymorphic, Literal(True))) report.add("polymorphic_declared", 1) if list(g.objects(term, RDFS.domain)): report.add("polymorphic_has_domain", 1, "%s is marked polymorphic and still declares a domain" % p["term"]) # 4 — declare the domains that were merely missing (TN-025) for d in spec.get("domains") or []: term, target = pr[d["term"]], pr[d["domain"]] if (term, None, None) not in g: report.add("domain_missing_term", 1, d["term"]) continue if (target, RDF.type, OWL.Class) not in g: report.add("domain_missing_class", 1, "%s -> %s" % (d["term"], d["domain"])) continue if (term, RDFS.domain, target) not in g: g.remove((term, RDFS.domain, None)) g.add((term, RDFS.domain, target)) report.add("domains_declared", 1) # 5 — ranges, where a polymorphic property still points at one class for r in spec.get("ranges") or []: term, target = pr[r["term"]], pr[r["range"]] if (term, RDFS.range, target) not in g: g.remove((term, RDFS.range, None)) g.add((term, RDFS.range, target)) report.add("ranges_declared", 1) # 6 — attach a layer root to its provenance axis (TN-027) for a in spec.get("attach") or []: term, parent = pr[a["term"]], pr[a["parent"]] if (term, RDFS.subClassOf, parent) not in g: g.add((term, RDFS.subClassOf, parent)) report.add("attached", 1, "%s -> %s" % (a["term"], a["parent"])) # 7 — provenance belongs to the concrete classes, and to nothing else (TN-026) prov = spec.get("provenance") or {} concrete = {c for c in g.subjects(RDF.type, OWL.Class) if (c, pr.isAbstract, Literal(False)) in g} if prov.get("strip_from_non_concrete"): for s in set(g.subjects(pr.authoringMode, None)): if s in concrete: continue g.remove((s, pr.authoringMode, None)) g.remove((s, pr.harvestSource, None)) report.add("provenance_stripped", 1, local(s, ns)) if prov.get("derive_from_axis"): for cls in sorted(concrete, key=str): up = ancestors(g, cls) if pr.CapturedObject in up: want = "HARVESTED" elif pr.DefinedObject in up: want = "ASSERTED" else: report.add("provenance_no_axis", 1, local(cls, ns)) continue if (cls, pr.authoringMode, Literal(want)) in g: continue g.remove((cls, pr.authoringMode, None)) g.add((cls, pr.authoringMode, Literal(want))) report.add("provenance_declared", 1) # 8 — instance corrections, each one surfaced by declaring a domain ex = Namespace("https://data.pernod-ricard.com/sodh/") for fix in spec.get("instance_fixes") or []: prop = pr[fix["property"]] dest = pr[fix["to"]] if fix.get("to") else None for subject in fix["subjects"]: node = ex[subject] for gi in instances: for _, _, value in list(gi.triples((node, prop, None))): gi.remove((node, prop, value)) if dest is not None: gi.add((node, dest, value)) report.add("instances_%sd" % fix["action"], 1, "%s %s" % (subject, fix["property"])) # 9 — bump the version (EV-014: the namespace itself never moves) target = URIRef(ns.rstrip("/") + "/" + spec["meta"]["to_version"]) for onto in set(g.subjects(RDF.type, OWL.Ontology)): if (onto, OWL.versionIRI, target) not in g: g.remove((onto, OWL.versionIRI, None)) g.add((onto, OWL.versionIRI, target)) report.add("version_bumped", 1, str(target)) def main(): ap = argparse.ArgumentParser() ap.add_argument("--apply", action="store_true") ap.add_argument("--ontology", default=os.path.join(HERE, "..", "..", "ontology", "pr_metamodel.ttl")) ap.add_argument("--instances", action="append", default=[]) ap.add_argument("--log-dir", default=os.path.join(HERE, "logs")) args = ap.parse_args() if not os.path.exists(args.ontology): print("MISSING INPUT: %s" % args.ontology) sys.exit(1) spec = yaml.safe_load(open(SPEC, encoding="utf-8")) checksums = OrderedDict((p, md5(p)) for p in [args.ontology] + args.instances) g = Graph() g.parse(args.ontology, format="turtle") instances = [] for path in args.instances: gi = Graph() gi.parse(path, format="turtle") instances.append(gi) check_version(g, spec, args.ontology) before = [len(g)] + [len(x) for x in instances] report = Report() migrate(g, instances, spec, report) after = [len(g)] + [len(x) for x in instances] print("MIGRATION %s -> %s %s" % (spec["meta"]["from_version"], spec["meta"]["to_version"], "APPLY" if args.apply else "DRY RUN")) print() for step, n in report.steps.items(): print(" %-28s %6d" % (step, n)) print(" %-28s %6d" % ("total changes", report.total)) print() for path, b, a in zip([args.ontology] + args.instances, before, after): print(" %-46s %6d -> %6d triples" % (os.path.basename(path), b, a)) if report.notes: print() for n in report.notes: print(" note: " + n) os.makedirs(args.log_dir, exist_ok=True) stamp = datetime.datetime.now().strftime("%Y%m%dT%H%M%S") log_path = os.path.join(args.log_dir, "migration_v2_1_%s.json" % stamp) json.dump({"attempt_timestamp": datetime.datetime.now().isoformat(timespec="seconds"), "mode": "apply" if args.apply else "dry-run", "from_version": spec["meta"]["from_version"], "to_version": spec["meta"]["to_version"], "input_checksums": checksums, "steps": report.steps, "total_changes": report.total, "triples_before": dict(zip([args.ontology] + args.instances, before)), "triples_after": dict(zip([args.ontology] + args.instances, after)), "notes": report.notes}, open(log_path, "w", encoding="utf-8"), indent=2) print("\n log: %s" % log_path) if not args.apply: print("\n DRY RUN — nothing written. Re-run with --apply when the counts " "above are what you expect.") return g.serialize(destination=args.ontology, format="turtle") for path, gi in zip(args.instances, instances): gi.serialize(destination=path, format="turtle") print("\n written. Replay this script now: a second run must report zero " "changes (EV-006).") if __name__ == "__main__": main()