#!/usr/bin/env python3 """ Migrate the Pernod Ricard Data MetaModel and the SODH instances to v2.2. USAGE python3 migrate_tbox_v2_2.py --ontology ... --instances ... python3 migrate_tbox_v2_2.py --ontology ... --instances ... --apply v2.2 turns the actor nodes into governance ASSIGNMENTS: a stable post carrying an identifier, a domain and a name, plus the person holding it as a value. It completes the twelve that existed, creates the ten sub-domain owners, the product owner, and one steward assignment per business object. DESIGN EV-004 dry run is the default EV-005 every edit goes through rdflib EV-006 guards test the target state, so a replay reports zero change EV-015 an execution log is written for every attempt """ import argparse import datetime import hashlib import json import os import re import sys from collections import OrderedDict import yaml from rdflib import Graph, Literal, Namespace, RDF, RDFS, OWL, URIRef HERE = os.path.dirname(os.path.abspath(__file__)) SPEC = os.path.join(HERE, "assignments_v2_2.yaml") class Report(object): def __init__(self): self.steps = OrderedDict() self.notes = [] def add(self, step, n=1, detail=None): self.steps[step] = self.steps.get(step, 0) + n if detail: self.notes.append("%s: %s" % (step, detail)) @property def total(self): return sum(self.steps.values()) def derive_label(name): """TN-018 and TN-023: a label is derived from the local name, never typed.""" spaced = re.sub(r"(?<=[a-z0-9])(?=[A-Z])|(?<=[A-Z])(?=[A-Z][a-z])", " ", name) return spaced[0].lower() + spaced[1:] def md5(path): h = hashlib.md5() with open(path, "rb") as fh: for chunk in iter(lambda: fh.read(65536), b""): h.update(chunk) return h.hexdigest() def rename_node(g, old, new, report, step): """EV-001 applied to an instance: rewrite every triple naming the node.""" if (new, None, None) in g and (old, None, None) not in g: return 0 n = 0 for s, p, o in list(g): ns, no = (new if s == old else s), (new if o == old else o) if (ns, no) != (s, o): g.remove((s, p, o)) g.add((ns, p, no)) n += 1 report.add(step, n) return n def assign(g, pr, node, role, identifier, name, held_by, domain, spec, report, step): """Write one assignment. The guard tests the identifier, so a replay is a no-op.""" if (node, pr.identifier, Literal(identifier)) in g: return False g.add((node, RDF.type, role)) g.remove((node, pr.identifier, None)) g.add((node, pr.identifier, Literal(identifier))) g.remove((node, pr.canonicalName, None)) g.add((node, pr.canonicalName, Literal(name))) g.remove((node, pr.personName, None)) if held_by: g.add((node, pr.personName, Literal(held_by))) g.remove((node, pr.status, None)) g.add((node, pr.status, Literal(spec["status"]))) g.remove((node, pr.version, None)) g.add((node, pr.version, Literal(spec["version"]))) if domain is not None: g.remove((node, pr.ownedByDomain, None)) g.add((node, pr.ownedByDomain, domain)) report.add(step, 1) return True def check_version(g, spec, path): """EV-010 applied to a migration script: refuse to run against the wrong input. A script announcing 2.1 -> 2.2 will happily work on a file still in 2.0 and report success, leaving a state that is neither version. The same silent failure the shapes are guarded against, one layer down. """ ns = spec["meta"]["namespace"] expected = URIRef(ns.rstrip("/") + "/" + spec["meta"]["from_version"]) target = URIRef(ns.rstrip("/") + "/" + spec["meta"]["to_version"]) found = set(g.objects(None, OWL.versionIRI)) if target in found: print("ALREADY AT %s — nothing to do." % spec["meta"]["to_version"]) sys.exit(0) if expected not in found: print("ABORT — %s declares %s; this script migrates from %s." % (os.path.basename(path), ", ".join(sorted(str(f) for f in found)) or "no version", expected)) sys.exit(1) def migrate(onto, data, spec, report): ns, ins = spec["meta"]["namespace"], spec["meta"]["instance_namespace"] pr, ex = Namespace(ns), Namespace(ins) # 1 — declare heldBy on Actor (procedure C1) for d in spec.get("declare_properties") or []: term = pr[d["term"]] if (term, RDF.type, getattr(OWL, d["kind"])) not in onto: onto.add((term, RDF.type, getattr(OWL, d["kind"]))) onto.add((term, RDFS.label, Literal(derive_label(d["term"])))) onto.add((term, RDFS.domain, pr[d["domain"]])) onto.add((term, RDFS.range, URIRef(d["range"]))) onto.add((term, RDFS.comment, Literal(" ".join(d["comment"].split())))) report.add("properties_declared", 1, d["term"]) # 2 — the short form of Data Steward is DST, and the rulebook now says so for u in spec.get("display_updates") or []: term = pr[u["term"]] if (term, pr.acronym, Literal(u["acronym"])) not in onto: onto.remove((term, pr.acronym, None)) onto.add((term, pr.acronym, Literal(u["acronym"]))) report.add("display_updated", 1, "%s -> %s" % (u["term"], u["acronym"])) # 3 — rename the instance nodes that carried the wrong short form for r in spec.get("instance_renames") or []: rename_node(data, ex[r["from"]], ex[r["to"]], report, "instances_renamed") # 4 — complete the assignments that already existed for c in spec.get("complete") or []: node = ex[c["node"]] if (node, None, None) not in data: report.add("complete_missing", 1, c["node"]) continue role = next(iter(data.objects(node, RDF.type)), None) assign(data, pr, node, role, c["identifier"], c["name"], c["held_by"], ex[c["domain"]], spec, report, "assignments_completed") # 5 — sub-domain owners sdo_of = {} for s in spec.get("create_sub_domain_owners") or []: node = ex[s["node"]] sdo_of[ex[s["sub_domain"]]] = node if assign(data, pr, node, pr.DataSubDomainOwner, s["identifier"], spec["sub_domain_owner_name"], s["held_by"], ex[s["domain"]], spec, report, "sub_domain_owners_created"): data.add((ex[s["sub_domain"]], pr.hasSubDomainOwner, node)) # 6 — product owners for p in spec.get("create_product_owners") or []: node = ex[p["node"]] product = ex[p["product"]] if (product, None, None) not in data: report.add("product_missing", 1, p["product"]) continue if assign(data, pr, node, pr.DataProductOwner, p["identifier"], spec["product_owner_name"], p["held_by"], ex[p["domain"]], spec, report, "product_owners_created"): data.add((product, pr.hasProductOwner, node)) # 7 — create the business objects that close a governance chain for b in spec.get("create_business_objects") or []: node = ex[b["node"]] if (node, pr.identifier, Literal(b["identifier"])) in data: continue data.add((node, RDF.type, pr.BusinessObject)) data.add((node, pr.identifier, Literal(b["identifier"]))) data.add((node, pr.canonicalName, Literal(b["name"]))) data.add((node, pr.isAbout, ex[b["is_about"]])) data.add((node, pr.belongsTo, ex[b["sub_domain"]])) data.add((node, pr.ownedByDomain, ex[b["domain"]])) data.add((node, pr.status, Literal(spec["status"]))) data.add((node, pr.version, Literal(spec["version"]))) report.add("business_objects_created", 1, b["node"]) # 8 — one steward assignment per business object, derived from the graph holders = spec.get("steward_holders") or {} steward_of = {} for bo in sorted(data.subjects(RDF.type, pr.BusinessObject), key=str): ident = next((str(v) for v in data.objects(bo, pr.identifier)), None) if not ident: report.add("steward_no_identifier", 1, str(bo).replace(ins, "")) continue suffix = ident.split("-", 1)[1] # 06.01-001 domain_key = "DD_" + suffix.split(".", 1)[0] # DD_06 holder = holders.get(domain_key) if holder is None: report.add("steward_no_holder", 1, domain_key) continue node = ex["DST_" + suffix.replace(".", "_").replace("-", "_")] steward_of[bo] = node if assign(data, pr, node, pr.DataSteward, "DST-" + suffix, spec["steward_name"], holder, ex[domain_key], spec, report, "stewards_created"): for _, _, old in list(data.triples((bo, pr.monitoredBy, None))): data.remove((bo, pr.monitoredBy, old)) data.add((bo, pr.monitoredBy, node)) # 9 — resolve every ownedBy pointing at a per-domain steward node res = spec.get("owned_by_resolution") or {} old_stewards = {ex[r["from"]] for r in (spec.get("instance_renames") or [])} | \ {ex[r["to"]] for r in (spec.get("instance_renames") or [])} def business_object_of(subject): """Follow the declared chains until a business object is reached.""" for chain in res.get("chains") or []: prop = pr[chain["via"]] if chain["direction"] == "inbound": for candidate in data.subjects(prop, subject): if (candidate, RDF.type, pr.BusinessObject) in data: return candidate else: for target in data.objects(subject, prop): if (target, RDF.type, pr.BusinessObject) in data: return target for candidate in data.subjects(pr.isAbout, target): if (candidate, RDF.type, pr.BusinessObject) in data: return candidate return None remove_on = {pr[c] for c in (res.get("remove_on") or [])} for subject, _, target in list(data.triples((None, pr.ownedBy, None))): if target not in old_stewards: continue types = set(data.objects(subject, RDF.type)) if types & remove_on: data.remove((subject, pr.ownedBy, target)) report.add("ownedby_removed", 1) continue if pr.DataSubDomain in types and res.get("repoint_sub_domains"): owner = sdo_of.get(subject) if owner is None: report.add("ownedby_unresolved", 1, "%s has no sub-domain owner" % str(subject).replace(ins, "")) continue data.remove((subject, pr.ownedBy, target)) data.add((subject, pr.ownedBy, owner)) report.add("ownedby_repointed_to_sdo", 1) continue bo = business_object_of(subject) owner = steward_of.get(bo) if bo is not None else None if owner is None: report.add("ownedby_unresolved", 1, "%s reaches no business object" % str(subject).replace(ins, "")) continue data.remove((subject, pr.ownedBy, target)) data.add((subject, pr.ownedBy, owner)) report.add("ownedby_repointed_to_steward", 1) # 10 — withdraw the per-domain steward nodes, once nothing names them (EV-003) if spec.get("retire_domain_stewards"): for r in spec.get("instance_renames") or []: node = ex[r["to"]] referrers = {s for s in data.subjects(None, node)} if referrers: report.add("steward_still_referenced", 1, "%s cited by %d subject(s)" % (r["to"], len(referrers))) continue n = 0 for t in list(data.triples((node, None, None))): data.remove(t) n += 1 if n: report.add("domain_stewards_retired", 1, r["to"]) # 11 — bump the version target = URIRef(ns.rstrip("/") + "/" + spec["meta"]["to_version"]) for o in set(onto.subjects(RDF.type, OWL.Ontology)): if (o, OWL.versionIRI, target) not in onto: onto.remove((o, OWL.versionIRI, None)) onto.add((o, OWL.versionIRI, target)) report.add("version_bumped", 1, str(target)) def main(): ap = argparse.ArgumentParser() ap.add_argument("--apply", action="store_true") ap.add_argument("--ontology", required=True) ap.add_argument("--instances", required=True) ap.add_argument("--log-dir", default=os.path.join(HERE, "logs")) args = ap.parse_args() for path in (args.ontology, args.instances): if not os.path.exists(path): print("MISSING INPUT: %s" % path) sys.exit(1) spec = yaml.safe_load(open(SPEC, encoding="utf-8")) checksums = OrderedDict((p, md5(p)) for p in (args.ontology, args.instances)) onto, data = Graph(), Graph() onto.parse(args.ontology, format="turtle") data.parse(args.instances, format="turtle") check_version(onto, spec, args.ontology) before = (len(onto), len(data)) report = Report() migrate(onto, data, spec, report) after = (len(onto), len(data)) print("MIGRATION %s -> %s %s" % (spec["meta"]["from_version"], spec["meta"]["to_version"], "APPLY" if args.apply else "DRY RUN")) print() for step, n in report.steps.items(): print(" %-28s %6d" % (step, n)) print(" %-28s %6d" % ("total changes", report.total)) print() for path, b, a in zip((args.ontology, args.instances), before, after): print(" %-46s %6d -> %6d triples" % (os.path.basename(path), b, a)) if report.notes: print() for n in report.notes: print(" note: " + n) os.makedirs(args.log_dir, exist_ok=True) stamp = datetime.datetime.now().strftime("%Y%m%dT%H%M%S") log_path = os.path.join(args.log_dir, "migration_v2_2_%s.json" % stamp) json.dump({"attempt_timestamp": datetime.datetime.now().isoformat(timespec="seconds"), "mode": "apply" if args.apply else "dry-run", "from_version": spec["meta"]["from_version"], "to_version": spec["meta"]["to_version"], "input_checksums": checksums, "steps": report.steps, "total_changes": report.total, "triples_before": dict(zip((args.ontology, args.instances), before)), "triples_after": dict(zip((args.ontology, args.instances), after)), "notes": report.notes}, open(log_path, "w", encoding="utf-8"), indent=2) print("\n log: %s" % log_path) if not args.apply: print("\n DRY RUN — nothing written. Re-run with --apply when the counts " "above are what you expect.") return onto.serialize(destination=args.ontology, format="turtle") data.serialize(destination=args.instances, format="turtle") print("\n written. Replay this script now: a second run must report zero " "changes (EV-006).") if __name__ == "__main__": main()