#!/usr/bin/env python3 """ Migrate the Pernod Ricard Data MetaModel from v1.6 to v2.0. USAGE python3 migrate_tbox_v2_0.py # dry run, writes nothing python3 migrate_tbox_v2_0.py --apply # writes in place python3 migrate_tbox_v2_0.py --apply --log-dir logs/ --ontology path to the ontology TTL (default ../ontology/pr_metamodel.ttl) --instances path to an instance TTL (repeatable) --shapes path to a shapes TTL (repeatable) DESIGN EV-004 dry run is the default; nothing is written without --apply EV-005 every edit goes through rdflib, never a regex on the text EV-006 guards test the target state, so a replay reports zero change EV-015 an execution log is written for every attempt Run on GrosseBertha, inside the pinned venv: . venv/bin/activate && python3 migrate_tbox_v2_0.py """ import argparse import datetime import hashlib import json import os import re import sys from collections import OrderedDict import yaml from rdflib import Graph, Literal, Namespace, RDF, RDFS, OWL, URIRef, XSD from rdflib.namespace import SH HERE = os.path.dirname(os.path.abspath(__file__)) SPEC = os.path.join(HERE, "renames.yaml") RULES = os.path.normpath(os.path.join(HERE, "..", "rules.yaml")) DCTERMS = Namespace("http://purl.org/dc/terms/") # ----------------------------------------------------------------- utilities class Report(object): """Counts every step, so that idempotence is provable and not merely hoped.""" def __init__(self): self.steps = OrderedDict() self.notes = [] def add(self, step, n, detail=None): self.steps[step] = self.steps.get(step, 0) + n if detail: self.notes.append("%s: %s" % (step, detail)) @property def total(self): return sum(self.steps.values()) def md5(path): h = hashlib.md5() with open(path, "rb") as f: for chunk in iter(lambda: f.read(65536), b""): h.update(chunk) return h.hexdigest() def local(uri, ns): s = str(uri) return s[len(ns):] if s.startswith(ns) else None def decamelise(name, is_class): """TN-018. Consecutive capitals are kept together: they carry an acronym.""" spaced = re.sub(r"(?<=[a-z0-9])(?=[A-Z])|(?<=[A-Z])(?=[A-Z][a-z])", " ", name) return spaced if is_class else spaced[0].lower() + spaced[1:] # ------------------------------------------------------------------ the work def rename(graph, pr, old, new, report, step): """EV-001. Rewrite every triple naming the identifier, in all three positions.""" src, dst = pr[old], pr[new] if (dst, None, None) in graph and (src, None, None) not in graph: return 0 # EV-006: already done n = 0 for s, p, o in list(graph): ns, np_, no = s, p, o if s == src: ns = dst if p == src: np_ = dst if o == src: no = dst if (ns, np_, no) != (s, p, o): graph.remove((s, p, o)) graph.add((ns, np_, no)) n += 1 report.add(step, n) return n def merge(graph, pr, src_name, dst_name, report): """The source disappears into an existing target, which keeps its declaration.""" src, dst = pr[src_name], pr[dst_name] n = 0 for s, p, o in list(graph.triples((None, src, None))): graph.remove((s, p, o)) graph.add((s, dst, o)) n += 1 for s, p, o in list(graph.triples((src, None, None))): graph.remove((s, p, o)) # drop the source declaration n += 1 report.add("merge", n) return n def drop_subject(graph, subject, report, step): """EV-003. Remove the block AND every reference naming it, or inference rebuilds it.""" n = 0 for t in list(graph.triples((subject, None, None))): graph.remove(t) n += 1 for t in list(graph.triples((None, None, subject))): graph.remove(t) n += 1 for t in list(graph.triples((None, subject, None))): graph.remove(t) n += 1 report.add(step, n) return n def count_instances(graphs, term): """EV-002. The proof required before any permanent withdrawal. Counted on INSTANCE graphs only. A mention inside a shapes graph is not an instantiation: a shape carrying sh:maxCount 0 on a term is a guard forbidding its reappearance, which is the opposite of a use, and it keeps working after the term is gone since it checks an identifier rather than a declaration. """ n = 0 for g in graphs: n += len(list(g.triples((None, RDF.type, term)))) n += len(list(g.triples((None, term, None)))) n += len(list(g.triples((None, None, term)))) return n def rewrite_queries(graph, renames, report): """Rename inside embedded SPARQL queries. A sh:select or sh:ask body is a literal, so no triple-level rename reaches it: the query keeps naming a term that no longer exists and the constraint silently stops matching. This is not the line-by-line editing EV-005 forbids: the graph is parsed, the literal is retrieved as a value, and what is rewritten is a structured query language where a prefixed name has clear token boundaries. """ n = 0 for prop in (SH.select, SH.ask, SH.construct): for s, p, o in list(graph.triples((None, prop, None))): text = str(o) new = text for old, target in renames: new = re.sub(r"\bpr:%s\b" % re.escape(old), "pr:%s" % target, new) if new != text: graph.remove((s, p, o)) graph.add((s, p, Literal(new))) n += 1 report.add("queries_rewritten", n) return n def migrate(onto, instances, shapes, spec, rules, report): ns = spec["meta"]["namespace"] pr = Namespace(ns) others = instances + shapes all_graphs = [onto] + others # 1 — deprecated terms are withdrawn outright (phase clause), proof first. # The proof is taken on instance graphs only: a mention in a shapes graph # is a guard against reappearance, not a use. # Reference cleaning likewise stops at the ontology and the instances. # EV-003 guards against phantom nodes rebuilt by inference in the model # graph; a mention inside a shape is a constraint on an identifier, in a # separate graph, taking part in no inference over the ontology. Removing # it would strip the sh:path from a guard and leave a property shape # carrying a maxCount and no path — an invalid shape, and the loss of the # very rule that forbids the term from coming back. if spec["structural"].get("drop_deprecated"): for subj in list(onto.subjects(OWL.deprecated, Literal(True))): name = local(subj, ns) or str(subj) used = count_instances(instances, subj) if used: report.add("deprecated_kept", 1, "%s instantiated %d times" % (name, used)) continue for g in [onto] + instances: drop_subject(g, subj, report, "deprecated_dropped") # 2 — vocabulary renames, in dependency order for order in (1, 2, 3): for r in [x for x in spec["renames"] if x["order"] == order]: for g in all_graphs: rename(g, pr, r["from"], r["to"], report, "rename_order_%d" % order) # 3 — instance-side alignment: forms the instances use that the vocabulary # never declared and that the TN-012 renames do not converge on. # Applied to the shapes too: they target these forms in sh:path. for r in spec.get("abox_renames") or []: for g in all_graphs: rename(g, pr, r["from"], r["to"], report, "abox_aligned") # 4 — embedded SPARQL queries carry term names as text, invisible to a # triple-level rename. Rewritten with the same map, after it. pairs = [(r["from"], r["to"]) for r in spec["renames"]] pairs += [(r["from"], r["to"]) for r in (spec.get("abox_renames") or [])] pairs += [(m["from"], m["into"]) for m in (spec.get("merges") or [])] for g in all_graphs: rewrite_queries(g, pairs, report) # 5 — merges for m in spec.get("merges") or []: for g in all_graphs: merge(g, pr, m["from"], m["into"], report) # 6 — reclassify display properties (TN-003) for rc in spec["structural"]["reclassify"]: term = pr[rc["term"]] if (term, RDF.type, OWL.AnnotationProperty) not in onto: onto.remove((term, RDF.type, getattr(OWL, rc["from"]))) onto.add((term, RDF.type, OWL.AnnotationProperty)) report.add("reclassified", 1, rc["term"]) # 7 — create the governance layer root (TN-027) for c in spec["structural"].get("create_classes") or []: term = pr[c["term"]] if (term, RDF.type, OWL.Class) not in onto: onto.add((term, RDF.type, OWL.Class)) onto.add((term, RDFS.subClassOf, pr[c["parent"]])) onto.add((term, RDFS.label, Literal(c["label"]))) onto.add((term, RDFS.comment, Literal(" ".join(c["comment"].split())))) onto.add((term, pr.isAbstract, Literal(True))) report.add("classes_created", 1, c["term"]) # 8 — reparent (TN-027) for rp in spec["structural"].get("reparent") or []: term, parent = pr[rp["term"]], pr[rp["parent"]] if (term, RDFS.subClassOf, parent) not in onto: onto.add((term, RDFS.subClassOf, parent)) report.add("reparented", 1, "%s -> %s" % (rp["term"], rp["parent"])) # 9 — sub-properties (TN-028) for sp in spec["structural"].get("subproperties") or []: term, parent = pr[sp["term"]], pr[sp["parent"]] if (term, RDFS.subPropertyOf, parent) not in onto: onto.add((term, RDFS.subPropertyOf, parent)) report.add("subproperties", 1, "%s -> %s" % (sp["term"], sp["parent"])) # 10 — controlled values become literals (TN-016, TN-017) for conv in spec["structural"].get("to_literal") or []: prop = pr[conv["property"]] if (prop, RDF.type, OWL.DatatypeProperty) not in onto: onto.remove((prop, RDF.type, OWL.ObjectProperty)) onto.add((prop, RDF.type, OWL.DatatypeProperty)) onto.remove((prop, RDFS.range, None)) onto.add((prop, RDFS.range, XSD.string)) onto.add((prop, RDFS.domain, pr[conv["domain"]])) report.add("to_literal_property", 1, conv["property"]) for g in all_graphs: # rewrite the asserted values for s, p, o in list(g.triples((None, prop, None))): name = local(o, ns) if name and name in conv["value_map"]: g.remove((s, p, o)) g.add((s, p, Literal(conv["value_map"][name]))) report.add("to_literal_values", 1) for ind in conv["drop_individuals"] + [conv["drop_class"]]: subj = pr[ind] if (subj, None, None) in onto: for g in [onto] + instances: # shapes keep their guards drop_subject(g, subj, report, "to_literal_dropped") # 11 — abstractness, from the rulebook annex (TN-006, TN-007) for a in rules["abstractness"]: term = pr[a["term"]] if (term, RDF.type, OWL.Class) not in onto: report.add("abstract_missing_class", 1, a["term"]) continue want = Literal(bool(a["is_abstract"])) if (term, pr.isAbstract, want) not in onto: onto.remove((term, pr.isAbstract, None)) onto.add((term, pr.isAbstract, want)) report.add("isAbstract_declared", 1) # 12 — display annotations, from the rulebook annex (TN-022) # The annex is authoritative AND exhaustive: a null value means any existing # annotation is removed. A short label identical to the label carries no # information and is one more thing to keep in step. for d in rules["display"]: term = pr[d["iri"]] for prop, value in ((pr.shortLabel, d.get("short_label")), (pr.acronym, d.get("acronym"))): current = list(onto.objects(term, prop)) if value: if (term, prop, Literal(value)) not in onto: onto.remove((term, prop, None)) onto.add((term, prop, Literal(value))) report.add("display_set", 1) elif current: onto.remove((term, prop, None)) report.add("display_removed", len(current)) # 13 — regenerate every label by derivation (TN-018, TN-023) kinds = (OWL.Class, OWL.ObjectProperty, OWL.DatatypeProperty, OWL.AnnotationProperty) for kind in kinds: for term in set(onto.subjects(RDF.type, kind)): name = local(term, ns) if not name: continue want = Literal(decamelise(name, kind == OWL.Class)) if (term, RDFS.label, want) not in onto: onto.remove((term, RDFS.label, None)) onto.add((term, RDFS.label, want)) report.add("labels_regenerated", 1) # 14 — bump the ontology version (EV-014: the namespace itself never moves) target = URIRef(ns.rstrip("/") + "/" + spec["meta"]["to_version"]) for onto_iri in set(onto.subjects(RDF.type, OWL.Ontology)): if (onto_iri, OWL.versionIRI, target) not in onto: onto.remove((onto_iri, OWL.versionIRI, None)) onto.add((onto_iri, OWL.versionIRI, target)) report.add("version_bumped", 1, str(target)) # 15 — the shapes declare the ontology version they target (EV-010). # The shapes graph carries no ontology node, so it is created here. so = spec["structural"].get("shapes_ontology") if so: node = URIRef(so["iri"]) for g in shapes: if (node, DCTERMS.conformsTo, target) not in g: g.remove((node, DCTERMS.conformsTo, None)) g.add((node, RDF.type, OWL.Ontology)) g.add((node, RDFS.label, Literal(so["label"]))) g.add((node, DCTERMS.conformsTo, target)) report.add("shapes_conformsTo", 1, str(target)) # ----------------------------------------------------------------------- main def main(): ap = argparse.ArgumentParser() ap.add_argument("--apply", action="store_true", help="write the files (default: dry run)") ap.add_argument("--ontology", default=os.path.join(HERE, "..", "ontology", "pr_metamodel.ttl")) ap.add_argument("--instances", action="append", default=[]) ap.add_argument("--shapes", action="append", default=[]) ap.add_argument("--log-dir", default=os.path.join(HERE, "logs")) args = ap.parse_args() spec = yaml.safe_load(open(SPEC, encoding="utf-8")) rules = yaml.safe_load(open(RULES, encoding="utf-8")) paths = [args.ontology] + args.instances + args.shapes missing = [p for p in paths if not os.path.exists(p)] if missing: print("MISSING INPUT") for p in missing: print(" " + p) sys.exit(1) inputs = OrderedDict((p, md5(p)) for p in paths) onto = Graph() onto.parse(args.ontology, format="turtle") instances, shapes = [], [] for p in args.instances: g = Graph() g.parse(p, format="turtle") instances.append(g) for p in args.shapes: g = Graph() g.parse(p, format="turtle") g.bind("dcterms", DCTERMS) shapes.append(g) others = instances + shapes other_paths = args.instances + args.shapes before = [len(onto)] + [len(g) for g in others] report = Report() migrate(onto, instances, shapes, spec, rules, report) after = [len(onto)] + [len(g) for g in others] print("MIGRATION %s -> %s %s" % (spec["meta"]["from_version"], spec["meta"]["to_version"], "APPLY" if args.apply else "DRY RUN")) print() for step, n in report.steps.items(): print(" %-28s %6d" % (step, n)) print(" %-28s %6d" % ("total changes", report.total)) print() for p, b, a in zip(paths, before, after): print(" %-46s %6d -> %6d triples" % (os.path.basename(p), b, a)) if report.notes: print() for n in report.notes: print(" note: " + n) log = { "attempt_timestamp": datetime.datetime.now().isoformat(timespec="seconds"), "mode": "apply" if args.apply else "dry-run", "from_version": spec["meta"]["from_version"], "to_version": spec["meta"]["to_version"], "input_checksums": inputs, "steps": report.steps, "total_changes": report.total, "triples_before": dict(zip(paths, before)), "triples_after": dict(zip(paths, after)), "notes": report.notes, } os.makedirs(args.log_dir, exist_ok=True) stamp = datetime.datetime.now().strftime("%Y%m%dT%H%M%S") log_path = os.path.join(args.log_dir, "migration_%s.json" % stamp) json.dump(log, open(log_path, "w", encoding="utf-8"), indent=2) print("\n log: %s" % log_path) if not args.apply: print("\n DRY RUN — nothing written. Re-run with --apply when the counts " "above are what you expect.") return onto.serialize(destination=args.ontology, format="turtle") for p, g in zip(other_paths, others): g.serialize(destination=p, format="turtle") print("\n written. Replay this script now: a second run must report zero " "changes (EV-006).") if __name__ == "__main__": main()