Master/data-meta-model
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
908826601a58e0c4bd1e3db39a69c8a011e8c002
data-meta-model/governance/migration/migrate_tbox_v2_0.py
T

441 lines
17 KiB
Python
Raw Blame History

#!/usr/bin/env python3
"""
Migrate the Pernod Ricard Data MetaModel from v1.6 to v2.0.
USAGE
python3 migrate_tbox_v2_0.py # dry run, writes nothing
python3 migrate_tbox_v2_0.py --apply # writes in place
python3 migrate_tbox_v2_0.py --apply --log-dir logs/
--ontology path to the ontology TTL (default ../ontology/pr_metamodel.ttl)
--instances path to an instance TTL (repeatable)
--shapes path to a shapes TTL (repeatable)
DESIGN
EV-004 dry run is the default; nothing is written without --apply
EV-005 every edit goes through rdflib, never a regex on the text
EV-006 guards test the target state, so a replay reports zero change
EV-015 an execution log is written for every attempt
Run on GrosseBertha, inside the pinned venv:
. venv/bin/activate && python3 migrate_tbox_v2_0.py
"""
import argparse
import datetime
import hashlib
import json
import os
import re
import sys
from collections import OrderedDict
import yaml
from rdflib import Graph, Literal, Namespace, RDF, RDFS, OWL, URIRef, XSD
from rdflib.namespace import SH
HERE = os.path.dirname(os.path.abspath(__file__))
SPEC = os.path.join(HERE, "renames.yaml")
RULES = os.path.normpath(os.path.join(HERE, "..", "rules.yaml"))
DCTERMS = Namespace("http://purl.org/dc/terms/")
# ----------------------------------------------------------------- utilities
class Report(object):
"""Counts every step, so that idempotence is provable and not merely hoped."""
def __init__(self):
self.steps = OrderedDict()
self.notes = []
def add(self, step, n, detail=None):
self.steps[step] = self.steps.get(step, 0) + n
if detail:
self.notes.append("%s: %s" % (step, detail))
@property
def total(self):
return sum(self.steps.values())
def md5(path):
h = hashlib.md5()
with open(path, "rb") as f:
for chunk in iter(lambda: f.read(65536), b""):
h.update(chunk)
return h.hexdigest()
def local(uri, ns):
s = str(uri)
return s[len(ns):] if s.startswith(ns) else None
def decamelise(name, is_class):
"""TN-018. Consecutive capitals are kept together: they carry an acronym."""
spaced = re.sub(r"(?<=[a-z0-9])(?=[A-Z])|(?<=[A-Z])(?=[A-Z][a-z])", " ", name)
return spaced if is_class else spaced[0].lower() + spaced[1:]
# ------------------------------------------------------------------ the work
def rename(graph, pr, old, new, report, step):
"""EV-001. Rewrite every triple naming the identifier, in all three positions."""
src, dst = pr[old], pr[new]
if (dst, None, None) in graph and (src, None, None) not in graph:
return 0 # EV-006: already done
n = 0
for s, p, o in list(graph):
ns, np_, no = s, p, o
if s == src:
ns = dst
if p == src:
np_ = dst
if o == src:
no = dst
if (ns, np_, no) != (s, p, o):
graph.remove((s, p, o))
graph.add((ns, np_, no))
n += 1
report.add(step, n)
return n
def merge(graph, pr, src_name, dst_name, report):
"""The source disappears into an existing target, which keeps its declaration."""
src, dst = pr[src_name], pr[dst_name]
n = 0
for s, p, o in list(graph.triples((None, src, None))):
graph.remove((s, p, o))
graph.add((s, dst, o))
n += 1
for s, p, o in list(graph.triples((src, None, None))):
graph.remove((s, p, o)) # drop the source declaration
n += 1
report.add("merge", n)
return n
def drop_subject(graph, subject, report, step):
"""EV-003. Remove the block AND every reference naming it, or inference rebuilds it."""
n = 0
for t in list(graph.triples((subject, None, None))):
graph.remove(t)
n += 1
for t in list(graph.triples((None, None, subject))):
graph.remove(t)
n += 1
for t in list(graph.triples((None, subject, None))):
graph.remove(t)
n += 1
report.add(step, n)
return n
def count_instances(graphs, term):
"""EV-002. The proof required before any permanent withdrawal.
Counted on INSTANCE graphs only. A mention inside a shapes graph is not an
instantiation: a shape carrying sh:maxCount 0 on a term is a guard forbidding
its reappearance, which is the opposite of a use, and it keeps working after
the term is gone since it checks an identifier rather than a declaration.
"""
n = 0
for g in graphs:
n += len(list(g.triples((None, RDF.type, term))))
n += len(list(g.triples((None, term, None))))
n += len(list(g.triples((None, None, term))))
return n
def rewrite_queries(graph, renames, report):
"""Rename inside embedded SPARQL queries.
A sh:select or sh:ask body is a literal, so no triple-level rename reaches
it: the query keeps naming a term that no longer exists and the constraint
silently stops matching. This is not the line-by-line editing EV-005
forbids: the graph is parsed, the literal is retrieved as a value, and what
is rewritten is a structured query language where a prefixed name has clear
token boundaries.
"""
n = 0
for prop in (SH.select, SH.ask, SH.construct):
for s, p, o in list(graph.triples((None, prop, None))):
text = str(o)
new = text
for old, target in renames:
new = re.sub(r"\bpr:%s\b" % re.escape(old), "pr:%s" % target, new)
if new != text:
graph.remove((s, p, o))
graph.add((s, p, Literal(new)))
n += 1
report.add("queries_rewritten", n)
return n
def migrate(onto, instances, shapes, spec, rules, report):
ns = spec["meta"]["namespace"]
pr = Namespace(ns)
others = instances + shapes
all_graphs = [onto] + others
# 1 — deprecated terms are withdrawn outright (phase clause), proof first.
# The proof is taken on instance graphs only: a mention in a shapes graph
# is a guard against reappearance, not a use.
# Reference cleaning likewise stops at the ontology and the instances.
# EV-003 guards against phantom nodes rebuilt by inference in the model
# graph; a mention inside a shape is a constraint on an identifier, in a
# separate graph, taking part in no inference over the ontology. Removing
# it would strip the sh:path from a guard and leave a property shape
# carrying a maxCount and no path — an invalid shape, and the loss of the
# very rule that forbids the term from coming back.
if spec["structural"].get("drop_deprecated"):
for subj in list(onto.subjects(OWL.deprecated, Literal(True))):
name = local(subj, ns) or str(subj)
used = count_instances(instances, subj)
if used:
report.add("deprecated_kept", 1, "%s instantiated %d times" % (name, used))
continue
for g in [onto] + instances:
drop_subject(g, subj, report, "deprecated_dropped")
# 2 — vocabulary renames, in dependency order
for order in (1, 2, 3):
for r in [x for x in spec["renames"] if x["order"] == order]:
for g in all_graphs:
rename(g, pr, r["from"], r["to"], report, "rename_order_%d" % order)
# 3 — instance-side alignment: forms the instances use that the vocabulary
# never declared and that the TN-012 renames do not converge on.
# Applied to the shapes too: they target these forms in sh:path.
for r in spec.get("abox_renames") or []:
for g in all_graphs:
rename(g, pr, r["from"], r["to"], report, "abox_aligned")
# 4 — embedded SPARQL queries carry term names as text, invisible to a
# triple-level rename. Rewritten with the same map, after it.
pairs = [(r["from"], r["to"]) for r in spec["renames"]]
pairs += [(r["from"], r["to"]) for r in (spec.get("abox_renames") or [])]
pairs += [(m["from"], m["into"]) for m in (spec.get("merges") or [])]
for g in all_graphs:
rewrite_queries(g, pairs, report)
# 5 — merges
for m in spec.get("merges") or []:
for g in all_graphs:
merge(g, pr, m["from"], m["into"], report)
# 6 — reclassify display properties (TN-003)
for rc in spec["structural"]["reclassify"]:
term = pr[rc["term"]]
if (term, RDF.type, OWL.AnnotationProperty) not in onto:
onto.remove((term, RDF.type, getattr(OWL, rc["from"])))
onto.add((term, RDF.type, OWL.AnnotationProperty))
report.add("reclassified", 1, rc["term"])
# 7 — create the governance layer root (TN-027)
for c in spec["structural"].get("create_classes") or []:
term = pr[c["term"]]
if (term, RDF.type, OWL.Class) not in onto:
onto.add((term, RDF.type, OWL.Class))
onto.add((term, RDFS.subClassOf, pr[c["parent"]]))
onto.add((term, RDFS.label, Literal(c["label"])))
onto.add((term, RDFS.comment, Literal(" ".join(c["comment"].split()))))
onto.add((term, pr.isAbstract, Literal(True)))
report.add("classes_created", 1, c["term"])
# 8 — reparent (TN-027)
for rp in spec["structural"].get("reparent") or []:
term, parent = pr[rp["term"]], pr[rp["parent"]]
if (term, RDFS.subClassOf, parent) not in onto:
onto.add((term, RDFS.subClassOf, parent))
report.add("reparented", 1, "%s -> %s" % (rp["term"], rp["parent"]))
# 9 — sub-properties (TN-028)
for sp in spec["structural"].get("subproperties") or []:
term, parent = pr[sp["term"]], pr[sp["parent"]]
if (term, RDFS.subPropertyOf, parent) not in onto:
onto.add((term, RDFS.subPropertyOf, parent))
report.add("subproperties", 1, "%s -> %s" % (sp["term"], sp["parent"]))
# 10 — controlled values become literals (TN-016, TN-017)
for conv in spec["structural"].get("to_literal") or []:
prop = pr[conv["property"]]
if (prop, RDF.type, OWL.DatatypeProperty) not in onto:
onto.remove((prop, RDF.type, OWL.ObjectProperty))
onto.add((prop, RDF.type, OWL.DatatypeProperty))
onto.remove((prop, RDFS.range, None))
onto.add((prop, RDFS.range, XSD.string))
onto.add((prop, RDFS.domain, pr[conv["domain"]]))
report.add("to_literal_property", 1, conv["property"])
for g in all_graphs: # rewrite the asserted values
for s, p, o in list(g.triples((None, prop, None))):
name = local(o, ns)
if name and name in conv["value_map"]:
g.remove((s, p, o))
g.add((s, p, Literal(conv["value_map"][name])))
report.add("to_literal_values", 1)
for ind in conv["drop_individuals"] + [conv["drop_class"]]:
subj = pr[ind]
if (subj, None, None) in onto:
for g in [onto] + instances: # shapes keep their guards
drop_subject(g, subj, report, "to_literal_dropped")
# 11 — abstractness, from the rulebook annex (TN-006, TN-007)
for a in rules["abstractness"]:
term = pr[a["term"]]
if (term, RDF.type, OWL.Class) not in onto:
report.add("abstract_missing_class", 1, a["term"])
continue
want = Literal(bool(a["is_abstract"]))
if (term, pr.isAbstract, want) not in onto:
onto.remove((term, pr.isAbstract, None))
onto.add((term, pr.isAbstract, want))
report.add("isAbstract_declared", 1)
# 12 — display annotations, from the rulebook annex (TN-022)
# The annex is authoritative AND exhaustive: a null value means any existing
# annotation is removed. A short label identical to the label carries no
# information and is one more thing to keep in step.
for d in rules["display"]:
term = pr[d["iri"]]
for prop, value in ((pr.shortLabel, d.get("short_label")),
(pr.acronym, d.get("acronym"))):
current = list(onto.objects(term, prop))
if value:
if (term, prop, Literal(value)) not in onto:
onto.remove((term, prop, None))
onto.add((term, prop, Literal(value)))
report.add("display_set", 1)
elif current:
onto.remove((term, prop, None))
report.add("display_removed", len(current))
# 13 — regenerate every label by derivation (TN-018, TN-023)
kinds = (OWL.Class, OWL.ObjectProperty, OWL.DatatypeProperty, OWL.AnnotationProperty)
for kind in kinds:
for term in set(onto.subjects(RDF.type, kind)):
name = local(term, ns)
if not name:
continue
want = Literal(decamelise(name, kind == OWL.Class))
if (term, RDFS.label, want) not in onto:
onto.remove((term, RDFS.label, None))
onto.add((term, RDFS.label, want))
report.add("labels_regenerated", 1)
# 14 — bump the ontology version (EV-014: the namespace itself never moves)
target = URIRef(ns.rstrip("/") + "/" + spec["meta"]["to_version"])
for onto_iri in set(onto.subjects(RDF.type, OWL.Ontology)):
if (onto_iri, OWL.versionIRI, target) not in onto:
onto.remove((onto_iri, OWL.versionIRI, None))
onto.add((onto_iri, OWL.versionIRI, target))
report.add("version_bumped", 1, str(target))
# 15 — the shapes declare the ontology version they target (EV-010).
# The shapes graph carries no ontology node, so it is created here.
so = spec["structural"].get("shapes_ontology")
if so:
node = URIRef(so["iri"])
for g in shapes:
if (node, DCTERMS.conformsTo, target) not in g:
g.remove((node, DCTERMS.conformsTo, None))
g.add((node, RDF.type, OWL.Ontology))
g.add((node, RDFS.label, Literal(so["label"])))
g.add((node, DCTERMS.conformsTo, target))
report.add("shapes_conformsTo", 1, str(target))
# ----------------------------------------------------------------------- main
def main():
ap = argparse.ArgumentParser()
ap.add_argument("--apply", action="store_true", help="write the files (default: dry run)")
ap.add_argument("--ontology", default=os.path.join(HERE, "..", "ontology", "pr_metamodel.ttl"))
ap.add_argument("--instances", action="append", default=[])
ap.add_argument("--shapes", action="append", default=[])
ap.add_argument("--log-dir", default=os.path.join(HERE, "logs"))
args = ap.parse_args()
spec = yaml.safe_load(open(SPEC, encoding="utf-8"))
rules = yaml.safe_load(open(RULES, encoding="utf-8"))
paths = [args.ontology] + args.instances + args.shapes
missing = [p for p in paths if not os.path.exists(p)]
if missing:
print("MISSING INPUT")
for p in missing:
print(" " + p)
sys.exit(1)
inputs = OrderedDict((p, md5(p)) for p in paths)
onto = Graph()
onto.parse(args.ontology, format="turtle")
instances, shapes = [], []
for p in args.instances:
g = Graph()
g.parse(p, format="turtle")
instances.append(g)
for p in args.shapes:
g = Graph()
g.parse(p, format="turtle")
g.bind("dcterms", DCTERMS)
shapes.append(g)
others = instances + shapes
other_paths = args.instances + args.shapes
before = [len(onto)] + [len(g) for g in others]
report = Report()
migrate(onto, instances, shapes, spec, rules, report)
after = [len(onto)] + [len(g) for g in others]
print("MIGRATION %s -> %s %s"
% (spec["meta"]["from_version"], spec["meta"]["to_version"],
"APPLY" if args.apply else "DRY RUN"))
print()
for step, n in report.steps.items():
print(" %-28s %6d" % (step, n))
print(" %-28s %6d" % ("total changes", report.total))
print()
for p, b, a in zip(paths, before, after):
print(" %-46s %6d -> %6d triples" % (os.path.basename(p), b, a))
if report.notes:
print()
for n in report.notes:
print(" note: " + n)
log = {
"attempt_timestamp": datetime.datetime.now().isoformat(timespec="seconds"),
"mode": "apply" if args.apply else "dry-run",
"from_version": spec["meta"]["from_version"],
"to_version": spec["meta"]["to_version"],
"input_checksums": inputs,
"steps": report.steps,
"total_changes": report.total,
"triples_before": dict(zip(paths, before)),
"triples_after": dict(zip(paths, after)),
"notes": report.notes,
}
os.makedirs(args.log_dir, exist_ok=True)
stamp = datetime.datetime.now().strftime("%Y%m%dT%H%M%S")
log_path = os.path.join(args.log_dir, "migration_%s.json" % stamp)
json.dump(log, open(log_path, "w", encoding="utf-8"), indent=2)
print("\n log: %s" % log_path)
if not args.apply:
print("\n DRY RUN — nothing written. Re-run with --apply when the counts "
"above are what you expect.")
return
onto.serialize(destination=args.ontology, format="turtle")
for p, g in zip(other_paths, others):
g.serialize(destination=p, format="turtle")
print("\n written. Replay this script now: a second run must report zero "
"changes (EV-006).")
if __name__ == "__main__":
main()
Reference in New Issue View Git Blame Copy Permalink