Master/data-meta-model
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

tbox: v2.2 - polymorphisme declare sur 19 proprietes, 17 domaines poses, provenance portee par les classes concretes, GovernanceLayerObject rattachee, affectations de gouvernance completes (12 completees, 10 SDO, 1 PO, 16 stewards par objet metier), Panel Coverage gouverne. Validation SHACL sans violation bloquante

Browse Source
This commit is contained in:
Bastien Gourdon
2026-08-07 10:31:16 +02:00
parent d90d2a316d
commit 0b7d3834d2
29 changed files with 3274 additions and 274 deletions
Download Patch File Download Diff File Expand all files Collapse all files
governance/migration/migrate_tbox_v2_2.py
+367
View File
@@ -0,0 +1,367 @@
#!/usr/bin/env python3
"""
Migrate the Pernod Ricard Data MetaModel and the SODH instances to v2.2.
USAGE
python3 migrate_tbox_v2_2.py --ontology ... --instances ...
python3 migrate_tbox_v2_2.py --ontology ... --instances ... --apply
v2.2 turns the actor nodes into governance ASSIGNMENTS: a stable post carrying
an identifier, a domain and a name, plus the person holding it as a value. It
completes the twelve that existed, creates the ten sub-domain owners, the
product owner, and one steward assignment per business object.
DESIGN
EV-004 dry run is the default
EV-005 every edit goes through rdflib
EV-006 guards test the target state, so a replay reports zero change
EV-015 an execution log is written for every attempt
"""
import argparse
import datetime
import hashlib
import json
import os
import re
import sys
from collections import OrderedDict
import yaml
from rdflib import Graph, Literal, Namespace, RDF, RDFS, OWL, URIRef
HERE = os.path.dirname(os.path.abspath(__file__))
SPEC = os.path.join(HERE, "assignments_v2_2.yaml")
class Report(object):
def __init__(self):
self.steps = OrderedDict()
self.notes = []
def add(self, step, n=1, detail=None):
self.steps[step] = self.steps.get(step, 0) + n
if detail:
self.notes.append("%s: %s" % (step, detail))
@property
def total(self):
return sum(self.steps.values())
def derive_label(name):
"""TN-018 and TN-023: a label is derived from the local name, never typed."""
spaced = re.sub(r"(?<=[a-z0-9])(?=[A-Z])|(?<=[A-Z])(?=[A-Z][a-z])", " ", name)
return spaced[0].lower() + spaced[1:]
def md5(path):
h = hashlib.md5()
with open(path, "rb") as fh:
for chunk in iter(lambda: fh.read(65536), b""):
h.update(chunk)
return h.hexdigest()
def rename_node(g, old, new, report, step):
"""EV-001 applied to an instance: rewrite every triple naming the node."""
if (new, None, None) in g and (old, None, None) not in g:
return 0
n = 0
for s, p, o in list(g):
ns, no = (new if s == old else s), (new if o == old else o)
if (ns, no) != (s, o):
g.remove((s, p, o))
g.add((ns, p, no))
n += 1
report.add(step, n)
return n
def assign(g, pr, node, role, identifier, name, held_by, domain, spec, report, step):
"""Write one assignment. The guard tests the identifier, so a replay is a no-op."""
if (node, pr.identifier, Literal(identifier)) in g:
return False
g.add((node, RDF.type, role))
g.remove((node, pr.identifier, None))
g.add((node, pr.identifier, Literal(identifier)))
g.remove((node, pr.canonicalName, None))
g.add((node, pr.canonicalName, Literal(name)))
g.remove((node, pr.personName, None))
if held_by:
g.add((node, pr.personName, Literal(held_by)))
g.remove((node, pr.status, None))
g.add((node, pr.status, Literal(spec["status"])))
g.remove((node, pr.version, None))
g.add((node, pr.version, Literal(spec["version"])))
if domain is not None:
g.remove((node, pr.ownedByDomain, None))
g.add((node, pr.ownedByDomain, domain))
report.add(step, 1)
return True
def check_version(g, spec, path):
"""EV-010 applied to a migration script: refuse to run against the wrong input.
A script announcing 2.1 -> 2.2 will happily work on a file still in 2.0 and
report success, leaving a state that is neither version. The same silent
failure the shapes are guarded against, one layer down.
"""
ns = spec["meta"]["namespace"]
expected = URIRef(ns.rstrip("/") + "/" + spec["meta"]["from_version"])
target = URIRef(ns.rstrip("/") + "/" + spec["meta"]["to_version"])
found = set(g.objects(None, OWL.versionIRI))
if target in found:
print("ALREADY AT %s — nothing to do." % spec["meta"]["to_version"])
sys.exit(0)
if expected not in found:
print("ABORT — %s declares %s; this script migrates from %s."
% (os.path.basename(path),
", ".join(sorted(str(f) for f in found)) or "no version",
expected))
sys.exit(1)
def migrate(onto, data, spec, report):
ns, ins = spec["meta"]["namespace"], spec["meta"]["instance_namespace"]
pr, ex = Namespace(ns), Namespace(ins)
# 1 — declare heldBy on Actor (procedure C1)
for d in spec.get("declare_properties") or []:
term = pr[d["term"]]
if (term, RDF.type, getattr(OWL, d["kind"])) not in onto:
onto.add((term, RDF.type, getattr(OWL, d["kind"])))
onto.add((term, RDFS.label, Literal(derive_label(d["term"]))))
onto.add((term, RDFS.domain, pr[d["domain"]]))
onto.add((term, RDFS.range, URIRef(d["range"])))
onto.add((term, RDFS.comment, Literal(" ".join(d["comment"].split()))))
report.add("properties_declared", 1, d["term"])
# 2 — the short form of Data Steward is DST, and the rulebook now says so
for u in spec.get("display_updates") or []:
term = pr[u["term"]]
if (term, pr.acronym, Literal(u["acronym"])) not in onto:
onto.remove((term, pr.acronym, None))
onto.add((term, pr.acronym, Literal(u["acronym"])))
report.add("display_updated", 1, "%s -> %s" % (u["term"], u["acronym"]))
# 3 — rename the instance nodes that carried the wrong short form
for r in spec.get("instance_renames") or []:
rename_node(data, ex[r["from"]], ex[r["to"]], report, "instances_renamed")
# 4 — complete the assignments that already existed
for c in spec.get("complete") or []:
node = ex[c["node"]]
if (node, None, None) not in data:
report.add("complete_missing", 1, c["node"])
continue
role = next(iter(data.objects(node, RDF.type)), None)
assign(data, pr, node, role, c["identifier"], c["name"],
c["held_by"], ex[c["domain"]], spec, report, "assignments_completed")
# 5 — sub-domain owners
sdo_of = {}
for s in spec.get("create_sub_domain_owners") or []:
node = ex[s["node"]]
sdo_of[ex[s["sub_domain"]]] = node
if assign(data, pr, node, pr.DataSubDomainOwner, s["identifier"],
spec["sub_domain_owner_name"], s["held_by"], ex[s["domain"]],
spec, report, "sub_domain_owners_created"):
data.add((ex[s["sub_domain"]], pr.hasSubDomainOwner, node))
# 6 — product owners
for p in spec.get("create_product_owners") or []:
node = ex[p["node"]]
product = ex[p["product"]]
if (product, None, None) not in data:
report.add("product_missing", 1, p["product"])
continue
if assign(data, pr, node, pr.DataProductOwner, p["identifier"],
spec["product_owner_name"], p["held_by"], ex[p["domain"]],
spec, report, "product_owners_created"):
data.add((product, pr.hasProductOwner, node))
# 7 — create the business objects that close a governance chain
for b in spec.get("create_business_objects") or []:
node = ex[b["node"]]
if (node, pr.identifier, Literal(b["identifier"])) in data:
continue
data.add((node, RDF.type, pr.BusinessObject))
data.add((node, pr.identifier, Literal(b["identifier"])))
data.add((node, pr.canonicalName, Literal(b["name"])))
data.add((node, pr.isAbout, ex[b["is_about"]]))
data.add((node, pr.belongsTo, ex[b["sub_domain"]]))
data.add((node, pr.ownedByDomain, ex[b["domain"]]))
data.add((node, pr.status, Literal(spec["status"])))
data.add((node, pr.version, Literal(spec["version"])))
report.add("business_objects_created", 1, b["node"])
# 8 — one steward assignment per business object, derived from the graph
holders = spec.get("steward_holders") or {}
steward_of = {}
for bo in sorted(data.subjects(RDF.type, pr.BusinessObject), key=str):
ident = next((str(v) for v in data.objects(bo, pr.identifier)), None)
if not ident:
report.add("steward_no_identifier", 1, str(bo).replace(ins, ""))
continue
suffix = ident.split("-", 1)[1] # 06.01-001
domain_key = "DD_" + suffix.split(".", 1)[0] # DD_06
holder = holders.get(domain_key)
if holder is None:
report.add("steward_no_holder", 1, domain_key)
continue
node = ex["DST_" + suffix.replace(".", "_").replace("-", "_")]
steward_of[bo] = node
if assign(data, pr, node, pr.DataSteward, "DST-" + suffix,
spec["steward_name"], holder, ex[domain_key],
spec, report, "stewards_created"):
for _, _, old in list(data.triples((bo, pr.monitoredBy, None))):
data.remove((bo, pr.monitoredBy, old))
data.add((bo, pr.monitoredBy, node))
# 9 — resolve every ownedBy pointing at a per-domain steward node
res = spec.get("owned_by_resolution") or {}
old_stewards = {ex[r["from"]] for r in (spec.get("instance_renames") or [])} | \
{ex[r["to"]] for r in (spec.get("instance_renames") or [])}
def business_object_of(subject):
"""Follow the declared chains until a business object is reached."""
for chain in res.get("chains") or []:
prop = pr[chain["via"]]
if chain["direction"] == "inbound":
for candidate in data.subjects(prop, subject):
if (candidate, RDF.type, pr.BusinessObject) in data:
return candidate
else:
for target in data.objects(subject, prop):
if (target, RDF.type, pr.BusinessObject) in data:
return target
for candidate in data.subjects(pr.isAbout, target):
if (candidate, RDF.type, pr.BusinessObject) in data:
return candidate
return None
remove_on = {pr[c] for c in (res.get("remove_on") or [])}
for subject, _, target in list(data.triples((None, pr.ownedBy, None))):
if target not in old_stewards:
continue
types = set(data.objects(subject, RDF.type))
if types & remove_on:
data.remove((subject, pr.ownedBy, target))
report.add("ownedby_removed", 1)
continue
if pr.DataSubDomain in types and res.get("repoint_sub_domains"):
owner = sdo_of.get(subject)
if owner is None:
report.add("ownedby_unresolved", 1,
"%s has no sub-domain owner" % str(subject).replace(ins, ""))
continue
data.remove((subject, pr.ownedBy, target))
data.add((subject, pr.ownedBy, owner))
report.add("ownedby_repointed_to_sdo", 1)
continue
bo = business_object_of(subject)
owner = steward_of.get(bo) if bo is not None else None
if owner is None:
report.add("ownedby_unresolved", 1,
"%s reaches no business object" % str(subject).replace(ins, ""))
continue
data.remove((subject, pr.ownedBy, target))
data.add((subject, pr.ownedBy, owner))
report.add("ownedby_repointed_to_steward", 1)
# 10 — withdraw the per-domain steward nodes, once nothing names them (EV-003)
if spec.get("retire_domain_stewards"):
for r in spec.get("instance_renames") or []:
node = ex[r["to"]]
referrers = {s for s in data.subjects(None, node)}
if referrers:
report.add("steward_still_referenced", 1,
"%s cited by %d subject(s)" % (r["to"], len(referrers)))
continue
n = 0
for t in list(data.triples((node, None, None))):
data.remove(t)
n += 1
if n:
report.add("domain_stewards_retired", 1, r["to"])
# 11 — bump the version
target = URIRef(ns.rstrip("/") + "/" + spec["meta"]["to_version"])
for o in set(onto.subjects(RDF.type, OWL.Ontology)):
if (o, OWL.versionIRI, target) not in onto:
onto.remove((o, OWL.versionIRI, None))
onto.add((o, OWL.versionIRI, target))
report.add("version_bumped", 1, str(target))
def main():
ap = argparse.ArgumentParser()
ap.add_argument("--apply", action="store_true")
ap.add_argument("--ontology", required=True)
ap.add_argument("--instances", required=True)
ap.add_argument("--log-dir", default=os.path.join(HERE, "logs"))
args = ap.parse_args()
for path in (args.ontology, args.instances):
if not os.path.exists(path):
print("MISSING INPUT: %s" % path)
sys.exit(1)
spec = yaml.safe_load(open(SPEC, encoding="utf-8"))
checksums = OrderedDict((p, md5(p)) for p in (args.ontology, args.instances))
onto, data = Graph(), Graph()
onto.parse(args.ontology, format="turtle")
data.parse(args.instances, format="turtle")
check_version(onto, spec, args.ontology)
before = (len(onto), len(data))
report = Report()
migrate(onto, data, spec, report)
after = (len(onto), len(data))
print("MIGRATION %s -> %s %s"
% (spec["meta"]["from_version"], spec["meta"]["to_version"],
"APPLY" if args.apply else "DRY RUN"))
print()
for step, n in report.steps.items():
print(" %-28s %6d" % (step, n))
print(" %-28s %6d" % ("total changes", report.total))
print()
for path, b, a in zip((args.ontology, args.instances), before, after):
print(" %-46s %6d -> %6d triples" % (os.path.basename(path), b, a))
if report.notes:
print()
for n in report.notes:
print(" note: " + n)
os.makedirs(args.log_dir, exist_ok=True)
stamp = datetime.datetime.now().strftime("%Y%m%dT%H%M%S")
log_path = os.path.join(args.log_dir, "migration_v2_2_%s.json" % stamp)
json.dump({"attempt_timestamp": datetime.datetime.now().isoformat(timespec="seconds"),
"mode": "apply" if args.apply else "dry-run",
"from_version": spec["meta"]["from_version"],
"to_version": spec["meta"]["to_version"],
"input_checksums": checksums,
"steps": report.steps, "total_changes": report.total,
"triples_before": dict(zip((args.ontology, args.instances), before)),
"triples_after": dict(zip((args.ontology, args.instances), after)),
"notes": report.notes},
open(log_path, "w", encoding="utf-8"), indent=2)
print("\n log: %s" % log_path)
if not args.apply:
print("\n DRY RUN — nothing written. Re-run with --apply when the counts "
"above are what you expect.")
return
onto.serialize(destination=args.ontology, format="turtle")
data.serialize(destination=args.instances, format="turtle")
print("\n written. Replay this script now: a second run must report zero "
"changes (EV-006).")
if __name__ == "__main__":
main()