Master/data-meta-model
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

tbox: v1.5 - PhysicalRelation devient DataStructure et point d'extension declare pour le non-tabulaire, hasSource devient sourcedFrom vers un System instancie, storedIn sert deux granularites et deprecie materializedAs, references devient qualifiedBy, primaryLocation sous-propriete de storedIn, hasShortLabel sur 34 classes. CuratedObject devient DefinedObject et ObservedObject devient CapturedObject jusque dans les IRI, curationMode devient authoringMode. Correction de fond : PhysicalLayerObject ne porte plus la provenance, chaque classe de la couche la declare - un System est defined, tout ce qui est en dessous est captured. instances: 5 systemes crees, 151 sourcedFrom, 15 BO nettoyes de hasSource, labels courts, versions alignees

Browse Source
This commit is contained in:
Bastien Gourdon
2026-07-30 15:13:03 +02:00
parent 76e95de471
commit f83b7d7491
9 changed files with 2751 additions and 1504 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/migrate_instances_v1_4.py
+176
View File
@@ -0,0 +1,176 @@
#!/usr/bin/env python3
"""
SODH - migration v1.3 -> v1.4
==============================
USAGE
python3 scripts/migrate_instances_v1_4.py # dry run
python3 scripts/migrate_instances_v1_4.py --apply # rewrite, .bak kept
1. SYSTEMS BECOME NODES. hasSource held a string: "SODH Gold", "MDM Product",
"Group Finance". Six systems are created and sourcedFrom points at them.
"Which objects depend on MDM Product?" was unanswerable while MDM Product
was text.
The systems are seeded here by hand, once. Everything below them --
databases, schemas, tables, columns -- is meant to be harvested, and this
script does not invent any of it.
2. Three hasSource values were not systems at all but migration notes ("DGO
proposal, split of a compound sub-domain"). They are dropped rather than
turned into a system: provenance of a record is not a property of the thing
the record describes.
3. materializedAs -> storedIn on the Data Objects. Nothing carries it yet,
done for completeness so the term never reappears.
"""
import os
import re
import shutil
import sys
REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
TTL = os.path.join(REPO, "instances", "sodh.ttl")
W = 78
# (iri, identifier, name, short label, system type, the string it replaces)
SYSTEMS = [
("ex:SYS_SODH_GOLD", "SYS-001", "Sell Out Data Hub - Gold layer", "SODH Gold",
"pr:DataPlatform", "SODH Gold"),
("ex:SYS_MDM_PRODUCT", "SYS-002", "Master Data Management - Product", "MDM Product",
"pr:MasterDataSystem", "MDM Product"),
("ex:SYS_MDM_CUSTOMER", "SYS-003", "Master Data Management - Customer", "MDM Customer",
"pr:MasterDataSystem", "MDM Customer"),
("ex:SYS_GROUP_FINANCE", "SYS-004", "Group Finance consolidation", "Group Finance",
"pr:SourceApplication", "Group Finance"),
("ex:SYS_CORPORATE_CALENDAR", "SYS-005", "Corporate standard calendar", "Standard Calendar",
"pr:SourceApplication", "Standard Calendar"),
]
# strings that were never a system: provenance notes about the record itself
NOT_A_SYSTEM = ["DGO proposal"]
def clauses(body):
out, buf, i, n = [], [], 0, len(body)
while i < n:
c = body[i]
if c == '"':
j = i + 1
while j < n and body[j] != '"':
j += 2 if body[j] == "\\" else 1
j = min(j + 1, n)
buf.append(body[i:j]); i = j; continue
if c == ";":
out.append("".join(buf)); buf = []; i += 1; continue
buf.append(c); i += 1
out.append("".join(buf))
return out
def set_prop(text, subject, prop, value):
pat = re.compile(r'(^%s a pr:\w+ ;)(.*?)(\.\s*\n)' % re.escape(subject), re.S | re.M)
m = pat.search(text)
if not m:
return text, False
kept = [c for c in clauses(m.group(2))
if c.strip() and not re.match(r'\s*%s\s' % re.escape(prop), c)]
if value is not None:
kept.insert(0, "\n %s %s " % (prop, value))
body = " ;".join(kept).rstrip() + "\n "
return text[:m.start()] + m.group(1) + body + m.group(3) + text[m.end():], True
def get_source(text, subject):
m = re.search(r'^%s a pr:\w+ ;(.*?)\.\s*\n' % re.escape(subject), text, re.S | re.M)
if not m:
return None
for c in clauses(m.group(1)):
mm = re.match(r'\s*pr:hasSource\s+"([^"]*)"', c)
if mm:
return mm.group(1)
return None
def system_blocks(todo):
head = ("# --- systems -----------------------------------------------------------\n"
"# Seeded by hand, once. A system was a string until now, which is why\n"
"# 'what depends on MDM Product' had no answer. Everything below a system --\n"
"# databases, schemas, tables, columns -- is meant to be harvested, and\n"
"# nothing here invents any of it.\n\n")
out = []
for iri, ident, name, short, stype, _ in todo:
out.append('%s a pr:System ;\n'
' pr:hasIdentifier "%s" ; pr:hasName "%s" ;\n'
' pr:hasShortLabel "%s" ;\n'
' pr:hasSystemType %s ;\n'
' pr:hasStatus "DRAFT" ; pr:hasVersion "1.4" .'
% (iri, ident, name, short, stype))
return head + "\n".join(out) + "\n"
def main():
apply_changes = "--apply" in sys.argv
if not os.path.exists(TTL):
print("Not found: %s" % TTL); sys.exit(2)
text = original = open(TTL, encoding="utf-8").read()
report = []
todo = [sy for sy in SYSTEMS if not re.search(r'^%s a ' % re.escape(sy[0]), text, re.M)]
if todo:
anchor = re.search(r'^ex:DD_\w+ a pr:DataDomain', text, re.M)
pos = anchor.start() if anchor else 0
text = text[:pos] + system_blocks(todo) + "\n" + text[pos:]
report.append("%d systems created%s" % (len(todo), "" if todo else " (already present)"))
by_string = {sy[5]: sy[0] for sy in SYSTEMS}
n_ok = n_drop = 0
for m in list(re.finditer(r'^(ex:\w+) a pr:\w+ ;', text, re.M)):
subj = m.group(1)
src = get_source(text, subj)
if src is None:
continue
if src in by_string:
text, _ = set_prop(text, subj, "pr:hasSource", None)
text, _ = set_prop(text, subj, "pr:sourcedFrom", by_string[src])
n_ok += 1
elif any(k in src for k in NOT_A_SYSTEM):
text, _ = set_prop(text, subj, "pr:hasSource", None)
n_drop += 1
report.append("sourcedFrom set on %d objects" % n_ok)
report.append("%d provenance notes dropped rather than turned into a system" % n_drop)
text, n = re.subn(r'\bpr:materializedAs\b', 'pr:storedIn', text)
report.append("materializedAs -> storedIn: %d occurrence(s)" % n)
print()
print("SODH MIGRATION v1.3 -> v1.4 %s" % ("APPLY" if apply_changes else "DRY RUN"))
print("=" * W)
for line in report:
print(" " + line)
print("=" * W)
left = re.findall(r'pr:hasSource\s+"([^"]*)"', text)
defined = set(re.findall(r'^(ex:\w+) a ', text, re.M))
dangling = sorted(set(re.findall(r'\bex:\w+', text)) - defined)
doubled = []
for m in re.finditer(r'^(ex:\w+) a pr:\w+ ;(.*?)\.\s*\n', text, re.S | re.M):
for p in ("pr:sourcedFrom", "pr:hasName", "pr:hasShortLabel"):
if len(re.findall(r'(?<![\w:])%s\s' % re.escape(p), m.group(2))) > 1:
doubled.append("%s/%s" % (m.group(1), p))
print(" hasSource left: %d %s" % (len(left), sorted(set(left)) if left else "(must be 0)"))
print(" sourcedFrom references: %d" % len(re.findall(r'pr:sourcedFrom', text)))
print(" dangling references: %s" % (", ".join(dangling) if dangling else "none"))
print(" duplicated single-valued properties: %s"
% (", ".join(doubled) if doubled else "none"))
print("=" * W)
if text != original and apply_changes:
shutil.copy2(TTL, TTL + ".bak")
open(TTL, "w", encoding="utf-8").write(text)
print(" written, backup at %s.bak" % os.path.basename(TTL))
elif text != original:
print(" dry run -- re-run with --apply to write")
print()
if __name__ == "__main__":
main()