#!/usr/bin/env python3 """ SODH - migration v1.3 -> v1.4 ============================== USAGE python3 scripts/migrate_instances_v1_4.py # dry run python3 scripts/migrate_instances_v1_4.py --apply # rewrite, .bak kept 1. SYSTEMS BECOME NODES. hasSource held a string: "SODH Gold", "MDM Product", "Group Finance". Six systems are created and sourcedFrom points at them. "Which objects depend on MDM Product?" was unanswerable while MDM Product was text. The systems are seeded here by hand, once. Everything below them -- databases, schemas, tables, columns -- is meant to be harvested, and this script does not invent any of it. 2. Three hasSource values were not systems at all but migration notes ("DGO proposal, split of a compound sub-domain"). They are dropped rather than turned into a system: provenance of a record is not a property of the thing the record describes. 3. materializedAs -> storedIn on the Data Objects. Nothing carries it yet, done for completeness so the term never reappears. """ import os import re import shutil import sys REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) TTL = os.path.join(REPO, "instances", "sodh.ttl") W = 78 # (iri, identifier, name, short label, system type, the string it replaces) SYSTEMS = [ ("ex:SYS_SODH_GOLD", "SYS-001", "Sell Out Data Hub - Gold layer", "SODH Gold", "pr:DataPlatform", "SODH Gold"), ("ex:SYS_MDM_PRODUCT", "SYS-002", "Master Data Management - Product", "MDM Product", "pr:MasterDataSystem", "MDM Product"), ("ex:SYS_MDM_CUSTOMER", "SYS-003", "Master Data Management - Customer", "MDM Customer", "pr:MasterDataSystem", "MDM Customer"), ("ex:SYS_GROUP_FINANCE", "SYS-004", "Group Finance consolidation", "Group Finance", "pr:SourceApplication", "Group Finance"), ("ex:SYS_CORPORATE_CALENDAR", "SYS-005", "Corporate standard calendar", "Standard Calendar", "pr:SourceApplication", "Standard Calendar"), ] # strings that were never a system: provenance notes about the record itself NOT_A_SYSTEM = ["DGO proposal"] def clauses(body): out, buf, i, n = [], [], 0, len(body) while i < n: c = body[i] if c == '"': j = i + 1 while j < n and body[j] != '"': j += 2 if body[j] == "\\" else 1 j = min(j + 1, n) buf.append(body[i:j]); i = j; continue if c == ";": out.append("".join(buf)); buf = []; i += 1; continue buf.append(c); i += 1 out.append("".join(buf)) return out def set_prop(text, subject, prop, value): pat = re.compile(r'(^%s a pr:\w+ ;)(.*?)(\.\s*\n)' % re.escape(subject), re.S | re.M) m = pat.search(text) if not m: return text, False kept = [c for c in clauses(m.group(2)) if c.strip() and not re.match(r'\s*%s\s' % re.escape(prop), c)] if value is not None: kept.insert(0, "\n %s %s " % (prop, value)) body = " ;".join(kept).rstrip() + "\n " return text[:m.start()] + m.group(1) + body + m.group(3) + text[m.end():], True def get_source(text, subject): m = re.search(r'^%s a pr:\w+ ;(.*?)\.\s*\n' % re.escape(subject), text, re.S | re.M) if not m: return None for c in clauses(m.group(1)): mm = re.match(r'\s*pr:hasSource\s+"([^"]*)"', c) if mm: return mm.group(1) return None def system_blocks(todo): head = ("# --- systems -----------------------------------------------------------\n" "# Seeded by hand, once. A system was a string until now, which is why\n" "# 'what depends on MDM Product' had no answer. Everything below a system --\n" "# databases, schemas, tables, columns -- is meant to be harvested, and\n" "# nothing here invents any of it.\n\n") out = [] for iri, ident, name, short, stype, _ in todo: out.append('%s a pr:System ;\n' ' pr:hasIdentifier "%s" ; pr:hasName "%s" ;\n' ' pr:hasShortLabel "%s" ;\n' ' pr:hasSystemType %s ;\n' ' pr:hasStatus "DRAFT" ; pr:hasVersion "1.4" .' % (iri, ident, name, short, stype)) return head + "\n".join(out) + "\n" def main(): apply_changes = "--apply" in sys.argv if not os.path.exists(TTL): print("Not found: %s" % TTL); sys.exit(2) text = original = open(TTL, encoding="utf-8").read() report = [] todo = [sy for sy in SYSTEMS if not re.search(r'^%s a ' % re.escape(sy[0]), text, re.M)] if todo: anchor = re.search(r'^ex:DD_\w+ a pr:DataDomain', text, re.M) pos = anchor.start() if anchor else 0 text = text[:pos] + system_blocks(todo) + "\n" + text[pos:] report.append("%d systems created%s" % (len(todo), "" if todo else " (already present)")) by_string = {sy[5]: sy[0] for sy in SYSTEMS} n_ok = n_drop = 0 for m in list(re.finditer(r'^(ex:\w+) a pr:\w+ ;', text, re.M)): subj = m.group(1) src = get_source(text, subj) if src is None: continue if src in by_string: text, _ = set_prop(text, subj, "pr:hasSource", None) text, _ = set_prop(text, subj, "pr:sourcedFrom", by_string[src]) n_ok += 1 elif any(k in src for k in NOT_A_SYSTEM): text, _ = set_prop(text, subj, "pr:hasSource", None) n_drop += 1 report.append("sourcedFrom set on %d objects" % n_ok) report.append("%d provenance notes dropped rather than turned into a system" % n_drop) text, n = re.subn(r'\bpr:materializedAs\b', 'pr:storedIn', text) report.append("materializedAs -> storedIn: %d occurrence(s)" % n) print() print("SODH MIGRATION v1.3 -> v1.4 %s" % ("APPLY" if apply_changes else "DRY RUN")) print("=" * W) for line in report: print(" " + line) print("=" * W) left = re.findall(r'pr:hasSource\s+"([^"]*)"', text) defined = set(re.findall(r'^(ex:\w+) a ', text, re.M)) dangling = sorted(set(re.findall(r'\bex:\w+', text)) - defined) doubled = [] for m in re.finditer(r'^(ex:\w+) a pr:\w+ ;(.*?)\.\s*\n', text, re.S | re.M): for p in ("pr:sourcedFrom", "pr:hasName", "pr:hasShortLabel"): if len(re.findall(r'(? 1: doubled.append("%s/%s" % (m.group(1), p)) print(" hasSource left: %d %s" % (len(left), sorted(set(left)) if left else "(must be 0)")) print(" sourcedFrom references: %d" % len(re.findall(r'pr:sourcedFrom', text))) print(" dangling references: %s" % (", ".join(dangling) if dangling else "none")) print(" duplicated single-valued properties: %s" % (", ".join(doubled) if doubled else "none")) print("=" * W) if text != original and apply_changes: shutil.copy2(TTL, TTL + ".bak") open(TTL, "w", encoding="utf-8").write(text) print(" written, backup at %s.bak" % os.path.basename(TTL)) elif text != original: print(" dry run -- re-run with --apply to write") print() if __name__ == "__main__": main()