data-meta-model/scripts/migrate_instances_v1_4.py

#!/usr/bin/env python3
"""
SODH - migration v1.3 -> v1.4
==============================
USAGE
    python3 scripts/migrate_instances_v1_4.py            # dry run
    python3 scripts/migrate_instances_v1_4.py --apply    # rewrite, .bak kept

1. SYSTEMS BECOME NODES. hasSource held a string: "SODH Gold", "MDM Product",
   "Group Finance". Six systems are created and sourcedFrom points at them.
   "Which objects depend on MDM Product?" was unanswerable while MDM Product
   was text.

   The systems are seeded here by hand, once. Everything below them --
   databases, schemas, tables, columns -- is meant to be harvested, and this
   script does not invent any of it.

2. Three hasSource values were not systems at all but migration notes ("DGO
   proposal, split of a compound sub-domain"). They are dropped rather than
   turned into a system: provenance of a record is not a property of the thing
   the record describes.

3. materializedAs -> storedIn on the Data Objects. Nothing carries it yet,
   done for completeness so the term never reappears.
"""
import os
import re
import shutil
import sys

REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
TTL = os.path.join(REPO, "instances", "sodh.ttl")
W = 78

# (iri, identifier, name, short label, system type, the string it replaces)
SYSTEMS = [
    ("ex:SYS_SODH_GOLD", "SYS-001", "Sell Out Data Hub - Gold layer", "SODH Gold",
     "pr:DataPlatform", "SODH Gold"),
    ("ex:SYS_MDM_PRODUCT", "SYS-002", "Master Data Management - Product", "MDM Product",
     "pr:MasterDataSystem", "MDM Product"),
    ("ex:SYS_MDM_CUSTOMER", "SYS-003", "Master Data Management - Customer", "MDM Customer",
     "pr:MasterDataSystem", "MDM Customer"),
    ("ex:SYS_GROUP_FINANCE", "SYS-004", "Group Finance consolidation", "Group Finance",
     "pr:SourceApplication", "Group Finance"),
    ("ex:SYS_CORPORATE_CALENDAR", "SYS-005", "Corporate standard calendar", "Standard Calendar",
     "pr:SourceApplication", "Standard Calendar"),
]

# strings that were never a system: provenance notes about the record itself
NOT_A_SYSTEM = ["DGO proposal"]


def clauses(body):
    out, buf, i, n = [], [], 0, len(body)
    while i < n:
        c = body[i]
        if c == '"':
            j = i + 1
            while j < n and body[j] != '"':
                j += 2 if body[j] == "\\" else 1
            j = min(j + 1, n)
            buf.append(body[i:j]); i = j; continue
        if c == ";":
            out.append("".join(buf)); buf = []; i += 1; continue
        buf.append(c); i += 1
    out.append("".join(buf))
    return out


def set_prop(text, subject, prop, value):
    pat = re.compile(r'(^%s a pr:\w+ ;)(.*?)(\.\s*\n)' % re.escape(subject), re.S | re.M)
    m = pat.search(text)
    if not m:
        return text, False
    kept = [c for c in clauses(m.group(2))
            if c.strip() and not re.match(r'\s*%s\s' % re.escape(prop), c)]
    if value is not None:
        kept.insert(0, "\n    %s %s " % (prop, value))
    body = " ;".join(kept).rstrip() + "\n    "
    return text[:m.start()] + m.group(1) + body + m.group(3) + text[m.end():], True


def get_source(text, subject):
    m = re.search(r'^%s a pr:\w+ ;(.*?)\.\s*\n' % re.escape(subject), text, re.S | re.M)
    if not m:
        return None
    for c in clauses(m.group(1)):
        mm = re.match(r'\s*pr:hasSource\s+"([^"]*)"', c)
        if mm:
            return mm.group(1)
    return None


def system_blocks(todo):
    head = ("# --- systems -----------------------------------------------------------\n"
            "# Seeded by hand, once. A system was a string until now, which is why\n"
            "# 'what depends on MDM Product' had no answer. Everything below a system --\n"
            "# databases, schemas, tables, columns -- is meant to be harvested, and\n"
            "# nothing here invents any of it.\n\n")
    out = []
    for iri, ident, name, short, stype, _ in todo:
        out.append('%s a pr:System ;\n'
                   '    pr:hasIdentifier "%s" ; pr:hasName "%s" ;\n'
                   '    pr:hasShortLabel "%s" ;\n'
                   '    pr:hasSystemType %s ;\n'
                   '    pr:hasStatus "DRAFT" ; pr:hasVersion "1.4" .'
                   % (iri, ident, name, short, stype))
    return head + "\n".join(out) + "\n"


def main():
    apply_changes = "--apply" in sys.argv
    if not os.path.exists(TTL):
        print("Not found: %s" % TTL); sys.exit(2)
    text = original = open(TTL, encoding="utf-8").read()
    report = []

    todo = [sy for sy in SYSTEMS if not re.search(r'^%s a ' % re.escape(sy[0]), text, re.M)]
    if todo:
        anchor = re.search(r'^ex:DD_\w+ a pr:DataDomain', text, re.M)
        pos = anchor.start() if anchor else 0
        text = text[:pos] + system_blocks(todo) + "\n" + text[pos:]
    report.append("%d systems created%s" % (len(todo), "" if todo else " (already present)"))

    by_string = {sy[5]: sy[0] for sy in SYSTEMS}
    n_ok = n_drop = 0
    for m in list(re.finditer(r'^(ex:\w+) a pr:\w+ ;', text, re.M)):
        subj = m.group(1)
        src = get_source(text, subj)
        if src is None:
            continue
        if src in by_string:
            text, _ = set_prop(text, subj, "pr:hasSource", None)
            text, _ = set_prop(text, subj, "pr:sourcedFrom", by_string[src])
            n_ok += 1
        elif any(k in src for k in NOT_A_SYSTEM):
            text, _ = set_prop(text, subj, "pr:hasSource", None)
            n_drop += 1
    report.append("sourcedFrom set on %d objects" % n_ok)
    report.append("%d provenance notes dropped rather than turned into a system" % n_drop)

    text, n = re.subn(r'\bpr:materializedAs\b', 'pr:storedIn', text)
    report.append("materializedAs -> storedIn: %d occurrence(s)" % n)

    print()
    print("SODH MIGRATION v1.3 -> v1.4   %s" % ("APPLY" if apply_changes else "DRY RUN"))
    print("=" * W)
    for line in report:
        print("  " + line)
    print("=" * W)
    left = re.findall(r'pr:hasSource\s+"([^"]*)"', text)
    defined = set(re.findall(r'^(ex:\w+) a ', text, re.M))
    dangling = sorted(set(re.findall(r'\bex:\w+', text)) - defined)
    doubled = []
    for m in re.finditer(r'^(ex:\w+) a pr:\w+ ;(.*?)\.\s*\n', text, re.S | re.M):
        for p in ("pr:sourcedFrom", "pr:hasName", "pr:hasShortLabel"):
            if len(re.findall(r'(?<![\w:])%s\s' % re.escape(p), m.group(2))) > 1:
                doubled.append("%s/%s" % (m.group(1), p))
    print("  hasSource left: %d   %s" % (len(left), sorted(set(left)) if left else "(must be 0)"))
    print("  sourcedFrom references: %d" % len(re.findall(r'pr:sourcedFrom', text)))
    print("  dangling references: %s" % (", ".join(dangling) if dangling else "none"))
    print("  duplicated single-valued properties: %s"
          % (", ".join(doubled) if doubled else "none"))
    print("=" * W)

    if text != original and apply_changes:
        shutil.copy2(TTL, TTL + ".bak")
        open(TTL, "w", encoding="utf-8").write(text)
        print("  written, backup at %s.bak" % os.path.basename(TTL))
    elif text != original:
        print("  dry run -- re-run with --apply to write")
    print()


if __name__ == "__main__":
    main()