Bastien Gourdon
203 lines
8.1 KiB
Python
203 lines
8.1 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
SODH - data fixes v1.2
|
|
=======================
|
|
Four corrections found while reviewing the viewer.
|
|
|
|
USAGE
|
|
python3 scripts/apply_fixes_v1_2.py # dry run
|
|
python3 scripts/apply_fixes_v1_2.py --apply # rewrite, .bak kept
|
|
|
|
1. DATA DOMAIN OWNERS
|
|
Six named owners added, plus the hasDomainOwner link. Ownership was
|
|
readable only through the DGL until now, which conflates two roles: the
|
|
owner is accountable for the domain, the DGL runs its governance.
|
|
|
|
2. hasSource CARRIED TWO MEANINGS
|
|
On a Data Element it named the system of origin (MDM Product, Group
|
|
Finance). On a Data Object it named the physical location (SODH Gold /
|
|
F_SELL_OUT). Same property, two concepts -- the defect we removed from
|
|
composedOf, still present one layer down. Split: physicalName holds the
|
|
relation name, hasSource holds the system. This is also what makes the
|
|
physical layer derivable: F_SELL_OUT plus 132 column names.
|
|
|
|
3. METRIC UNITS ARE PLURAL
|
|
Five mother metrics compute elements in several units (9L and L, EUR and
|
|
LC and USD). A metric states every unit its variants are expressed in; a
|
|
Data Element states exactly one. Recomputed from the elements rather than
|
|
asserted, so the two can never drift.
|
|
|
|
4. Sell Out Distribution -> Retail Distribution, matching the concept: sell
|
|
out counts units sold, distribution measures presence in store.
|
|
"""
|
|
import os
|
|
import re
|
|
import shutil
|
|
import sys
|
|
|
|
REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
|
TTL = os.path.join(REPO, "instances", "sodh.ttl")
|
|
W = 78
|
|
|
|
OWNERS = [
|
|
("ex:DDO_DD_06", "Ludovic Cottier", "ex:DD_06"),
|
|
("ex:DDO_DD_04", "David Carpio", "ex:DD_04"),
|
|
("ex:DDO_DD_10", "Alberto Lupano", "ex:DD_10"),
|
|
("ex:DDO_DD_05", "David Carpio", "ex:DD_05"),
|
|
("ex:DDO_DD_16", "Julien Soisson", "ex:DD_16"),
|
|
("ex:DDO_DD_21", "Vincent Meunier", "ex:DD_21"),
|
|
]
|
|
|
|
RENAME = {"ex:BO_06_01_003": ("Sell Out Distribution", "Retail Distribution")}
|
|
|
|
# system of origin, once the physical location is taken out of it
|
|
SOURCE_FIX = {
|
|
"Snowflake (SODH Gold)": "SODH Gold",
|
|
"SODH Gold (flag)": "SODH Gold",
|
|
}
|
|
|
|
|
|
def clauses(body):
|
|
out, buf, i, n = [], [], 0, len(body)
|
|
while i < n:
|
|
c = body[i]
|
|
if c == '"':
|
|
j = i + 1
|
|
while j < n and body[j] != '"':
|
|
j += 2 if body[j] == "\\" else 1
|
|
j = min(j + 1, n)
|
|
buf.append(body[i:j]); i = j; continue
|
|
if c == ";":
|
|
out.append("".join(buf)); buf = []; i += 1; continue
|
|
buf.append(c); i += 1
|
|
out.append("".join(buf))
|
|
return out
|
|
|
|
|
|
def set_prop(text, subject, prop, value):
|
|
"""Clause-level set: survives mid-line properties and semicolons in literals."""
|
|
pat = re.compile(r'(^%s a pr:\w+ ;)(.*?)(\.\s*\n)' % re.escape(subject), re.S | re.M)
|
|
m = pat.search(text)
|
|
if not m:
|
|
return text, False
|
|
kept = [c for c in clauses(m.group(2))
|
|
if c.strip() and not re.match(r'\s*%s\s' % re.escape(prop), c)]
|
|
if value is not None:
|
|
kept.insert(0, "\n %s %s " % (prop, value))
|
|
body = " ;".join(kept).rstrip() + "\n "
|
|
return text[:m.start()] + m.group(1) + body + m.group(3) + text[m.end():], True
|
|
|
|
|
|
def get_prop(text, subject, prop):
|
|
m = re.search(r'^%s a pr:\w+ ;(.*?)\.\s*\n' % re.escape(subject), text, re.S | re.M)
|
|
if not m:
|
|
return None
|
|
for c in clauses(m.group(1)):
|
|
mm = re.match(r'\s*%s\s+(.*)$' % re.escape(prop), c, re.S)
|
|
if mm:
|
|
return mm.group(1).strip()
|
|
return None
|
|
|
|
|
|
def main():
|
|
apply_changes = "--apply" in sys.argv
|
|
if not os.path.exists(TTL):
|
|
print("Not found: %s" % TTL); sys.exit(2)
|
|
text = original = open(TTL, encoding="utf-8").read()
|
|
report = []
|
|
|
|
# ---- 1. domain owners -------------------------------------------------
|
|
todo = [o for o in OWNERS if not re.search(r'^%s a ' % re.escape(o[0]), text, re.M)]
|
|
if todo:
|
|
blocks = "\n".join('%s a pr:DataDomainOwner ; pr:hasName "%s" .' % (i, n)
|
|
for i, n, _ in todo)
|
|
anchor = re.search(r'^ex:ST_\w+ a pr:DataSteward', text, re.M)
|
|
pos = anchor.start() if anchor else 0
|
|
text = (text[:pos]
|
|
+ "# --- data domain owners ----------------------------------------------\n"
|
|
+ "# Accountable for the domain. Distinct from the DGL, who runs its\n"
|
|
+ "# governance: two roles that hasDGL alone was conflating.\n"
|
|
+ blocks + "\n\n" + text[pos:])
|
|
for i, n, dd in OWNERS:
|
|
text, _ = set_prop(text, dd, "pr:hasDomainOwner", i)
|
|
report.append("%d data domain owners named and linked" % len(OWNERS))
|
|
|
|
# ---- 2. physical location out of hasSource ----------------------------
|
|
n_do = 0
|
|
for m in list(re.finditer(r'^(ex:DO_\w+) a pr:DataObject ;', text, re.M)):
|
|
subj = m.group(1)
|
|
src = get_prop(text, subj, "pr:hasSource")
|
|
if src and "/" in src:
|
|
system, relation = [p.strip().strip('"') for p in src.strip('"').split("/", 1)]
|
|
text, _ = set_prop(text, subj, "pr:hasSource", '"%s"' % system)
|
|
text, _ = set_prop(text, subj, "pr:physicalName", '"%s"' % relation)
|
|
n_do += 1
|
|
report.append("%d data objects: physical relation moved out of hasSource" % n_do)
|
|
|
|
n_de = 0
|
|
for old, new in SOURCE_FIX.items():
|
|
text, k = re.subn(r'pr:hasSource "%s"' % re.escape(old),
|
|
'pr:hasSource "%s"' % new, text)
|
|
n_de += k
|
|
report.append("%d source labels normalised to the system of origin" % n_de)
|
|
|
|
# ---- 3. metric units, recomputed from the elements ---------------------
|
|
de_unit = {}
|
|
for m in re.finditer(r'^(ex:DE_\w+) a pr:DataElement ;(.*?)\.\s*\n', text, re.S | re.M):
|
|
u = re.search(r'pr:hasUnit\s+"([^"]*)"', m.group(2))
|
|
if u:
|
|
de_unit[m.group(1)] = u.group(1)
|
|
wired = {}
|
|
for m in re.finditer(r'(ex:M_\w+)\s+pr:computedBy(.*?)\.', text, re.S):
|
|
wired[m.group(1)] = re.findall(r'ex:DE_\w+', m.group(2))
|
|
n_m = 0
|
|
for metric, des in wired.items():
|
|
units = sorted({de_unit[d] for d in des if d in de_unit and de_unit[d] != "-"})
|
|
if not units:
|
|
continue
|
|
value = " , ".join('"%s"' % u for u in units)
|
|
before = get_prop(text, metric, "pr:hasUnit")
|
|
text, ok = set_prop(text, metric, "pr:hasUnit", value)
|
|
if ok and before != value:
|
|
n_m += 1
|
|
report.append("%d metrics now list every unit their elements use" % n_m)
|
|
|
|
# ---- 4. rename --------------------------------------------------------
|
|
for subj, (old, new) in RENAME.items():
|
|
if get_prop(text, subj, "pr:hasName") == '"%s"' % old:
|
|
text, _ = set_prop(text, subj, "pr:hasName", '"%s"' % new)
|
|
report.append("renamed %s -> %s" % (old, new))
|
|
|
|
# ---- report -----------------------------------------------------------
|
|
print()
|
|
print("SODH DATA FIXES v1.2 %s" % ("APPLY" if apply_changes else "DRY RUN"))
|
|
print("=" * W)
|
|
for line in report:
|
|
print(" " + line)
|
|
print("=" * W)
|
|
multi = len(re.findall(r'pr:hasUnit\s+"[^"]*"\s*,', text))
|
|
phys = len(re.findall(r'^ex:DO_\w+ a pr:DataObject ;(?:(?!\.\s*\n).)*?physicalName',
|
|
text, re.S | re.M))
|
|
doubled = []
|
|
for m in re.finditer(r'^(ex:\w+) a pr:\w+ ;(.*?)\.\s*\n', text, re.S | re.M):
|
|
for p in ("pr:hasName", "pr:hasSource", "pr:physicalName", "pr:hasDomainOwner"):
|
|
if len(re.findall(r'(?<![\w:])%s\s' % re.escape(p), m.group(2))) > 1:
|
|
doubled.append("%s/%s" % (m.group(1), p))
|
|
print(" metrics with several units %d | data objects with a physical name %d"
|
|
% (multi, phys))
|
|
print(" duplicated single-valued properties: %s"
|
|
% (", ".join(doubled) if doubled else "none"))
|
|
print("=" * W)
|
|
|
|
if text != original and apply_changes:
|
|
shutil.copy2(TTL, TTL + ".bak")
|
|
open(TTL, "w", encoding="utf-8").write(text)
|
|
print(" written, backup at %s.bak" % os.path.basename(TTL))
|
|
elif text != original:
|
|
print(" dry run -- re-run with --apply to write")
|
|
print()
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|