tbox: v1.2 - physicalizedIn devient storedIn, hasSource ramene a un seul sens (systeme detenant la donnee faisant autorite) et interdit sur les Business Concepts. Table supprime, MaterializedView / constrainsRelation / approximateRowCount / hasVendor deprecies. shapes: controle que la source d'une metrique correspond a celle de ses elements, unite multiple sur Metric et unique sur DataElement, hasProductOwner en avertissement. instances: Data Product SODH avec contrat et deux interfaces, 6 proprietaires de domaine nommes, Sell Out Distribution devient Retail Distribution. generated: les deux viewers partagent desormais le meme moteur
This commit is contained in:
Bastien Gourdon
@@ -0,0 +1,202 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
SODH - data fixes v1.2
|
||||
=======================
|
||||
Four corrections found while reviewing the viewer.
|
||||
|
||||
USAGE
|
||||
python3 scripts/apply_fixes_v1_2.py # dry run
|
||||
python3 scripts/apply_fixes_v1_2.py --apply # rewrite, .bak kept
|
||||
|
||||
1. DATA DOMAIN OWNERS
|
||||
Six named owners added, plus the hasDomainOwner link. Ownership was
|
||||
readable only through the DGL until now, which conflates two roles: the
|
||||
owner is accountable for the domain, the DGL runs its governance.
|
||||
|
||||
2. hasSource CARRIED TWO MEANINGS
|
||||
On a Data Element it named the system of origin (MDM Product, Group
|
||||
Finance). On a Data Object it named the physical location (SODH Gold /
|
||||
F_SELL_OUT). Same property, two concepts -- the defect we removed from
|
||||
composedOf, still present one layer down. Split: physicalName holds the
|
||||
relation name, hasSource holds the system. This is also what makes the
|
||||
physical layer derivable: F_SELL_OUT plus 132 column names.
|
||||
|
||||
3. METRIC UNITS ARE PLURAL
|
||||
Five mother metrics compute elements in several units (9L and L, EUR and
|
||||
LC and USD). A metric states every unit its variants are expressed in; a
|
||||
Data Element states exactly one. Recomputed from the elements rather than
|
||||
asserted, so the two can never drift.
|
||||
|
||||
4. Sell Out Distribution -> Retail Distribution, matching the concept: sell
|
||||
out counts units sold, distribution measures presence in store.
|
||||
"""
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import sys
|
||||
|
||||
REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
TTL = os.path.join(REPO, "instances", "sodh.ttl")
|
||||
W = 78
|
||||
|
||||
OWNERS = [
|
||||
("ex:DDO_DD_06", "Ludovic Cottier", "ex:DD_06"),
|
||||
("ex:DDO_DD_04", "David Carpio", "ex:DD_04"),
|
||||
("ex:DDO_DD_10", "Alberto Lupano", "ex:DD_10"),
|
||||
("ex:DDO_DD_05", "David Carpio", "ex:DD_05"),
|
||||
("ex:DDO_DD_16", "Julien Soisson", "ex:DD_16"),
|
||||
("ex:DDO_DD_21", "Vincent Meunier", "ex:DD_21"),
|
||||
]
|
||||
|
||||
RENAME = {"ex:BO_06_01_003": ("Sell Out Distribution", "Retail Distribution")}
|
||||
|
||||
# system of origin, once the physical location is taken out of it
|
||||
SOURCE_FIX = {
|
||||
"Snowflake (SODH Gold)": "SODH Gold",
|
||||
"SODH Gold (flag)": "SODH Gold",
|
||||
}
|
||||
|
||||
|
||||
def clauses(body):
|
||||
out, buf, i, n = [], [], 0, len(body)
|
||||
while i < n:
|
||||
c = body[i]
|
||||
if c == '"':
|
||||
j = i + 1
|
||||
while j < n and body[j] != '"':
|
||||
j += 2 if body[j] == "\\" else 1
|
||||
j = min(j + 1, n)
|
||||
buf.append(body[i:j]); i = j; continue
|
||||
if c == ";":
|
||||
out.append("".join(buf)); buf = []; i += 1; continue
|
||||
buf.append(c); i += 1
|
||||
out.append("".join(buf))
|
||||
return out
|
||||
|
||||
|
||||
def set_prop(text, subject, prop, value):
|
||||
"""Clause-level set: survives mid-line properties and semicolons in literals."""
|
||||
pat = re.compile(r'(^%s a pr:\w+ ;)(.*?)(\.\s*\n)' % re.escape(subject), re.S | re.M)
|
||||
m = pat.search(text)
|
||||
if not m:
|
||||
return text, False
|
||||
kept = [c for c in clauses(m.group(2))
|
||||
if c.strip() and not re.match(r'\s*%s\s' % re.escape(prop), c)]
|
||||
if value is not None:
|
||||
kept.insert(0, "\n %s %s " % (prop, value))
|
||||
body = " ;".join(kept).rstrip() + "\n "
|
||||
return text[:m.start()] + m.group(1) + body + m.group(3) + text[m.end():], True
|
||||
|
||||
|
||||
def get_prop(text, subject, prop):
|
||||
m = re.search(r'^%s a pr:\w+ ;(.*?)\.\s*\n' % re.escape(subject), text, re.S | re.M)
|
||||
if not m:
|
||||
return None
|
||||
for c in clauses(m.group(1)):
|
||||
mm = re.match(r'\s*%s\s+(.*)$' % re.escape(prop), c, re.S)
|
||||
if mm:
|
||||
return mm.group(1).strip()
|
||||
return None
|
||||
|
||||
|
||||
def main():
|
||||
apply_changes = "--apply" in sys.argv
|
||||
if not os.path.exists(TTL):
|
||||
print("Not found: %s" % TTL); sys.exit(2)
|
||||
text = original = open(TTL, encoding="utf-8").read()
|
||||
report = []
|
||||
|
||||
# ---- 1. domain owners -------------------------------------------------
|
||||
todo = [o for o in OWNERS if not re.search(r'^%s a ' % re.escape(o[0]), text, re.M)]
|
||||
if todo:
|
||||
blocks = "\n".join('%s a pr:DataDomainOwner ; pr:hasName "%s" .' % (i, n)
|
||||
for i, n, _ in todo)
|
||||
anchor = re.search(r'^ex:ST_\w+ a pr:DataSteward', text, re.M)
|
||||
pos = anchor.start() if anchor else 0
|
||||
text = (text[:pos]
|
||||
+ "# --- data domain owners ----------------------------------------------\n"
|
||||
+ "# Accountable for the domain. Distinct from the DGL, who runs its\n"
|
||||
+ "# governance: two roles that hasDGL alone was conflating.\n"
|
||||
+ blocks + "\n\n" + text[pos:])
|
||||
for i, n, dd in OWNERS:
|
||||
text, _ = set_prop(text, dd, "pr:hasDomainOwner", i)
|
||||
report.append("%d data domain owners named and linked" % len(OWNERS))
|
||||
|
||||
# ---- 2. physical location out of hasSource ----------------------------
|
||||
n_do = 0
|
||||
for m in list(re.finditer(r'^(ex:DO_\w+) a pr:DataObject ;', text, re.M)):
|
||||
subj = m.group(1)
|
||||
src = get_prop(text, subj, "pr:hasSource")
|
||||
if src and "/" in src:
|
||||
system, relation = [p.strip().strip('"') for p in src.strip('"').split("/", 1)]
|
||||
text, _ = set_prop(text, subj, "pr:hasSource", '"%s"' % system)
|
||||
text, _ = set_prop(text, subj, "pr:physicalName", '"%s"' % relation)
|
||||
n_do += 1
|
||||
report.append("%d data objects: physical relation moved out of hasSource" % n_do)
|
||||
|
||||
n_de = 0
|
||||
for old, new in SOURCE_FIX.items():
|
||||
text, k = re.subn(r'pr:hasSource "%s"' % re.escape(old),
|
||||
'pr:hasSource "%s"' % new, text)
|
||||
n_de += k
|
||||
report.append("%d source labels normalised to the system of origin" % n_de)
|
||||
|
||||
# ---- 3. metric units, recomputed from the elements ---------------------
|
||||
de_unit = {}
|
||||
for m in re.finditer(r'^(ex:DE_\w+) a pr:DataElement ;(.*?)\.\s*\n', text, re.S | re.M):
|
||||
u = re.search(r'pr:hasUnit\s+"([^"]*)"', m.group(2))
|
||||
if u:
|
||||
de_unit[m.group(1)] = u.group(1)
|
||||
wired = {}
|
||||
for m in re.finditer(r'(ex:M_\w+)\s+pr:computedBy(.*?)\.', text, re.S):
|
||||
wired[m.group(1)] = re.findall(r'ex:DE_\w+', m.group(2))
|
||||
n_m = 0
|
||||
for metric, des in wired.items():
|
||||
units = sorted({de_unit[d] for d in des if d in de_unit and de_unit[d] != "-"})
|
||||
if not units:
|
||||
continue
|
||||
value = " , ".join('"%s"' % u for u in units)
|
||||
before = get_prop(text, metric, "pr:hasUnit")
|
||||
text, ok = set_prop(text, metric, "pr:hasUnit", value)
|
||||
if ok and before != value:
|
||||
n_m += 1
|
||||
report.append("%d metrics now list every unit their elements use" % n_m)
|
||||
|
||||
# ---- 4. rename --------------------------------------------------------
|
||||
for subj, (old, new) in RENAME.items():
|
||||
if get_prop(text, subj, "pr:hasName") == '"%s"' % old:
|
||||
text, _ = set_prop(text, subj, "pr:hasName", '"%s"' % new)
|
||||
report.append("renamed %s -> %s" % (old, new))
|
||||
|
||||
# ---- report -----------------------------------------------------------
|
||||
print()
|
||||
print("SODH DATA FIXES v1.2 %s" % ("APPLY" if apply_changes else "DRY RUN"))
|
||||
print("=" * W)
|
||||
for line in report:
|
||||
print(" " + line)
|
||||
print("=" * W)
|
||||
multi = len(re.findall(r'pr:hasUnit\s+"[^"]*"\s*,', text))
|
||||
phys = len(re.findall(r'^ex:DO_\w+ a pr:DataObject ;(?:(?!\.\s*\n).)*?physicalName',
|
||||
text, re.S | re.M))
|
||||
doubled = []
|
||||
for m in re.finditer(r'^(ex:\w+) a pr:\w+ ;(.*?)\.\s*\n', text, re.S | re.M):
|
||||
for p in ("pr:hasName", "pr:hasSource", "pr:physicalName", "pr:hasDomainOwner"):
|
||||
if len(re.findall(r'(?<![\w:])%s\s' % re.escape(p), m.group(2))) > 1:
|
||||
doubled.append("%s/%s" % (m.group(1), p))
|
||||
print(" metrics with several units %d | data objects with a physical name %d"
|
||||
% (multi, phys))
|
||||
print(" duplicated single-valued properties: %s"
|
||||
% (", ".join(doubled) if doubled else "none"))
|
||||
print("=" * W)
|
||||
|
||||
if text != original and apply_changes:
|
||||
shutil.copy2(TTL, TTL + ".bak")
|
||||
open(TTL, "w", encoding="utf-8").write(text)
|
||||
print(" written, backup at %s.bak" % os.path.basename(TTL))
|
||||
elif text != original:
|
||||
print(" dry run -- re-run with --apply to write")
|
||||
print()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user