Compare commits
3 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 76e95de471 | |||
| 7793def268 | |||
| 45ca9ffb68 |
+831
-550
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
+1346
-388
File diff suppressed because it is too large
Load Diff
+66
-21
@@ -7,11 +7,45 @@
|
||||
@prefix pr: <https://ontology.pernod-ricard.com/metamodel/> .
|
||||
|
||||
#################################################################
|
||||
# PERNOD RICARD DATA META MODEL - T-Box v1.1
|
||||
# PERNOD RICARD DATA META MODEL - T-Box v1.2
|
||||
# Owner: Bastien Gourdon - Data Governance Office
|
||||
#
|
||||
# WHAT CHANGED vs v1.0
|
||||
# WHAT CHANGED vs v1.1
|
||||
# --------------------
|
||||
# A. physicalizedIn -> storedIn. "Physicalized in" is jargon; the relation
|
||||
# says where the data sits, and "stored in" says that to anyone.
|
||||
#
|
||||
# B. hasSource MEANS ONE THING: the system that holds the authoritative data.
|
||||
# It was drifting into a third meaning on Business Concepts and Metrics,
|
||||
# where it named the DOCUMENT a record came from ("SODH back-doc v0.7").
|
||||
# A business concept has no data source -- it has an owning domain, which
|
||||
# is its authority, and a definition published in the glossary. The
|
||||
# property is now forbidden on concepts by SHACL.
|
||||
#
|
||||
# C. FIVE TERMS REMOVED OR DEPRECATED, none of them instantiated anywhere:
|
||||
# Table deleted -- deprecated since v1.0, replaced by
|
||||
# BaseTable, and never used. A deprecation that
|
||||
# never ends stops meaning anything.
|
||||
# constrainsRelation derivable: constrainsField -> livesIn gives the
|
||||
# relation. An asserted edge redundant with an
|
||||
# existing path is a future divergence.
|
||||
# MaterializedView a class standing in for an attribute; being
|
||||
# derived is what dependsOn already says.
|
||||
# approximateRowCount volumetry with no identified use. queryCount30d
|
||||
# and lastQueriedOn carry the rationalisation case.
|
||||
# hasVendor marginal.
|
||||
#
|
||||
# D. hasProductOwner drops from BLOCKING to a warning in the shapes. The SODH
|
||||
# product owner is genuinely not appointed yet, and a blocking rule that
|
||||
# cannot be satisfied truthfully forces invention -- the BR-004 lesson.
|
||||
#
|
||||
# Environment and deployedIn stay, unused. Without them a DEV table and its
|
||||
# PROD twin are indistinguishable and lineage pollutes silently on the first
|
||||
# multi-environment harvest. Unused because nothing is harvested yet, not
|
||||
# because they are useless.
|
||||
#
|
||||
# WHAT CHANGED vs v1.0 (kept for reference)
|
||||
# -----------------------------------------
|
||||
# A. RELATIONS SIMPLIFIED AND GIVEN HONEST NAMES
|
||||
# belongsToDomain + belongsToSubDomain -> belongsTo (one property,
|
||||
# scoped per class by SHACL; reusable if a domain family appears).
|
||||
@@ -117,12 +151,12 @@
|
||||
|
||||
<https://ontology.pernod-ricard.com/metamodel/>
|
||||
a owl:Ontology ;
|
||||
owl:versionIRI <https://ontology.pernod-ricard.com/metamodel/1.1> ;
|
||||
owl:priorVersion <https://ontology.pernod-ricard.com/metamodel/1.0> ;
|
||||
owl:versionIRI <https://ontology.pernod-ricard.com/metamodel/1.2> ;
|
||||
owl:priorVersion <https://ontology.pernod-ricard.com/metamodel/1.1> ;
|
||||
dcterms:title "Pernod Ricard Data Meta Model" ;
|
||||
dcterms:creator "Bastien Gourdon" ;
|
||||
dcterms:modified "2026-07-26"^^xsd:date ;
|
||||
owl:versionInfo "1.1.0-draft-2026-07-26" ;
|
||||
dcterms:modified "2026-07-27"^^xsd:date ;
|
||||
owl:versionInfo "1.2.0-draft-2026-07-27" ;
|
||||
rdfs:comment """OWL T-Box. Six layers (Ownership, Business, Logical, Delivery, Physical, Consumption) crossed with a provenance axis (Curated vs Observed). Curated objects are asserted by governance and reviewed; Observed objects are harvested from Snowflake, Tableau and Looker and are rebuilt, never edited. Bridge annotation properties are declared for future alignment with domain ontologies.""" .
|
||||
|
||||
|
||||
@@ -294,18 +328,14 @@ pr:View a owl:Class ; rdfs:subClassOf pr:PhysicalRelation ;
|
||||
rdfs:comment "Depends on other relations through dependsOn, so impact is transitive." .
|
||||
|
||||
pr:MaterializedView a owl:Class ; rdfs:subClassOf pr:PhysicalRelation ;
|
||||
rdfs:label "Materialized View" .
|
||||
rdfs:label "Materialized View (deprecated)" ;
|
||||
owl:deprecated true ; dcterms:isReplacedBy pr:View ;
|
||||
rdfs:comment "A class standing in for an attribute: what distinguishes it is being derived, which dependsOn already states." .
|
||||
|
||||
pr:ExternalTable a owl:Class ; rdfs:subClassOf pr:PhysicalRelation ;
|
||||
rdfs:label "External Table" ;
|
||||
rdfs:comment "Backed by files outside the warehouse. Marks the boundary where lineage leaves Snowflake." .
|
||||
|
||||
pr:Table a owl:Class ; rdfs:subClassOf pr:PhysicalRelation ;
|
||||
rdfs:label "Table (deprecated)" ;
|
||||
owl:deprecated true ;
|
||||
dcterms:isReplacedBy pr:BaseTable ;
|
||||
rdfs:comment "v0.9 term. Kept so that existing assertions still classify. Use BaseTable." .
|
||||
|
||||
pr:Field a owl:Class ; rdfs:subClassOf pr:PhysicalLayerObject ;
|
||||
rdfs:label "Field" ;
|
||||
pr:harvestSource "SNOWFLAKE.INFORMATION_SCHEMA.COLUMNS" ;
|
||||
@@ -550,11 +580,15 @@ pr:materializedAs a owl:ObjectProperty ;
|
||||
pr:curationMode "PROPOSED_CONFIRMED" ;
|
||||
rdfs:comment "Data Object to physical relation. Not functional: the same Data Object exists in DEV, UAT and PROD, and sometimes across several tables." .
|
||||
|
||||
pr:physicalizedIn a owl:ObjectProperty ;
|
||||
rdfs:label "physicalized in" ;
|
||||
pr:storedIn a owl:ObjectProperty ;
|
||||
rdfs:label "stored in" ;
|
||||
rdfs:domain pr:DataElement ; rdfs:range pr:Field ;
|
||||
pr:curationMode "PROPOSED_CONFIRMED" ;
|
||||
rdfs:comment "Data Element to physical column, 1..n. Proposed by the reconciliation job on fullyQualifiedName and type, confirmed by a Data Steward. This single edge is what makes end-to-end impact analysis possible." .
|
||||
rdfs:comment "Data Element to physical column, 1..n. Proposed by the reconciliation job on fullyQualifiedName and type, confirmed by a Data Steward. This single edge is what makes end-to-end impact analysis possible. Renamed from physicalizedIn, which was jargon: the relation says where the data sits." .
|
||||
|
||||
pr:physicalizedIn a owl:ObjectProperty ;
|
||||
rdfs:label "physicalized in (deprecated)" ;
|
||||
owl:deprecated true ; dcterms:isReplacedBy pr:storedIn .
|
||||
|
||||
|
||||
#################################################################
|
||||
@@ -597,8 +631,9 @@ pr:deployedIn a owl:ObjectProperty , owl:FunctionalProperty ;
|
||||
rdfs:domain pr:ObservedObject ; rdfs:range pr:Environment .
|
||||
|
||||
pr:constrainsRelation a owl:ObjectProperty , owl:FunctionalProperty ;
|
||||
rdfs:label "constrains relation" ;
|
||||
rdfs:domain pr:KeyConstraint ; rdfs:range pr:PhysicalRelation .
|
||||
rdfs:label "constrains relation (deprecated)" ;
|
||||
owl:deprecated true ;
|
||||
rdfs:comment "Derivable: constrainsField then livesIn gives the relation. An asserted edge redundant with an existing path is a divergence waiting to happen." .
|
||||
|
||||
pr:constrainsField a owl:ObjectProperty ;
|
||||
rdfs:label "constrains field" ;
|
||||
@@ -738,7 +773,14 @@ pr:hasStatus a owl:DatatypeProperty ; rdfs:label "status" ; rdfs:range xsd:s
|
||||
pr:hasVersion a owl:DatatypeProperty ; rdfs:label "version" ; rdfs:range xsd:string .
|
||||
pr:createdOn a owl:DatatypeProperty ; rdfs:label "created on" ; rdfs:range xsd:date .
|
||||
pr:lastReviewedOn a owl:DatatypeProperty ; rdfs:label "last reviewed on" ; rdfs:range xsd:date .
|
||||
pr:hasSource a owl:DatatypeProperty ; rdfs:label "back-doc source" ; rdfs:range xsd:string .
|
||||
pr:hasSource a owl:DatatypeProperty ; rdfs:label "source of truth" ; rdfs:range xsd:string ;
|
||||
rdfs:comment """The system holding the authoritative data for this object: SODH Gold, MDM Product, Group Finance.
|
||||
|
||||
ONE MEANING ONLY. It had drifted into naming the document a record came from ("SODH back-doc v0.7"), and into packing a system and a relation into one string ("SODH Gold / F_SELL_OUT"). The relation belongs in physicalName; the document belongs nowhere -- provenance of a record is not a property of the thing it describes.
|
||||
|
||||
MUST NOT appear on a Business Concept: a concept has no data source. Its authority is its owning domain and its definition is published in the glossary. Enforced by SHACL.
|
||||
|
||||
On a Metric it is asserted rather than derived, by exception, because "where can I get this metric" is a first-order question and the derivation chain (computedBy, then the data object holding the element) will have gaps before harvesting. A shape checks the assertion against the elements.""" .
|
||||
pr:arbitrationStatus a owl:DatatypeProperty ; rdfs:label "arbitration status" ; rdfs:range xsd:string ;
|
||||
rdfs:comment "TO_ARBITRATE blocks publication (OW-007). Use this on the object rather than tracking arbitration in a side spreadsheet." .
|
||||
|
||||
@@ -798,12 +840,15 @@ pr:hasPhysicalDataType a owl:DatatypeProperty ; rdfs:label "physical data type"
|
||||
pr:isNullable a owl:DatatypeProperty ; rdfs:label "is nullable" ; rdfs:range xsd:boolean .
|
||||
pr:ordinalPosition a owl:DatatypeProperty ; rdfs:label "ordinal position" ; rdfs:range xsd:integer .
|
||||
pr:hasViewDefinition a owl:DatatypeProperty ; rdfs:label "view definition" ; rdfs:range xsd:string .
|
||||
pr:hasVendor a owl:DatatypeProperty ; rdfs:label "vendor" ; rdfs:range xsd:string .
|
||||
pr:hasVendor a owl:DatatypeProperty ; rdfs:label "vendor (deprecated)" ; rdfs:range xsd:string ;
|
||||
owl:deprecated true .
|
||||
pr:definedBy a owl:DatatypeProperty ; rdfs:label "defined by (code URI)" ; rdfs:range xsd:anyURI ;
|
||||
rdfs:comment "Git URI of the code behind a Transformation. Connects the graph to the repository." .
|
||||
|
||||
# ---- Usage evidence (the rationalisation argument)
|
||||
pr:approximateRowCount a owl:DatatypeProperty ; rdfs:label "approximate row count" ; rdfs:range xsd:integer .
|
||||
pr:approximateRowCount a owl:DatatypeProperty ; rdfs:label "approximate row count (deprecated)" ; rdfs:range xsd:integer ;
|
||||
owl:deprecated true ;
|
||||
rdfs:comment "Volumetry with no identified use. queryCount30d and lastQueriedOn carry the rationalisation argument; row count does not." .
|
||||
pr:lastQueriedOn a owl:DatatypeProperty ; rdfs:label "last queried on" ; rdfs:range xsd:dateTime ;
|
||||
pr:harvestSource "SNOWFLAKE.ACCOUNT_USAGE.ACCESS_HISTORY" ;
|
||||
rdfs:comment "Lets the model say 'documented but unused for eight months'. A rationalisation argument that costs nothing and is denominated in warehouse spend." .
|
||||
|
||||
@@ -95,7 +95,7 @@ NEW_CONCEPTS = [
|
||||
|
||||
# ------------------------------------------------------------------- new object
|
||||
NEW_OBJECTS = [
|
||||
("ex:BO_04_01_002", "BO-04.01-002", "Outlet", "ex:DD_04", "ex:ST_DD_04", "ex:SD_04_01",
|
||||
("ex:BO_04_02_001", "BO-04.02-001", "Outlet", "ex:DD_04", "ex:ST_DD_04", "ex:SD_04_02",
|
||||
"ex:BC_04_01_002", [], "MDM Customer -> SODH [split from BO-04.01-001, proposed to DD-04]"),
|
||||
("ex:BO_16_01_002", "BO-16.01-002", "Exchange Rate", "ex:DD_16", "ex:ST_DD_16", "ex:SD_16_01",
|
||||
"ex:BC_16_01_002", [], "Group Finance -> SODH [split from BO-16.01-001, proposed to DD-16]"),
|
||||
@@ -176,7 +176,13 @@ def main():
|
||||
continue
|
||||
block = m.group(1)
|
||||
|
||||
# strip what a previous run may have written, so re-running is a no-op.
|
||||
# Duplicated clauses carrying the SAME value merge under RDF set
|
||||
# semantics: the graph stays valid and SHACL sees nothing, while the
|
||||
# file quietly grows a redundant line on every pass.
|
||||
new_block = re.sub(r'[ \t]*pr:composedOf\s+[^";]*;[ \t]*\n', '', block)
|
||||
new_block = re.sub(r'[ \t]*pr:(aboutConcept|usesConcept)\s+[^";]*;[ \t]*\n',
|
||||
'', new_block)
|
||||
clause = ' pr:aboutConcept %s ;\n' % about
|
||||
if uses:
|
||||
clause += ' pr:usesConcept %s ;\n' % " , ".join(uses)
|
||||
@@ -204,27 +210,38 @@ def main():
|
||||
" renamed=%s" % newname if newname else ""))
|
||||
|
||||
# ---- 3. insert the missing concepts --------------------------------------
|
||||
# idempotent: identical duplicated blocks merge in RDF, so SHACL would
|
||||
# never flag them -- the file grows while the graph stays valid.
|
||||
todo_c = [c for c in NEW_CONCEPTS
|
||||
if not re.search(r'^%s a ' % re.escape(c[0]), text, re.M)]
|
||||
anchor = re.search(r'\n(?=ex:BO_\w+ a pr:BusinessObject ;)', text)
|
||||
if anchor:
|
||||
blocks = "\n".join(concept_block(*c) for c in NEW_CONCEPTS)
|
||||
if anchor and todo_c:
|
||||
blocks = "\n".join(concept_block(*c) for c in todo_c)
|
||||
text = (text[:anchor.start()] + "\n\n" +
|
||||
"# --- concepts added in v1.1 migration -------------------------------\n" +
|
||||
"# Six of these belong to other domains: DRAFT definitions proposed by the\n" +
|
||||
"# DGO, flagged TO_ARBITRATE. OW-007 blocks publication until ratified.\n" +
|
||||
blocks + "\n" + text[anchor.start():])
|
||||
report.append("inserted %d Business Concepts (%d TO_ARBITRATE)"
|
||||
% (len(NEW_CONCEPTS), sum(1 for c in NEW_CONCEPTS if c[6])))
|
||||
% (len(todo_c), sum(1 for c in todo_c if c[6])))
|
||||
elif not todo_c:
|
||||
report.append("all %d Business Concepts already present, skipped"
|
||||
% len(NEW_CONCEPTS))
|
||||
|
||||
# ---- 4. append the objects produced by the splits ------------------------
|
||||
todo_o = [o for o in NEW_OBJECTS
|
||||
if not re.search(r'^%s a ' % re.escape(o[0]), text, re.M)]
|
||||
tail = re.search(r'\n(?=ex:M_\w+ a pr:Metric ;|ex:DO_\w+ a pr:DataObject ;)', text)
|
||||
if tail:
|
||||
blocks = "\n".join(object_block(*o) for o in NEW_OBJECTS)
|
||||
if tail and todo_o:
|
||||
blocks = "\n".join(object_block(*o) for o in todo_o)
|
||||
text = (text[:tail.start()] + "\n\n" +
|
||||
"# --- Business Objects from the v1.1 splits ---------------------------\n" +
|
||||
"# 'Customer & Outlet' and 'Currency & Exchange Rates' had no single\n" +
|
||||
"# subject. Split proposed to DD-04 and DD-16, hence TO_ARBITRATE.\n" +
|
||||
blocks + "\n" + text[tail.start():])
|
||||
report.append("inserted %d Business Objects from splits" % len(NEW_OBJECTS))
|
||||
report.append("inserted %d Business Objects from splits" % len(todo_o))
|
||||
elif not todo_o:
|
||||
report.append("Business Objects from splits already present, skipped")
|
||||
|
||||
# ---- report --------------------------------------------------------------
|
||||
print()
|
||||
@@ -233,6 +250,13 @@ def main():
|
||||
for line in report:
|
||||
print(" " + line)
|
||||
print("=" * W)
|
||||
doubled = []
|
||||
for m in re.finditer(r'^(ex:\w+) a pr:\w+ ;(.*?)\.\s*\n', text, re.S | re.M):
|
||||
for prop in ("pr:hasName", "pr:aboutConcept", "pr:belongsTo"):
|
||||
if len(re.findall(r'(?<![\w:])%s\s' % re.escape(prop), m.group(2))) > 1:
|
||||
doubled.append("%s/%s" % (m.group(1), prop))
|
||||
print(" duplicated single-valued properties: %s"
|
||||
% (", ".join(doubled) if doubled else "none"))
|
||||
bo = len(re.findall(r'a pr:BusinessObject', text))
|
||||
bc = len(re.findall(r'a pr:BusinessConcept', text))
|
||||
print(" Business Objects %d | Business Concepts %d | composedOf left %d"
|
||||
|
||||
@@ -0,0 +1,482 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
SODH - BR-013 alignment, last step of the v1.1 migration
|
||||
=========================================================
|
||||
Regenerates the Metric and Data Element sections of instances/sodh.ttl from the
|
||||
v0.7 back-doc, which has been ahead of the TTL since the divergence found at
|
||||
audit time. This is the migration that puts the flow back the right way round:
|
||||
after it, the TTL is the source and the workbook becomes a generated artefact.
|
||||
|
||||
USAGE
|
||||
python3 scripts/apply_br013_sodh.py # dry run
|
||||
python3 scripts/apply_br013_sodh.py --apply # rewrite, .bak kept
|
||||
|
||||
WHAT IT FIXES
|
||||
37 BLOCKING DataElementMeaningShape -- no element had a route to meaning
|
||||
15 BLOCKING physicalName on a Metric -- layer leak
|
||||
15 WARNING hasGranularity on a Metric
|
||||
6 WARNING monitoredBy on a Data Object
|
||||
|
||||
WHAT IT DOES
|
||||
1. 15 metrics -> 10 mother metrics carrying the harmonized calculation
|
||||
rule, each with a formula, a unit and a measured Concept. No physical
|
||||
name, no granularity: both belong elsewhere now.
|
||||
2. 95 measure Data Elements generated, each computedBy its mother metric,
|
||||
each carrying the physical name the metric used to hold.
|
||||
3. 37 dimensional Data Elements given a represents towards their Concept.
|
||||
With 2, this closes the XOR: every element reaches meaning by exactly
|
||||
one route.
|
||||
4. Three concepts created because the XOR forces them to be named --
|
||||
Country, Marketing Entity, Fiscal Period had elements but no notion.
|
||||
TO_ARBITRATE like the others.
|
||||
5. Steward removed from the 6 Data Objects: it is inherited now.
|
||||
6. hasGrainElement on the 4 DIMENSION Data Objects only.
|
||||
|
||||
WHY FACT TABLES GET NO GRAIN ELEMENTS
|
||||
A fact table's grain is carried by its foreign keys, and OW-006 says a
|
||||
foreign key is not a Data Element -- so a fact object has no element of its
|
||||
own to point at. Its grain is already expressed, by references towards the
|
||||
dimension objects it joins. Forcing hasGrainElement onto facts would mean
|
||||
either breaking OW-006 or pointing at another object's elements, which
|
||||
GrainConsistencyShape rejects. Nothing is lost: the information is there.
|
||||
"""
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import sys
|
||||
|
||||
REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
TTL = os.path.join(REPO, "instances", "sodh.ttl")
|
||||
BACKDOC = os.path.join(REPO, "instances", "SODH_data.xlsx")
|
||||
W = 78
|
||||
|
||||
# ---- mother metric -> the Concept it measures ------------------------------
|
||||
METRIC_CONCEPT = {
|
||||
"M-06.01-001": "ex:BC_06_01_001", # Sell Out Volume -> Sell Out
|
||||
"M-06.01-002": "ex:BC_06_01_001", # Sell Out Value -> Sell Out
|
||||
"M-06.01-003": "ex:BC_06_01_004", # Numeric Distribution -> Retail Distribution
|
||||
"M-06.01-004": "ex:BC_06_01_004", # Weighted Distribution -> Retail Distribution
|
||||
"M-06.01-005": "ex:BC_06_01_004", # Total Distribution Pts -> Retail Distribution
|
||||
"M-06.01-006": "ex:BC_06_01_002", # Forward Stock Volume -> Retailer Stock
|
||||
"M-06.01-007": "ex:BC_06_01_002", # Days of Coverage -> Retailer Stock
|
||||
"M-06.01-008": "ex:BC_06_01_002", # Stock Share -> Retailer Stock
|
||||
"M-06.01-009": "ex:BC_06_01_005", # Baseline Volume -> Sell Out Baseline
|
||||
"M-06.01-010": "ex:BC_06_01_005", # Incremental Volume -> Sell Out Baseline
|
||||
}
|
||||
|
||||
# ---- dimensional element -> the Concept it represents ----------------------
|
||||
# Exact ids first, then id prefixes. Every dimensional element must land
|
||||
# somewhere: that is what the XOR is for.
|
||||
DE_CONCEPT_EXACT = {
|
||||
"DE-04.01-0006": "ex:BC_04_01_002", # Outlet Type -> Outlet
|
||||
"DE-04.01-0007": "ex:BC_04_01_002", # Channel -> Outlet
|
||||
"DE-04.01-0010": "ex:BC_04_01_002", # Point of Sale Count -> Outlet
|
||||
"DE-04.01-0008": "ex:BC_04_03_001", # Country Code -> Country
|
||||
"DE-04.01-0009": "ex:BC_04_03_001", # Country Name -> Country
|
||||
"DE-16.01-0001": "ex:BC_16_01_001", # Currency Code -> Currency
|
||||
"DE-16.01-0002": "ex:BC_16_01_002", # FX Rate to-Euro -> Exchange Rate
|
||||
"DE-16.01-0003": "ex:BC_16_01_002", # FX Rate to-USD -> Exchange Rate
|
||||
"DE-16.02-0001": "ex:BC_16_03_001", # Marketing Entity -> Marketing Entity
|
||||
"DE-16.02-0002": "ex:BC_16_02_001", # Fiscal Period Code -> Fiscal Period
|
||||
"DE-16.02-0003": "ex:BC_16_02_001", # Fiscal Quarter Code -> Fiscal Period
|
||||
}
|
||||
DE_CONCEPT_PREFIX = [
|
||||
("DE-10.01", "ex:BC_10_01_001"), # product hierarchy -> Product
|
||||
("DE-04.01", "ex:BC_04_01_001"), # trade hierarchy -> Customer
|
||||
("DE-21.01", "ex:BC_21_01_001"), # calendar -> Calendar Date
|
||||
("DE-05.01", "ex:BC_05_01_001"), # promotion flags -> Promotion
|
||||
]
|
||||
|
||||
# ---- concepts the XOR forces us to name ------------------------------------
|
||||
NEW_CONCEPTS = [
|
||||
# Final identifiers from the outset. Creating them under one id and
|
||||
# renumbering them later broke idempotence: the second run no longer found
|
||||
# the original id and recreated the concept alongside the renamed one.
|
||||
("ex:BC_04_03_001", "BC-04.03-001", "Country", "ex:DD_04", "ex:ST_DD_04",
|
||||
"A sovereign territory used as the geographic frame for retail measurement, "
|
||||
"identified by its ISO 3166 code. Distinct from Customer and Outlet: it is where "
|
||||
"they operate, not what they are."),
|
||||
("ex:BC_16_03_001", "BC-16.03-001", "Marketing Entity", "ex:DD_16", "ex:ST_DD_16",
|
||||
"An organisational unit of the group holding commercial responsibility for a market, "
|
||||
"and the level at which financial results are consolidated."),
|
||||
("ex:BC_16_02_001", "BC-16.02-001", "Fiscal Period", "ex:DD_16", "ex:ST_DD_16",
|
||||
"A reporting interval of the Pernod Ricard fiscal year, which runs July to June. "
|
||||
"Distinct from Calendar Date: the fiscal frame does not align with the Gregorian one."),
|
||||
]
|
||||
|
||||
# ---- grain of the DIMENSION objects (fact grain lives in references) -------
|
||||
GRAIN = {
|
||||
"ex:DO_10_01_001": ["ex:DE_10_01_0005"], # Product Dimension -> SKU Code
|
||||
"ex:DO_04_01_001": ["ex:DE_04_01_0005"], # Customer Dimension -> Customer Tier-2 Code
|
||||
"ex:DO_16_01_001": ["ex:DE_16_01_0001"], # Currency Dimension -> Currency Code
|
||||
"ex:DO_21_01_001": ["ex:DE_21_01_0001"], # Calendar Dimension -> Calendar Date
|
||||
}
|
||||
|
||||
UNIT_FIX = {"9L / L": "9L", "Currency EUR/USD/LC": "EUR", "%": "%",
|
||||
"Index": "Index", "Days": "Days"}
|
||||
|
||||
|
||||
def iri(ident):
|
||||
"""BO-06.01-001 -> ex:BO_06_01_001"""
|
||||
return "ex:" + ident.replace("-", "_", 1).replace(".", "_").replace("-", "_")
|
||||
|
||||
|
||||
def esc(text):
|
||||
return text.replace("\\", "\\\\").replace('"', '\\"')
|
||||
|
||||
|
||||
def _xlsx_rows(path):
|
||||
"""
|
||||
Minimal .xlsx reader: zipfile + ElementTree, no third-party dependency.
|
||||
|
||||
openpyxl would do this in three lines, but the validation venv is already
|
||||
pinned tightly (pyshacl 0.26.0 for Python 3.9) and adding a dependency to a
|
||||
migration script that runs once is a poor trade. Handles what a back-doc
|
||||
needs: shared strings, inline strings, numbers, and sheet names.
|
||||
"""
|
||||
import zipfile
|
||||
import xml.etree.ElementTree as ET
|
||||
|
||||
NS = "{http://schemas.openxmlformats.org/spreadsheetml/2006/main}"
|
||||
REL = "{http://schemas.openxmlformats.org/officeDocument/2006/relationships}"
|
||||
PKG = "{http://schemas.openxmlformats.org/package/2006/relationships}"
|
||||
|
||||
with zipfile.ZipFile(path) as z:
|
||||
shared = []
|
||||
if "xl/sharedStrings.xml" in z.namelist():
|
||||
for si in ET.fromstring(z.read("xl/sharedStrings.xml")):
|
||||
shared.append("".join(t.text or "" for t in si.iter(NS + "t")))
|
||||
|
||||
rels = {}
|
||||
for rel in ET.fromstring(z.read("xl/_rels/workbook.xml.rels")):
|
||||
rels[rel.get("Id")] = rel.get("Target").lstrip("/")
|
||||
|
||||
sheets = []
|
||||
for sh in ET.fromstring(z.read("xl/workbook.xml")).iter(NS + "sheet"):
|
||||
target = rels.get(sh.get(REL + "id"), "")
|
||||
if not target.startswith("xl/"):
|
||||
target = "xl/" + target
|
||||
sheets.append((sh.get("name"), target))
|
||||
|
||||
for name, target in sheets:
|
||||
if target not in z.namelist():
|
||||
continue
|
||||
rows = []
|
||||
for row in ET.fromstring(z.read(target)).iter(NS + "row"):
|
||||
cells = []
|
||||
for c in row.iter(NS + "c"):
|
||||
v = c.find(NS + "v")
|
||||
if c.get("t") == "s" and v is not None:
|
||||
cells.append(shared[int(v.text)])
|
||||
elif c.get("t") == "inlineStr":
|
||||
cells.append("".join(t.text or "" for t in c.iter(NS + "t")))
|
||||
else:
|
||||
cells.append(v.text if v is not None else "")
|
||||
rows.append([(x or "").strip() for x in cells])
|
||||
yield name, rows
|
||||
|
||||
|
||||
def _text_rows(path):
|
||||
"""Tab-separated fallback, for back-docs exported as plain text."""
|
||||
sheet, rows = None, []
|
||||
for raw in open(path, encoding="utf-8", errors="replace"):
|
||||
if raw.startswith("## Sheet:"):
|
||||
if sheet:
|
||||
yield sheet, rows
|
||||
sheet, rows = raw.split(":", 1)[1].strip(), []
|
||||
elif sheet is not None:
|
||||
rows.append([c.strip() for c in raw.rstrip("\n").split("\t")])
|
||||
if sheet:
|
||||
yield sheet, rows
|
||||
|
||||
|
||||
def parse_backdoc(path):
|
||||
"""
|
||||
Read the back-doc, whichever form it takes.
|
||||
|
||||
A real .xlsx is a ZIP (magic PK\x03\x04); some pipelines hand over a
|
||||
tab-separated text export of the same content. Sniff rather than assume:
|
||||
guessing from the extension is what made this script fail the first time.
|
||||
"""
|
||||
with open(path, "rb") as fh:
|
||||
is_zip = fh.read(4) == b"PK\x03\x04"
|
||||
reader = _xlsx_rows if is_zip else _text_rows
|
||||
|
||||
out = {"metrics": [], "elements": []}
|
||||
for name, rows in reader(path):
|
||||
key = None
|
||||
if name.strip().startswith("6"):
|
||||
key = "metrics"
|
||||
elif name.strip().startswith("7"):
|
||||
key = "elements"
|
||||
if not key:
|
||||
continue
|
||||
for cells in rows:
|
||||
if len(cells) < 5 or not cells[0]:
|
||||
continue
|
||||
if key == "metrics" and cells[0].startswith("M-"):
|
||||
out[key].append(cells)
|
||||
elif key == "elements" and cells[0].startswith("DE-"):
|
||||
out[key].append(cells)
|
||||
if not out["metrics"] or not out["elements"]:
|
||||
raise SystemExit(
|
||||
"Back-doc read but empty: %d metrics, %d elements.\n"
|
||||
"Expected sheets starting with '6.' (Mother Metrics) and '7.' "
|
||||
"(Data Elements) in %s" % (len(out["metrics"]), len(out["elements"]), path))
|
||||
return out
|
||||
|
||||
|
||||
def concept_for(de_id):
|
||||
if de_id in DE_CONCEPT_EXACT:
|
||||
return DE_CONCEPT_EXACT[de_id]
|
||||
for prefix, bc in DE_CONCEPT_PREFIX:
|
||||
if de_id.startswith(prefix):
|
||||
return bc
|
||||
return None
|
||||
|
||||
|
||||
def build_metrics(metrics):
|
||||
out = ["# --- mother metrics, BR-013 -----------------------------------------",
|
||||
"# One harmonized calculation rule each. The granular variants are Data",
|
||||
"# Elements linked by computedBy. No physical name (layer leak) and no",
|
||||
"# granularity (a calculation rule has no rows).", ""]
|
||||
for r in metrics:
|
||||
mid, name, bo, formula, unit = r[0], r[1], r[2], r[3], r[4]
|
||||
unit = UNIT_FIX.get(unit.split(" (")[0], unit.split(" (")[0])
|
||||
out.append("%s a pr:Metric ;" % iri(mid))
|
||||
out.append(' pr:hasIdentifier "%s" ; pr:hasName "%s" ;' % (mid, esc(name)))
|
||||
out.append(" pr:owningDomain ex:DD_06 ; pr:ownedBy ex:ST_DD_06 ;")
|
||||
out.append(' pr:hasFormula "%s" ;' % esc(formula))
|
||||
out.append(' pr:hasUnit "%s" ;' % esc(unit))
|
||||
out.append(" pr:measures %s ;" % METRIC_CONCEPT[mid])
|
||||
out.append(' pr:hasStatus "DRAFT" ; pr:hasVersion "1.1" ; '
|
||||
'pr:hasSource "SODH back-doc v0.7" .')
|
||||
return "\n".join(out)
|
||||
|
||||
|
||||
def build_elements(elements):
|
||||
measure, dimensional = [], []
|
||||
for r in elements:
|
||||
(deid, name, domain, computed, fmt, unit, source, phys, do) = r[:9]
|
||||
dom = "ex:DD_" + deid.split("-")[1].split(".")[0]
|
||||
block = ["%s a pr:DataElement ;" % iri(deid),
|
||||
' pr:hasIdentifier "%s" ; pr:hasName "%s" ;' % (deid, esc(name)),
|
||||
" pr:owningDomain %s ;" % dom,
|
||||
' pr:hasFormat "%s" ;' % esc(fmt)]
|
||||
if unit and unit != "-":
|
||||
block.append(' pr:hasUnit "%s" ;' % esc(unit))
|
||||
if phys and phys != "-":
|
||||
block.append(' pr:physicalName "%s" ;' % esc(phys))
|
||||
if computed == "(dimensional)":
|
||||
bc = concept_for(deid)
|
||||
if not bc:
|
||||
raise SystemExit("No concept mapped for %s -- the XOR would fail." % deid)
|
||||
block.append(" pr:represents %s ;" % bc)
|
||||
block.append(' pr:hasStatus "DRAFT" ; pr:hasVersion "1.1" ; '
|
||||
'pr:hasSource "%s" .' % esc(source))
|
||||
dimensional.append("\n".join(block))
|
||||
else:
|
||||
block.append(' pr:hasStatus "DRAFT" ; pr:hasVersion "1.1" ; '
|
||||
'pr:hasSource "%s" .' % esc(source))
|
||||
measure.append("\n".join(block))
|
||||
return measure, dimensional
|
||||
|
||||
|
||||
def build_computed_by(elements):
|
||||
"""Metric -> its granular elements. Asserted from the metric side."""
|
||||
by_metric = {}
|
||||
name_to_id = {r[1]: r[0] for r in parse_backdoc(BACKDOC)["metrics"]}
|
||||
for r in elements:
|
||||
if r[3] == "(dimensional)":
|
||||
continue
|
||||
mid = name_to_id.get(r[3])
|
||||
if mid:
|
||||
by_metric.setdefault(mid, []).append(iri(r[0]))
|
||||
out = ["# --- BR-013 wiring: each mother metric and the variants that implement it",
|
||||
""]
|
||||
for mid, des in by_metric.items():
|
||||
chunks = [des[i:i + 4] for i in range(0, len(des), 4)]
|
||||
lines = [" , ".join(c) for c in chunks]
|
||||
out.append("%s pr:computedBy %s ." % (iri(mid), " ,\n ".join(lines)))
|
||||
return "\n".join(out), sum(len(v) for v in by_metric.values())
|
||||
|
||||
|
||||
def rebuild_has_element(text, elements):
|
||||
"""
|
||||
Re-attach every Data Element to its Data Object, from the back-doc column.
|
||||
|
||||
Without this the 95 generated measure elements would be orphans: no path to
|
||||
the physical layer, and no steward, since stewardship is inherited through
|
||||
the object. Not caught by any shape -- which is why it is worth doing here
|
||||
rather than waiting for the validator to complain.
|
||||
"""
|
||||
by_do = {}
|
||||
for r in elements:
|
||||
do = r[8].strip()
|
||||
if do and do != "-":
|
||||
by_do.setdefault(iri(do), []).append(iri(r[0]))
|
||||
n = 0
|
||||
for do, des in by_do.items():
|
||||
pat = re.compile(r'(^%s a pr:DataObject ;.*?)\n\s*pr:hasElement[^;]*;' % re.escape(do),
|
||||
re.S | re.M)
|
||||
chunks = [des[i:i + 4] for i in range(0, len(des), 4)]
|
||||
clause = "\n pr:hasElement " + " ,\n ".join(" , ".join(c) for c in chunks) + " ;"
|
||||
if pat.search(text):
|
||||
text = pat.sub(lambda m: m.group(1) + clause, text, count=1)
|
||||
n += 1
|
||||
return text, n, {k: len(v) for k, v in by_do.items()}
|
||||
|
||||
|
||||
def rebuild_has_metric(text, metrics):
|
||||
"""
|
||||
Re-point hasMetric on the Business Objects at the mother metrics.
|
||||
|
||||
Removing the old metric definitions is not enough: the Business Objects
|
||||
still name them, and because hasMetric has rdfs:range pr:Metric, RDFS
|
||||
entailment types those dangling IRIs as metrics. They then become focus
|
||||
nodes carrying no identifier, no name, no formula -- ten ghosts, six
|
||||
violations each. Deleting a subject means deleting what points at it.
|
||||
"""
|
||||
by_bo = {}
|
||||
for r in metrics:
|
||||
by_bo.setdefault(iri(r[2]), []).append(iri(r[0]))
|
||||
|
||||
text = re.sub(r'\n\s*pr:hasMetric[^;]*;', '', text)
|
||||
n = 0
|
||||
for bo, ms in by_bo.items():
|
||||
pat = re.compile(r'(^%s a pr:BusinessObject ;.*?)(\n\s*pr:hasStatus)' % re.escape(bo),
|
||||
re.S | re.M)
|
||||
if pat.search(text):
|
||||
text = pat.sub(r'\1\n pr:hasMetric %s ;\2' % " , ".join(ms), text, count=1)
|
||||
n += 1
|
||||
return text, n, by_bo
|
||||
|
||||
|
||||
def concept_block(c):
|
||||
i, ident, name, dom, st, definition = c
|
||||
return ("%s a pr:BusinessConcept ;\n"
|
||||
' pr:hasIdentifier "%s" ; pr:hasName "%s" ;\n'
|
||||
" pr:owningDomain %s ; pr:ownedBy %s ;\n"
|
||||
' pr:hasBusinessDefinition "%s" ;\n'
|
||||
' pr:arbitrationStatus "TO_ARBITRATE" ;\n'
|
||||
' pr:hasStatus "DRAFT" ; pr:hasVersion "1.1" ; '
|
||||
'pr:hasSource "DGO proposal, pending ratification by the owning domain" .'
|
||||
% (i, ident, esc(name), dom, st, esc(definition)))
|
||||
|
||||
|
||||
def main():
|
||||
apply_changes = "--apply" in sys.argv
|
||||
for f in (TTL, BACKDOC):
|
||||
if not os.path.exists(f):
|
||||
print("Not found: %s" % f)
|
||||
sys.exit(2)
|
||||
|
||||
doc = parse_backdoc(BACKDOC)
|
||||
text = original = open(TTL, encoding="utf-8").read()
|
||||
report = []
|
||||
|
||||
# 1. drop the old metric and data element blocks
|
||||
n_old_m = len(re.findall(r'^ex:M_\w+ a pr:Metric ;', text, re.M))
|
||||
n_old_de = len(re.findall(r'^ex:DE_\w+ a pr:DataElement ;', text, re.M))
|
||||
text = re.sub(r'^ex:M_\w+ a pr:Metric ;.*?\.\s*\n(?=^ex:|\Z)', '', text, flags=re.S | re.M)
|
||||
text = re.sub(r'^ex:DE_\w+ a pr:DataElement ;.*?\.\s*\n(?=^ex:|\Z)', '', text, flags=re.S | re.M)
|
||||
report.append("removed %d old metrics and %d old data elements" % (n_old_m, n_old_de))
|
||||
|
||||
# 2. steward is inherited, not declared on a Data Object
|
||||
n_st = 0
|
||||
for m in re.finditer(r'^ex:DO_\w+ a pr:DataObject ;.*?\.\s*\n', text, re.S | re.M):
|
||||
block = m.group(0)
|
||||
new = re.sub(r'\s*pr:monitoredBy\s+ex:\w+\s*;', ' ;', block)
|
||||
new = re.sub(r';\s*;', ' ;', new)
|
||||
if new != block:
|
||||
n_st += 1
|
||||
text = text.replace(block, new, 1)
|
||||
report.append("removed monitoredBy from %d Data Objects (inherited now)" % n_st)
|
||||
|
||||
# 3. grain, dimension objects only
|
||||
n_g = 0
|
||||
for do, grain in GRAIN.items():
|
||||
pat = re.compile(r'(^%s a pr:DataObject ;.*?)(\n\s*pr:hasStatus)' % re.escape(do),
|
||||
re.S | re.M)
|
||||
if pat.search(text):
|
||||
text = pat.sub(r'\1\n pr:hasGrainElement %s ;\2' % " , ".join(grain), text, count=1)
|
||||
n_g += 1
|
||||
report.append("grain declared on %d dimension Data Objects "
|
||||
"(fact grain stays in references)" % n_g)
|
||||
|
||||
# 4. the three concepts the XOR forces us to name
|
||||
# idempotent: a concept already in the file is left alone. Duplicated
|
||||
# blocks would be INVISIBLE to SHACL -- identical triples merge under RDF
|
||||
# set semantics, so the graph validates while the file carries redundant
|
||||
# text. A script that can be re-run must check before it inserts.
|
||||
todo = [c for c in NEW_CONCEPTS
|
||||
if not re.search(r'^%s a ' % re.escape(c[0]), text, re.M)]
|
||||
if len(todo) < len(NEW_CONCEPTS):
|
||||
report.append("skipped %d concept(s) already present"
|
||||
% (len(NEW_CONCEPTS) - len(todo)))
|
||||
anchor = re.search(r'\n(?=ex:BO_\w+ a pr:BusinessObject ;)', text)
|
||||
blocks = "\n".join(concept_block(c) for c in todo)
|
||||
if todo:
|
||||
text = (text[:anchor.start()] + "\n\n"
|
||||
+ "# --- concepts required by the meaning XOR ---------------------------\n"
|
||||
+ "# Country, Marketing Entity and Fiscal Period had Data Elements but no\n"
|
||||
+ "# notion behind them. The rule forced them to be named.\n"
|
||||
+ blocks + "\n" + text[anchor.start():])
|
||||
report.append("added %d concepts required by the XOR (TO_ARBITRATE)" % len(todo))
|
||||
|
||||
# 5. metrics, elements, wiring
|
||||
measure, dimensional = build_elements(doc["elements"])
|
||||
wiring, n_wired = build_computed_by(doc["elements"])
|
||||
text = text.rstrip() + "\n\n\n" + build_metrics(doc["metrics"]) + "\n\n"
|
||||
text += ("# --- dimensional data elements --------------------------------------\n"
|
||||
"# Each represents the Concept it carries: the first of the two routes\n"
|
||||
"# to business meaning.\n\n" + "\n".join(dimensional) + "\n\n")
|
||||
text += ("# --- granular measure data elements ---------------------------------\n"
|
||||
"# Same calculation rule as their mother metric, different analysis\n"
|
||||
"# context. They reach meaning through computedBy, never directly.\n\n"
|
||||
+ "\n".join(measure) + "\n\n" + wiring + "\n")
|
||||
report.append("wrote %d mother metrics, %d dimensional and %d measure elements"
|
||||
% (len(doc["metrics"]), len(dimensional), len(measure)))
|
||||
report.append("wired %d computedBy links" % n_wired)
|
||||
|
||||
text, n_bo, per_bo = rebuild_has_metric(text, doc["metrics"])
|
||||
report.append("re-pointed hasMetric on %d Business Objects (%s)"
|
||||
% (n_bo, ", ".join("%s=%d" % (k.replace("ex:BO_", "BO-"), len(v))
|
||||
for k, v in sorted(per_bo.items()))))
|
||||
|
||||
text, n_do, counts = rebuild_has_element(text, doc["elements"])
|
||||
report.append("re-attached elements to %d Data Objects (%s)"
|
||||
% (n_do, ", ".join("%s=%d" % (k.replace("ex:DO_", "DO-"), v)
|
||||
for k, v in sorted(counts.items()))))
|
||||
|
||||
print()
|
||||
print("SODH BR-013 ALIGNMENT %s" % ("APPLY" if apply_changes else "DRY RUN"))
|
||||
print("=" * W)
|
||||
for line in report:
|
||||
print(" " + line)
|
||||
print("=" * W)
|
||||
defined = set(re.findall(r'^(ex:M_\w+) a pr:Metric ;', text, re.M))
|
||||
referenced = set(re.findall(r'ex:M_\w+', text))
|
||||
dangling = referenced - defined
|
||||
print(" dangling metric references: %s"
|
||||
% (", ".join(sorted(dangling)) if dangling else "none"))
|
||||
print(" Metrics %d | Data Elements %d | Concepts %d | computedBy %d | "
|
||||
"physicalName on Metric %d"
|
||||
% (len(re.findall(r'a pr:Metric', text)),
|
||||
len(re.findall(r'a pr:DataElement', text)),
|
||||
len(re.findall(r'a pr:BusinessConcept', text)),
|
||||
n_wired,
|
||||
len(re.findall(r'a pr:Metric ;[^.]*?physicalName', text, re.S))))
|
||||
print("=" * W)
|
||||
|
||||
if text != original and apply_changes:
|
||||
shutil.copy2(TTL, TTL + ".bak")
|
||||
open(TTL, "w", encoding="utf-8").write(text)
|
||||
print(" written, backup at %s.bak" % os.path.basename(TTL))
|
||||
elif text != original:
|
||||
print(" dry run -- re-run with --apply to write")
|
||||
print()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,202 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
SODH - data fixes v1.2
|
||||
=======================
|
||||
Four corrections found while reviewing the viewer.
|
||||
|
||||
USAGE
|
||||
python3 scripts/apply_fixes_v1_2.py # dry run
|
||||
python3 scripts/apply_fixes_v1_2.py --apply # rewrite, .bak kept
|
||||
|
||||
1. DATA DOMAIN OWNERS
|
||||
Six named owners added, plus the hasDomainOwner link. Ownership was
|
||||
readable only through the DGL until now, which conflates two roles: the
|
||||
owner is accountable for the domain, the DGL runs its governance.
|
||||
|
||||
2. hasSource CARRIED TWO MEANINGS
|
||||
On a Data Element it named the system of origin (MDM Product, Group
|
||||
Finance). On a Data Object it named the physical location (SODH Gold /
|
||||
F_SELL_OUT). Same property, two concepts -- the defect we removed from
|
||||
composedOf, still present one layer down. Split: physicalName holds the
|
||||
relation name, hasSource holds the system. This is also what makes the
|
||||
physical layer derivable: F_SELL_OUT plus 132 column names.
|
||||
|
||||
3. METRIC UNITS ARE PLURAL
|
||||
Five mother metrics compute elements in several units (9L and L, EUR and
|
||||
LC and USD). A metric states every unit its variants are expressed in; a
|
||||
Data Element states exactly one. Recomputed from the elements rather than
|
||||
asserted, so the two can never drift.
|
||||
|
||||
4. Sell Out Distribution -> Retail Distribution, matching the concept: sell
|
||||
out counts units sold, distribution measures presence in store.
|
||||
"""
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import sys
|
||||
|
||||
REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
TTL = os.path.join(REPO, "instances", "sodh.ttl")
|
||||
W = 78
|
||||
|
||||
OWNERS = [
|
||||
("ex:DDO_DD_06", "Ludovic Cottier", "ex:DD_06"),
|
||||
("ex:DDO_DD_04", "David Carpio", "ex:DD_04"),
|
||||
("ex:DDO_DD_10", "Alberto Lupano", "ex:DD_10"),
|
||||
("ex:DDO_DD_05", "David Carpio", "ex:DD_05"),
|
||||
("ex:DDO_DD_16", "Julien Soisson", "ex:DD_16"),
|
||||
("ex:DDO_DD_21", "Vincent Meunier", "ex:DD_21"),
|
||||
]
|
||||
|
||||
RENAME = {"ex:BO_06_01_003": ("Sell Out Distribution", "Retail Distribution")}
|
||||
|
||||
# system of origin, once the physical location is taken out of it
|
||||
SOURCE_FIX = {
|
||||
"Snowflake (SODH Gold)": "SODH Gold",
|
||||
"SODH Gold (flag)": "SODH Gold",
|
||||
}
|
||||
|
||||
|
||||
def clauses(body):
|
||||
out, buf, i, n = [], [], 0, len(body)
|
||||
while i < n:
|
||||
c = body[i]
|
||||
if c == '"':
|
||||
j = i + 1
|
||||
while j < n and body[j] != '"':
|
||||
j += 2 if body[j] == "\\" else 1
|
||||
j = min(j + 1, n)
|
||||
buf.append(body[i:j]); i = j; continue
|
||||
if c == ";":
|
||||
out.append("".join(buf)); buf = []; i += 1; continue
|
||||
buf.append(c); i += 1
|
||||
out.append("".join(buf))
|
||||
return out
|
||||
|
||||
|
||||
def set_prop(text, subject, prop, value):
|
||||
"""Clause-level set: survives mid-line properties and semicolons in literals."""
|
||||
pat = re.compile(r'(^%s a pr:\w+ ;)(.*?)(\.\s*\n)' % re.escape(subject), re.S | re.M)
|
||||
m = pat.search(text)
|
||||
if not m:
|
||||
return text, False
|
||||
kept = [c for c in clauses(m.group(2))
|
||||
if c.strip() and not re.match(r'\s*%s\s' % re.escape(prop), c)]
|
||||
if value is not None:
|
||||
kept.insert(0, "\n %s %s " % (prop, value))
|
||||
body = " ;".join(kept).rstrip() + "\n "
|
||||
return text[:m.start()] + m.group(1) + body + m.group(3) + text[m.end():], True
|
||||
|
||||
|
||||
def get_prop(text, subject, prop):
|
||||
m = re.search(r'^%s a pr:\w+ ;(.*?)\.\s*\n' % re.escape(subject), text, re.S | re.M)
|
||||
if not m:
|
||||
return None
|
||||
for c in clauses(m.group(1)):
|
||||
mm = re.match(r'\s*%s\s+(.*)$' % re.escape(prop), c, re.S)
|
||||
if mm:
|
||||
return mm.group(1).strip()
|
||||
return None
|
||||
|
||||
|
||||
def main():
|
||||
apply_changes = "--apply" in sys.argv
|
||||
if not os.path.exists(TTL):
|
||||
print("Not found: %s" % TTL); sys.exit(2)
|
||||
text = original = open(TTL, encoding="utf-8").read()
|
||||
report = []
|
||||
|
||||
# ---- 1. domain owners -------------------------------------------------
|
||||
todo = [o for o in OWNERS if not re.search(r'^%s a ' % re.escape(o[0]), text, re.M)]
|
||||
if todo:
|
||||
blocks = "\n".join('%s a pr:DataDomainOwner ; pr:hasName "%s" .' % (i, n)
|
||||
for i, n, _ in todo)
|
||||
anchor = re.search(r'^ex:ST_\w+ a pr:DataSteward', text, re.M)
|
||||
pos = anchor.start() if anchor else 0
|
||||
text = (text[:pos]
|
||||
+ "# --- data domain owners ----------------------------------------------\n"
|
||||
+ "# Accountable for the domain. Distinct from the DGL, who runs its\n"
|
||||
+ "# governance: two roles that hasDGL alone was conflating.\n"
|
||||
+ blocks + "\n\n" + text[pos:])
|
||||
for i, n, dd in OWNERS:
|
||||
text, _ = set_prop(text, dd, "pr:hasDomainOwner", i)
|
||||
report.append("%d data domain owners named and linked" % len(OWNERS))
|
||||
|
||||
# ---- 2. physical location out of hasSource ----------------------------
|
||||
n_do = 0
|
||||
for m in list(re.finditer(r'^(ex:DO_\w+) a pr:DataObject ;', text, re.M)):
|
||||
subj = m.group(1)
|
||||
src = get_prop(text, subj, "pr:hasSource")
|
||||
if src and "/" in src:
|
||||
system, relation = [p.strip().strip('"') for p in src.strip('"').split("/", 1)]
|
||||
text, _ = set_prop(text, subj, "pr:hasSource", '"%s"' % system)
|
||||
text, _ = set_prop(text, subj, "pr:physicalName", '"%s"' % relation)
|
||||
n_do += 1
|
||||
report.append("%d data objects: physical relation moved out of hasSource" % n_do)
|
||||
|
||||
n_de = 0
|
||||
for old, new in SOURCE_FIX.items():
|
||||
text, k = re.subn(r'pr:hasSource "%s"' % re.escape(old),
|
||||
'pr:hasSource "%s"' % new, text)
|
||||
n_de += k
|
||||
report.append("%d source labels normalised to the system of origin" % n_de)
|
||||
|
||||
# ---- 3. metric units, recomputed from the elements ---------------------
|
||||
de_unit = {}
|
||||
for m in re.finditer(r'^(ex:DE_\w+) a pr:DataElement ;(.*?)\.\s*\n', text, re.S | re.M):
|
||||
u = re.search(r'pr:hasUnit\s+"([^"]*)"', m.group(2))
|
||||
if u:
|
||||
de_unit[m.group(1)] = u.group(1)
|
||||
wired = {}
|
||||
for m in re.finditer(r'(ex:M_\w+)\s+pr:computedBy(.*?)\.', text, re.S):
|
||||
wired[m.group(1)] = re.findall(r'ex:DE_\w+', m.group(2))
|
||||
n_m = 0
|
||||
for metric, des in wired.items():
|
||||
units = sorted({de_unit[d] for d in des if d in de_unit and de_unit[d] != "-"})
|
||||
if not units:
|
||||
continue
|
||||
value = " , ".join('"%s"' % u for u in units)
|
||||
before = get_prop(text, metric, "pr:hasUnit")
|
||||
text, ok = set_prop(text, metric, "pr:hasUnit", value)
|
||||
if ok and before != value:
|
||||
n_m += 1
|
||||
report.append("%d metrics now list every unit their elements use" % n_m)
|
||||
|
||||
# ---- 4. rename --------------------------------------------------------
|
||||
for subj, (old, new) in RENAME.items():
|
||||
if get_prop(text, subj, "pr:hasName") == '"%s"' % old:
|
||||
text, _ = set_prop(text, subj, "pr:hasName", '"%s"' % new)
|
||||
report.append("renamed %s -> %s" % (old, new))
|
||||
|
||||
# ---- report -----------------------------------------------------------
|
||||
print()
|
||||
print("SODH DATA FIXES v1.2 %s" % ("APPLY" if apply_changes else "DRY RUN"))
|
||||
print("=" * W)
|
||||
for line in report:
|
||||
print(" " + line)
|
||||
print("=" * W)
|
||||
multi = len(re.findall(r'pr:hasUnit\s+"[^"]*"\s*,', text))
|
||||
phys = len(re.findall(r'^ex:DO_\w+ a pr:DataObject ;(?:(?!\.\s*\n).)*?physicalName',
|
||||
text, re.S | re.M))
|
||||
doubled = []
|
||||
for m in re.finditer(r'^(ex:\w+) a pr:\w+ ;(.*?)\.\s*\n', text, re.S | re.M):
|
||||
for p in ("pr:hasName", "pr:hasSource", "pr:physicalName", "pr:hasDomainOwner"):
|
||||
if len(re.findall(r'(?<![\w:])%s\s' % re.escape(p), m.group(2))) > 1:
|
||||
doubled.append("%s/%s" % (m.group(1), p))
|
||||
print(" metrics with several units %d | data objects with a physical name %d"
|
||||
% (multi, phys))
|
||||
print(" duplicated single-valued properties: %s"
|
||||
% (", ".join(doubled) if doubled else "none"))
|
||||
print("=" * W)
|
||||
|
||||
if text != original and apply_changes:
|
||||
shutil.copy2(TTL, TTL + ".bak")
|
||||
open(TTL, "w", encoding="utf-8").write(text)
|
||||
print(" written, backup at %s.bak" % os.path.basename(TTL))
|
||||
elif text != original:
|
||||
print(" dry run -- re-run with --apply to write")
|
||||
print()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,261 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
SODH - structure proposed for the borrowed domains
|
||||
===================================================
|
||||
Gives the nine TO_ARBITRATE concepts a home: a Business Object and a
|
||||
Sub-Domain each, per the structure arbitrated with Bastien on 2026-07-27.
|
||||
|
||||
USAGE
|
||||
python3 scripts/apply_structure_sodh.py # dry run
|
||||
python3 scripts/apply_structure_sodh.py --apply # rewrite, .bak kept
|
||||
|
||||
STILL A PROPOSAL
|
||||
Everything created here stays TO_ARBITRATE. DD-04, DD-10, DD-16 and DD-21
|
||||
own these perimeters; the DGO is proposing a coherent structure, not
|
||||
ratifying one. Replacing padding with an equally unilateral structure
|
||||
would trade one problem for a tidier version of the same problem. The flag
|
||||
comes off when Helene Puchot, Anas El Kesri and Gaelle Seret say so, and
|
||||
OW-007 blocks publication until then.
|
||||
|
||||
IDENTIFIERS ARE FINAL FROM CREATION
|
||||
Identifiers encode the sub-domain (BO-04.02-001 lives in SD-04.02), so the
|
||||
upstream scripts now mint them correctly rather than having this one
|
||||
renumber afterwards. Renaming after creation broke idempotence: the script
|
||||
that created an object stopped recognising its own output and made a second
|
||||
copy on the next run.
|
||||
|
||||
COMPOUND SUB-DOMAINS SPLIT TOO
|
||||
'Customer & Outlet' and 'Marketing Entity & Fiscal Calendar' have the same
|
||||
defect as the Business Objects that were split earlier: a name joined by
|
||||
'&' has no single subject. Same rule, one level up.
|
||||
"""
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import sys
|
||||
|
||||
REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
TTL = os.path.join(REPO, "instances", "sodh.ttl")
|
||||
W = 78
|
||||
|
||||
# No IRI renaming here. Objects are created with their final identifiers by
|
||||
# the upstream scripts: renumbering after the fact broke idempotence, because
|
||||
# the creating script then no longer recognised its own output.
|
||||
# ---- sub-domains ----------------------------------------------------------
|
||||
SD_RENAME = {
|
||||
"ex:SD_04_01": "Customer Structure",
|
||||
"ex:SD_16_01": "Currency & Exchange Rate",
|
||||
"ex:SD_16_02": "Fiscal Calendar",
|
||||
"ex:SD_21_01": "Standard Calendar",
|
||||
}
|
||||
SD_NEW = [
|
||||
("ex:SD_04_02", "SD-04.02", "Outlet & Points of Sale", "ex:DD_04"),
|
||||
("ex:SD_04_03", "SD-04.03", "Geography", "ex:DD_04"),
|
||||
("ex:SD_16_03", "SD-16.03", "Organisation Structure", "ex:DD_16"),
|
||||
]
|
||||
|
||||
# ---- business objects -----------------------------------------------------
|
||||
BO_RENAME = {
|
||||
"ex:BO_10_01_001": "Product Hierarchy",
|
||||
"ex:BO_04_01_001": "Customer Hierarchy",
|
||||
"ex:BO_21_01_001": "Analysis Period",
|
||||
}
|
||||
BO_NEW = [
|
||||
("ex:BO_04_03_001", "BO-04.03-001", "Country", "ex:DD_04", "ex:ST_DD_04",
|
||||
"ex:SD_04_03", "ex:BC_04_03_001"),
|
||||
("ex:BO_16_02_001", "BO-16.02-001", "Fiscal Period", "ex:DD_16", "ex:ST_DD_16",
|
||||
"ex:SD_16_02", "ex:BC_16_02_001"),
|
||||
("ex:BO_16_03_001", "BO-16.03-001", "Marketing Entity", "ex:DD_16", "ex:ST_DD_16",
|
||||
"ex:SD_16_03", "ex:BC_16_03_001"),
|
||||
]
|
||||
|
||||
# ---- where each object finally sits ---------------------------------------
|
||||
BELONGS = {
|
||||
"ex:BO_10_01_001": "ex:SD_10_01",
|
||||
"ex:BO_04_01_001": "ex:SD_04_01",
|
||||
"ex:BO_04_02_001": "ex:SD_04_02",
|
||||
"ex:BO_04_03_001": "ex:SD_04_03",
|
||||
"ex:BO_16_01_001": "ex:SD_16_01",
|
||||
"ex:BO_16_01_002": "ex:SD_16_01",
|
||||
"ex:BO_16_02_001": "ex:SD_16_02",
|
||||
"ex:BO_16_03_001": "ex:SD_16_03",
|
||||
"ex:BO_21_01_001": "ex:SD_21_01",
|
||||
}
|
||||
|
||||
# ---- subject of each borrowed object --------------------------------------
|
||||
ABOUT = {
|
||||
"ex:BO_10_01_001": "ex:BC_10_01_001", # Product Hierarchy -> Product
|
||||
"ex:BO_04_01_001": "ex:BC_04_01_001", # Customer Hierarchy -> Customer
|
||||
"ex:BO_04_02_001": "ex:BC_04_01_002", # Outlet -> Outlet
|
||||
"ex:BO_04_03_001": "ex:BC_04_03_001", # Country -> Country
|
||||
"ex:BO_16_01_001": "ex:BC_16_01_001", # Currency -> Currency
|
||||
"ex:BO_16_01_002": "ex:BC_16_01_002", # Exchange Rate -> Exchange Rate
|
||||
"ex:BO_16_02_001": "ex:BC_16_02_001", # Fiscal Period -> Fiscal Period
|
||||
"ex:BO_16_03_001": "ex:BC_16_03_001", # Marketing Entity -> Marketing Entity
|
||||
"ex:BO_21_01_001": "ex:BC_21_01_001", # Analysis Period -> Calendar Date
|
||||
}
|
||||
|
||||
MARKETING_NOTE = ("Named Marketing Entity, not Management Entity: the element is "
|
||||
"Marketing Entity Code and the column is MARKETING_ENTITY_CD. "
|
||||
"To be confirmed by DD-16.")
|
||||
|
||||
|
||||
def sd_block(i, ident, name, domain):
|
||||
return ('%s a pr:SubDomain ;\n'
|
||||
' pr:hasIdentifier "%s" ; pr:hasName "%s" ;\n'
|
||||
' pr:belongsTo %s ; pr:owningDomain %s ;\n'
|
||||
' pr:arbitrationStatus "TO_ARBITRATE" ;\n'
|
||||
' pr:hasStatus "DRAFT" ; pr:hasVersion "1.1" ; '
|
||||
'pr:hasSource "DGO proposal, split of a compound sub-domain" .'
|
||||
% (i, ident, name, domain, domain))
|
||||
|
||||
|
||||
def bo_block(i, ident, name, domain, steward, sd, bc):
|
||||
return ('%s a pr:BusinessObject ;\n'
|
||||
' pr:hasIdentifier "%s" ; pr:hasName "%s" ;\n'
|
||||
' pr:owningDomain %s ; pr:ownedBy %s ;\n'
|
||||
' pr:belongsTo %s ; pr:monitoredBy %s ;\n'
|
||||
' pr:aboutConcept %s ;\n'
|
||||
' pr:arbitrationStatus "TO_ARBITRATE" ;\n'
|
||||
' pr:hasStatus "DRAFT" ; pr:hasVersion "1.1" ; '
|
||||
'pr:hasSource "DGO proposal, pending ratification by the owning domain" .'
|
||||
% (i, ident, name, domain, steward, sd, steward, bc))
|
||||
|
||||
|
||||
def _split_clauses(body):
|
||||
"""Split a Turtle predicate list on ';' that sit outside string literals."""
|
||||
out, buf, i, n = [], [], 0, len(body)
|
||||
while i < n:
|
||||
c = body[i]
|
||||
if c == '"':
|
||||
j = i + 1
|
||||
while j < n and body[j] != '"':
|
||||
j += 2 if body[j] == "\\" else 1
|
||||
j = min(j + 1, n)
|
||||
buf.append(body[i:j]); i = j; continue
|
||||
if c == ";":
|
||||
out.append("".join(buf)); buf = []; i += 1; continue
|
||||
buf.append(c); i += 1
|
||||
out.append("".join(buf))
|
||||
return out
|
||||
|
||||
|
||||
def set_prop(text, subject, prop, value):
|
||||
"""
|
||||
Set or replace a single-valued property inside a subject block.
|
||||
|
||||
Clause-level, not regex-level: a property can sit mid-line
|
||||
(pr:hasIdentifier "X" ; pr:hasName "Y" ;) and its value can contain a
|
||||
semicolon inside the literal. A line-anchored pattern silently misses both
|
||||
and APPENDS instead of replacing, which produces a second value and a
|
||||
cardinality violation -- exactly what happened on the first run here.
|
||||
"""
|
||||
pat = re.compile(r'(^%s a pr:\w+ ;)(.*?)(\.\s*\n)' % re.escape(subject), re.S | re.M)
|
||||
m = pat.search(text)
|
||||
if not m:
|
||||
return text, False
|
||||
kept = [c for c in _split_clauses(m.group(2))
|
||||
if c.strip() and not re.match(r'\s*%s\s' % re.escape(prop), c)]
|
||||
kept.insert(0, "\n %s %s " % (prop, value))
|
||||
body = " ;".join(kept)
|
||||
if not body.endswith("\n"):
|
||||
body = body.rstrip() + "\n "
|
||||
return text[:m.start()] + m.group(1) + body + m.group(3) + text[m.end():], True
|
||||
|
||||
|
||||
def main():
|
||||
apply_changes = "--apply" in sys.argv
|
||||
if not os.path.exists(TTL):
|
||||
print("Not found: %s" % TTL)
|
||||
sys.exit(2)
|
||||
text = original = open(TTL, encoding="utf-8").read()
|
||||
report = []
|
||||
|
||||
# 2. sub-domains
|
||||
for sd, name in SD_RENAME.items():
|
||||
text, ok = set_prop(text, sd, "pr:hasName", '"%s"' % name)
|
||||
if ok:
|
||||
report.append("renamed %s -> %s" % (sd.replace("ex:SD_", "SD-"), name))
|
||||
|
||||
todo_sd = [s for s in SD_NEW if not re.search(r'^%s a ' % re.escape(s[0]), text, re.M)]
|
||||
if todo_sd:
|
||||
anchor = re.search(r'\n(?=ex:BC_\w+ a pr:BusinessConcept ;|ex:BO_\w+ a pr:BusinessObject ;)',
|
||||
text)
|
||||
blocks = "\n".join(sd_block(*s) for s in todo_sd)
|
||||
text = (text[:anchor.start()] + "\n\n"
|
||||
+ "# --- sub-domains from the v1.1 structure proposal --------------------\n"
|
||||
+ "# 'Customer & Outlet' and 'Marketing Entity & Fiscal Calendar' had the\n"
|
||||
+ "# same defect as the compound Business Objects: a name joined by '&'\n"
|
||||
+ "# has no single subject. Same rule, one level up.\n"
|
||||
+ blocks + "\n" + text[anchor.start():])
|
||||
report.append("created %d sub-domains%s"
|
||||
% (len(todo_sd), "" if todo_sd else " (already present)"))
|
||||
|
||||
# 3. new business objects
|
||||
todo_bo = [b for b in BO_NEW if not re.search(r'^%s a ' % re.escape(b[0]), text, re.M)]
|
||||
if todo_bo:
|
||||
anchor = re.search(r'\n(?=ex:DO_\w+ a pr:DataObject ;|ex:M_\w+ a pr:Metric ;)', text)
|
||||
blocks = "\n".join(bo_block(*b) for b in todo_bo)
|
||||
text = (text[:anchor.start()] + "\n\n"
|
||||
+ "# --- business objects for the borrowed concepts -----------------------\n"
|
||||
+ "# Country, Fiscal Period and Marketing Entity had a concept but nothing\n"
|
||||
+ "# to hold it. Proposed to DD-04 and DD-16, hence TO_ARBITRATE.\n"
|
||||
+ blocks + "\n" + text[anchor.start():])
|
||||
report.append("created %d business objects%s"
|
||||
% (len(todo_bo), "" if todo_bo else " (already present)"))
|
||||
|
||||
# 4. renames, placement, subject
|
||||
for bo, name in BO_RENAME.items():
|
||||
text, _ = set_prop(text, bo, "pr:hasName", '"%s"' % name)
|
||||
report.append("renamed %d business objects to the agreed nomenclature" % len(BO_RENAME))
|
||||
|
||||
n_b = n_a = 0
|
||||
for bo, sd in BELONGS.items():
|
||||
text, ok = set_prop(text, bo, "pr:belongsTo", sd)
|
||||
n_b += ok
|
||||
for bo, bc in ABOUT.items():
|
||||
text, ok = set_prop(text, bo, "pr:aboutConcept", bc)
|
||||
n_a += ok
|
||||
report.append("placed %d objects in their sub-domain, %d subjects confirmed" % (n_b, n_a))
|
||||
|
||||
# 5. the naming caveat, recorded on the object rather than in a side note
|
||||
text, ok = set_prop(text, "ex:BC_16_03_001", "pr:hasBusinessRule", '"%s"' % MARKETING_NOTE)
|
||||
if ok:
|
||||
report.append("recorded the Marketing/Management naming caveat on the concept")
|
||||
|
||||
# ---- report
|
||||
print()
|
||||
print("SODH STRUCTURE PROPOSAL %s" % ("APPLY" if apply_changes else "DRY RUN"))
|
||||
print("=" * W)
|
||||
for line in report:
|
||||
print(" " + line)
|
||||
print("=" * W)
|
||||
defined = set(re.findall(r'^(ex:\w+) a ', text, re.M))
|
||||
dangling = sorted(set(re.findall(r'\bex:\w+', text)) - defined)
|
||||
print(" dangling references: %s" % (", ".join(dangling) if dangling else "none"))
|
||||
for cls in ("SubDomain", "BusinessObject", "BusinessConcept"):
|
||||
ids = re.findall(r'^(ex:\w+) a pr:%s\b' % cls, text, re.M)
|
||||
dup = [i for i in set(ids) if ids.count(i) > 1]
|
||||
print(" %-16s %3d duplicates %d" % (cls, len(ids), len(dup)))
|
||||
doubled = []
|
||||
for m in re.finditer(r'^(ex:\w+) a pr:\w+ ;(.*?)\.\s*\n', text, re.S | re.M):
|
||||
for prop in ("pr:hasName", "pr:hasIdentifier", "pr:belongsTo", "pr:aboutConcept"):
|
||||
if len(re.findall(r'%s\s' % re.escape(prop), m.group(2))) > 1:
|
||||
doubled.append("%s/%s" % (m.group(1), prop))
|
||||
print(" duplicated single-valued properties: %s"
|
||||
% (", ".join(doubled) if doubled else "none"))
|
||||
arb = len(re.findall(r'pr:arbitrationStatus "TO_ARBITRATE"', text))
|
||||
print(" TO_ARBITRATE %3d (nothing publishes until the domains ratify)" % arb)
|
||||
print("=" * W)
|
||||
|
||||
if text != original and apply_changes:
|
||||
shutil.copy2(TTL, TTL + ".bak")
|
||||
open(TTL, "w", encoding="utf-8").write(text)
|
||||
print(" written, backup at %s.bak" % os.path.basename(TTL))
|
||||
elif text != original:
|
||||
print(" dry run -- re-run with --apply to write")
|
||||
print()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
File diff suppressed because it is too large
Load Diff
+997
-751
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,207 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
SODH - migration v1.1 -> v1.2
|
||||
==============================
|
||||
USAGE
|
||||
python3 scripts/migrate_instances_v1_2.py # dry run
|
||||
python3 scripts/migrate_instances_v1_2.py --apply # rewrite, .bak kept
|
||||
|
||||
WHAT IT DOES
|
||||
1. physicalizedIn -> storedIn everywhere (no instances yet, done for
|
||||
completeness so the term never appears again).
|
||||
2. hasSource removed from every Business Concept. A concept has no source
|
||||
of truth: its authority is its owning domain, its definition belongs in
|
||||
the glossary. Asking which system defines it has no answer, because
|
||||
every system that uses it would be a candidate.
|
||||
3. hasSource on Metrics recomputed from the elements that implement them.
|
||||
It said "SODH back-doc v0.7" -- the provenance of the record, not the
|
||||
source of the data. Asserted rather than derived, by exception, because
|
||||
"where can I get this metric" is a first-order question and the
|
||||
derivation chain will have gaps until harvesting; a shape then checks
|
||||
the assertion against the elements.
|
||||
4. The SODH Data Product, its contract and its two interfaces.
|
||||
|
||||
WHY THE PRODUCT OWNER IS EMPTY
|
||||
Nobody is appointed. The field stays absent and the shape reports a
|
||||
warning rather than a violation. Inventing a name to satisfy a blocking
|
||||
rule is exactly how BR-004 produced padding in v0.6.
|
||||
"""
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import sys
|
||||
|
||||
REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
TTL = os.path.join(REPO, "instances", "sodh.ttl")
|
||||
W = 78
|
||||
|
||||
PRODUCT = "ex:DP_06_01_001"
|
||||
CONTRACT = "ex:DC_06_01_001"
|
||||
IFACE_WH = "ex:DI_06_01_001"
|
||||
IFACE_FILE = "ex:DI_06_01_002"
|
||||
|
||||
|
||||
def clauses(body):
|
||||
out, buf, i, n = [], [], 0, len(body)
|
||||
while i < n:
|
||||
c = body[i]
|
||||
if c == '"':
|
||||
j = i + 1
|
||||
while j < n and body[j] != '"':
|
||||
j += 2 if body[j] == "\\" else 1
|
||||
j = min(j + 1, n)
|
||||
buf.append(body[i:j]); i = j; continue
|
||||
if c == ";":
|
||||
out.append("".join(buf)); buf = []; i += 1; continue
|
||||
buf.append(c); i += 1
|
||||
out.append("".join(buf))
|
||||
return out
|
||||
|
||||
|
||||
def set_prop(text, subject, prop, value):
|
||||
pat = re.compile(r'(^%s a pr:\w+ ;)(.*?)(\.\s*\n)' % re.escape(subject), re.S | re.M)
|
||||
m = pat.search(text)
|
||||
if not m:
|
||||
return text, False
|
||||
kept = [c for c in clauses(m.group(2))
|
||||
if c.strip() and not re.match(r'\s*%s\s' % re.escape(prop), c)]
|
||||
if value is not None:
|
||||
kept.insert(0, "\n %s %s " % (prop, value))
|
||||
body = " ;".join(kept).rstrip() + "\n "
|
||||
return text[:m.start()] + m.group(1) + body + m.group(3) + text[m.end():], True
|
||||
|
||||
|
||||
def get_prop(text, subject, prop):
|
||||
m = re.search(r'^%s a pr:\w+ ;(.*?)\.\s*\n' % re.escape(subject), text, re.S | re.M)
|
||||
if not m:
|
||||
return None
|
||||
for c in clauses(m.group(1)):
|
||||
mm = re.match(r'\s*%s\s+(.*)$' % re.escape(prop), c, re.S)
|
||||
if mm:
|
||||
return mm.group(1).strip()
|
||||
return None
|
||||
|
||||
|
||||
def product_blocks(objects):
|
||||
packages = " ,\n ".join(objects)
|
||||
return """# --- data product ------------------------------------------------------
|
||||
# The unit of delivery: what carries a version, an owner, a changelog and a
|
||||
# list of consumers to warn. It has no single technical counterpart on purpose
|
||||
# -- rename the schema and the product is unchanged; change its SLA and it is
|
||||
# not. Its physical footprint is derivable through packages, never asserted.
|
||||
|
||||
%s a pr:DataProduct ;
|
||||
pr:hasIdentifier "DP-06.01-001" ; pr:hasName "Sell Out Data Hub - Gold" ;
|
||||
pr:owningDomain ex:DD_06 ;
|
||||
pr:operatedBy ex:SD_06_01 ;
|
||||
pr:packages %s ;
|
||||
pr:governedBy %s ;
|
||||
pr:exposes %s , %s ;
|
||||
pr:hasStatus "DRAFT" ; pr:hasVersion "1.2" ; pr:hasSource "SODH Gold" .
|
||||
|
||||
%s a pr:DataContract ;
|
||||
pr:hasIdentifier "DC-06.01-001" ; pr:hasName "Sell Out Data Hub - Gold service commitment" ;
|
||||
pr:owningDomain ex:DD_06 ;
|
||||
pr:hasBusinessRule "Refreshed weekly, available Tuesday 06:00 CET. Panel coverage published with every load. Schema stable within a minor version; 60 days notice on any breaking change. Support through the Sales Performance data team." ;
|
||||
pr:hasStatus "DRAFT" ; pr:hasVersion "1.2" .
|
||||
|
||||
%s a pr:DataInterface ;
|
||||
pr:hasIdentifier "DI-06.01-001" ; pr:hasName "SODH Gold consumption schema" ;
|
||||
pr:owningDomain ex:DD_06 ;
|
||||
pr:hasBusinessDefinition "Read access to the Gold layer through Snowflake grants. The channel BI tools and analysts consume."@en ;
|
||||
pr:hasStatus "DRAFT" ; pr:hasVersion "1.2" ; pr:hasSource "SODH Gold" .
|
||||
|
||||
%s a pr:DataInterface ;
|
||||
pr:hasIdentifier "DI-06.01-002" ; pr:hasName "Panel partner weekly extract" ;
|
||||
pr:owningDomain ex:DD_06 ;
|
||||
pr:hasBusinessDefinition "File extract delivered to the panel provider each week. Outside Snowflake, and the boundary where lineage leaves the warehouse."@en ;
|
||||
pr:hasStatus "DRAFT" ; pr:hasVersion "1.2" ; pr:hasSource "SODH Gold" .
|
||||
""" % (PRODUCT, packages, CONTRACT, IFACE_WH, IFACE_FILE,
|
||||
CONTRACT, IFACE_WH, IFACE_FILE)
|
||||
|
||||
|
||||
def main():
|
||||
apply_changes = "--apply" in sys.argv
|
||||
if not os.path.exists(TTL):
|
||||
print("Not found: %s" % TTL); sys.exit(2)
|
||||
text = original = open(TTL, encoding="utf-8").read()
|
||||
report = []
|
||||
|
||||
# 1. term rename
|
||||
text, n = re.subn(r'\bpr:physicalizedIn\b', 'pr:storedIn', text)
|
||||
report.append("physicalizedIn -> storedIn: %d occurrence(s)" % n)
|
||||
|
||||
# 2. no source of truth on a concept
|
||||
n_bc = 0
|
||||
for m in list(re.finditer(r'^(ex:BC_\w+) a pr:BusinessConcept ;', text, re.M)):
|
||||
if get_prop(text, m.group(1), "pr:hasSource") is not None:
|
||||
text, _ = set_prop(text, m.group(1), "pr:hasSource", None)
|
||||
n_bc += 1
|
||||
report.append("hasSource removed from %d business concepts" % n_bc)
|
||||
|
||||
# 3. metric source of truth, from the elements that implement it
|
||||
de_src = {}
|
||||
for m in re.finditer(r'^(ex:DE_\w+) a pr:DataElement ;(.*?)\.\s*\n', text, re.S | re.M):
|
||||
v = re.search(r'pr:hasSource\s+"([^"]*)"', m.group(2))
|
||||
if v:
|
||||
de_src[m.group(1)] = v.group(1)
|
||||
wired = {}
|
||||
for m in re.finditer(r'(ex:M_\w+)\s+pr:computedBy(.*?)\.', text, re.S):
|
||||
wired[m.group(1)] = re.findall(r'ex:DE_\w+', m.group(2))
|
||||
n_m, mixed = 0, []
|
||||
for metric, des in wired.items():
|
||||
srcs = sorted({de_src[d] for d in des if d in de_src})
|
||||
if not srcs:
|
||||
continue
|
||||
if len(srcs) > 1:
|
||||
mixed.append(metric)
|
||||
text, ok = set_prop(text, metric, "pr:hasSource",
|
||||
" , ".join('"%s"' % x for x in srcs))
|
||||
n_m += ok
|
||||
report.append("source of truth recomputed on %d metrics%s"
|
||||
% (n_m, "" if not mixed else
|
||||
" (%d blend several systems)" % len(mixed)))
|
||||
|
||||
# 4. the data product
|
||||
if not re.search(r'^%s a ' % re.escape(PRODUCT), text, re.M):
|
||||
objs = re.findall(r'^(ex:DO_\w+) a pr:DataObject ;', text, re.M)
|
||||
text = text.rstrip() + "\n\n\n" + product_blocks(objs)
|
||||
report.append("data product created, packaging %d data objects" % len(objs))
|
||||
else:
|
||||
report.append("data product already present, skipped")
|
||||
|
||||
# ---- report
|
||||
print()
|
||||
print("SODH MIGRATION v1.1 -> v1.2 %s" % ("APPLY" if apply_changes else "DRY RUN"))
|
||||
print("=" * W)
|
||||
for line in report:
|
||||
print(" " + line)
|
||||
print("=" * W)
|
||||
bc_src = len(re.findall(r'^ex:BC_\w+ a pr:BusinessConcept ;(?:(?!\.\s*\n).)*?hasSource',
|
||||
text, re.S | re.M))
|
||||
defined = set(re.findall(r'^(ex:\w+) a ', text, re.M))
|
||||
dangling = sorted(set(re.findall(r'\bex:\w+', text)) - defined)
|
||||
doubled = []
|
||||
for m in re.finditer(r'^(ex:\w+) a pr:\w+ ;(.*?)\.\s*\n', text, re.S | re.M):
|
||||
for p in ("pr:hasName", "pr:hasSource", "pr:operatedBy", "pr:governedBy"):
|
||||
if len(re.findall(r'(?<![\w:])%s\s' % re.escape(p), m.group(2))) > 1:
|
||||
doubled.append("%s/%s" % (m.group(1), p))
|
||||
print(" concepts still carrying a source: %d (must be 0)" % bc_src)
|
||||
print(" physicalizedIn left: %d (must be 0)"
|
||||
% len(re.findall(r'pr:physicalizedIn', text)))
|
||||
print(" dangling references: %s" % (", ".join(dangling) if dangling else "none"))
|
||||
print(" duplicated single-valued properties: %s"
|
||||
% (", ".join(doubled) if doubled else "none"))
|
||||
print("=" * W)
|
||||
|
||||
if text != original and apply_changes:
|
||||
shutil.copy2(TTL, TTL + ".bak")
|
||||
open(TTL, "w", encoding="utf-8").write(text)
|
||||
print(" written, backup at %s.bak" % os.path.basename(TTL))
|
||||
elif text != original:
|
||||
print(" dry run -- re-run with --apply to write")
|
||||
print()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -236,13 +236,41 @@ def main():
|
||||
meta_shacl=False,
|
||||
)
|
||||
|
||||
def shape_label(g, res, SH):
|
||||
"""
|
||||
A readable name for the shape that fired.
|
||||
|
||||
sh:sourceShape on a property constraint is a blank node, which prints as
|
||||
'ne775aabb...' and tells the reader nothing. Fall back, in order, to the
|
||||
constrained path, then to the named NodeShape that owns the blank node,
|
||||
then to the constraint component.
|
||||
"""
|
||||
shape = g.value(res, SH.sourceShape)
|
||||
if shape is not None and not str(shape).startswith("n"):
|
||||
name = str(shape).rsplit("/", 1)[-1]
|
||||
if name and not name.startswith("N"):
|
||||
return name
|
||||
path = g.value(res, SH.resultPath)
|
||||
if path is not None:
|
||||
owner = None
|
||||
for s_, p_, o_ in g.triples((None, SH.property, shape)):
|
||||
owner = s_
|
||||
break
|
||||
base = str(path).rsplit("/", 1)[-1].rsplit("#", 1)[-1]
|
||||
if owner is not None and not str(owner).startswith("n"):
|
||||
return "%s / %s" % (str(owner).rsplit("/", 1)[-1], base)
|
||||
return "path %s" % base
|
||||
comp = g.value(res, SH.sourceConstraintComponent)
|
||||
if comp is not None:
|
||||
return str(comp).rsplit("#", 1)[-1]
|
||||
return "(unnamed shape)"
|
||||
|
||||
rows = []
|
||||
for res in results.subjects(RDF.type, SH.ValidationResult):
|
||||
sev = str(results.value(res, SH.resultSeverity)).rsplit("#", 1)[-1]
|
||||
focus = str(results.value(res, SH.focusNode)).rsplit("/", 1)[-1]
|
||||
msg = str(results.value(res, SH.resultMessage) or "")
|
||||
src = str(results.value(res, SH.sourceShape) or "").rsplit("/", 1)[-1]
|
||||
rows.append((sev, src, focus, msg))
|
||||
rows.append((sev, shape_label(results, res, SH), focus, msg))
|
||||
|
||||
violations = sum(1 for r in rows if r[0] == "Violation")
|
||||
warnings = len(rows) - violations
|
||||
|
||||
@@ -40,7 +40,7 @@
|
||||
|
||||
<https://ontology.pernod-ricard.com/metamodel/shapes/>
|
||||
dcterms:title "PR Data Meta Model - SHACL Shapes" ;
|
||||
dcterms:modified "2026-07-26"^^xsd:date ;
|
||||
dcterms:modified "2026-07-27"^^xsd:date ;
|
||||
rdfs:comment "Executable form of the rulebook. Validated by scripts/run_shacl_validation.py." .
|
||||
|
||||
|
||||
@@ -231,7 +231,12 @@ prs:BusinessConceptShape a sh:NodeShape ;
|
||||
sh:property [
|
||||
sh:path pr:hasFormula ; sh:maxCount 0 ;
|
||||
sh:severity sh:Violation ;
|
||||
sh:message "BR-003 -- the presence of a formula is what makes something a Metric. A Concept with a formula is a misclassified Metric." ] .
|
||||
sh:message "BR-003 -- the presence of a formula is what makes something a Metric. A Concept with a formula is a misclassified Metric." ] ;
|
||||
|
||||
sh:property [
|
||||
sh:path pr:hasSource ; sh:maxCount 0 ;
|
||||
sh:severity sh:Violation ;
|
||||
sh:message "A Business Concept has no source of truth. Its authority is its owning domain and its definition belongs in the glossary. Naming a system here would ask which of the systems that USE the concept defines it -- a question with no answer." ] .
|
||||
|
||||
|
||||
prs:MetricShape a sh:NodeShape ;
|
||||
@@ -244,9 +249,9 @@ prs:MetricShape a sh:NodeShape ;
|
||||
sh:message "BR-003 -- a Metric is a measurement with a calculation rule. No formula, no Metric." ] ;
|
||||
|
||||
sh:property [
|
||||
sh:path pr:hasUnit ; sh:minCount 1 ; sh:maxCount 1 ;
|
||||
sh:path pr:hasUnit ; sh:minCount 1 ;
|
||||
sh:severity sh:Violation ;
|
||||
sh:message "A Metric states its unit." ] ;
|
||||
sh:message "A Metric states every unit its variants are expressed in. NOT capped at one: Sell Out Volume is computed in 9L and in L, Sell Out Value in EUR, USD and local currency. The single-unit rule belongs on the Data Element, where it holds -- see DataElementShape." ] ;
|
||||
|
||||
sh:property [
|
||||
sh:path pr:measures ; sh:class pr:BusinessConcept ;
|
||||
@@ -265,6 +270,38 @@ prs:MetricShape a sh:NodeShape ;
|
||||
sh:message "A Metric must not carry a physical name -- that is a layer leak. Under BR-013 the physical name belongs on the Data Element." ] .
|
||||
|
||||
|
||||
# The source of truth of a Metric is asserted by exception, then checked.
|
||||
# "Where can I get this metric" is a first-order question and the derivation
|
||||
# chain will have gaps before harvesting, so the assertion stands on its own --
|
||||
# but it must agree with the elements that implement the metric, or the two
|
||||
# will drift the day an element moves.
|
||||
prs:MetricSourceShape a sh:NodeShape ;
|
||||
sh:targetClass pr:Metric ;
|
||||
sh:sparql [
|
||||
sh:severity sh:Violation ;
|
||||
sh:message "A Metric's source of truth must match the source of at least one element that computes it. If they disagree, either the metric points at the wrong system or an element has moved." ;
|
||||
sh:select """
|
||||
PREFIX pr: <https://ontology.pernod-ricard.com/metamodel/>
|
||||
SELECT $this WHERE {
|
||||
$this pr:hasSource ?src ; pr:computedBy ?de .
|
||||
FILTER NOT EXISTS {
|
||||
$this pr:computedBy ?any .
|
||||
?any pr:hasSource ?src .
|
||||
}
|
||||
}""" ] ;
|
||||
sh:sparql [
|
||||
sh:severity sh:Warning ;
|
||||
sh:message "Elements computing one Metric come from several systems of truth. Legitimate for a metric blending sources, worth a look otherwise." ;
|
||||
sh:select """
|
||||
PREFIX pr: <https://ontology.pernod-ricard.com/metamodel/>
|
||||
SELECT $this WHERE {
|
||||
$this pr:computedBy ?a , ?b .
|
||||
?a pr:hasSource ?sa .
|
||||
?b pr:hasSource ?sb .
|
||||
FILTER(?sa != ?sb)
|
||||
}""" ] .
|
||||
|
||||
|
||||
# BR-013, finally executable. v0.1.1 declared it BLOCKING with no shape.
|
||||
prs:MotherMetricShape a sh:NodeShape ;
|
||||
sh:targetClass pr:Metric ;
|
||||
@@ -306,7 +343,13 @@ prs:DataObjectShape a sh:NodeShape ;
|
||||
sh:property [
|
||||
sh:path pr:monitoredBy ; sh:maxCount 0 ;
|
||||
sh:severity sh:Warning ;
|
||||
sh:message "Stewardship is declared on the Business Object and inherited through represents. Declaring it here allows two answers to the same question." ] .
|
||||
sh:message "Stewardship is declared on the Business Object and inherited through represents. Declaring it here allows two answers to the same question." ] ;
|
||||
|
||||
sh:property [
|
||||
sh:path pr:hasSource ; sh:maxCount 1 ;
|
||||
sh:not [ sh:pattern "/" ] ;
|
||||
sh:severity sh:Warning ;
|
||||
sh:message "hasSource names the SYSTEM of origin, nothing else. A value like 'SODH Gold / F_SELL_OUT' packs a system and a physical relation into one string -- the physical relation belongs in physicalName." ] .
|
||||
|
||||
|
||||
# A declared grain element must actually belong to the object it grains.
|
||||
@@ -340,6 +383,11 @@ prs:DataElementShape a sh:NodeShape ;
|
||||
sh:path pr:fullyQualifiedName ; sh:maxCount 1 ;
|
||||
sh:severity sh:Info ] ;
|
||||
|
||||
sh:property [
|
||||
sh:path pr:hasUnit ; sh:maxCount 1 ;
|
||||
sh:severity sh:Violation ;
|
||||
sh:message "A Data Element carries exactly one unit. This is the counterpart of the Metric rule: the mother metric lists the units its variants use, each variant states the single unit it is actually expressed in. An element with two units is two elements." ] ;
|
||||
|
||||
sh:property [
|
||||
sh:path pr:represents ; sh:class pr:BusinessConcept ;
|
||||
sh:maxCount 1 ;
|
||||
@@ -381,9 +429,9 @@ prs:DataProductShape a sh:NodeShape ;
|
||||
|
||||
sh:property [
|
||||
sh:path pr:hasProductOwner ; sh:class pr:DataProductOwner ;
|
||||
sh:minCount 1 ; sh:maxCount 1 ;
|
||||
sh:severity sh:Violation ;
|
||||
sh:message "A Data Product has exactly one Product Owner." ] ;
|
||||
sh:maxCount 1 ;
|
||||
sh:severity sh:Warning ;
|
||||
sh:message "A Data Product should name exactly one Product Owner. A warning rather than a block: the SODH owner is genuinely not appointed, and a blocking rule that cannot be satisfied truthfully is how BR-004 produced padding." ] ;
|
||||
|
||||
sh:property [
|
||||
sh:path pr:packages ; sh:class pr:DataObject ;
|
||||
|
||||
Reference in New Issue
Block a user