tbox: v1.5 - PhysicalRelation devient DataStructure et point d'extension declare pour le non-tabulaire, hasSource devient sourcedFrom vers un System instancie, storedIn sert deux granularites et deprecie materializedAs, references devient qualifiedBy, primaryLocation sous-propriete de storedIn, hasShortLabel sur 34 classes. CuratedObject devient DefinedObject et ObservedObject devient CapturedObject jusque dans les IRI, curationMode devient authoringMode. Correction de fond : PhysicalLayerObject ne porte plus la provenance, chaque classe de la couche la declare - un System est defined, tout ce qui est en dessous est captured. instances: 5 systemes crees, 151 sourcedFrom, 15 BO nettoyes de hasSource, labels courts, versions alignees
This commit is contained in:
Bastien Gourdon
@@ -35,13 +35,13 @@ DOMAIN_STYLE = OrderedDict([
|
||||
])
|
||||
|
||||
MULTI = {"composedOf", "usesConcept", "hasElement", "hasMetric", "measures",
|
||||
"computedBy", "references", "hasGrainElement", "monitoredBy", "hasSynonym",
|
||||
"computedBy", "qualifiedBy", "hasGrainElement", "monitoredBy", "hasSynonym",
|
||||
"hasUnit", "hasSource", "packages", "exposes"}
|
||||
|
||||
SUBJECT_DOMAIN = "DD_06" # the hub owner: everything else is a dependency
|
||||
|
||||
# Every user-facing label in one place. Hyphenated throughout, and never the
|
||||
# raw class name: "Physical Relation" and "Physicalized in" told a business
|
||||
# raw class name: "Data Structure" and "Physicalized in" told a business
|
||||
# reader nothing.
|
||||
LABELS = {
|
||||
"DataDomain": "Domain", "SubDomain": "Sub-domain",
|
||||
@@ -49,14 +49,17 @@ LABELS = {
|
||||
"Metric": "Metric", "DataObject": "Data object", "DataElement": "Data element",
|
||||
"BaseTable": "Base table", "Field": "Field",
|
||||
"DataProduct": "Data product", "DataContract": "Data contract",
|
||||
"System": "System",
|
||||
"DataInterface": "Data interface",
|
||||
}
|
||||
|
||||
# Which ownership roles a card shows. The Steward section is gone: one section,
|
||||
# different roles per object, so the reader always looks in the same place.
|
||||
ROLES = {
|
||||
"DataDomain": [("Domain owner", "owner"), ("Data-governance lead", "dgl")],
|
||||
"SubDomain": [("Sub-domain owner", "subOwner"), ("Data-governance lead", "dgl_up")],
|
||||
# role labels mirror the short labels the T-Box carries on the actor
|
||||
# classes, so the two viewers name the same role the same way
|
||||
"DataDomain": [("Domain owner", "owner"), ("Data governance lead", "dgl")],
|
||||
"SubDomain": [("Sub-domain owner", "subOwner"), ("Data governance lead", "dgl_up")],
|
||||
"BusinessObject": [("Steward", "steward")],
|
||||
"BusinessConcept": [("Steward", "steward")],
|
||||
"Metric": [("Steward", "steward")],
|
||||
@@ -180,7 +183,10 @@ def build(subjects):
|
||||
nodes[subj] = {
|
||||
"id": subj,
|
||||
"kind": t,
|
||||
# the model carries both names now: hasShortLabel for screens,
|
||||
# hasName for the record. The viewer stops keeping its own list.
|
||||
"label": LABELS.get(t, t),
|
||||
"shortName": rec.get("hasShortLabel", ""),
|
||||
"ident": rec.get("hasIdentifier", ""),
|
||||
"name": rec.get("hasName", subj.replace("ex:", "")),
|
||||
"domain": domain_of(subj, rec, subjects),
|
||||
@@ -193,6 +199,8 @@ def build(subjects):
|
||||
"physical": rec.get("physicalName", ""),
|
||||
"format": rec.get("hasFormat", ""),
|
||||
"source": ", ".join(as_list(rec.get("hasSource"))),
|
||||
"sourcedFrom": rec.get("sourcedFrom", ""),
|
||||
"fqn": "",
|
||||
"version": rec.get("hasVersion", ""),
|
||||
"operatedBy": rec.get("operatedBy", ""),
|
||||
"governedBy": rec.get("governedBy", ""),
|
||||
@@ -208,7 +216,7 @@ def build(subjects):
|
||||
"computedBy": as_list(rec.get("computedBy")),
|
||||
"hasElement": as_list(rec.get("hasElement")),
|
||||
"grain": as_list(rec.get("hasGrainElement")),
|
||||
"references": as_list(rec.get("references")),
|
||||
"qualifiedBy": as_list(rec.get("qualifiedBy")),
|
||||
"steward": as_list(rec.get("monitoredBy")) or as_list(rec.get("ownedBy")),
|
||||
"dgl": rec.get("hasDGL", ""),
|
||||
"owner": rec.get("hasDomainOwner", ""),
|
||||
@@ -239,6 +247,7 @@ def build(subjects):
|
||||
phys.setdefault(pid, {"id": pid, "kind": "BaseTable",
|
||||
"label": LABELS["BaseTable"],
|
||||
"name": n["physical"], "domain": n["domain"],
|
||||
"fqn": n["physical"],
|
||||
"derived": True, "backs": []})
|
||||
phys[pid]["backs"].append(n["id"])
|
||||
n["materializedAs"] = pid
|
||||
@@ -251,6 +260,7 @@ def build(subjects):
|
||||
phys.setdefault(pid, {"id": pid, "kind": "Field",
|
||||
"label": LABELS["Field"], "name": n["physical"],
|
||||
"domain": n["domain"], "derived": True,
|
||||
"fqn": (rel + "." if rel else "") + n["physical"],
|
||||
"relation": "phys:" + rel if rel else "",
|
||||
"backs": []})
|
||||
phys[pid]["backs"].append(n["id"])
|
||||
@@ -258,13 +268,14 @@ def build(subjects):
|
||||
for p in phys.values():
|
||||
p.setdefault("ident", ""); p.setdefault("arb", False)
|
||||
p.setdefault("label", LABELS.get(p["kind"], p["kind"]))
|
||||
p.setdefault("shortName", "")
|
||||
for k in ("definition","rule","formula","physical","format","source","belongsTo",
|
||||
"about","represents","status","dgl","owner","subOwner","activation",
|
||||
"version","operatedBy","governedBy","productOwner","storedIn",
|
||||
"materializedAs","relation","stewardFrom"):
|
||||
"materializedAs","relation","stewardFrom","fqn","sourcedFrom"):
|
||||
p.setdefault(k, "")
|
||||
for k in ("uses","measures","hasMetric","computedBy","hasElement","grain",
|
||||
"references","steward","units","computedFrom","packages","exposes",
|
||||
"qualifiedBy","steward","units","computedFrom","packages","exposes",
|
||||
"steward_inherited"):
|
||||
p.setdefault(k, [])
|
||||
p.setdefault("route", "")
|
||||
@@ -777,7 +788,7 @@ var G = {mode:"strata", doms:[], sel:null, hot:null, labels:true,
|
||||
show:{DataDomain:true, SubDomain:true, BusinessObject:true,
|
||||
BusinessConcept:true, Metric:true, DataObject:true,
|
||||
DataElement:false, DataProduct:true, DataContract:true,
|
||||
DataInterface:true, BaseTable:true, Field:false},
|
||||
DataInterface:true, System:true, BaseTable:true, Field:false},
|
||||
view:{x:0,y:0,s:1}, nodes:[], edges:[]};
|
||||
var NW=126, NH=32, LANEPAD=26, ROWGAP=15, BANDGAP=34, GUT=142;
|
||||
|
||||
@@ -787,12 +798,12 @@ var ROW = {DataDomain:0, SubDomain:1, BusinessObject:2,
|
||||
BusinessConcept:0, Metric:1,
|
||||
DataObject:0, DataElement:1,
|
||||
DataProduct:0, DataContract:1, DataInterface:1,
|
||||
BaseTable:0, Field:1};
|
||||
System:0, BaseTable:1, Field:2};
|
||||
var LAYER = {DataDomain:0, SubDomain:0, BusinessObject:0,
|
||||
BusinessConcept:1, Metric:1,
|
||||
DataObject:2, DataElement:2,
|
||||
DataProduct:3, DataInterface:3, DataContract:3,
|
||||
BaseTable:4, Field:4};
|
||||
System:4, BaseTable:4, Field:4};
|
||||
var BANDS = [
|
||||
{t:"Ownership & Categorization", g:"who is accountable"},
|
||||
{t:"Business", g:"what it means"},
|
||||
@@ -805,7 +816,7 @@ var KINDS = [["DataDomain","Domains"],["SubDomain","Sub-domains"],
|
||||
["Metric","Metrics"],["DataObject","Data objects"],
|
||||
["DataElement","Data elements"],["DataProduct","Data products"],
|
||||
["DataContract","Data contracts"],["DataInterface","Data interfaces"],
|
||||
["BaseTable","Base tables"],["Field","Fields"]];
|
||||
["System","Systems"],["BaseTable","Base tables"],["Field","Fields"]];
|
||||
|
||||
function subjectColour(dom){ return dom === M.subject ? "#1E4B9B" : "#8A6A28"; }
|
||||
|
||||
@@ -831,8 +842,8 @@ function buildGraph(){
|
||||
n.measures.forEach(function(c){ edge(n.id, c, "measures"); });
|
||||
n.computedBy.forEach(function(d){ edge(n.id, d, "computed by"); });
|
||||
n.hasElement.forEach(function(d){ edge(n.id, d, "has element"); });
|
||||
n.references.forEach(function(d){ edge(n.id, d, "references"); });
|
||||
if(n.materializedAs) edge(n.id, n.materializedAs, "materialized as");
|
||||
n.qualifiedBy.forEach(function(d){ edge(n.id, d, "qualified by"); });
|
||||
if(n.sourcedFrom) edge(n.id, n.sourcedFrom, "sourced from");
|
||||
if(n.storedIn) edge(n.id, n.storedIn, "stored in");
|
||||
if(n.operatedBy) edge(n.id, n.operatedBy, "operated by");
|
||||
if(n.governedBy) edge(n.id, n.governedBy, "governed by");
|
||||
@@ -1113,10 +1124,12 @@ function renderGraphShell(){
|
||||
}).join("")
|
||||
+ '<div class="gsec">Options</div>'
|
||||
+ '<label><input type="checkbox" id="g-lab" checked> Relation names</label>'
|
||||
+ '<select id="g-dom"><option value="">All domains</option>'
|
||||
+ M.domains.map(function(d){ return '<option value="'+d.code+'"'
|
||||
+ (G.dom===d.code?" selected":"") + '>'+esc(d.label)+'</option>'; }).join("")
|
||||
+ '</select>'
|
||||
+ '<div class="gsec">Domains</div>'
|
||||
+ M.domains.map(function(d){
|
||||
return '<label><input type="checkbox" data-dm="' + d.code + '"'
|
||||
+ (!G.doms.length || G.doms.indexOf(d.code) > -1 ? " checked" : "")
|
||||
+ '> <i style="background:' + (d.code === M.subject ? "#1E4B9B" : "#8A6A28")
|
||||
+ '"></i>' + esc(d.label) + '</label>'; }).join("")
|
||||
+ '<div class="mono" id="g-count"></div></div>');
|
||||
h.push('<div class="gzoom"><button id="g-in">+</button>'
|
||||
+ '<button id="g-out">\u2212</button><button id="g-fit" style="font-size:12px">'
|
||||
@@ -1222,7 +1235,7 @@ var HIST = [];
|
||||
|
||||
/* reverse indexes, built once */
|
||||
var PARENT_DO = {}, SUBS_OF = {}, BOS_OF = {}, BACKS = {},
|
||||
METRICS_OF = {}, REALISED_BY = {};
|
||||
METRICS_OF = {}, REALISED_BY = {}, ABOUT_OF = {}, USES_OF = {};
|
||||
M.nodes.forEach(function(n){
|
||||
n.hasElement.forEach(function(de){ PARENT_DO[de] = n.id; });
|
||||
if(n.kind === "SubDomain" && n.belongsTo) (SUBS_OF[n.belongsTo] = SUBS_OF[n.belongsTo]||[]).push(n.id);
|
||||
@@ -1230,6 +1243,8 @@ M.nodes.forEach(function(n){
|
||||
(n.backs||[]).forEach(function(b){ (BACKS[n.id] = BACKS[n.id]||[]).push(b); });
|
||||
/* a concept needs to show the metrics that measure it, not only the reverse */
|
||||
n.measures.forEach(function(bc){ (METRICS_OF[bc] = METRICS_OF[bc]||[]).push(n.id); });
|
||||
if(n.about) (ABOUT_OF[n.about] = ABOUT_OF[n.about]||[]).push(n.id);
|
||||
n.uses.forEach(function(u){ (USES_OF[u] = USES_OF[u]||[]).push(n.id); });
|
||||
if(n.kind === "DataObject" && n.represents)
|
||||
(REALISED_BY[n.represents] = REALISED_BY[n.represents]||[]).push(n.id);
|
||||
});
|
||||
@@ -1256,8 +1271,15 @@ function paint(id){
|
||||
+ esc(byId[HIST[HIST.length-2]].name) + '</button>');
|
||||
h.push('<div class="kind" style="color:' + subjectColour(n.domain) + '">'
|
||||
+ esc(n.label || n.kind) + '</div>');
|
||||
h.push('<h3>' + esc(n.name) + '</h3>');
|
||||
h.push('<div class="iri">' + esc(n.ident || n.id) + '</div>');
|
||||
h.push('<h3>' + esc(n.shortName || n.name) + '</h3>');
|
||||
if(n.shortName && n.shortName !== n.name)
|
||||
h.push('<div class="row" style="border:0;padding:0 0 8px"><span class="mono">'
|
||||
+ esc(n.name) + '</span><span></span></div>');
|
||||
/* a derived physical node has no governance identifier -- showing its
|
||||
internal id as a subtitle read like one. The qualified name is an
|
||||
attribute, and belongs in the attributes block with everything else. */
|
||||
if(n.ident) h.push('<div class="iri">' + esc(n.ident) + '</div>');
|
||||
else h.push('<div style="height:8px"></div>');
|
||||
if(n.arb) h.push('<div class="note"><b>Awaiting arbitration.</b> Proposed by the '
|
||||
+ 'Data Governance Office; ' + esc(domLabel(n.domain))
|
||||
+ ' owns this perimeter and has not ratified it. Cannot be published (OW-007).</div>');
|
||||
@@ -1270,15 +1292,20 @@ function paint(id){
|
||||
var rows = [];
|
||||
function at(label, v){ if(v) rows.push([label, v]); }
|
||||
at("Status", n.status);
|
||||
at("Source-of-truth", n.source);
|
||||
if(n.kind !== "DataProduct") at("Source-of-truth", n.source);
|
||||
if(n.kind === "DataElement"){
|
||||
at("Base table", byId[n.storedIn] && byId[byId[n.storedIn].relation]
|
||||
? byId[byId[n.storedIn].relation].name : "");
|
||||
at("Table field", n.physical);
|
||||
at("Unit", n.units.join(", "));
|
||||
at("Format", n.format);
|
||||
} else if(n.kind === "DataObject"){
|
||||
at("Base table", n.physical);
|
||||
} else if(n.kind === "BaseTable" || n.kind === "Field"){
|
||||
at("Fully qualified name", n.fqn);
|
||||
} else {
|
||||
at("Unit" + (n.units.length > 1 ? "s" : ""), n.units.join(", "));
|
||||
at("Format", n.format);
|
||||
if(n.kind !== "DataElement") at("Physical name", n.physical);
|
||||
}
|
||||
at("Activation", n.activation.replace(/([A-Z])/g," $1").trim());
|
||||
at("Version", n.version);
|
||||
@@ -1287,7 +1314,8 @@ function paint(id){
|
||||
+ esc(r[1])+'</span></div>'; }).join(""));
|
||||
|
||||
/* ---- 2. what it means ---- */
|
||||
if(n.definition) h.push('<h4>Definition</h4><p>' + esc(n.definition) + '</p>');
|
||||
h.push('<h4>What it is for</h4><p>' + (n.definition ? esc(n.definition)
|
||||
: '<span class="mono">No definition yet.</span>') + '</p>');
|
||||
if(n.formula) h.push('<h4>Calculation rule</h4><p class="mono">'
|
||||
+ esc(n.formula) + '</p>');
|
||||
if(n.rule) h.push('<h4>' + (n.kind === "DataContract" ? "Commitment" : "Open question")
|
||||
@@ -1304,8 +1332,11 @@ function paint(id){
|
||||
else if(r[1] === "productOwner") ids = n.productOwner ? [n.productOwner] : [];
|
||||
else if(Array.isArray(n[r[1]])) ids = n[r[1]];
|
||||
else if(n[r[1]]) ids = [n[r[1]]];
|
||||
var names = ids.map(function(i){ return byId[i] ? esc(byId[i].name) : ""; })
|
||||
.filter(Boolean);
|
||||
/* actors carry a short label too; the ownership block shows it rather
|
||||
than the full record name */
|
||||
var names = ids.map(function(i){
|
||||
return byId[i] ? esc(byId[i].shortName || byId[i].name) : ""; })
|
||||
.filter(Boolean);
|
||||
var inherited = (r[1] === "steward_inherited" && n.stewardFrom && names.length);
|
||||
return '<div class="row"><span>' + r[0] + '</span><span class="mono">'
|
||||
+ (names.length ? names.join(", ") : "to be defined")
|
||||
@@ -1331,22 +1362,24 @@ function paint(id){
|
||||
link("Business objects", BOS_OF[n.id]);
|
||||
link("Is about", [n.about]);
|
||||
link("Also uses", n.uses);
|
||||
/* a concept is the subject of the objects that point at it: shown before
|
||||
the metrics, because the objects are the coarser grain */
|
||||
link("Defines", ABOUT_OF[n.id]);
|
||||
link("Used by", USES_OF[n.id]);
|
||||
link("Metrics", (n.hasMetric || []).concat(METRICS_OF[n.id] || []));
|
||||
link("Measures", n.measures);
|
||||
link("Represents", [n.represents]);
|
||||
link("Implements metric", n.computedFrom);
|
||||
link("Governed by", [n.governedBy]);
|
||||
link("Exposes", n.exposes);
|
||||
link("Implements", n.computedFrom);
|
||||
link("Packages", n.packages);
|
||||
link("Data objects", REALISED_BY[n.id]);
|
||||
link("References", n.references);
|
||||
link("Qualified by", n.qualifiedBy);
|
||||
link("Grain", n.grain);
|
||||
link("Held by data object", [PARENT_DO[n.id]]);
|
||||
link("Held by", [PARENT_DO[n.id]]);
|
||||
link("Computed by", n.computedBy);
|
||||
if(n.hasElement.length) links.push(["Data elements", n.hasElement]);
|
||||
link("Materialized as", [n.materializedAs]);
|
||||
link("Stored in", [n.storedIn]);
|
||||
link("Base table", [n.relation]);
|
||||
link("Sourced from", [n.sourcedFrom]);
|
||||
link("Held by", [n.relation]);
|
||||
link("Provides data for", BACKS[n.id]);
|
||||
|
||||
links.forEach(function(l){
|
||||
|
||||
+147
-69
@@ -13,7 +13,7 @@ Same visual language as the SODH viewer, with one substitution. Where SODH
|
||||
lays its columns out by owning domain, the T-Box has no domains -- its
|
||||
orthogonal axis is PROVENANCE: what governance asserts against what a
|
||||
harvester observes. The two-colour code carries that instead, blue for
|
||||
curated and dark gold for observed, and the seam is what crosses between them.
|
||||
defined and dark gold for captured, and the seam is what crosses between them.
|
||||
|
||||
No dependency: parses the Turtle subset this file uses, so it runs on the DSM
|
||||
system python without a venv.
|
||||
@@ -33,8 +33,8 @@ WANTED = {
|
||||
"rdfs:domain": "domain", "rdfs:range": "range",
|
||||
"owl:inverseOf": "inverseOf", "owl:deprecated": "deprecated",
|
||||
"dcterms:isReplacedBy": "isReplacedBy",
|
||||
"pr:harvestSource": "harvestSource", "pr:curationMode": "curationMode",
|
||||
"pr:hasAcronym": "acronym",
|
||||
"pr:harvestSource": "harvestSource", "pr:authoringMode": "authoringMode",
|
||||
"pr:hasAcronym": "acronym", "pr:hasShortLabel": "shortLabel",
|
||||
}
|
||||
|
||||
LAYER_ROOT = OrderedDict([
|
||||
@@ -43,8 +43,8 @@ LAYER_ROOT = OrderedDict([
|
||||
("pr:BusinessLayerObject", ("business", "Business", "what it means")),
|
||||
("pr:LogicalLayerObject", ("logical", "Logical",
|
||||
"how it is structured")),
|
||||
("pr:DeliveryLayerObject", ("delivery", "Delivery",
|
||||
"what we promise, to whom")),
|
||||
("pr:DeliveryLayerObject", ("delivery", "Delivery",
|
||||
"what we promise, to whom, until when")),
|
||||
("pr:PhysicalLayerObject", ("physical", "Physical",
|
||||
"what actually runs")),
|
||||
("pr:ConsumptionLayerObject", ("consumption", "Consumption",
|
||||
@@ -213,19 +213,24 @@ def build(subjects):
|
||||
layer = "vocab"
|
||||
else:
|
||||
layer = "roots"
|
||||
prov = ("observed" if "pr:ObservedObject" in anc
|
||||
else "curated" if "pr:CuratedObject" in anc else "structural")
|
||||
# provenance is INFERRED from the type-of chain, never asserted:
|
||||
# a Sub-domain is Defined because it is a kind of Ownership Layer
|
||||
# Object, which is a kind of Defined Object. No separate edge exists
|
||||
# because none is needed -- the subclass chain already says it.
|
||||
prov = ("captured" if "pr:CapturedObject" in anc
|
||||
else "defined" if "pr:DefinedObject" in anc else "structural")
|
||||
depth, p, guard = 0, parents.get(term), 0
|
||||
while p and p in classes and guard < 8:
|
||||
depth += 1; guard += 1; p = parents.get(p)
|
||||
nodes[term] = {
|
||||
"id": term, "kind": "Class",
|
||||
"label": first(rec, "label", term.split(":")[1]),
|
||||
"shortLabel": first(rec, "shortLabel"),
|
||||
"layer": layer, "prov": prov, "depth": depth,
|
||||
"parent": parents.get(term, ""),
|
||||
"comment": first(rec, "comment"),
|
||||
"harvest": first(rec, "harvestSource"),
|
||||
"curation": first(rec, "curationMode"),
|
||||
"authoring": first(rec, "authoringMode"),
|
||||
"acronym": first(rec, "acronym"),
|
||||
"deprecated": first(rec, "deprecated") in ("true", "True"),
|
||||
"replacedBy": first(rec, "isReplacedBy"),
|
||||
@@ -240,11 +245,12 @@ def build(subjects):
|
||||
return {
|
||||
"id": term, "kind": kind,
|
||||
"label": first(rec, "label", term.split(":")[1]),
|
||||
"shortLabel": first(rec, "shortLabel"),
|
||||
"comment": first(rec, "comment"),
|
||||
"domain": first(rec, "domain"), "range": first(rec, "range"),
|
||||
"inverseOf": first(rec, "inverseOf"),
|
||||
"subPropertyOf": first(rec, "subPropertyOf"),
|
||||
"curation": first(rec, "curationMode"),
|
||||
"authoring": first(rec, "authoringMode"),
|
||||
"harvest": first(rec, "harvestSource"),
|
||||
"functional": "owl:FunctionalProperty" in rec.get("type", []),
|
||||
"transitive": "owl:TransitiveProperty" in rec.get("type", []),
|
||||
@@ -298,7 +304,7 @@ def main():
|
||||
c = len([n for n in live_c if n["layer"] == l["id"]])
|
||||
if c:
|
||||
print(" %-28s %d" % (l["title"], c))
|
||||
for p in ("curated", "observed", "structural"):
|
||||
for p in ("defined", "captured", "structural"):
|
||||
c = len([n for n in live_c if n["prov"] == p])
|
||||
print(" %-28s %d" % (p, c))
|
||||
|
||||
@@ -528,10 +534,33 @@ var G = {mode:"strata", provs:[], sel:null, hot:null, labels:true,
|
||||
var NW=126, NH=32, LANEPAD=26, ROWGAP=15, BANDGAP=34, GUT=142;
|
||||
|
||||
function provColour(p){
|
||||
return p === "observed" ? "#8A6A28" : p === "curated" ? "#1E4B9B" : "#6B6B6B";
|
||||
return p === "captured" ? "#8A6A28" : p === "defined" ? "#1E4B9B" : "#6B6B6B";
|
||||
}
|
||||
var LANES = [["curated","Curated \u00b7 asserted by governance","#1E4B9B"],
|
||||
["observed","Observed \u00b7 harvested nightly","#8A6A28"],
|
||||
/* Scoped relations: these carry no rdfs:domain or rdfs:range on purpose,
|
||||
because each serves several classes with different targets and the scope
|
||||
belongs in the SHACL shapes. Both the graph and the cards read this list,
|
||||
so an edge drawn in one is never missing from the other. */
|
||||
var SCOPED_PAIRS = [
|
||||
["pr:belongsTo", "pr:SubDomain", "pr:DataDomain", "belongs to"],
|
||||
["pr:belongsTo", "pr:BusinessObject", "pr:SubDomain", "belongs to"],
|
||||
["pr:represents", "pr:DataObject", "pr:BusinessObject", "represents"],
|
||||
["pr:represents", "pr:DataElement", "pr:BusinessConcept", "represents"],
|
||||
["pr:monitoredBy", "pr:BusinessObject", "pr:DataSteward", "monitored by"],
|
||||
["pr:owningDomain", "pr:DefinedObject", "pr:DataDomain", "owning domain"],
|
||||
["pr:ownedBy", "pr:DefinedObject", "pr:Actor", "owned by"]
|
||||
];
|
||||
var ORDER = ["pr:MetaModelObject", "pr:DefinedObject", "pr:CapturedObject",
|
||||
"pr:OwnershipLayerObject", "pr:DataDomain", "pr:SubDomain", "pr:BusinessObject",
|
||||
"pr:BusinessLayerObject", "pr:BusinessConcept", "pr:Metric", "pr:KPI",
|
||||
"pr:LogicalLayerObject", "pr:DataObject", "pr:DataElement",
|
||||
"pr:DeliveryLayerObject", "pr:DataProduct", "pr:DataContract", "pr:DataInterface",
|
||||
"pr:PhysicalLayerObject", "pr:System", "pr:Database", "pr:Schema",
|
||||
"pr:DataStructure", "pr:BaseTable", "pr:View", "pr:ExternalTable", "pr:Field",
|
||||
"pr:KeyConstraint", "pr:PrimaryKey", "pr:ForeignKeyConstraint", "pr:Transformation",
|
||||
"pr:ConsumptionLayerObject", "pr:BIWorkspace", "pr:BIDataSource", "pr:BIField",
|
||||
"pr:BIReport"];
|
||||
var LANES = [["defined","Defined \u00b7 someone decided it","#1E4B9B"],
|
||||
["captured","Captured \u00b7 read from a running system","#8A6A28"],
|
||||
["structural","Structural","#6B6B6B"]];
|
||||
function live(a){ return a.filter(function(x){ return !x.deprecated; }); }
|
||||
function layerTitle(id){
|
||||
@@ -544,18 +573,43 @@ function buildGraph(){
|
||||
live(M.nodes).forEach(function(n){
|
||||
if(!G.layers[n.layer]) return;
|
||||
if(G.provs.length && G.provs.indexOf(n.prov) === -1) return;
|
||||
/* row = rank inside the layer, so the hierarchy reads DOWNWARD:
|
||||
layer root, then Domain, then Sub-domain, then Business object.
|
||||
Depth alone put siblings side by side, which read left to right. */
|
||||
var rank = ORDER.indexOf(n.id);
|
||||
keep[n.id] = {n:n, x:0, y:0,
|
||||
layer: M.layers.map(function(l){return l.id;}).indexOf(n.layer),
|
||||
row: Math.min(n.depth, 2)};
|
||||
row: rank > -1 ? rank : 90 + Math.min(n.depth, 5)};
|
||||
});
|
||||
var E = [];
|
||||
/* Properties deliberately left without rdfs:domain/range serve several
|
||||
classes with different targets, and their scope lives in the SHACL
|
||||
shapes instead. The graph reads domain and range, so those edges were
|
||||
simply absent -- Business Object to Sub-domain among them. The pairs are
|
||||
declared here, mirroring the shapes, rather than added to the T-Box where
|
||||
they would over-constrain the property. */
|
||||
SCOPED_PAIRS.forEach(function(sc){
|
||||
if(byId[sc[0]] && !byId[sc[0]].deprecated && keep[sc[1]] && keep[sc[2]]
|
||||
&& sc[1] !== sc[2])
|
||||
E.push({s:sc[1], t:sc[2], l:sc[3], cross:false});
|
||||
});
|
||||
var _unused = [
|
||||
["pr:belongsTo", "pr:SubDomain", "pr:DataDomain", "belongs to"],
|
||||
["pr:belongsTo", "pr:BusinessObject", "pr:SubDomain", "belongs to"],
|
||||
["pr:represents", "pr:DataObject", "pr:BusinessObject", "represents"],
|
||||
["pr:represents", "pr:DataElement", "pr:BusinessConcept", "represents"],
|
||||
["pr:monitoredBy", "pr:BusinessObject", "pr:DataSteward", "monitored by"],
|
||||
["pr:owningDomain", "pr:DefinedObject", "pr:DataDomain", "owning domain"],
|
||||
["pr:ownedBy", "pr:DefinedObject", "pr:Actor", "owned by"]
|
||||
];
|
||||
live(M.relations).forEach(function(r){
|
||||
if(keep[r.domain] && keep[r.range] && r.domain !== r.range)
|
||||
E.push({s:r.domain, t:r.range, l:r.label, cross:r.seam});
|
||||
});
|
||||
|
||||
live(M.nodes).forEach(function(n){
|
||||
if(n.parent && keep[n.id] && keep[n.parent])
|
||||
E.push({s:n.id, t:n.parent, l:"specialises", cross:false});
|
||||
E.push({s:n.id, t:n.parent, l:"type of", cross:false});
|
||||
});
|
||||
G.nodes = Object.keys(keep).map(function(k){ return keep[k]; });
|
||||
G.edges = E;
|
||||
@@ -569,19 +623,15 @@ function layoutGraph(idx){
|
||||
if(G.mode === "network"){ layoutForce(idx); return; }
|
||||
var lanes = LANES.map(function(L){ return L[0]; })
|
||||
.filter(function(c){ return G.nodes.some(function(n){ return n.n.prov === c; }); });
|
||||
/* One class per row. A lane is one column wide, so a layer reads DOWNWARDS:
|
||||
the layer class on top, then Domain, then Sub-domain, then Business
|
||||
Object. That is how a hierarchy is read; laying them left to right made
|
||||
the reader scan across for something that descends. */
|
||||
var laneW = {}, x = GUT;
|
||||
G._lanes = [];
|
||||
lanes.forEach(function(code){
|
||||
var members = G.nodes.filter(function(n){ return n.n.prov === code; });
|
||||
var widest = 1;
|
||||
M.layers.forEach(function(_, li){
|
||||
[0,1,2].forEach(function(ri){
|
||||
var c = members.filter(function(n){ return n.layer===li && n.row===ri; }).length;
|
||||
if(c) widest = Math.max(widest, Math.min(c, Math.ceil(Math.sqrt(c*1.6))));
|
||||
});
|
||||
});
|
||||
var w = widest*(NW+12) + LANEPAD*2;
|
||||
laneW[code] = {x:x, w:w, per:widest};
|
||||
var w = NW + LANEPAD*2;
|
||||
laneW[code] = {x:x, w:w};
|
||||
var L = LANES.filter(function(z){ return z[0] === code; })[0];
|
||||
G._lanes.push({code:code, x:x, w:w, label:L[1], colour:L[2]});
|
||||
x += w;
|
||||
@@ -591,30 +641,23 @@ function layoutGraph(idx){
|
||||
var y = 0;
|
||||
M.layers.forEach(function(b, li){
|
||||
b._y = y;
|
||||
var maxRows = 0;
|
||||
var rowTops = {};
|
||||
var cursor = y + 34;
|
||||
[0,1,2].forEach(function(ri){
|
||||
var any = G.nodes.filter(function(n){ return n.layer===li && n.row===ri; });
|
||||
if(!any.length){ rowTops[ri] = cursor; return; }
|
||||
rowTops[ri] = cursor;
|
||||
var deepest = 0;
|
||||
lanes.forEach(function(code){
|
||||
var g = any.filter(function(n){ return n.n.prov === code; });
|
||||
if(!g.length) return;
|
||||
var per = laneW[code].per;
|
||||
deepest = Math.max(deepest, Math.ceil(g.length/per));
|
||||
g.sort(function(p,q){ return p.n.label.localeCompare(q.n.label); });
|
||||
g.forEach(function(n, i){
|
||||
n.x = laneW[code].x + LANEPAD + (i%per)*(NW+12) + NW/2;
|
||||
n.y = cursor + Math.floor(i/per)*(NH+ROWGAP) + NH/2;
|
||||
});
|
||||
var any = G.nodes.filter(function(n){ return n.layer === li; });
|
||||
if(!any.length){ b._h = 0; return; }
|
||||
var cursor = y + 36, rows = 0;
|
||||
lanes.forEach(function(code){
|
||||
var g = any.filter(function(n){ return n.n.prov === code; });
|
||||
if(!g.length) return;
|
||||
g.sort(function(p, q){
|
||||
return ((ORDER.indexOf(p.n.id)+1)||99) - ((ORDER.indexOf(q.n.id)+1)||99)
|
||||
|| p.n.label.localeCompare(q.n.label); });
|
||||
g.forEach(function(n, i){
|
||||
n.x = laneW[code].x + laneW[code].w/2;
|
||||
n.y = cursor + i*(NH+ROWGAP) + NH/2;
|
||||
});
|
||||
cursor += deepest*(NH+ROWGAP) + ROWGAP;
|
||||
maxRows += deepest;
|
||||
rows = Math.max(rows, g.length);
|
||||
});
|
||||
b._h = maxRows ? (cursor - y + 12) : 0;
|
||||
y += b._h + (b._h ? BANDGAP : 0);
|
||||
b._h = rows*(NH+ROWGAP) + 48;
|
||||
y += b._h + BANDGAP;
|
||||
});
|
||||
G._height = y;
|
||||
}
|
||||
@@ -676,7 +719,7 @@ function renderGraph(){
|
||||
if(G.mode === "strata" && G.nodes.length){
|
||||
/* domain lanes: alternating tint, so a column reads as one domain */
|
||||
(G._lanes||[]).forEach(function(l, i){
|
||||
var isSubject = l.code === "curated";
|
||||
var isSubject = l.code === "defined";
|
||||
bL.appendChild(el("rect", {x:l.x, y:-6, width:l.w, height:G._height+10,
|
||||
fill:l.colour, opacity:isSubject ? .05 : .028}));
|
||||
if(i) bL.appendChild(el("line", {x1:l.x, y1:-6, x2:l.x, y2:G._height+4,
|
||||
@@ -733,7 +776,7 @@ function renderGraph(){
|
||||
specialisations a shape, never instantiated */
|
||||
g.appendChild(el("rect",{width:NW, height:NH, rx:3, fill:"#FFFFFF",
|
||||
stroke:c, "stroke-dasharray": n.n.abstract ? "5 3" : "none"}));
|
||||
var lines = wrapText(n.n.label, 21);
|
||||
var lines = wrapText(n.n.shortLabel || n.n.label, 21);
|
||||
lines.forEach(function(t,i){
|
||||
var e = el("text",{x:NW/2, y:NH/2+3.5+(i-(lines.length-1)/2)*10, fill:"#1A1A1A"});
|
||||
e.textContent = t; g.appendChild(e);
|
||||
@@ -807,33 +850,33 @@ function zoomG(f){
|
||||
function kpi(v,l,w){ return '<div class="kpi'+(w?' warn':'')+'"><b>'+v+'</b><span>'+l+'</span></div>'; }
|
||||
function renderScope(){
|
||||
var cls=live(M.nodes), rel=live(M.relations), att=live(M.attributes);
|
||||
var cur=cls.filter(function(n){return n.prov==="curated";});
|
||||
var obs=cls.filter(function(n){return n.prov==="observed";});
|
||||
var cur=cls.filter(function(n){return n.prov==="defined";});
|
||||
var obs=cls.filter(function(n){return n.prov==="captured";});
|
||||
var str=cls.filter(function(n){return n.prov==="structural";});
|
||||
var seam=rel.filter(function(r){return r.seam;});
|
||||
var dep=M.nodes.filter(function(n){return n.deprecated;})
|
||||
.concat(M.relations.concat(M.attributes).filter(function(r){return r.deprecated;}));
|
||||
var h=[];
|
||||
h.push('<p class="lede">The layer says what an object <b>is</b>. The provenance says <b>how it got here</b>.</p>');
|
||||
h.push('<p class="sublede">Curated objects are asserted by governance and reviewed through a pull request. '
|
||||
+ 'Observed objects are harvested from Snowflake, Tableau and Looker, rebuilt on every run and never edited '
|
||||
h.push('<p class="sublede">Defined objects are written by governance and reviewed through a pull request. '
|
||||
+ 'Captured objects are read from Snowflake, Tableau and Looker, rebuilt on every run and never edited '
|
||||
+ 'by hand. Without that split the ownership rules would demand an owner, a status and a version on every '
|
||||
+ 'one of some fifty thousand warehouse columns.</p>');
|
||||
h.push('<div class="kpis">'+kpi(cls.length,"Classes")+kpi(rel.length,"Relations")
|
||||
+kpi(att.length,"Attributes")+kpi(M.bridges.length,"Bridge properties")
|
||||
+kpi(seam.length,"Seam edges",true)+kpi(dep.length,"Deprecated",true)+'</div>');
|
||||
h.push('<h2 class="sec">Curated against observed</h2><div class="bar">'
|
||||
h.push('<h2 class="sec">Defined against captured</h2><div class="bar">'
|
||||
+'<div style="width:'+(cur.length/cls.length*100)+'%;background:#1E4B9B"></div>'
|
||||
+'<div style="width:'+(obs.length/cls.length*100)+'%;background:#8A6A28"></div>'
|
||||
+'<div style="width:'+(str.length/cls.length*100)+'%;background:#B9B4A7"></div></div>');
|
||||
h.push('<div class="legend">'
|
||||
+'<span><i style="background:#1E4B9B"></i>Curated \u2014 '+cur.length+'</span>'
|
||||
+'<span><i style="background:#8A6A28"></i>Observed \u2014 '+obs.length+'</span>'
|
||||
+'<span><i style="background:#1E4B9B"></i>Defined \u2014 '+cur.length+'</span>'
|
||||
+'<span><i style="background:#8A6A28"></i>Captured \u2014 '+obs.length+'</span>'
|
||||
+'<span><i style="background:#B9B4A7"></i>Structural \u2014 '+str.length+'</span></div>');
|
||||
h.push('<div class="note" style="max-width:880px;margin-bottom:26px"><b>Only '+seam.length
|
||||
+' relations cross between the two.</b> '+seam.map(function(r){return esc(r.label);}).join(" and ")
|
||||
+' \u2014 both proposed by the reconciliation job and confirmed by a steward. Everywhere else the '
|
||||
+'graph is either pure governance or pure machine, which is what keeps the curation cost bounded.</div>');
|
||||
+'graph is either pure governance or pure machine, which is what keeps the mapping effort bounded.</div>');
|
||||
h.push('<h2 class="sec">The layers</h2><div class="grid">');
|
||||
M.layers.forEach(function(l){
|
||||
var g=cls.filter(function(n){return n.layer===l.id;});
|
||||
@@ -861,9 +904,12 @@ function renderModel(){
|
||||
M.layers.forEach(function(l){
|
||||
var g=live(M.nodes).filter(function(n){return n.layer===l.id;});
|
||||
if(!g.length) return;
|
||||
g.sort(function(a,b){ return a.depth-b.depth || a.label.localeCompare(b.label); });
|
||||
g.sort(function(a,b){
|
||||
return a.depth - b.depth
|
||||
|| ((ORDER.indexOf(a.id)+1)||99) - ((ORDER.indexOf(b.id)+1)||99)
|
||||
|| a.label.localeCompare(b.label); });
|
||||
h.push('<h2 class="sec" style="color:'+provColour(g[0].prov)+'">'+esc(l.title)+' \u00b7 '+esc(l.tag)+'</h2>');
|
||||
h.push('<table><thead><tr><th>Class</th><th>Provenance</th><th class="hide-s">Specialises</th>'
|
||||
h.push('<table><thead><tr><th>Class</th><th>Provenance</th><th class="hide-s">Type of</th>'
|
||||
+'<th class="hide-s">What it is for</th></tr></thead><tbody>');
|
||||
g.forEach(function(n){
|
||||
h.push('<tr data-id="'+n.id+'"><td><span class="dot" style="background:'+provColour(n.prov)
|
||||
@@ -903,7 +949,7 @@ function propRows(){
|
||||
if(p.functional) t.push("exactly one");
|
||||
if(p.transitive) t.push("transitive");
|
||||
if(p.seam) t.push("seam");
|
||||
if(p.curation) t.push(p.curation.toLowerCase().replace(/_/g," "));
|
||||
if(p.authoring) t.push(p.authoring.toLowerCase().replace(/_/g," "));
|
||||
h.push('<tr data-id="'+p.id+'"><td><b>'+esc(p.label)+'</b><div class="mono">'+esc(p.id)+'</div></td>'
|
||||
+'<td class="hide-s">'+(byId[p.domain]?esc(clean(byId[p.domain].label)):'<span class="mono">any</span>')+'</td>'
|
||||
+'<td class="hide-s">'+(byId[p.range]?esc(clean(byId[p.range].label))
|
||||
@@ -939,8 +985,8 @@ function renderDep(){
|
||||
/* ------------------------------------------------------------ SHELL */
|
||||
function renderGraphShell(){
|
||||
var h=['<p class="lede">The meta model as a graph.</p>'
|
||||
+'<p class="sublede">Columns are provenance: <b style="color:#1E4B9B">curated</b> on the left, '
|
||||
+'<b style="color:#8A6A28">observed</b> on the right. Rows are layers. A <b>navy edge</b> crosses '
|
||||
+'<p class="sublede">Columns are provenance: <b style="color:#1E4B9B">defined</b> on the left, '
|
||||
+'<b style="color:#8A6A28">captured</b> on the right. Rows are layers. A <b>navy edge</b> crosses '
|
||||
+'between the two \u2014 that is the seam, the only place a steward confirms anything. Dashed outlines '
|
||||
+'are abstract classes, never instantiated.</p>'];
|
||||
h.push('<div class="gwrap"><svg id="g-svg" xmlns="http://www.w3.org/2000/svg"><defs>'
|
||||
@@ -1076,6 +1122,29 @@ M.relations.forEach(function(r){
|
||||
if(r.deprecated) return;
|
||||
if(r.domain) (FROM[r.domain]=FROM[r.domain]||[]).push(r.id);
|
||||
if(r.range) (TO[r.range]=TO[r.range]||[]).push(r.id); });
|
||||
/* The same scoped pairs the graph uses. Without them a Business Object shows
|
||||
no incoming relation at all, because belongsTo and represents carry no
|
||||
rdfs:domain -- their scope lives in the shapes. The card was silent about
|
||||
edges the graph was drawing. */
|
||||
SCOPED_PAIRS.forEach(function(sc){
|
||||
if(!byId[sc[0]] || byId[sc[0]].deprecated) return;
|
||||
(FROM[sc[1]] = FROM[sc[1]]||[]).push(sc[0]);
|
||||
(TO[sc[2]] = TO[sc[2]]||[]).push(sc[0]);
|
||||
});
|
||||
/* the scoped properties again: without this a Sub-domain card showed no
|
||||
incoming Business Object, and a Business Object card had no incoming list
|
||||
at all, because neither relation declares a domain or a range */
|
||||
[["pr:belongsTo","pr:SubDomain","pr:DataDomain"],
|
||||
["pr:belongsTo","pr:BusinessObject","pr:SubDomain"],
|
||||
["pr:represents","pr:DataObject","pr:BusinessObject"],
|
||||
["pr:represents","pr:DataElement","pr:BusinessConcept"],
|
||||
["pr:monitoredBy","pr:BusinessObject","pr:DataSteward"],
|
||||
["pr:owningDomain","pr:DefinedObject","pr:DataDomain"],
|
||||
["pr:ownedBy","pr:DefinedObject","pr:Actor"]].forEach(function(sc){
|
||||
if(!byId[sc[0]] || byId[sc[0]].deprecated) return;
|
||||
if((FROM[sc[1]]||[]).indexOf(sc[0])===-1) (FROM[sc[1]]=FROM[sc[1]]||[]).push(sc[0]);
|
||||
if((TO[sc[2]]||[]).indexOf(sc[0])===-1) (TO[sc[2]]=TO[sc[2]]||[]).push(sc[0]);
|
||||
});
|
||||
M.attributes.forEach(function(a){
|
||||
if(a.deprecated||!a.domain) return;
|
||||
(ATTRS[a.domain]=ATTRS[a.domain]||[]).push(a.id); });
|
||||
@@ -1104,7 +1173,10 @@ function paint(id){
|
||||
: n.kind==="Attribute" ? "Attribute" : "Bridge property";
|
||||
h.push('<div class="kind" style="color:'+provColour(n.prov||"structural")+'">'
|
||||
+ kindLabel + (n.layer ? ' \u00b7 ' + esc(layerTitle(n.layer)) : '') + '</div>');
|
||||
h.push('<h3>'+esc(clean(n.label))+'</h3>');
|
||||
h.push('<h3>'+esc(n.shortLabel || clean(n.label))+'</h3>');
|
||||
if(n.shortLabel && n.shortLabel !== clean(n.label))
|
||||
h.push('<div class="row" style="border:0;padding:0 0 6px"><span class="mono">'
|
||||
+ esc(clean(n.label)) + '</span><span></span></div>');
|
||||
h.push('<div class="iri">'+esc(n.id)+'</div>');
|
||||
if(n.deprecated) h.push('<div class="note"><b>Deprecated.</b> Kept so assertions written '
|
||||
+ 'against it keep classifying (LC-002)'
|
||||
@@ -1117,13 +1189,13 @@ function paint(id){
|
||||
if(n.kind==="Class"){
|
||||
at("Provenance", n.prov);
|
||||
at("Instantiated", n.abstract ? "never \u2014 abstract" : "yes");
|
||||
at("Curation mode", (n.curation||"").toLowerCase().replace(/_/g," "));
|
||||
at("Authoring mode", (n.authoring||"").toLowerCase().replace(/_/g," "));
|
||||
at("Acronym", n.acronym);
|
||||
} else {
|
||||
at("Cardinality", n.functional ? "exactly one" : "many");
|
||||
at("Transitive", n.transitive ? "yes" : "");
|
||||
at("Crosses the seam", n.seam ? "yes" : "");
|
||||
at("Curation mode", (n.curation||"").toLowerCase().replace(/_/g," "));
|
||||
at("Authoring mode", (n.authoring||"").toLowerCase().replace(/_/g," "));
|
||||
if(!byId[n.range]) at("Range", n.range);
|
||||
if(!n.domain) at("Domain", "any \u2014 scoped by SHACL per class");
|
||||
}
|
||||
@@ -1132,12 +1204,18 @@ function paint(id){
|
||||
}).join(""));
|
||||
|
||||
/* 2. what it is for */
|
||||
if(n.comment) h.push('<h4>What it is for</h4><p>'+esc(n.comment).replace(/\n/g,"<br>")+'</p>');
|
||||
h.push('<h4>What it is for</h4><p>' + (n.comment
|
||||
? esc(n.comment).replace(/\n/g,"<br>")
|
||||
: '<em style="color:var(--muted)">No definition yet \u2014 this term needs one '
|
||||
+ 'before it can be published.</em>') + '</p>');
|
||||
if(n.harvest) h.push('<h4>Harvested from</h4><div class="harv">'+esc(n.harvest)+'</div>');
|
||||
|
||||
/* 3. related terms, from the coarsest down */
|
||||
function link(label, ids){
|
||||
ids=(ids||[]).filter(function(i){return byId[i];});
|
||||
/* deprecated terms live on their own tab; showing them in a link list
|
||||
invites someone to use one */
|
||||
ids=(ids||[]).filter(function(i){
|
||||
return byId[i] && (!byId[i].deprecated || label === "Replaced by"); });
|
||||
if(!ids.length) return;
|
||||
h.push('<h4>'+label+'</h4>');
|
||||
ids.slice(0,16).forEach(function(i){
|
||||
@@ -1148,8 +1226,8 @@ function paint(id){
|
||||
if(ids.length>16) h.push('<div class="row"><span class="mono">\u2026 and '
|
||||
+(ids.length-16)+' more</span><span></span></div>');
|
||||
}
|
||||
link("Specialises", [n.parent, n.subPropertyOf]);
|
||||
link("Specialised by", CHILDREN[n.id]);
|
||||
link("Type of", [n.parent, n.subPropertyOf]);
|
||||
link("Types", CHILDREN[n.id]);
|
||||
link("Replaced by", [n.replacedBy]);
|
||||
link("From", [n.domain]);
|
||||
link("To", [n.range]);
|
||||
|
||||
@@ -0,0 +1,148 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
SODH - migration v1.2 -> v1.3
|
||||
==============================
|
||||
USAGE
|
||||
python3 scripts/migrate_instances_v1_3.py # dry run
|
||||
python3 scripts/migrate_instances_v1_3.py --apply # rewrite, .bak kept
|
||||
|
||||
1. references -> qualifiedBy. The name now says why the link exists rather
|
||||
than how it is implemented.
|
||||
|
||||
2. hasSource removed from every Business Object. Same argument as for the
|
||||
Business Concepts in v1.2, and the values showed why it was needed: across
|
||||
the 15 objects the property held physical locations ("SODH Gold /
|
||||
F_SELL_OUT"), migration notes ("MDM Product -> SODH [concepts TBC by
|
||||
DD-10]") and arbitration remarks ("DGO proposal, pending ratification"),
|
||||
all under one name. A Business Object is a governance grouping; its data
|
||||
lives in the Data Objects that represent it, and those carry the system.
|
||||
|
||||
3. Short labels where the long name is unwieldy on screen. The Data Product
|
||||
was called "Sell Out Data Hub - Gold" on its own card and "SODH Gold"
|
||||
everywhere else, because the viewers each kept a display dictionary.
|
||||
|
||||
4. Instance versions aligned on the model version. Business Objects were
|
||||
still stamped 0.6 while the model had moved to 1.3.
|
||||
"""
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import sys
|
||||
|
||||
REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
TTL = os.path.join(REPO, "instances", "sodh.ttl")
|
||||
W = 78
|
||||
|
||||
SHORT_LABELS = {
|
||||
"ex:DP_06_01_001": "SODH Gold",
|
||||
"ex:DC_06_01_001": "SODH Gold contract",
|
||||
"ex:DI_06_01_001": "Consumption schema",
|
||||
"ex:DI_06_01_002": "Panel extract",
|
||||
"ex:DO_06_01_001": "Sell Out Volume",
|
||||
"ex:DO_06_01_003": "Sell Out Distribution",
|
||||
"ex:DO_10_01_001": "Product",
|
||||
"ex:DO_04_01_001": "Customer",
|
||||
"ex:DO_16_01_001": "Currency",
|
||||
"ex:DO_21_01_001": "Calendar",
|
||||
}
|
||||
MODEL_VERSION = "1.3"
|
||||
|
||||
|
||||
def clauses(body):
|
||||
out, buf, i, n = [], [], 0, len(body)
|
||||
while i < n:
|
||||
c = body[i]
|
||||
if c == '"':
|
||||
j = i + 1
|
||||
while j < n and body[j] != '"':
|
||||
j += 2 if body[j] == "\\" else 1
|
||||
j = min(j + 1, n)
|
||||
buf.append(body[i:j]); i = j; continue
|
||||
if c == ";":
|
||||
out.append("".join(buf)); buf = []; i += 1; continue
|
||||
buf.append(c); i += 1
|
||||
out.append("".join(buf))
|
||||
return out
|
||||
|
||||
|
||||
def set_prop(text, subject, prop, value):
|
||||
pat = re.compile(r'(^%s a pr:\w+ ;)(.*?)(\.\s*\n)' % re.escape(subject), re.S | re.M)
|
||||
m = pat.search(text)
|
||||
if not m:
|
||||
return text, False
|
||||
kept = [c for c in clauses(m.group(2))
|
||||
if c.strip() and not re.match(r'\s*%s\s' % re.escape(prop), c)]
|
||||
if value is not None:
|
||||
kept.insert(0, "\n %s %s " % (prop, value))
|
||||
body = " ;".join(kept).rstrip() + "\n "
|
||||
return text[:m.start()] + m.group(1) + body + m.group(3) + text[m.end():], True
|
||||
|
||||
|
||||
def has_prop(text, subject, prop):
|
||||
m = re.search(r'^%s a pr:\w+ ;(.*?)\.\s*\n' % re.escape(subject), text, re.S | re.M)
|
||||
return bool(m) and any(re.match(r'\s*%s\s' % re.escape(prop), c)
|
||||
for c in clauses(m.group(1)))
|
||||
|
||||
|
||||
def main():
|
||||
apply_changes = "--apply" in sys.argv
|
||||
if not os.path.exists(TTL):
|
||||
print("Not found: %s" % TTL); sys.exit(2)
|
||||
text = original = open(TTL, encoding="utf-8").read()
|
||||
report = []
|
||||
|
||||
text, n = re.subn(r'\bpr:references\b', 'pr:qualifiedBy', text)
|
||||
report.append("references -> qualifiedBy: %d occurrence(s)" % n)
|
||||
|
||||
n_bo = 0
|
||||
for m in list(re.finditer(r'^(ex:BO_\w+) a pr:BusinessObject ;', text, re.M)):
|
||||
if has_prop(text, m.group(1), "pr:hasSource"):
|
||||
text, _ = set_prop(text, m.group(1), "pr:hasSource", None)
|
||||
n_bo += 1
|
||||
report.append("hasSource removed from %d business objects" % n_bo)
|
||||
|
||||
n_sl = 0
|
||||
for subj, label in SHORT_LABELS.items():
|
||||
if re.search(r'^%s a ' % re.escape(subj), text, re.M):
|
||||
text, ok = set_prop(text, subj, "pr:hasShortLabel", '"%s"' % label)
|
||||
n_sl += ok
|
||||
report.append("short label set on %d objects" % n_sl)
|
||||
|
||||
n_v = 0
|
||||
for m in list(re.finditer(r'^(ex:\w+) a pr:\w+ ;', text, re.M)):
|
||||
if has_prop(text, m.group(1), "pr:hasVersion"):
|
||||
text, ok = set_prop(text, m.group(1), "pr:hasVersion",
|
||||
'"%s"' % MODEL_VERSION)
|
||||
n_v += ok
|
||||
report.append("version aligned on %s for %d objects" % (MODEL_VERSION, n_v))
|
||||
|
||||
print()
|
||||
print("SODH MIGRATION v1.2 -> v1.3 %s" % ("APPLY" if apply_changes else "DRY RUN"))
|
||||
print("=" * W)
|
||||
for line in report:
|
||||
print(" " + line)
|
||||
print("=" * W)
|
||||
left_bo = len([1 for m in re.finditer(r'^(ex:BO_\w+) a pr:BusinessObject ;', text, re.M)
|
||||
if has_prop(text, m.group(1), "pr:hasSource")])
|
||||
doubled = []
|
||||
for m in re.finditer(r'^(ex:\w+) a pr:\w+ ;(.*?)\.\s*\n', text, re.S | re.M):
|
||||
for p in ("pr:hasName", "pr:hasShortLabel", "pr:hasVersion", "pr:hasSource"):
|
||||
if len(re.findall(r'(?<![\w:])%s\s' % re.escape(p), m.group(2))) > 1:
|
||||
doubled.append("%s/%s" % (m.group(1), p))
|
||||
print(" business objects still carrying a source: %d (must be 0)" % left_bo)
|
||||
print(" references left: %d (must be 0)" % len(re.findall(r'pr:references', text)))
|
||||
print(" duplicated single-valued properties: %s"
|
||||
% (", ".join(doubled) if doubled else "none"))
|
||||
print("=" * W)
|
||||
|
||||
if text != original and apply_changes:
|
||||
shutil.copy2(TTL, TTL + ".bak")
|
||||
open(TTL, "w", encoding="utf-8").write(text)
|
||||
print(" written, backup at %s.bak" % os.path.basename(TTL))
|
||||
elif text != original:
|
||||
print(" dry run -- re-run with --apply to write")
|
||||
print()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,176 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
SODH - migration v1.3 -> v1.4
|
||||
==============================
|
||||
USAGE
|
||||
python3 scripts/migrate_instances_v1_4.py # dry run
|
||||
python3 scripts/migrate_instances_v1_4.py --apply # rewrite, .bak kept
|
||||
|
||||
1. SYSTEMS BECOME NODES. hasSource held a string: "SODH Gold", "MDM Product",
|
||||
"Group Finance". Six systems are created and sourcedFrom points at them.
|
||||
"Which objects depend on MDM Product?" was unanswerable while MDM Product
|
||||
was text.
|
||||
|
||||
The systems are seeded here by hand, once. Everything below them --
|
||||
databases, schemas, tables, columns -- is meant to be harvested, and this
|
||||
script does not invent any of it.
|
||||
|
||||
2. Three hasSource values were not systems at all but migration notes ("DGO
|
||||
proposal, split of a compound sub-domain"). They are dropped rather than
|
||||
turned into a system: provenance of a record is not a property of the thing
|
||||
the record describes.
|
||||
|
||||
3. materializedAs -> storedIn on the Data Objects. Nothing carries it yet,
|
||||
done for completeness so the term never reappears.
|
||||
"""
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import sys
|
||||
|
||||
REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
TTL = os.path.join(REPO, "instances", "sodh.ttl")
|
||||
W = 78
|
||||
|
||||
# (iri, identifier, name, short label, system type, the string it replaces)
|
||||
SYSTEMS = [
|
||||
("ex:SYS_SODH_GOLD", "SYS-001", "Sell Out Data Hub - Gold layer", "SODH Gold",
|
||||
"pr:DataPlatform", "SODH Gold"),
|
||||
("ex:SYS_MDM_PRODUCT", "SYS-002", "Master Data Management - Product", "MDM Product",
|
||||
"pr:MasterDataSystem", "MDM Product"),
|
||||
("ex:SYS_MDM_CUSTOMER", "SYS-003", "Master Data Management - Customer", "MDM Customer",
|
||||
"pr:MasterDataSystem", "MDM Customer"),
|
||||
("ex:SYS_GROUP_FINANCE", "SYS-004", "Group Finance consolidation", "Group Finance",
|
||||
"pr:SourceApplication", "Group Finance"),
|
||||
("ex:SYS_CORPORATE_CALENDAR", "SYS-005", "Corporate standard calendar", "Standard Calendar",
|
||||
"pr:SourceApplication", "Standard Calendar"),
|
||||
]
|
||||
|
||||
# strings that were never a system: provenance notes about the record itself
|
||||
NOT_A_SYSTEM = ["DGO proposal"]
|
||||
|
||||
|
||||
def clauses(body):
|
||||
out, buf, i, n = [], [], 0, len(body)
|
||||
while i < n:
|
||||
c = body[i]
|
||||
if c == '"':
|
||||
j = i + 1
|
||||
while j < n and body[j] != '"':
|
||||
j += 2 if body[j] == "\\" else 1
|
||||
j = min(j + 1, n)
|
||||
buf.append(body[i:j]); i = j; continue
|
||||
if c == ";":
|
||||
out.append("".join(buf)); buf = []; i += 1; continue
|
||||
buf.append(c); i += 1
|
||||
out.append("".join(buf))
|
||||
return out
|
||||
|
||||
|
||||
def set_prop(text, subject, prop, value):
|
||||
pat = re.compile(r'(^%s a pr:\w+ ;)(.*?)(\.\s*\n)' % re.escape(subject), re.S | re.M)
|
||||
m = pat.search(text)
|
||||
if not m:
|
||||
return text, False
|
||||
kept = [c for c in clauses(m.group(2))
|
||||
if c.strip() and not re.match(r'\s*%s\s' % re.escape(prop), c)]
|
||||
if value is not None:
|
||||
kept.insert(0, "\n %s %s " % (prop, value))
|
||||
body = " ;".join(kept).rstrip() + "\n "
|
||||
return text[:m.start()] + m.group(1) + body + m.group(3) + text[m.end():], True
|
||||
|
||||
|
||||
def get_source(text, subject):
|
||||
m = re.search(r'^%s a pr:\w+ ;(.*?)\.\s*\n' % re.escape(subject), text, re.S | re.M)
|
||||
if not m:
|
||||
return None
|
||||
for c in clauses(m.group(1)):
|
||||
mm = re.match(r'\s*pr:hasSource\s+"([^"]*)"', c)
|
||||
if mm:
|
||||
return mm.group(1)
|
||||
return None
|
||||
|
||||
|
||||
def system_blocks(todo):
|
||||
head = ("# --- systems -----------------------------------------------------------\n"
|
||||
"# Seeded by hand, once. A system was a string until now, which is why\n"
|
||||
"# 'what depends on MDM Product' had no answer. Everything below a system --\n"
|
||||
"# databases, schemas, tables, columns -- is meant to be harvested, and\n"
|
||||
"# nothing here invents any of it.\n\n")
|
||||
out = []
|
||||
for iri, ident, name, short, stype, _ in todo:
|
||||
out.append('%s a pr:System ;\n'
|
||||
' pr:hasIdentifier "%s" ; pr:hasName "%s" ;\n'
|
||||
' pr:hasShortLabel "%s" ;\n'
|
||||
' pr:hasSystemType %s ;\n'
|
||||
' pr:hasStatus "DRAFT" ; pr:hasVersion "1.4" .'
|
||||
% (iri, ident, name, short, stype))
|
||||
return head + "\n".join(out) + "\n"
|
||||
|
||||
|
||||
def main():
|
||||
apply_changes = "--apply" in sys.argv
|
||||
if not os.path.exists(TTL):
|
||||
print("Not found: %s" % TTL); sys.exit(2)
|
||||
text = original = open(TTL, encoding="utf-8").read()
|
||||
report = []
|
||||
|
||||
todo = [sy for sy in SYSTEMS if not re.search(r'^%s a ' % re.escape(sy[0]), text, re.M)]
|
||||
if todo:
|
||||
anchor = re.search(r'^ex:DD_\w+ a pr:DataDomain', text, re.M)
|
||||
pos = anchor.start() if anchor else 0
|
||||
text = text[:pos] + system_blocks(todo) + "\n" + text[pos:]
|
||||
report.append("%d systems created%s" % (len(todo), "" if todo else " (already present)"))
|
||||
|
||||
by_string = {sy[5]: sy[0] for sy in SYSTEMS}
|
||||
n_ok = n_drop = 0
|
||||
for m in list(re.finditer(r'^(ex:\w+) a pr:\w+ ;', text, re.M)):
|
||||
subj = m.group(1)
|
||||
src = get_source(text, subj)
|
||||
if src is None:
|
||||
continue
|
||||
if src in by_string:
|
||||
text, _ = set_prop(text, subj, "pr:hasSource", None)
|
||||
text, _ = set_prop(text, subj, "pr:sourcedFrom", by_string[src])
|
||||
n_ok += 1
|
||||
elif any(k in src for k in NOT_A_SYSTEM):
|
||||
text, _ = set_prop(text, subj, "pr:hasSource", None)
|
||||
n_drop += 1
|
||||
report.append("sourcedFrom set on %d objects" % n_ok)
|
||||
report.append("%d provenance notes dropped rather than turned into a system" % n_drop)
|
||||
|
||||
text, n = re.subn(r'\bpr:materializedAs\b', 'pr:storedIn', text)
|
||||
report.append("materializedAs -> storedIn: %d occurrence(s)" % n)
|
||||
|
||||
print()
|
||||
print("SODH MIGRATION v1.3 -> v1.4 %s" % ("APPLY" if apply_changes else "DRY RUN"))
|
||||
print("=" * W)
|
||||
for line in report:
|
||||
print(" " + line)
|
||||
print("=" * W)
|
||||
left = re.findall(r'pr:hasSource\s+"([^"]*)"', text)
|
||||
defined = set(re.findall(r'^(ex:\w+) a ', text, re.M))
|
||||
dangling = sorted(set(re.findall(r'\bex:\w+', text)) - defined)
|
||||
doubled = []
|
||||
for m in re.finditer(r'^(ex:\w+) a pr:\w+ ;(.*?)\.\s*\n', text, re.S | re.M):
|
||||
for p in ("pr:sourcedFrom", "pr:hasName", "pr:hasShortLabel"):
|
||||
if len(re.findall(r'(?<![\w:])%s\s' % re.escape(p), m.group(2))) > 1:
|
||||
doubled.append("%s/%s" % (m.group(1), p))
|
||||
print(" hasSource left: %d %s" % (len(left), sorted(set(left)) if left else "(must be 0)"))
|
||||
print(" sourcedFrom references: %d" % len(re.findall(r'pr:sourcedFrom', text)))
|
||||
print(" dangling references: %s" % (", ".join(dangling) if dangling else "none"))
|
||||
print(" duplicated single-valued properties: %s"
|
||||
% (", ".join(doubled) if doubled else "none"))
|
||||
print("=" * W)
|
||||
|
||||
if text != original and apply_changes:
|
||||
shutil.copy2(TTL, TTL + ".bak")
|
||||
open(TTL, "w", encoding="utf-8").write(text)
|
||||
print(" written, backup at %s.bak" % os.path.basename(TTL))
|
||||
elif text != original:
|
||||
print(" dry run -- re-run with --apply to write")
|
||||
print()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user