From 08c99e144824c82618187d42cc31acd0546253d7 Mon Sep 17 00:00:00 2001 From: Eddie Soehnel Date: Tue, 24 Mar 2026 16:01:04 -0600 Subject: [PATCH] system ontology graph added a lot to it based on learnings today with creating for LOD and how this affects how I code and build systems --- specs/3_product_development_s/.gitkeep.md | 1 - .../system-ontology-framework/app.js | 286 ++++++ .../graph_edges_nodes.json | 942 ++++++++++++++++++ .../system-ontology-framework/index.html | 170 ++++ .../system-ontology-framework.md | 172 ++++ .../system-ontology-validators.csv | 28 + .../system_ontology - edges.csv | 1 + .../system_ontology - nodes.csv | 1 + 8 files changed, 1600 insertions(+), 1 deletion(-) delete mode 100644 specs/3_product_development_s/.gitkeep.md create mode 100644 specs/3_product_development_s/system-ontology-framework/app.js create mode 100644 specs/3_product_development_s/system-ontology-framework/graph_edges_nodes.json create mode 100644 specs/3_product_development_s/system-ontology-framework/index.html create mode 100644 specs/3_product_development_s/system-ontology-framework/system-ontology-framework.md create mode 100644 specs/3_product_development_s/system-ontology-framework/system-ontology-validators.csv create mode 100644 specs/3_product_development_s/system-ontology-framework/system_ontology - edges.csv create mode 100644 specs/3_product_development_s/system-ontology-framework/system_ontology - nodes.csv diff --git a/specs/3_product_development_s/.gitkeep.md b/specs/3_product_development_s/.gitkeep.md deleted file mode 100644 index c693f13..0000000 --- a/specs/3_product_development_s/.gitkeep.md +++ /dev/null @@ -1 +0,0 @@ -keep \ No newline at end of file diff --git a/specs/3_product_development_s/system-ontology-framework/app.js b/specs/3_product_development_s/system-ontology-framework/app.js new file mode 100644 index 0000000..c461f67 --- /dev/null +++ b/specs/3_product_development_s/system-ontology-framework/app.js @@ -0,0 +1,286 @@ +console.log('APP.JS LOADED'); + +const GRAPH_URL = 'lod_graph_cytoscape.json'; + +function cleanValue(value) { + if (value == null) { + return null; + } + + if (typeof value === 'string') { + const trimmed = value.trim(); + if (!trimmed || trimmed.toLowerCase() === 'nan' || trimmed.toLowerCase() === 'undefined') { + return null; + } + return trimmed; + } + + if (typeof value === 'number' && Number.isNaN(value)) { + return null; + } + + return value; +} + +function normalizeId(value) { + const cleaned = cleanValue(value); + return cleaned == null ? null : String(cleaned).trim(); +} + +function pickNodeId(data, index) { + const rawId = normalizeId(data['node-id']) + || normalizeId(data.label) + || normalizeId(data.name) + || normalizeId(data.id); + + if (!rawId || rawId.startsWith('{')) { + return 'node-' + (index + 1); + } + + return rawId; +} + +function pickLabel(data, fallbackId) { + return normalizeId(data.label) + || normalizeId(data['node-id']) + || normalizeId(data.name) + || normalizeId(data['sub-module']) + || fallbackId; +} + +function normalizePayload(payload) { + if (Array.isArray(payload)) { + return payload; + } + + if (Array.isArray(payload && payload.elements)) { + return payload.elements; + } + + const nodes = Array.isArray(payload && payload.nodes) ? payload.nodes : []; + const edges = Array.isArray(payload && payload.edges) ? payload.edges : []; + return nodes.concat(edges); +} + +function buildGraph(rawItems) { + const nodes = []; + const edges = []; + const nodeIds = new Set(); + + rawItems.forEach(function (item, index) { + const data = (item && item.data) || item || {}; + const source = normalizeId(data.source || data.from); + const target = normalizeId(data.target || data.to); + + if (source && target) { + edges.push({ + data: { + id: normalizeId(data.id) || 'edge-' + (edges.length + 1), + source: source, + target: target, + label: normalizeId(data.label) || '', + 'edge-type': normalizeId(data['edge-type']) || normalizeId(data.label) || '', + entity: normalizeId(data.entity), + execution: normalizeId(data.execution), + schedule: normalizeId(data.schedule), + criticality: normalizeId(data.criticality) + } + }); + return; + } + + const id = pickNodeId(data, index); + if (!id || nodeIds.has(id)) { + return; + } + + nodeIds.add(id); + nodes.push({ + data: { + id: id, + label: pickLabel(data, id), + type: normalizeId(data.type), + module: normalizeId(data.module), + submodule: normalizeId(data['sub-module']), + codebase: normalizeId(data['code-base']), + environment: normalizeId(data['execution-environment']) + } + }); + }); + + const filteredEdges = edges.filter(function (edge) { + return nodeIds.has(edge.data.source) && nodeIds.has(edge.data.target); + }); + + return { nodes: nodes, edges: filteredEdges }; +} + +function setStatus(message, isError) { + const cyContainer = document.getElementById('cy'); + cyContainer.innerHTML = ''; + cyContainer.style.display = 'grid'; + cyContainer.style.placeItems = 'center'; + cyContainer.style.color = isError ? '#9f1239' : '#1f2937'; + cyContainer.style.font = '600 16px/1.4 system-ui, sans-serif'; + cyContainer.textContent = message; +} + +function colorForType(type) { + const normalized = (type || '').toLowerCase(); + + if (normalized === 'ui') { + return '#2563eb'; + } + + if (normalized === 'script') { + return '#0f766e'; + } + + if (normalized === 'datastore') { + return '#a16207'; + } + + return '#475569'; +} + +async function loadGraph() { + try { + const res = await fetch(GRAPH_URL, { cache: 'no-store' }); + if (!res.ok) { + throw new Error('Fetch failed: ' + res.status + ' ' + res.statusText); + } + + const rawText = await res.text(); + const sanitizedText = rawText.replace(/\bNaN\b/g, 'null'); + const payload = JSON.parse(sanitizedText); + const rawItems = normalizePayload(payload); + const graph = buildGraph(rawItems); + + console.log('GRAPH COUNTS:', { + rawItems: rawItems.length, + nodes: graph.nodes.length, + edges: graph.edges.length + }); + + if (!graph.nodes.length) { + setStatus('No valid nodes found in lod_graph_cytoscape.json', true); + return; + } + + document.getElementById('cy').style.display = 'block'; + + graph.nodes.forEach(function (node) { + node.data.color = colorForType(node.data.type); + }); + + const cy = cytoscape({ + container: document.getElementById('cy'), + elements: graph.nodes.concat(graph.edges), + style: [ + { + selector: 'node', + style: { + label: 'data(label)', + 'background-color': 'data(color)', + color: '#ffffff', + 'text-wrap': 'wrap', + 'text-max-width': 120, + 'text-valign': 'center', + 'text-halign': 'center', + 'font-size': 12, + 'font-weight': 700, + 'text-outline-width': 2, + 'text-outline-color': '#0f172a', + 'text-outline-opacity': 0.18, + width: 72, + height: 72, + 'border-width': 2, + 'border-color': '#0f172a', + 'border-opacity': 0.28, + 'overlay-opacity': 0, + 'shadow-blur': 18, + 'shadow-color': '#0f172a', + 'shadow-opacity': 0.12, + 'shadow-offset-x': 0, + 'shadow-offset-y': 6 + } + }, + { + selector: 'node[type = "UI"]', + style: { + shape: 'round-rectangle' + } + }, + { + selector: 'node[type = "Datastore"]', + style: { + shape: 'diamond' + } + }, + { + selector: 'node[type = "Script"]', + style: { + shape: 'ellipse' + } + }, + { + selector: 'edge', + style: { + width: 3, + label: 'data(label)', + color: '#334155', + 'font-size': 10, + 'font-weight': 600, + 'text-background-color': '#ffffff', + 'text-background-opacity': 0.9, + 'text-background-padding': 3, + 'text-rotation': 'autorotate', + 'curve-style': 'bezier', + 'line-color': '#64748b', + 'target-arrow-color': '#64748b', + 'target-arrow-shape': 'triangle', + 'arrow-scale': 1.1, + 'overlay-opacity': 0 + } + }, + { + selector: 'edge[edge-type = "writes"]', + style: { + 'line-style': 'dashed' + } + }, + { + selector: 'edge[edge-type = "returns"]', + style: { + 'line-color': '#7c3aed', + 'target-arrow-color': '#7c3aed' + } + }, + { + selector: 'edge[edge-type = "navigates"]', + style: { + 'line-color': '#db2777', + 'target-arrow-color': '#db2777' + } + } + ], + layout: { + name: 'breadthfirst', + directed: true, + spacingFactor: 1.15, + padding: 60, + fit: true, + animate: false + } + }); + + cy.ready(function () { + console.log('GRAPH READY:', cy.elements().length); + }); + } catch (error) { + console.error('GRAPH LOAD FAILED:', error); + setStatus('Graph load failed: ' + error.message, true); + } +} + +loadGraph(); diff --git a/specs/3_product_development_s/system-ontology-framework/graph_edges_nodes.json b/specs/3_product_development_s/system-ontology-framework/graph_edges_nodes.json new file mode 100644 index 0000000..47a0bac --- /dev/null +++ b/specs/3_product_development_s/system-ontology-framework/graph_edges_nodes.json @@ -0,0 +1,942 @@ +{ + "nodes": [ + { + "module": "customer-db", + "sub-module": "login", + "active": "prod", + "node-id": "file:login.js", + "location": "srv/apps/lod-api/ui/static/", + "code-base": "js", + "type": "Script", + "retry-policy": "no", + "indempotent": "yes", + "logging": "yes", + "metrics": NaN, + "alerts": NaN, + "execution-environment": "browser/client", + "vm-environment": "lod-dev-app, lod-prod-app", + "owner": "system", + "external-connections": NaN, + "system-version": "1.0.0", + "component-version": "1.0.0", + "script-version": "1.0.0" + }, + { + "module": "customer-db", + "sub-module": "login", + "active": "prod", + "node-id": "file:login.html", + "location": "srv/apps/lod-api/ui/static/", + "code-base": "html", + "type": "UI", + "retry-policy": "no", + "indempotent": "yes", + "logging": "yes", + "metrics": NaN, + "alerts": NaN, + "execution-environment": "browser/client", + "vm-environment": "lod-dev-app, lod-prod-app", + "owner": "system", + "external-connections": NaN, + "system-version": "1.0.0", + "component-version": "1.0.0", + "script-version": "1.0.0" + }, + { + "module": "customer-db", + "sub-module": "login", + "active": "prod", + "node-id": "login API", + "location": "srv/apps/lod-api/ui/static/", + "code-base": "py", + "type": "Script", + "retry-policy": "no", + "indempotent": "yes", + "logging": "yes", + "metrics": NaN, + "alerts": NaN, + "execution-environment": "browser/client", + "vm-environment": "lod-dev-app, lod-prod-app", + "owner": "system", + "external-connections": NaN, + "system-version": "1.0.0", + "component-version": "1.0.0", + "script-version": "1.0.0" + }, + { + "module": "customer-db", + "sub-module": "login", + "active": "prod", + "node-id": "auth system", + "location": "srv/apps/lod-api/ui/static/", + "code-base": "py", + "type": "Script", + "retry-policy": "no", + "indempotent": "yes", + "logging": "yes", + "metrics": NaN, + "alerts": NaN, + "execution-environment": "backend/vm", + "vm-environment": "lod-dev-app, lod-prod-app", + "owner": "system", + "external-connections": NaN, + "system-version": "1.0.0", + "component-version": "1.0.0", + "script-version": "1.0.0" + }, + { + "module": "customer-db", + "sub-module": "login", + "active": "prod", + "node-id": "environment config", + "location": "srv/apps/lod-api/ui/static/", + "code-base": ".env", + "type": "Datastore", + "retry-policy": "no", + "indempotent": "yes", + "logging": "yes", + "metrics": NaN, + "alerts": NaN, + "execution-environment": "infrastructure", + "vm-environment": "lod-dev-app, lod-prod-app", + "owner": "system", + "external-connections": NaN, + "system-version": "1.0.0", + "component-version": "1.0.0", + "script-version": "1.0.0" + }, + { + "module": "customer-db", + "sub-module": "login", + "active": "prod", + "node-id": "localStorage ", + "location": "srv/apps/lod-api/ui/static/", + "code-base": "browser DOM (in memory)", + "type": "Datastore", + "retry-policy": "no", + "indempotent": "yes", + "logging": "yes", + "metrics": NaN, + "alerts": NaN, + "execution-environment": "browser/client", + "vm-environment": "lod-dev-app, lod-prod-app", + "owner": "system", + "external-connections": NaN, + "system-version": "1.0.0", + "component-version": "1.0.0", + "script-version": "1.0.0" + }, + { + "module": "customer-db", + "sub-module": "login", + "active": "prod", + "node-id": "file:dashboard.html", + "location": "srv/apps/lod-api/ui/static/", + "code-base": "html", + "type": "UI", + "retry-policy": "no", + "indempotent": "yes", + "logging": "yes", + "metrics": NaN, + "alerts": NaN, + "execution-environment": "browser/client", + "vm-environment": "lod-dev-app, lod-prod-app", + "owner": "system", + "external-connections": NaN, + "system-version": "1.0.0", + "component-version": "1.0.0", + "script-version": "1.0.0" + }, + { + "module": "customer-db", + "sub-module": "login", + "active": "prod", + "node-id": "DOM (#msg)", + "location": "srv/apps/lod-api/ui/static/", + "code-base": "browser DOM (in memory)", + "type": "Datastore", + "retry-policy": "no", + "indempotent": "yes", + "logging": "yes", + "metrics": NaN, + "alerts": NaN, + "execution-environment": "browser/client", + "vm-environment": "lod-dev-app, lod-prod-app", + "owner": "system", + "external-connections": NaN, + "system-version": "1.0.0", + "component-version": "1.0.0", + "script-version": "1.0.0" + }, + { + "module": "customer-db", + "sub-module": "login", + "active": "prod", + "node-id": NaN, + "location": NaN, + "code-base": NaN, + "type": NaN, + "retry-policy": NaN, + "indempotent": NaN, + "logging": NaN, + "metrics": NaN, + "alerts": NaN, + "execution-environment": NaN, + "vm-environment": NaN, + "owner": NaN, + "external-connections": NaN, + "system-version": NaN, + "component-version": NaN, + "script-version": NaN + }, + { + "module": "customer-db", + "sub-module": "login", + "active": "prod", + "node-id": NaN, + "location": NaN, + "code-base": NaN, + "type": NaN, + "retry-policy": NaN, + "indempotent": NaN, + "logging": NaN, + "metrics": NaN, + "alerts": NaN, + "execution-environment": NaN, + "vm-environment": NaN, + "owner": NaN, + "external-connections": NaN, + "system-version": NaN, + "component-version": NaN, + "script-version": NaN + }, + { + "module": "customer-db", + "sub-module": "login", + "active": "prod", + "node-id": NaN, + "location": NaN, + "code-base": NaN, + "type": NaN, + "retry-policy": NaN, + "indempotent": NaN, + "logging": NaN, + "metrics": NaN, + "alerts": NaN, + "execution-environment": NaN, + "vm-environment": NaN, + "owner": NaN, + "external-connections": NaN, + "system-version": NaN, + "component-version": NaN, + "script-version": NaN + }, + { + "module": "customer-db", + "sub-module": "login", + "active": "prod", + "node-id": NaN, + "location": NaN, + "code-base": NaN, + "type": NaN, + "retry-policy": NaN, + "indempotent": NaN, + "logging": NaN, + "metrics": NaN, + "alerts": NaN, + "execution-environment": NaN, + "vm-environment": NaN, + "owner": NaN, + "external-connections": NaN, + "system-version": NaN, + "component-version": NaN, + "script-version": NaN + }, + { + "module": "customer-db", + "sub-module": "login", + "active": "prod", + "node-id": NaN, + "location": NaN, + "code-base": NaN, + "type": NaN, + "retry-policy": NaN, + "indempotent": NaN, + "logging": NaN, + "metrics": NaN, + "alerts": NaN, + "execution-environment": NaN, + "vm-environment": NaN, + "owner": NaN, + "external-connections": NaN, + "system-version": NaN, + "component-version": NaN, + "script-version": NaN + }, + { + "module": "customer-db", + "sub-module": "login", + "active": "prod", + "node-id": NaN, + "location": NaN, + "code-base": NaN, + "type": NaN, + "retry-policy": NaN, + "indempotent": NaN, + "logging": NaN, + "metrics": NaN, + "alerts": NaN, + "execution-environment": NaN, + "vm-environment": NaN, + "owner": NaN, + "external-connections": NaN, + "system-version": NaN, + "component-version": NaN, + "script-version": NaN + }, + { + "module": "customer-db", + "sub-module": "login", + "active": "prod", + "node-id": NaN, + "location": NaN, + "code-base": NaN, + "type": NaN, + "retry-policy": NaN, + "indempotent": NaN, + "logging": NaN, + "metrics": NaN, + "alerts": NaN, + "execution-environment": NaN, + "vm-environment": NaN, + "owner": NaN, + "external-connections": NaN, + "system-version": NaN, + "component-version": NaN, + "script-version": NaN + }, + { + "module": "customer-db", + "sub-module": "login", + "active": "prod", + "node-id": NaN, + "location": NaN, + "code-base": NaN, + "type": NaN, + "retry-policy": NaN, + "indempotent": NaN, + "logging": NaN, + "metrics": NaN, + "alerts": NaN, + "execution-environment": NaN, + "vm-environment": NaN, + "owner": NaN, + "external-connections": NaN, + "system-version": NaN, + "component-version": NaN, + "script-version": NaN + }, + { + "module": "customer-db", + "sub-module": "login", + "active": "prod", + "node-id": NaN, + "location": NaN, + "code-base": NaN, + "type": NaN, + "retry-policy": NaN, + "indempotent": NaN, + "logging": NaN, + "metrics": NaN, + "alerts": NaN, + "execution-environment": NaN, + "vm-environment": NaN, + "owner": NaN, + "external-connections": NaN, + "system-version": NaN, + "component-version": NaN, + "script-version": NaN + }, + { + "module": "customer-db", + "sub-module": "login", + "active": "prod", + "node-id": NaN, + "location": NaN, + "code-base": NaN, + "type": NaN, + "retry-policy": NaN, + "indempotent": NaN, + "logging": NaN, + "metrics": NaN, + "alerts": NaN, + "execution-environment": NaN, + "vm-environment": NaN, + "owner": NaN, + "external-connections": NaN, + "system-version": NaN, + "component-version": NaN, + "script-version": NaN + }, + { + "module": "customer-db", + "sub-module": "login", + "active": "prod", + "node-id": NaN, + "location": NaN, + "code-base": NaN, + "type": NaN, + "retry-policy": NaN, + "indempotent": NaN, + "logging": NaN, + "metrics": NaN, + "alerts": NaN, + "execution-environment": NaN, + "vm-environment": NaN, + "owner": NaN, + "external-connections": NaN, + "system-version": NaN, + "component-version": NaN, + "script-version": NaN + }, + { + "module": "customer-db", + "sub-module": "login", + "active": "prod", + "node-id": NaN, + "location": NaN, + "code-base": NaN, + "type": NaN, + "retry-policy": NaN, + "indempotent": NaN, + "logging": NaN, + "metrics": NaN, + "alerts": NaN, + "execution-environment": NaN, + "vm-environment": NaN, + "owner": NaN, + "external-connections": NaN, + "system-version": NaN, + "component-version": NaN, + "script-version": NaN + }, + { + "module": "customer-db", + "sub-module": "login", + "active": "prod", + "node-id": NaN, + "location": NaN, + "code-base": NaN, + "type": NaN, + "retry-policy": NaN, + "indempotent": NaN, + "logging": NaN, + "metrics": NaN, + "alerts": NaN, + "execution-environment": NaN, + "vm-environment": NaN, + "owner": NaN, + "external-connections": NaN, + "system-version": NaN, + "component-version": NaN, + "script-version": NaN + }, + { + "module": "customer-db", + "sub-module": "login", + "active": "prod", + "node-id": NaN, + "location": NaN, + "code-base": NaN, + "type": NaN, + "retry-policy": NaN, + "indempotent": NaN, + "logging": NaN, + "metrics": NaN, + "alerts": NaN, + "execution-environment": NaN, + "vm-environment": NaN, + "owner": NaN, + "external-connections": NaN, + "system-version": NaN, + "component-version": NaN, + "script-version": NaN + }, + { + "module": "customer-db", + "sub-module": "login", + "active": "prod", + "node-id": NaN, + "location": NaN, + "code-base": NaN, + "type": NaN, + "retry-policy": NaN, + "indempotent": NaN, + "logging": NaN, + "metrics": NaN, + "alerts": NaN, + "execution-environment": NaN, + "vm-environment": NaN, + "owner": NaN, + "external-connections": NaN, + "system-version": NaN, + "component-version": NaN, + "script-version": NaN + }, + { + "module": "customer-db", + "sub-module": "login", + "active": "prod", + "node-id": NaN, + "location": NaN, + "code-base": NaN, + "type": NaN, + "retry-policy": NaN, + "indempotent": NaN, + "logging": NaN, + "metrics": NaN, + "alerts": NaN, + "execution-environment": NaN, + "vm-environment": NaN, + "owner": NaN, + "external-connections": NaN, + "system-version": NaN, + "component-version": NaN, + "script-version": NaN + }, + { + "module": "customer-db", + "sub-module": "login", + "active": "prod", + "node-id": NaN, + "location": NaN, + "code-base": NaN, + "type": NaN, + "retry-policy": NaN, + "indempotent": NaN, + "logging": NaN, + "metrics": NaN, + "alerts": NaN, + "execution-environment": NaN, + "vm-environment": NaN, + "owner": NaN, + "external-connections": NaN, + "system-version": NaN, + "component-version": NaN, + "script-version": NaN + }, + { + "module": "customer-db", + "sub-module": "login", + "active": "prod", + "node-id": NaN, + "location": NaN, + "code-base": NaN, + "type": NaN, + "retry-policy": NaN, + "indempotent": NaN, + "logging": NaN, + "metrics": NaN, + "alerts": NaN, + "execution-environment": NaN, + "vm-environment": NaN, + "owner": NaN, + "external-connections": NaN, + "system-version": NaN, + "component-version": NaN, + "script-version": NaN + } + ], + "edges": [ + { + "module": "customer-db", + "active": "prod", + "edge-id": "login.html-login.js-calls", + "from-node": "login.html", + "to-node": "login.js", + "edge-type": "calls", + "entity": "user", + "execution": "user", + "schedule": "on submit", + "state-transition": "submitted", + "dependencies": "login.html", + "criticality": "high", + "execution-environment": "browser/client" + }, + { + "module": "customer-db", + "active": "prod", + "edge-id": "login.js-login.js-validates", + "from-node": "login.js", + "to-node": "login.js", + "edge-type": "validates", + "entity": "user", + "execution": "event", + "schedule": "on submit", + "state-transition": "validated", + "dependencies": "login.html", + "criticality": "high", + "execution-environment": "browser/client" + }, + { + "module": "customer-db", + "active": "prod", + "edge-id": "login.js-login API (/login) (API)-calls", + "from-node": "login.js", + "to-node": "login API (/login) (API)", + "edge-type": "calls", + "entity": "credentials", + "execution": "user", + "schedule": "on submit", + "state-transition": "validated", + "dependencies": "login.html", + "criticality": "high", + "execution-environment": "backend/vm" + }, + { + "module": "customer-db", + "active": "prod", + "edge-id": "login API (/login) (API)-environment config (DataStore / External)-reads", + "from-node": "login API (/login) (API)", + "to-node": "environment config (DataStore / External)", + "edge-type": "reads", + "entity": "credentials", + "execution": "user", + "schedule": "on submit", + "state-transition": "validated", + "dependencies": "login.js", + "criticality": "high", + "execution-environment": "backend/vm" + }, + { + "module": "customer-db", + "active": "prod", + "edge-id": "login API (/login) (API)-auth system (Script / Backend logic)-validates", + "from-node": "login API (/login) (API)", + "to-node": "auth system (Script / Backend logic)", + "edge-type": "validates", + "entity": "credentials", + "execution": "user", + "schedule": "on submit", + "state-transition": "authenticated", + "dependencies": "login.js", + "criticality": "high", + "execution-environment": "infrastructure" + }, + { + "module": "customer-db", + "active": "prod", + "edge-id": "login API (/login) (API)-login.js-returns", + "from-node": "login API (/login) (API)", + "to-node": "login.js", + "edge-type": "returns", + "entity": "token", + "execution": "user", + "schedule": "on submit", + "state-transition": "authenticated", + "dependencies": "login.js", + "criticality": "high", + "execution-environment": "backend/vm" + }, + { + "module": "customer-db", + "active": "prod", + "edge-id": "login.js-localStorage (DataStore)-writes", + "from-node": "login.js", + "to-node": "localStorage (DataStore)", + "edge-type": "writes", + "entity": "token", + "execution": "user", + "schedule": "on submit", + "state-transition": "authenticated", + "dependencies": "login.js", + "criticality": "high", + "execution-environment": "browser/client" + }, + { + "module": "customer-db", + "active": "prod", + "edge-id": "login.js-dashboard.html (UI)-navigates", + "from-node": "login.js", + "to-node": "dashboard.html (UI)", + "edge-type": "navigates", + "entity": "token, session", + "execution": "user", + "schedule": "on submit", + "state-transition": "authenticated", + "dependencies": "login.js", + "criticality": "high", + "execution-environment": "browser/client" + }, + { + "module": "customer-db", + "active": "prod", + "edge-id": "login.js-DOM (#msg) (UI component, optional)-writes", + "from-node": "login.js", + "to-node": "DOM (#msg) (UI component, optional)", + "edge-type": "writes", + "entity": "error message", + "execution": "user", + "schedule": "on submit", + "state-transition": "authenticated", + "dependencies": "login.js", + "criticality": "high", + "execution-environment": "browser/client" + }, + { + "module": "customer-db", + "active": "prod", + "edge-id": NaN, + "from-node": NaN, + "to-node": NaN, + "edge-type": NaN, + "entity": NaN, + "execution": NaN, + "schedule": NaN, + "state-transition": NaN, + "dependencies": NaN, + "criticality": NaN, + "execution-environment": NaN + }, + { + "module": "customer-db", + "active": "prod", + "edge-id": NaN, + "from-node": NaN, + "to-node": NaN, + "edge-type": NaN, + "entity": NaN, + "execution": NaN, + "schedule": NaN, + "state-transition": NaN, + "dependencies": NaN, + "criticality": NaN, + "execution-environment": NaN + }, + { + "module": "customer-db", + "active": "prod", + "edge-id": NaN, + "from-node": NaN, + "to-node": NaN, + "edge-type": NaN, + "entity": NaN, + "execution": NaN, + "schedule": NaN, + "state-transition": NaN, + "dependencies": NaN, + "criticality": NaN, + "execution-environment": NaN + }, + { + "module": "customer-db", + "active": "prod", + "edge-id": NaN, + "from-node": NaN, + "to-node": NaN, + "edge-type": NaN, + "entity": NaN, + "execution": NaN, + "schedule": NaN, + "state-transition": NaN, + "dependencies": NaN, + "criticality": NaN, + "execution-environment": NaN + }, + { + "module": "customer-db", + "active": "prod", + "edge-id": NaN, + "from-node": NaN, + "to-node": NaN, + "edge-type": NaN, + "entity": NaN, + "execution": NaN, + "schedule": NaN, + "state-transition": NaN, + "dependencies": NaN, + "criticality": NaN, + "execution-environment": NaN + }, + { + "module": "customer-db", + "active": "prod", + "edge-id": NaN, + "from-node": NaN, + "to-node": NaN, + "edge-type": NaN, + "entity": NaN, + "execution": NaN, + "schedule": NaN, + "state-transition": NaN, + "dependencies": NaN, + "criticality": NaN, + "execution-environment": NaN + }, + { + "module": "customer-db", + "active": "prod", + "edge-id": NaN, + "from-node": NaN, + "to-node": NaN, + "edge-type": NaN, + "entity": NaN, + "execution": NaN, + "schedule": NaN, + "state-transition": NaN, + "dependencies": NaN, + "criticality": NaN, + "execution-environment": NaN + }, + { + "module": "customer-db", + "active": "prod", + "edge-id": NaN, + "from-node": NaN, + "to-node": NaN, + "edge-type": NaN, + "entity": NaN, + "execution": NaN, + "schedule": NaN, + "state-transition": NaN, + "dependencies": NaN, + "criticality": NaN, + "execution-environment": NaN + }, + { + "module": "customer-db", + "active": "prod", + "edge-id": NaN, + "from-node": NaN, + "to-node": NaN, + "edge-type": NaN, + "entity": NaN, + "execution": NaN, + "schedule": NaN, + "state-transition": NaN, + "dependencies": NaN, + "criticality": NaN, + "execution-environment": NaN + }, + { + "module": "customer-db", + "active": "prod", + "edge-id": NaN, + "from-node": NaN, + "to-node": NaN, + "edge-type": NaN, + "entity": NaN, + "execution": NaN, + "schedule": NaN, + "state-transition": NaN, + "dependencies": NaN, + "criticality": NaN, + "execution-environment": NaN + }, + { + "module": "customer-db", + "active": "prod", + "edge-id": NaN, + "from-node": NaN, + "to-node": NaN, + "edge-type": NaN, + "entity": NaN, + "execution": NaN, + "schedule": NaN, + "state-transition": NaN, + "dependencies": NaN, + "criticality": NaN, + "execution-environment": NaN + }, + { + "module": "customer-db", + "active": "prod", + "edge-id": NaN, + "from-node": NaN, + "to-node": NaN, + "edge-type": NaN, + "entity": NaN, + "execution": NaN, + "schedule": NaN, + "state-transition": NaN, + "dependencies": NaN, + "criticality": NaN, + "execution-environment": NaN + }, + { + "module": "customer-db", + "active": "prod", + "edge-id": NaN, + "from-node": NaN, + "to-node": NaN, + "edge-type": NaN, + "entity": NaN, + "execution": NaN, + "schedule": NaN, + "state-transition": NaN, + "dependencies": NaN, + "criticality": NaN, + "execution-environment": NaN + }, + { + "module": "customer-db", + "active": "prod", + "edge-id": NaN, + "from-node": NaN, + "to-node": NaN, + "edge-type": NaN, + "entity": NaN, + "execution": NaN, + "schedule": NaN, + "state-transition": NaN, + "dependencies": NaN, + "criticality": NaN, + "execution-environment": NaN + }, + { + "module": "customer-db", + "active": "prod", + "edge-id": NaN, + "from-node": NaN, + "to-node": NaN, + "edge-type": NaN, + "entity": NaN, + "execution": NaN, + "schedule": NaN, + "state-transition": NaN, + "dependencies": NaN, + "criticality": NaN, + "execution-environment": NaN + }, + { + "module": "customer-db", + "active": "prod", + "edge-id": NaN, + "from-node": NaN, + "to-node": NaN, + "edge-type": NaN, + "entity": NaN, + "execution": NaN, + "schedule": NaN, + "state-transition": NaN, + "dependencies": NaN, + "criticality": NaN, + "execution-environment": NaN + }, + { + "module": "customer-db", + "active": "prod", + "edge-id": NaN, + "from-node": NaN, + "to-node": NaN, + "edge-type": NaN, + "entity": NaN, + "execution": NaN, + "schedule": NaN, + "state-transition": NaN, + "dependencies": NaN, + "criticality": NaN, + "execution-environment": NaN + } + ] +} \ No newline at end of file diff --git a/specs/3_product_development_s/system-ontology-framework/index.html b/specs/3_product_development_s/system-ontology-framework/index.html new file mode 100644 index 0000000..6f5cfac --- /dev/null +++ b/specs/3_product_development_s/system-ontology-framework/index.html @@ -0,0 +1,170 @@ + + + + + + Login Graph + + + + + + +
+
+

System Graph

+

Login Flow Map

+

Cytoscape view of the login-side app graph, showing browser UI, scripts, datastores, and the relationships between them.

+
+ UI node + Script node + Datastore node + Return edge + Navigation edge +
+
+ +
+
+
+
+ + + + diff --git a/specs/3_product_development_s/system-ontology-framework/system-ontology-framework.md b/specs/3_product_development_s/system-ontology-framework/system-ontology-framework.md new file mode 100644 index 0000000..639005f --- /dev/null +++ b/specs/3_product_development_s/system-ontology-framework/system-ontology-framework.md @@ -0,0 +1,172 @@ +# C4 Model (industry standard) + +1. Context (system in the world) +2. Container (apps/services) +3. Component (modules inside apps) +4. Code (classes/functions) + +## Tools + +### graphical interface (lucidchart) +- start here to map the system +- this is for high-level thinking, presentation, human UI + +### spreadsheet (System catalog / service registry / data dictionary hybrid) + +As youi build the system and its files and components, you create a machine-readable system graph that is a declarative system model where everything is described, and everything else is derived from it: +- diagrams +- monitoring +- orchestration +- agents +The spreadsheet is the SOURCE OF TRUTH. JSON is a derived artifact. The Cytoscape graph is a VIEW. Never manually edit JSON long-term — generate it. + +Ai can use this to: +1. generate diagrams automatially +2. detect issueso +- orphaned components +- circular dependencies +- unused scripts +- data inconsistencies +3. build agents +- “Find all nodes where operation=VALIDATE and auto-optimize” +4. simulate system +- what happens if ... +- what breaks if ... +5.generate code scaffolding +- FastAPI routes +- cron jobs +- validators +6. build inference layer +See template. + +### Process +1. build graphical interface for human documenting components and flow. Build in small, disrete modules you can put a box around that is on its own. Define flows, outcomes. Then, define nodes and edges - ontology draft +2. Code the components +3. Have AI create the actual graph in the spreadsheet based on the templates - see in same folder for this file. Create in csv, copy and save as csv, open csv in spreadsheet, copy to gsheets file. Try to fit to validators sheet +4. Have AI build the corresponding json file. Invalid JSON (e.g. NaN) → graph will not render. See template of prior one created - graph_edges_nodes.json This is ontology final +4. Create browser-based cytoscape graph viewer - instructions below +5. Fix mismatches between ontologt draft and ontology final + +#### Example: + +Add Customer Module +Trigger: user submits form +Input: customer data +Process: + validate → write → index update +Output: + new customer JSON + +Graph: + +UI → API → validate → storage → index → response + +If a module does more than one of these → it’s too big. + +#### Practical Rules for Your Future Builds +1. Rule 1 — Always start with flow + +Before coding: + +User action → API → validation → storage → response + +2. Rule 2 — Cap module size + +If a module: + +touches > 5 files +has > 1 primary route + +👉 split it + +3. Rule 3 — Every module must be drawable as a box + +If you can’t draw a clean box around it: + +👉 it’s not a module yet + +4. Rule 4 — Graph is not documentation + +This is important: + +Your graph is not a diagram — it is a system constraint + +You should be able to say: + +“this edge is invalid” +“this node should not exist” + +### System Graph + +1. Create project in WSL +mkdir ~/(name)-system-graph +cd ~/(name)-system-graph +2. create index.html - adjust file name being rendered (step 4) +3. create app.js - adjust file name being rendered (step 4) +4. convert spreadsheet to JSON and create _(name)_graph_cytoscape.json +- - JSON MUST be a flat array of Cytoscape elements: + +[ + { data: { id, label } }, + { data: { source, target } } +] + +NOT: +{ nodes: [...], edges: [...] } +- - Node IDs must be: +- - - exact string match +- - -trimmed (no trailing spaces) +- - - no annotations (remove "(API)", "(DB)", etc.) +Edges must reference these exact IDs. +5. copy to WSl folder cp /mnt/c/Users//Desktop/lod_graph_cytoscape.json ~/system-graph/ +5. run on wsl: python3 -m http.server 8000 +6. http://localhost:8000 +7. Always edit the file the server is serving (WSL). Invalid JSON (e.g. NaN) → graph will not render. Node IDs must match edge references exactly. Use fetch debug to verify actual runtime data +- - fetch('lod_graph_cytoscape.json') + .then(r => r.text()) + .then(console.log) +8. Edit and adjust scripts as needed using codex desktop or in terminal, then output the files to desktop to save cp ~/system-graph/app.js ~/system-graph/index.html ~/system-graph/lod_graph_cytoscape.json /mnt/c/Users/edsoe/Desktop/ + + +## Other + +### Add environment boundary rule (this was THE issue) + +You mention copying, but not the underlying principle. + +Add: + +The browser only sees files served by the WSL server. + +Editing files on Desktop has no effect until copied into WSL. + +Always verify the runtime file, not the edited file. + +### Add minimal test graph (for sanity check) + +This is a must-have reset tool: + +[ + { "data": { "id": "A", "label": "A" } }, + { "data": { "id": "B", "label": "B" } }, + { "data": { "id": "e1", "source": "A", "target": "B" } } +] + +Use when debugging pipeline. + +### Add layout note (optional but useful) +Default layout: cose (force-directed) + +If graph looks flat or clustered: +- adjust layout +- or check connectivity + +### Common Failure Modes +- | Symptom | Cause | +| -------------------- | ----------------------------------- | +| Blank graph | JSON invalid OR fetch failed | +| “Unexpected token N” | NaN in JSON | +| A→B→C won’t change | editing wrong file (WSL vs Desktop) | +| Graph collapsed | edge references missing nodes | +| No console logs | app.js not loaded | + diff --git a/specs/3_product_development_s/system-ontology-framework/system-ontology-validators.csv b/specs/3_product_development_s/system-ontology-framework/system-ontology-validators.csv new file mode 100644 index 0000000..ef6c7ea --- /dev/null +++ b/specs/3_product_development_s/system-ontology-framework/system-ontology-validators.csv @@ -0,0 +1,28 @@ +module,sub-module,active,node-id,code-base,type,retry-policy-indempotent,logging,metrics,execution-environment,vm-environment,owner,from-node-to-node,edge-type,entity,execution,schedule,state-transition,dependencies,observability-hooks,criticality +customer-db,login,prod,file:login.js,html,UI,yes,yes,,browser/client,lod-dev-app,system,file:login.js,calls,attachment file,user,1-minute,auth dependency loaded,file:login.js,logs,low +,customers,dev,file:login.html,js,Script,no,no,,backend/vm,lod-prod-app,ingestion-agent,file:login.html,reads,attachment record,event,3-minutes,authenticated,file:login.html,metrics,medium +,,ideation,login API,py,Datastore,,print-based,,infrastructure,,inference-agent,login API,writes,auth token,cron,5-minutes,authorized,login API,alerts,high +,,archived,auth system,.env,Cron,,,,,,,auth system,triggers,chunk,manual,manual,chunked,auth system,, +,,,environment config,browser DOM (in memory),Template,,,,,,,environment config,validates,credentials,,on change,config loaded,environment config,, +,,,localStorage ,,API,,,,,,,localStorage ,transforms,customer index,,on load/reload,customer created/updated/deleted,localStorage ,, +,,,file:dashboard.html,,Schema,,,,,,,file:dashboard.html,queues,customer json,,on login,customer loaded,file:dashboard.html,, +,,,DOM (#msg),,External,,,,,,,DOM (#msg),depends_on,customer record,,on mutation,customer loaded/saved,DOM (#msg),, +,,,file:auth.py,,Service,,,,,,,file:auth.py,returns,dog index,,on request,dog index built,file:auth.py,, +,,,file:customer.py,,Config,,,,,,,file:customer.py,navigates,error message,,on submit,error,file:customer.py,, +,,,file:customers_index.py,,,,,,,,,file:customers_index.py,imports,PID,,on upload,file persisted/json updated,file:customers_index.py,, +,,,file:dogs.py,,,,,,,,,file:dogs.py,,session,,on upload,index built,file:dogs.py,, +,,,file:upload.py,,,,,,,,,file:upload.py,,token,,on upload,index rebuild callable,file:upload.py,, +,,,file:validators.py,,,,,,,,,file:validators.py,,user,,on upload,indexed,file:validators.py,, +,,,file:main.py,,,,,,,,,file:main.py,,validation rules,,,inferred,file:main.py,, +,,,file:requirements.txt,,,,,,,,,file:requirements.txt,,validator request,,,rules dependency loaded,file:requirements.txt,, +,,,.venv,,,,,,,,,.venv,,,,,rules returned,.venv,, +,,,_pycache_,,,,,,,,,_pycache_,,,,,storage dependency loaded,_pycache_,, +,,,file:storage.py,,,,,,,,,file:storage.py,,,,,submitted,file:storage.py,, +,,,file:validate.py,,,,,,,,,file:validate.py,,,,,validated,file:validate.py,, +,,,file:validators_config.py,,,,,,,,,file:validators_config.py,,,,,validated,file:validators_config.py,, +,,,file:add_customer.js,,,,,,,,,file:add_customer.js,,,,,validation dependency loaded,file:add_customer.js,, +,,,file:app.js,,,,,,,,,file:app.js,,,,,,file:app.js,, +,,,file:dogs.html,,,,,,,,,file:dogs.html,,,,,,file:dogs.html,, +,,,file:dogs.js,,,,,,,,,file:dogs.js,,,,,,file:dogs.js,, +,,,file:edit_customer.html,,,,,,,,,file:edit_customer.html,,,,,,file:edit_customer.html,, +,,,file:validators-ui.js,,,,,,,,,file:validators-ui.js,,,,,,file:validators-ui.js,, \ No newline at end of file diff --git a/specs/3_product_development_s/system-ontology-framework/system_ontology - edges.csv b/specs/3_product_development_s/system-ontology-framework/system_ontology - edges.csv new file mode 100644 index 0000000..51b5f26 --- /dev/null +++ b/specs/3_product_development_s/system-ontology-framework/system_ontology - edges.csv @@ -0,0 +1 @@ +module,active,edge-id,from-node,to-node,edge-type,entity,execution,schedule,state-transition,dependencies,criticality,execution-environment diff --git a/specs/3_product_development_s/system-ontology-framework/system_ontology - nodes.csv b/specs/3_product_development_s/system-ontology-framework/system_ontology - nodes.csv new file mode 100644 index 0000000..31051bf --- /dev/null +++ b/specs/3_product_development_s/system-ontology-framework/system_ontology - nodes.csv @@ -0,0 +1 @@ +module,sub-module,active,node-id,location,code-base,type,retry-policy,indempotent,logging,metrics,alerts,execution-environment,vm-environment,owner,external-connections,system-version,component-version,script-version