observability.md: full 7-step audit, first pass this session

1. Live-verify: journald persistent-storage config confirmed on two
   hosts (1G vs 500M cap, matches the doc's claim exactly), Alloy
   active confirmed across all 11 fleet hosts.
2. Fix KB: Alloy Agent Model was stale - version had moved from
   v1.18.0 to v1.18.1 on all 8 'newer' hosts since last checked,
   role's own default still unreconciled at 1.17.1. Fixed and
   confirmed via re-fetch.
3. Pulled the full Observability Pipeline tree: Alloy Agent Model,
   Log Flow, Metrics Flow.
4. Replaced both mermaid diagrams (broke this repo's Nextcloud-SVG
   convention) with real diagrams built through the proper round-trip.
5. Evidence: real journald config dumps, plus 3 real Grafana
   screenshots (host metrics dashboard, detailed Loki log stream
   showing job=technitium-dns/docker/system, and the fleet-wide log
   volume dashboard) - one screenshot round caught and fixed an
   unredacted internal IP before being used.
6/7. Compelling-proof test passes; sanitize-check clean.

assets/observability-log-fanout.svg

@@ -0,0 +1,4 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!-- Do not edit this file with editors other than draw.io -->
+<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
+<svg xmlns="http://www.w3.org/2000/svg" style="background: transparent; background-color: transparent; color-scheme: light dark;" xmlns:xlink="http://www.w3.org/1999/xlink" version="1.1" width="872px" height="273px" viewBox="0 0 872 273" content="&lt;mxfile host=&quot;embed.diagrams.net&quot; scale=&quot;1&quot; border=&quot;0&quot;&gt;&#10;  &lt;diagram name=&quot;Log Fanout&quot; id=&quot;obs-logfanout-1&quot;&gt;&#10;    &lt;mxGraphModel dx=&quot;1414&quot; dy=&quot;791&quot; grid=&quot;1&quot; gridSize=&quot;10&quot; guides=&quot;1&quot; tooltips=&quot;1&quot; connect=&quot;1&quot; arrows=&quot;1&quot; fold=&quot;1&quot; page=&quot;1&quot; pageScale=&quot;1&quot; pageWidth=&quot;1000&quot; pageHeight=&quot;320&quot; math=&quot;0&quot; shadow=&quot;0&quot;&gt;&#10;      &lt;root&gt;&#10;        &lt;mxCell id=&quot;0&quot; /&gt;&#10;        &lt;mxCell id=&quot;1&quot; parent=&quot;0&quot; /&gt;&#10;        &lt;mxCell id=&quot;a1&quot; parent=&quot;1&quot; style=&quot;rounded=1;whiteSpace=wrap;html=1;fillColor=#dae8fc;strokeColor=#6c8ebf;fontColor=#333333;fontSize=12;fontStyle=1;shadow=1;&quot; value=&quot;journald&amp;lt;br&amp;gt;&amp;lt;font style=&amp;quot;font-size: 10px;&amp;quot;&amp;gt;system logs&amp;lt;/font&amp;gt;&quot; vertex=&quot;1&quot;&gt;&#10;          &lt;mxGeometry height=&quot;60&quot; width=&quot;200&quot; x=&quot;40&quot; y=&quot;30&quot; as=&quot;geometry&quot; /&gt;&#10;        &lt;/mxCell&gt;&#10;        &lt;mxCell id=&quot;b1&quot; parent=&quot;1&quot; style=&quot;rounded=1;whiteSpace=wrap;html=1;fillColor=#d5e8d4;strokeColor=#82b366;fontColor=#333333;fontSize=12;fontStyle=1;shadow=1;&quot; value=&quot;Loki&amp;lt;br&amp;gt;&amp;lt;font style=&amp;quot;font-size: 10px;&amp;quot;&amp;gt;job=system&amp;lt;/font&amp;gt;&quot; vertex=&quot;1&quot;&gt;&#10;          &lt;mxGeometry height=&quot;60&quot; width=&quot;200&quot; x=&quot;700&quot; y=&quot;30&quot; as=&quot;geometry&quot; /&gt;&#10;        &lt;/mxCell&gt;&#10;        &lt;mxCell id=&quot;e1&quot; edge=&quot;1&quot; parent=&quot;1&quot; source=&quot;a1&quot; style=&quot;html=1;endArrow=block;strokeColor=#6c8ebf;strokeWidth=2;rounded=1;&quot; target=&quot;b1&quot;&gt;&#10;          &lt;mxGeometry relative=&quot;1&quot; as=&quot;geometry&quot; /&gt;&#10;        &lt;/mxCell&gt;&#10;        &lt;mxCell id=&quot;a2&quot; parent=&quot;1&quot; style=&quot;rounded=1;whiteSpace=wrap;html=1;fillColor=#dae8fc;strokeColor=#6c8ebf;fontColor=#333333;fontSize=12;fontStyle=1;shadow=1;&quot; value=&quot;Docker API&amp;lt;br&amp;gt;&amp;lt;font style=&amp;quot;font-size: 10px;&amp;quot;&amp;gt;container logs, survives recreation&amp;lt;/font&amp;gt;&quot; vertex=&quot;1&quot;&gt;&#10;          &lt;mxGeometry height=&quot;60&quot; width=&quot;200&quot; x=&quot;40&quot; y=&quot;130&quot; as=&quot;geometry&quot; /&gt;&#10;        &lt;/mxCell&gt;&#10;        &lt;mxCell id=&quot;b2&quot; parent=&quot;1&quot; style=&quot;rounded=1;whiteSpace=wrap;html=1;fillColor=#d5e8d4;strokeColor=#82b366;fontColor=#333333;fontSize=12;fontStyle=1;shadow=1;&quot; value=&quot;Loki&amp;lt;br&amp;gt;&amp;lt;font style=&amp;quot;font-size: 10px;&amp;quot;&amp;gt;job=docker&amp;lt;/font&amp;gt;&quot; vertex=&quot;1&quot;&gt;&#10;          &lt;mxGeometry height=&quot;60&quot; width=&quot;200&quot; x=&quot;700&quot; y=&quot;130&quot; as=&quot;geometry&quot; /&gt;&#10;        &lt;/mxCell&gt;&#10;        &lt;mxCell id=&quot;e2&quot; edge=&quot;1&quot; parent=&quot;1&quot; source=&quot;a2&quot; style=&quot;html=1;endArrow=block;strokeColor=#6c8ebf;strokeWidth=2;rounded=1;&quot; target=&quot;b2&quot;&gt;&#10;          &lt;mxGeometry relative=&quot;1&quot; as=&quot;geometry&quot; /&gt;&#10;        &lt;/mxCell&gt;&#10;        &lt;mxCell id=&quot;a3&quot; parent=&quot;1&quot; style=&quot;rounded=1;whiteSpace=wrap;html=1;fillColor=#dae8fc;strokeColor=#6c8ebf;fontColor=#333333;fontSize=12;fontStyle=1;shadow=1;&quot; value=&quot;DNS query log&amp;lt;br&amp;gt;&amp;lt;font style=&amp;quot;font-size: 10px;&amp;quot;&amp;gt;daily rotation&amp;lt;/font&amp;gt;&quot; vertex=&quot;1&quot;&gt;&#10;          &lt;mxGeometry height=&quot;60&quot; width=&quot;200&quot; x=&quot;40&quot; y=&quot;230&quot; as=&quot;geometry&quot; /&gt;&#10;        &lt;/mxCell&gt;&#10;        &lt;mxCell id=&quot;b3&quot; parent=&quot;1&quot; style=&quot;rounded=1;whiteSpace=wrap;html=1;fillColor=#d5e8d4;strokeColor=#82b366;fontColor=#333333;fontSize=12;fontStyle=1;shadow=1;&quot; value=&quot;Loki&amp;lt;br&amp;gt;&amp;lt;font style=&amp;quot;font-size: 10px;&amp;quot;&amp;gt;job=dns, real event time&amp;lt;/font&amp;gt;&quot; vertex=&quot;1&quot;&gt;&#10;          &lt;mxGeometry height=&quot;60&quot; width=&quot;200&quot; x=&quot;700&quot; y=&quot;230&quot; as=&quot;geometry&quot; /&gt;&#10;        &lt;/mxCell&gt;&#10;        &lt;mxCell id=&quot;e3&quot; edge=&quot;1&quot; parent=&quot;1&quot; source=&quot;a3&quot; style=&quot;html=1;endArrow=block;strokeColor=#6c8ebf;strokeWidth=2;rounded=1;&quot; target=&quot;b3Line truncated

assets/observability-pipeline.svg

@@ -0,0 +1,4 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!-- Do not edit this file with editors other than draw.io -->
+<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
+<svg xmlns="http://www.w3.org/2000/svg" style="background: transparent; background-color: transparent; color-scheme: light dark;" xmlns:xlink="http://www.w3.org/1999/xlink" version="1.1" width="838px" height="324px" viewBox="0 0 838 324" content="&lt;mxfile host=&quot;embed.diagrams.net&quot; scale=&quot;1&quot; border=&quot;0&quot;&gt;&#10;  &lt;diagram name=&quot;Pipeline&quot; id=&quot;obs-pipeline-1&quot;&gt;&#10;    &lt;mxGraphModel dx=&quot;1178&quot; dy=&quot;659&quot; grid=&quot;1&quot; gridSize=&quot;10&quot; guides=&quot;1&quot; tooltips=&quot;1&quot; connect=&quot;1&quot; arrows=&quot;1&quot; fold=&quot;1&quot; page=&quot;1&quot; pageScale=&quot;1&quot; pageWidth=&quot;1000&quot; pageHeight=&quot;420&quot; math=&quot;0&quot; shadow=&quot;0&quot;&gt;&#10;      &lt;root&gt;&#10;        &lt;mxCell id=&quot;0&quot; /&gt;&#10;        &lt;mxCell id=&quot;1&quot; parent=&quot;0&quot; /&gt;&#10;        &lt;mxCell id=&quot;srcgroup&quot; parent=&quot;1&quot; style=&quot;rounded=1;whiteSpace=wrap;html=1;startSize=30;swimlane;fillColor=none;strokeColor=#666666;dashed=1;fontSize=13;fontStyle=1;fontColor=#333333;align=center;&quot; value=&quot;PER-HOST SOURCES&quot; vertex=&quot;1&quot;&gt;&#10;          &lt;mxGeometry height=&quot;320&quot; width=&quot;220&quot; x=&quot;30&quot; y=&quot;40&quot; as=&quot;geometry&quot; /&gt;&#10;        &lt;/mxCell&gt;&#10;        &lt;mxCell id=&quot;s1&quot; parent=&quot;srcgroup&quot; style=&quot;rounded=1;whiteSpace=wrap;html=1;fillColor=#dae8fc;strokeColor=#6c8ebf;fontColor=#333333;fontSize=12;fontStyle=1;shadow=1;&quot; value=&quot;journald&amp;lt;br&amp;gt;&amp;lt;font style=&amp;quot;font-size: 10px;&amp;quot;&amp;gt;persistent, 14-day retention&amp;lt;/font&amp;gt;&quot; vertex=&quot;1&quot;&gt;&#10;          &lt;mxGeometry height=&quot;60&quot; width=&quot;180&quot; x=&quot;20&quot; y=&quot;30&quot; as=&quot;geometry&quot; /&gt;&#10;        &lt;/mxCell&gt;&#10;        &lt;mxCell id=&quot;s2&quot; parent=&quot;srcgroup&quot; style=&quot;rounded=1;whiteSpace=wrap;html=1;fillColor=#dae8fc;strokeColor=#6c8ebf;fontColor=#333333;fontSize=12;fontStyle=1;shadow=1;&quot; value=&quot;Docker container&amp;lt;br&amp;gt;&amp;lt;font style=&amp;quot;font-size: 10px;&amp;quot;&amp;gt;stdout/stderr, docker_hosts only&amp;lt;/font&amp;gt;&quot; vertex=&quot;1&quot;&gt;&#10;          &lt;mxGeometry height=&quot;60&quot; width=&quot;180&quot; x=&quot;20&quot; y=&quot;105&quot; as=&quot;geometry&quot; /&gt;&#10;        &lt;/mxCell&gt;&#10;        &lt;mxCell id=&quot;s3&quot; parent=&quot;srcgroup&quot; style=&quot;rounded=1;whiteSpace=wrap;html=1;fillColor=#d5e8d4;strokeColor=#82b366;fontColor=#333333;fontSize=12;fontStyle=1;shadow=1;&quot; value=&quot;Node metrics&amp;lt;br&amp;gt;&amp;lt;font style=&amp;quot;font-size: 10px;&amp;quot;&amp;gt;CPU / mem / disk / net&amp;lt;/font&amp;gt;&quot; vertex=&quot;1&quot;&gt;&#10;          &lt;mxGeometry height=&quot;60&quot; width=&quot;180&quot; x=&quot;20&quot; y=&quot;180&quot; as=&quot;geometry&quot; /&gt;&#10;        &lt;/mxCell&gt;&#10;        &lt;mxCell id=&quot;s4&quot; parent=&quot;srcgroup&quot; style=&quot;rounded=1;whiteSpace=wrap;html=1;fillColor=#d5e8d4;strokeColor=#82b366;fontColor=#333333;fontSize=12;fontStyle=1;shadow=1;&quot; value=&quot;Native app metrics&amp;lt;br&amp;gt;&amp;lt;font style=&amp;quot;font-size: 10px;&amp;quot;&amp;gt;5 apps expose endpoints&amp;lt;/font&amp;gt;&quot; vertex=&quot;1&quot;&gt;&#10;          &lt;mxGeometry height=&quot;60&quot; width=&quot;180&quot; x=&quot;20&quot; y=&quot;255&quot; as=&quot;geometry&quot; /&gt;&#10;        &lt;/mxCell&gt;&#10;        &lt;mxCell id=&quot;agent&quot; parent=&quot;1&quot; style=&quot;rounded=1;whiteSpace=wrap;html=1;fillColor=#fff2cc;strokeColor=#d6b656;fontColor=#333333;fontSize=13;fontStyle=1;shadow=1;&quot; value=&quot;Grafana Alloy&amp;lt;br&amp;gt;&amp;lt;font style=&amp;quot;font-size: 10px;&amp;quot;&amp;gt;native systemd service&amp;lt;/font&amp;gt;&quot; vertex=&quot;1&quot;&gt;&#10;          &lt;mxGeometry height=&quot;80&quot; width=&quot;200&quot; x=&quot;360&quot; y=&quot;160&quot; as=&quot;geometry&quot; /&gt;&#10;        &lt;/mxCell&gt;&#10;        &lt;mxCell id=&quot;loki&quot; parent=&quot;1&quot; style=&quot;rounded=1;whiteSpace=wrap;html=1;fillColor=#dae8fc;strokeColor=#6c8ebf;fontColor=#333333;fontSize=13;fontStyle=1;shadow=1;&quot; value=&quot;Loki&quot; vertex=&quot;1&quot;&gt;&#10;          &lt;mxGeometry height=&quot;60&quot; width=&quot;160&quot; x=&quot;700&quot; y=&quot;70&quot; as=&quot;geometry&quot; /&gt;&#10;        &lt;/mxCell&gt;&#10;        &lt;mxCell id=&quot;prom&quot; parent=&quot;1&quot; style=&quot;rounded=1;whiteSpace=wrap;html=1;fillColor=#d5e8d4;strokeColor=#82b366;fontColor=#333333;fontSize=13;fontStyle=1;shadow=1;&quot; value=&quot;Prometheus&quot; vertex=&quot;1&quot;&gt;&#10;          &lt;mxGeometry height=&quot;60&quot; width=&quot;160&quot; x=&quot;700&quot; y=&quot;270&quot; as=&quot;geometry&quot; /&gt;&#10;    Line truncated