SSR dashboard: new errors-and-hit-rate tile fed by /__stats samples in vitec:ssr-health (stat_* columns in healthlog)

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

packages/vitec/Classes/Command/SsrHealthSampleCommand.php

@@ -57,6 +57,36 @@ final class SsrHealthSampleCommand extends Command
         } catch (\Throwable $e) {
             $row['response_ms'] = (int)round((microtime(true) - $start) * 1000);
         }
+
+        // Cumulative node counters from /__stats (-1 = no data this sample,
+        // e.g. node down or release without the endpoint). The errors tile
+        // computes 24h deltas from these, skipping -1 rows; the counters
+        // reset whenever the node restarts, which the delta logic clamps.
+        $stats = [
+            'stat_hit' => -1, 'stat_miss' => -1, 'stat_stale' => -1,
+            'stat_bypass' => -1, 'stat_render_errors' => -1, 'stat_backend_errors' => -1,
+        ];
+        if ($row['ok'] === 1 && ($statsEndpoint = $this->purgeService->getStatsEndpoint()) !== '') {
+            try {
+                $statsResponse = $this->requestFactory->request($statsEndpoint, 'GET', [
+                    'timeout' => 4, 'connect_timeout' => 2, 'http_errors' => false,
+                ]);
+                if ($statsResponse->getStatusCode() === 200) {
+                    $s = json_decode((string)$statsResponse->getBody(), true) ?: [];
+                    $stats = [
+                        'stat_hit' => (int)($s['hit'] ?? -1),
+                        'stat_miss' => (int)($s['miss'] ?? -1),
+                        'stat_stale' => (int)($s['stale'] ?? -1),
+                        'stat_bypass' => (int)($s['bypass'] ?? -1),
+                        'stat_render_errors' => (int)($s['renderErrors'] ?? -1),
+                        'stat_backend_errors' => (int)($s['backendErrors'] ?? -1),
+                    ];
+                }
+            } catch (\Throwable) {
+                // Stats are optional - the health sample must never fail on them.
+            }
+        }
+        $row = array_merge($row, $stats);
         // Previous samples, newest first, read BEFORE this one is inserted:
         // [0] is the state every transition decision compares against.
         $failoverAfter = $this->purgeService->getFailoverAfter();
@@ -81,6 +111,17 @@ final class SsrHealthSampleCommand extends Command
             $row['cache_mb'],
             $row['response_ms']
         ));
+        if ($row['stat_hit'] >= 0) {
+            $output->writeln(sprintf(
+                'stats: hit=%d miss=%d stale=%d bypass=%d renderErrors=%d backendErrors=%d',
+                $row['stat_hit'],
+                $row['stat_miss'],
+                $row['stat_stale'],
+                $row['stat_bypass'],
+                $row['stat_render_errors'],
+                $row['stat_backend_errors']
+            ));
+        }
         $prevOk = $previous === [] ? null : $previous[0];
         if ($prevOk !== null && $prevOk !== $row['ok']) {
             if ($row['ok'] === 1) {

packages/vitec/Classes/Service/SsrPurgeService.php

@@ -42,6 +42,21 @@ final class SsrPurgeService
         return $this->getConfig()['SSR_HEALTH_ENDPOINT'] ?? '';
     }
 
+    /**
+     * The /__stats sibling of the health endpoint (cumulative hit/miss/
+     * error counters, in the node since release 20261109-211544). Derived
+     * from SSR_HEALTH_ENDPOINT; SSR_STATS_ENDPOINT overrides explicitly.
+     */
+    public function getStatsEndpoint(): string
+    {
+        $explicit = $this->getConfig()['SSR_STATS_ENDPOINT'] ?? '';
+        if ($explicit !== '') {
+            return $explicit;
+        }
+        $health = $this->getHealthEndpoint();
+        return $health === '' ? '' : str_replace('/__health', '/__stats', $health);
+    }
+
     /**
      * Comma-separated recipient list for vitec:ssr-health transition and
      * failover alerts (SSR_ALERT_EMAIL); empty string = alerting off.

packages/vitec/Classes/Widgets/SsrErrorsWidget.php

@@ -0,0 +1,159 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Evomedien\Vitec\Widgets;
+
+use TYPO3\CMS\Core\Database\ConnectionPool;
+use TYPO3\CMS\Dashboard\Widgets\WidgetConfigurationInterface;
+use TYPO3\CMS\Dashboard\Widgets\WidgetInterface;
+
+/**
+ * Dashboard tile: SSR error counters and cache hit rate.
+ *
+ * Source: the cumulative /__stats counters of the SSR node, sampled every
+ * 5 minutes by vitec:ssr-health into tx_vitec_ssrhealthlog (stat_* columns,
+ * -1 = no data for that sample). The counters reset whenever the node
+ * restarts (deploy!), so "totals" are "since node start" and the 24h error
+ * figures are sums of positive deltas between consecutive samples - a
+ * reset simply contributes nothing.
+ */
+final class SsrErrorsWidget implements WidgetInterface
+{
+    public function __construct(
+        private readonly WidgetConfigurationInterface $configuration,
+        private readonly ConnectionPool $connectionPool,
+    ) {}
+
+    public function renderWidgetContent(): string
+    {
+        $latest = $this->latestStatsRow();
+        if ($latest === null) {
+            return '<p class="text-muted">No /__stats samples yet. They arrive with the next '
+                . '<code>vitec:ssr-health</code> cron run (every 5 minutes) once the node release '
+                . 'provides <code>/__stats</code>.</p>';
+        }
+
+        $served = (int)$latest['stat_hit'] + (int)$latest['stat_miss'] + (int)$latest['stat_stale'];
+        $hitRate = $served > 0
+            ? number_format(((int)$latest['stat_hit'] + (int)$latest['stat_stale']) / $served * 100, 1) . ' %'
+            : '-';
+
+        [$newRender24h, $newBackend24h] = $this->errorDeltas24h();
+        $down24h = $this->downSamples24h();
+
+        $rows = [
+            ['Cache hit rate (since node start)', htmlspecialchars($hitRate)
+                . ' <small>(hit ' . (int)$latest['stat_hit']
+                . ' / stale ' . (int)$latest['stat_stale']
+                . ' / miss ' . (int)$latest['stat_miss'] . ')</small>'],
+            ['Render errors (new in 24 h / total)', $this->errorCell($newRender24h, (int)$latest['stat_render_errors'])],
+            ['Backend errors (new in 24 h / total)', $this->errorCell($newBackend24h, (int)$latest['stat_backend_errors'])],
+            ['DOWN samples (24 h)', $down24h['count'] > 0
+                ? '<span class="badge badge-danger">' . $down24h['count'] . '</span> <small>last: '
+                    . htmlspecialchars($down24h['last']) . '</small>'
+                : '<span class="badge badge-success">0</span>'],
+            ['Sample age', htmlspecialchars($this->age((int)$latest['crdate']))],
+        ];
+
+        $html = '<table class="table table-striped table-sm"><tbody>';
+        foreach ($rows as [$label, $value]) {
+            $html .= '<tr><th style="width:55%">' . $label . '</th><td>' . $value . '</td></tr>';
+        }
+        $html .= '</tbody></table>';
+        $html .= '<p class="text-muted" style="font-size:11px">Totals are cumulative since the node '
+            . 'last (re)started; a deploy resets them. Error details: '
+            . '<code>journalctl -u vitec-ssr</code> on the VPS.</p>';
+        return $html;
+    }
+
+    public function getOptions(): array
+    {
+        return [];
+    }
+
+    private function errorCell(?int $new24h, int $total): string
+    {
+        $newBadge = $new24h === null
+            ? '<span class="badge badge-secondary">n/a</span>'
+            : ($new24h > 0
+                ? '<span class="badge badge-danger">+' . $new24h . '</span>'
+                : '<span class="badge badge-success">+0</span>');
+        return $newBadge . ' <small>/ ' . $total . ' total</small>';
+    }
+
+    private function latestStatsRow(): ?array
+    {
+        try {
+            $row = $this->connectionPool->getConnectionForTable('tx_vitec_ssrhealthlog')
+                ->executeQuery(
+                    'SELECT crdate, stat_hit, stat_miss, stat_stale, stat_render_errors, stat_backend_errors'
+                    . ' FROM tx_vitec_ssrhealthlog WHERE stat_hit >= 0 ORDER BY crdate DESC, uid DESC LIMIT 1'
+                )->fetchAssociative();
+            return $row === false ? null : $row;
+        } catch (\Throwable) {
+            return null; // Column missing before database:updateschema - render the empty state.
+        }
+    }
+
+    /**
+     * New errors in the last 24 h = sum of positive deltas between
+     * consecutive stats samples. Rows with -1 (node down / no stats) are
+     * skipped; counter resets produce a negative step, clamped to 0.
+     *
+     * @return array{0: ?int, 1: ?int} [renderErrors, backendErrors]
+     */
+    private function errorDeltas24h(): array
+    {
+        try {
+            $rows = $this->connectionPool->getConnectionForTable('tx_vitec_ssrhealthlog')
+                ->executeQuery(
+                    'SELECT stat_render_errors, stat_backend_errors FROM tx_vitec_ssrhealthlog'
+                    . ' WHERE crdate > ' . (time() - 86400) . ' AND stat_hit >= 0 ORDER BY crdate ASC, uid ASC'
+                )->fetchAllAssociative();
+        } catch (\Throwable) {
+            return [null, null];
+        }
+        if (count($rows) < 2) {
+            return [null, null];
+        }
+        $render = 0;
+        $backend = 0;
+        $prev = null;
+        foreach ($rows as $r) {
+            if ($prev !== null) {
+                $render += max(0, (int)$r['stat_render_errors'] - (int)$prev['stat_render_errors']);
+                $backend += max(0, (int)$r['stat_backend_errors'] - (int)$prev['stat_backend_errors']);
+            }
+            $prev = $r;
+        }
+        return [$render, $backend];
+    }
+
+    /**
+     * @return array{count: int, last: string}
+     */
+    private function downSamples24h(): array
+    {
+        try {
+            $conn = $this->connectionPool->getConnectionForTable('tx_vitec_ssrhealthlog');
+            $count = (int)$conn->executeQuery(
+                'SELECT COUNT(*) FROM tx_vitec_ssrhealthlog WHERE crdate > ' . (time() - 86400) . ' AND ok = 0'
+            )->fetchOne();
+            $last = $count > 0
+                ? date('d.m. H:i', (int)$conn->executeQuery(
+                    'SELECT MAX(crdate) FROM tx_vitec_ssrhealthlog WHERE ok = 0'
+                )->fetchOne())
+                : '-';
+            return ['count' => $count, 'last' => $last];
+        } catch (\Throwable) {
+            return ['count' => 0, 'last' => '-'];
+        }
+    }
+
+    private function age(int $crdate): string
+    {
+        $minutes = (int)floor((time() - $crdate) / 60);
+        return $minutes < 1 ? 'just now' : $minutes . ' min';
+    }
+}