SSR health: alert mails on ok/down transitions + auto-failover to SPA after N consecutive misses (SSR_ALERT_EMAIL / SSR_FAILOVER_AFTER in var/vitec-ssr-purge.env)

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

packages/vitec/Classes/Command/SsrHealthSampleCommand.php

@@ -4,6 +4,7 @@ declare(strict_types=1);
 
 namespace Evomedien\Vitec\Command;
 
+use Evomedien\Vitec\Service\SsrFlagService;
 use Evomedien\Vitec\Service\SsrPurgeService;
 use Symfony\Component\Console\Attribute\AsCommand;
 use Symfony\Component\Console\Command\Command;
@@ -11,10 +12,13 @@ use Symfony\Component\Console\Input\InputInterface;
 use Symfony\Component\Console\Output\OutputInterface;
 use TYPO3\CMS\Core\Database\ConnectionPool;
 use TYPO3\CMS\Core\Http\RequestFactory;
+use TYPO3\CMS\Core\Mail\Mailer;
+use TYPO3\CMS\Core\Mail\MailMessage;
+use TYPO3\CMS\Core\Utility\GeneralUtility;
 
 #[AsCommand(
     name: 'vitec:ssr-health',
-    description: 'Sample the SSR node /__health into tx_vitec_ssrhealthlog (run via scheduler for history)',
+    description: 'Sample the SSR node /__health into tx_vitec_ssrhealthlog; mails on ok/down transitions, auto-failover to SPA after N consecutive misses',
 )]
 final class SsrHealthSampleCommand extends Command
 {
@@ -22,6 +26,7 @@ final class SsrHealthSampleCommand extends Command
 
     public function __construct(
         private readonly SsrPurgeService $purgeService,
+        private readonly SsrFlagService $flagService,
         private readonly RequestFactory $requestFactory,
         private readonly ConnectionPool $connectionPool,
     ) {
@@ -52,6 +57,16 @@ final class SsrHealthSampleCommand extends Command
         } catch (\Throwable $e) {
             $row['response_ms'] = (int)round((microtime(true) - $start) * 1000);
         }
+        // Previous samples, newest first, read BEFORE this one is inserted:
+        // [0] is the state every transition decision compares against.
+        $failoverAfter = $this->purgeService->getFailoverAfter();
+        $previous = array_map('intval', $this->connectionPool
+            ->getQueryBuilderForTable('tx_vitec_ssrhealthlog')
+            ->select('ok')->from('tx_vitec_ssrhealthlog')
+            ->orderBy('crdate', 'DESC')->addOrderBy('uid', 'DESC')
+            ->setMaxResults($failoverAfter)
+            ->executeQuery()->fetchFirstColumn());
+
         $conn = $this->connectionPool->getConnectionForTable('tx_vitec_ssrhealthlog');
         $conn->insert('tx_vitec_ssrhealthlog', $row);
         $conn->executeStatement(
@@ -66,6 +81,108 @@ final class SsrHealthSampleCommand extends Command
             $row['cache_mb'],
             $row['response_ms']
         ));
+        $prevOk = $previous === [] ? null : $previous[0];
+        if ($prevOk !== null && $prevOk !== $row['ok']) {
+            if ($row['ok'] === 1) {
+                $this->sendAlert(
+                    '[VITEC SSR] Entwarnung: SSR-Server wieder erreichbar',
+                    "Die 5-Minuten-Messung erreicht den SSR-Server wieder (Status ok).\n\n"
+                    . 'Zeitpunkt:   ' . date('d.m.Y H:i:s') . "\n"
+                    . 'Release:     ' . ($row['release_id'] ?: '-') . "\n"
+                    . 'Antwortzeit: ' . $row['response_ms'] . " ms\n\n"
+                    . 'Aktueller Stand: Schalter vitec.ssr.enabled = '
+                    . ($this->flagService->isEnabledInSettings() ? 'AN' : 'AUS')
+                    . ', Flag-Datei ' . ($this->flagService->isFlagSet() ? 'vorhanden (SSR aktiv)' : 'fehlt (SPA-Auslieferung)') . ".\n"
+                    . "Falls der Auto-Failover ausgelöst wurde, bleibt die Seite bewusst auf SPA.\n"
+                    . "Wieder einschalten: TYPO3-Backend > Site Management > Settings > SSR delivery auf AN,\n"
+                    . "danach Komplett-Purge (vendor/bin/typo3 vitec:ssr-purge --all).\n",
+                    $output
+                );
+            } else {
+                $this->sendAlert(
+                    '[VITEC SSR] ALARM: SSR-Server antwortet nicht',
+                    "Die 5-Minuten-Messung konnte den SSR-Server nicht erreichen\n"
+                    . "(oder /__health meldete einen Fehler).\n\n"
+                    . 'Zeitpunkt:   ' . date('d.m.Y H:i:s') . "\n"
+                    . 'Endpoint:    ' . $endpoint . "\n"
+                    . 'Antwortzeit: ' . $row['response_ms'] . " ms\n\n"
+                    . 'Solange die Flag-Datei existiert, leitet dev.vitec.com weiter zum SSR-Server. Nach '
+                    . $failoverAfter . ' Fehlmessungen in Folge (ca. ' . ($failoverAfter * 5) . " Minuten) schaltet\n"
+                    . "der Auto-Failover automatisch auf SPA-Auslieferung um - die Seite bleibt erreichbar.\n\n"
+                    . "Diese Mail kommt nur beim Zustandswechsel ok -> down, nicht bei jeder Messung.\n",
+                    $output
+                );
+            }
+        }
+
+        if ($row['ok'] === 0 && $this->flagService->isFlagSet()) {
+            $consecutive = 1;
+            foreach ($previous as $ok) {
+                if ($ok !== 0) {
+                    break;
+                }
+                $consecutive++;
+            }
+            if ($consecutive >= $failoverAfter) {
+                try {
+                    $this->flagService->disableSsr();
+                    $output->writeln(sprintf(
+                        '<error>AUTO-FAILOVER after %d consecutive misses: vitec.ssr.enabled=false, caches flushed, flag removed</error>',
+                        $consecutive
+                    ));
+                    $this->sendAlert(
+                        '[VITEC SSR] AUTO-FAILOVER: Seite auf SPA umgeschaltet',
+                        'AUTO-FAILOVER AUSGELÖST: ' . $consecutive . " Messungen in Folge down.\n\n"
+                        . "dev.vitec.com liefert ab sofort die SPA aus (Rendering im Browser).\n"
+                        . "Für Besucher bleibt die Seite erreichbar.\n\n"
+                        . "Was automatisch passiert ist:\n"
+                        . "- Site-Setting vitec.ssr.enabled auf false gestellt (settings.yaml)\n"
+                        . "- Caches geleert\n"
+                        . "- Flag-Datei var/vitec-ssr-on.flag gelöscht\n\n"
+                        . "Wieder einschalten (bewusst NUR von Hand), sobald der SSR-Server wieder läuft:\n"
+                        . "1. Auf dem VPS prüfen: systemctl status vitec-ssr und /__health\n"
+                        . "2. TYPO3-Backend > Site Management > Settings > SSR delivery wieder AN\n"
+                        . "3. Komplett-Purge: vendor/bin/typo3 vitec:ssr-purge --all\n",
+                        $output
+                    );
+                } catch (\Throwable $e) {
+                    $output->writeln('<error>auto-failover FAILED: ' . $e->getMessage() . '</error>');
+                    $this->sendAlert(
+                        '[VITEC SSR] FEHLER: Auto-Failover fehlgeschlagen',
+                        "Der Auto-Failover sollte auslösen, ist aber fehlgeschlagen:\n"
+                        . $e->getMessage() . "\n\n"
+                        . "Bitte von Hand prüfen: Site-Setting vitec.ssr.enabled, Datei var/vitec-ssr-on.flag.\n"
+                        . "Not-Aus ohne TYPO3: Flag-Datei löschen UND Setting auf false stellen\n"
+                        . "(sonst legt die SyncSsrFlag-Middleware die Flag-Datei sofort wieder an).\n",
+                        $output
+                    );
+                }
+            }
+        }
+
         return Command::SUCCESS;
     }
+
+    /**
+     * Mails are best effort: a mail problem must never break the sampler
+     * (the cron would stop logging history). Recipients come from
+     * SSR_ALERT_EMAIL in var/vitec-ssr-purge.env - unset means "alerts
+     * off", which the output states so the cron log shows why.
+     */
+    private function sendAlert(string $subject, string $body, OutputInterface $output): void
+    {
+        $recipients = GeneralUtility::trimExplode(',', $this->purgeService->getAlertEmail(), true);
+        if ($recipients === []) {
+            $output->writeln('<comment>transition detected but SSR_ALERT_EMAIL is not configured - no mail sent</comment>');
+            return;
+        }
+        try {
+            $mail = GeneralUtility::makeInstance(MailMessage::class);
+            $mail->to(...$recipients)->subject($subject)->text($body);
+            GeneralUtility::makeInstance(Mailer::class)->send($mail);
+            $output->writeln('alert mail sent: ' . $subject . ' -> ' . implode(', ', $recipients));
+        } catch (\Throwable $e) {
+            $output->writeln('<error>alert mail failed: ' . $e->getMessage() . '</error>');
+        }
+    }
 }

packages/vitec/Classes/Service/SsrFlagService.php

@@ -4,9 +4,11 @@ declare(strict_types=1);
 
 namespace Evomedien\Vitec\Service;
 
+use TYPO3\CMS\Core\Cache\CacheManager;
 use TYPO3\CMS\Core\Core\Environment;
 use TYPO3\CMS\Core\Exception\SiteNotFoundException;
 use TYPO3\CMS\Core\Site\SiteFinder;
+use TYPO3\CMS\Core\Utility\GeneralUtility;
 
 /**
  * Keeps the SSR routing flag file in sync with the site setting
@@ -91,6 +93,48 @@ final class SsrFlagService
         return $desired;
     }
 
+    /**
+     * Auto-failover (vitec:ssr-health): turns SSR delivery hard OFF.
+     *
+     * Removing only the flag file would not stick - the self-healing
+     * SyncSsrFlag middleware recreates it from the still-true site setting
+     * on the very next request. So the setting itself is flipped to false
+     * in config/sites/<identifier>/settings.yaml (the file the backend
+     * settings editor writes, flat dotted keys), caches are flushed so the
+     * compiled site settings forget the old value, and only then the flag
+     * is removed. Re-enabling is deliberately manual (backend switch or
+     * settings.yaml + cache flush).
+     */
+    public function disableSsr(): void
+    {
+        $changed = false;
+        foreach ($this->siteFinder->getAllSites() as $site) {
+            $file = Environment::getConfigPath() . '/sites/' . $site->getIdentifier() . '/settings.yaml';
+            if (!is_file($file)) {
+                continue;
+            }
+            $yaml = (string)file_get_contents($file);
+            $new = preg_replace('/^(vitec\.ssr\.enabled\s*:).*$/m', '$1 false', $yaml, 1, $count);
+            if ($new !== null && $count > 0 && $new !== $yaml) {
+                if (file_put_contents($file, $new) === false) {
+                    throw new \RuntimeException('Could not write ' . $file, 1762800001);
+                }
+                $changed = true;
+            }
+        }
+        if (!$changed && $this->isEnabledInSettings()) {
+            throw new \RuntimeException(
+                'vitec.ssr.enabled not found in any config/sites/*/settings.yaml - cannot disable it, '
+                . 'and the SyncSsrFlag middleware would recreate the flag from the setting',
+                1762800002
+            );
+        }
+        if ($changed) {
+            GeneralUtility::makeInstance(CacheManager::class)->flushCaches();
+        }
+        @unlink($this->getFlagPath());
+    }
+
     /**
      * Sync variant for the request hot path: never lets an IO hiccup
      * (read-only FS, concurrent unlink ...) break request handling.

packages/vitec/Classes/Service/SsrPurgeService.php

@@ -13,7 +13,8 @@ use TYPO3\CMS\Core\Http\RequestFactory;
  * keeps a log of every attempt in tx_vitec_ssrpurgelog.
  *
  * Endpoint + secret live OUTSIDE the repository in var/vitec-ssr-purge.env
- * (KEY=VALUE lines: SSR_PURGE_ENDPOINT, SSR_PURGE_SECRET, SSR_HEALTH_ENDPOINT)
+ * (KEY=VALUE lines: SSR_PURGE_ENDPOINT, SSR_PURGE_SECRET, SSR_HEALTH_ENDPOINT,
+ * SSR_ALERT_EMAIL, SSR_FAILOVER_AFTER)
  * - same pattern as the routing flag file, nothing secret in git.
  */
 final class SsrPurgeService
@@ -41,6 +42,25 @@ final class SsrPurgeService
         return $this->getConfig()['SSR_HEALTH_ENDPOINT'] ?? '';
     }
 
+    /**
+     * Comma-separated recipient list for vitec:ssr-health transition and
+     * failover alerts (SSR_ALERT_EMAIL); empty string = alerting off.
+     */
+    public function getAlertEmail(): string
+    {
+        return trim($this->getConfig()['SSR_ALERT_EMAIL'] ?? '');
+    }
+
+    /**
+     * Consecutive DOWN samples after which vitec:ssr-health fails over to
+     * the SPA (SSR_FAILOVER_AFTER, default 2 = ~10 minutes at the 5-minute
+     * cron cadence). Never below 1.
+     */
+    public function getFailoverAfter(): int
+    {
+        return max(1, (int)($this->getConfig()['SSR_FAILOVER_AFTER'] ?? 2));
+    }
+
     /**
      * Normalizes, dedupes and sends the paths; logs the outcome.
      * Never throws - editors' saves must not fail on a purge problem.