diff --git a/packages/vitec/Classes/Command/SsrHealthSampleCommand.php b/packages/vitec/Classes/Command/SsrHealthSampleCommand.php index 6c321ee..8ca9ec9 100644 --- a/packages/vitec/Classes/Command/SsrHealthSampleCommand.php +++ b/packages/vitec/Classes/Command/SsrHealthSampleCommand.php @@ -4,6 +4,7 @@ declare(strict_types=1); namespace Evomedien\Vitec\Command; +use Evomedien\Vitec\Service\SsrFlagService; use Evomedien\Vitec\Service\SsrPurgeService; use Symfony\Component\Console\Attribute\AsCommand; use Symfony\Component\Console\Command\Command; @@ -11,10 +12,13 @@ use Symfony\Component\Console\Input\InputInterface; use Symfony\Component\Console\Output\OutputInterface; use TYPO3\CMS\Core\Database\ConnectionPool; use TYPO3\CMS\Core\Http\RequestFactory; +use TYPO3\CMS\Core\Mail\Mailer; +use TYPO3\CMS\Core\Mail\MailMessage; +use TYPO3\CMS\Core\Utility\GeneralUtility; #[AsCommand( name: 'vitec:ssr-health', - description: 'Sample the SSR node /__health into tx_vitec_ssrhealthlog (run via scheduler for history)', + description: 'Sample the SSR node /__health into tx_vitec_ssrhealthlog; mails on ok/down transitions, auto-failover to SPA after N consecutive misses', )] final class SsrHealthSampleCommand extends Command { @@ -22,6 +26,7 @@ final class SsrHealthSampleCommand extends Command public function __construct( private readonly SsrPurgeService $purgeService, + private readonly SsrFlagService $flagService, private readonly RequestFactory $requestFactory, private readonly ConnectionPool $connectionPool, ) { @@ -52,6 +57,16 @@ final class SsrHealthSampleCommand extends Command } catch (\Throwable $e) { $row['response_ms'] = (int)round((microtime(true) - $start) * 1000); } + // Previous samples, newest first, read BEFORE this one is inserted: + // [0] is the state every transition decision compares against. + $failoverAfter = $this->purgeService->getFailoverAfter(); + $previous = array_map('intval', $this->connectionPool + ->getQueryBuilderForTable('tx_vitec_ssrhealthlog') + ->select('ok')->from('tx_vitec_ssrhealthlog') + ->orderBy('crdate', 'DESC')->addOrderBy('uid', 'DESC') + ->setMaxResults($failoverAfter) + ->executeQuery()->fetchFirstColumn()); + $conn = $this->connectionPool->getConnectionForTable('tx_vitec_ssrhealthlog'); $conn->insert('tx_vitec_ssrhealthlog', $row); $conn->executeStatement( @@ -66,6 +81,108 @@ final class SsrHealthSampleCommand extends Command $row['cache_mb'], $row['response_ms'] )); + $prevOk = $previous === [] ? null : $previous[0]; + if ($prevOk !== null && $prevOk !== $row['ok']) { + if ($row['ok'] === 1) { + $this->sendAlert( + '[VITEC SSR] Entwarnung: SSR-Server wieder erreichbar', + "Die 5-Minuten-Messung erreicht den SSR-Server wieder (Status ok).\n\n" + . 'Zeitpunkt: ' . date('d.m.Y H:i:s') . "\n" + . 'Release: ' . ($row['release_id'] ?: '-') . "\n" + . 'Antwortzeit: ' . $row['response_ms'] . " ms\n\n" + . 'Aktueller Stand: Schalter vitec.ssr.enabled = ' + . ($this->flagService->isEnabledInSettings() ? 'AN' : 'AUS') + . ', Flag-Datei ' . ($this->flagService->isFlagSet() ? 'vorhanden (SSR aktiv)' : 'fehlt (SPA-Auslieferung)') . ".\n" + . "Falls der Auto-Failover ausgelöst wurde, bleibt die Seite bewusst auf SPA.\n" + . "Wieder einschalten: TYPO3-Backend > Site Management > Settings > SSR delivery auf AN,\n" + . "danach Komplett-Purge (vendor/bin/typo3 vitec:ssr-purge --all).\n", + $output + ); + } else { + $this->sendAlert( + '[VITEC SSR] ALARM: SSR-Server antwortet nicht', + "Die 5-Minuten-Messung konnte den SSR-Server nicht erreichen\n" + . "(oder /__health meldete einen Fehler).\n\n" + . 'Zeitpunkt: ' . date('d.m.Y H:i:s') . "\n" + . 'Endpoint: ' . $endpoint . "\n" + . 'Antwortzeit: ' . $row['response_ms'] . " ms\n\n" + . 'Solange die Flag-Datei existiert, leitet dev.vitec.com weiter zum SSR-Server. Nach ' + . $failoverAfter . ' Fehlmessungen in Folge (ca. ' . ($failoverAfter * 5) . " Minuten) schaltet\n" + . "der Auto-Failover automatisch auf SPA-Auslieferung um - die Seite bleibt erreichbar.\n\n" + . "Diese Mail kommt nur beim Zustandswechsel ok -> down, nicht bei jeder Messung.\n", + $output + ); + } + } + + if ($row['ok'] === 0 && $this->flagService->isFlagSet()) { + $consecutive = 1; + foreach ($previous as $ok) { + if ($ok !== 0) { + break; + } + $consecutive++; + } + if ($consecutive >= $failoverAfter) { + try { + $this->flagService->disableSsr(); + $output->writeln(sprintf( + 'AUTO-FAILOVER after %d consecutive misses: vitec.ssr.enabled=false, caches flushed, flag removed', + $consecutive + )); + $this->sendAlert( + '[VITEC SSR] AUTO-FAILOVER: Seite auf SPA umgeschaltet', + 'AUTO-FAILOVER AUSGELÖST: ' . $consecutive . " Messungen in Folge down.\n\n" + . "dev.vitec.com liefert ab sofort die SPA aus (Rendering im Browser).\n" + . "Für Besucher bleibt die Seite erreichbar.\n\n" + . "Was automatisch passiert ist:\n" + . "- Site-Setting vitec.ssr.enabled auf false gestellt (settings.yaml)\n" + . "- Caches geleert\n" + . "- Flag-Datei var/vitec-ssr-on.flag gelöscht\n\n" + . "Wieder einschalten (bewusst NUR von Hand), sobald der SSR-Server wieder läuft:\n" + . "1. Auf dem VPS prüfen: systemctl status vitec-ssr und /__health\n" + . "2. TYPO3-Backend > Site Management > Settings > SSR delivery wieder AN\n" + . "3. Komplett-Purge: vendor/bin/typo3 vitec:ssr-purge --all\n", + $output + ); + } catch (\Throwable $e) { + $output->writeln('auto-failover FAILED: ' . $e->getMessage() . ''); + $this->sendAlert( + '[VITEC SSR] FEHLER: Auto-Failover fehlgeschlagen', + "Der Auto-Failover sollte auslösen, ist aber fehlgeschlagen:\n" + . $e->getMessage() . "\n\n" + . "Bitte von Hand prüfen: Site-Setting vitec.ssr.enabled, Datei var/vitec-ssr-on.flag.\n" + . "Not-Aus ohne TYPO3: Flag-Datei löschen UND Setting auf false stellen\n" + . "(sonst legt die SyncSsrFlag-Middleware die Flag-Datei sofort wieder an).\n", + $output + ); + } + } + } + return Command::SUCCESS; } + + /** + * Mails are best effort: a mail problem must never break the sampler + * (the cron would stop logging history). Recipients come from + * SSR_ALERT_EMAIL in var/vitec-ssr-purge.env - unset means "alerts + * off", which the output states so the cron log shows why. + */ + private function sendAlert(string $subject, string $body, OutputInterface $output): void + { + $recipients = GeneralUtility::trimExplode(',', $this->purgeService->getAlertEmail(), true); + if ($recipients === []) { + $output->writeln('transition detected but SSR_ALERT_EMAIL is not configured - no mail sent'); + return; + } + try { + $mail = GeneralUtility::makeInstance(MailMessage::class); + $mail->to(...$recipients)->subject($subject)->text($body); + GeneralUtility::makeInstance(Mailer::class)->send($mail); + $output->writeln('alert mail sent: ' . $subject . ' -> ' . implode(', ', $recipients)); + } catch (\Throwable $e) { + $output->writeln('alert mail failed: ' . $e->getMessage() . ''); + } + } } diff --git a/packages/vitec/Classes/Service/SsrFlagService.php b/packages/vitec/Classes/Service/SsrFlagService.php index 00358bf..1f3abba 100644 --- a/packages/vitec/Classes/Service/SsrFlagService.php +++ b/packages/vitec/Classes/Service/SsrFlagService.php @@ -4,9 +4,11 @@ declare(strict_types=1); namespace Evomedien\Vitec\Service; +use TYPO3\CMS\Core\Cache\CacheManager; use TYPO3\CMS\Core\Core\Environment; use TYPO3\CMS\Core\Exception\SiteNotFoundException; use TYPO3\CMS\Core\Site\SiteFinder; +use TYPO3\CMS\Core\Utility\GeneralUtility; /** * Keeps the SSR routing flag file in sync with the site setting @@ -91,6 +93,48 @@ final class SsrFlagService return $desired; } + /** + * Auto-failover (vitec:ssr-health): turns SSR delivery hard OFF. + * + * Removing only the flag file would not stick - the self-healing + * SyncSsrFlag middleware recreates it from the still-true site setting + * on the very next request. So the setting itself is flipped to false + * in config/sites//settings.yaml (the file the backend + * settings editor writes, flat dotted keys), caches are flushed so the + * compiled site settings forget the old value, and only then the flag + * is removed. Re-enabling is deliberately manual (backend switch or + * settings.yaml + cache flush). + */ + public function disableSsr(): void + { + $changed = false; + foreach ($this->siteFinder->getAllSites() as $site) { + $file = Environment::getConfigPath() . '/sites/' . $site->getIdentifier() . '/settings.yaml'; + if (!is_file($file)) { + continue; + } + $yaml = (string)file_get_contents($file); + $new = preg_replace('/^(vitec\.ssr\.enabled\s*:).*$/m', '$1 false', $yaml, 1, $count); + if ($new !== null && $count > 0 && $new !== $yaml) { + if (file_put_contents($file, $new) === false) { + throw new \RuntimeException('Could not write ' . $file, 1762800001); + } + $changed = true; + } + } + if (!$changed && $this->isEnabledInSettings()) { + throw new \RuntimeException( + 'vitec.ssr.enabled not found in any config/sites/*/settings.yaml - cannot disable it, ' + . 'and the SyncSsrFlag middleware would recreate the flag from the setting', + 1762800002 + ); + } + if ($changed) { + GeneralUtility::makeInstance(CacheManager::class)->flushCaches(); + } + @unlink($this->getFlagPath()); + } + /** * Sync variant for the request hot path: never lets an IO hiccup * (read-only FS, concurrent unlink ...) break request handling. diff --git a/packages/vitec/Classes/Service/SsrPurgeService.php b/packages/vitec/Classes/Service/SsrPurgeService.php index 96edae7..e4eac19 100644 --- a/packages/vitec/Classes/Service/SsrPurgeService.php +++ b/packages/vitec/Classes/Service/SsrPurgeService.php @@ -13,7 +13,8 @@ use TYPO3\CMS\Core\Http\RequestFactory; * keeps a log of every attempt in tx_vitec_ssrpurgelog. * * Endpoint + secret live OUTSIDE the repository in var/vitec-ssr-purge.env - * (KEY=VALUE lines: SSR_PURGE_ENDPOINT, SSR_PURGE_SECRET, SSR_HEALTH_ENDPOINT) + * (KEY=VALUE lines: SSR_PURGE_ENDPOINT, SSR_PURGE_SECRET, SSR_HEALTH_ENDPOINT, + * SSR_ALERT_EMAIL, SSR_FAILOVER_AFTER) * - same pattern as the routing flag file, nothing secret in git. */ final class SsrPurgeService @@ -41,6 +42,25 @@ final class SsrPurgeService return $this->getConfig()['SSR_HEALTH_ENDPOINT'] ?? ''; } + /** + * Comma-separated recipient list for vitec:ssr-health transition and + * failover alerts (SSR_ALERT_EMAIL); empty string = alerting off. + */ + public function getAlertEmail(): string + { + return trim($this->getConfig()['SSR_ALERT_EMAIL'] ?? ''); + } + + /** + * Consecutive DOWN samples after which vitec:ssr-health fails over to + * the SPA (SSR_FAILOVER_AFTER, default 2 = ~10 minutes at the 5-minute + * cron cadence). Never below 1. + */ + public function getFailoverAfter(): int + { + return max(1, (int)($this->getConfig()['SSR_FAILOVER_AFTER'] ?? 2)); + } + /** * Normalizes, dedupes and sends the paths; logs the outcome. * Never throws - editors' saves must not fail on a purge problem.