addOption( 'test-mail', null, InputOption::VALUE_OPTIONAL, 'Send a sample alert mail through the real alert path and exit (no health sample). Optional address; default: the configured alert recipients', false ); } protected function execute(InputInterface $input, OutputInterface $output): int { $testMail = $input->getOption('test-mail'); if ($testMail !== false) { return $this->sendTestMail((string)$testMail, $output); } $endpoint = $this->purgeService->getHealthEndpoint(); if ($endpoint === '') { $output->writeln('SSR_HEALTH_ENDPOINT missing in var/vitec-ssr-purge.env'); return Command::INVALID; } $row = ['crdate' => time(), 'ok' => 0, 'release_id' => '', 'cached_pages' => 0, 'cache_mb' => 0, 'response_ms' => 0]; $start = microtime(true); try { $response = $this->requestFactory->request($endpoint, 'GET', [ 'timeout' => 4, 'connect_timeout' => 2, 'http_errors' => false, ]); $row['response_ms'] = (int)round((microtime(true) - $start) * 1000); if ($response->getStatusCode() === 200) { $data = json_decode((string)$response->getBody(), true); $row['ok'] = !empty($data['ok']) ? 1 : 0; $row['release_id'] = substr((string)($data['release']['release'] ?? ''), 0, 100); $row['cached_pages'] = (int)($data['cachedPages'] ?? 0); $row['cache_mb'] = (int)($data['cacheMB'] ?? 0); } } catch (\Throwable $e) { $row['response_ms'] = (int)round((microtime(true) - $start) * 1000); } // Cumulative node counters from /__stats (-1 = no data this sample, // e.g. node down or release without the endpoint). The errors tile // computes 24h deltas from these, skipping -1 rows; the counters // reset whenever the node restarts, which the delta logic clamps. $stats = [ 'stat_hit' => -1, 'stat_miss' => -1, 'stat_stale' => -1, 'stat_bypass' => -1, 'stat_render_errors' => -1, 'stat_backend_errors' => -1, ]; if ($row['ok'] === 1 && ($statsEndpoint = $this->purgeService->getStatsEndpoint()) !== '') { try { $statsResponse = $this->requestFactory->request($statsEndpoint, 'GET', [ 'timeout' => 4, 'connect_timeout' => 2, 'http_errors' => false, ]); if ($statsResponse->getStatusCode() === 200) { $s = json_decode((string)$statsResponse->getBody(), true) ?: []; $stats = [ 'stat_hit' => (int)($s['hit'] ?? -1), 'stat_miss' => (int)($s['miss'] ?? -1), 'stat_stale' => (int)($s['stale'] ?? -1), 'stat_bypass' => (int)($s['bypass'] ?? -1), 'stat_render_errors' => (int)($s['renderErrors'] ?? -1), 'stat_backend_errors' => (int)($s['backendErrors'] ?? -1), ]; } } catch (\Throwable) { // Stats are optional - the health sample must never fail on them. } } $row = array_merge($row, $stats); // Previous samples, newest first, read BEFORE this one is inserted: // [0] is the state every transition decision compares against. $failoverAfter = $this->purgeService->getFailoverAfter(); $previous = array_map('intval', $this->connectionPool ->getQueryBuilderForTable('tx_vitec_ssrhealthlog') ->select('ok')->from('tx_vitec_ssrhealthlog') ->orderBy('crdate', 'DESC')->addOrderBy('uid', 'DESC') ->setMaxResults($failoverAfter) ->executeQuery()->fetchFirstColumn()); $conn = $this->connectionPool->getConnectionForTable('tx_vitec_ssrhealthlog'); $conn->insert('tx_vitec_ssrhealthlog', $row); $conn->executeStatement( 'DELETE FROM tx_vitec_ssrhealthlog WHERE crdate < ?', [time() - self::KEEP_DAYS * 86400] ); $output->writeln(sprintf( '%s release=%s pages=%d cache=%dMB %dms', $row['ok'] ? 'ok' : 'DOWN', $row['release_id'] ?: '-', $row['cached_pages'], $row['cache_mb'], $row['response_ms'] )); if ($row['stat_hit'] >= 0) { $output->writeln(sprintf( 'stats: hit=%d miss=%d stale=%d bypass=%d renderErrors=%d backendErrors=%d', $row['stat_hit'], $row['stat_miss'], $row['stat_stale'], $row['stat_bypass'], $row['stat_render_errors'], $row['stat_backend_errors'] )); } $prevOk = $previous === [] ? null : $previous[0]; if ($prevOk !== null && $prevOk !== $row['ok']) { if ($row['ok'] === 1) { $this->sendAlert( '[VITEC SSR] Recovered: SSR server reachable again', "The 5-minute health check reaches the SSR server again (status ok).\n\n" . 'Time: ' . date('d.m.Y H:i:s') . "\n" . 'Release: ' . ($row['release_id'] ?: '-') . "\n" . 'Response time: ' . $row['response_ms'] . " ms\n\n" . 'Current state: switch vitec.ssr.enabled = ' . ($this->flagService->isEnabledInSettings() ? 'ON' : 'OFF') . ', flag file ' . ($this->flagService->isFlagSet() ? 'present (SSR active)' : 'missing (SPA delivery)') . ".\n" . "If the auto-failover was triggered, the site deliberately stays on SPA.\n" . "Re-enable: TYPO3 backend > Site Management > Settings > SSR delivery ON,\n" . "then a full purge (vendor/bin/typo3 vitec:ssr-purge --all).\n", $output ); } else { $this->sendAlert( '[VITEC SSR] ALERT: SSR server not responding', "The 5-minute health check could not reach the SSR server\n" . "(or /__health reported an error).\n\n" . 'Time: ' . date('d.m.Y H:i:s') . "\n" . 'Endpoint: ' . $endpoint . "\n" . 'Response time: ' . $row['response_ms'] . " ms\n\n" . 'As long as the flag file exists, dev.vitec.com keeps routing to the SSR server. After ' . $failoverAfter . ' failed checks in a row (about ' . ($failoverAfter * 5) . " minutes) the\n" . "auto-failover switches to SPA delivery automatically - the site stays reachable.\n\n" . "This mail is sent only on the ok -> down transition, not for every check.\n", $output ); } } if ($row['ok'] === 0 && $this->flagService->isFlagSet()) { $consecutive = 1; foreach ($previous as $ok) { if ($ok !== 0) { break; } $consecutive++; } if ($consecutive >= $failoverAfter) { try { $this->flagService->disableSsr(); $output->writeln(sprintf( 'AUTO-FAILOVER after %d consecutive misses: vitec.ssr.enabled=false, caches flushed, flag removed', $consecutive )); $this->sendAlert( '[VITEC SSR] AUTO-FAILOVER: site switched to SPA', 'AUTO-FAILOVER TRIGGERED: ' . $consecutive . " consecutive DOWN checks.\n\n" . "dev.vitec.com now serves the SPA (rendering in the browser).\n" . "The site stays available for visitors.\n\n" . "What happened automatically:\n" . "- site setting vitec.ssr.enabled set to false (settings.yaml)\n" . "- caches flushed\n" . "- flag file var/vitec-ssr-on.flag removed\n\n" . "Re-enable (deliberately MANUAL only) once the SSR server is back:\n" . "1. Check on the VPS: systemctl status vitec-ssr and /__health\n" . "2. TYPO3 backend > Site Management > Settings > SSR delivery back ON\n" . "3. Full purge: vendor/bin/typo3 vitec:ssr-purge --all\n", $output ); } catch (\Throwable $e) { $output->writeln('auto-failover FAILED: ' . $e->getMessage() . ''); $this->sendAlert( '[VITEC SSR] ERROR: auto-failover FAILED', "The auto-failover should have triggered but failed:\n" . $e->getMessage() . "\n\n" . "Please check manually: site setting vitec.ssr.enabled, file var/vitec-ssr-on.flag.\n" . "Emergency off without TYPO3: delete the flag file AND set the setting to false\n" . "(otherwise the SyncSsrFlag middleware recreates the flag file immediately).\n", $output ); } } } return Command::SUCCESS; } /** * Deliverability check (e.g. mail-tester.com): same sender, mailer and * plain-text format as a real alert, clearly marked as a test. */ private function sendTestMail(string $address, OutputInterface $output): int { $override = []; if ($address !== '') { if (!GeneralUtility::validEmail($address)) { $output->writeln('invalid address: ' . $address . ''); return Command::INVALID; } $override = [$address]; } $this->sendAlert( '[VITEC SSR] TEST: alert mail check', "This is a test of the VITEC SSR alert mail - nothing is wrong.\n\n" . "Real alerts look like this one and are sent when the 5-minute health\n" . "check sees the SSR server go down or come back, and when the\n" . "auto-failover switches the site to SPA delivery.\n\n" . 'Time: ' . date('d.m.Y H:i:s') . "\n" . 'Current state: switch vitec.ssr.enabled = ' . ($this->flagService->isEnabledInSettings() ? 'ON' : 'OFF') . ', flag file ' . ($this->flagService->isFlagSet() ? 'present (SSR active)' : 'missing (SPA delivery)') . ".\n\n" . "Recipients are maintained in the TYPO3 backend:\n" . "Site Management > Settings > VITEC > SSR alert mail recipients.\n", $output, $override ); return Command::SUCCESS; } /** * Mails are best effort: a mail problem must never break the sampler * (the cron would stop logging history). Recipients come from the site * setting vitec.ssr.alertEmails (Site Management › Settings › VITEC), * fallback SSR_ALERT_EMAIL in var/vitec-ssr-purge.env - none at all * means "alerts off", which the output states so the cron log shows why. */ private function sendAlert(string $subject, string $body, OutputInterface $output, array $override = []): void { $alert = $this->purgeService->getAlertRecipients(); if ($alert['invalid'] !== [] && $override === []) { $output->writeln('skipped invalid alert addresses: ' . implode(', ', $alert['invalid']) . ''); } $recipients = $override !== [] ? $override : $alert['recipients']; if ($recipients === []) { $output->writeln('transition detected but no alert recipients are configured (Site settings › VITEC or SSR_ALERT_EMAIL) - no mail sent'); return; } try { $mail = GeneralUtility::makeInstance(MailMessage::class); $mail->to(...$recipients)->subject($subject)->text($body); GeneralUtility::makeInstance(Mailer::class)->send($mail); $output->writeln('alert mail sent: ' . $subject . ' -> ' . implode(', ', $recipients)); } catch (\Throwable $e) { $output->writeln('alert mail failed: ' . $e->getMessage() . ''); } } }