orascheandClaude Opus 5.5
90cb2faaf9
New site setting vitec.ssr.alertEmails (Site Management, Settings, VITEC; one address per entry). vitec:ssr-health reads the recipients from there, invalid entries are skipped and logged, SSR_ALERT_EMAIL in the env file stays as fallback. The cockpit shows the recipients in the live band. Pre-filled with the address that was in the env file. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
235 lines
12 KiB
PHP
235 lines
12 KiB
PHP
<?php
|
||
|
||
declare(strict_types=1);
|
||
|
||
namespace Evomedien\Vitec\Command;
|
||
|
||
use Evomedien\Vitec\Service\SsrFlagService;
|
||
use Evomedien\Vitec\Service\SsrPurgeService;
|
||
use Symfony\Component\Console\Attribute\AsCommand;
|
||
use Symfony\Component\Console\Command\Command;
|
||
use Symfony\Component\Console\Input\InputInterface;
|
||
use Symfony\Component\Console\Output\OutputInterface;
|
||
use TYPO3\CMS\Core\Database\ConnectionPool;
|
||
use TYPO3\CMS\Core\Http\RequestFactory;
|
||
use TYPO3\CMS\Core\Mail\Mailer;
|
||
use TYPO3\CMS\Core\Mail\MailMessage;
|
||
use TYPO3\CMS\Core\Utility\GeneralUtility;
|
||
|
||
#[AsCommand(
|
||
name: 'vitec:ssr-health',
|
||
description: 'Sample the SSR node /__health into tx_vitec_ssrhealthlog; mails on ok/down transitions, auto-failover to SPA after N consecutive misses',
|
||
)]
|
||
final class SsrHealthSampleCommand extends Command
|
||
{
|
||
private const KEEP_DAYS = 14;
|
||
|
||
public function __construct(
|
||
private readonly SsrPurgeService $purgeService,
|
||
private readonly SsrFlagService $flagService,
|
||
private readonly RequestFactory $requestFactory,
|
||
private readonly ConnectionPool $connectionPool,
|
||
) {
|
||
parent::__construct();
|
||
}
|
||
|
||
protected function execute(InputInterface $input, OutputInterface $output): int
|
||
{
|
||
$endpoint = $this->purgeService->getHealthEndpoint();
|
||
if ($endpoint === '') {
|
||
$output->writeln('<error>SSR_HEALTH_ENDPOINT missing in var/vitec-ssr-purge.env</error>');
|
||
return Command::INVALID;
|
||
}
|
||
$row = ['crdate' => time(), 'ok' => 0, 'release_id' => '', 'cached_pages' => 0, 'cache_mb' => 0, 'response_ms' => 0];
|
||
$start = microtime(true);
|
||
try {
|
||
$response = $this->requestFactory->request($endpoint, 'GET', [
|
||
'timeout' => 4, 'connect_timeout' => 2, 'http_errors' => false,
|
||
]);
|
||
$row['response_ms'] = (int)round((microtime(true) - $start) * 1000);
|
||
if ($response->getStatusCode() === 200) {
|
||
$data = json_decode((string)$response->getBody(), true);
|
||
$row['ok'] = !empty($data['ok']) ? 1 : 0;
|
||
$row['release_id'] = substr((string)($data['release']['release'] ?? ''), 0, 100);
|
||
$row['cached_pages'] = (int)($data['cachedPages'] ?? 0);
|
||
$row['cache_mb'] = (int)($data['cacheMB'] ?? 0);
|
||
}
|
||
} catch (\Throwable $e) {
|
||
$row['response_ms'] = (int)round((microtime(true) - $start) * 1000);
|
||
}
|
||
|
||
// Cumulative node counters from /__stats (-1 = no data this sample,
|
||
// e.g. node down or release without the endpoint). The errors tile
|
||
// computes 24h deltas from these, skipping -1 rows; the counters
|
||
// reset whenever the node restarts, which the delta logic clamps.
|
||
$stats = [
|
||
'stat_hit' => -1, 'stat_miss' => -1, 'stat_stale' => -1,
|
||
'stat_bypass' => -1, 'stat_render_errors' => -1, 'stat_backend_errors' => -1,
|
||
];
|
||
if ($row['ok'] === 1 && ($statsEndpoint = $this->purgeService->getStatsEndpoint()) !== '') {
|
||
try {
|
||
$statsResponse = $this->requestFactory->request($statsEndpoint, 'GET', [
|
||
'timeout' => 4, 'connect_timeout' => 2, 'http_errors' => false,
|
||
]);
|
||
if ($statsResponse->getStatusCode() === 200) {
|
||
$s = json_decode((string)$statsResponse->getBody(), true) ?: [];
|
||
$stats = [
|
||
'stat_hit' => (int)($s['hit'] ?? -1),
|
||
'stat_miss' => (int)($s['miss'] ?? -1),
|
||
'stat_stale' => (int)($s['stale'] ?? -1),
|
||
'stat_bypass' => (int)($s['bypass'] ?? -1),
|
||
'stat_render_errors' => (int)($s['renderErrors'] ?? -1),
|
||
'stat_backend_errors' => (int)($s['backendErrors'] ?? -1),
|
||
];
|
||
}
|
||
} catch (\Throwable) {
|
||
// Stats are optional - the health sample must never fail on them.
|
||
}
|
||
}
|
||
$row = array_merge($row, $stats);
|
||
// Previous samples, newest first, read BEFORE this one is inserted:
|
||
// [0] is the state every transition decision compares against.
|
||
$failoverAfter = $this->purgeService->getFailoverAfter();
|
||
$previous = array_map('intval', $this->connectionPool
|
||
->getQueryBuilderForTable('tx_vitec_ssrhealthlog')
|
||
->select('ok')->from('tx_vitec_ssrhealthlog')
|
||
->orderBy('crdate', 'DESC')->addOrderBy('uid', 'DESC')
|
||
->setMaxResults($failoverAfter)
|
||
->executeQuery()->fetchFirstColumn());
|
||
|
||
$conn = $this->connectionPool->getConnectionForTable('tx_vitec_ssrhealthlog');
|
||
$conn->insert('tx_vitec_ssrhealthlog', $row);
|
||
$conn->executeStatement(
|
||
'DELETE FROM tx_vitec_ssrhealthlog WHERE crdate < ?',
|
||
[time() - self::KEEP_DAYS * 86400]
|
||
);
|
||
$output->writeln(sprintf(
|
||
'%s release=%s pages=%d cache=%dMB %dms',
|
||
$row['ok'] ? '<info>ok</info>' : '<error>DOWN</error>',
|
||
$row['release_id'] ?: '-',
|
||
$row['cached_pages'],
|
||
$row['cache_mb'],
|
||
$row['response_ms']
|
||
));
|
||
if ($row['stat_hit'] >= 0) {
|
||
$output->writeln(sprintf(
|
||
'stats: hit=%d miss=%d stale=%d bypass=%d renderErrors=%d backendErrors=%d',
|
||
$row['stat_hit'],
|
||
$row['stat_miss'],
|
||
$row['stat_stale'],
|
||
$row['stat_bypass'],
|
||
$row['stat_render_errors'],
|
||
$row['stat_backend_errors']
|
||
));
|
||
}
|
||
$prevOk = $previous === [] ? null : $previous[0];
|
||
if ($prevOk !== null && $prevOk !== $row['ok']) {
|
||
if ($row['ok'] === 1) {
|
||
$this->sendAlert(
|
||
'[VITEC SSR] Recovered: SSR server reachable again',
|
||
"The 5-minute health check reaches the SSR server again (status ok).\n\n"
|
||
. 'Time: ' . date('d.m.Y H:i:s') . "\n"
|
||
. 'Release: ' . ($row['release_id'] ?: '-') . "\n"
|
||
. 'Response time: ' . $row['response_ms'] . " ms\n\n"
|
||
. 'Current state: switch vitec.ssr.enabled = '
|
||
. ($this->flagService->isEnabledInSettings() ? 'ON' : 'OFF')
|
||
. ', flag file ' . ($this->flagService->isFlagSet() ? 'present (SSR active)' : 'missing (SPA delivery)') . ".\n"
|
||
. "If the auto-failover was triggered, the site deliberately stays on SPA.\n"
|
||
. "Re-enable: TYPO3 backend > Site Management > Settings > SSR delivery ON,\n"
|
||
. "then a full purge (vendor/bin/typo3 vitec:ssr-purge --all).\n",
|
||
$output
|
||
);
|
||
} else {
|
||
$this->sendAlert(
|
||
'[VITEC SSR] ALERT: SSR server not responding',
|
||
"The 5-minute health check could not reach the SSR server\n"
|
||
. "(or /__health reported an error).\n\n"
|
||
. 'Time: ' . date('d.m.Y H:i:s') . "\n"
|
||
. 'Endpoint: ' . $endpoint . "\n"
|
||
. 'Response time: ' . $row['response_ms'] . " ms\n\n"
|
||
. 'As long as the flag file exists, dev.vitec.com keeps routing to the SSR server. After '
|
||
. $failoverAfter . ' failed checks in a row (about ' . ($failoverAfter * 5) . " minutes) the\n"
|
||
. "auto-failover switches to SPA delivery automatically - the site stays reachable.\n\n"
|
||
. "This mail is sent only on the ok -> down transition, not for every check.\n",
|
||
$output
|
||
);
|
||
}
|
||
}
|
||
|
||
if ($row['ok'] === 0 && $this->flagService->isFlagSet()) {
|
||
$consecutive = 1;
|
||
foreach ($previous as $ok) {
|
||
if ($ok !== 0) {
|
||
break;
|
||
}
|
||
$consecutive++;
|
||
}
|
||
if ($consecutive >= $failoverAfter) {
|
||
try {
|
||
$this->flagService->disableSsr();
|
||
$output->writeln(sprintf(
|
||
'<error>AUTO-FAILOVER after %d consecutive misses: vitec.ssr.enabled=false, caches flushed, flag removed</error>',
|
||
$consecutive
|
||
));
|
||
$this->sendAlert(
|
||
'[VITEC SSR] AUTO-FAILOVER: site switched to SPA',
|
||
'AUTO-FAILOVER TRIGGERED: ' . $consecutive . " consecutive DOWN checks.\n\n"
|
||
. "dev.vitec.com now serves the SPA (rendering in the browser).\n"
|
||
. "The site stays available for visitors.\n\n"
|
||
. "What happened automatically:\n"
|
||
. "- site setting vitec.ssr.enabled set to false (settings.yaml)\n"
|
||
. "- caches flushed\n"
|
||
. "- flag file var/vitec-ssr-on.flag removed\n\n"
|
||
. "Re-enable (deliberately MANUAL only) once the SSR server is back:\n"
|
||
. "1. Check on the VPS: systemctl status vitec-ssr and /__health\n"
|
||
. "2. TYPO3 backend > Site Management > Settings > SSR delivery back ON\n"
|
||
. "3. Full purge: vendor/bin/typo3 vitec:ssr-purge --all\n",
|
||
$output
|
||
);
|
||
} catch (\Throwable $e) {
|
||
$output->writeln('<error>auto-failover FAILED: ' . $e->getMessage() . '</error>');
|
||
$this->sendAlert(
|
||
'[VITEC SSR] ERROR: auto-failover FAILED',
|
||
"The auto-failover should have triggered but failed:\n"
|
||
. $e->getMessage() . "\n\n"
|
||
. "Please check manually: site setting vitec.ssr.enabled, file var/vitec-ssr-on.flag.\n"
|
||
. "Emergency off without TYPO3: delete the flag file AND set the setting to false\n"
|
||
. "(otherwise the SyncSsrFlag middleware recreates the flag file immediately).\n",
|
||
$output
|
||
);
|
||
}
|
||
}
|
||
}
|
||
|
||
return Command::SUCCESS;
|
||
}
|
||
|
||
/**
|
||
* Mails are best effort: a mail problem must never break the sampler
|
||
* (the cron would stop logging history). Recipients come from the site
|
||
* setting vitec.ssr.alertEmails (Site Management › Settings › VITEC),
|
||
* fallback SSR_ALERT_EMAIL in var/vitec-ssr-purge.env - none at all
|
||
* means "alerts off", which the output states so the cron log shows why.
|
||
*/
|
||
private function sendAlert(string $subject, string $body, OutputInterface $output): void
|
||
{
|
||
$alert = $this->purgeService->getAlertRecipients();
|
||
if ($alert['invalid'] !== []) {
|
||
$output->writeln('<comment>skipped invalid alert addresses: ' . implode(', ', $alert['invalid']) . '</comment>');
|
||
}
|
||
$recipients = $alert['recipients'];
|
||
if ($recipients === []) {
|
||
$output->writeln('<comment>transition detected but no alert recipients are configured (Site settings › VITEC or SSR_ALERT_EMAIL) - no mail sent</comment>');
|
||
return;
|
||
}
|
||
try {
|
||
$mail = GeneralUtility::makeInstance(MailMessage::class);
|
||
$mail->to(...$recipients)->subject($subject)->text($body);
|
||
GeneralUtility::makeInstance(Mailer::class)->send($mail);
|
||
$output->writeln('alert mail sent: ' . $subject . ' -> ' . implode(', ', $recipients));
|
||
} catch (\Throwable $e) {
|
||
$output->writeln('<error>alert mail failed: ' . $e->getMessage() . '</error>');
|
||
}
|
||
}
|
||
}
|