189 lines
9.4 KiB
PHP
189 lines
9.4 KiB
PHP
<?php
|
|
|
|
declare(strict_types=1);
|
|
|
|
namespace Evomedien\Vitec\Command;
|
|
|
|
use Evomedien\Vitec\Service\SsrFlagService;
|
|
use Evomedien\Vitec\Service\SsrPurgeService;
|
|
use Symfony\Component\Console\Attribute\AsCommand;
|
|
use Symfony\Component\Console\Command\Command;
|
|
use Symfony\Component\Console\Input\InputInterface;
|
|
use Symfony\Component\Console\Output\OutputInterface;
|
|
use TYPO3\CMS\Core\Database\ConnectionPool;
|
|
use TYPO3\CMS\Core\Http\RequestFactory;
|
|
use TYPO3\CMS\Core\Mail\Mailer;
|
|
use TYPO3\CMS\Core\Mail\MailMessage;
|
|
use TYPO3\CMS\Core\Utility\GeneralUtility;
|
|
|
|
#[AsCommand(
|
|
name: 'vitec:ssr-health',
|
|
description: 'Sample the SSR node /__health into tx_vitec_ssrhealthlog; mails on ok/down transitions, auto-failover to SPA after N consecutive misses',
|
|
)]
|
|
final class SsrHealthSampleCommand extends Command
|
|
{
|
|
private const KEEP_DAYS = 14;
|
|
|
|
public function __construct(
|
|
private readonly SsrPurgeService $purgeService,
|
|
private readonly SsrFlagService $flagService,
|
|
private readonly RequestFactory $requestFactory,
|
|
private readonly ConnectionPool $connectionPool,
|
|
) {
|
|
parent::__construct();
|
|
}
|
|
|
|
protected function execute(InputInterface $input, OutputInterface $output): int
|
|
{
|
|
$endpoint = $this->purgeService->getHealthEndpoint();
|
|
if ($endpoint === '') {
|
|
$output->writeln('<error>SSR_HEALTH_ENDPOINT missing in var/vitec-ssr-purge.env</error>');
|
|
return Command::INVALID;
|
|
}
|
|
$row = ['crdate' => time(), 'ok' => 0, 'release_id' => '', 'cached_pages' => 0, 'cache_mb' => 0, 'response_ms' => 0];
|
|
$start = microtime(true);
|
|
try {
|
|
$response = $this->requestFactory->request($endpoint, 'GET', [
|
|
'timeout' => 4, 'connect_timeout' => 2, 'http_errors' => false,
|
|
]);
|
|
$row['response_ms'] = (int)round((microtime(true) - $start) * 1000);
|
|
if ($response->getStatusCode() === 200) {
|
|
$data = json_decode((string)$response->getBody(), true);
|
|
$row['ok'] = !empty($data['ok']) ? 1 : 0;
|
|
$row['release_id'] = substr((string)($data['release']['release'] ?? ''), 0, 100);
|
|
$row['cached_pages'] = (int)($data['cachedPages'] ?? 0);
|
|
$row['cache_mb'] = (int)($data['cacheMB'] ?? 0);
|
|
}
|
|
} catch (\Throwable $e) {
|
|
$row['response_ms'] = (int)round((microtime(true) - $start) * 1000);
|
|
}
|
|
// Previous samples, newest first, read BEFORE this one is inserted:
|
|
// [0] is the state every transition decision compares against.
|
|
$failoverAfter = $this->purgeService->getFailoverAfter();
|
|
$previous = array_map('intval', $this->connectionPool
|
|
->getQueryBuilderForTable('tx_vitec_ssrhealthlog')
|
|
->select('ok')->from('tx_vitec_ssrhealthlog')
|
|
->orderBy('crdate', 'DESC')->addOrderBy('uid', 'DESC')
|
|
->setMaxResults($failoverAfter)
|
|
->executeQuery()->fetchFirstColumn());
|
|
|
|
$conn = $this->connectionPool->getConnectionForTable('tx_vitec_ssrhealthlog');
|
|
$conn->insert('tx_vitec_ssrhealthlog', $row);
|
|
$conn->executeStatement(
|
|
'DELETE FROM tx_vitec_ssrhealthlog WHERE crdate < ?',
|
|
[time() - self::KEEP_DAYS * 86400]
|
|
);
|
|
$output->writeln(sprintf(
|
|
'%s release=%s pages=%d cache=%dMB %dms',
|
|
$row['ok'] ? '<info>ok</info>' : '<error>DOWN</error>',
|
|
$row['release_id'] ?: '-',
|
|
$row['cached_pages'],
|
|
$row['cache_mb'],
|
|
$row['response_ms']
|
|
));
|
|
$prevOk = $previous === [] ? null : $previous[0];
|
|
if ($prevOk !== null && $prevOk !== $row['ok']) {
|
|
if ($row['ok'] === 1) {
|
|
$this->sendAlert(
|
|
'[VITEC SSR] Entwarnung: SSR-Server wieder erreichbar',
|
|
"Die 5-Minuten-Messung erreicht den SSR-Server wieder (Status ok).\n\n"
|
|
. 'Zeitpunkt: ' . date('d.m.Y H:i:s') . "\n"
|
|
. 'Release: ' . ($row['release_id'] ?: '-') . "\n"
|
|
. 'Antwortzeit: ' . $row['response_ms'] . " ms\n\n"
|
|
. 'Aktueller Stand: Schalter vitec.ssr.enabled = '
|
|
. ($this->flagService->isEnabledInSettings() ? 'AN' : 'AUS')
|
|
. ', Flag-Datei ' . ($this->flagService->isFlagSet() ? 'vorhanden (SSR aktiv)' : 'fehlt (SPA-Auslieferung)') . ".\n"
|
|
. "Falls der Auto-Failover ausgelöst wurde, bleibt die Seite bewusst auf SPA.\n"
|
|
. "Wieder einschalten: TYPO3-Backend > Site Management > Settings > SSR delivery auf AN,\n"
|
|
. "danach Komplett-Purge (vendor/bin/typo3 vitec:ssr-purge --all).\n",
|
|
$output
|
|
);
|
|
} else {
|
|
$this->sendAlert(
|
|
'[VITEC SSR] ALARM: SSR-Server antwortet nicht',
|
|
"Die 5-Minuten-Messung konnte den SSR-Server nicht erreichen\n"
|
|
. "(oder /__health meldete einen Fehler).\n\n"
|
|
. 'Zeitpunkt: ' . date('d.m.Y H:i:s') . "\n"
|
|
. 'Endpoint: ' . $endpoint . "\n"
|
|
. 'Antwortzeit: ' . $row['response_ms'] . " ms\n\n"
|
|
. 'Solange die Flag-Datei existiert, leitet dev.vitec.com weiter zum SSR-Server. Nach '
|
|
. $failoverAfter . ' Fehlmessungen in Folge (ca. ' . ($failoverAfter * 5) . " Minuten) schaltet\n"
|
|
. "der Auto-Failover automatisch auf SPA-Auslieferung um - die Seite bleibt erreichbar.\n\n"
|
|
. "Diese Mail kommt nur beim Zustandswechsel ok -> down, nicht bei jeder Messung.\n",
|
|
$output
|
|
);
|
|
}
|
|
}
|
|
|
|
if ($row['ok'] === 0 && $this->flagService->isFlagSet()) {
|
|
$consecutive = 1;
|
|
foreach ($previous as $ok) {
|
|
if ($ok !== 0) {
|
|
break;
|
|
}
|
|
$consecutive++;
|
|
}
|
|
if ($consecutive >= $failoverAfter) {
|
|
try {
|
|
$this->flagService->disableSsr();
|
|
$output->writeln(sprintf(
|
|
'<error>AUTO-FAILOVER after %d consecutive misses: vitec.ssr.enabled=false, caches flushed, flag removed</error>',
|
|
$consecutive
|
|
));
|
|
$this->sendAlert(
|
|
'[VITEC SSR] AUTO-FAILOVER: Seite auf SPA umgeschaltet',
|
|
'AUTO-FAILOVER AUSGELÖST: ' . $consecutive . " Messungen in Folge down.\n\n"
|
|
. "dev.vitec.com liefert ab sofort die SPA aus (Rendering im Browser).\n"
|
|
. "Für Besucher bleibt die Seite erreichbar.\n\n"
|
|
. "Was automatisch passiert ist:\n"
|
|
. "- Site-Setting vitec.ssr.enabled auf false gestellt (settings.yaml)\n"
|
|
. "- Caches geleert\n"
|
|
. "- Flag-Datei var/vitec-ssr-on.flag gelöscht\n\n"
|
|
. "Wieder einschalten (bewusst NUR von Hand), sobald der SSR-Server wieder läuft:\n"
|
|
. "1. Auf dem VPS prüfen: systemctl status vitec-ssr und /__health\n"
|
|
. "2. TYPO3-Backend > Site Management > Settings > SSR delivery wieder AN\n"
|
|
. "3. Komplett-Purge: vendor/bin/typo3 vitec:ssr-purge --all\n",
|
|
$output
|
|
);
|
|
} catch (\Throwable $e) {
|
|
$output->writeln('<error>auto-failover FAILED: ' . $e->getMessage() . '</error>');
|
|
$this->sendAlert(
|
|
'[VITEC SSR] FEHLER: Auto-Failover fehlgeschlagen',
|
|
"Der Auto-Failover sollte auslösen, ist aber fehlgeschlagen:\n"
|
|
. $e->getMessage() . "\n\n"
|
|
. "Bitte von Hand prüfen: Site-Setting vitec.ssr.enabled, Datei var/vitec-ssr-on.flag.\n"
|
|
. "Not-Aus ohne TYPO3: Flag-Datei löschen UND Setting auf false stellen\n"
|
|
. "(sonst legt die SyncSsrFlag-Middleware die Flag-Datei sofort wieder an).\n",
|
|
$output
|
|
);
|
|
}
|
|
}
|
|
}
|
|
|
|
return Command::SUCCESS;
|
|
}
|
|
|
|
/**
|
|
* Mails are best effort: a mail problem must never break the sampler
|
|
* (the cron would stop logging history). Recipients come from
|
|
* SSR_ALERT_EMAIL in var/vitec-ssr-purge.env - unset means "alerts
|
|
* off", which the output states so the cron log shows why.
|
|
*/
|
|
private function sendAlert(string $subject, string $body, OutputInterface $output): void
|
|
{
|
|
$recipients = GeneralUtility::trimExplode(',', $this->purgeService->getAlertEmail(), true);
|
|
if ($recipients === []) {
|
|
$output->writeln('<comment>transition detected but SSR_ALERT_EMAIL is not configured - no mail sent</comment>');
|
|
return;
|
|
}
|
|
try {
|
|
$mail = GeneralUtility::makeInstance(MailMessage::class);
|
|
$mail->to(...$recipients)->subject($subject)->text($body);
|
|
GeneralUtility::makeInstance(Mailer::class)->send($mail);
|
|
$output->writeln('alert mail sent: ' . $subject . ' -> ' . implode(', ', $recipients));
|
|
} catch (\Throwable $e) {
|
|
$output->writeln('<error>alert mail failed: ' . $e->getMessage() . '</error>');
|
|
}
|
|
}
|
|
}
|