Add Solr search: EXT:solr 14 integration and JSON search endpoint

Apache Solr 10 runs on a dedicated VPS behind a Caddy HTTPS proxy
(vitecsolr.evomedien.de) because the managed webserver only allows
outgoing standard ports. Credentials stay out of git: the site config
carries %env()% placeholders resolved via putenv() in the git-ignored
config/system/additional.php.

- composer: apache-solr-for-typo3/solr 14.0.0-RC1 (the only line
  compatible with TYPO3 14; final 14.0.0 will arrive via composer update)
- config.yaml: read connection https/443, core_en per language,
  env placeholder credentials; .gitignore covers additional.php
- setup.typoscript: config.index_enable = 1 (page indexing silently
  refuses without it), solr_pi_results JSON renderer registration,
  results highlighting, and content field extraction from tt_content
  rows - the headless JSON output has no TYPO3SEARCH markers, so the
  default page content extraction indexed an empty content field
- SearchJsonRenderer (renderer #27): JSON output for the search plugin
  on /search. GET q/page in; { query, page, resultsPerPage, numFound,
  totalPages, results[], suggestions } out. Disables the page cache per
  request: config.no_cache is gone in TYPO3 v14, and q/page are
  excluded from cHash, so cached variants would collide
- ext_localconf.php: q and page added to cacheHash excludedParameters
- SolrIndexCommand (vitec:solr-index): works the index queue from the
  CLI with connection diagnostics and a --debug single-step mode -
  EXT:solr 14 ships no console commands and the backend button indexes
  one item per click

packages/vitec/Classes/Command/SolrIndexCommand.php

@@ -0,0 +1,130 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Evomedien\Vitec\Command;
+
+use ApacheSolrForTypo3\Solr\ConnectionManager;
+use ApacheSolrForTypo3\Solr\Domain\Index\IndexService;
+use ApacheSolrForTypo3\Solr\Domain\Site\SiteRepository;
+use ApacheSolrForTypo3\Solr\IndexQueue\IndexingService;
+use ApacheSolrForTypo3\Solr\IndexQueue\Queue;
+use Symfony\Component\Console\Attribute\AsCommand;
+use Symfony\Component\Console\Command\Command;
+use Symfony\Component\Console\Input\InputInterface;
+use Symfony\Component\Console\Input\InputOption;
+use Symfony\Component\Console\Output\OutputInterface;
+use Throwable;
+use TYPO3\CMS\Core\Core\Bootstrap;
+use TYPO3\CMS\Core\Utility\GeneralUtility;
+
+/**
+ * Works the Solr index queue from the CLI - EXT:solr 14 ships no console
+ * commands, its indexing runs via backend module clicks (1 item each!) or a
+ * scheduler task. This command fills that gap for cron usage and prints the
+ * resolved connection state first, because a missing connection makes the
+ * indexer skip items silently (no error, no queue entry).
+ */
+#[AsCommand(
+    name: 'vitec:solr-index',
+    description: 'Work the Solr index queue and show connection diagnostics',
+)]
+class SolrIndexCommand extends Command
+{
+    protected function configure(): void
+    {
+        $this->addOption('limit', 'l', InputOption::VALUE_REQUIRED, 'Max queue items to index in this run', '20');
+        $this->addOption('root', 'r', InputOption::VALUE_REQUIRED, 'Root page uid of the site', '1');
+        $this->addOption('debug', 'd', InputOption::VALUE_NONE, 'Single-step the first queue item verbosely');
+    }
+
+    protected function execute(InputInterface $input, OutputInterface $output): int
+    {
+        Bootstrap::initializeBackendAuthentication();
+
+        $rootPageId = (int)$input->getOption('root');
+        $limit = (int)$input->getOption('limit');
+
+        $siteRepository = GeneralUtility::makeInstance(SiteRepository::class);
+        $site = $siteRepository->getSiteByRootPageId($rootPageId);
+
+        $configs = $site->getAllSolrConnectionConfigurations();
+        $output->writeln('Connection configurations: ' . count($configs));
+        foreach ($configs as $languageId => $config) {
+            $read = $config['read'];
+            $output->writeln(sprintf(
+                '  lang %d: %s://%s:%d%s core=%s auth=%s',
+                $languageId,
+                $read['scheme'],
+                $read['host'],
+                $read['port'],
+                $read['path'],
+                $read['core'],
+                $read['username'] !== '' ? 'yes' : 'no',
+            ));
+        }
+        if ($configs === []) {
+            $output->writeln('<error>No connection configuration resolved - check site config.</error>');
+            return Command::FAILURE;
+        }
+
+        if ($input->getOption('debug')) {
+            $queue = GeneralUtility::makeInstance(Queue::class);
+            $items = $queue->getItemsToIndex($site, 5);
+            $output->writeln('getItemsToIndex(5): ' . count($items) . ' items');
+            if ($items !== []) {
+                $first = $items[0];
+                $output->writeln('  first: type=' . $first->getType() . ' uid=' . $first->getRecordUid());
+                $svc = GeneralUtility::getContainer()->get(IndexingService::class);
+                $ref = new \ReflectionObject($svc);
+
+                $call = static function (string $method, array $args) use ($svc, $ref) {
+                    $m = $ref->getMethod($method);
+                    return $m->invokeArgs($svc, $args);
+                };
+
+                $conns = $call('getPageSolrConnections', [$first]);
+                $output->writeln('  getPageSolrConnections: ' . count($conns) . ' connections (langs: ' . implode(',', array_keys($conns)) . ')');
+                if ($conns === []) {
+                    return Command::SUCCESS;
+                }
+
+                $groups = $call('findUserGroupsForPage', [$first, 0]);
+                $output->writeln('  findUserGroupsForPage(lang 0): ' . json_encode($groups));
+
+                $accessRootline = $call('buildAccessRootline', [$first, 0, $groups[0] ?? 0]);
+                $params = $call('buildPageParameters', [$first]);
+                $instructions = new \ApacheSolrForTypo3\Solr\IndexQueue\IndexingInstructions(
+                    items: [$first],
+                    action: \ApacheSolrForTypo3\Solr\IndexQueue\IndexingInstructions::ACTION_INDEX_PAGE,
+                    language: 0,
+                    userGroup: (int)($groups[0] ?? 0),
+                    accessRootline: $accessRootline,
+                    parameters: $params,
+                );
+                $response = $call('executeSubRequest', [$first, 0, $instructions]);
+                if ($response === null) {
+                    $output->writeln('  executeSubRequest => NULL (siehe Log)');
+                } else {
+                    $output->writeln('  executeSubRequest => HTTP ' . $response->getStatusCode()
+                        . ' content-type=' . $response->getHeaderLine('Content-Type'));
+                    $body = (string)$response->getBody();
+                    $output->writeln('  body (600 Zeichen): ' . substr($body, 0, 600));
+                }
+            }
+            return Command::SUCCESS;
+        }
+
+        $indexService = GeneralUtility::makeInstance(IndexService::class, $site);
+        try {
+            $success = $indexService->indexItems($limit);
+        } catch (Throwable $e) {
+            $output->writeln('<error>indexItems threw: ' . $e->getMessage() . '</error>');
+            $output->writeln($e->getTraceAsString());
+            return Command::FAILURE;
+        }
+        $output->writeln('indexItems(' . $limit . ') returned: ' . ($success ? 'success' : 'with errors'));
+
+        return Command::SUCCESS;
+    }
+}

packages/vitec/Classes/DataProcessing/ContainerChildrenProcessor.php

@@ -20,6 +20,7 @@ use Evomedien\Vitec\UserFunc\CustomerlogosJsonRenderer;
 use Evomedien\Vitec\UserFunc\FormsJsonRenderer;
 use Evomedien\Vitec\UserFunc\ModelcardJsonRenderer;
 use Evomedien\Vitec\UserFunc\NewsJsonRenderer;
+use Evomedien\Vitec\UserFunc\SearchJsonRenderer;
 use TYPO3\CMS\Core\Database\Connection;
 use TYPO3\CMS\Core\Database\ConnectionPool;
 use TYPO3\CMS\Core\Utility\GeneralUtility;
@@ -132,6 +133,7 @@ final class ContainerChildrenProcessor implements DataProcessorInterface
         'vitec_contactform' => [FormsJsonRenderer::class, 'form'],
         'vitec_demoform' => [FormsJsonRenderer::class, 'form'],
         'vitec_helpdeskform' => [FormsJsonRenderer::class, 'form'],
+        'solr_pi_results' => [SearchJsonRenderer::class, 'search'],
         'news_pi1' => [NewsJsonRenderer::class, 'news'],
         'news_newsliststicky' => [NewsJsonRenderer::class, 'news'],
         'news_newsselectedlist' => [NewsJsonRenderer::class, 'news'],

packages/vitec/Classes/Service/ContentElementResolver.php

@@ -22,6 +22,7 @@ use Evomedien\Vitec\UserFunc\CustomerlogosJsonRenderer;
 use Evomedien\Vitec\UserFunc\FormsJsonRenderer;
 use Evomedien\Vitec\UserFunc\ModelcardJsonRenderer;
 use Evomedien\Vitec\UserFunc\NewsJsonRenderer;
+use Evomedien\Vitec\UserFunc\SearchJsonRenderer;
 use Evomedien\Vitec\UserFunc\ContainerBackgroundRenderer;
 use TYPO3\CMS\Core\Database\ConnectionPool;
 use TYPO3\CMS\Core\Utility\GeneralUtility;
@@ -130,6 +131,7 @@ final class ContentElementResolver
         'vitec_contactform' => [FormsJsonRenderer::class, 'form'],
         'vitec_demoform' => [FormsJsonRenderer::class, 'form'],
         'vitec_helpdeskform' => [FormsJsonRenderer::class, 'form'],
+        'solr_pi_results' => [SearchJsonRenderer::class, 'search'],
         'news_pi1' => [NewsJsonRenderer::class, 'news'],
         'news_newsliststicky' => [NewsJsonRenderer::class, 'news'],
         'news_newsselectedlist' => [NewsJsonRenderer::class, 'news'],

packages/vitec/Classes/UserFunc/SearchJsonRenderer.php

@@ -0,0 +1,227 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Evomedien\Vitec\UserFunc;
+
+use ApacheSolrForTypo3\Solr\ConnectionManager;
+use ApacheSolrForTypo3\Solr\Domain\Search\ResultSet\SearchResultSetService;
+use ApacheSolrForTypo3\Solr\Domain\Search\SearchRequestBuilder;
+use ApacheSolrForTypo3\Solr\Search;
+use ApacheSolrForTypo3\Solr\System\Configuration\ConfigurationManager;
+use ApacheSolrForTypo3\Solr\System\Service\ConfigurationService;
+use Doctrine\DBAL\ParameterType;
+use TYPO3\CMS\Core\Attribute\AsAllowedCallable;
+use TYPO3\CMS\Core\Database\ConnectionPool;
+use TYPO3\CMS\Core\Utility\GeneralUtility;
+use TYPO3\CMS\Frontend\ContentObject\ContentObjectRenderer;
+
+/**
+ * UserFunc: render EXT:solr search results as JSON (headless).
+ *
+ * Output under content.search:
+ *   {
+ *     "query":          the search terms as used,
+ *     "page":           1-based current page,
+ *     "resultsPerPage": page size (plugin.tx_solr.search.results.resultsPerPage
+ *                       or the FlexForm override on the plugin),
+ *     "numFound":       total hits,
+ *     "totalPages":     ceil(numFound / resultsPerPage),
+ *     "results":        [ { "title", "url", "type", "teaser" } ],
+ *     "suggestions":    spellcheck alternatives ("did you mean"), [] if none
+ *   }
+ *
+ * Request parameters (GET, on the page carrying the plugin):
+ *   q     = search terms; without it the empty shape above is returned
+ *           (numFound 0) so the frontend always sees the same structure
+ *   page  = 1-based page number, optional
+ * The EXT:solr namespace (tx_solr[q], tx_solr[page]) is accepted as a
+ * fallback so classic solr URLs keep working.
+ *
+ * The search itself is EXT:solr's own pipeline (SearchRequestBuilder ->
+ * SearchResultSetService), i.e. identical to what the Fluid plugin runs -
+ * only the rendering is JSON instead of HTML. `teaser` carries the
+ * highlighted fragment when highlighting is enabled, otherwise a plain
+ * 250-char excerpt of the indexed content.
+ *
+ * `render()` = top-level plugin / page discovery. `renderForRecord()` = one
+ * specific tt_content row (reused by ContainerChildrenProcessor for nested
+ * plugins). Exception-safe.
+ */
+class SearchJsonRenderer
+{
+    private ?ContentObjectRenderer $cObj = null;
+
+    /** Same duck-typed setter contract as the other renderers (see MarketListJsonRenderer). */
+    public function setContentObjectRenderer(ContentObjectRenderer $cObj): void
+    {
+        $this->cObj = $cObj;
+    }
+
+    private const CTYPE = 'solr_pi_results';
+    private const TEASER_FALLBACK_LENGTH = 250;
+
+    #[AsAllowedCallable]
+    public function render(string $content, array $conf): string
+    {
+        $row = is_array($this->cObj?->data ?? null) ? $this->cObj->data : null;
+        if ($row && (string)($row['CType'] ?? '') === self::CTYPE) {
+            return $this->renderForRecord($row);
+        }
+
+        $pageId = 0;
+        $request = $GLOBALS['TYPO3_REQUEST'] ?? null;
+        if ($request !== null) {
+            $pageInfo = $request->getAttribute('frontend.page.information');
+            if ($pageInfo !== null) {
+                $pageId = (int)$pageInfo->getId();
+            }
+        }
+        if ($pageId <= 0) {
+            $pageId = (int)($GLOBALS['TSFE']->id ?? 0);
+        }
+        if ($pageId <= 0) {
+            return '';
+        }
+
+        $qb = GeneralUtility::makeInstance(ConnectionPool::class)->getQueryBuilderForTable('tt_content');
+        $ces = $qb
+            ->select('*')
+            ->from('tt_content')
+            ->where(
+                $qb->expr()->eq('pid', $qb->createNamedParameter($pageId, ParameterType::INTEGER)),
+                $qb->expr()->eq('CType', $qb->createNamedParameter(self::CTYPE, ParameterType::STRING)),
+                $qb->expr()->eq('deleted', 0),
+                $qb->expr()->eq('hidden', 0)
+            )
+            ->executeQuery()
+            ->fetchAllAssociative();
+
+        if (empty($ces)) {
+            return '';
+        }
+
+        return $this->renderForRecord($ces[0]);
+    }
+
+    /**
+     * @param array<string,mixed> $contentElement
+     */
+    public function renderForRecord(array $contentElement): string
+    {
+        try {
+            $request = $GLOBALS['TYPO3_REQUEST'] ?? null;
+            if ($request === null) {
+                return '';
+            }
+
+            // Search responses depend on the q/page GET parameters, which are
+            // excluded from cHash - cached variants would collide (config.no_cache
+            // is gone in TYPO3 v14, so the cache is disabled per request here).
+            $request->getAttribute('frontend.cache.instruction')
+                ?->disableCache('vitec search: response varies by q/page query parameters');
+
+            $params = $request->getQueryParams();
+            $solrNamespace = (array)($params['tx_solr'] ?? []);
+            $query = trim((string)($params['q'] ?? $solrNamespace['q'] ?? ''));
+            $page = max(1, (int)($params['page'] ?? $solrNamespace['page'] ?? 1));
+
+            if ($query === '') {
+                return (string)json_encode($this->emptyResult());
+            }
+
+            $languageId = (int)($request->getAttribute('language')?->getLanguageId() ?? 0);
+            $pageId = (int)($request->getAttribute('frontend.page.information')?->getId() ?? 0);
+
+            $typoScriptConfiguration = GeneralUtility::makeInstance(ConfigurationManager::class)
+                ->getTypoScriptFromRequest($request);
+            if (!empty($contentElement['pi_flexform'])) {
+                GeneralUtility::makeInstance(ConfigurationService::class)->overrideConfigurationWithFlexFormSettings(
+                    (string)$contentElement['pi_flexform'],
+                    $typoScriptConfiguration,
+                );
+            }
+
+            $connection = GeneralUtility::makeInstance(ConnectionManager::class)
+                ->getConnectionByTypo3Site($request->getAttribute('site'), $languageId);
+            $search = GeneralUtility::makeInstance(Search::class, $connection);
+            $searchService = GeneralUtility::makeInstance(
+                SearchResultSetService::class,
+                $typoScriptConfiguration,
+                $search,
+            );
+            $searchRequest = GeneralUtility::makeInstance(SearchRequestBuilder::class, $typoScriptConfiguration)
+                ->buildForSearch(['q' => $query, 'page' => $page], $pageId, $languageId);
+
+            $resultSet = $searchService->search($searchRequest);
+
+            $highlighted = null;
+            try {
+                $highlighted = $resultSet->getUsedSearch()?->getHighlightedContent();
+            } catch (\Throwable) {
+                // no highlighting available - the excerpt fallback below covers it
+            }
+
+            $results = [];
+            foreach ($resultSet->getSearchResults() as $document) {
+                $id = (string)$document->getId();
+                if ($highlighted !== null && !empty($highlighted->{$id}->content[0])) {
+                    $teaser = implode(' ... ', $highlighted->{$id}->content);
+                } else {
+                    $teaser = mb_substr(trim((string)$document->getContent()), 0, self::TEASER_FALLBACK_LENGTH);
+                }
+                $results[] = [
+                    'title' => (string)$document->getTitle(),
+                    'url' => (string)$document->getUrl(),
+                    'type' => (string)$document->getType(),
+                    'teaser' => $teaser,
+                ];
+            }
+
+            $suggestions = [];
+            try {
+                foreach ($resultSet->getSpellCheckingSuggestions() as $suggestion) {
+                    $word = method_exists($suggestion, 'getSuggestion')
+                        ? (string)$suggestion->getSuggestion()
+                        : '';
+                    if ($word !== '') {
+                        $suggestions[] = $word;
+                    }
+                }
+            } catch (\Throwable) {
+                // spellchecking disabled or unavailable - not essential
+            }
+
+            $numFound = $resultSet->getAllResultCount();
+            $resultsPerPage = $resultSet->getUsedResultsPerPage() ?: count($results);
+
+            return (string)json_encode([
+                'query' => $query,
+                'page' => $page,
+                'resultsPerPage' => $resultsPerPage,
+                'numFound' => $numFound,
+                'totalPages' => $resultsPerPage > 0 ? (int)ceil($numFound / $resultsPerPage) : 0,
+                'results' => $results,
+                'suggestions' => array_values(array_unique($suggestions)),
+            ]);
+        } catch (\Throwable $e) {
+            return '';
+        }
+    }
+
+    /**
+     * @return array<string,mixed>
+     */
+    private function emptyResult(): array
+    {
+        return [
+            'query' => '',
+            'page' => 1,
+            'resultsPerPage' => 0,
+            'numFound' => 0,
+            'totalPages' => 0,
+            'results' => [],
+            'suggestions' => [],
+        ];
+    }
+}

packages/vitec/Configuration/Sets/Vitecset/setup.typoscript

@@ -1,4 +1,6 @@
 config {
+  # EXT:solr indexes pages only when indexing is explicitly enabled
+  index_enable = 1
   pageTitleProviders {
     vitec {
       provider = Evomedien\Vitec\PageTitle\ProductPageTitleProvider
@@ -32,6 +34,19 @@ tt_content {
         }
     }
 
+    # EXT:solr search results as JSON - the search endpoint for the React frontend
+    solr_pi_results < lib.contentElementWithHeader
+    solr_pi_results {
+        fields {
+            content {
+                fields {
+                    search = USER
+                    search.userFunc = Evomedien\Vitec\UserFunc\SearchJsonRenderer->render
+                }
+            }
+        }
+    }
+
     news_pi1 < lib.contentElementWithHeader
     news_pi1 {
         fields {
@@ -278,3 +293,46 @@ config.recordLinks {
         }
     }
 }
+
+# Highlighted teaser fragments for the JSON search results (SearchJsonRenderer)
+plugin.tx_solr.search.results.resultsHighlighting = 1
+plugin.tx_solr.search.results.resultsHighlighting.wrap = <mark>|</mark>
+
+
+# Headless has no TYPO3SEARCH_begin/end markers in its JSON output, so the
+# default page content extraction indexes an EMPTY content field. Build the
+# content from the page's tt_content rows instead: header, subheader, bodytext
+# cover core CTypes and the Content Blocks (their text fields reuse bodytext),
+# vitec_quote is the only Content Blocks text column of its own. SOLR_CONTENT
+# strips markup before indexing. Collection children (FAQ items, cards) live in
+# their own tables and are not covered yet.
+plugin.tx_solr.index.queue.pages.fields {
+    content = SOLR_CONTENT
+    content {
+        cObject = COA
+        cObject {
+            10 = CONTENT
+            10 {
+                table = tt_content
+                select {
+                    orderBy = sorting
+                }
+                renderObj = COA
+                renderObj {
+                    10 = TEXT
+                    10.field = header
+                    10.noTrimWrap = || |
+                    20 = TEXT
+                    20.field = subheader
+                    20.noTrimWrap = || |
+                    30 = TEXT
+                    30.field = bodytext
+                    30.noTrimWrap = || |
+                    40 = TEXT
+                    40.field = vitec_quote
+                    40.noTrimWrap = || |
+                }
+            }
+        }
+    }
+}

packages/vitec/ext_localconf.php

@@ -250,3 +250,10 @@ $GLOBALS['TYPO3_CONF_VARS']['SYS']['formEngine']['nodeRegistry'][1750000000] = [
     );
 
 })();
+
+// The JSON search endpoint (SearchJsonRenderer) uses plain `q` and `page` GET
+// parameters. Without this exclusion TYPO3 demands a cHash for them and answers
+// with an error page; the no_cache condition in the Vitecset TypoScript makes
+// sure search responses are rendered fresh instead of sticking in the page cache.
+$GLOBALS['TYPO3_CONF_VARS']['FE']['cacheHash']['excludedParameters'][] = 'q';
+$GLOBALS['TYPO3_CONF_VARS']['FE']['cacheHash']['excludedParameters'][] = 'page';