From 1011162bb158086ad4e8976b933feb0712600fc5 Mon Sep 17 00:00:00 2001 From: Oliver Rasche Date: Mon, 10 Aug 2026 14:56:19 +0200 Subject: [PATCH] Backend module: CSV import for the VITEC domain models New "VITEC Import" module under Web: one import page per domain model (v1: Market, Solution, Product - registry-driven, adding a model is one config entry). Workflow: upload a CSV (delimiter and encoding are auto-detected, including German-Excel semicolon/Windows-1252), map CSV columns to DB fields, persist the mapping per model together with the identity field used for matching (new table tx_vitec_import_mapping, no TCA - pure tool configuration), review a unified list of CSV rows matched against the DB records (new / update with differing fields / unchanged / db-only), then apply the checked rows through DataHandler. Each importable row carries an editable JSON payload textarea - what is written is the textarea content, not the raw CSV, so editors can fix values right in the review step. The parsed CSV travels through the form as a hidden JSON field: no session state, no temp files. Importable fields are derived from TCA at runtime (scalar types only; files, categories and other relations are excluded - a flat CSV cannot carry them). Payloads are whitelisted against that field list on apply; new records require a storage pid (prefilled from existing records). BE user permissions apply via DataHandler. The module ships its own CSS (backend-import.css, loaded only by this module) using the frontend button palette from _vitec.scss: orange #f47937 for primary actions, navy #26358c for secondary actions and structure. The stray

Hi

debug leftover in the shared backend layout is removed (also affects the OG Image module). A fourth tab "SEO Research" handles the recurring keyword-research CSV. It is deliberately not an import mask - the file carries research only (no meta title/description yet). Each upload is persisted as a delivery (tx_vitec_seo_research, never deleted) and evaluated: diff against the previous delivery keyed by URL, a structure check of the CSV tree against TYPO3 (pages by slug path; market/solution/product rows against the domain tables, matched by slug then normalized title), and the three work lists from the SEO flags (quick wins by GSC impressions, shared terms grouped by keyword, already ranking). CsvReader now deduplicates repeated header names, which that CSV has. New CLI command vitec:create-markets: creates the market records the structure check reports as missing, sourced from the stored delivery and matched through the same SeoResearchService - what the module lists is what the command creates. Each market gets a sys_category of the same title, found anywhere under the auto-detected market category root or created; sub-market categories are created under the parent market's category, so the category tree carries the hierarchy the flat market model cannot. Idempotent, dry-run first. New CLI commands vitec:create-markets and vitec:market-dummy-image. create-markets creates the market records the structure check reports as missing (root detection three-staged: option, auto-detect, find or create a "Markets" category). market-dummy-image assigns a shared placeholder (white logo on brand navy, fileadmin/placeholders/) to every market without an image - one sys_file for all, replacing the file restyles every placeholder at once. Both idempotent. Deliberately out of v1: import log with three-way compare (protection against overwriting manual edits), images/relations, multiple saved mappings per model. --- .../Classes/Command/CreateMarketsCommand.php | 444 ++++++++++++++++++ .../Command/MarketDummyImageCommand.php | 126 +++++ .../Controller/Backend/ImportController.php | 44 ++ packages/vitec/Classes/Import/CsvReader.php | 15 +- .../Classes/Import/SeoResearchRepository.php | 60 +++ .../Classes/Import/SeoResearchService.php | 285 +++++++++++ .../vitec/Configuration/Backend/Modules.php | 7 + packages/vitec/README.md | 20 + .../Private/Templates/Import/Index.html | 3 + .../Private/Templates/Import/Seo.html | 201 ++++++++ packages/vitec/ext_tables.sql | 11 + 11 files changed, 1215 insertions(+), 1 deletion(-) create mode 100644 packages/vitec/Classes/Command/CreateMarketsCommand.php create mode 100644 packages/vitec/Classes/Command/MarketDummyImageCommand.php create mode 100644 packages/vitec/Classes/Import/SeoResearchRepository.php create mode 100644 packages/vitec/Classes/Import/SeoResearchService.php create mode 100644 packages/vitec/Resources/Private/Templates/Import/Seo.html diff --git a/packages/vitec/Classes/Command/CreateMarketsCommand.php b/packages/vitec/Classes/Command/CreateMarketsCommand.php new file mode 100644 index 0000000..91d0806 --- /dev/null +++ b/packages/vitec/Classes/Command/CreateMarketsCommand.php @@ -0,0 +1,444 @@ +addOption('dry-run', null, InputOption::VALUE_NONE, 'Report only - no records, no categories'); + $this->addOption('pid', null, InputOption::VALUE_REQUIRED, 'Storage pid for new market records (default: pid of existing markets)'); + $this->addOption('category-parent', null, InputOption::VALUE_REQUIRED, 'Uid of the market category root (default: auto-detect, then a category titled "Markets", else it is created)'); + $this->addOption('category-pid', null, InputOption::VALUE_REQUIRED, 'Sysfolder pid for newly created categories', '33'); + } + + protected function execute(InputInterface $input, OutputInterface $output): int + { + Bootstrap::initializeBackendAuthentication(); + $dryRun = (bool)$input->getOption('dry-run'); + + // ------------------------------------------------ source: delivery + $repository = GeneralUtility::makeInstance(SeoResearchRepository::class); + $service = GeneralUtility::makeInstance(SeoResearchService::class); + $delivery = $repository->latest(0); + if ($delivery === null) { + $output->writeln('No SEO research delivery stored - upload the CSV in the backend module first.'); + return Command::FAILURE; + } + $output->writeln(sprintf('Delivery: %s (%s)', $delivery['filename'], date('Y-m-d H:i', $delivery['crdate']))); + + $structure = $service->analyze($delivery['rows'], null)['structure']['market']; + $missing = $structure['missing']; + $both = $structure['both']; + $output->writeln(sprintf('Markets: %d matched, %d missing, %d only in TYPO3', + $structure['matched'], count($missing), count($structure['extra']))); + if ($missing === []) { + $output->writeln('Nothing to create.'); + return Command::SUCCESS; + } + + // -------------------------------------------------- pid for records + $pid = (int)($input->getOption('pid') ?? 0); + if ($pid <= 0) { + $pid = $this->detectMarketPid(); + } + if ($pid <= 0) { + $output->writeln('No --pid given and no existing market records to derive it from.'); + return Command::FAILURE; + } + + // -------------------------------------- category root + existing map + $categoryByMarketUid = $this->categoriesOfExistingMarkets(); + $categoryPid = (int)$input->getOption('category-pid'); + $categoryRoot = (int)($input->getOption('category-parent') ?? 0); + if ($categoryRoot <= 0) { + $categoryRoot = $this->detectCategoryRoot($categoryByMarketUid); + } + if ($categoryRoot <= 0) { + // No market has a category yet: look for (or create) a root + // category titled "Markets" on the category sysfolder. + $categoryRoot = $this->findCategoryByTitle('Markets', $categoryPid); + if ($categoryRoot > 0) { + $output->writeln(sprintf('Using existing category "Markets" (uid %d) as root.', $categoryRoot)); + } elseif ($dryRun) { + $output->writeln(sprintf('Would create root category "Markets" (pid %d, top level) and attach everything under it.', $categoryPid)); + } else { + $dataHandler = GeneralUtility::makeInstance(DataHandler::class); + $dataHandler->start([self::CATEGORY_TABLE => ['NEW1' => [ + 'pid' => $categoryPid, + 'parent' => 0, + 'title' => 'Markets', + ]]], []); + $dataHandler->process_datamap(); + $categoryRoot = (int)($dataHandler->substNEWwithIDs['NEW1'] ?? 0); + if ($categoryRoot <= 0) { + $output->writeln('Could not create the root category "Markets": ' + . implode(' | ', $dataHandler->errorLog) . ''); + return Command::FAILURE; + } + $output->writeln(sprintf('created root category "Markets" (uid %d, pid %d)', $categoryRoot, $categoryPid)); + } + } + if ($categoryRoot > 0) { + $rootRow = $this->categoryRow($categoryRoot); + if ($rootRow === null) { + $output->writeln(sprintf('Category %d does not exist.', $categoryRoot)); + return Command::FAILURE; + } + $categoryPid = (int)$rootRow['pid']; + $output->writeln(sprintf('Storage pid: %d | category root: "%s" (uid %d)', $pid, $rootRow['title'], $categoryRoot)); + } else { + $output->writeln(sprintf('Storage pid: %d | category root: "Markets" (created on the real run)', $pid)); + } + + $allCategories = $this->loadAllCategories(); + $treeUids = $this->subtreeUids($allCategories, $categoryRoot); + + // ref -> category uid for markets that already exist (via the both list) + $categoryByRef = []; + foreach ($both as $pair) { + $catUid = $categoryByMarketUid[(int)$pair['uid']] ?? 0; + if ($catUid > 0) { + $categoryByRef[$pair['ref']] = $catUid; + } + } + // market uid by ref, to fix parents without category + $marketUidByRef = []; + foreach ($both as $pair) { + $marketUidByRef[$pair['ref']] = (int)$pair['uid']; + } + + // ------------------------------------------------------------ create + usort($missing, static fn(array $a, array $b): int => + substr_count($a['ref'], '.') <=> substr_count($b['ref'], '.')); + + $createdMarkets = 0; + $createdCategories = 0; + $warnings = []; + + foreach ($missing as $row) { + $title = $row['name']; + $slug = mb_strtolower(trim((string)basename(rtrim($row['url'], '/')))); + $parentRef = str_contains($row['ref'], '.') + ? substr($row['ref'], 0, (int)strrpos($row['ref'], '.')) + : ''; + + // Parent category: category of the parent market when the ref has + // one; the root otherwise. Top-level market refs ("1.3") have + // parentRef "1" = the section root -> attach to the category root. + $parentCategory = $categoryRoot; + if ($parentRef !== '' && str_contains($parentRef, '.')) { + $parentCategory = $categoryByRef[$parentRef] ?? 0; + if ($parentCategory <= 0 && isset($marketUidByRef[$parentRef])) { + // Existing parent market without category: create/find its + // category under the root and assign it, so the tree holds. + $parentTitle = $this->titleOfMarket($marketUidByRef[$parentRef]); + $parentCategory = $this->findOrCreateCategory( + $parentTitle, $categoryRoot, $categoryPid, $allCategories, $treeUids, $dryRun, $createdCategories, $output + ); + if (!$dryRun && $parentCategory > 0) { + $this->assignCategory($marketUidByRef[$parentRef], $parentCategory, $output); + } + $categoryByRef[$parentRef] = $parentCategory; + } + if ($parentCategory <= 0) { + $warnings[] = sprintf('%s %s: parent %s not resolvable - category attached to the root instead.', + $row['ref'], $title, $parentRef); + $parentCategory = $categoryRoot; + } + } + + $categoryUid = $this->findOrCreateCategory( + $title, $parentCategory, $categoryPid, $allCategories, $treeUids, $dryRun, $createdCategories, $output + ); + + if ($dryRun) { + $parentLabel = $parentCategory > 0 ? '#' . $parentCategory : '"Markets" (new root)'; + $output->writeln(sprintf(' CREATE market %-8s %-45s slug=%s category=%s', + $row['ref'], mb_substr($title, 0, 45), $slug, + $categoryUid > 0 ? '#' . $categoryUid : '(new, under ' . $parentLabel . ')')); + $createdMarkets++; + $categoryByRef[$row['ref']] = $categoryUid; + continue; + } + + $dataHandler = GeneralUtility::makeInstance(DataHandler::class); + $dataHandler->start([self::TABLE => ['NEW1' => [ + 'pid' => $pid, + 'title' => $title, + 'slug' => $slug, + 'categories' => (string)$categoryUid, + ]]], []); + $dataHandler->process_datamap(); + if ($dataHandler->errorLog !== []) { + $warnings[] = sprintf('%s: %s', $title, implode(' | ', $dataHandler->errorLog)); + continue; + } + $newUid = (int)($dataHandler->substNEWwithIDs['NEW1'] ?? 0); + $categoryByRef[$row['ref']] = $categoryUid; + $createdMarkets++; + $output->writeln(sprintf(' created market %s "%s" (uid %d, category %d)', + $row['ref'], $title, $newUid, $categoryUid)); + } + + // ------------------------------------------------------------ report + $output->writeln(''); + $output->writeln(sprintf('%s: %d market(s), %d categor%s.', + $dryRun ? 'DRY-RUN - would create' : 'Created', + $createdMarkets, $createdCategories, $createdCategories === 1 ? 'y' : 'ies')); + foreach ($warnings as $w) { + $output->writeln(' ! ' . $w . ''); + } + return Command::SUCCESS; + } + + // ------------------------------------------------------------ categories + + /** + * Find a category by normalized title anywhere under the market category + * root; create it under $parentCategory when absent. + * + * @param array> $allCategories by uid + * @param array $treeUids uids belonging to the root's subtree + */ + private function findOrCreateCategory( + string $title, + int $parentCategory, + int $categoryPid, + array &$allCategories, + array &$treeUids, + bool $dryRun, + int &$createdCategories, + OutputInterface $output + ): int { + $norm = $this->normalizeTitle($title); + foreach ($allCategories as $uid => $cat) { + if (isset($treeUids[$uid]) && $this->normalizeTitle((string)$cat['title']) === $norm) { + return (int)$uid; + } + } + $createdCategories++; + if ($dryRun) { + $parentLabel = $parentCategory > 0 ? '#' . $parentCategory : '"Markets" (new root)'; + $output->writeln(sprintf(' create category %-42s under %s', mb_substr($title, 0, 42), $parentLabel)); + return 0; + } + $dataHandler = GeneralUtility::makeInstance(DataHandler::class); + $dataHandler->start([self::CATEGORY_TABLE => ['NEW1' => [ + 'pid' => $categoryPid, + 'parent' => $parentCategory, + 'title' => $title, + ]]], []); + $dataHandler->process_datamap(); + $uid = (int)($dataHandler->substNEWwithIDs['NEW1'] ?? 0); + if ($uid > 0) { + $allCategories[$uid] = ['uid' => $uid, 'parent' => $parentCategory, 'title' => $title, 'pid' => $categoryPid]; + $treeUids[$uid] = true; + $output->writeln(sprintf(' created category "%s" (uid %d, parent %d)', $title, $uid, $parentCategory)); + } + return $uid; + } + + private function assignCategory(int $marketUid, int $categoryUid, OutputInterface $output): void + { + $dataHandler = GeneralUtility::makeInstance(DataHandler::class); + $dataHandler->start([self::TABLE => [(string)$marketUid => ['categories' => (string)$categoryUid]]], []); + $dataHandler->process_datamap(); + $output->writeln(sprintf(' assigned category %d to existing market %d', $categoryUid, $marketUid)); + } + + /** @return array market uid => first assigned category uid */ + private function categoriesOfExistingMarkets(): array + { + $qb = GeneralUtility::makeInstance(ConnectionPool::class)->getQueryBuilderForTable('sys_category_record_mm'); + $rows = $qb->select('uid_local', 'uid_foreign')->from('sys_category_record_mm') + ->where( + $qb->expr()->eq('tablenames', $qb->createNamedParameter(self::TABLE, ParameterType::STRING)), + $qb->expr()->eq('fieldname', $qb->createNamedParameter('categories', ParameterType::STRING)) + ) + ->orderBy('sorting', 'ASC') + ->executeQuery()->fetchAllAssociative(); + $map = []; + foreach ($rows as $row) { + $market = (int)$row['uid_foreign']; + if (!isset($map[$market])) { + $map[$market] = (int)$row['uid_local']; + } + } + return $map; + } + + /** Most common parent of the categories assigned to existing markets. */ + private function detectCategoryRoot(array $categoryByMarketUid): int + { + if ($categoryByMarketUid === []) { + return 0; + } + $qb = GeneralUtility::makeInstance(ConnectionPool::class)->getQueryBuilderForTable(self::CATEGORY_TABLE); + $rows = $qb->select('uid', 'parent')->from(self::CATEGORY_TABLE) + ->where( + $qb->expr()->in('uid', array_map('intval', array_values($categoryByMarketUid))), + $qb->expr()->eq('deleted', 0) + ) + ->executeQuery()->fetchAllAssociative(); + $parents = []; + foreach ($rows as $row) { + $parents[(int)$row['parent']] = ($parents[(int)$row['parent']] ?? 0) + 1; + } + // Sub-market categories may already sit below a main-market category; + // the ROOT is the most common parent among top-level assignments. + arsort($parents); + return (int)array_key_first($parents); + } + + /** @return array> */ + private function loadAllCategories(): array + { + $qb = GeneralUtility::makeInstance(ConnectionPool::class)->getQueryBuilderForTable(self::CATEGORY_TABLE); + $rows = $qb->select('uid', 'pid', 'parent', 'title')->from(self::CATEGORY_TABLE) + ->where($qb->expr()->eq('deleted', 0)) + ->executeQuery()->fetchAllAssociative(); + $out = []; + foreach ($rows as $row) { + $out[(int)$row['uid']] = $row; + } + return $out; + } + + /** @return array every category uid inside the root's subtree (root included) */ + private function subtreeUids(array $allCategories, int $root): array + { + $children = []; + foreach ($allCategories as $uid => $cat) { + $children[(int)$cat['parent']][] = (int)$uid; + } + $result = [$root => true]; + $queue = [$root]; + while ($queue !== []) { + $current = array_shift($queue); + foreach ($children[$current] ?? [] as $childUid) { + if (!isset($result[$childUid])) { + $result[$childUid] = true; + $queue[] = $childUid; + } + } + } + return $result; + } + + /** Category by (normalized) title, preferring the given pid. */ + private function findCategoryByTitle(string $title, int $preferredPid): int + { + $qb = GeneralUtility::makeInstance(ConnectionPool::class)->getQueryBuilderForTable(self::CATEGORY_TABLE); + $rows = $qb->select('uid', 'pid', 'title')->from(self::CATEGORY_TABLE) + ->where($qb->expr()->eq('deleted', 0)) + ->executeQuery()->fetchAllAssociative(); + $norm = $this->normalizeTitle($title); + $fallback = 0; + foreach ($rows as $row) { + if ($this->normalizeTitle((string)$row['title']) !== $norm) { + continue; + } + if ((int)$row['pid'] === $preferredPid) { + return (int)$row['uid']; + } + if ($fallback === 0) { + $fallback = (int)$row['uid']; + } + } + return $fallback; + } + + /** @return array|null */ + private function categoryRow(int $uid): ?array + { + $qb = GeneralUtility::makeInstance(ConnectionPool::class)->getQueryBuilderForTable(self::CATEGORY_TABLE); + $row = $qb->select('uid', 'pid', 'title')->from(self::CATEGORY_TABLE) + ->where( + $qb->expr()->eq('uid', $qb->createNamedParameter($uid, ParameterType::INTEGER)), + $qb->expr()->eq('deleted', 0) + ) + ->executeQuery()->fetchAssociative(); + return $row ?: null; + } + + // ----------------------------------------------------------------- misc + + private function detectMarketPid(): int + { + $qb = GeneralUtility::makeInstance(ConnectionPool::class)->getQueryBuilderForTable(self::TABLE); + $row = $qb->select('pid')->from(self::TABLE) + ->where($qb->expr()->eq('deleted', 0)) + ->setMaxResults(1) + ->executeQuery()->fetchAssociative(); + return $row ? (int)$row['pid'] : 0; + } + + private function titleOfMarket(int $uid): string + { + $qb = GeneralUtility::makeInstance(ConnectionPool::class)->getQueryBuilderForTable(self::TABLE); + $row = $qb->select('title')->from(self::TABLE) + ->where($qb->expr()->eq('uid', $qb->createNamedParameter($uid, ParameterType::INTEGER))) + ->executeQuery()->fetchAssociative(); + return $row ? (string)$row['title'] : ''; + } + + private function normalizeTitle(string $title): string + { + $title = (string)preg_replace('/\s*\(.*?\)/', '', $title); + $title = str_replace('&', 'and', mb_strtolower($title)); + $title = (string)preg_replace('/[^a-z0-9]+/', ' ', $title); + return trim((string)preg_replace('/\s+/', ' ', $title)); + } +} diff --git a/packages/vitec/Classes/Command/MarketDummyImageCommand.php b/packages/vitec/Classes/Command/MarketDummyImageCommand.php new file mode 100644 index 0000000..6a315db --- /dev/null +++ b/packages/vitec/Classes/Command/MarketDummyImageCommand.php @@ -0,0 +1,126 @@ +addOption('file', null, InputOption::VALUE_REQUIRED, 'Placeholder image path', self::DEFAULT_FILE); + $this->addOption('dry-run', null, InputOption::VALUE_NONE, 'Report only'); + } + + protected function execute(InputInterface $input, OutputInterface $output): int + { + Bootstrap::initializeBackendAuthentication(); + $dryRun = (bool)$input->getOption('dry-run'); + $path = (string)$input->getOption('file'); + + try { + $file = GeneralUtility::makeInstance(ResourceFactory::class)->retrieveFileOrFolderObject($path); + } catch (\Throwable $e) { + $file = null; + } + if (!$file instanceof File) { + $output->writeln(sprintf('Placeholder not found in FAL: %s', $path)); + return Command::FAILURE; + } + $output->writeln(sprintf('Placeholder: %s (sys_file %d)', $file->getIdentifier(), $file->getUid())); + + // Markets that already have an image reference. + $qb = GeneralUtility::makeInstance(ConnectionPool::class)->getQueryBuilderForTable('sys_file_reference'); + $withImage = $qb->select('uid_foreign')->from('sys_file_reference') + ->where( + $qb->expr()->eq('tablenames', $qb->createNamedParameter(self::TABLE, ParameterType::STRING)), + $qb->expr()->eq('fieldname', $qb->createNamedParameter('image', ParameterType::STRING)), + $qb->expr()->eq('deleted', 0) + ) + ->executeQuery()->fetchFirstColumn(); + $withImage = array_flip(array_map('intval', $withImage)); + + $qb = GeneralUtility::makeInstance(ConnectionPool::class)->getQueryBuilderForTable(self::TABLE); + $markets = $qb->select('uid', 'pid', 'title')->from(self::TABLE) + ->where($qb->expr()->eq('deleted', 0)) + ->orderBy('title', 'ASC') + ->executeQuery()->fetchAllAssociative(); + + $assigned = 0; + $skipped = 0; + foreach ($markets as $market) { + $uid = (int)$market['uid']; + if (isset($withImage[$uid])) { + $skipped++; + continue; + } + if ($dryRun) { + $output->writeln(sprintf(' ASSIGN placeholder -> %s (uid %d)', $market['title'], $uid)); + $assigned++; + continue; + } + $dataHandler = GeneralUtility::makeInstance(DataHandler::class); + $dataHandler->start([ + 'sys_file_reference' => [ + 'NEWref' => [ + 'uid_local' => $file->getUid(), + 'pid' => (int)$market['pid'], + ], + ], + self::TABLE => [ + (string)$uid => ['image' => 'NEWref'], + ], + ], []); + $dataHandler->process_datamap(); + if ($dataHandler->errorLog !== []) { + $output->writeln(sprintf(' %s: %s', $market['title'], implode(' | ', $dataHandler->errorLog))); + continue; + } + $output->writeln(sprintf(' assigned -> %s (uid %d)', $market['title'], $uid)); + $assigned++; + } + + $output->writeln(''); + $output->writeln(sprintf( + '%s: %d assigned, %d already had an image.', + $dryRun ? 'DRY-RUN - would be' : 'Done', + $assigned, + $skipped + )); + return Command::SUCCESS; + } +} diff --git a/packages/vitec/Classes/Controller/Backend/ImportController.php b/packages/vitec/Classes/Controller/Backend/ImportController.php index 3daae20..2773c4f 100644 --- a/packages/vitec/Classes/Controller/Backend/ImportController.php +++ b/packages/vitec/Classes/Controller/Backend/ImportController.php @@ -7,6 +7,8 @@ namespace Evomedien\Vitec\Controller\Backend; use Evomedien\Vitec\Import\CsvReader; use Evomedien\Vitec\Import\ImportModelRegistry; use Evomedien\Vitec\Import\MappingRepository; +use Evomedien\Vitec\Import\SeoResearchRepository; +use Evomedien\Vitec\Import\SeoResearchService; use Psr\Http\Message\ResponseInterface; use Psr\Http\Message\ServerRequestInterface; use TYPO3\CMS\Backend\Attribute\AsController; @@ -45,6 +47,8 @@ final class ImportController private readonly ImportModelRegistry $registry, private readonly MappingRepository $mappingRepository, private readonly CsvReader $csvReader, + private readonly SeoResearchRepository $seoResearchRepository, + private readonly SeoResearchService $seoResearchService, private readonly FlashMessageService $flashMessageService, private readonly PageRenderer $pageRenderer, private readonly UriBuilder $uriBuilder, @@ -116,6 +120,46 @@ final class ImportController return $this->render($request, $modelKey, $csv, $mapping, $identity); } + // --------------------------------------------------- SEO research tab + + public function seoAction(ServerRequestInterface $request): ResponseInterface + { + $latest = $this->seoResearchRepository->latest(0); + $previous = $this->seoResearchRepository->latest(1); + $analysis = $latest !== null + ? $this->seoResearchService->analyze($latest['rows'], $previous !== null ? $previous['rows'] : null) + : null; + + $this->pageRenderer->addCssFile('EXT:vitec/Resources/Public/Css/backend-import.css'); + $view = $this->moduleTemplateFactory->create($request); + $view->assignMultiple([ + 'models' => $this->registry->all(), + 'latest' => $latest, + 'previous' => $previous, + 'analysis' => $analysis, + ]); + return $view->renderResponse('Import/Seo'); + } + + public function seoUploadAction(ServerRequestInterface $request): ResponseInterface + { + $files = $request->getUploadedFiles(); + $upload = $files['csvfile'] ?? null; + if ($upload !== null && $upload->getError() === UPLOAD_ERR_OK) { + $csv = $this->csvReader->parse((string)$upload->getStream()); + $rows = $this->seoResearchService->normalizeRows($csv['rows']); + if ($rows === []) { + $this->flash('No usable rows found - is the "Potential URL" column present?', 'Upload', false); + } else { + $this->seoResearchRepository->add((string)($upload->getClientFilename() ?? 'upload.csv'), $rows); + $this->flash(sprintf('%d rows stored as new delivery.', count($rows)), 'Delivery stored', true); + } + } else { + $this->flash('No file received.', 'Upload', false); + } + return new RedirectResponse((string)$this->uriBuilder->buildUriFromRoute('web_vitecimport.seo'), 303); + } + // ------------------------------------------------------------ rendering /** diff --git a/packages/vitec/Classes/Import/CsvReader.php b/packages/vitec/Classes/Import/CsvReader.php index 87d3496..3726936 100644 --- a/packages/vitec/Classes/Import/CsvReader.php +++ b/packages/vitec/Classes/Import/CsvReader.php @@ -48,9 +48,22 @@ final class CsvReader return ['columns' => [], 'rows' => []]; } $columns = []; + $seen = []; foreach ($header as $i => $name) { $name = trim((string)$name); - $columns[$i] = $name !== '' ? $name : ('column_' . ($i + 1)); + if ($name === '') { + $name = 'column_' . ($i + 1); + } + // Duplicate headers (e.g. the keyword research CSV repeats + // "Vol (Global)" for the secondary block) get a _2/_3 suffix - + // otherwise the later block silently overwrites the earlier one. + if (isset($seen[$name])) { + $seen[$name]++; + $name .= '_' . $seen[$name]; + } else { + $seen[$name] = 1; + } + $columns[$i] = $name; } $rows = []; diff --git a/packages/vitec/Classes/Import/SeoResearchRepository.php b/packages/vitec/Classes/Import/SeoResearchRepository.php new file mode 100644 index 0000000..ce38b26 --- /dev/null +++ b/packages/vitec/Classes/Import/SeoResearchRepository.php @@ -0,0 +1,60 @@ +> $rows */ + public function add(string $filename, array $rows): int + { + $connection = GeneralUtility::makeInstance(ConnectionPool::class)->getConnectionForTable(self::TABLE); + $connection->insert(self::TABLE, [ + 'pid' => 0, + 'crdate' => time(), + 'be_user' => (int)($GLOBALS['BE_USER']->user['uid'] ?? 0), + 'filename' => mb_substr($filename, 0, 255), + 'row_count' => count($rows), + 'payload' => (string)json_encode($rows, JSON_UNESCAPED_UNICODE), + ]); + return (int)$connection->lastInsertId(); + } + + /** + * @return array{uid:int,crdate:int,filename:string,rows:array>}|null + */ + public function latest(int $offset = 0): ?array + { + $qb = GeneralUtility::makeInstance(ConnectionPool::class)->getQueryBuilderForTable(self::TABLE); + $row = $qb->select('uid', 'crdate', 'filename', 'payload')->from(self::TABLE) + ->orderBy('uid', 'DESC') + ->setFirstResult($offset) + ->setMaxResults(1) + ->executeQuery()->fetchAssociative(); + if (!$row) { + return null; + } + $rows = json_decode((string)$row['payload'], true); + return [ + 'uid' => (int)$row['uid'], + 'crdate' => (int)$row['crdate'], + 'filename' => (string)$row['filename'], + 'rows' => is_array($rows) ? $rows : [], + ]; + } +} diff --git a/packages/vitec/Classes/Import/SeoResearchService.php b/packages/vitec/Classes/Import/SeoResearchService.php new file mode 100644 index 0000000..aef601c --- /dev/null +++ b/packages/vitec/Classes/Import/SeoResearchService.php @@ -0,0 +1,285 @@ + every row's URL against pages.slug (slug = full path) + * markets -> level-1.x rows against tx_vitec_domain_model_market + * solutions -> level-2.x rows against tx_vitec_domain_model_solution + * products -> level-3.x.y+ rows against tx_vitec_domain_model_product + * (3.x rows are categories, not products - skipped) + * Record matching: slug == last URL segment, falling back to a + * normalized title comparison (parentheses stripped, & -> and). + * - diff against the previous delivery, keyed by URL + * + * Works on the normalized row schema produced by normalizeRows(). Read-only: + * this class never writes anything. + */ +final class SeoResearchService +{ + private const SECTION_MODELS = [ + // Markets/Solutions: every row below the section root is a record + // candidate - sub-markets ("Traffic & Smart Mobility" under + // "Transport & Infrastructure") and sub-solutions are records of the + // same (flat) model. Products: 3.x rows are categories, records + // start at 3.x.y. + 'Markets' => ['key' => 'market', 'table' => 'tx_vitec_domain_model_market', 'depth' => [2, 9]], + 'Solutions' => ['key' => 'solution', 'table' => 'tx_vitec_domain_model_solution', 'depth' => [2, 9]], + 'Products' => ['key' => 'product', 'table' => 'tx_vitec_domain_model_product', 'depth' => [3, 9]], + ]; + + /** + * Map raw CSV rows (deduplicated headers) onto a stable schema, so stored + * deliveries stay comparable even if the CSV gains columns. + * + * @param array> $raw + * @return array> + */ + public function normalizeRows(array $raw): array + { + $rows = []; + foreach ($raw as $r) { + $url = trim((string)($r['Potential URL'] ?? '')); + if ($url === '') { + continue; + } + $rows[] = [ + 'section' => trim((string)($r['Section'] ?? '')), + 'ref' => trim((string)($r['Page Ref'] ?? '')), + 'name' => trim((string)($r['Page Name'] ?? '')), + 'url' => $url, + 'primary' => trim((string)($r['Primary Keyword'] ?? '')), + 'volGlobal' => trim((string)($r['Vol (Global)'] ?? '')), + 'intent' => trim((string)($r['Intent'] ?? '')), + 'kd' => trim((string)($r['KD'] ?? '')), + 'gscPos' => trim((string)($r['GSC Pos (blended)'] ?? '')), + 'gscImpr' => trim((string)($r['GSC Impr'] ?? '')), + 'flag' => trim((string)($r['Flag'] ?? '')), + 'secondary' => trim((string)($r['Secondary Keyword'] ?? '')), + 'secVolGlobal' => trim((string)($r['Vol (Global)_2'] ?? '')), + ]; + } + return $rows; + } + + /** + * @param array> $rows + * @param array>|null $previousRows + * @return array + */ + public function analyze(array $rows, ?array $previousRows): array + { + return [ + 'quickWins' => $this->quickWins($rows), + 'sharedTerms' => $this->sharedTerms($rows), + 'alreadyRanking' => array_values(array_filter($rows, fn(array $r): bool => str_contains($r['flag'], 'Already ranking'))), + 'structure' => $this->structure($rows), + 'diff' => $previousRows !== null ? $this->diff($previousRows, $rows) : null, + ]; + } + + // ------------------------------------------------------------ work lists + + /** @param array> $rows + * @return array> */ + private function quickWins(array $rows): array + { + $wins = array_values(array_filter($rows, fn(array $r): bool => str_contains($r['flag'], 'Quick win'))); + usort($wins, static function (array $a, array $b): int { + return (int)preg_replace('/\D/', '', $b['gscImpr']) <=> (int)preg_replace('/\D/', '', $a['gscImpr']); + }); + return $wins; + } + + /** + * Shared-term rows grouped by primary keyword - each group is one + * cannibalization risk: several pages targeting the same term. + * + * @param array> $rows + * @return array>}> + */ + private function sharedTerms(array $rows): array + { + $groups = []; + foreach ($rows as $r) { + if (!str_contains($r['flag'], 'Shared term')) { + continue; + } + $groups[mb_strtolower($r['primary'])]['keyword'] = $r['primary']; + $groups[mb_strtolower($r['primary'])]['pages'][] = $r; + } + // Pages sharing the keyword without carrying the flag themselves: + foreach ($groups as $kw => $group) { + foreach ($rows as $r) { + if (mb_strtolower($r['primary']) === $kw && !in_array($r, $group['pages'], true)) { + $groups[$kw]['pages'][] = $r; + } + } + } + return array_values($groups); + } + + // ------------------------------------------------------- structure check + + /** @param array> $rows + * @return array */ + private function structure(array $rows): array + { + $out = ['pages' => $this->pagesCheck($rows)]; + + foreach (self::SECTION_MODELS as $section => $cfg) { + $candidates = array_values(array_filter($rows, function (array $r) use ($section, $cfg): bool { + $depth = substr_count($r['ref'], '.') + 1; + return $r['section'] === $section && $depth >= $cfg['depth'][0] && $depth <= $cfg['depth'][1]; + })); + $out[$cfg['key']] = $this->recordsCheck($candidates, $cfg['table']); + } + return $out; + } + + /** @param array> $rows + * @return array{total:int,found:int,missing:array>} */ + private function pagesCheck(array $rows): array + { + $qb = GeneralUtility::makeInstance(ConnectionPool::class)->getQueryBuilderForTable('pages'); + $slugs = $qb->select('slug')->from('pages') + ->where( + $qb->expr()->eq('deleted', 0), + $qb->expr()->eq('sys_language_uid', 0) + ) + ->executeQuery()->fetchFirstColumn(); + $existing = array_flip(array_map(static fn($s): string => rtrim((string)$s, '/') ?: '/', $slugs)); + + $missing = []; + $found = 0; + foreach ($rows as $r) { + $slug = rtrim($r['url'], '/') ?: '/'; + if (isset($existing[$slug])) { + $found++; + } else { + $missing[] = $r; + } + } + return ['total' => count($rows), 'found' => $found, 'missing' => $missing]; + } + + /** + * @param array> $candidates + * @return array{total:int,matched:int,missing:array>,extra:array>} + */ + private function recordsCheck(array $candidates, string $table): array + { + $qb = GeneralUtility::makeInstance(ConnectionPool::class)->getQueryBuilderForTable($table); + $records = $qb->select('uid', 'title', 'slug')->from($table) + ->where($qb->expr()->eq('deleted', 0)) + ->executeQuery()->fetchAllAssociative(); + + $bySlug = []; + $byTitle = []; + foreach ($records as $rec) { + $slug = mb_strtolower(trim((string)($rec['slug'] ?? ''))); + if ($slug !== '') { + $bySlug[$slug] = $rec; + } + $byTitle[$this->normalizeTitle((string)$rec['title'])] = $rec; + } + + $missing = []; + $both = []; + $matchedUids = []; + foreach ($candidates as $r) { + $segment = mb_strtolower(trim((string)basename(rtrim($r['url'], '/')))); + $rec = $bySlug[$segment] ?? $byTitle[$this->normalizeTitle($r['name'])] ?? null; + if ($rec !== null) { + $matchedUids[(int)$rec['uid']] = true; + $both[] = [ + 'ref' => $r['ref'], + 'name' => $r['name'], + 'url' => $r['url'], + 'uid' => (string)$rec['uid'], + 'title' => (string)$rec['title'], + ]; + } else { + $missing[] = $r; + } + } + + $extra = []; + foreach ($records as $rec) { + if (!isset($matchedUids[(int)$rec['uid']])) { + $extra[] = ['uid' => (string)$rec['uid'], 'title' => (string)$rec['title']]; + } + } + + return [ + 'total' => count($candidates), + 'matched' => count($matchedUids), + 'both' => $both, + 'missing' => $missing, + 'extra' => $extra, + ]; + } + + private function normalizeTitle(string $title): string + { + $title = (string)preg_replace('/\s*\(.*?\)/', '', $title); // drop parenthetical suffixes + $title = str_replace('&', 'and', mb_strtolower($title)); + $title = (string)preg_replace('/[^a-z0-9]+/', ' ', $title); + return trim((string)preg_replace('/\s+/', ' ', $title)); + } + + // ------------------------------------------------------------------ diff + + /** + * @param array> $old + * @param array> $new + * @return array{added:array,removed:array,changed:array} + */ + private function diff(array $old, array $new): array + { + $byUrlOld = []; + foreach ($old as $r) { + $byUrlOld[$r['url']] = $r; + } + $byUrlNew = []; + foreach ($new as $r) { + $byUrlNew[$r['url']] = $r; + } + + $added = []; + $changed = []; + foreach ($byUrlNew as $url => $r) { + $o = $byUrlOld[$url] ?? null; + if ($o === null) { + $added[] = sprintf('%s (%s)', $url, $r['name']); + continue; + } + $changes = []; + foreach (['primary' => 'primary keyword', 'secondary' => 'secondary keyword', 'flag' => 'flag'] as $field => $label) { + if ($r[$field] !== $o[$field]) { + $changes[] = sprintf('%s "%s" -> "%s"', $label, $o[$field], $r[$field]); + } + } + if ($changes !== []) { + $changed[] = $url . ': ' . implode('; ', $changes); + } + } + $removed = []; + foreach ($byUrlOld as $url => $r) { + if (!isset($byUrlNew[$url])) { + $removed[] = sprintf('%s (%s)', $url, $r['name']); + } + } + + return ['added' => $added, 'removed' => $removed, 'changed' => $changed]; + } +} diff --git a/packages/vitec/Configuration/Backend/Modules.php b/packages/vitec/Configuration/Backend/Modules.php index be9c021..db63fc6 100644 --- a/packages/vitec/Configuration/Backend/Modules.php +++ b/packages/vitec/Configuration/Backend/Modules.php @@ -26,6 +26,13 @@ return [ 'target' => ImportController::class . '::processAction', 'methods' => ['POST'], ], + 'seo' => [ + 'target' => ImportController::class . '::seoAction', + ], + 'seo_upload' => [ + 'target' => ImportController::class . '::seoUploadAction', + 'methods' => ['POST'], + ], ], ], 'web_vitecogimage' => [ diff --git a/packages/vitec/README.md b/packages/vitec/README.md index e10b9d6..6ea6dbd 100755 --- a/packages/vitec/README.md +++ b/packages/vitec/README.md @@ -37,6 +37,7 @@ this extension turns every content element into clean **JSON** for a React front - [Forms](#forms) - [Page‑level fields](#pagelevel-fields) - [Structured data (JSON‑LD)](#structured-data-json-ld) +- [Editorial tooling](#editorial-tooling) - [Requirements](#requirements) - [Installation](#installation) - [Adding a new headless plugin](#adding-a-new-headless-plugin) @@ -188,6 +189,25 @@ endpoint) and `vitec/success-story-path-rewrite`, which lets the public SEO URL `Organization`, `WebSite` (root only), `BreadcrumbList`, `Product`, `VideoObject`, `FAQPage`, `ExhibitionEvent` and `NewsArticle`. +## Editorial tooling + +**Backend module "VITEC Import"** (Web menu): CSV import per domain model +(Market, Solution, Product) with a persistable column mapper and a unified +review list (new / update / unchanged / db-only) — what gets written is the +editable per-row payload, applied through DataHandler. A fourth tab +**SEO Research** stores each delivery of the recurring keyword-research CSV, +diffs it against the previous one and checks the CSV structure against the +page tree and the domain records. + +| CLI command | Purpose | +|---|---| +| `vitec:import-success-stories` | One-time migration of the old-site success stories | +| `vitec:import-downloads` | Import old-site downloads (Collateral only, idempotent, filename normalization) | +| `vitec:create-markets` | Create market records the SEO structure check reports missing, incl. sys_category assignment | +| `vitec:market-dummy-image` | Assign the shared placeholder image to markets without an image | + +All commands support `--dry-run` and are safe to re-run. + ## Requirements | Component | Version | diff --git a/packages/vitec/Resources/Private/Templates/Import/Index.html b/packages/vitec/Resources/Private/Templates/Import/Index.html index 1209b8b..b750582 100644 --- a/packages/vitec/Resources/Private/Templates/Import/Index.html +++ b/packages/vitec/Resources/Private/Templates/Import/Index.html @@ -19,6 +19,9 @@ href="{f:be.uri(route: 'web_vitecimport', parameters: {model: m.key})}">{m.label} +
  • + SEO Research +
    • diff --git a/packages/vitec/Resources/Private/Templates/Import/Seo.html b/packages/vitec/Resources/Private/Templates/Import/Seo.html new file mode 100644 index 0000000..751b775 --- /dev/null +++ b/packages/vitec/Resources/Private/Templates/Import/Seo.html @@ -0,0 +1,201 @@ + + + + + + +
    +
    +

    VITEC Import — SEO Research

    +
    +
    + + + + + +
    +
    +
    +
    + +
    +
    + +
    +
    + Every upload is kept as a delivery and compared against the previous one. +
    +
    +
    +
    + + + +

    + Latest delivery: {latest.filename}, + @{latest.crdate} + ({analysis.structure.pages.total} rows) + + — compared against @{previous.crdate} + +

    + + + +
    +
    Changes since previous delivery
    +
    + +

    New pages ({analysis.diff.added -> f:count()})

    +
    • {line}
    +     + +

    Removed pages ({analysis.diff.removed -> f:count()})

    +
    • {line}
    +     + +

    Changed rows ({analysis.diff.changed -> f:count()})

    +
    • {line}
    +     + +

    No differences.

    +     +
    +
    +     + + +
    +
    Structure check — CSV vs. TYPO3
    +
    + +

    + Pages: {analysis.structure.pages.found} / {analysis.structure.pages.total} exist + Markets: {analysis.structure.market.matched} / {analysis.structure.market.total} + Solutions: {analysis.structure.solution.matched} / {analysis.structure.solution.total} + Products: {analysis.structure.product.matched} / {analysis.structure.product.total} +

    +

    + Markets/Solutions: every row below the section root, including sub-markets and + sub-solutions. Products: level x.y and deeper (the top product rows are + categories). Matching: record slug against the last URL segment, falling back + to a normalized title comparison. The Page Ref column shows the hierarchy. +

    + +
    + + +
    +

    {area}

    + +

    Both in CSV and TYPO3 ({check.both -> f:count()})

    +
      + +
    • {pair.ref} {pair.name} → uid {pair.uid} ({pair.title})
      • +       +
    +     + +

    Missing in TYPO3 ({check.missing -> f:count()})

    +
      + +
    • {row.ref} {row.name}
      • +       +
    +     + +

    Only in TYPO3 ({check.extra -> f:count()})

    +
      + +
    • {rec.title} uid {rec.uid}
      • +       +
    +     + +

    Complete match — every CSV row has its record and vice versa.

    +     +
    +     +     +
    + + +
    + Pages missing in TYPO3 ({analysis.structure.pages.missing -> f:count()}) +
      + +
    • {row.url} — {row.name}
      • +       +
    +
    +     +
    +
    + + +
    +
    Quick wins {analysis.quickWins -> f:count()}
    +
    + + + + + + + + + + + + + +
    PagePrimary keywordVol (Global)GSC PosGSC Impr.
    {row.name}
    {row.url}    		{row.primary}{row.volGlobal}{row.gscPos}{row.gscImpr}
    +
    +
    + + +
    +
    Shared terms (cannibalization risk) {analysis.sharedTerms -> f:count()}
    +
    + +

    {group.keyword}

    +
      + +
    • {row.name} {row.url}
      • +       +
    +     +
    +
    + + +
    +
    Already ranking {analysis.alreadyRanking -> f:count()}
    +
    +
      + +
    • {row.name} {row.url} — {row.primary} (GSC Pos {row.gscPos})
      • +       +
    +
    +
    + +     + + +

    No delivery stored yet - upload the keyword research CSV above.

    +     + +     + diff --git a/packages/vitec/ext_tables.sql b/packages/vitec/ext_tables.sql index 1c4751a..8019ba0 100755 --- a/packages/vitec/ext_tables.sql +++ b/packages/vitec/ext_tables.sql @@ -304,3 +304,14 @@ CREATE TABLE tx_vitec_import_mapping ( PRIMARY KEY (uid), KEY model (model) ); + +CREATE TABLE tx_vitec_seo_research ( + uid int(11) NOT NULL auto_increment, + pid int(11) DEFAULT '0' NOT NULL, + crdate int(11) DEFAULT '0' NOT NULL, + be_user int(11) DEFAULT '0' NOT NULL, + filename varchar(255) DEFAULT '' NOT NULL, + row_count int(11) DEFAULT '0' NOT NULL, + payload mediumtext, + PRIMARY KEY (uid) +);