diff --git a/packages/vitec/Classes/Command/CreateMarketsCommand.php b/packages/vitec/Classes/Command/CreateMarketsCommand.php new file mode 100644 index 0000000..91d0806 --- /dev/null +++ b/packages/vitec/Classes/Command/CreateMarketsCommand.php @@ -0,0 +1,444 @@ +addOption('dry-run', null, InputOption::VALUE_NONE, 'Report only - no records, no categories'); + $this->addOption('pid', null, InputOption::VALUE_REQUIRED, 'Storage pid for new market records (default: pid of existing markets)'); + $this->addOption('category-parent', null, InputOption::VALUE_REQUIRED, 'Uid of the market category root (default: auto-detect, then a category titled "Markets", else it is created)'); + $this->addOption('category-pid', null, InputOption::VALUE_REQUIRED, 'Sysfolder pid for newly created categories', '33'); + } + + protected function execute(InputInterface $input, OutputInterface $output): int + { + Bootstrap::initializeBackendAuthentication(); + $dryRun = (bool)$input->getOption('dry-run'); + + // ------------------------------------------------ source: delivery + $repository = GeneralUtility::makeInstance(SeoResearchRepository::class); + $service = GeneralUtility::makeInstance(SeoResearchService::class); + $delivery = $repository->latest(0); + if ($delivery === null) { + $output->writeln('No SEO research delivery stored - upload the CSV in the backend module first.'); + return Command::FAILURE; + } + $output->writeln(sprintf('Delivery: %s (%s)', $delivery['filename'], date('Y-m-d H:i', $delivery['crdate']))); + + $structure = $service->analyze($delivery['rows'], null)['structure']['market']; + $missing = $structure['missing']; + $both = $structure['both']; + $output->writeln(sprintf('Markets: %d matched, %d missing, %d only in TYPO3', + $structure['matched'], count($missing), count($structure['extra']))); + if ($missing === []) { + $output->writeln('Nothing to create.'); + return Command::SUCCESS; + } + + // -------------------------------------------------- pid for records + $pid = (int)($input->getOption('pid') ?? 0); + if ($pid <= 0) { + $pid = $this->detectMarketPid(); + } + if ($pid <= 0) { + $output->writeln('No --pid given and no existing market records to derive it from.'); + return Command::FAILURE; + } + + // -------------------------------------- category root + existing map + $categoryByMarketUid = $this->categoriesOfExistingMarkets(); + $categoryPid = (int)$input->getOption('category-pid'); + $categoryRoot = (int)($input->getOption('category-parent') ?? 0); + if ($categoryRoot <= 0) { + $categoryRoot = $this->detectCategoryRoot($categoryByMarketUid); + } + if ($categoryRoot <= 0) { + // No market has a category yet: look for (or create) a root + // category titled "Markets" on the category sysfolder. + $categoryRoot = $this->findCategoryByTitle('Markets', $categoryPid); + if ($categoryRoot > 0) { + $output->writeln(sprintf('Using existing category "Markets" (uid %d) as root.', $categoryRoot)); + } elseif ($dryRun) { + $output->writeln(sprintf('Would create root category "Markets" (pid %d, top level) and attach everything under it.', $categoryPid)); + } else { + $dataHandler = GeneralUtility::makeInstance(DataHandler::class); + $dataHandler->start([self::CATEGORY_TABLE => ['NEW1' => [ + 'pid' => $categoryPid, + 'parent' => 0, + 'title' => 'Markets', + ]]], []); + $dataHandler->process_datamap(); + $categoryRoot = (int)($dataHandler->substNEWwithIDs['NEW1'] ?? 0); + if ($categoryRoot <= 0) { + $output->writeln('Could not create the root category "Markets": ' + . implode(' | ', $dataHandler->errorLog) . ''); + return Command::FAILURE; + } + $output->writeln(sprintf('created root category "Markets" (uid %d, pid %d)', $categoryRoot, $categoryPid)); + } + } + if ($categoryRoot > 0) { + $rootRow = $this->categoryRow($categoryRoot); + if ($rootRow === null) { + $output->writeln(sprintf('Category %d does not exist.', $categoryRoot)); + return Command::FAILURE; + } + $categoryPid = (int)$rootRow['pid']; + $output->writeln(sprintf('Storage pid: %d | category root: "%s" (uid %d)', $pid, $rootRow['title'], $categoryRoot)); + } else { + $output->writeln(sprintf('Storage pid: %d | category root: "Markets" (created on the real run)', $pid)); + } + + $allCategories = $this->loadAllCategories(); + $treeUids = $this->subtreeUids($allCategories, $categoryRoot); + + // ref -> category uid for markets that already exist (via the both list) + $categoryByRef = []; + foreach ($both as $pair) { + $catUid = $categoryByMarketUid[(int)$pair['uid']] ?? 0; + if ($catUid > 0) { + $categoryByRef[$pair['ref']] = $catUid; + } + } + // market uid by ref, to fix parents without category + $marketUidByRef = []; + foreach ($both as $pair) { + $marketUidByRef[$pair['ref']] = (int)$pair['uid']; + } + + // ------------------------------------------------------------ create + usort($missing, static fn(array $a, array $b): int => + substr_count($a['ref'], '.') <=> substr_count($b['ref'], '.')); + + $createdMarkets = 0; + $createdCategories = 0; + $warnings = []; + + foreach ($missing as $row) { + $title = $row['name']; + $slug = mb_strtolower(trim((string)basename(rtrim($row['url'], '/')))); + $parentRef = str_contains($row['ref'], '.') + ? substr($row['ref'], 0, (int)strrpos($row['ref'], '.')) + : ''; + + // Parent category: category of the parent market when the ref has + // one; the root otherwise. Top-level market refs ("1.3") have + // parentRef "1" = the section root -> attach to the category root. + $parentCategory = $categoryRoot; + if ($parentRef !== '' && str_contains($parentRef, '.')) { + $parentCategory = $categoryByRef[$parentRef] ?? 0; + if ($parentCategory <= 0 && isset($marketUidByRef[$parentRef])) { + // Existing parent market without category: create/find its + // category under the root and assign it, so the tree holds. + $parentTitle = $this->titleOfMarket($marketUidByRef[$parentRef]); + $parentCategory = $this->findOrCreateCategory( + $parentTitle, $categoryRoot, $categoryPid, $allCategories, $treeUids, $dryRun, $createdCategories, $output + ); + if (!$dryRun && $parentCategory > 0) { + $this->assignCategory($marketUidByRef[$parentRef], $parentCategory, $output); + } + $categoryByRef[$parentRef] = $parentCategory; + } + if ($parentCategory <= 0) { + $warnings[] = sprintf('%s %s: parent %s not resolvable - category attached to the root instead.', + $row['ref'], $title, $parentRef); + $parentCategory = $categoryRoot; + } + } + + $categoryUid = $this->findOrCreateCategory( + $title, $parentCategory, $categoryPid, $allCategories, $treeUids, $dryRun, $createdCategories, $output + ); + + if ($dryRun) { + $parentLabel = $parentCategory > 0 ? '#' . $parentCategory : '"Markets" (new root)'; + $output->writeln(sprintf(' CREATE market %-8s %-45s slug=%s category=%s', + $row['ref'], mb_substr($title, 0, 45), $slug, + $categoryUid > 0 ? '#' . $categoryUid : '(new, under ' . $parentLabel . ')')); + $createdMarkets++; + $categoryByRef[$row['ref']] = $categoryUid; + continue; + } + + $dataHandler = GeneralUtility::makeInstance(DataHandler::class); + $dataHandler->start([self::TABLE => ['NEW1' => [ + 'pid' => $pid, + 'title' => $title, + 'slug' => $slug, + 'categories' => (string)$categoryUid, + ]]], []); + $dataHandler->process_datamap(); + if ($dataHandler->errorLog !== []) { + $warnings[] = sprintf('%s: %s', $title, implode(' | ', $dataHandler->errorLog)); + continue; + } + $newUid = (int)($dataHandler->substNEWwithIDs['NEW1'] ?? 0); + $categoryByRef[$row['ref']] = $categoryUid; + $createdMarkets++; + $output->writeln(sprintf(' created market %s "%s" (uid %d, category %d)', + $row['ref'], $title, $newUid, $categoryUid)); + } + + // ------------------------------------------------------------ report + $output->writeln(''); + $output->writeln(sprintf('%s: %d market(s), %d categor%s.', + $dryRun ? 'DRY-RUN - would create' : 'Created', + $createdMarkets, $createdCategories, $createdCategories === 1 ? 'y' : 'ies')); + foreach ($warnings as $w) { + $output->writeln(' ! ' . $w . ''); + } + return Command::SUCCESS; + } + + // ------------------------------------------------------------ categories + + /** + * Find a category by normalized title anywhere under the market category + * root; create it under $parentCategory when absent. + * + * @param array> $allCategories by uid + * @param array $treeUids uids belonging to the root's subtree + */ + private function findOrCreateCategory( + string $title, + int $parentCategory, + int $categoryPid, + array &$allCategories, + array &$treeUids, + bool $dryRun, + int &$createdCategories, + OutputInterface $output + ): int { + $norm = $this->normalizeTitle($title); + foreach ($allCategories as $uid => $cat) { + if (isset($treeUids[$uid]) && $this->normalizeTitle((string)$cat['title']) === $norm) { + return (int)$uid; + } + } + $createdCategories++; + if ($dryRun) { + $parentLabel = $parentCategory > 0 ? '#' . $parentCategory : '"Markets" (new root)'; + $output->writeln(sprintf(' create category %-42s under %s', mb_substr($title, 0, 42), $parentLabel)); + return 0; + } + $dataHandler = GeneralUtility::makeInstance(DataHandler::class); + $dataHandler->start([self::CATEGORY_TABLE => ['NEW1' => [ + 'pid' => $categoryPid, + 'parent' => $parentCategory, + 'title' => $title, + ]]], []); + $dataHandler->process_datamap(); + $uid = (int)($dataHandler->substNEWwithIDs['NEW1'] ?? 0); + if ($uid > 0) { + $allCategories[$uid] = ['uid' => $uid, 'parent' => $parentCategory, 'title' => $title, 'pid' => $categoryPid]; + $treeUids[$uid] = true; + $output->writeln(sprintf(' created category "%s" (uid %d, parent %d)', $title, $uid, $parentCategory)); + } + return $uid; + } + + private function assignCategory(int $marketUid, int $categoryUid, OutputInterface $output): void + { + $dataHandler = GeneralUtility::makeInstance(DataHandler::class); + $dataHandler->start([self::TABLE => [(string)$marketUid => ['categories' => (string)$categoryUid]]], []); + $dataHandler->process_datamap(); + $output->writeln(sprintf(' assigned category %d to existing market %d', $categoryUid, $marketUid)); + } + + /** @return array market uid => first assigned category uid */ + private function categoriesOfExistingMarkets(): array + { + $qb = GeneralUtility::makeInstance(ConnectionPool::class)->getQueryBuilderForTable('sys_category_record_mm'); + $rows = $qb->select('uid_local', 'uid_foreign')->from('sys_category_record_mm') + ->where( + $qb->expr()->eq('tablenames', $qb->createNamedParameter(self::TABLE, ParameterType::STRING)), + $qb->expr()->eq('fieldname', $qb->createNamedParameter('categories', ParameterType::STRING)) + ) + ->orderBy('sorting', 'ASC') + ->executeQuery()->fetchAllAssociative(); + $map = []; + foreach ($rows as $row) { + $market = (int)$row['uid_foreign']; + if (!isset($map[$market])) { + $map[$market] = (int)$row['uid_local']; + } + } + return $map; + } + + /** Most common parent of the categories assigned to existing markets. */ + private function detectCategoryRoot(array $categoryByMarketUid): int + { + if ($categoryByMarketUid === []) { + return 0; + } + $qb = GeneralUtility::makeInstance(ConnectionPool::class)->getQueryBuilderForTable(self::CATEGORY_TABLE); + $rows = $qb->select('uid', 'parent')->from(self::CATEGORY_TABLE) + ->where( + $qb->expr()->in('uid', array_map('intval', array_values($categoryByMarketUid))), + $qb->expr()->eq('deleted', 0) + ) + ->executeQuery()->fetchAllAssociative(); + $parents = []; + foreach ($rows as $row) { + $parents[(int)$row['parent']] = ($parents[(int)$row['parent']] ?? 0) + 1; + } + // Sub-market categories may already sit below a main-market category; + // the ROOT is the most common parent among top-level assignments. + arsort($parents); + return (int)array_key_first($parents); + } + + /** @return array> */ + private function loadAllCategories(): array + { + $qb = GeneralUtility::makeInstance(ConnectionPool::class)->getQueryBuilderForTable(self::CATEGORY_TABLE); + $rows = $qb->select('uid', 'pid', 'parent', 'title')->from(self::CATEGORY_TABLE) + ->where($qb->expr()->eq('deleted', 0)) + ->executeQuery()->fetchAllAssociative(); + $out = []; + foreach ($rows as $row) { + $out[(int)$row['uid']] = $row; + } + return $out; + } + + /** @return array every category uid inside the root's subtree (root included) */ + private function subtreeUids(array $allCategories, int $root): array + { + $children = []; + foreach ($allCategories as $uid => $cat) { + $children[(int)$cat['parent']][] = (int)$uid; + } + $result = [$root => true]; + $queue = [$root]; + while ($queue !== []) { + $current = array_shift($queue); + foreach ($children[$current] ?? [] as $childUid) { + if (!isset($result[$childUid])) { + $result[$childUid] = true; + $queue[] = $childUid; + } + } + } + return $result; + } + + /** Category by (normalized) title, preferring the given pid. */ + private function findCategoryByTitle(string $title, int $preferredPid): int + { + $qb = GeneralUtility::makeInstance(ConnectionPool::class)->getQueryBuilderForTable(self::CATEGORY_TABLE); + $rows = $qb->select('uid', 'pid', 'title')->from(self::CATEGORY_TABLE) + ->where($qb->expr()->eq('deleted', 0)) + ->executeQuery()->fetchAllAssociative(); + $norm = $this->normalizeTitle($title); + $fallback = 0; + foreach ($rows as $row) { + if ($this->normalizeTitle((string)$row['title']) !== $norm) { + continue; + } + if ((int)$row['pid'] === $preferredPid) { + return (int)$row['uid']; + } + if ($fallback === 0) { + $fallback = (int)$row['uid']; + } + } + return $fallback; + } + + /** @return array|null */ + private function categoryRow(int $uid): ?array + { + $qb = GeneralUtility::makeInstance(ConnectionPool::class)->getQueryBuilderForTable(self::CATEGORY_TABLE); + $row = $qb->select('uid', 'pid', 'title')->from(self::CATEGORY_TABLE) + ->where( + $qb->expr()->eq('uid', $qb->createNamedParameter($uid, ParameterType::INTEGER)), + $qb->expr()->eq('deleted', 0) + ) + ->executeQuery()->fetchAssociative(); + return $row ?: null; + } + + // ----------------------------------------------------------------- misc + + private function detectMarketPid(): int + { + $qb = GeneralUtility::makeInstance(ConnectionPool::class)->getQueryBuilderForTable(self::TABLE); + $row = $qb->select('pid')->from(self::TABLE) + ->where($qb->expr()->eq('deleted', 0)) + ->setMaxResults(1) + ->executeQuery()->fetchAssociative(); + return $row ? (int)$row['pid'] : 0; + } + + private function titleOfMarket(int $uid): string + { + $qb = GeneralUtility::makeInstance(ConnectionPool::class)->getQueryBuilderForTable(self::TABLE); + $row = $qb->select('title')->from(self::TABLE) + ->where($qb->expr()->eq('uid', $qb->createNamedParameter($uid, ParameterType::INTEGER))) + ->executeQuery()->fetchAssociative(); + return $row ? (string)$row['title'] : ''; + } + + private function normalizeTitle(string $title): string + { + $title = (string)preg_replace('/\s*\(.*?\)/', '', $title); + $title = str_replace('&', 'and', mb_strtolower($title)); + $title = (string)preg_replace('/[^a-z0-9]+/', ' ', $title); + return trim((string)preg_replace('/\s+/', ' ', $title)); + } +} diff --git a/packages/vitec/Classes/Command/MarketDummyImageCommand.php b/packages/vitec/Classes/Command/MarketDummyImageCommand.php new file mode 100644 index 0000000..6a315db --- /dev/null +++ b/packages/vitec/Classes/Command/MarketDummyImageCommand.php @@ -0,0 +1,126 @@ +addOption('file', null, InputOption::VALUE_REQUIRED, 'Placeholder image path', self::DEFAULT_FILE); + $this->addOption('dry-run', null, InputOption::VALUE_NONE, 'Report only'); + } + + protected function execute(InputInterface $input, OutputInterface $output): int + { + Bootstrap::initializeBackendAuthentication(); + $dryRun = (bool)$input->getOption('dry-run'); + $path = (string)$input->getOption('file'); + + try { + $file = GeneralUtility::makeInstance(ResourceFactory::class)->retrieveFileOrFolderObject($path); + } catch (\Throwable $e) { + $file = null; + } + if (!$file instanceof File) { + $output->writeln(sprintf('Placeholder not found in FAL: %s', $path)); + return Command::FAILURE; + } + $output->writeln(sprintf('Placeholder: %s (sys_file %d)', $file->getIdentifier(), $file->getUid())); + + // Markets that already have an image reference. + $qb = GeneralUtility::makeInstance(ConnectionPool::class)->getQueryBuilderForTable('sys_file_reference'); + $withImage = $qb->select('uid_foreign')->from('sys_file_reference') + ->where( + $qb->expr()->eq('tablenames', $qb->createNamedParameter(self::TABLE, ParameterType::STRING)), + $qb->expr()->eq('fieldname', $qb->createNamedParameter('image', ParameterType::STRING)), + $qb->expr()->eq('deleted', 0) + ) + ->executeQuery()->fetchFirstColumn(); + $withImage = array_flip(array_map('intval', $withImage)); + + $qb = GeneralUtility::makeInstance(ConnectionPool::class)->getQueryBuilderForTable(self::TABLE); + $markets = $qb->select('uid', 'pid', 'title')->from(self::TABLE) + ->where($qb->expr()->eq('deleted', 0)) + ->orderBy('title', 'ASC') + ->executeQuery()->fetchAllAssociative(); + + $assigned = 0; + $skipped = 0; + foreach ($markets as $market) { + $uid = (int)$market['uid']; + if (isset($withImage[$uid])) { + $skipped++; + continue; + } + if ($dryRun) { + $output->writeln(sprintf(' ASSIGN placeholder -> %s (uid %d)', $market['title'], $uid)); + $assigned++; + continue; + } + $dataHandler = GeneralUtility::makeInstance(DataHandler::class); + $dataHandler->start([ + 'sys_file_reference' => [ + 'NEWref' => [ + 'uid_local' => $file->getUid(), + 'pid' => (int)$market['pid'], + ], + ], + self::TABLE => [ + (string)$uid => ['image' => 'NEWref'], + ], + ], []); + $dataHandler->process_datamap(); + if ($dataHandler->errorLog !== []) { + $output->writeln(sprintf(' %s: %s', $market['title'], implode(' | ', $dataHandler->errorLog))); + continue; + } + $output->writeln(sprintf(' assigned -> %s (uid %d)', $market['title'], $uid)); + $assigned++; + } + + $output->writeln(''); + $output->writeln(sprintf( + '%s: %d assigned, %d already had an image.', + $dryRun ? 'DRY-RUN - would be' : 'Done', + $assigned, + $skipped + )); + return Command::SUCCESS; + } +} diff --git a/packages/vitec/Classes/Controller/Backend/ImportController.php b/packages/vitec/Classes/Controller/Backend/ImportController.php index 3daae20..2773c4f 100644 --- a/packages/vitec/Classes/Controller/Backend/ImportController.php +++ b/packages/vitec/Classes/Controller/Backend/ImportController.php @@ -7,6 +7,8 @@ namespace Evomedien\Vitec\Controller\Backend; use Evomedien\Vitec\Import\CsvReader; use Evomedien\Vitec\Import\ImportModelRegistry; use Evomedien\Vitec\Import\MappingRepository; +use Evomedien\Vitec\Import\SeoResearchRepository; +use Evomedien\Vitec\Import\SeoResearchService; use Psr\Http\Message\ResponseInterface; use Psr\Http\Message\ServerRequestInterface; use TYPO3\CMS\Backend\Attribute\AsController; @@ -45,6 +47,8 @@ final class ImportController private readonly ImportModelRegistry $registry, private readonly MappingRepository $mappingRepository, private readonly CsvReader $csvReader, + private readonly SeoResearchRepository $seoResearchRepository, + private readonly SeoResearchService $seoResearchService, private readonly FlashMessageService $flashMessageService, private readonly PageRenderer $pageRenderer, private readonly UriBuilder $uriBuilder, @@ -116,6 +120,46 @@ final class ImportController return $this->render($request, $modelKey, $csv, $mapping, $identity); } + // --------------------------------------------------- SEO research tab + + public function seoAction(ServerRequestInterface $request): ResponseInterface + { + $latest = $this->seoResearchRepository->latest(0); + $previous = $this->seoResearchRepository->latest(1); + $analysis = $latest !== null + ? $this->seoResearchService->analyze($latest['rows'], $previous !== null ? $previous['rows'] : null) + : null; + + $this->pageRenderer->addCssFile('EXT:vitec/Resources/Public/Css/backend-import.css'); + $view = $this->moduleTemplateFactory->create($request); + $view->assignMultiple([ + 'models' => $this->registry->all(), + 'latest' => $latest, + 'previous' => $previous, + 'analysis' => $analysis, + ]); + return $view->renderResponse('Import/Seo'); + } + + public function seoUploadAction(ServerRequestInterface $request): ResponseInterface + { + $files = $request->getUploadedFiles(); + $upload = $files['csvfile'] ?? null; + if ($upload !== null && $upload->getError() === UPLOAD_ERR_OK) { + $csv = $this->csvReader->parse((string)$upload->getStream()); + $rows = $this->seoResearchService->normalizeRows($csv['rows']); + if ($rows === []) { + $this->flash('No usable rows found - is the "Potential URL" column present?', 'Upload', false); + } else { + $this->seoResearchRepository->add((string)($upload->getClientFilename() ?? 'upload.csv'), $rows); + $this->flash(sprintf('%d rows stored as new delivery.', count($rows)), 'Delivery stored', true); + } + } else { + $this->flash('No file received.', 'Upload', false); + } + return new RedirectResponse((string)$this->uriBuilder->buildUriFromRoute('web_vitecimport.seo'), 303); + } + // ------------------------------------------------------------ rendering /** diff --git a/packages/vitec/Classes/Import/CsvReader.php b/packages/vitec/Classes/Import/CsvReader.php index 87d3496..3726936 100644 --- a/packages/vitec/Classes/Import/CsvReader.php +++ b/packages/vitec/Classes/Import/CsvReader.php @@ -48,9 +48,22 @@ final class CsvReader return ['columns' => [], 'rows' => []]; } $columns = []; + $seen = []; foreach ($header as $i => $name) { $name = trim((string)$name); - $columns[$i] = $name !== '' ? $name : ('column_' . ($i + 1)); + if ($name === '') { + $name = 'column_' . ($i + 1); + } + // Duplicate headers (e.g. the keyword research CSV repeats + // "Vol (Global)" for the secondary block) get a _2/_3 suffix - + // otherwise the later block silently overwrites the earlier one. + if (isset($seen[$name])) { + $seen[$name]++; + $name .= '_' . $seen[$name]; + } else { + $seen[$name] = 1; + } + $columns[$i] = $name; } $rows = []; diff --git a/packages/vitec/Classes/Import/SeoResearchRepository.php b/packages/vitec/Classes/Import/SeoResearchRepository.php new file mode 100644 index 0000000..ce38b26 --- /dev/null +++ b/packages/vitec/Classes/Import/SeoResearchRepository.php @@ -0,0 +1,60 @@ +> $rows */ + public function add(string $filename, array $rows): int + { + $connection = GeneralUtility::makeInstance(ConnectionPool::class)->getConnectionForTable(self::TABLE); + $connection->insert(self::TABLE, [ + 'pid' => 0, + 'crdate' => time(), + 'be_user' => (int)($GLOBALS['BE_USER']->user['uid'] ?? 0), + 'filename' => mb_substr($filename, 0, 255), + 'row_count' => count($rows), + 'payload' => (string)json_encode($rows, JSON_UNESCAPED_UNICODE), + ]); + return (int)$connection->lastInsertId(); + } + + /** + * @return array{uid:int,crdate:int,filename:string,rows:array>}|null + */ + public function latest(int $offset = 0): ?array + { + $qb = GeneralUtility::makeInstance(ConnectionPool::class)->getQueryBuilderForTable(self::TABLE); + $row = $qb->select('uid', 'crdate', 'filename', 'payload')->from(self::TABLE) + ->orderBy('uid', 'DESC') + ->setFirstResult($offset) + ->setMaxResults(1) + ->executeQuery()->fetchAssociative(); + if (!$row) { + return null; + } + $rows = json_decode((string)$row['payload'], true); + return [ + 'uid' => (int)$row['uid'], + 'crdate' => (int)$row['crdate'], + 'filename' => (string)$row['filename'], + 'rows' => is_array($rows) ? $rows : [], + ]; + } +} diff --git a/packages/vitec/Classes/Import/SeoResearchService.php b/packages/vitec/Classes/Import/SeoResearchService.php new file mode 100644 index 0000000..aef601c --- /dev/null +++ b/packages/vitec/Classes/Import/SeoResearchService.php @@ -0,0 +1,285 @@ + every row's URL against pages.slug (slug = full path) + * markets -> level-1.x rows against tx_vitec_domain_model_market + * solutions -> level-2.x rows against tx_vitec_domain_model_solution + * products -> level-3.x.y+ rows against tx_vitec_domain_model_product + * (3.x rows are categories, not products - skipped) + * Record matching: slug == last URL segment, falling back to a + * normalized title comparison (parentheses stripped, & -> and). + * - diff against the previous delivery, keyed by URL + * + * Works on the normalized row schema produced by normalizeRows(). Read-only: + * this class never writes anything. + */ +final class SeoResearchService +{ + private const SECTION_MODELS = [ + // Markets/Solutions: every row below the section root is a record + // candidate - sub-markets ("Traffic & Smart Mobility" under + // "Transport & Infrastructure") and sub-solutions are records of the + // same (flat) model. Products: 3.x rows are categories, records + // start at 3.x.y. + 'Markets' => ['key' => 'market', 'table' => 'tx_vitec_domain_model_market', 'depth' => [2, 9]], + 'Solutions' => ['key' => 'solution', 'table' => 'tx_vitec_domain_model_solution', 'depth' => [2, 9]], + 'Products' => ['key' => 'product', 'table' => 'tx_vitec_domain_model_product', 'depth' => [3, 9]], + ]; + + /** + * Map raw CSV rows (deduplicated headers) onto a stable schema, so stored + * deliveries stay comparable even if the CSV gains columns. + * + * @param array> $raw + * @return array> + */ + public function normalizeRows(array $raw): array + { + $rows = []; + foreach ($raw as $r) { + $url = trim((string)($r['Potential URL'] ?? '')); + if ($url === '') { + continue; + } + $rows[] = [ + 'section' => trim((string)($r['Section'] ?? '')), + 'ref' => trim((string)($r['Page Ref'] ?? '')), + 'name' => trim((string)($r['Page Name'] ?? '')), + 'url' => $url, + 'primary' => trim((string)($r['Primary Keyword'] ?? '')), + 'volGlobal' => trim((string)($r['Vol (Global)'] ?? '')), + 'intent' => trim((string)($r['Intent'] ?? '')), + 'kd' => trim((string)($r['KD'] ?? '')), + 'gscPos' => trim((string)($r['GSC Pos (blended)'] ?? '')), + 'gscImpr' => trim((string)($r['GSC Impr'] ?? '')), + 'flag' => trim((string)($r['Flag'] ?? '')), + 'secondary' => trim((string)($r['Secondary Keyword'] ?? '')), + 'secVolGlobal' => trim((string)($r['Vol (Global)_2'] ?? '')), + ]; + } + return $rows; + } + + /** + * @param array> $rows + * @param array>|null $previousRows + * @return array + */ + public function analyze(array $rows, ?array $previousRows): array + { + return [ + 'quickWins' => $this->quickWins($rows), + 'sharedTerms' => $this->sharedTerms($rows), + 'alreadyRanking' => array_values(array_filter($rows, fn(array $r): bool => str_contains($r['flag'], 'Already ranking'))), + 'structure' => $this->structure($rows), + 'diff' => $previousRows !== null ? $this->diff($previousRows, $rows) : null, + ]; + } + + // ------------------------------------------------------------ work lists + + /** @param array> $rows + * @return array> */ + private function quickWins(array $rows): array + { + $wins = array_values(array_filter($rows, fn(array $r): bool => str_contains($r['flag'], 'Quick win'))); + usort($wins, static function (array $a, array $b): int { + return (int)preg_replace('/\D/', '', $b['gscImpr']) <=> (int)preg_replace('/\D/', '', $a['gscImpr']); + }); + return $wins; + } + + /** + * Shared-term rows grouped by primary keyword - each group is one + * cannibalization risk: several pages targeting the same term. + * + * @param array> $rows + * @return array>}> + */ + private function sharedTerms(array $rows): array + { + $groups = []; + foreach ($rows as $r) { + if (!str_contains($r['flag'], 'Shared term')) { + continue; + } + $groups[mb_strtolower($r['primary'])]['keyword'] = $r['primary']; + $groups[mb_strtolower($r['primary'])]['pages'][] = $r; + } + // Pages sharing the keyword without carrying the flag themselves: + foreach ($groups as $kw => $group) { + foreach ($rows as $r) { + if (mb_strtolower($r['primary']) === $kw && !in_array($r, $group['pages'], true)) { + $groups[$kw]['pages'][] = $r; + } + } + } + return array_values($groups); + } + + // ------------------------------------------------------- structure check + + /** @param array> $rows + * @return array */ + private function structure(array $rows): array + { + $out = ['pages' => $this->pagesCheck($rows)]; + + foreach (self::SECTION_MODELS as $section => $cfg) { + $candidates = array_values(array_filter($rows, function (array $r) use ($section, $cfg): bool { + $depth = substr_count($r['ref'], '.') + 1; + return $r['section'] === $section && $depth >= $cfg['depth'][0] && $depth <= $cfg['depth'][1]; + })); + $out[$cfg['key']] = $this->recordsCheck($candidates, $cfg['table']); + } + return $out; + } + + /** @param array> $rows + * @return array{total:int,found:int,missing:array>} */ + private function pagesCheck(array $rows): array + { + $qb = GeneralUtility::makeInstance(ConnectionPool::class)->getQueryBuilderForTable('pages'); + $slugs = $qb->select('slug')->from('pages') + ->where( + $qb->expr()->eq('deleted', 0), + $qb->expr()->eq('sys_language_uid', 0) + ) + ->executeQuery()->fetchFirstColumn(); + $existing = array_flip(array_map(static fn($s): string => rtrim((string)$s, '/') ?: '/', $slugs)); + + $missing = []; + $found = 0; + foreach ($rows as $r) { + $slug = rtrim($r['url'], '/') ?: '/'; + if (isset($existing[$slug])) { + $found++; + } else { + $missing[] = $r; + } + } + return ['total' => count($rows), 'found' => $found, 'missing' => $missing]; + } + + /** + * @param array> $candidates + * @return array{total:int,matched:int,missing:array>,extra:array>} + */ + private function recordsCheck(array $candidates, string $table): array + { + $qb = GeneralUtility::makeInstance(ConnectionPool::class)->getQueryBuilderForTable($table); + $records = $qb->select('uid', 'title', 'slug')->from($table) + ->where($qb->expr()->eq('deleted', 0)) + ->executeQuery()->fetchAllAssociative(); + + $bySlug = []; + $byTitle = []; + foreach ($records as $rec) { + $slug = mb_strtolower(trim((string)($rec['slug'] ?? ''))); + if ($slug !== '') { + $bySlug[$slug] = $rec; + } + $byTitle[$this->normalizeTitle((string)$rec['title'])] = $rec; + } + + $missing = []; + $both = []; + $matchedUids = []; + foreach ($candidates as $r) { + $segment = mb_strtolower(trim((string)basename(rtrim($r['url'], '/')))); + $rec = $bySlug[$segment] ?? $byTitle[$this->normalizeTitle($r['name'])] ?? null; + if ($rec !== null) { + $matchedUids[(int)$rec['uid']] = true; + $both[] = [ + 'ref' => $r['ref'], + 'name' => $r['name'], + 'url' => $r['url'], + 'uid' => (string)$rec['uid'], + 'title' => (string)$rec['title'], + ]; + } else { + $missing[] = $r; + } + } + + $extra = []; + foreach ($records as $rec) { + if (!isset($matchedUids[(int)$rec['uid']])) { + $extra[] = ['uid' => (string)$rec['uid'], 'title' => (string)$rec['title']]; + } + } + + return [ + 'total' => count($candidates), + 'matched' => count($matchedUids), + 'both' => $both, + 'missing' => $missing, + 'extra' => $extra, + ]; + } + + private function normalizeTitle(string $title): string + { + $title = (string)preg_replace('/\s*\(.*?\)/', '', $title); // drop parenthetical suffixes + $title = str_replace('&', 'and', mb_strtolower($title)); + $title = (string)preg_replace('/[^a-z0-9]+/', ' ', $title); + return trim((string)preg_replace('/\s+/', ' ', $title)); + } + + // ------------------------------------------------------------------ diff + + /** + * @param array> $old + * @param array> $new + * @return array{added:array,removed:array,changed:array} + */ + private function diff(array $old, array $new): array + { + $byUrlOld = []; + foreach ($old as $r) { + $byUrlOld[$r['url']] = $r; + } + $byUrlNew = []; + foreach ($new as $r) { + $byUrlNew[$r['url']] = $r; + } + + $added = []; + $changed = []; + foreach ($byUrlNew as $url => $r) { + $o = $byUrlOld[$url] ?? null; + if ($o === null) { + $added[] = sprintf('%s (%s)', $url, $r['name']); + continue; + } + $changes = []; + foreach (['primary' => 'primary keyword', 'secondary' => 'secondary keyword', 'flag' => 'flag'] as $field => $label) { + if ($r[$field] !== $o[$field]) { + $changes[] = sprintf('%s "%s" -> "%s"', $label, $o[$field], $r[$field]); + } + } + if ($changes !== []) { + $changed[] = $url . ': ' . implode('; ', $changes); + } + } + $removed = []; + foreach ($byUrlOld as $url => $r) { + if (!isset($byUrlNew[$url])) { + $removed[] = sprintf('%s (%s)', $url, $r['name']); + } + } + + return ['added' => $added, 'removed' => $removed, 'changed' => $changed]; + } +} diff --git a/packages/vitec/Configuration/Backend/Modules.php b/packages/vitec/Configuration/Backend/Modules.php index be9c021..db63fc6 100644 --- a/packages/vitec/Configuration/Backend/Modules.php +++ b/packages/vitec/Configuration/Backend/Modules.php @@ -26,6 +26,13 @@ return [ 'target' => ImportController::class . '::processAction', 'methods' => ['POST'], ], + 'seo' => [ + 'target' => ImportController::class . '::seoAction', + ], + 'seo_upload' => [ + 'target' => ImportController::class . '::seoUploadAction', + 'methods' => ['POST'], + ], ], ], 'web_vitecogimage' => [ diff --git a/packages/vitec/README.md b/packages/vitec/README.md index e10b9d6..6ea6dbd 100755 --- a/packages/vitec/README.md +++ b/packages/vitec/README.md @@ -37,6 +37,7 @@ this extension turns every content element into clean **JSON** for a React front - [Forms](#forms) - [Page‑level fields](#pagelevel-fields) - [Structured data (JSON‑LD)](#structured-data-json-ld) +- [Editorial tooling](#editorial-tooling) - [Requirements](#requirements) - [Installation](#installation) - [Adding a new headless plugin](#adding-a-new-headless-plugin) @@ -188,6 +189,25 @@ endpoint) and `vitec/success-story-path-rewrite`, which lets the public SEO URL `Organization`, `WebSite` (root only), `BreadcrumbList`, `Product`, `VideoObject`, `FAQPage`, `ExhibitionEvent` and `NewsArticle`. +## Editorial tooling + +**Backend module "VITEC Import"** (Web menu): CSV import per domain model +(Market, Solution, Product) with a persistable column mapper and a unified +review list (new / update / unchanged / db-only) — what gets written is the +editable per-row payload, applied through DataHandler. A fourth tab +**SEO Research** stores each delivery of the recurring keyword-research CSV, +diffs it against the previous one and checks the CSV structure against the +page tree and the domain records. + +| CLI command | Purpose | +|---|---| +| `vitec:import-success-stories` | One-time migration of the old-site success stories | +| `vitec:import-downloads` | Import old-site downloads (Collateral only, idempotent, filename normalization) | +| `vitec:create-markets` | Create market records the SEO structure check reports missing, incl. sys_category assignment | +| `vitec:market-dummy-image` | Assign the shared placeholder image to markets without an image | + +All commands support `--dry-run` and are safe to re-run. + ## Requirements | Component | Version | diff --git a/packages/vitec/Resources/Private/Templates/Import/Index.html b/packages/vitec/Resources/Private/Templates/Import/Index.html index 1209b8b..b750582 100644 --- a/packages/vitec/Resources/Private/Templates/Import/Index.html +++ b/packages/vitec/Resources/Private/Templates/Import/Index.html @@ -19,6 +19,9 @@ href="{f:be.uri(route: 'web_vitecimport', parameters: {model: m.key})}">{m.label} +
  • + SEO Research +
    • diff --git a/packages/vitec/Resources/Private/Templates/Import/Seo.html b/packages/vitec/Resources/Private/Templates/Import/Seo.html new file mode 100644 index 0000000..751b775 --- /dev/null +++ b/packages/vitec/Resources/Private/Templates/Import/Seo.html @@ -0,0 +1,201 @@ + + + + + + +
    +
    +

    VITEC Import — SEO Research

    +
    +
    + + + + + +
    +
    +
    +
    + +
    +
    + +
    +
    + Every upload is kept as a delivery and compared against the previous one. +
    +
    +
    +
    + + + +

    + Latest delivery: {latest.filename}, + @{latest.crdate} + ({analysis.structure.pages.total} rows) + + — compared against @{previous.crdate} + +

    + + + +
    +
    Changes since previous delivery
    +
    + +

    New pages ({analysis.diff.added -> f:count()})

    +
    • {line}
    +     + +

    Removed pages ({analysis.diff.removed -> f:count()})

    +
    • {line}
    +     + +

    Changed rows ({analysis.diff.changed -> f:count()})

    +
    • {line}
    +     + +

    No differences.

    +     +
    +
    +     + + +
    +
    Structure check — CSV vs. TYPO3
    +
    + +

    + Pages: {analysis.structure.pages.found} / {analysis.structure.pages.total} exist + Markets: {analysis.structure.market.matched} / {analysis.structure.market.total} + Solutions: {analysis.structure.solution.matched} / {analysis.structure.solution.total} + Products: {analysis.structure.product.matched} / {analysis.structure.product.total} +

    +

    + Markets/Solutions: every row below the section root, including sub-markets and + sub-solutions. Products: level x.y and deeper (the top product rows are + categories). Matching: record slug against the last URL segment, falling back + to a normalized title comparison. The Page Ref column shows the hierarchy. +

    + +
    + + +
    +

    {area}

    + +

    Both in CSV and TYPO3 ({check.both -> f:count()})

    +
      + +
    • {pair.ref} {pair.name} → uid {pair.uid} ({pair.title})
      • +       +
    +     + +

    Missing in TYPO3 ({check.missing -> f:count()})

    +
      + +
    • {row.ref} {row.name}
      • +       +
    +     + +

    Only in TYPO3 ({check.extra -> f:count()})

    +
      + +
    • {rec.title} uid {rec.uid}
      • +       +
    +     + +

    Complete match — every CSV row has its record and vice versa.

    +     +
    +     +     +
    + + +
    + Pages missing in TYPO3 ({analysis.structure.pages.missing -> f:count()}) +
      + +
    • {row.url} — {row.name}
      • +       +
    +
    +     +
    +
    + + +
    +
    Quick wins {analysis.quickWins -> f:count()}
    +
    + + + + + + + + + + + + + +
    PagePrimary keywordVol (Global)GSC PosGSC Impr.
    {row.name}
    {row.url}    		{row.primary}{row.volGlobal}{row.gscPos}{row.gscImpr}
    +
    +
    + + +
    +
    Shared terms (cannibalization risk) {analysis.sharedTerms -> f:count()}
    +
    + +

    {group.keyword}

    +
      + +
    • {row.name} {row.url}
      • +       +
    +     +
    +
    + + +
    +
    Already ranking {analysis.alreadyRanking -> f:count()}
    +
    +
      + +
    • {row.name} {row.url} — {row.primary} (GSC Pos {row.gscPos})
      • +       +
    +
    +
    + +     + + +

    No delivery stored yet - upload the keyword research CSV above.

    +     + +     + diff --git a/packages/vitec/ext_tables.sql b/packages/vitec/ext_tables.sql index 1c4751a..8019ba0 100755 --- a/packages/vitec/ext_tables.sql +++ b/packages/vitec/ext_tables.sql @@ -304,3 +304,14 @@ CREATE TABLE tx_vitec_import_mapping ( PRIMARY KEY (uid), KEY model (model) ); + +CREATE TABLE tx_vitec_seo_research ( + uid int(11) NOT NULL auto_increment, + pid int(11) DEFAULT '0' NOT NULL, + crdate int(11) DEFAULT '0' NOT NULL, + be_user int(11) DEFAULT '0' NOT NULL, + filename varchar(255) DEFAULT '' NOT NULL, + row_count int(11) DEFAULT '0' NOT NULL, + payload mediumtext, + PRIMARY KEY (uid) +);