addOption('dry-run', null, InputOption::VALUE_NONE, 'Report only - no records, no categories');
$this->addOption('pid', null, InputOption::VALUE_REQUIRED, 'Storage pid for new market records (default: pid of existing markets)');
$this->addOption('category-parent', null, InputOption::VALUE_REQUIRED, 'Uid of the market category root (default: auto-detect, then a category titled "Markets", else it is created)');
$this->addOption('category-pid', null, InputOption::VALUE_REQUIRED, 'Sysfolder pid for newly created categories', '33');
}
protected function execute(InputInterface $input, OutputInterface $output): int
{
Bootstrap::initializeBackendAuthentication();
$dryRun = (bool)$input->getOption('dry-run');
// ------------------------------------------------ source: delivery
$repository = GeneralUtility::makeInstance(SeoResearchRepository::class);
$service = GeneralUtility::makeInstance(SeoResearchService::class);
$delivery = $repository->latest(0);
if ($delivery === null) {
$output->writeln('No SEO research delivery stored - upload the CSV in the backend module first.');
return Command::FAILURE;
}
$output->writeln(sprintf('Delivery: %s (%s)', $delivery['filename'], date('Y-m-d H:i', $delivery['crdate'])));
$structure = $service->analyze($delivery['rows'], null)['structure']['market'];
$missing = $structure['missing'];
$both = $structure['both'];
$output->writeln(sprintf('Markets: %d matched, %d missing, %d only in TYPO3',
$structure['matched'], count($missing), count($structure['extra'])));
if ($missing === []) {
$output->writeln('Nothing to create.');
return Command::SUCCESS;
}
// -------------------------------------------------- pid for records
$pid = (int)($input->getOption('pid') ?? 0);
if ($pid <= 0) {
$pid = $this->detectMarketPid();
}
if ($pid <= 0) {
$output->writeln('No --pid given and no existing market records to derive it from.');
return Command::FAILURE;
}
// -------------------------------------- category root + existing map
$categoryByMarketUid = $this->categoriesOfExistingMarkets();
$categoryPid = (int)$input->getOption('category-pid');
$categoryRoot = (int)($input->getOption('category-parent') ?? 0);
if ($categoryRoot <= 0) {
$categoryRoot = $this->detectCategoryRoot($categoryByMarketUid);
}
if ($categoryRoot <= 0) {
// No market has a category yet: look for (or create) a root
// category titled "Markets" on the category sysfolder.
$categoryRoot = $this->findCategoryByTitle('Markets', $categoryPid);
if ($categoryRoot > 0) {
$output->writeln(sprintf('Using existing category "Markets" (uid %d) as root.', $categoryRoot));
} elseif ($dryRun) {
$output->writeln(sprintf('Would create root category "Markets" (pid %d, top level) and attach everything under it.', $categoryPid));
} else {
$dataHandler = GeneralUtility::makeInstance(DataHandler::class);
$dataHandler->start([self::CATEGORY_TABLE => ['NEW1' => [
'pid' => $categoryPid,
'parent' => 0,
'title' => 'Markets',
]]], []);
$dataHandler->process_datamap();
$categoryRoot = (int)($dataHandler->substNEWwithIDs['NEW1'] ?? 0);
if ($categoryRoot <= 0) {
$output->writeln('Could not create the root category "Markets": '
. implode(' | ', $dataHandler->errorLog) . '');
return Command::FAILURE;
}
$output->writeln(sprintf('created root category "Markets" (uid %d, pid %d)', $categoryRoot, $categoryPid));
}
}
if ($categoryRoot > 0) {
$rootRow = $this->categoryRow($categoryRoot);
if ($rootRow === null) {
$output->writeln(sprintf('Category %d does not exist.', $categoryRoot));
return Command::FAILURE;
}
$categoryPid = (int)$rootRow['pid'];
$output->writeln(sprintf('Storage pid: %d | category root: "%s" (uid %d)', $pid, $rootRow['title'], $categoryRoot));
} else {
$output->writeln(sprintf('Storage pid: %d | category root: "Markets" (created on the real run)', $pid));
}
$allCategories = $this->loadAllCategories();
$treeUids = $this->subtreeUids($allCategories, $categoryRoot);
// ref -> category uid for markets that already exist (via the both list)
$categoryByRef = [];
foreach ($both as $pair) {
$catUid = $categoryByMarketUid[(int)$pair['uid']] ?? 0;
if ($catUid > 0) {
$categoryByRef[$pair['ref']] = $catUid;
}
}
// market uid by ref, to fix parents without category
$marketUidByRef = [];
foreach ($both as $pair) {
$marketUidByRef[$pair['ref']] = (int)$pair['uid'];
}
// ------------------------------------------------------------ create
usort($missing, static fn(array $a, array $b): int =>
substr_count($a['ref'], '.') <=> substr_count($b['ref'], '.'));
$createdMarkets = 0;
$createdCategories = 0;
$warnings = [];
foreach ($missing as $row) {
$title = $row['name'];
$slug = mb_strtolower(trim((string)basename(rtrim($row['url'], '/'))));
$parentRef = str_contains($row['ref'], '.')
? substr($row['ref'], 0, (int)strrpos($row['ref'], '.'))
: '';
// Parent category: category of the parent market when the ref has
// one; the root otherwise. Top-level market refs ("1.3") have
// parentRef "1" = the section root -> attach to the category root.
$parentCategory = $categoryRoot;
if ($parentRef !== '' && str_contains($parentRef, '.')) {
$parentCategory = $categoryByRef[$parentRef] ?? 0;
if ($parentCategory <= 0 && isset($marketUidByRef[$parentRef])) {
// Existing parent market without category: create/find its
// category under the root and assign it, so the tree holds.
$parentTitle = $this->titleOfMarket($marketUidByRef[$parentRef]);
$parentCategory = $this->findOrCreateCategory(
$parentTitle, $categoryRoot, $categoryPid, $allCategories, $treeUids, $dryRun, $createdCategories, $output
);
if (!$dryRun && $parentCategory > 0) {
$this->assignCategory($marketUidByRef[$parentRef], $parentCategory, $output);
}
$categoryByRef[$parentRef] = $parentCategory;
}
if ($parentCategory <= 0) {
$warnings[] = sprintf('%s %s: parent %s not resolvable - category attached to the root instead.',
$row['ref'], $title, $parentRef);
$parentCategory = $categoryRoot;
}
}
$categoryUid = $this->findOrCreateCategory(
$title, $parentCategory, $categoryPid, $allCategories, $treeUids, $dryRun, $createdCategories, $output
);
if ($dryRun) {
$parentLabel = $parentCategory > 0 ? '#' . $parentCategory : '"Markets" (new root)';
$output->writeln(sprintf(' CREATE market %-8s %-45s slug=%s category=%s',
$row['ref'], mb_substr($title, 0, 45), $slug,
$categoryUid > 0 ? '#' . $categoryUid : '(new, under ' . $parentLabel . ')'));
$createdMarkets++;
$categoryByRef[$row['ref']] = $categoryUid;
continue;
}
$dataHandler = GeneralUtility::makeInstance(DataHandler::class);
$dataHandler->start([self::TABLE => ['NEW1' => [
'pid' => $pid,
'title' => $title,
'slug' => $slug,
'categories' => (string)$categoryUid,
]]], []);
$dataHandler->process_datamap();
if ($dataHandler->errorLog !== []) {
$warnings[] = sprintf('%s: %s', $title, implode(' | ', $dataHandler->errorLog));
continue;
}
$newUid = (int)($dataHandler->substNEWwithIDs['NEW1'] ?? 0);
$categoryByRef[$row['ref']] = $categoryUid;
$createdMarkets++;
$output->writeln(sprintf(' created market %s "%s" (uid %d, category %d)',
$row['ref'], $title, $newUid, $categoryUid));
}
// ------------------------------------------------------------ report
$output->writeln('');
$output->writeln(sprintf('%s: %d market(s), %d categor%s.',
$dryRun ? 'DRY-RUN - would create' : 'Created',
$createdMarkets, $createdCategories, $createdCategories === 1 ? 'y' : 'ies'));
foreach ($warnings as $w) {
$output->writeln(' ! ' . $w . '');
}
return Command::SUCCESS;
}
// ------------------------------------------------------------ categories
/**
* Find a category by normalized title anywhere under the market category
* root; create it under $parentCategory when absent.
*
* @param array> $allCategories by uid
* @param array $treeUids uids belonging to the root's subtree
*/
private function findOrCreateCategory(
string $title,
int $parentCategory,
int $categoryPid,
array &$allCategories,
array &$treeUids,
bool $dryRun,
int &$createdCategories,
OutputInterface $output
): int {
$norm = $this->normalizeTitle($title);
foreach ($allCategories as $uid => $cat) {
if (isset($treeUids[$uid]) && $this->normalizeTitle((string)$cat['title']) === $norm) {
return (int)$uid;
}
}
$createdCategories++;
if ($dryRun) {
$parentLabel = $parentCategory > 0 ? '#' . $parentCategory : '"Markets" (new root)';
$output->writeln(sprintf(' create category %-42s under %s', mb_substr($title, 0, 42), $parentLabel));
return 0;
}
$dataHandler = GeneralUtility::makeInstance(DataHandler::class);
$dataHandler->start([self::CATEGORY_TABLE => ['NEW1' => [
'pid' => $categoryPid,
'parent' => $parentCategory,
'title' => $title,
]]], []);
$dataHandler->process_datamap();
$uid = (int)($dataHandler->substNEWwithIDs['NEW1'] ?? 0);
if ($uid > 0) {
$allCategories[$uid] = ['uid' => $uid, 'parent' => $parentCategory, 'title' => $title, 'pid' => $categoryPid];
$treeUids[$uid] = true;
$output->writeln(sprintf(' created category "%s" (uid %d, parent %d)', $title, $uid, $parentCategory));
}
return $uid;
}
private function assignCategory(int $marketUid, int $categoryUid, OutputInterface $output): void
{
$dataHandler = GeneralUtility::makeInstance(DataHandler::class);
$dataHandler->start([self::TABLE => [(string)$marketUid => ['categories' => (string)$categoryUid]]], []);
$dataHandler->process_datamap();
$output->writeln(sprintf(' assigned category %d to existing market %d', $categoryUid, $marketUid));
}
/** @return array market uid => first assigned category uid */
private function categoriesOfExistingMarkets(): array
{
$qb = GeneralUtility::makeInstance(ConnectionPool::class)->getQueryBuilderForTable('sys_category_record_mm');
$rows = $qb->select('uid_local', 'uid_foreign')->from('sys_category_record_mm')
->where(
$qb->expr()->eq('tablenames', $qb->createNamedParameter(self::TABLE, ParameterType::STRING)),
$qb->expr()->eq('fieldname', $qb->createNamedParameter('categories', ParameterType::STRING))
)
->orderBy('sorting', 'ASC')
->executeQuery()->fetchAllAssociative();
$map = [];
foreach ($rows as $row) {
$market = (int)$row['uid_foreign'];
if (!isset($map[$market])) {
$map[$market] = (int)$row['uid_local'];
}
}
return $map;
}
/** Most common parent of the categories assigned to existing markets. */
private function detectCategoryRoot(array $categoryByMarketUid): int
{
if ($categoryByMarketUid === []) {
return 0;
}
$qb = GeneralUtility::makeInstance(ConnectionPool::class)->getQueryBuilderForTable(self::CATEGORY_TABLE);
$rows = $qb->select('uid', 'parent')->from(self::CATEGORY_TABLE)
->where(
$qb->expr()->in('uid', array_map('intval', array_values($categoryByMarketUid))),
$qb->expr()->eq('deleted', 0)
)
->executeQuery()->fetchAllAssociative();
$parents = [];
foreach ($rows as $row) {
$parents[(int)$row['parent']] = ($parents[(int)$row['parent']] ?? 0) + 1;
}
// Sub-market categories may already sit below a main-market category;
// the ROOT is the most common parent among top-level assignments.
arsort($parents);
return (int)array_key_first($parents);
}
/** @return array> */
private function loadAllCategories(): array
{
$qb = GeneralUtility::makeInstance(ConnectionPool::class)->getQueryBuilderForTable(self::CATEGORY_TABLE);
$rows = $qb->select('uid', 'pid', 'parent', 'title')->from(self::CATEGORY_TABLE)
->where($qb->expr()->eq('deleted', 0))
->executeQuery()->fetchAllAssociative();
$out = [];
foreach ($rows as $row) {
$out[(int)$row['uid']] = $row;
}
return $out;
}
/** @return array every category uid inside the root's subtree (root included) */
private function subtreeUids(array $allCategories, int $root): array
{
$children = [];
foreach ($allCategories as $uid => $cat) {
$children[(int)$cat['parent']][] = (int)$uid;
}
$result = [$root => true];
$queue = [$root];
while ($queue !== []) {
$current = array_shift($queue);
foreach ($children[$current] ?? [] as $childUid) {
if (!isset($result[$childUid])) {
$result[$childUid] = true;
$queue[] = $childUid;
}
}
}
return $result;
}
/** Category by (normalized) title, preferring the given pid. */
private function findCategoryByTitle(string $title, int $preferredPid): int
{
$qb = GeneralUtility::makeInstance(ConnectionPool::class)->getQueryBuilderForTable(self::CATEGORY_TABLE);
$rows = $qb->select('uid', 'pid', 'title')->from(self::CATEGORY_TABLE)
->where($qb->expr()->eq('deleted', 0))
->executeQuery()->fetchAllAssociative();
$norm = $this->normalizeTitle($title);
$fallback = 0;
foreach ($rows as $row) {
if ($this->normalizeTitle((string)$row['title']) !== $norm) {
continue;
}
if ((int)$row['pid'] === $preferredPid) {
return (int)$row['uid'];
}
if ($fallback === 0) {
$fallback = (int)$row['uid'];
}
}
return $fallback;
}
/** @return array|null */
private function categoryRow(int $uid): ?array
{
$qb = GeneralUtility::makeInstance(ConnectionPool::class)->getQueryBuilderForTable(self::CATEGORY_TABLE);
$row = $qb->select('uid', 'pid', 'title')->from(self::CATEGORY_TABLE)
->where(
$qb->expr()->eq('uid', $qb->createNamedParameter($uid, ParameterType::INTEGER)),
$qb->expr()->eq('deleted', 0)
)
->executeQuery()->fetchAssociative();
return $row ?: null;
}
// ----------------------------------------------------------------- misc
private function detectMarketPid(): int
{
$qb = GeneralUtility::makeInstance(ConnectionPool::class)->getQueryBuilderForTable(self::TABLE);
$row = $qb->select('pid')->from(self::TABLE)
->where($qb->expr()->eq('deleted', 0))
->setMaxResults(1)
->executeQuery()->fetchAssociative();
return $row ? (int)$row['pid'] : 0;
}
private function titleOfMarket(int $uid): string
{
$qb = GeneralUtility::makeInstance(ConnectionPool::class)->getQueryBuilderForTable(self::TABLE);
$row = $qb->select('title')->from(self::TABLE)
->where($qb->expr()->eq('uid', $qb->createNamedParameter($uid, ParameterType::INTEGER)))
->executeQuery()->fetchAssociative();
return $row ? (string)$row['title'] : '';
}
private function normalizeTitle(string $title): string
{
$title = (string)preg_replace('/\s*\(.*?\)/', '', $title);
$title = str_replace('&', 'and', mb_strtolower($title));
$title = (string)preg_replace('/[^a-z0-9]+/', ' ', $title);
return trim((string)preg_replace('/\s+/', ' ', $title));
}
}