VITEC-website/packages/vitec/Classes/Command/CreateMarketsCommand.php

<?php

declare(strict_types=1);

namespace Evomedien\Vitec\Command;

use Doctrine\DBAL\ParameterType;
use Evomedien\Vitec\Import\SeoResearchRepository;
use Evomedien\Vitec\Import\SeoResearchService;
use Symfony\Component\Console\Attribute\AsCommand;
use Symfony\Component\Console\Command\Command;
use Symfony\Component\Console\Input\InputInterface;
use Symfony\Component\Console\Input\InputOption;
use Symfony\Component\Console\Output\OutputInterface;
use TYPO3\CMS\Core\Core\Bootstrap;
use TYPO3\CMS\Core\Database\ConnectionPool;
use TYPO3\CMS\Core\DataHandling\DataHandler;
use TYPO3\CMS\Core\Utility\GeneralUtility;

/**
 * Creates the market records that the SEO-research structure check reports
 * as "Missing in TYPO3" - including their sys_category assignment.
 *
 * Source is the latest stored keyword-research delivery
 * (tx_vitec_seo_research), evaluated through the very same
 * SeoResearchService the backend module uses: whatever the module lists as
 * missing is exactly what this command creates. Per market:
 *
 *   - record:   title = CSV page name, slug = last URL segment,
 *               pid = pid of the existing market records (or --pid)
 *   - category: a sys_category of the same title, looked up anywhere under
 *               the market category root; created when absent. Sub-markets
 *               get their category created UNDER the parent market's
 *               category (Page Ref hierarchy, e.g. 1.1.1 under 1.1), so the
 *               category tree carries the hierarchy the flat market model
 *               cannot.
 *
 * The category root is auto-detected as the most common parent of the
 * categories assigned to existing markets; override with --category-parent.
 * If an existing parent market has no category yet, it is assigned the
 * (found or created) category, otherwise its sub-markets could not be
 * attached to the tree.
 *
 * Idempotent: matched markets are skipped; re-running creates nothing twice.
 *
 *   vendor/bin/typo3 vitec:create-markets --dry-run
 *   vendor/bin/typo3 vitec:create-markets --category-parent=42
 */
#[AsCommand(
    name: 'vitec:create-markets',
    description: 'Create market records missing vs. the stored SEO research, with sys_category assignment'
)]
final class CreateMarketsCommand extends Command
{
    private const TABLE = 'tx_vitec_domain_model_market';
    private const CATEGORY_TABLE = 'sys_category';

    protected function configure(): void
    {
        $this->addOption('dry-run', null, InputOption::VALUE_NONE, 'Report only - no records, no categories');
        $this->addOption('pid', null, InputOption::VALUE_REQUIRED, 'Storage pid for new market records (default: pid of existing markets)');
        $this->addOption('category-parent', null, InputOption::VALUE_REQUIRED, 'Uid of the market category root (default: auto-detect, then a category titled "Markets", else it is created)');
        $this->addOption('category-pid', null, InputOption::VALUE_REQUIRED, 'Sysfolder pid for newly created categories', '33');
    }

    protected function execute(InputInterface $input, OutputInterface $output): int
    {
        Bootstrap::initializeBackendAuthentication();
        $dryRun = (bool)$input->getOption('dry-run');

        // ------------------------------------------------ source: delivery
        $repository = GeneralUtility::makeInstance(SeoResearchRepository::class);
        $service = GeneralUtility::makeInstance(SeoResearchService::class);
        $delivery = $repository->latest(0);
        if ($delivery === null) {
            $output->writeln('<error>No SEO research delivery stored - upload the CSV in the backend module first.</error>');
            return Command::FAILURE;
        }
        $output->writeln(sprintf('Delivery: %s (%s)', $delivery['filename'], date('Y-m-d H:i', $delivery['crdate'])));

        $structure = $service->analyze($delivery['rows'], null)['structure']['market'];
        $missing = $structure['missing'];
        $both = $structure['both'];
        $output->writeln(sprintf('Markets: %d matched, %d missing, %d only in TYPO3',
            $structure['matched'], count($missing), count($structure['extra'])));
        if ($missing === []) {
            $output->writeln('Nothing to create.');
            return Command::SUCCESS;
        }

        // -------------------------------------------------- pid for records
        $pid = (int)($input->getOption('pid') ?? 0);
        if ($pid <= 0) {
            $pid = $this->detectMarketPid();
        }
        if ($pid <= 0) {
            $output->writeln('<error>No --pid given and no existing market records to derive it from.</error>');
            return Command::FAILURE;
        }

        // -------------------------------------- category root + existing map
        $categoryByMarketUid = $this->categoriesOfExistingMarkets();
        $categoryPid = (int)$input->getOption('category-pid');
        $categoryRoot = (int)($input->getOption('category-parent') ?? 0);
        if ($categoryRoot <= 0) {
            $categoryRoot = $this->detectCategoryRoot($categoryByMarketUid);
        }
        if ($categoryRoot <= 0) {
            // No market has a category yet: look for (or create) a root
            // category titled "Markets" on the category sysfolder.
            $categoryRoot = $this->findCategoryByTitle('Markets', $categoryPid);
            if ($categoryRoot > 0) {
                $output->writeln(sprintf('Using existing category "Markets" (uid %d) as root.', $categoryRoot));
            } elseif ($dryRun) {
                $output->writeln(sprintf('Would create root category "Markets" (pid %d, top level) and attach everything under it.', $categoryPid));
            } else {
                $dataHandler = GeneralUtility::makeInstance(DataHandler::class);
                $dataHandler->start([self::CATEGORY_TABLE => ['NEW1' => [
                    'pid' => $categoryPid,
                    'parent' => 0,
                    'title' => 'Markets',
                ]]], []);
                $dataHandler->process_datamap();
                $categoryRoot = (int)($dataHandler->substNEWwithIDs['NEW1'] ?? 0);
                if ($categoryRoot <= 0) {
                    $output->writeln('<error>Could not create the root category "Markets": '
                        . implode(' | ', $dataHandler->errorLog) . '</error>');
                    return Command::FAILURE;
                }
                $output->writeln(sprintf('<info>created root category "Markets" (uid %d, pid %d)</info>', $categoryRoot, $categoryPid));
            }
        }
        if ($categoryRoot > 0) {
            $rootRow = $this->categoryRow($categoryRoot);
            if ($rootRow === null) {
                $output->writeln(sprintf('<error>Category %d does not exist.</error>', $categoryRoot));
                return Command::FAILURE;
            }
            $categoryPid = (int)$rootRow['pid'];
            $output->writeln(sprintf('Storage pid: %d | category root: "%s" (uid %d)', $pid, $rootRow['title'], $categoryRoot));
        } else {
            $output->writeln(sprintf('Storage pid: %d | category root: "Markets" (created on the real run)', $pid));
        }

        $allCategories = $this->loadAllCategories();
        $treeUids = $this->subtreeUids($allCategories, $categoryRoot);

        // ref -> category uid for markets that already exist (via the both list)
        $categoryByRef = [];
        foreach ($both as $pair) {
            $catUid = $categoryByMarketUid[(int)$pair['uid']] ?? 0;
            if ($catUid > 0) {
                $categoryByRef[$pair['ref']] = $catUid;
            }
        }
        // market uid by ref, to fix parents without category
        $marketUidByRef = [];
        foreach ($both as $pair) {
            $marketUidByRef[$pair['ref']] = (int)$pair['uid'];
        }

        // ------------------------------------------------------------ create
        usort($missing, static fn(array $a, array $b): int =>
            substr_count($a['ref'], '.') <=> substr_count($b['ref'], '.'));

        $createdMarkets = 0;
        $createdCategories = 0;
        $warnings = [];

        foreach ($missing as $row) {
            $title = $row['name'];
            $slug = mb_strtolower(trim((string)basename(rtrim($row['url'], '/'))));
            $parentRef = str_contains($row['ref'], '.')
                ? substr($row['ref'], 0, (int)strrpos($row['ref'], '.'))
                : '';

            // Parent category: category of the parent market when the ref has
            // one; the root otherwise. Top-level market refs ("1.3") have
            // parentRef "1" = the section root -> attach to the category root.
            $parentCategory = $categoryRoot;
            if ($parentRef !== '' && str_contains($parentRef, '.')) {
                $parentCategory = $categoryByRef[$parentRef] ?? 0;
                if ($parentCategory <= 0 && isset($marketUidByRef[$parentRef])) {
                    // Existing parent market without category: create/find its
                    // category under the root and assign it, so the tree holds.
                    $parentTitle = $this->titleOfMarket($marketUidByRef[$parentRef]);
                    $parentCategory = $this->findOrCreateCategory(
                        $parentTitle, $categoryRoot, $categoryPid, $allCategories, $treeUids, $dryRun, $createdCategories, $output
                    );
                    if (!$dryRun && $parentCategory > 0) {
                        $this->assignCategory($marketUidByRef[$parentRef], $parentCategory, $output);
                    }
                    $categoryByRef[$parentRef] = $parentCategory;
                }
                if ($parentCategory <= 0) {
                    $warnings[] = sprintf('%s %s: parent %s not resolvable - category attached to the root instead.',
                        $row['ref'], $title, $parentRef);
                    $parentCategory = $categoryRoot;
                }
            }

            $categoryUid = $this->findOrCreateCategory(
                $title, $parentCategory, $categoryPid, $allCategories, $treeUids, $dryRun, $createdCategories, $output
            );

            if ($dryRun) {
                $parentLabel = $parentCategory > 0 ? '#' . $parentCategory : '"Markets" (new root)';
                $output->writeln(sprintf('  CREATE market %-8s %-45s slug=%s category=%s',
                    $row['ref'], mb_substr($title, 0, 45), $slug,
                    $categoryUid > 0 ? '#' . $categoryUid : '(new, under ' . $parentLabel . ')'));
                $createdMarkets++;
                $categoryByRef[$row['ref']] = $categoryUid;
                continue;
            }

            $dataHandler = GeneralUtility::makeInstance(DataHandler::class);
            $dataHandler->start([self::TABLE => ['NEW1' => [
                'pid' => $pid,
                'title' => $title,
                'slug' => $slug,
                'categories' => (string)$categoryUid,
            ]]], []);
            $dataHandler->process_datamap();
            if ($dataHandler->errorLog !== []) {
                $warnings[] = sprintf('%s: %s', $title, implode(' | ', $dataHandler->errorLog));
                continue;
            }
            $newUid = (int)($dataHandler->substNEWwithIDs['NEW1'] ?? 0);
            $categoryByRef[$row['ref']] = $categoryUid;
            $createdMarkets++;
            $output->writeln(sprintf('  <info>created market %s "%s" (uid %d, category %d)</info>',
                $row['ref'], $title, $newUid, $categoryUid));
        }

        // ------------------------------------------------------------ report
        $output->writeln('');
        $output->writeln(sprintf('%s: %d market(s), %d categor%s.',
            $dryRun ? 'DRY-RUN - would create' : 'Created',
            $createdMarkets, $createdCategories, $createdCategories === 1 ? 'y' : 'ies'));
        foreach ($warnings as $w) {
            $output->writeln('  <comment>! ' . $w . '</comment>');
        }
        return Command::SUCCESS;
    }

    // ------------------------------------------------------------ categories

    /**
     * Find a category by normalized title anywhere under the market category
     * root; create it under $parentCategory when absent.
     *
     * @param array<int,array<string,mixed>> $allCategories by uid
     * @param array<int,bool> $treeUids uids belonging to the root's subtree
     */
    private function findOrCreateCategory(
        string $title,
        int $parentCategory,
        int $categoryPid,
        array &$allCategories,
        array &$treeUids,
        bool $dryRun,
        int &$createdCategories,
        OutputInterface $output
    ): int {
        $norm = $this->normalizeTitle($title);
        foreach ($allCategories as $uid => $cat) {
            if (isset($treeUids[$uid]) && $this->normalizeTitle((string)$cat['title']) === $norm) {
                return (int)$uid;
            }
        }
        $createdCategories++;
        if ($dryRun) {
            $parentLabel = $parentCategory > 0 ? '#' . $parentCategory : '"Markets" (new root)';
            $output->writeln(sprintf('  create category %-42s under %s', mb_substr($title, 0, 42), $parentLabel));
            return 0;
        }
        $dataHandler = GeneralUtility::makeInstance(DataHandler::class);
        $dataHandler->start([self::CATEGORY_TABLE => ['NEW1' => [
            'pid' => $categoryPid,
            'parent' => $parentCategory,
            'title' => $title,
        ]]], []);
        $dataHandler->process_datamap();
        $uid = (int)($dataHandler->substNEWwithIDs['NEW1'] ?? 0);
        if ($uid > 0) {
            $allCategories[$uid] = ['uid' => $uid, 'parent' => $parentCategory, 'title' => $title, 'pid' => $categoryPid];
            $treeUids[$uid] = true;
            $output->writeln(sprintf('  <info>created category "%s" (uid %d, parent %d)</info>', $title, $uid, $parentCategory));
        }
        return $uid;
    }

    private function assignCategory(int $marketUid, int $categoryUid, OutputInterface $output): void
    {
        $dataHandler = GeneralUtility::makeInstance(DataHandler::class);
        $dataHandler->start([self::TABLE => [(string)$marketUid => ['categories' => (string)$categoryUid]]], []);
        $dataHandler->process_datamap();
        $output->writeln(sprintf('  <info>assigned category %d to existing market %d</info>', $categoryUid, $marketUid));
    }

    /** @return array<int,int> market uid => first assigned category uid */
    private function categoriesOfExistingMarkets(): array
    {
        $qb = GeneralUtility::makeInstance(ConnectionPool::class)->getQueryBuilderForTable('sys_category_record_mm');
        $rows = $qb->select('uid_local', 'uid_foreign')->from('sys_category_record_mm')
            ->where(
                $qb->expr()->eq('tablenames', $qb->createNamedParameter(self::TABLE, ParameterType::STRING)),
                $qb->expr()->eq('fieldname', $qb->createNamedParameter('categories', ParameterType::STRING))
            )
            ->orderBy('sorting', 'ASC')
            ->executeQuery()->fetchAllAssociative();
        $map = [];
        foreach ($rows as $row) {
            $market = (int)$row['uid_foreign'];
            if (!isset($map[$market])) {
                $map[$market] = (int)$row['uid_local'];
            }
        }
        return $map;
    }

    /** Most common parent of the categories assigned to existing markets. */
    private function detectCategoryRoot(array $categoryByMarketUid): int
    {
        if ($categoryByMarketUid === []) {
            return 0;
        }
        $qb = GeneralUtility::makeInstance(ConnectionPool::class)->getQueryBuilderForTable(self::CATEGORY_TABLE);
        $rows = $qb->select('uid', 'parent')->from(self::CATEGORY_TABLE)
            ->where(
                $qb->expr()->in('uid', array_map('intval', array_values($categoryByMarketUid))),
                $qb->expr()->eq('deleted', 0)
            )
            ->executeQuery()->fetchAllAssociative();
        $parents = [];
        foreach ($rows as $row) {
            $parents[(int)$row['parent']] = ($parents[(int)$row['parent']] ?? 0) + 1;
        }
        // Sub-market categories may already sit below a main-market category;
        // the ROOT is the most common parent among top-level assignments.
        arsort($parents);
        return (int)array_key_first($parents);
    }

    /** @return array<int,array<string,mixed>> */
    private function loadAllCategories(): array
    {
        $qb = GeneralUtility::makeInstance(ConnectionPool::class)->getQueryBuilderForTable(self::CATEGORY_TABLE);
        $rows = $qb->select('uid', 'pid', 'parent', 'title')->from(self::CATEGORY_TABLE)
            ->where($qb->expr()->eq('deleted', 0))
            ->executeQuery()->fetchAllAssociative();
        $out = [];
        foreach ($rows as $row) {
            $out[(int)$row['uid']] = $row;
        }
        return $out;
    }

    /** @return array<int,bool> every category uid inside the root's subtree (root included) */
    private function subtreeUids(array $allCategories, int $root): array
    {
        $children = [];
        foreach ($allCategories as $uid => $cat) {
            $children[(int)$cat['parent']][] = (int)$uid;
        }
        $result = [$root => true];
        $queue = [$root];
        while ($queue !== []) {
            $current = array_shift($queue);
            foreach ($children[$current] ?? [] as $childUid) {
                if (!isset($result[$childUid])) {
                    $result[$childUid] = true;
                    $queue[] = $childUid;
                }
            }
        }
        return $result;
    }

    /** Category by (normalized) title, preferring the given pid. */
    private function findCategoryByTitle(string $title, int $preferredPid): int
    {
        $qb = GeneralUtility::makeInstance(ConnectionPool::class)->getQueryBuilderForTable(self::CATEGORY_TABLE);
        $rows = $qb->select('uid', 'pid', 'title')->from(self::CATEGORY_TABLE)
            ->where($qb->expr()->eq('deleted', 0))
            ->executeQuery()->fetchAllAssociative();
        $norm = $this->normalizeTitle($title);
        $fallback = 0;
        foreach ($rows as $row) {
            if ($this->normalizeTitle((string)$row['title']) !== $norm) {
                continue;
            }
            if ((int)$row['pid'] === $preferredPid) {
                return (int)$row['uid'];
            }
            if ($fallback === 0) {
                $fallback = (int)$row['uid'];
            }
        }
        return $fallback;
    }

    /** @return array<string,mixed>|null */
    private function categoryRow(int $uid): ?array
    {
        $qb = GeneralUtility::makeInstance(ConnectionPool::class)->getQueryBuilderForTable(self::CATEGORY_TABLE);
        $row = $qb->select('uid', 'pid', 'title')->from(self::CATEGORY_TABLE)
            ->where(
                $qb->expr()->eq('uid', $qb->createNamedParameter($uid, ParameterType::INTEGER)),
                $qb->expr()->eq('deleted', 0)
            )
            ->executeQuery()->fetchAssociative();
        return $row ?: null;
    }

    // ----------------------------------------------------------------- misc

    private function detectMarketPid(): int
    {
        $qb = GeneralUtility::makeInstance(ConnectionPool::class)->getQueryBuilderForTable(self::TABLE);
        $row = $qb->select('pid')->from(self::TABLE)
            ->where($qb->expr()->eq('deleted', 0))
            ->setMaxResults(1)
            ->executeQuery()->fetchAssociative();
        return $row ? (int)$row['pid'] : 0;
    }

    private function titleOfMarket(int $uid): string
    {
        $qb = GeneralUtility::makeInstance(ConnectionPool::class)->getQueryBuilderForTable(self::TABLE);
        $row = $qb->select('title')->from(self::TABLE)
            ->where($qb->expr()->eq('uid', $qb->createNamedParameter($uid, ParameterType::INTEGER)))
            ->executeQuery()->fetchAssociative();
        return $row ? (string)$row['title'] : '';
    }

    private function normalizeTitle(string $title): string
    {
        $title = (string)preg_replace('/\s*\(.*?\)/', '', $title);
        $title = str_replace('&', 'and', mb_strtolower($title));
        $title = (string)preg_replace('/[^a-z0-9]+/', ' ', $title);
        return trim((string)preg_replace('/\s+/', ' ', $title));
    }
}