addOption('file', null, InputOption::VALUE_REQUIRED, 'Export JSON', 'migrations/news_export.json'); $this->addOption('dry-run', null, InputOption::VALUE_NONE, 'Report only - nothing is written'); $this->addOption('pid', null, InputOption::VALUE_REQUIRED, 'Storage pid for the news records (0 = auto-detect)', '0'); $this->addOption('force', null, InputOption::VALUE_NONE, 'Update records that already exist'); $this->addOption('only', null, InputOption::VALUE_REQUIRED, 'Import only the record with this path_segment'); $this->addOption('category-parent', null, InputOption::VALUE_REQUIRED, 'Parent uid for categories that have to be created (0 = root)', '0'); $this->addOption('skip-categories', null, InputOption::VALUE_NONE, 'Do not touch category assignments'); $this->addOption('category', null, InputOption::VALUE_REQUIRED, 'Assign these sys_category uids (comma separated) to every record and ignore the old taxonomy'); } protected function execute(InputInterface $input, OutputInterface $output): int { // DataHandler checks backend permissions, and on CLI there is no backend // user - without this every write fails with "Attempt to modify table // ... without permission". Same first line as the other import commands. Bootstrap::initializeBackendAuthentication(); $dryRun = (bool)$input->getOption('dry-run'); if ($dryRun) { $output->writeln('DRY RUN - nothing will be written.'); } $export = $this->loadExport((string)$input->getOption('file'), $output); if ($export === null) { return Command::FAILURE; } $news = $export['news'] ?? []; $output->writeln(sprintf('Export: %d news records, written %s', count($news), (string)($export['meta']['exportedAt'] ?? '?'))); $pid = (int)$input->getOption('pid') ?: $this->detectPid(); if ($pid <= 0) { $output->writeln('No storage pid found. Pass --pid= with the news sysfolder.'); return Command::FAILURE; } $output->writeln(sprintf('Storage pid: %d', $pid)); $pageMap = $this->buildPageMap(); $output->writeln(sprintf('Imported pages found via tx_impexp_origuid: %d', count($pageMap))); if ($pageMap === []) { $output->writeln('No imported pages found - every "page as news" record will be skipped.'); } // A fixed category list wins over the old taxonomy. The source rubrics // are a 2012-era scheme whose leaf names collide with the market tree // here, so pinning everything to one editorial category is both safer // and closer to how the new site is organised. $fixedCategories = $this->parseUidList((string)$input->getOption('category')); $categoryMap = []; if ($fixedCategories !== '') { $output->writeln(sprintf('Categories: every record gets %s; the old taxonomy is ignored.', $fixedCategories)); $this->reportCategoryTitles($fixedCategories, $output); } elseif (!$input->getOption('skip-categories')) { $categoryMap = $this->buildCategoryMap($export, (int)$input->getOption('category-parent'), $pid, $dryRun, $output); } $only = (string)$input->getOption('only'); $force = (bool)$input->getOption('force'); // Pass 1 - default language. Translations need their parent to exist. $created = $updated = $existing = $skipped = 0; $uidByOldUid = []; $missingPages = []; foreach ($news as $row) { if ((int)($row['sys_language_uid'] ?? 0) > 0) { continue; } if ($only !== '' && (string)($row['path_segment'] ?? '') !== $only) { continue; } $oldUid = (int)$row['uid']; $existingUid = $this->findByImportId($oldUid); if ($existingUid > 0) { $uidByOldUid[$oldUid] = $existingUid; if (!$force) { $existing++; continue; } } $data = $this->buildPayload($row, $pid, $categoryMap, $export, $fixedCategories); if ((int)($row['type'] ?? 0) === 1) { $oldPage = $this->extractPageUid((string)($row['internalurl'] ?? '')); $newPage = $pageMap[$oldPage] ?? 0; if ($newPage === 0) { $missingPages[] = sprintf('%s (old page %d)', (string)$row['path_segment'], $oldPage); $skipped++; continue; } $data['internalurl'] = 't3://page?uid=' . $newPage; $output->writeln(sprintf(' page-news %-52s -> page %d', mb_substr((string)$row['path_segment'], 0, 52), $newPage)); } $newUid = $this->write($existingUid, $data, $dryRun, $output, (string)$row['path_segment']); if ($newUid === 0) { continue; } $uidByOldUid[$oldUid] = $newUid; $existingUid > 0 ? $updated++ : $created++; } // Pass 2 - translations, now that every parent has a uid on this side. $translations = 0; foreach ($news as $row) { if ((int)($row['sys_language_uid'] ?? 0) <= 0) { continue; } $parentOld = (int)($row['l10n_parent'] ?? 0); $parentNew = $uidByOldUid[$parentOld] ?? $this->findByImportId($parentOld); if ($parentNew === 0) { $output->writeln(sprintf(' translation of old uid %d skipped: parent not imported', $parentOld)); $skipped++; continue; } $oldUid = (int)$row['uid']; $existingUid = $this->findByImportId($oldUid); if ($existingUid > 0 && !$force) { $existing++; continue; } $data = $this->buildPayload($row, $pid, $categoryMap, $export, $fixedCategories); $data['sys_language_uid'] = (int)$row['sys_language_uid']; $data['l10n_parent'] = $parentNew; if ($this->write($existingUid, $data, $dryRun, $output, (string)$row['path_segment']) > 0) { $translations++; } } $output->writeln(''); $output->writeln(sprintf('%s: %d created, %d updated, %d already present, %d translations, %d skipped', $dryRun ? 'DRY-RUN' : 'Done', $created, $updated, $existing, $translations, $skipped)); foreach ($missingPages as $line) { $output->writeln('no imported page for: ' . $line . ''); } if (!$force && $existing > 0) { $output->writeln('Use --force to update the records that already exist.'); } return Command::SUCCESS; } /** @return array|null */ private function loadExport(string $file, OutputInterface $output): ?array { $path = $file; if (!is_file($path)) { $path = rtrim(\TYPO3\CMS\Core\Core\Environment::getProjectPath(), '/') . '/' . ltrim($file, '/'); } if (!is_file($path)) { $output->writeln(sprintf('Export not found: %s', $file)); return null; } $data = json_decode((string)file_get_contents($path), true); if (!is_array($data) || !isset($data['news'])) { $output->writeln('Export could not be parsed or carries no "news" block.'); return null; } return $data; } /** * Payload for one record. Empty strings are kept out so a field the export * dropped as empty does not overwrite something an editor filled in here. * * @param array $row * @param array $categoryMap * @param array $export * @return array */ private function buildPayload(array $row, int $pid, array $categoryMap, array $export, string $fixedCategories = ''): array { $data = [ 'pid' => $pid, 'hidden' => (int)($row['hidden'] ?? 0), 'import_source' => self::IMPORT_SOURCE, 'import_id' => (int)$row['uid'], ]; foreach (self::PLAIN_FIELDS as $field) { if (array_key_exists($field, $row) && $row[$field] !== null && $row[$field] !== '') { $data[$field] = $row[$field]; } } if ($fixedCategories !== '') { $data['categories'] = $fixedCategories; } elseif ($categoryMap !== []) { $categories = []; foreach ($export['categoryAssignments'] ?? [] as $mm) { if ((int)$mm['uid_foreign'] !== (int)$row['uid']) { continue; } $mapped = $categoryMap[(int)$mm['category_uid']] ?? 0; if ($mapped > 0) { $categories[] = $mapped; } } if ($categories !== []) { $data['categories'] = implode(',', array_unique($categories)); } } return $data; } /** @param array $data */ private function write(int $existingUid, array $data, bool $dryRun, OutputInterface $output, string $label): int { if ($dryRun) { return $existingUid > 0 ? $existingUid : -1; } $id = $existingUid > 0 ? (string)$existingUid : 'NEW' . substr(md5($label . microtime()), 0, 12); $dataHandler = GeneralUtility::makeInstance(DataHandler::class); $dataHandler->start([self::TABLE => [$id => $data]], []); $dataHandler->process_datamap(); if ($dataHandler->errorLog !== []) { $output->writeln(sprintf(' %s: %s', $label, implode(' | ', $dataHandler->errorLog))); return 0; } return $existingUid > 0 ? $existingUid : (int)($dataHandler->substNEWwithIDs[$id] ?? 0); } /** old page uid (from tx_impexp_origuid) => uid on this site */ private function buildPageMap(): array { $qb = GeneralUtility::makeInstance(ConnectionPool::class)->getQueryBuilderForTable('pages'); $qb->getRestrictions()->removeAll(); $rows = $qb->select('uid', 'tx_impexp_origuid')->from('pages') ->where( $qb->expr()->gt('tx_impexp_origuid', $qb->createNamedParameter(0, ParameterType::INTEGER)), $qb->expr()->eq('deleted', 0) ) ->executeQuery()->fetchAllAssociative(); $map = []; foreach ($rows as $row) { $map[(int)$row['tx_impexp_origuid']] = (int)$row['uid']; } return $map; } /** `t3://page?uid=N` or a bare uid - both forms occur in the source data. */ private function extractPageUid(string $internalUrl): int { $internalUrl = trim($internalUrl); if (preg_match('#t3://page\?uid=(\d+)#i', $internalUrl, $m)) { return (int)$m[1]; } return ctype_digit($internalUrl) ? (int)$internalUrl : 0; } /** * Old category uid => uid here. Matched by title, because uids are * meaningless across installations. Missing categories are created below * --category-parent so the assignments do not silently vanish. * * @param array $export * @return array */ private function buildCategoryMap(array $export, int $parent, int $pid, bool $dryRun, OutputInterface $output): array { $categories = $export['categories'] ?? []; if ($categories === []) { return []; } $qb = GeneralUtility::makeInstance(ConnectionPool::class)->getQueryBuilderForTable(self::CATEGORY_TABLE); $qb->getRestrictions()->removeAll(); $rows = $qb->select('uid', 'title', 'parent')->from(self::CATEGORY_TABLE) ->where($qb->expr()->eq('deleted', 0)) ->executeQuery()->fetchAllAssociative(); $titleByUid = []; foreach ($rows as $row) { $titleByUid[(int)$row['uid']] = (string)$row['title']; } // With --category-parent the title match is restricted to that subtree. // Matching site-wide is what makes this dangerous: the old news rubrics // carry market names (Education, Government, Healthcare, ...) that also // exist in the market taxonomy, and a bare title match would file press // releases under markets. $scope = $parent > 0 ? $this->collectDescendants($parent, $rows) : null; $byTitle = []; foreach ($rows as $row) { if ($scope !== null && !isset($scope[(int)$row['uid']])) { continue; } $byTitle[mb_strtolower(trim((string)$row['title']))] = (int)$row['uid']; } if ($scope !== null) { $output->writeln(sprintf('Category match restricted to the subtree below uid %d ("%s"), %d candidates.', $parent, $titleByUid[$parent] ?? '?', count($byTitle))); } else { $output->writeln('Category match is site-wide - pass --category-parent= to restrict it to one subtree.'); } $map = []; $found = $missing = 0; foreach ($categories as $cat) { $title = trim((string)$cat['title']); $key = mb_strtolower($title); if (isset($byTitle[$key])) { $hit = $byTitle[$key]; $map[(int)$cat['uid']] = $hit; $found++; $parentUid = 0; foreach ($rows as $row) { if ((int)$row['uid'] === $hit) { $parentUid = (int)$row['parent']; break; } } $output->writeln(sprintf(' category "%s" -> uid %d (below "%s")', $title, $hit, $parentUid > 0 ? ($titleByUid[$parentUid] ?? '?') : 'root')); continue; } $output->writeln(sprintf(' category "%s" does not exist here%s', $title, $dryRun ? '' : ' - creating')); $missing++; if ($dryRun) { continue; } $id = 'NEW' . substr(md5('cat' . $title), 0, 12); $dataHandler = GeneralUtility::makeInstance(DataHandler::class); $dataHandler->start([self::CATEGORY_TABLE => [$id => ['pid' => $parent > 0 ? 0 : $pid, 'parent' => $parent, 'title' => $title]]], []); $dataHandler->process_datamap(); $newUid = (int)($dataHandler->substNEWwithIDs[$id] ?? 0); if ($newUid > 0) { $map[(int)$cat['uid']] = $newUid; $byTitle[$key] = $newUid; } } $output->writeln(sprintf('Categories: %d matched, %d missing', $found, $missing)); return $map; } /** * The category plus every descendant, as a uid => true lookup. Iterative and * seen-guarded so a parent pointing back up the tree cannot loop. * * @param array> $rows * @return array */ private function collectDescendants(int $uid, array $rows): array { $childrenByParent = []; foreach ($rows as $row) { $childrenByParent[(int)$row['parent']][] = (int)$row['uid']; } $collected = [$uid => true]; $queue = [$uid]; while ($queue !== []) { $current = array_shift($queue); foreach ($childrenByParent[$current] ?? [] as $child) { if (isset($collected[$child])) { continue; } $collected[$child] = true; $queue[] = $child; } } return $collected; } /** "54, 55" -> "54,55"; anything non-numeric is dropped. */ private function parseUidList(string $value): string { $uids = []; foreach (explode(',', $value) as $part) { $part = trim($part); if ($part !== '' && ctype_digit($part) && (int)$part > 0) { $uids[] = (int)$part; } } return implode(',', array_unique($uids)); } /** * Prints the titles behind the uids given via --category, so a typo shows up * in the dry run instead of after 218 records have been filed wrongly. */ private function reportCategoryTitles(string $uidList, OutputInterface $output): void { foreach (explode(',', $uidList) as $uid) { $qb = GeneralUtility::makeInstance(ConnectionPool::class)->getQueryBuilderForTable(self::CATEGORY_TABLE); $qb->getRestrictions()->removeAll(); $row = $qb->select('uid', 'title', 'parent')->from(self::CATEGORY_TABLE) ->where( $qb->expr()->eq('uid', $qb->createNamedParameter((int)$uid, ParameterType::INTEGER)), $qb->expr()->eq('deleted', 0) ) ->executeQuery()->fetchAssociative(); if (!$row) { $output->writeln(sprintf(' uid %s: no such category', $uid)); continue; } $output->writeln(sprintf(' uid %s = "%s"', $uid, (string)$row['title'])); } } private function findByImportId(int $oldUid): int { $qb = GeneralUtility::makeInstance(ConnectionPool::class)->getQueryBuilderForTable(self::TABLE); $qb->getRestrictions()->removeAll(); $uid = $qb->select('uid')->from(self::TABLE) ->where( $qb->expr()->eq('import_id', $qb->createNamedParameter($oldUid, ParameterType::INTEGER)), $qb->expr()->eq('import_source', $qb->createNamedParameter(self::IMPORT_SOURCE)), $qb->expr()->eq('deleted', 0) ) ->setMaxResults(1)->executeQuery()->fetchOne(); return (int)($uid ?: 0); } /** Most-used pid of the news records already here; 0 when there are none. */ private function detectPid(): int { $qb = GeneralUtility::makeInstance(ConnectionPool::class)->getQueryBuilderForTable(self::TABLE); $qb->getRestrictions()->removeAll(); $rows = $qb->select('pid')->addSelectLiteral('COUNT(*) AS cnt')->from(self::TABLE) ->where($qb->expr()->eq('deleted', 0)) ->groupBy('pid')->orderBy('cnt', 'DESC')->setMaxResults(1) ->executeQuery()->fetchAssociative(); return (int)($rows['pid'] ?? 0); } }