492 lines
21 KiB
PHP
492 lines
21 KiB
PHP
<?php
|
|
|
|
declare(strict_types=1);
|
|
|
|
namespace Evomedien\Vitec\Command;
|
|
|
|
use Doctrine\DBAL\ParameterType;
|
|
use Symfony\Component\Console\Attribute\AsCommand;
|
|
use Symfony\Component\Console\Command\Command;
|
|
use Symfony\Component\Console\Input\InputInterface;
|
|
use Symfony\Component\Console\Input\InputOption;
|
|
use Symfony\Component\Console\Output\OutputInterface;
|
|
use TYPO3\CMS\Core\Core\Bootstrap;
|
|
use TYPO3\CMS\Core\Database\ConnectionPool;
|
|
use TYPO3\CMS\Core\DataHandling\DataHandler;
|
|
use TYPO3\CMS\Core\Utility\GeneralUtility;
|
|
|
|
/**
|
|
* Imports the news of the old vitec.com into EXT:news on this installation.
|
|
*
|
|
* Source is migrations/news_export.json, written by migrations/export_news.php
|
|
* on the old site (plain PDO, read-only). Both installations run EXT:news, so
|
|
* this is a record-level migration rather than a translation between models.
|
|
*
|
|
* Identity: `import_source` + `import_id` — the two fields EXT:news carries for
|
|
* exactly this purpose. Matching on them rather than on `path_segment` keeps a
|
|
* re-run correct even after the editors have changed a slug. Existing records
|
|
* are left alone unless --force is given, so the command can be run repeatedly
|
|
* while the old site keeps publishing.
|
|
*
|
|
* Two record types come over:
|
|
* - type 0 (171 records) — ordinary articles, everything is in the record
|
|
* - type 1 (47 records) — "page as news": the body lives on a TYPO3 page that
|
|
* was brought over by T3D import beforehand. `internalurl` still points at
|
|
* the page uid of the OLD site, so it is rewritten via `tx_impexp_origuid`,
|
|
* which impexp wrote on every imported page. A record whose page cannot be
|
|
* found is skipped and reported — a news item linking into the void is worse
|
|
* than a visible gap.
|
|
*
|
|
* Deliberately not imported: tags (the tag table is empty on the source, the 149
|
|
* MM rows are orphans of a deleted record), `related` and `related_links` (none
|
|
* exist), and the single `fal_media` reference (one PDF, faster by hand).
|
|
*
|
|
* ALWAYS run --dry-run first.
|
|
*/
|
|
#[AsCommand(
|
|
name: 'vitec:import-news',
|
|
description: 'Imports the old-site news into EXT:news, linking the "page as news" records to their imported pages.'
|
|
)]
|
|
class ImportNewsCommand extends Command
|
|
{
|
|
private const TABLE = 'tx_news_domain_model_news';
|
|
private const CATEGORY_TABLE = 'sys_category';
|
|
private const IMPORT_SOURCE = 'vitec-legacy';
|
|
|
|
/** Scalar fields carried over 1:1 when present in the export. */
|
|
private const PLAIN_FIELDS = [
|
|
'title', 'teaser', 'bodytext', 'datetime', 'archive', 'istopnews',
|
|
'author', 'author_email', 'path_segment', 'type', 'externalurl',
|
|
'description', 'keywords', 'alternative_title', 'sitemap_changefreq',
|
|
'sitemap_priority',
|
|
];
|
|
|
|
protected function configure(): void
|
|
{
|
|
$this->addOption('file', null, InputOption::VALUE_REQUIRED, 'Export JSON', 'migrations/news_export.json');
|
|
$this->addOption('dry-run', null, InputOption::VALUE_NONE, 'Report only - nothing is written');
|
|
$this->addOption('pid', null, InputOption::VALUE_REQUIRED, 'Storage pid for the news records (0 = auto-detect)', '0');
|
|
$this->addOption('force', null, InputOption::VALUE_NONE, 'Update records that already exist');
|
|
$this->addOption('only', null, InputOption::VALUE_REQUIRED, 'Import only the record with this path_segment');
|
|
$this->addOption('category-parent', null, InputOption::VALUE_REQUIRED, 'Parent uid for categories that have to be created (0 = root)', '0');
|
|
$this->addOption('skip-categories', null, InputOption::VALUE_NONE, 'Do not touch category assignments');
|
|
$this->addOption('category', null, InputOption::VALUE_REQUIRED, 'Assign these sys_category uids (comma separated) to every record and ignore the old taxonomy');
|
|
}
|
|
|
|
protected function execute(InputInterface $input, OutputInterface $output): int
|
|
{
|
|
// DataHandler checks backend permissions, and on CLI there is no backend
|
|
// user - without this every write fails with "Attempt to modify table
|
|
// ... without permission". Same first line as the other import commands.
|
|
Bootstrap::initializeBackendAuthentication();
|
|
|
|
$dryRun = (bool)$input->getOption('dry-run');
|
|
if ($dryRun) {
|
|
$output->writeln('<comment>DRY RUN - nothing will be written.</comment>');
|
|
}
|
|
|
|
$export = $this->loadExport((string)$input->getOption('file'), $output);
|
|
if ($export === null) {
|
|
return Command::FAILURE;
|
|
}
|
|
$news = $export['news'] ?? [];
|
|
$output->writeln(sprintf('Export: <info>%d</info> news records, written %s', count($news), (string)($export['meta']['exportedAt'] ?? '?')));
|
|
|
|
$pid = (int)$input->getOption('pid') ?: $this->detectPid();
|
|
if ($pid <= 0) {
|
|
$output->writeln('<error>No storage pid found. Pass --pid=<uid> with the news sysfolder.</error>');
|
|
return Command::FAILURE;
|
|
}
|
|
$output->writeln(sprintf('Storage pid: <info>%d</info>', $pid));
|
|
|
|
$pageMap = $this->buildPageMap();
|
|
$output->writeln(sprintf('Imported pages found via tx_impexp_origuid: <info>%d</info>', count($pageMap)));
|
|
if ($pageMap === []) {
|
|
$output->writeln('<comment>No imported pages found - every "page as news" record will be skipped.</comment>');
|
|
}
|
|
|
|
// A fixed category list wins over the old taxonomy. The source rubrics
|
|
// are a 2012-era scheme whose leaf names collide with the market tree
|
|
// here, so pinning everything to one editorial category is both safer
|
|
// and closer to how the new site is organised.
|
|
$fixedCategories = $this->parseUidList((string)$input->getOption('category'));
|
|
$categoryMap = [];
|
|
if ($fixedCategories !== '') {
|
|
$output->writeln(sprintf('Categories: every record gets <info>%s</info>; the old taxonomy is ignored.', $fixedCategories));
|
|
$this->reportCategoryTitles($fixedCategories, $output);
|
|
} elseif (!$input->getOption('skip-categories')) {
|
|
$categoryMap = $this->buildCategoryMap($export, (int)$input->getOption('category-parent'), $pid, $dryRun, $output);
|
|
}
|
|
|
|
$only = (string)$input->getOption('only');
|
|
$force = (bool)$input->getOption('force');
|
|
|
|
// Pass 1 - default language. Translations need their parent to exist.
|
|
$created = $updated = $existing = $skipped = 0;
|
|
$uidByOldUid = [];
|
|
$missingPages = [];
|
|
|
|
foreach ($news as $row) {
|
|
if ((int)($row['sys_language_uid'] ?? 0) > 0) {
|
|
continue;
|
|
}
|
|
if ($only !== '' && (string)($row['path_segment'] ?? '') !== $only) {
|
|
continue;
|
|
}
|
|
|
|
$oldUid = (int)$row['uid'];
|
|
$existingUid = $this->findByImportId($oldUid);
|
|
if ($existingUid > 0) {
|
|
$uidByOldUid[$oldUid] = $existingUid;
|
|
if (!$force) {
|
|
$existing++;
|
|
continue;
|
|
}
|
|
}
|
|
|
|
$data = $this->buildPayload($row, $pid, $categoryMap, $export, $fixedCategories);
|
|
|
|
if ((int)($row['type'] ?? 0) === 1) {
|
|
$oldPage = $this->extractPageUid((string)($row['internalurl'] ?? ''));
|
|
$newPage = $pageMap[$oldPage] ?? 0;
|
|
if ($newPage === 0) {
|
|
$missingPages[] = sprintf('%s (old page %d)', (string)$row['path_segment'], $oldPage);
|
|
$skipped++;
|
|
continue;
|
|
}
|
|
$data['internalurl'] = 't3://page?uid=' . $newPage;
|
|
$output->writeln(sprintf(' <info>page-news</info> %-52s -> page %d', mb_substr((string)$row['path_segment'], 0, 52), $newPage));
|
|
}
|
|
|
|
$newUid = $this->write($existingUid, $data, $dryRun, $output, (string)$row['path_segment']);
|
|
if ($newUid === 0) {
|
|
continue;
|
|
}
|
|
$uidByOldUid[$oldUid] = $newUid;
|
|
$existingUid > 0 ? $updated++ : $created++;
|
|
}
|
|
|
|
// Pass 2 - translations, now that every parent has a uid on this side.
|
|
$translations = 0;
|
|
foreach ($news as $row) {
|
|
if ((int)($row['sys_language_uid'] ?? 0) <= 0) {
|
|
continue;
|
|
}
|
|
$parentOld = (int)($row['l10n_parent'] ?? 0);
|
|
$parentNew = $uidByOldUid[$parentOld] ?? $this->findByImportId($parentOld);
|
|
if ($parentNew === 0) {
|
|
$output->writeln(sprintf(' <comment>translation of old uid %d skipped: parent not imported</comment>', $parentOld));
|
|
$skipped++;
|
|
continue;
|
|
}
|
|
$oldUid = (int)$row['uid'];
|
|
$existingUid = $this->findByImportId($oldUid);
|
|
if ($existingUid > 0 && !$force) {
|
|
$existing++;
|
|
continue;
|
|
}
|
|
$data = $this->buildPayload($row, $pid, $categoryMap, $export, $fixedCategories);
|
|
$data['sys_language_uid'] = (int)$row['sys_language_uid'];
|
|
$data['l10n_parent'] = $parentNew;
|
|
if ($this->write($existingUid, $data, $dryRun, $output, (string)$row['path_segment']) > 0) {
|
|
$translations++;
|
|
}
|
|
}
|
|
|
|
$output->writeln('');
|
|
$output->writeln(sprintf('%s: <info>%d created</info>, %d updated, %d already present, %d translations, %d skipped',
|
|
$dryRun ? 'DRY-RUN' : 'Done', $created, $updated, $existing, $translations, $skipped));
|
|
foreach ($missingPages as $line) {
|
|
$output->writeln('<comment>no imported page for: ' . $line . '</comment>');
|
|
}
|
|
if (!$force && $existing > 0) {
|
|
$output->writeln('<comment>Use --force to update the records that already exist.</comment>');
|
|
}
|
|
|
|
return Command::SUCCESS;
|
|
}
|
|
|
|
/** @return array<string,mixed>|null */
|
|
private function loadExport(string $file, OutputInterface $output): ?array
|
|
{
|
|
$path = $file;
|
|
if (!is_file($path)) {
|
|
$path = rtrim(\TYPO3\CMS\Core\Core\Environment::getProjectPath(), '/') . '/' . ltrim($file, '/');
|
|
}
|
|
if (!is_file($path)) {
|
|
$output->writeln(sprintf('<error>Export not found: %s</error>', $file));
|
|
return null;
|
|
}
|
|
$data = json_decode((string)file_get_contents($path), true);
|
|
if (!is_array($data) || !isset($data['news'])) {
|
|
$output->writeln('<error>Export could not be parsed or carries no "news" block.</error>');
|
|
return null;
|
|
}
|
|
return $data;
|
|
}
|
|
|
|
/**
|
|
* Payload for one record. Empty strings are kept out so a field the export
|
|
* dropped as empty does not overwrite something an editor filled in here.
|
|
*
|
|
* @param array<string,mixed> $row
|
|
* @param array<int,int> $categoryMap
|
|
* @param array<string,mixed> $export
|
|
* @return array<string,mixed>
|
|
*/
|
|
private function buildPayload(array $row, int $pid, array $categoryMap, array $export, string $fixedCategories = ''): array
|
|
{
|
|
$data = [
|
|
'pid' => $pid,
|
|
'hidden' => (int)($row['hidden'] ?? 0),
|
|
'import_source' => self::IMPORT_SOURCE,
|
|
'import_id' => (int)$row['uid'],
|
|
];
|
|
foreach (self::PLAIN_FIELDS as $field) {
|
|
if (array_key_exists($field, $row) && $row[$field] !== null && $row[$field] !== '') {
|
|
$data[$field] = $row[$field];
|
|
}
|
|
}
|
|
if ($fixedCategories !== '') {
|
|
$data['categories'] = $fixedCategories;
|
|
} elseif ($categoryMap !== []) {
|
|
$categories = [];
|
|
foreach ($export['categoryAssignments'] ?? [] as $mm) {
|
|
if ((int)$mm['uid_foreign'] !== (int)$row['uid']) {
|
|
continue;
|
|
}
|
|
$mapped = $categoryMap[(int)$mm['category_uid']] ?? 0;
|
|
if ($mapped > 0) {
|
|
$categories[] = $mapped;
|
|
}
|
|
}
|
|
if ($categories !== []) {
|
|
$data['categories'] = implode(',', array_unique($categories));
|
|
}
|
|
}
|
|
return $data;
|
|
}
|
|
|
|
/** @param array<string,mixed> $data */
|
|
private function write(int $existingUid, array $data, bool $dryRun, OutputInterface $output, string $label): int
|
|
{
|
|
if ($dryRun) {
|
|
return $existingUid > 0 ? $existingUid : -1;
|
|
}
|
|
$id = $existingUid > 0 ? (string)$existingUid : 'NEW' . substr(md5($label . microtime()), 0, 12);
|
|
$dataHandler = GeneralUtility::makeInstance(DataHandler::class);
|
|
$dataHandler->start([self::TABLE => [$id => $data]], []);
|
|
$dataHandler->process_datamap();
|
|
if ($dataHandler->errorLog !== []) {
|
|
$output->writeln(sprintf(' <error>%s: %s</error>', $label, implode(' | ', $dataHandler->errorLog)));
|
|
return 0;
|
|
}
|
|
return $existingUid > 0 ? $existingUid : (int)($dataHandler->substNEWwithIDs[$id] ?? 0);
|
|
}
|
|
|
|
/** old page uid (from tx_impexp_origuid) => uid on this site */
|
|
private function buildPageMap(): array
|
|
{
|
|
$qb = GeneralUtility::makeInstance(ConnectionPool::class)->getQueryBuilderForTable('pages');
|
|
$qb->getRestrictions()->removeAll();
|
|
$rows = $qb->select('uid', 'tx_impexp_origuid')->from('pages')
|
|
->where(
|
|
$qb->expr()->gt('tx_impexp_origuid', $qb->createNamedParameter(0, ParameterType::INTEGER)),
|
|
$qb->expr()->eq('deleted', 0)
|
|
)
|
|
->executeQuery()->fetchAllAssociative();
|
|
|
|
$map = [];
|
|
foreach ($rows as $row) {
|
|
$map[(int)$row['tx_impexp_origuid']] = (int)$row['uid'];
|
|
}
|
|
return $map;
|
|
}
|
|
|
|
/** `t3://page?uid=N` or a bare uid - both forms occur in the source data. */
|
|
private function extractPageUid(string $internalUrl): int
|
|
{
|
|
$internalUrl = trim($internalUrl);
|
|
if (preg_match('#t3://page\?uid=(\d+)#i', $internalUrl, $m)) {
|
|
return (int)$m[1];
|
|
}
|
|
return ctype_digit($internalUrl) ? (int)$internalUrl : 0;
|
|
}
|
|
|
|
/**
|
|
* Old category uid => uid here. Matched by title, because uids are
|
|
* meaningless across installations. Missing categories are created below
|
|
* --category-parent so the assignments do not silently vanish.
|
|
*
|
|
* @param array<string,mixed> $export
|
|
* @return array<int,int>
|
|
*/
|
|
private function buildCategoryMap(array $export, int $parent, int $pid, bool $dryRun, OutputInterface $output): array
|
|
{
|
|
$categories = $export['categories'] ?? [];
|
|
if ($categories === []) {
|
|
return [];
|
|
}
|
|
|
|
$qb = GeneralUtility::makeInstance(ConnectionPool::class)->getQueryBuilderForTable(self::CATEGORY_TABLE);
|
|
$qb->getRestrictions()->removeAll();
|
|
$rows = $qb->select('uid', 'title', 'parent')->from(self::CATEGORY_TABLE)
|
|
->where($qb->expr()->eq('deleted', 0))
|
|
->executeQuery()->fetchAllAssociative();
|
|
|
|
$titleByUid = [];
|
|
foreach ($rows as $row) {
|
|
$titleByUid[(int)$row['uid']] = (string)$row['title'];
|
|
}
|
|
|
|
// With --category-parent the title match is restricted to that subtree.
|
|
// Matching site-wide is what makes this dangerous: the old news rubrics
|
|
// carry market names (Education, Government, Healthcare, ...) that also
|
|
// exist in the market taxonomy, and a bare title match would file press
|
|
// releases under markets.
|
|
$scope = $parent > 0 ? $this->collectDescendants($parent, $rows) : null;
|
|
|
|
$byTitle = [];
|
|
foreach ($rows as $row) {
|
|
if ($scope !== null && !isset($scope[(int)$row['uid']])) {
|
|
continue;
|
|
}
|
|
$byTitle[mb_strtolower(trim((string)$row['title']))] = (int)$row['uid'];
|
|
}
|
|
if ($scope !== null) {
|
|
$output->writeln(sprintf('Category match restricted to the subtree below uid %d ("%s"), %d candidates.',
|
|
$parent, $titleByUid[$parent] ?? '?', count($byTitle)));
|
|
} else {
|
|
$output->writeln('<comment>Category match is site-wide - pass --category-parent=<uid> to restrict it to one subtree.</comment>');
|
|
}
|
|
|
|
$map = [];
|
|
$found = $missing = 0;
|
|
foreach ($categories as $cat) {
|
|
$title = trim((string)$cat['title']);
|
|
$key = mb_strtolower($title);
|
|
if (isset($byTitle[$key])) {
|
|
$hit = $byTitle[$key];
|
|
$map[(int)$cat['uid']] = $hit;
|
|
$found++;
|
|
$parentUid = 0;
|
|
foreach ($rows as $row) {
|
|
if ((int)$row['uid'] === $hit) {
|
|
$parentUid = (int)$row['parent'];
|
|
break;
|
|
}
|
|
}
|
|
$output->writeln(sprintf(' category "%s" -> <info>uid %d</info> (below "%s")',
|
|
$title, $hit, $parentUid > 0 ? ($titleByUid[$parentUid] ?? '?') : 'root'));
|
|
continue;
|
|
}
|
|
$output->writeln(sprintf(' category <comment>"%s"</comment> does not exist here%s', $title, $dryRun ? '' : ' - creating'));
|
|
$missing++;
|
|
if ($dryRun) {
|
|
continue;
|
|
}
|
|
$id = 'NEW' . substr(md5('cat' . $title), 0, 12);
|
|
$dataHandler = GeneralUtility::makeInstance(DataHandler::class);
|
|
$dataHandler->start([self::CATEGORY_TABLE => [$id => ['pid' => $parent > 0 ? 0 : $pid, 'parent' => $parent, 'title' => $title]]], []);
|
|
$dataHandler->process_datamap();
|
|
$newUid = (int)($dataHandler->substNEWwithIDs[$id] ?? 0);
|
|
if ($newUid > 0) {
|
|
$map[(int)$cat['uid']] = $newUid;
|
|
$byTitle[$key] = $newUid;
|
|
}
|
|
}
|
|
$output->writeln(sprintf('Categories: <info>%d matched</info>, %d missing', $found, $missing));
|
|
return $map;
|
|
}
|
|
|
|
/**
|
|
* The category plus every descendant, as a uid => true lookup. Iterative and
|
|
* seen-guarded so a parent pointing back up the tree cannot loop.
|
|
*
|
|
* @param array<int,array<string,mixed>> $rows
|
|
* @return array<int,true>
|
|
*/
|
|
private function collectDescendants(int $uid, array $rows): array
|
|
{
|
|
$childrenByParent = [];
|
|
foreach ($rows as $row) {
|
|
$childrenByParent[(int)$row['parent']][] = (int)$row['uid'];
|
|
}
|
|
$collected = [$uid => true];
|
|
$queue = [$uid];
|
|
while ($queue !== []) {
|
|
$current = array_shift($queue);
|
|
foreach ($childrenByParent[$current] ?? [] as $child) {
|
|
if (isset($collected[$child])) {
|
|
continue;
|
|
}
|
|
$collected[$child] = true;
|
|
$queue[] = $child;
|
|
}
|
|
}
|
|
return $collected;
|
|
}
|
|
|
|
/** "54, 55" -> "54,55"; anything non-numeric is dropped. */
|
|
private function parseUidList(string $value): string
|
|
{
|
|
$uids = [];
|
|
foreach (explode(',', $value) as $part) {
|
|
$part = trim($part);
|
|
if ($part !== '' && ctype_digit($part) && (int)$part > 0) {
|
|
$uids[] = (int)$part;
|
|
}
|
|
}
|
|
return implode(',', array_unique($uids));
|
|
}
|
|
|
|
/**
|
|
* Prints the titles behind the uids given via --category, so a typo shows up
|
|
* in the dry run instead of after 218 records have been filed wrongly.
|
|
*/
|
|
private function reportCategoryTitles(string $uidList, OutputInterface $output): void
|
|
{
|
|
foreach (explode(',', $uidList) as $uid) {
|
|
$qb = GeneralUtility::makeInstance(ConnectionPool::class)->getQueryBuilderForTable(self::CATEGORY_TABLE);
|
|
$qb->getRestrictions()->removeAll();
|
|
$row = $qb->select('uid', 'title', 'parent')->from(self::CATEGORY_TABLE)
|
|
->where(
|
|
$qb->expr()->eq('uid', $qb->createNamedParameter((int)$uid, ParameterType::INTEGER)),
|
|
$qb->expr()->eq('deleted', 0)
|
|
)
|
|
->executeQuery()->fetchAssociative();
|
|
if (!$row) {
|
|
$output->writeln(sprintf(' <error>uid %s: no such category</error>', $uid));
|
|
continue;
|
|
}
|
|
$output->writeln(sprintf(' uid %s = <info>"%s"</info>', $uid, (string)$row['title']));
|
|
}
|
|
}
|
|
|
|
private function findByImportId(int $oldUid): int
|
|
{
|
|
$qb = GeneralUtility::makeInstance(ConnectionPool::class)->getQueryBuilderForTable(self::TABLE);
|
|
$qb->getRestrictions()->removeAll();
|
|
$uid = $qb->select('uid')->from(self::TABLE)
|
|
->where(
|
|
$qb->expr()->eq('import_id', $qb->createNamedParameter($oldUid, ParameterType::INTEGER)),
|
|
$qb->expr()->eq('import_source', $qb->createNamedParameter(self::IMPORT_SOURCE)),
|
|
$qb->expr()->eq('deleted', 0)
|
|
)
|
|
->setMaxResults(1)->executeQuery()->fetchOne();
|
|
return (int)($uid ?: 0);
|
|
}
|
|
|
|
/** Most-used pid of the news records already here; 0 when there are none. */
|
|
private function detectPid(): int
|
|
{
|
|
$qb = GeneralUtility::makeInstance(ConnectionPool::class)->getQueryBuilderForTable(self::TABLE);
|
|
$qb->getRestrictions()->removeAll();
|
|
$rows = $qb->select('pid')->addSelectLiteral('COUNT(*) AS cnt')->from(self::TABLE)
|
|
->where($qb->expr()->eq('deleted', 0))
|
|
->groupBy('pid')->orderBy('cnt', 'DESC')->setMaxResults(1)
|
|
->executeQuery()->fetchAssociative();
|
|
return (int)($rows['pid'] ?? 0);
|
|
}
|
|
}
|