Workbook import phase 1: manifest-driven batch command
Implements the 18.09. strategy's phase 1 on top of the existing single importer (which stays the one write path): - new vitec:import-product-workbooks: runs vitec:import-product-workbook per manifest entry (in-process sub-command), aggregates a per-file report (writes/unchanged/na, URL-mismatch and error flags, details on -v or failure) plus a summary; without --apply every run is a dry run; --only filters - migrations/products_xlsx/ now holds working copies of all 100 agency workbooks in their category folders (git-ignored) plus the TRACKED manifest.json: workbook path -> product slug, family entries mark the K-level landing pages (never imported), product null = unresolved. Generated from the 18.09. analysis + sitemap column K: 68 product / 17 family / 15 open - --init-manifest resolves open entries against the full product table (hidden records included): URL segment/filename vs slug, normalized title, unique slug prefix; only unambiguous hits are written back - single command compare is now tolerant of the RTE storage transform (whitespace collapse + <p>-wrap simulation), fixing the known every-run-WRITE false positive on re-imports Post-dry-run hardening, applied the same day (69 products, 785 field writes across two runs, zero errors, verified against the live JSON): - PROTECTED_FIELDS title/urltitle in the single command: the stored module mapping would have overwritten six canonical sitemap-V2 product titles - the stored mapping predated the 18.09. defaults and silently dropped `highlights` on every product; the single command now merges DEFAULT_MAPPING under the stored mapping (defaults as base, stored as per-field override) - known limitation for phase 2: the g: composed fields report WRITE on every run (RTE transform alters ul/strong markup beyond the p-wrap normalisation) - the change watch must compare against stored imported values, not the database Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
orascheandClaude Fable 5 committed
1 parent
6811463343
commit
6961254e57
5 files changed
+661
-9
No files matched your search
@@ -42,6 +42,15 @@ final class ImportProductWorkbookCommand extends Command
|
||||
{
|
||||
private const TABLE = 'tx_vitec_domain_model_product';
|
||||
|
||||
/**
|
||||
* Never written by the workbook import, whatever the stored mapping
|
||||
* says: product titles are canonical per sitemap V2 (decision
|
||||
* 2026-09-04) - the 2026-09-24 batch dry run showed a stored module
|
||||
* mapping would have renamed six products. urltitle would only receive
|
||||
* a duplicate of the hero one-liner.
|
||||
*/
|
||||
private const PROTECTED_FIELDS = ['title', 'urltitle'];
|
||||
|
||||
public function __construct(
|
||||
private readonly ProductXlsxReader $reader,
|
||||
private readonly MappingRepository $mappingRepository,
|
||||
@@ -89,20 +98,30 @@ final class ImportProductWorkbookCommand extends Command
|
||||
$output->writeln(sprintf('<comment>URL mismatch: /%s (workbook) vs /%s (record) - check the target!</comment>', $url, $slug));
|
||||
}
|
||||
|
||||
$mapping = array_filter($this->mappingRepository->load('product_xlsx')['mapping'] ?? []);
|
||||
if ($mapping === []) {
|
||||
$mapping = ProductTextImportController::DEFAULT_MAPPING;
|
||||
}
|
||||
// Defaults as the base, the stored module mapping as per-field
|
||||
// override - a field the module never saved (its dropdown was empty
|
||||
// on an old-format workbook, e.g. `highlights`) still imports with
|
||||
// its default selector. Found in the 2026-09-24 batch run: the
|
||||
// stored mapping predated the highlights default and silently
|
||||
// dropped the field on all 69 products.
|
||||
$mapping = array_merge(
|
||||
ProductTextImportController::DEFAULT_MAPPING,
|
||||
array_filter($this->mappingRepository->load('product_xlsx')['mapping'] ?? [])
|
||||
);
|
||||
|
||||
$data = [];
|
||||
foreach ($mapping as $field => $selector) {
|
||||
if (in_array($field, self::PROTECTED_FIELDS, true)) {
|
||||
$output->writeln(sprintf('%-20s %-34s -> protected (canonical, never written)', $field, $selector));
|
||||
continue;
|
||||
}
|
||||
$value = $this->reader->valueFor($parsed, $selector);
|
||||
if ($value === null) {
|
||||
$output->writeln(sprintf('%-20s %-34s -> not in this file', $field, $selector));
|
||||
continue;
|
||||
}
|
||||
$old = (string)($product[$field] ?? '');
|
||||
if (trim($old) === trim($value)) {
|
||||
if ($this->isUnchanged($old, $value)) {
|
||||
$output->writeln(sprintf('%-20s %-34s -> unchanged', $field, $selector));
|
||||
continue;
|
||||
}
|
||||
@@ -134,6 +153,22 @@ final class ImportProductWorkbookCommand extends Command
|
||||
return Command::SUCCESS;
|
||||
}
|
||||
|
||||
/**
|
||||
* Compare tolerant of the RTE storage transform, which wraps bare
|
||||
* content in <p>...</p> and strips newlines on save - without this every
|
||||
* re-run reported already-imported richtext fields as WRITE (the known
|
||||
* false-positive from the module's preview).
|
||||
*/
|
||||
private function isUnchanged(string $old, string $new): bool
|
||||
{
|
||||
$normalize = static fn(string $s): string => trim((string)preg_replace('/\s+/u', ' ', $s));
|
||||
if ($normalize($old) === $normalize($new)) {
|
||||
return true;
|
||||
}
|
||||
|
||||
return $normalize($old) === $normalize('<p>' . $new . '</p>');
|
||||
}
|
||||
|
||||
/** @return array<string,mixed>|null */
|
||||
private function loadProduct(string $slugOrUid): ?array
|
||||
{
|
||||
|
||||
@@ -0,0 +1,271 @@
|
||||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
|
||||
namespace Evomedien\Vitec\Command;
|
||||
|
||||
use Evomedien\Vitec\Import\ProductXlsxReader;
|
||||
use Symfony\Component\Console\Attribute\AsCommand;
|
||||
use Symfony\Component\Console\Command\Command;
|
||||
use Symfony\Component\Console\Input\ArrayInput;
|
||||
use Symfony\Component\Console\Input\InputInterface;
|
||||
use Symfony\Component\Console\Input\InputOption;
|
||||
use Symfony\Component\Console\Output\BufferedOutput;
|
||||
use Symfony\Component\Console\Output\OutputInterface;
|
||||
use TYPO3\CMS\Core\Core\Bootstrap;
|
||||
use TYPO3\CMS\Core\Database\ConnectionPool;
|
||||
use TYPO3\CMS\Core\Utility\GeneralUtility;
|
||||
|
||||
/**
|
||||
* Phase-1 batch around vitec:import-product-workbook: apply every agency
|
||||
* workbook under migrations/products_xlsx/ to its product record, driven by
|
||||
* the tracked manifest.json (workbook path -> product slug; family entries
|
||||
* are the K-level landing pages and are never imported; product null means
|
||||
* unresolved - reported and skipped).
|
||||
*
|
||||
* The heavy lifting stays in the single command (same reader, same mapping,
|
||||
* same DataHandler path, workbook parses stored for the module) - this
|
||||
* command only orchestrates it per manifest entry and aggregates a report.
|
||||
*
|
||||
* Without --apply every run is a dry run. --init-manifest resolves entries
|
||||
* with product null against the FULL product table (hidden records too):
|
||||
* workbook-URL segment or filename against slug, normalized title, and as a
|
||||
* last resort a UNIQUE slug prefix (mgw-ace -> mgw-ace-encoder). Only
|
||||
* unambiguous hits are written back to the manifest.
|
||||
*
|
||||
* vendor/bin/typo3 vitec:import-product-workbooks --init-manifest
|
||||
* vendor/bin/typo3 vitec:import-product-workbooks
|
||||
* vendor/bin/typo3 vitec:import-product-workbooks --only=aligo
|
||||
* vendor/bin/typo3 vitec:import-product-workbooks --apply
|
||||
*/
|
||||
#[AsCommand(
|
||||
name: 'vitec:import-product-workbooks',
|
||||
description: 'Batch: apply all agency workbooks to their product records via manifest.json (dry-run without --apply)'
|
||||
)]
|
||||
final class ImportProductWorkbooksCommand extends Command
|
||||
{
|
||||
private const TABLE = 'tx_vitec_domain_model_product';
|
||||
|
||||
public function __construct(private readonly ProductXlsxReader $reader)
|
||||
{
|
||||
parent::__construct();
|
||||
}
|
||||
|
||||
protected function configure(): void
|
||||
{
|
||||
$this->addOption('dir', null, InputOption::VALUE_REQUIRED, 'Workbook directory holding manifest.json', 'migrations/products_xlsx');
|
||||
$this->addOption('apply', null, InputOption::VALUE_NONE, 'Actually write - without this every run is a dry run');
|
||||
$this->addOption('only', null, InputOption::VALUE_REQUIRED, 'Process only entries whose path or product slug contains this substring');
|
||||
$this->addOption('init-manifest', null, InputOption::VALUE_NONE, 'Resolve product:null manifest entries against the full product table (writes the manifest, imports nothing)');
|
||||
}
|
||||
|
||||
protected function execute(InputInterface $input, OutputInterface $output): int
|
||||
{
|
||||
Bootstrap::initializeBackendAuthentication();
|
||||
|
||||
$dir = rtrim((string)$input->getOption('dir'), '/');
|
||||
$manifestFile = $dir . '/manifest.json';
|
||||
if (!is_file($manifestFile)) {
|
||||
$output->writeln('<error>Manifest not found: ' . $manifestFile . '</error>');
|
||||
return Command::FAILURE;
|
||||
}
|
||||
$manifest = json_decode((string)file_get_contents($manifestFile), true);
|
||||
$entries = $manifest['workbooks'] ?? null;
|
||||
if (!is_array($entries)) {
|
||||
$output->writeln('<error>Manifest malformed (no workbooks key)</error>');
|
||||
return Command::FAILURE;
|
||||
}
|
||||
|
||||
if ((bool)$input->getOption('init-manifest')) {
|
||||
return $this->initManifest($dir, $manifestFile, $manifest, $output);
|
||||
}
|
||||
|
||||
$apply = (bool)$input->getOption('apply');
|
||||
$only = (string)($input->getOption('only') ?? '');
|
||||
$single = $this->getApplication()?->find('vitec:import-product-workbook');
|
||||
if ($single === null) {
|
||||
$output->writeln('<error>vitec:import-product-workbook not available</error>');
|
||||
return Command::FAILURE;
|
||||
}
|
||||
|
||||
$output->writeln(sprintf('<info>%d manifest entries · %s</info>', count($entries), $apply ? 'APPLY' : 'DRY-RUN'));
|
||||
|
||||
$families = 0;
|
||||
$unresolved = [];
|
||||
$processed = 0;
|
||||
$totalWrites = 0;
|
||||
$errors = 0;
|
||||
foreach ($entries as $relPath => $entry) {
|
||||
$slug = $entry['product'] ?? null;
|
||||
if (!empty($entry['family'])) {
|
||||
$families++;
|
||||
continue;
|
||||
}
|
||||
if ($slug === null || $slug === '') {
|
||||
$unresolved[] = $relPath;
|
||||
continue;
|
||||
}
|
||||
if ($only !== '' && !str_contains($relPath, $only) && !str_contains((string)$slug, $only)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
$file = $dir . '/' . $relPath;
|
||||
$buffer = new BufferedOutput();
|
||||
$arguments = ['file' => $file, 'product' => (string)$slug];
|
||||
if (!$apply) {
|
||||
$arguments['--dry-run'] = true;
|
||||
}
|
||||
try {
|
||||
$exit = $single->run(new ArrayInput($arguments), $buffer);
|
||||
} catch (\Throwable $e) {
|
||||
$exit = 1;
|
||||
$buffer->writeln('EXCEPTION: ' . $e->getMessage());
|
||||
}
|
||||
$text = $buffer->fetch();
|
||||
$writes = substr_count($text, '-> WRITE');
|
||||
$unchanged = substr_count($text, '-> unchanged');
|
||||
$missing = substr_count($text, '-> not in this file');
|
||||
$mismatch = str_contains($text, 'URL mismatch');
|
||||
$processed++;
|
||||
$totalWrites += $writes;
|
||||
|
||||
$flags = [];
|
||||
if ($mismatch) {
|
||||
$flags[] = 'URL-MISMATCH';
|
||||
}
|
||||
if ($exit !== 0) {
|
||||
$flags[] = 'ERROR';
|
||||
$errors++;
|
||||
}
|
||||
$output->writeln(sprintf(
|
||||
'%-72s -> %-28s %d write, %d unchanged, %d n/a%s',
|
||||
mb_substr($relPath, 0, 72),
|
||||
mb_substr((string)$slug, 0, 28),
|
||||
$writes,
|
||||
$unchanged,
|
||||
$missing,
|
||||
$flags !== [] ? ' [' . implode(', ', $flags) . ']' : ''
|
||||
));
|
||||
if ($exit !== 0 || $mismatch || $output->isVerbose()) {
|
||||
foreach (explode("\n", trim($text)) as $line) {
|
||||
$output->writeln(' ' . $line);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
$output->writeln('');
|
||||
$output->writeln(sprintf(
|
||||
'<info>%d workbooks processed · %d field writes%s · %d errors · %d family workbooks skipped · %d unresolved</info>',
|
||||
$processed,
|
||||
$totalWrites,
|
||||
$apply ? '' : ' (dry run - nothing written)',
|
||||
$errors,
|
||||
$families,
|
||||
count($unresolved)
|
||||
));
|
||||
foreach ($unresolved as $relPath) {
|
||||
$output->writeln(' <comment>unresolved: ' . $relPath . '</comment>');
|
||||
}
|
||||
if ($apply && $totalWrites > 0) {
|
||||
$output->writeln('<info>Flush the cache: vendor/bin/typo3 cache:flush</info>');
|
||||
}
|
||||
|
||||
return $errors === 0 ? Command::SUCCESS : Command::FAILURE;
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve product:null entries against the full product table. Writes
|
||||
* only unambiguous hits back into the manifest; everything else stays
|
||||
* null with its note.
|
||||
*
|
||||
* @param array<string,mixed> $manifest
|
||||
*/
|
||||
private function initManifest(string $dir, string $manifestFile, array $manifest, OutputInterface $output): int
|
||||
{
|
||||
$connection = GeneralUtility::makeInstance(ConnectionPool::class)->getConnectionForTable(self::TABLE);
|
||||
$products = $connection->fetchAllAssociative(
|
||||
'SELECT uid, slug, title, hidden FROM ' . self::TABLE . ' WHERE deleted = 0'
|
||||
);
|
||||
$bySlug = [];
|
||||
$byTitle = [];
|
||||
foreach ($products as $product) {
|
||||
$bySlug[(string)$product['slug']] = $product;
|
||||
$byTitle[$this->normalize((string)$product['title'])] = $product;
|
||||
}
|
||||
|
||||
$resolved = 0;
|
||||
foreach ($manifest['workbooks'] as $relPath => $entry) {
|
||||
if (!empty($entry['family']) || (($entry['product'] ?? null) !== null)) {
|
||||
continue;
|
||||
}
|
||||
$candidates = [];
|
||||
$file = $dir . '/' . $relPath;
|
||||
if (is_file($file)) {
|
||||
try {
|
||||
$parsed = $this->reader->parse($file);
|
||||
$segment = basename(rtrim((string)($parsed['meta']['url'] ?? ''), '/'));
|
||||
if ($segment !== '') {
|
||||
$candidates[] = $segment;
|
||||
}
|
||||
} catch (\Throwable $e) {
|
||||
// unreadable file - fall through to the filename
|
||||
}
|
||||
}
|
||||
$stem = (string)preg_replace('/(_?Product[_ ]Page[_ ]Content|_US_English)/i', '', pathinfo($relPath, PATHINFO_FILENAME));
|
||||
$candidates[] = $this->slugify($stem);
|
||||
|
||||
$hit = null;
|
||||
$via = '';
|
||||
foreach ($candidates as $candidate) {
|
||||
if (isset($bySlug[$candidate])) {
|
||||
$hit = $bySlug[$candidate];
|
||||
$via = 'slug';
|
||||
break;
|
||||
}
|
||||
if (isset($byTitle[$this->normalize($candidate)])) {
|
||||
$hit = $byTitle[$this->normalize($candidate)];
|
||||
$via = 'titel';
|
||||
break;
|
||||
}
|
||||
$prefixHits = array_values(array_filter(
|
||||
$products,
|
||||
static fn(array $p): bool => str_starts_with((string)$p['slug'], $candidate . '-')
|
||||
));
|
||||
if (count($prefixHits) === 1) {
|
||||
$hit = $prefixHits[0];
|
||||
$via = 'slug-prefix';
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if ($hit !== null) {
|
||||
$manifest['workbooks'][$relPath] = [
|
||||
'product' => (string)$hit['slug'],
|
||||
'note' => sprintf('auto-resolved (%s%s) 2026-09-24', $via, ((int)$hit['hidden'] === 1) ? ', Record versteckt!' : ''),
|
||||
];
|
||||
$resolved++;
|
||||
$output->writeln(sprintf(' %-70s -> %s (%s%s)', mb_substr($relPath, 0, 70), $hit['slug'], $via, ((int)$hit['hidden'] === 1) ? ', HIDDEN' : ''));
|
||||
} else {
|
||||
$output->writeln(' <comment>' . $relPath . ' -> weiterhin ungeklaert</comment>');
|
||||
}
|
||||
}
|
||||
|
||||
file_put_contents($manifestFile, json_encode($manifest, JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE) . "\n");
|
||||
$output->writeln(sprintf('<info>%d entries resolved, manifest updated.</info>', $resolved));
|
||||
|
||||
return Command::SUCCESS;
|
||||
}
|
||||
|
||||
private function normalize(string $title): string
|
||||
{
|
||||
$title = str_replace(['&', '-', '_'], ['and', ' ', ' '], mb_strtolower($title));
|
||||
$title = (string)preg_replace('/\s*\(.*?\)/', '', $title);
|
||||
$title = (string)preg_replace('/[^a-z0-9]+/', ' ', $title);
|
||||
return trim((string)preg_replace('/\s+/', ' ', $title));
|
||||
}
|
||||
|
||||
private function slugify(string $value): string
|
||||
{
|
||||
return trim((string)preg_replace('/[^a-z0-9]+/', '-', str_replace('&', 'and', mb_strtolower($value))), '-');
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user