Oliver Rasche
7bd1161abf
VITEC Import module: Products tab is now a searchable/sortable product overview. "Edit Product" opens an XLSX upload with per-field source mapping (component + cell), old/new preview and checkbox apply via DataHandler; the mapping and manual matches are persisted and preselect the next workbook. Category workbooks are detected and rejected. - new: ProductXlsxReader (PhpSpreadsheet), ProductTextImportController, Products/ProductTexts templates, 4 module routes - related products: per-pair card text in new side table tx_vitec_product_related_text (survives MM rewrites), emitted as `cardtext` in the product JSON; missing MM relations added add-only - card copy read from Body Copy (D) with fallback to CTA/Card Copy (E) - the workbooks fill either depending on row type - product detail page <title> now uses seotitle with title fallback (provider made singleton and fed from the JSON renderer) - per-user recent-search badges; last loaded workbook stored per product (tx_vitec_product_workbook), Edit Product reopens on it - composer: add phpoffice/phpspreadsheet ^5.9 - diagnostics: migrations/check_workbook.php
480 lines
19 KiB
PHP
480 lines
19 KiB
PHP
<?php
|
|
|
|
declare(strict_types=1);
|
|
|
|
namespace Evomedien\Vitec\Controller\Backend;
|
|
|
|
use Evomedien\Vitec\Import\CsvReader;
|
|
use Evomedien\Vitec\Import\ImportModelRegistry;
|
|
use Evomedien\Vitec\Import\MappingRepository;
|
|
use Evomedien\Vitec\Import\SeoResearchRepository;
|
|
use Evomedien\Vitec\Import\SeoResearchService;
|
|
use Psr\Http\Message\ResponseInterface;
|
|
use Psr\Http\Message\ServerRequestInterface;
|
|
use TYPO3\CMS\Backend\Attribute\AsController;
|
|
use TYPO3\CMS\Backend\Routing\UriBuilder;
|
|
use TYPO3\CMS\Backend\Template\ModuleTemplateFactory;
|
|
use TYPO3\CMS\Core\DataHandling\DataHandler;
|
|
use TYPO3\CMS\Core\Http\RedirectResponse;
|
|
use TYPO3\CMS\Core\Messaging\FlashMessage;
|
|
use TYPO3\CMS\Core\Messaging\FlashMessageService;
|
|
use TYPO3\CMS\Core\Page\PageRenderer;
|
|
use TYPO3\CMS\Core\Type\ContextualFeedbackSeverity;
|
|
use TYPO3\CMS\Core\Utility\GeneralUtility;
|
|
|
|
/**
|
|
* Backend module: CSV import for the VITEC domain models.
|
|
*
|
|
* Workflow per model page (decisions 2026-08-06):
|
|
* 1. Upload a CSV; delimiter and encoding are auto-detected.
|
|
* 2. Map CSV columns to DB fields; the mapping (and the identity field
|
|
* used for matching) is persisted per model via MappingRepository.
|
|
* 3. A unified list shows CSV rows matched against the DB records:
|
|
* new / update (with the differing fields) / unchanged / db-only.
|
|
* Rows the identity match misses can be linked by hand to an existing
|
|
* record (alias select on every "new" row); the links persist per
|
|
* model next to the column mapping and are re-applied on upload.
|
|
* Each importable row carries an editable JSON payload textarea -
|
|
* what is applied is the textarea, not the raw CSV.
|
|
* 4. Checked rows are written through DataHandler (BE user permissions
|
|
* apply). New records are created on the configured pid.
|
|
*
|
|
* The parsed CSV travels through the form as a hidden JSON field, so the
|
|
* module needs no server-side session state.
|
|
*/
|
|
#[AsController]
|
|
final class ImportController
|
|
{
|
|
public function __construct(
|
|
private readonly ModuleTemplateFactory $moduleTemplateFactory,
|
|
private readonly ImportModelRegistry $registry,
|
|
private readonly MappingRepository $mappingRepository,
|
|
private readonly CsvReader $csvReader,
|
|
private readonly SeoResearchRepository $seoResearchRepository,
|
|
private readonly SeoResearchService $seoResearchService,
|
|
private readonly FlashMessageService $flashMessageService,
|
|
private readonly PageRenderer $pageRenderer,
|
|
private readonly UriBuilder $uriBuilder,
|
|
) {}
|
|
|
|
public function indexAction(ServerRequestInterface $request): ResponseInterface
|
|
{
|
|
$modelKey = (string)($request->getQueryParams()['model'] ?? 'market');
|
|
if (!$this->registry->has($modelKey)) {
|
|
$modelKey = 'market';
|
|
}
|
|
return $this->render($request, $modelKey, null, null, null);
|
|
}
|
|
|
|
public function processAction(ServerRequestInterface $request): ResponseInterface
|
|
{
|
|
$body = (array)$request->getParsedBody();
|
|
$modelKey = (string)($body['model'] ?? '');
|
|
if (!$this->registry->has($modelKey)) {
|
|
return new RedirectResponse((string)$this->uriBuilder->buildUriFromRoute('web_vitecimport'), 303);
|
|
}
|
|
$op = (string)($body['op'] ?? 'preview');
|
|
|
|
// CSV: fresh upload wins, otherwise the hidden state field.
|
|
$csv = null;
|
|
$files = $request->getUploadedFiles();
|
|
$upload = $files['csvfile'] ?? null;
|
|
if ($upload !== null && $upload->getError() === UPLOAD_ERR_OK) {
|
|
$csv = $this->csvReader->parse((string)$upload->getStream());
|
|
if ($csv['columns'] === []) {
|
|
$this->flash('The file could not be parsed as CSV.', 'Upload', false);
|
|
$csv = null;
|
|
} else {
|
|
$this->flash(
|
|
sprintf('%d rows, %d columns detected.', count($csv['rows']), count($csv['columns'])),
|
|
'CSV loaded',
|
|
true
|
|
);
|
|
}
|
|
} elseif (($body['csvstate'] ?? '') !== '') {
|
|
$decoded = json_decode((string)$body['csvstate'], true);
|
|
if (is_array($decoded) && isset($decoded['columns'], $decoded['rows'])) {
|
|
$csv = $decoded;
|
|
}
|
|
}
|
|
|
|
// Mapping from the form, when present.
|
|
$mapping = null;
|
|
$identity = null;
|
|
if (isset($body['map']) && is_array($body['map'])) {
|
|
$mapping = [];
|
|
foreach ($body['map'] as $column => $field) {
|
|
if ((string)$field !== '') {
|
|
$mapping[(string)$column] = (string)$field;
|
|
}
|
|
}
|
|
$identity = (string)($body['identity'] ?? '');
|
|
}
|
|
|
|
if ($op === 'saveMapping' && $mapping !== null && $identity !== '') {
|
|
$this->mappingRepository->save($modelKey, $mapping, $identity);
|
|
$this->flash('Mapping stored for this model - it will be preselected on the next upload.', 'Mapping saved', true);
|
|
}
|
|
|
|
if ($op === 'saveAliases' && $csv !== null && $mapping !== null && ($identity ?? '') !== '') {
|
|
$this->saveAliases(
|
|
$modelKey,
|
|
$csv,
|
|
$mapping,
|
|
(string)$identity,
|
|
is_array($body['alias'] ?? null) ? $body['alias'] : []
|
|
);
|
|
}
|
|
|
|
if ($op === 'apply') {
|
|
$this->apply($modelKey, $body);
|
|
}
|
|
|
|
return $this->render($request, $modelKey, $csv, $mapping, $identity);
|
|
}
|
|
|
|
// --------------------------------------------------- SEO research tab
|
|
|
|
public function seoAction(ServerRequestInterface $request): ResponseInterface
|
|
{
|
|
$latest = $this->seoResearchRepository->latest(0);
|
|
$previous = $this->seoResearchRepository->latest(1);
|
|
$analysis = $latest !== null
|
|
? $this->seoResearchService->analyze($latest['rows'], $previous !== null ? $previous['rows'] : null)
|
|
: null;
|
|
|
|
$this->pageRenderer->addCssFile('EXT:vitec/Resources/Public/Css/backend-import.css');
|
|
$view = $this->moduleTemplateFactory->create($request);
|
|
$view->assignMultiple([
|
|
'models' => $this->registry->all(),
|
|
'latest' => $latest,
|
|
'previous' => $previous,
|
|
'analysis' => $analysis,
|
|
]);
|
|
return $view->renderResponse('Import/Seo');
|
|
}
|
|
|
|
public function seoUploadAction(ServerRequestInterface $request): ResponseInterface
|
|
{
|
|
$files = $request->getUploadedFiles();
|
|
$upload = $files['csvfile'] ?? null;
|
|
if ($upload !== null && $upload->getError() === UPLOAD_ERR_OK) {
|
|
$csv = $this->csvReader->parse((string)$upload->getStream());
|
|
$rows = $this->seoResearchService->normalizeRows($csv['rows']);
|
|
if ($rows === []) {
|
|
$this->flash('No usable rows found - is the "Potential URL" column present?', 'Upload', false);
|
|
} else {
|
|
$this->seoResearchRepository->add((string)($upload->getClientFilename() ?? 'upload.csv'), $rows);
|
|
$this->flash(sprintf('%d rows stored as new delivery.', count($rows)), 'Delivery stored', true);
|
|
}
|
|
} else {
|
|
$this->flash('No file received.', 'Upload', false);
|
|
}
|
|
return new RedirectResponse((string)$this->uriBuilder->buildUriFromRoute('web_vitecimport.seo'), 303);
|
|
}
|
|
|
|
// ------------------------------------------------------------ rendering
|
|
|
|
/**
|
|
* @param array{columns:array<int,string>,rows:array<int,array<string,string>>}|null $csv
|
|
* @param array<string,string>|null $mapping
|
|
*/
|
|
private function render(
|
|
ServerRequestInterface $request,
|
|
string $modelKey,
|
|
?array $csv,
|
|
?array $mapping,
|
|
?string $identity
|
|
): ResponseInterface {
|
|
$model = $this->registry->get($modelKey);
|
|
$fields = $this->registry->importableFields($model['table']);
|
|
$saved = $this->mappingRepository->load($modelKey);
|
|
|
|
if ($mapping === null) {
|
|
$mapping = $saved['mapping'] ?? [];
|
|
if ($mapping === [] && $csv !== null) {
|
|
$mapping = $this->guessMapping($csv['columns'], $fields);
|
|
}
|
|
}
|
|
if ($identity === null || $identity === '') {
|
|
$identity = $saved['identity'] ?? (isset($fields['slug']) ? 'slug' : 'title');
|
|
}
|
|
|
|
['rows' => $union, 'candidates' => $aliasCandidates] =
|
|
$this->buildUnion($model['table'], $fields, $csv, $mapping, $identity, $saved['aliases'] ?? []);
|
|
|
|
$counts = ['new' => 0, 'update' => 0, 'unchanged' => 0, 'dbonly' => 0];
|
|
foreach ($union as $row) {
|
|
$counts[$row['status']]++;
|
|
}
|
|
|
|
// Column meta for the mapping panel: sample value + current target.
|
|
$columns = [];
|
|
foreach ($csv['columns'] ?? [] as $column) {
|
|
$columns[] = [
|
|
'name' => $column,
|
|
'sample' => (string)($csv['rows'][0][$column] ?? ''),
|
|
'target' => $mapping[$column] ?? '',
|
|
];
|
|
}
|
|
|
|
$this->pageRenderer->addCssFile('EXT:vitec/Resources/Public/Css/backend-import.css');
|
|
$view = $this->moduleTemplateFactory->create($request);
|
|
$view->assignMultiple([
|
|
'models' => $this->registry->all(),
|
|
'model' => $model,
|
|
'fields' => $fields,
|
|
'columns' => $columns,
|
|
'hasCsv' => $csv !== null,
|
|
'csvRowCount' => count($csv['rows'] ?? []),
|
|
'csvstate' => $csv !== null ? (string)json_encode($csv, JSON_UNESCAPED_UNICODE) : '',
|
|
'identity' => $identity,
|
|
'mappingSaved' => ($saved['mapping'] ?? []) !== [],
|
|
'aliasCandidates' => $aliasCandidates,
|
|
'union' => $union,
|
|
'counts' => $counts,
|
|
'pid' => $this->registry->detectPid($model['table']),
|
|
]);
|
|
|
|
return $view->renderResponse('Import/Index');
|
|
}
|
|
|
|
/**
|
|
* Name-equality guess for a first-time mapping.
|
|
*
|
|
* @param array<int,string> $columns
|
|
* @param array<string,string> $fields
|
|
* @return array<string,string>
|
|
*/
|
|
private function guessMapping(array $columns, array $fields): array
|
|
{
|
|
$mapping = [];
|
|
foreach ($columns as $column) {
|
|
$normalized = mb_strtolower(trim($column));
|
|
if (isset($fields[$normalized])) {
|
|
$mapping[$column] = $normalized;
|
|
}
|
|
}
|
|
return $mapping;
|
|
}
|
|
|
|
/**
|
|
* Merge CSV rows and DB records into one status-annotated list.
|
|
*
|
|
* Matching order per CSV row: identity field first, then the stored
|
|
* aliases. Alias candidates are all records the identity match did not
|
|
* claim - those are what the per-row select in the template offers.
|
|
*
|
|
* @param array<string,string> $fields
|
|
* @param array{columns:array<int,string>,rows:array<int,array<string,string>>}|null $csv
|
|
* @param array<string,string> $mapping
|
|
* @param array<string,int> $aliases
|
|
* @return array{rows: array<int,array<string,mixed>>, candidates: array<int,array{uid:int,title:string}>}
|
|
*/
|
|
private function buildUnion(string $table, array $fields, ?array $csv, array $mapping, string $identity, array $aliases): array
|
|
{
|
|
$mappedFields = array_values(array_unique(array_values($mapping)));
|
|
$records = $this->registry->loadRecords($table, array_keys($fields));
|
|
|
|
$byIdentity = [];
|
|
foreach ($records as $uid => $record) {
|
|
$key = mb_strtolower(trim((string)($record[$identity] ?? '')));
|
|
if ($key !== '' && !isset($byIdentity[$key])) {
|
|
$byIdentity[$key] = $uid;
|
|
}
|
|
}
|
|
|
|
$union = [];
|
|
$matchedUids = [];
|
|
$directUids = [];
|
|
$index = 0;
|
|
|
|
foreach ($csv['rows'] ?? [] as $raw) {
|
|
$payload = [];
|
|
foreach ($mapping as $column => $field) {
|
|
$payload[$field] = (string)($raw[$column] ?? '');
|
|
}
|
|
$identityValue = mb_strtolower(trim((string)($payload[$identity] ?? '')));
|
|
$uid = $identityValue !== '' ? ($byIdentity[$identityValue] ?? 0) : 0;
|
|
$viaAlias = false;
|
|
if ($uid === 0 && $identityValue !== '') {
|
|
$aliasUid = (int)($aliases[$identityValue] ?? 0);
|
|
if ($aliasUid > 0 && isset($records[$aliasUid])) {
|
|
$uid = $aliasUid;
|
|
$viaAlias = true;
|
|
}
|
|
}
|
|
|
|
if ($uid > 0) {
|
|
$matchedUids[$uid] = true;
|
|
if (!$viaAlias) {
|
|
$directUids[$uid] = true;
|
|
}
|
|
$record = $records[$uid];
|
|
$diff = [];
|
|
foreach ($payload as $field => $value) {
|
|
if (trim($value) !== trim((string)($record[$field] ?? ''))) {
|
|
$diff[] = $field;
|
|
}
|
|
}
|
|
$status = $diff === [] ? 'unchanged' : 'update';
|
|
} else {
|
|
$diff = array_keys($payload);
|
|
$status = 'new';
|
|
}
|
|
|
|
$union[] = [
|
|
'index' => $index++,
|
|
'status' => $status,
|
|
'uid' => $uid,
|
|
'label' => (string)($payload['title'] ?? ($payload[$identity] ?? 'row')),
|
|
'diff' => implode(', ', $diff),
|
|
'payload' => (string)json_encode($payload, JSON_PRETTY_PRINT | JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES),
|
|
'checked' => $status === 'new' || $status === 'update',
|
|
'viaAlias' => $viaAlias,
|
|
'aliasable' => $viaAlias || $status === 'new',
|
|
'aliasUid' => $viaAlias ? $uid : 0,
|
|
];
|
|
}
|
|
|
|
foreach ($records as $uid => $record) {
|
|
if (isset($matchedUids[$uid])) {
|
|
continue;
|
|
}
|
|
$union[] = [
|
|
'index' => $index++,
|
|
'status' => 'dbonly',
|
|
'uid' => $uid,
|
|
'label' => (string)($record['title'] ?? $uid),
|
|
'diff' => '',
|
|
'payload' => '',
|
|
'checked' => false,
|
|
];
|
|
}
|
|
|
|
$candidates = [];
|
|
foreach ($records as $uid => $record) {
|
|
if (isset($directUids[$uid])) {
|
|
continue;
|
|
}
|
|
$candidates[] = ['uid' => $uid, 'title' => (string)($record['title'] ?? $uid)];
|
|
}
|
|
usort($candidates, static fn(array $a, array $b): int => strcasecmp($a['title'], $b['title']));
|
|
|
|
return ['rows' => $union, 'candidates' => $candidates];
|
|
}
|
|
|
|
/**
|
|
* Merge the alias selects into the stored aliases. Key is the CSV
|
|
* identity value (lowercased, like the matcher sees it), value the
|
|
* record uid; uid 0 removes a link. Untouched keys stay - a CSV that
|
|
* lacks a row must not lose that row's link.
|
|
*
|
|
* @param array{columns:array<int,string>,rows:array<int,array<string,string>>} $csv
|
|
* @param array<string,string> $mapping
|
|
* @param array<int|string,mixed> $input row index => uid, from the form
|
|
*/
|
|
private function saveAliases(string $modelKey, array $csv, array $mapping, string $identity, array $input): void
|
|
{
|
|
$aliases = $this->mappingRepository->load($modelKey)['aliases'] ?? [];
|
|
$changed = 0;
|
|
foreach ($input as $rowIndex => $selectedUid) {
|
|
$raw = $csv['rows'][(int)$rowIndex] ?? null;
|
|
if ($raw === null) {
|
|
continue;
|
|
}
|
|
$payload = [];
|
|
foreach ($mapping as $column => $field) {
|
|
$payload[$field] = (string)($raw[$column] ?? '');
|
|
}
|
|
$key = mb_strtolower(trim((string)($payload[$identity] ?? '')));
|
|
if ($key === '') {
|
|
continue;
|
|
}
|
|
$uid = (int)$selectedUid;
|
|
$current = (int)($aliases[$key] ?? 0);
|
|
if ($uid > 0 && $uid !== $current) {
|
|
$aliases[$key] = $uid;
|
|
$changed++;
|
|
} elseif ($uid === 0 && $current > 0) {
|
|
unset($aliases[$key]);
|
|
$changed++;
|
|
}
|
|
}
|
|
$this->mappingRepository->saveAliases($modelKey, $aliases);
|
|
$this->flash(
|
|
sprintf('%d record link(s) changed, %d stored for this model in total.', $changed, count($aliases)),
|
|
'Record links saved',
|
|
true
|
|
);
|
|
}
|
|
|
|
// ---------------------------------------------------------------- apply
|
|
|
|
/** @param array<string,mixed> $body */
|
|
private function apply(string $modelKey, array $body): void
|
|
{
|
|
$model = $this->registry->get($modelKey);
|
|
$fields = $this->registry->importableFields($model['table']);
|
|
$pid = (int)($body['pid'] ?? 0);
|
|
$rows = is_array($body['rows'] ?? null) ? $body['rows'] : [];
|
|
|
|
$datamap = [];
|
|
$errors = [];
|
|
foreach ($rows as $i => $row) {
|
|
if (empty($row['selected'])) {
|
|
continue;
|
|
}
|
|
$payload = json_decode((string)($row['payload'] ?? ''), true);
|
|
if (!is_array($payload)) {
|
|
$errors[] = sprintf('Row %s: payload is not valid JSON - skipped.', $i);
|
|
continue;
|
|
}
|
|
// Whitelist against the TCA-derived field list.
|
|
$payload = array_intersect_key($payload, $fields);
|
|
$payload = array_map(static fn($v): string => is_scalar($v) ? (string)$v : '', $payload);
|
|
if ($payload === []) {
|
|
continue;
|
|
}
|
|
$uid = (int)($row['uid'] ?? 0);
|
|
if ($uid > 0) {
|
|
$datamap[(string)$uid] = $payload;
|
|
} elseif ($pid > 0) {
|
|
$datamap['NEW' . $i] = ['pid' => $pid] + $payload;
|
|
} else {
|
|
$errors[] = sprintf('Row %s: no storage pid for a new record - skipped.', $i);
|
|
}
|
|
}
|
|
|
|
$created = 0;
|
|
$updated = 0;
|
|
if ($datamap !== []) {
|
|
$dataHandler = GeneralUtility::makeInstance(DataHandler::class);
|
|
$dataHandler->start([$model['table'] => $datamap], []);
|
|
$dataHandler->process_datamap();
|
|
$errors = array_merge($errors, $dataHandler->errorLog);
|
|
$created = count($dataHandler->substNEWwithIDs);
|
|
$updated = count(array_filter(array_keys($datamap), 'is_numeric'));
|
|
}
|
|
|
|
$this->flash(
|
|
sprintf('%d created, %d updated, %d error(s).', $created, $updated, count($errors))
|
|
. ($errors !== [] ? ' ' . implode(' | ', array_slice($errors, 0, 5)) : ''),
|
|
'Import finished',
|
|
$errors === []
|
|
);
|
|
}
|
|
|
|
private function flash(string $message, string $title, bool $ok): void
|
|
{
|
|
$flashMessage = GeneralUtility::makeInstance(
|
|
FlashMessage::class,
|
|
$message,
|
|
$title,
|
|
$ok ? ContextualFeedbackSeverity::OK : ContextualFeedbackSeverity::ERROR,
|
|
true
|
|
);
|
|
$this->flashMessageService->getMessageQueueByIdentifier()->addMessage($flashMessage);
|
|
}
|
|
}
|