Oliver Rasche
5e3be3ec42
New "VITEC Import" module under Web: one import page per domain model (v1: Market, Solution, Product - registry-driven, adding a model is one config entry). Workflow: upload a CSV (delimiter and encoding are auto-detected, including German-Excel semicolon/Windows-1252), map CSV columns to DB fields, persist the mapping per model together with the identity field used for matching (new table tx_vitec_import_mapping, no TCA - pure tool configuration), review a unified list of CSV rows matched against the DB records (new / update with differing fields / unchanged / db-only), then apply the checked rows through DataHandler. Each importable row carries an editable JSON payload textarea - what is written is the textarea content, not the raw CSV, so editors can fix values right in the review step. The parsed CSV travels through the form as a hidden JSON field: no session state, no temp files. Importable fields are derived from TCA at runtime (scalar types only; files, categories and other relations are excluded - a flat CSV cannot carry them). Payloads are whitelisted against that field list on apply; new records require a storage pid (prefilled from existing records). BE user permissions apply via DataHandler. The module ships its own CSS (backend-import.css, loaded only by this module) using the frontend button palette from _vitec.scss: orange #f47937 for primary actions, navy #26358c for secondary actions and structure. The stray <h2>Hi</h2> debug leftover in the shared backend layout is removed (also affects the OG Image module). Deliberately out of v1: import log with three-way compare (protection against overwriting manual edits), images/relations, multiple saved mappings per model. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
347 lines
13 KiB
PHP
347 lines
13 KiB
PHP
<?php
|
|
|
|
declare(strict_types=1);
|
|
|
|
namespace Evomedien\Vitec\Controller\Backend;
|
|
|
|
use Evomedien\Vitec\Import\CsvReader;
|
|
use Evomedien\Vitec\Import\ImportModelRegistry;
|
|
use Evomedien\Vitec\Import\MappingRepository;
|
|
use Psr\Http\Message\ResponseInterface;
|
|
use Psr\Http\Message\ServerRequestInterface;
|
|
use TYPO3\CMS\Backend\Attribute\AsController;
|
|
use TYPO3\CMS\Backend\Routing\UriBuilder;
|
|
use TYPO3\CMS\Backend\Template\ModuleTemplateFactory;
|
|
use TYPO3\CMS\Core\DataHandling\DataHandler;
|
|
use TYPO3\CMS\Core\Http\RedirectResponse;
|
|
use TYPO3\CMS\Core\Messaging\FlashMessage;
|
|
use TYPO3\CMS\Core\Messaging\FlashMessageService;
|
|
use TYPO3\CMS\Core\Page\PageRenderer;
|
|
use TYPO3\CMS\Core\Type\ContextualFeedbackSeverity;
|
|
use TYPO3\CMS\Core\Utility\GeneralUtility;
|
|
|
|
/**
|
|
* Backend module: CSV import for the VITEC domain models.
|
|
*
|
|
* Workflow per model page (decisions 2026-08-06):
|
|
* 1. Upload a CSV; delimiter and encoding are auto-detected.
|
|
* 2. Map CSV columns to DB fields; the mapping (and the identity field
|
|
* used for matching) is persisted per model via MappingRepository.
|
|
* 3. A unified list shows CSV rows matched against the DB records:
|
|
* new / update (with the differing fields) / unchanged / db-only.
|
|
* Each importable row carries an editable JSON payload textarea -
|
|
* what is applied is the textarea, not the raw CSV.
|
|
* 4. Checked rows are written through DataHandler (BE user permissions
|
|
* apply). New records are created on the configured pid.
|
|
*
|
|
* The parsed CSV travels through the form as a hidden JSON field, so the
|
|
* module needs no server-side session state.
|
|
*/
|
|
#[AsController]
|
|
final class ImportController
|
|
{
|
|
public function __construct(
|
|
private readonly ModuleTemplateFactory $moduleTemplateFactory,
|
|
private readonly ImportModelRegistry $registry,
|
|
private readonly MappingRepository $mappingRepository,
|
|
private readonly CsvReader $csvReader,
|
|
private readonly FlashMessageService $flashMessageService,
|
|
private readonly PageRenderer $pageRenderer,
|
|
private readonly UriBuilder $uriBuilder,
|
|
) {}
|
|
|
|
public function indexAction(ServerRequestInterface $request): ResponseInterface
|
|
{
|
|
$modelKey = (string)($request->getQueryParams()['model'] ?? 'market');
|
|
if (!$this->registry->has($modelKey)) {
|
|
$modelKey = 'market';
|
|
}
|
|
return $this->render($request, $modelKey, null, null, null);
|
|
}
|
|
|
|
public function processAction(ServerRequestInterface $request): ResponseInterface
|
|
{
|
|
$body = (array)$request->getParsedBody();
|
|
$modelKey = (string)($body['model'] ?? '');
|
|
if (!$this->registry->has($modelKey)) {
|
|
return new RedirectResponse((string)$this->uriBuilder->buildUriFromRoute('web_vitecimport'), 303);
|
|
}
|
|
$op = (string)($body['op'] ?? 'preview');
|
|
|
|
// CSV: fresh upload wins, otherwise the hidden state field.
|
|
$csv = null;
|
|
$files = $request->getUploadedFiles();
|
|
$upload = $files['csvfile'] ?? null;
|
|
if ($upload !== null && $upload->getError() === UPLOAD_ERR_OK) {
|
|
$csv = $this->csvReader->parse((string)$upload->getStream());
|
|
if ($csv['columns'] === []) {
|
|
$this->flash('The file could not be parsed as CSV.', 'Upload', false);
|
|
$csv = null;
|
|
} else {
|
|
$this->flash(
|
|
sprintf('%d rows, %d columns detected.', count($csv['rows']), count($csv['columns'])),
|
|
'CSV loaded',
|
|
true
|
|
);
|
|
}
|
|
} elseif (($body['csvstate'] ?? '') !== '') {
|
|
$decoded = json_decode((string)$body['csvstate'], true);
|
|
if (is_array($decoded) && isset($decoded['columns'], $decoded['rows'])) {
|
|
$csv = $decoded;
|
|
}
|
|
}
|
|
|
|
// Mapping from the form, when present.
|
|
$mapping = null;
|
|
$identity = null;
|
|
if (isset($body['map']) && is_array($body['map'])) {
|
|
$mapping = [];
|
|
foreach ($body['map'] as $column => $field) {
|
|
if ((string)$field !== '') {
|
|
$mapping[(string)$column] = (string)$field;
|
|
}
|
|
}
|
|
$identity = (string)($body['identity'] ?? '');
|
|
}
|
|
|
|
if ($op === 'saveMapping' && $mapping !== null && $identity !== '') {
|
|
$this->mappingRepository->save($modelKey, $mapping, $identity);
|
|
$this->flash('Mapping stored for this model - it will be preselected on the next upload.', 'Mapping saved', true);
|
|
}
|
|
|
|
if ($op === 'apply') {
|
|
$this->apply($modelKey, $body);
|
|
}
|
|
|
|
return $this->render($request, $modelKey, $csv, $mapping, $identity);
|
|
}
|
|
|
|
// ------------------------------------------------------------ rendering
|
|
|
|
/**
|
|
* @param array{columns:array<int,string>,rows:array<int,array<string,string>>}|null $csv
|
|
* @param array<string,string>|null $mapping
|
|
*/
|
|
private function render(
|
|
ServerRequestInterface $request,
|
|
string $modelKey,
|
|
?array $csv,
|
|
?array $mapping,
|
|
?string $identity
|
|
): ResponseInterface {
|
|
$model = $this->registry->get($modelKey);
|
|
$fields = $this->registry->importableFields($model['table']);
|
|
$saved = $this->mappingRepository->load($modelKey);
|
|
|
|
if ($mapping === null) {
|
|
$mapping = $saved['mapping'] ?? [];
|
|
if ($mapping === [] && $csv !== null) {
|
|
$mapping = $this->guessMapping($csv['columns'], $fields);
|
|
}
|
|
}
|
|
if ($identity === null || $identity === '') {
|
|
$identity = $saved['identity'] ?? (isset($fields['slug']) ? 'slug' : 'title');
|
|
}
|
|
|
|
$union = $this->buildUnion($model['table'], $fields, $csv, $mapping, $identity);
|
|
|
|
$counts = ['new' => 0, 'update' => 0, 'unchanged' => 0, 'dbonly' => 0];
|
|
foreach ($union as $row) {
|
|
$counts[$row['status']]++;
|
|
}
|
|
|
|
// Column meta for the mapping panel: sample value + current target.
|
|
$columns = [];
|
|
foreach ($csv['columns'] ?? [] as $column) {
|
|
$columns[] = [
|
|
'name' => $column,
|
|
'sample' => (string)($csv['rows'][0][$column] ?? ''),
|
|
'target' => $mapping[$column] ?? '',
|
|
];
|
|
}
|
|
|
|
$this->pageRenderer->addCssFile('EXT:vitec/Resources/Public/Css/backend-import.css');
|
|
$view = $this->moduleTemplateFactory->create($request);
|
|
$view->assignMultiple([
|
|
'models' => $this->registry->all(),
|
|
'model' => $model,
|
|
'fields' => $fields,
|
|
'columns' => $columns,
|
|
'hasCsv' => $csv !== null,
|
|
'csvRowCount' => count($csv['rows'] ?? []),
|
|
'csvstate' => $csv !== null ? (string)json_encode($csv, JSON_UNESCAPED_UNICODE) : '',
|
|
'identity' => $identity,
|
|
'mappingSaved' => $saved !== null,
|
|
'union' => $union,
|
|
'counts' => $counts,
|
|
'pid' => $this->registry->detectPid($model['table']),
|
|
]);
|
|
|
|
return $view->renderResponse('Import/Index');
|
|
}
|
|
|
|
/**
|
|
* Name-equality guess for a first-time mapping.
|
|
*
|
|
* @param array<int,string> $columns
|
|
* @param array<string,string> $fields
|
|
* @return array<string,string>
|
|
*/
|
|
private function guessMapping(array $columns, array $fields): array
|
|
{
|
|
$mapping = [];
|
|
foreach ($columns as $column) {
|
|
$normalized = mb_strtolower(trim($column));
|
|
if (isset($fields[$normalized])) {
|
|
$mapping[$column] = $normalized;
|
|
}
|
|
}
|
|
return $mapping;
|
|
}
|
|
|
|
/**
|
|
* Merge CSV rows and DB records into one status-annotated list.
|
|
*
|
|
* @param array<string,string> $fields
|
|
* @param array{columns:array<int,string>,rows:array<int,array<string,string>>}|null $csv
|
|
* @param array<string,string> $mapping
|
|
* @return array<int,array<string,mixed>>
|
|
*/
|
|
private function buildUnion(string $table, array $fields, ?array $csv, array $mapping, string $identity): array
|
|
{
|
|
$mappedFields = array_values(array_unique(array_values($mapping)));
|
|
$records = $this->registry->loadRecords($table, array_keys($fields));
|
|
|
|
$byIdentity = [];
|
|
foreach ($records as $uid => $record) {
|
|
$key = mb_strtolower(trim((string)($record[$identity] ?? '')));
|
|
if ($key !== '' && !isset($byIdentity[$key])) {
|
|
$byIdentity[$key] = $uid;
|
|
}
|
|
}
|
|
|
|
$union = [];
|
|
$matchedUids = [];
|
|
$index = 0;
|
|
|
|
foreach ($csv['rows'] ?? [] as $raw) {
|
|
$payload = [];
|
|
foreach ($mapping as $column => $field) {
|
|
$payload[$field] = (string)($raw[$column] ?? '');
|
|
}
|
|
$identityValue = mb_strtolower(trim((string)($payload[$identity] ?? '')));
|
|
$uid = $identityValue !== '' ? ($byIdentity[$identityValue] ?? 0) : 0;
|
|
|
|
if ($uid > 0) {
|
|
$matchedUids[$uid] = true;
|
|
$record = $records[$uid];
|
|
$diff = [];
|
|
foreach ($payload as $field => $value) {
|
|
if (trim($value) !== trim((string)($record[$field] ?? ''))) {
|
|
$diff[] = $field;
|
|
}
|
|
}
|
|
$status = $diff === [] ? 'unchanged' : 'update';
|
|
} else {
|
|
$diff = array_keys($payload);
|
|
$status = 'new';
|
|
}
|
|
|
|
$union[] = [
|
|
'index' => $index++,
|
|
'status' => $status,
|
|
'uid' => $uid,
|
|
'label' => (string)($payload['title'] ?? ($payload[$identity] ?? 'row')),
|
|
'diff' => implode(', ', $diff),
|
|
'payload' => (string)json_encode($payload, JSON_PRETTY_PRINT | JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES),
|
|
'checked' => $status === 'new' || $status === 'update',
|
|
];
|
|
}
|
|
|
|
foreach ($records as $uid => $record) {
|
|
if (isset($matchedUids[$uid])) {
|
|
continue;
|
|
}
|
|
$union[] = [
|
|
'index' => $index++,
|
|
'status' => 'dbonly',
|
|
'uid' => $uid,
|
|
'label' => (string)($record['title'] ?? $uid),
|
|
'diff' => '',
|
|
'payload' => '',
|
|
'checked' => false,
|
|
];
|
|
}
|
|
|
|
return $union;
|
|
}
|
|
|
|
// ---------------------------------------------------------------- apply
|
|
|
|
/** @param array<string,mixed> $body */
|
|
private function apply(string $modelKey, array $body): void
|
|
{
|
|
$model = $this->registry->get($modelKey);
|
|
$fields = $this->registry->importableFields($model['table']);
|
|
$pid = (int)($body['pid'] ?? 0);
|
|
$rows = is_array($body['rows'] ?? null) ? $body['rows'] : [];
|
|
|
|
$datamap = [];
|
|
$errors = [];
|
|
foreach ($rows as $i => $row) {
|
|
if (empty($row['selected'])) {
|
|
continue;
|
|
}
|
|
$payload = json_decode((string)($row['payload'] ?? ''), true);
|
|
if (!is_array($payload)) {
|
|
$errors[] = sprintf('Row %s: payload is not valid JSON - skipped.', $i);
|
|
continue;
|
|
}
|
|
// Whitelist against the TCA-derived field list.
|
|
$payload = array_intersect_key($payload, $fields);
|
|
$payload = array_map(static fn($v): string => is_scalar($v) ? (string)$v : '', $payload);
|
|
if ($payload === []) {
|
|
continue;
|
|
}
|
|
$uid = (int)($row['uid'] ?? 0);
|
|
if ($uid > 0) {
|
|
$datamap[(string)$uid] = $payload;
|
|
} elseif ($pid > 0) {
|
|
$datamap['NEW' . $i] = ['pid' => $pid] + $payload;
|
|
} else {
|
|
$errors[] = sprintf('Row %s: no storage pid for a new record - skipped.', $i);
|
|
}
|
|
}
|
|
|
|
$created = 0;
|
|
$updated = 0;
|
|
if ($datamap !== []) {
|
|
$dataHandler = GeneralUtility::makeInstance(DataHandler::class);
|
|
$dataHandler->start([$model['table'] => $datamap], []);
|
|
$dataHandler->process_datamap();
|
|
$errors = array_merge($errors, $dataHandler->errorLog);
|
|
$created = count($dataHandler->substNEWwithIDs);
|
|
$updated = count(array_filter(array_keys($datamap), 'is_numeric'));
|
|
}
|
|
|
|
$this->flash(
|
|
sprintf('%d created, %d updated, %d error(s).', $created, $updated, count($errors))
|
|
. ($errors !== [] ? ' ' . implode(' | ', array_slice($errors, 0, 5)) : ''),
|
|
'Import finished',
|
|
$errors === []
|
|
);
|
|
}
|
|
|
|
private function flash(string $message, string $title, bool $ok): void
|
|
{
|
|
$flashMessage = GeneralUtility::makeInstance(
|
|
FlashMessage::class,
|
|
$message,
|
|
$title,
|
|
$ok ? ContextualFeedbackSeverity::OK : ContextualFeedbackSeverity::ERROR,
|
|
true
|
|
);
|
|
$this->flashMessageService->getMessageQueueByIdentifier()->addMessage($flashMessage);
|
|
}
|
|
}
|