From 10c0328bf25d1134f89f77c5bd338454d43c2839 Mon Sep 17 00:00:00 2001 From: Oliver Rasche Date: Fri, 7 Aug 2026 12:53:21 +0200 Subject: [PATCH] Backend module: CSV import for the VITEC domain models New "VITEC Import" module under Web: one import page per domain model (v1: Market, Solution, Product - registry-driven, adding a model is one config entry). Workflow: upload a CSV (delimiter and encoding are auto-detected, including German-Excel semicolon/Windows-1252), map CSV columns to DB fields, persist the mapping per model together with the identity field used for matching (new table tx_vitec_import_mapping, no TCA - pure tool configuration), review a unified list of CSV rows matched against the DB records (new / update with differing fields / unchanged / db-only), then apply the checked rows through DataHandler. Each importable row carries an editable JSON payload textarea - what is written is the textarea content, not the raw CSV, so editors can fix values right in the review step. The parsed CSV travels through the form as a hidden JSON field: no session state, no temp files. Importable fields are derived from TCA at runtime (scalar types only; files, categories and other relations are excluded - a flat CSV cannot carry them). Payloads are whitelisted against that field list on apply; new records require a storage pid (prefilled from existing records). BE user permissions apply via DataHandler. Deliberately out of v1: import log with three-way compare (protection against overwriting manual edits), images/relations, multiple saved mappings per model. Co-Authored-By: Claude Opus 5 --- .../Controller/Backend/ImportController.php | 343 ++++++++++++++++++ packages/vitec/Classes/Import/CsvReader.php | 78 ++++ .../Classes/Import/ImportModelRegistry.php | 115 ++++++ .../Classes/Import/MappingRepository.php | 65 ++++ .../vitec/Configuration/Backend/Modules.php | 23 ++ packages/vitec/Configuration/Icons.php | 4 + .../Private/Templates/Import/Index.html | 186 ++++++++++ .../Resources/Public/Icons/vitec-import.svg | 5 + packages/vitec/ext_tables.sql | 12 + 9 files changed, 831 insertions(+) create mode 100644 packages/vitec/Classes/Controller/Backend/ImportController.php create mode 100644 packages/vitec/Classes/Import/CsvReader.php create mode 100644 packages/vitec/Classes/Import/ImportModelRegistry.php create mode 100644 packages/vitec/Classes/Import/MappingRepository.php create mode 100644 packages/vitec/Resources/Private/Templates/Import/Index.html create mode 100644 packages/vitec/Resources/Public/Icons/vitec-import.svg diff --git a/packages/vitec/Classes/Controller/Backend/ImportController.php b/packages/vitec/Classes/Controller/Backend/ImportController.php new file mode 100644 index 0000000..024eda2 --- /dev/null +++ b/packages/vitec/Classes/Controller/Backend/ImportController.php @@ -0,0 +1,343 @@ +getQueryParams()['model'] ?? 'market'); + if (!$this->registry->has($modelKey)) { + $modelKey = 'market'; + } + return $this->render($request, $modelKey, null, null, null); + } + + public function processAction(ServerRequestInterface $request): ResponseInterface + { + $body = (array)$request->getParsedBody(); + $modelKey = (string)($body['model'] ?? ''); + if (!$this->registry->has($modelKey)) { + return new RedirectResponse((string)$this->uriBuilder->buildUriFromRoute('web_vitecimport'), 303); + } + $op = (string)($body['op'] ?? 'preview'); + + // CSV: fresh upload wins, otherwise the hidden state field. + $csv = null; + $files = $request->getUploadedFiles(); + $upload = $files['csvfile'] ?? null; + if ($upload !== null && $upload->getError() === UPLOAD_ERR_OK) { + $csv = $this->csvReader->parse((string)$upload->getStream()); + if ($csv['columns'] === []) { + $this->flash('The file could not be parsed as CSV.', 'Upload', false); + $csv = null; + } else { + $this->flash( + sprintf('%d rows, %d columns detected.', count($csv['rows']), count($csv['columns'])), + 'CSV loaded', + true + ); + } + } elseif (($body['csvstate'] ?? '') !== '') { + $decoded = json_decode((string)$body['csvstate'], true); + if (is_array($decoded) && isset($decoded['columns'], $decoded['rows'])) { + $csv = $decoded; + } + } + + // Mapping from the form, when present. + $mapping = null; + $identity = null; + if (isset($body['map']) && is_array($body['map'])) { + $mapping = []; + foreach ($body['map'] as $column => $field) { + if ((string)$field !== '') { + $mapping[(string)$column] = (string)$field; + } + } + $identity = (string)($body['identity'] ?? ''); + } + + if ($op === 'saveMapping' && $mapping !== null && $identity !== '') { + $this->mappingRepository->save($modelKey, $mapping, $identity); + $this->flash('Mapping stored for this model - it will be preselected on the next upload.', 'Mapping saved', true); + } + + if ($op === 'apply') { + $this->apply($modelKey, $body); + } + + return $this->render($request, $modelKey, $csv, $mapping, $identity); + } + + // ------------------------------------------------------------ rendering + + /** + * @param array{columns:array,rows:array>}|null $csv + * @param array|null $mapping + */ + private function render( + ServerRequestInterface $request, + string $modelKey, + ?array $csv, + ?array $mapping, + ?string $identity + ): ResponseInterface { + $model = $this->registry->get($modelKey); + $fields = $this->registry->importableFields($model['table']); + $saved = $this->mappingRepository->load($modelKey); + + if ($mapping === null) { + $mapping = $saved['mapping'] ?? []; + if ($mapping === [] && $csv !== null) { + $mapping = $this->guessMapping($csv['columns'], $fields); + } + } + if ($identity === null || $identity === '') { + $identity = $saved['identity'] ?? (isset($fields['slug']) ? 'slug' : 'title'); + } + + $union = $this->buildUnion($model['table'], $fields, $csv, $mapping, $identity); + + $counts = ['new' => 0, 'update' => 0, 'unchanged' => 0, 'dbonly' => 0]; + foreach ($union as $row) { + $counts[$row['status']]++; + } + + // Column meta for the mapping panel: sample value + current target. + $columns = []; + foreach ($csv['columns'] ?? [] as $column) { + $columns[] = [ + 'name' => $column, + 'sample' => (string)($csv['rows'][0][$column] ?? ''), + 'target' => $mapping[$column] ?? '', + ]; + } + + $view = $this->moduleTemplateFactory->create($request); + $view->assignMultiple([ + 'models' => $this->registry->all(), + 'model' => $model, + 'fields' => $fields, + 'columns' => $columns, + 'hasCsv' => $csv !== null, + 'csvRowCount' => count($csv['rows'] ?? []), + 'csvstate' => $csv !== null ? (string)json_encode($csv, JSON_UNESCAPED_UNICODE) : '', + 'identity' => $identity, + 'mappingSaved' => $saved !== null, + 'union' => $union, + 'counts' => $counts, + 'pid' => $this->registry->detectPid($model['table']), + ]); + + return $view->renderResponse('Import/Index'); + } + + /** + * Name-equality guess for a first-time mapping. + * + * @param array $columns + * @param array $fields + * @return array + */ + private function guessMapping(array $columns, array $fields): array + { + $mapping = []; + foreach ($columns as $column) { + $normalized = mb_strtolower(trim($column)); + if (isset($fields[$normalized])) { + $mapping[$column] = $normalized; + } + } + return $mapping; + } + + /** + * Merge CSV rows and DB records into one status-annotated list. + * + * @param array $fields + * @param array{columns:array,rows:array>}|null $csv + * @param array $mapping + * @return array> + */ + private function buildUnion(string $table, array $fields, ?array $csv, array $mapping, string $identity): array + { + $mappedFields = array_values(array_unique(array_values($mapping))); + $records = $this->registry->loadRecords($table, array_keys($fields)); + + $byIdentity = []; + foreach ($records as $uid => $record) { + $key = mb_strtolower(trim((string)($record[$identity] ?? ''))); + if ($key !== '' && !isset($byIdentity[$key])) { + $byIdentity[$key] = $uid; + } + } + + $union = []; + $matchedUids = []; + $index = 0; + + foreach ($csv['rows'] ?? [] as $raw) { + $payload = []; + foreach ($mapping as $column => $field) { + $payload[$field] = (string)($raw[$column] ?? ''); + } + $identityValue = mb_strtolower(trim((string)($payload[$identity] ?? ''))); + $uid = $identityValue !== '' ? ($byIdentity[$identityValue] ?? 0) : 0; + + if ($uid > 0) { + $matchedUids[$uid] = true; + $record = $records[$uid]; + $diff = []; + foreach ($payload as $field => $value) { + if (trim($value) !== trim((string)($record[$field] ?? ''))) { + $diff[] = $field; + } + } + $status = $diff === [] ? 'unchanged' : 'update'; + } else { + $diff = array_keys($payload); + $status = 'new'; + } + + $union[] = [ + 'index' => $index++, + 'status' => $status, + 'uid' => $uid, + 'label' => (string)($payload['title'] ?? ($payload[$identity] ?? 'row')), + 'diff' => implode(', ', $diff), + 'payload' => (string)json_encode($payload, JSON_PRETTY_PRINT | JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES), + 'checked' => $status === 'new' || $status === 'update', + ]; + } + + foreach ($records as $uid => $record) { + if (isset($matchedUids[$uid])) { + continue; + } + $union[] = [ + 'index' => $index++, + 'status' => 'dbonly', + 'uid' => $uid, + 'label' => (string)($record['title'] ?? $uid), + 'diff' => '', + 'payload' => '', + 'checked' => false, + ]; + } + + return $union; + } + + // ---------------------------------------------------------------- apply + + /** @param array $body */ + private function apply(string $modelKey, array $body): void + { + $model = $this->registry->get($modelKey); + $fields = $this->registry->importableFields($model['table']); + $pid = (int)($body['pid'] ?? 0); + $rows = is_array($body['rows'] ?? null) ? $body['rows'] : []; + + $datamap = []; + $errors = []; + foreach ($rows as $i => $row) { + if (empty($row['selected'])) { + continue; + } + $payload = json_decode((string)($row['payload'] ?? ''), true); + if (!is_array($payload)) { + $errors[] = sprintf('Row %s: payload is not valid JSON - skipped.', $i); + continue; + } + // Whitelist against the TCA-derived field list. + $payload = array_intersect_key($payload, $fields); + $payload = array_map(static fn($v): string => is_scalar($v) ? (string)$v : '', $payload); + if ($payload === []) { + continue; + } + $uid = (int)($row['uid'] ?? 0); + if ($uid > 0) { + $datamap[(string)$uid] = $payload; + } elseif ($pid > 0) { + $datamap['NEW' . $i] = ['pid' => $pid] + $payload; + } else { + $errors[] = sprintf('Row %s: no storage pid for a new record - skipped.', $i); + } + } + + $created = 0; + $updated = 0; + if ($datamap !== []) { + $dataHandler = GeneralUtility::makeInstance(DataHandler::class); + $dataHandler->start([$model['table'] => $datamap], []); + $dataHandler->process_datamap(); + $errors = array_merge($errors, $dataHandler->errorLog); + $created = count($dataHandler->substNEWwithIDs); + $updated = count(array_filter(array_keys($datamap), 'is_numeric')); + } + + $this->flash( + sprintf('%d created, %d updated, %d error(s).', $created, $updated, count($errors)) + . ($errors !== [] ? ' ' . implode(' | ', array_slice($errors, 0, 5)) : ''), + 'Import finished', + $errors === [] + ); + } + + private function flash(string $message, string $title, bool $ok): void + { + $flashMessage = GeneralUtility::makeInstance( + FlashMessage::class, + $message, + $title, + $ok ? ContextualFeedbackSeverity::OK : ContextualFeedbackSeverity::ERROR, + true + ); + $this->flashMessageService->getMessageQueueByIdentifier()->addMessage($flashMessage); + } +} diff --git a/packages/vitec/Classes/Import/CsvReader.php b/packages/vitec/Classes/Import/CsvReader.php new file mode 100644 index 0000000..87d3496 --- /dev/null +++ b/packages/vitec/Classes/Import/CsvReader.php @@ -0,0 +1,78 @@ +, rows: array>} + */ + public function parse(string $content): array + { + if (str_starts_with($content, "\xEF\xBB\xBF")) { + $content = substr($content, 3); + } + if (!mb_check_encoding($content, 'UTF-8')) { + $content = mb_convert_encoding($content, 'UTF-8', 'Windows-1252'); + } + + $firstLine = strtok($content, "\r\n") ?: ''; + $delimiter = ';'; + $best = 0; + foreach ([';', ',', "\t"] as $candidate) { + $count = substr_count($firstLine, $candidate); + if ($count > $best) { + $best = $count; + $delimiter = $candidate; + } + } + + $stream = fopen('php://temp', 'r+'); + fwrite($stream, $content); + rewind($stream); + + $header = fgetcsv($stream, null, $delimiter, '"', ''); + if (!is_array($header)) { + fclose($stream); + return ['columns' => [], 'rows' => []]; + } + $columns = []; + foreach ($header as $i => $name) { + $name = trim((string)$name); + $columns[$i] = $name !== '' ? $name : ('column_' . ($i + 1)); + } + + $rows = []; + while (($data = fgetcsv($stream, null, $delimiter, '"', '')) !== false) { + if ($data === [null] || $data === ['']) { + continue; // blank line + } + $row = []; + $hasContent = false; + foreach ($columns as $i => $name) { + $value = trim((string)($data[$i] ?? '')); + $row[$name] = $value; + if ($value !== '') { + $hasContent = true; + } + } + if ($hasContent) { + $rows[] = $row; + } + } + fclose($stream); + + return ['columns' => array_values($columns), 'rows' => $rows]; + } +} diff --git a/packages/vitec/Classes/Import/ImportModelRegistry.php b/packages/vitec/Classes/Import/ImportModelRegistry.php new file mode 100644 index 0000000..e778120 --- /dev/null +++ b/packages/vitec/Classes/Import/ImportModelRegistry.php @@ -0,0 +1,115 @@ + ['table' => 'tx_vitec_domain_model_market', 'label' => 'Markets'], + 'solution' => ['table' => 'tx_vitec_domain_model_solution', 'label' => 'Solutions'], + 'product' => ['table' => 'tx_vitec_domain_model_product', 'label' => 'Products'], + ]; + + /** Scalar TCA column types a CSV cell can populate. */ + private const IMPORTABLE_TYPES = ['input', 'text', 'slug', 'number', 'email', 'link', 'check', 'color']; + + private const EXCLUDED_FIELDS = [ + 'sys_language_uid', 'l10n_parent', 'l10n_diffsource', 'l10n_source', + 'l18n_parent', 'l18n_diffsource', 'starttime', 'endtime', 'hidden', + ]; + + /** @return array */ + public function all(): array + { + $out = []; + foreach (self::MODELS as $key => $cfg) { + $out[$key] = $cfg + ['key' => $key]; + } + return $out; + } + + public function has(string $key): bool + { + return isset(self::MODELS[$key]); + } + + /** @return array{key:string,table:string,label:string} */ + public function get(string $key): array + { + return self::MODELS[$key] + ['key' => $key]; + } + + /** + * Importable fields of a table, derived from TCA. + * + * @return array fieldName => human label + */ + public function importableFields(string $table): array + { + $fields = []; + foreach ($GLOBALS['TCA'][$table]['columns'] ?? [] as $name => $column) { + $type = (string)($column['config']['type'] ?? ''); + if (!in_array($type, self::IMPORTABLE_TYPES, true)) { + continue; + } + if (in_array($name, self::EXCLUDED_FIELDS, true)) { + continue; + } + if (($column['config']['renderType'] ?? '') === 'inputDateTime') { + continue; + } + $label = (string)($column['label'] ?? $name); + if (str_starts_with($label, 'LLL:')) { + $translated = $GLOBALS['LANG'] ?? null ? (string)$GLOBALS['LANG']->sL($label) : ''; + $label = $translated !== '' ? $translated : $name; + } + $fields[$name] = $label !== '' ? $label : $name; + } + return $fields; + } + + /** Pid of the first existing record - the sysfolder the model lives in. */ + public function detectPid(string $table): int + { + $qb = GeneralUtility::makeInstance(ConnectionPool::class)->getQueryBuilderForTable($table); + $row = $qb->select('pid')->from($table) + ->where($qb->expr()->eq('deleted', 0)) + ->setMaxResults(1) + ->executeQuery()->fetchAssociative(); + return $row ? (int)$row['pid'] : 0; + } + + /** + * All live records with the given fields, indexed by uid. + * + * @param array $fields + * @return array> + */ + public function loadRecords(string $table, array $fields): array + { + $select = array_values(array_unique(array_merge(['uid', 'title'], $fields))); + $qb = GeneralUtility::makeInstance(ConnectionPool::class)->getQueryBuilderForTable($table); + $rows = $qb->select(...$select)->from($table) + ->where($qb->expr()->eq('deleted', 0)) + ->orderBy('title', 'ASC') + ->executeQuery()->fetchAllAssociative(); + $out = []; + foreach ($rows as $row) { + $out[(int)$row['uid']] = $row; + } + return $out; + } +} diff --git a/packages/vitec/Classes/Import/MappingRepository.php b/packages/vitec/Classes/Import/MappingRepository.php new file mode 100644 index 0000000..9998156 --- /dev/null +++ b/packages/vitec/Classes/Import/MappingRepository.php @@ -0,0 +1,65 @@ + DB-field mapping per import model, so editors + * configure the mapper once and reuse it on every delivery. + * + * Plain table without TCA (tx_vitec_import_mapping) - the records are pure + * tool configuration, never edited through FormEngine. + */ +final class MappingRepository +{ + private const TABLE = 'tx_vitec_import_mapping'; + + /** + * @return array{mapping: array, identity: string}|null + */ + public function load(string $model): ?array + { + $qb = GeneralUtility::makeInstance(ConnectionPool::class)->getQueryBuilderForTable(self::TABLE); + $row = $qb->select('mapping', 'identity_field')->from(self::TABLE) + ->where($qb->expr()->eq('model', $qb->createNamedParameter($model, ParameterType::STRING))) + ->setMaxResults(1) + ->executeQuery()->fetchAssociative(); + if (!$row) { + return null; + } + $mapping = json_decode((string)$row['mapping'], true); + if (!is_array($mapping)) { + return null; + } + return [ + 'mapping' => array_map('strval', $mapping), + 'identity' => (string)$row['identity_field'], + ]; + } + + /** @param array $mapping */ + public function save(string $model, array $mapping, string $identity): void + { + $connection = GeneralUtility::makeInstance(ConnectionPool::class)->getConnectionForTable(self::TABLE); + $values = [ + 'mapping' => (string)json_encode($mapping, JSON_UNESCAPED_UNICODE), + 'identity_field' => $identity, + 'tstamp' => time(), + ]; + $exists = $connection->count('uid', self::TABLE, ['model' => $model]) > 0; + if ($exists) { + $connection->update(self::TABLE, $values, ['model' => $model]); + } else { + $connection->insert(self::TABLE, $values + [ + 'model' => $model, + 'pid' => 0, + 'crdate' => time(), + ]); + } + } +} diff --git a/packages/vitec/Configuration/Backend/Modules.php b/packages/vitec/Configuration/Backend/Modules.php index 445dd4f..be9c021 100644 --- a/packages/vitec/Configuration/Backend/Modules.php +++ b/packages/vitec/Configuration/Backend/Modules.php @@ -2,9 +2,32 @@ declare(strict_types=1); +use Evomedien\Vitec\Controller\Backend\ImportController; use Evomedien\Vitec\Controller\OgImageController; return [ + 'web_vitecimport' => [ + 'parent' => 'web', + 'position' => ['after' => 'web_info'], + 'access' => 'user', + 'workspaces' => 'live', + 'path' => '/module/web/vitec-import', + 'labels' => [ + 'title' => 'VITEC Import', + 'shortDescription' => 'CSV import for the VITEC domain models', + ], + 'extensionName' => 'Vitec', + 'iconIdentifier' => 'vitec-import', + 'routes' => [ + '_default' => [ + 'target' => ImportController::class . '::indexAction', + ], + 'process' => [ + 'target' => ImportController::class . '::processAction', + 'methods' => ['POST'], + ], + ], + ], 'web_vitecogimage' => [ 'parent' => 'web', 'position' => ['after' => 'web_info'], diff --git a/packages/vitec/Configuration/Icons.php b/packages/vitec/Configuration/Icons.php index 2977dbf..1491eed 100755 --- a/packages/vitec/Configuration/Icons.php +++ b/packages/vitec/Configuration/Icons.php @@ -84,6 +84,10 @@ return [ 'provider' => SvgIconProvider::class, 'source' => 'EXT:vitec/Resources/Public/Icons/vitec-plugin-marketlist.svg', ], + 'vitec-import' => [ + 'provider' => SvgIconProvider::class, + 'source' => 'EXT:vitec/Resources/Public/Icons/vitec-import.svg', + ], 'vitec-plugin-customerlogos' => [ 'provider' => SvgIconProvider::class, 'source' => 'EXT:vitec/Resources/Public/Icons/vitec-plugin-customerlogos.svg', diff --git a/packages/vitec/Resources/Private/Templates/Import/Index.html b/packages/vitec/Resources/Private/Templates/Import/Index.html new file mode 100644 index 0000000..548d0f8 --- /dev/null +++ b/packages/vitec/Resources/Private/Templates/Import/Index.html @@ -0,0 +1,186 @@ + + + + + + +
+
+

VITEC Import — {model.label}

+
+
+ + + + + +
+
+
+ +
+ +
+
+ +
+
+ Delimiter (";" or ",") and encoding (UTF-8 / Windows-1252) are detected automatically. +
+
+
+
+ + + +
+ + + + +
+
+ Field mapping + saved mapping loaded +
+
+ + + + + + + + + + + + + + + + + +
CSV columnSample (row 1)Database field
{col.name}{col.sample} + +
+ +
+
+
+ +
+
+ +
+
+ +
+
+
+
+ + +
+
+ Records +   {counts.new} new + {counts.update} update + {counts.unchanged} unchanged + {counts.dbonly} only in DB +
+
+ + + + + + + + + + + + + + + + + + + +
StatusRecordImport data (editable JSON — this is what gets written)
+ + + + + + new + update + unchanged + only in DB + + + {row.label} +
uid {row.uid} + +
changes: {row.diff} + + 		+ + + + +
+ +
+
+
+
+ +
+
+
+
+
+ + + + + +
+
Current records ({counts.dbonly})
+
+ + + + + + + +
uidTitle
{row.uid}{row.label}
+
+
+ + + + + diff --git a/packages/vitec/Resources/Public/Icons/vitec-import.svg b/packages/vitec/Resources/Public/Icons/vitec-import.svg new file mode 100644 index 0000000..860f8fb --- /dev/null +++ b/packages/vitec/Resources/Public/Icons/vitec-import.svg @@ -0,0 +1,5 @@ + + + + + diff --git a/packages/vitec/ext_tables.sql b/packages/vitec/ext_tables.sql index 784eb07..1c4751a 100755 --- a/packages/vitec/ext_tables.sql +++ b/packages/vitec/ext_tables.sql @@ -292,3 +292,15 @@ CREATE TABLE tx_vitec_form_submission ( delivery_status varchar(20) DEFAULT '' NOT NULL, delivery_error text ); + +CREATE TABLE tx_vitec_import_mapping ( + uid int(11) NOT NULL auto_increment, + pid int(11) DEFAULT '0' NOT NULL, + tstamp int(11) DEFAULT '0' NOT NULL, + crdate int(11) DEFAULT '0' NOT NULL, + model varchar(64) DEFAULT '' NOT NULL, + identity_field varchar(64) DEFAULT 'slug' NOT NULL, + mapping text, + PRIMARY KEY (uid), + KEY model (model) +);