Backend module: CSV import for the VITEC domain models

New "VITEC Import" module under Web: one import page per domain model
(v1: Market, Solution, Product - registry-driven, adding a model is one
config entry). Workflow: upload a CSV (delimiter and encoding are
auto-detected, including German-Excel semicolon/Windows-1252), map CSV
columns to DB fields, persist the mapping per model together with the
identity field used for matching (new table tx_vitec_import_mapping,
no TCA - pure tool configuration), review a unified list of CSV rows
matched against the DB records (new / update with differing fields /
unchanged / db-only), then apply the checked rows through DataHandler.

Each importable row carries an editable JSON payload textarea - what
is written is the textarea content, not the raw CSV, so editors can
fix values right in the review step. The parsed CSV travels through
the form as a hidden JSON field: no session state, no temp files.
Importable fields are derived from TCA at runtime (scalar types only;
files, categories and other relations are excluded - a flat CSV
cannot carry them). Payloads are whitelisted against that field list
on apply; new records require a storage pid (prefilled from existing
records). BE user permissions apply via DataHandler.

Deliberately out of v1: import log with three-way compare (protection
against overwriting manual edits), images/relations, multiple saved
mappings per model.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

packages/vitec/Classes/Import/CsvReader.php

@@ -0,0 +1,78 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Evomedien\Vitec\Import;
+
+/**
+ * Tolerant CSV parser for editor-supplied files.
+ *
+ * German Excel exports ship with semicolon delimiters and Windows-1252
+ * encoding; other tools use comma/UTF-8. Both are detected instead of
+ * configured: encoding by BOM/UTF-8 validity, delimiter by counting
+ * candidates in the header line. Multiline quoted values (e.g. RTE
+ * descriptions) are handled by fgetcsv on a temp stream.
+ */
+final class CsvReader
+{
+    /**
+     * @return array{columns: array<int,string>, rows: array<int,array<string,string>>}
+     */
+    public function parse(string $content): array
+    {
+        if (str_starts_with($content, "\xEF\xBB\xBF")) {
+            $content = substr($content, 3);
+        }
+        if (!mb_check_encoding($content, 'UTF-8')) {
+            $content = mb_convert_encoding($content, 'UTF-8', 'Windows-1252');
+        }
+
+        $firstLine = strtok($content, "\r\n") ?: '';
+        $delimiter = ';';
+        $best = 0;
+        foreach ([';', ',', "\t"] as $candidate) {
+            $count = substr_count($firstLine, $candidate);
+            if ($count > $best) {
+                $best = $count;
+                $delimiter = $candidate;
+            }
+        }
+
+        $stream = fopen('php://temp', 'r+');
+        fwrite($stream, $content);
+        rewind($stream);
+
+        $header = fgetcsv($stream, null, $delimiter, '"', '');
+        if (!is_array($header)) {
+            fclose($stream);
+            return ['columns' => [], 'rows' => []];
+        }
+        $columns = [];
+        foreach ($header as $i => $name) {
+            $name = trim((string)$name);
+            $columns[$i] = $name !== '' ? $name : ('column_' . ($i + 1));
+        }
+
+        $rows = [];
+        while (($data = fgetcsv($stream, null, $delimiter, '"', '')) !== false) {
+            if ($data === [null] || $data === ['']) {
+                continue; // blank line
+            }
+            $row = [];
+            $hasContent = false;
+            foreach ($columns as $i => $name) {
+                $value = trim((string)($data[$i] ?? ''));
+                $row[$name] = $value;
+                if ($value !== '') {
+                    $hasContent = true;
+                }
+            }
+            if ($hasContent) {
+                $rows[] = $row;
+            }
+        }
+        fclose($stream);
+
+        return ['columns' => array_values($columns), 'rows' => $rows];
+    }
+}

packages/vitec/Classes/Import/ImportModelRegistry.php

@@ -0,0 +1,115 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Evomedien\Vitec\Import;
+
+use TYPO3\CMS\Core\Database\ConnectionPool;
+use TYPO3\CMS\Core\Utility\GeneralUtility;
+
+/**
+ * Registry of the domain models the CSV import module can write to.
+ *
+ * Adding a model = one entry in MODELS. The importable fields are derived
+ * from the live TCA at runtime, so new columns show up in the mapper
+ * automatically. Only scalar column types are offered - files, categories
+ * and other relations cannot be carried by a flat CSV and are excluded.
+ */
+final class ImportModelRegistry
+{
+    private const MODELS = [
+        'market' => ['table' => 'tx_vitec_domain_model_market', 'label' => 'Markets'],
+        'solution' => ['table' => 'tx_vitec_domain_model_solution', 'label' => 'Solutions'],
+        'product' => ['table' => 'tx_vitec_domain_model_product', 'label' => 'Products'],
+    ];
+
+    /** Scalar TCA column types a CSV cell can populate. */
+    private const IMPORTABLE_TYPES = ['input', 'text', 'slug', 'number', 'email', 'link', 'check', 'color'];
+
+    private const EXCLUDED_FIELDS = [
+        'sys_language_uid', 'l10n_parent', 'l10n_diffsource', 'l10n_source',
+        'l18n_parent', 'l18n_diffsource', 'starttime', 'endtime', 'hidden',
+    ];
+
+    /** @return array<string,array{key:string,table:string,label:string}> */
+    public function all(): array
+    {
+        $out = [];
+        foreach (self::MODELS as $key => $cfg) {
+            $out[$key] = $cfg + ['key' => $key];
+        }
+        return $out;
+    }
+
+    public function has(string $key): bool
+    {
+        return isset(self::MODELS[$key]);
+    }
+
+    /** @return array{key:string,table:string,label:string} */
+    public function get(string $key): array
+    {
+        return self::MODELS[$key] + ['key' => $key];
+    }
+
+    /**
+     * Importable fields of a table, derived from TCA.
+     *
+     * @return array<string,string> fieldName => human label
+     */
+    public function importableFields(string $table): array
+    {
+        $fields = [];
+        foreach ($GLOBALS['TCA'][$table]['columns'] ?? [] as $name => $column) {
+            $type = (string)($column['config']['type'] ?? '');
+            if (!in_array($type, self::IMPORTABLE_TYPES, true)) {
+                continue;
+            }
+            if (in_array($name, self::EXCLUDED_FIELDS, true)) {
+                continue;
+            }
+            if (($column['config']['renderType'] ?? '') === 'inputDateTime') {
+                continue;
+            }
+            $label = (string)($column['label'] ?? $name);
+            if (str_starts_with($label, 'LLL:')) {
+                $translated = $GLOBALS['LANG'] ?? null ? (string)$GLOBALS['LANG']->sL($label) : '';
+                $label = $translated !== '' ? $translated : $name;
+            }
+            $fields[$name] = $label !== '' ? $label : $name;
+        }
+        return $fields;
+    }
+
+    /** Pid of the first existing record - the sysfolder the model lives in. */
+    public function detectPid(string $table): int
+    {
+        $qb = GeneralUtility::makeInstance(ConnectionPool::class)->getQueryBuilderForTable($table);
+        $row = $qb->select('pid')->from($table)
+            ->where($qb->expr()->eq('deleted', 0))
+            ->setMaxResults(1)
+            ->executeQuery()->fetchAssociative();
+        return $row ? (int)$row['pid'] : 0;
+    }
+
+    /**
+     * All live records with the given fields, indexed by uid.
+     *
+     * @param array<int,string> $fields
+     * @return array<int,array<string,mixed>>
+     */
+    public function loadRecords(string $table, array $fields): array
+    {
+        $select = array_values(array_unique(array_merge(['uid', 'title'], $fields)));
+        $qb = GeneralUtility::makeInstance(ConnectionPool::class)->getQueryBuilderForTable($table);
+        $rows = $qb->select(...$select)->from($table)
+            ->where($qb->expr()->eq('deleted', 0))
+            ->orderBy('title', 'ASC')
+            ->executeQuery()->fetchAllAssociative();
+        $out = [];
+        foreach ($rows as $row) {
+            $out[(int)$row['uid']] = $row;
+        }
+        return $out;
+    }
+}

packages/vitec/Classes/Import/MappingRepository.php

@@ -0,0 +1,65 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Evomedien\Vitec\Import;
+
+use Doctrine\DBAL\ParameterType;
+use TYPO3\CMS\Core\Database\ConnectionPool;
+use TYPO3\CMS\Core\Utility\GeneralUtility;
+
+/**
+ * Persists one CSV-column -> DB-field mapping per import model, so editors
+ * configure the mapper once and reuse it on every delivery.
+ *
+ * Plain table without TCA (tx_vitec_import_mapping) - the records are pure
+ * tool configuration, never edited through FormEngine.
+ */
+final class MappingRepository
+{
+    private const TABLE = 'tx_vitec_import_mapping';
+
+    /**
+     * @return array{mapping: array<string,string>, identity: string}|null
+     */
+    public function load(string $model): ?array
+    {
+        $qb = GeneralUtility::makeInstance(ConnectionPool::class)->getQueryBuilderForTable(self::TABLE);
+        $row = $qb->select('mapping', 'identity_field')->from(self::TABLE)
+            ->where($qb->expr()->eq('model', $qb->createNamedParameter($model, ParameterType::STRING)))
+            ->setMaxResults(1)
+            ->executeQuery()->fetchAssociative();
+        if (!$row) {
+            return null;
+        }
+        $mapping = json_decode((string)$row['mapping'], true);
+        if (!is_array($mapping)) {
+            return null;
+        }
+        return [
+            'mapping' => array_map('strval', $mapping),
+            'identity' => (string)$row['identity_field'],
+        ];
+    }
+
+    /** @param array<string,string> $mapping */
+    public function save(string $model, array $mapping, string $identity): void
+    {
+        $connection = GeneralUtility::makeInstance(ConnectionPool::class)->getConnectionForTable(self::TABLE);
+        $values = [
+            'mapping' => (string)json_encode($mapping, JSON_UNESCAPED_UNICODE),
+            'identity_field' => $identity,
+            'tstamp' => time(),
+        ];
+        $exists = $connection->count('uid', self::TABLE, ['model' => $model]) > 0;
+        if ($exists) {
+            $connection->update(self::TABLE, $values, ['model' => $model]);
+        } else {
+            $connection->insert(self::TABLE, $values + [
+                'model' => $model,
+                'pid' => 0,
+                'crdate' => time(),
+            ]);
+        }
+    }
+}