____-. convention * - legacy names are normalized on fetch so every imported file matches * the convention: __NN_A -> __NN-A, ___NN -> __NN, __NNA -> __NN-A, * and a bare __NN gets -A appended as its initial revision; the * filepath fallback in the renderers remains as a safety net * - filetype[0] -> existing `categories` field (categories under * --category-parent, matched via sys_category.filetype, then title) * - type[0] -> new `type` field (categories under --type-parent, * matched via sys_category.type, then title) * - records without title or file_url are skipped; records sharing one * file_url are merged (kept: filled fileprefix, then lowest uid) * - existing records (matched by slug) are skipped unless --force * * Usage: * vendor/bin/typo3 vitec:import-downloads --dry-run * vendor/bin/typo3 vitec:import-downloads --only=aligo-datasheet --force * * Records are stored on pid 29 (the downloads sysfolder) unless --pid says * otherwise. */ #[AsCommand( name: 'vitec:import-downloads', description: 'Import downloads from the old-website JSON export (Collateral directory only)' )] final class ImportDownloadsCommand extends Command { private const TABLE = 'tx_vitec_domain_model_download'; private const CATEGORY_TABLE = 'sys_category'; private const COLLATERAL_URL_PREFIX = 'https://www.vitec.com/fileadmin/downloads/Collateral/'; private const TARGET_REL_DIR = 'fileadmin/downloads/Collateral'; private const CONVENTION_PATTERN = '/^(.+)__([A-Za-z]+)__(\d+)-([A-Za-z]+)\.[A-Za-z0-9]+$/'; private const TYPE_PARENT_TITLE = 'Download Type'; protected function configure(): void { $this->addOption('url', null, InputOption::VALUE_REQUIRED, 'Export URL', 'https://www.vitec.com/import'); $this->addOption('file', null, InputOption::VALUE_REQUIRED, 'Read export from a local JSON/HTML file instead of the URL'); $this->addOption('pid', null, InputOption::VALUE_REQUIRED, 'Storage pid for new download records', '29'); $this->addOption('dry-run', null, InputOption::VALUE_NONE, 'Report only - no files, no records, no categories'); $this->addOption('only', null, InputOption::VALUE_REQUIRED, 'Import only the record with this slug'); $this->addOption('force', null, InputOption::VALUE_NONE, 'Update records that already exist (matched by slug)'); $this->addOption('skip-files', null, InputOption::VALUE_NONE, 'Do not download files, only write records'); $this->addOption('category-parent', null, InputOption::VALUE_REQUIRED, 'Parent uid of the filetype categories', '4'); $this->addOption('type-parent', null, InputOption::VALUE_REQUIRED, 'Parent uid of the type categories (0 = find/create "' . self::TYPE_PARENT_TITLE . '")', '0'); } protected function execute(InputInterface $input, OutputInterface $output): int { Bootstrap::initializeBackendAuthentication(); $dryRun = (bool)$input->getOption('dry-run'); $force = (bool)$input->getOption('force'); $skipFiles = (bool)$input->getOption('skip-files'); $only = trim((string)$input->getOption('only')); // ------------------------------------------------------------- load $raw = $this->loadRaw($input, $output); if ($raw === null) { return Command::FAILURE; } $rows = $this->extractRows($raw, $output); if ($rows === null) { return Command::FAILURE; } $output->writeln(sprintf('Export: %d records', count($rows))); // ---------------------------------------------------------- prepare $outOfScope = []; $skipped = []; $inScope = []; foreach ($rows as $r) { $title = trim((string)($r['title'] ?? '')); $fileUrl = trim((string)($r['file_url'] ?? '')); $slug = trim((string)($r['slug'] ?? '')); if ($title === '' || $fileUrl === '' || $slug === '') { $skipped[] = sprintf('uid=%s (no title/file_url/slug)', $r['uid'] ?? '?'); continue; } if (!str_starts_with($fileUrl, self::COLLATERAL_URL_PREFIX)) { $outOfScope[] = sprintf('uid=%s %s', $r['uid'] ?? '?', $fileUrl); continue; } $inScope[] = $r; } // merge records sharing one file_url: keep filled fileprefix, then lowest uid $byUrl = []; $merged = []; foreach ($inScope as $r) { $url = (string)$r['file_url']; if (!isset($byUrl[$url])) { $byUrl[$url] = $r; continue; } $kept = $byUrl[$url]; $keepNew = trim((string)($r['fileprefix'] ?? '')) !== '' && trim((string)($kept['fileprefix'] ?? '')) === ''; if (!$keepNew && trim((string)($r['fileprefix'] ?? '')) === trim((string)($kept['fileprefix'] ?? ''))) { $keepNew = (int)$r['uid'] < (int)$kept['uid']; } if ($keepNew) { $merged[] = sprintf('uid=%d superseded by uid=%d (%s)', (int)$kept['uid'], (int)$r['uid'], basename($url)); $byUrl[$url] = $r; } else { $merged[] = sprintf('uid=%d superseded by uid=%d (%s)', (int)$r['uid'], (int)$kept['uid'], basename($url)); } } $work = array_values($byUrl); if ($only !== '') { $work = array_values(array_filter($work, static fn(array $r): bool => (string)$r['slug'] === $only)); if ($work === []) { $output->writeln(sprintf('--only=%s matches nothing in scope.', $only)); return Command::FAILURE; } } $output->writeln(sprintf( 'In scope: %d (out of scope: %d, skipped: %d, merged duplicates: %d)', count($work), count($outOfScope), count($skipped), count($merged) )); foreach ($skipped as $s) { $output->writeln(' skip ' . $s); } foreach ($merged as $m) { $output->writeln(' merge ' . $m); } if ($output->isVerbose()) { foreach ($outOfScope as $o) { $output->writeln(' outside ' . $o); } } // -------------------------------------------------------------- pid $pid = (int)($input->getOption('pid') ?? 0); if ($pid <= 0) { $pid = $this->detectPid(); } if ($pid <= 0) { $output->writeln('No --pid given and no existing download records to derive it from.'); return Command::FAILURE; } $output->writeln(sprintf('Storage pid: %d', $pid)); // -------------------------------------------------- category lookup $categoryParent = (int)$input->getOption('category-parent'); $typeParent = (int)$input->getOption('type-parent'); $filetypeValues = $this->collectValues($work, 'filetype'); $typeValues = $this->collectValues($work, 'type'); if ($typeParent <= 0) { $typeParent = $this->findOrCreateTypeParent($categoryParent, $dryRun, $output); } $filetypeMap = $this->resolveCategories($filetypeValues, $categoryParent, 'filetype', $dryRun, $output); $typeMap = $this->resolveCategories($typeValues, $typeParent, 'type', $dryRun, $output); // ------------------------------------------------------------ files $targetDir = Environment::getPublicPath() . '/' . self::TARGET_REL_DIR; if (!$dryRun && !$skipFiles && !is_dir($targetDir)) { GeneralUtility::mkdir_deep($targetDir); } $created = $updated = $existing = $filesFetched = $filesPresent = $fileErrors = 0; $nonConvention = []; $renamed = []; foreach ($work as $r) { $slug = (string)$r['slug']; $fileUrl = (string)$r['file_url']; $filename = basename(parse_url($fileUrl, PHP_URL_PATH) ?: ''); if ($filename === '') { $output->writeln(sprintf(' %s: cannot derive filename from %s', $slug, $fileUrl)); continue; } // Normalize legacy separators so the runtime convention matches // on the new site (decision 2026-08-06: options A + C). $normalized = $this->normalizeFilename($filename); if ($normalized !== $filename) { $renamed[] = sprintf('%s -> %s', $filename, $normalized); $filename = $normalized; } if (!preg_match(self::CONVENTION_PATTERN, $filename)) { $nonConvention[] = $filename; } // --- file $targetFile = $targetDir . '/' . $filename; if (!$skipFiles) { if (is_file($targetFile)) { $filesPresent++; } elseif ($dryRun) { $filesFetched++; // would fetch } else { if ($this->fetchFile($fileUrl, $targetFile, $output)) { $filesFetched++; } else { $fileErrors++; $output->writeln(sprintf(' %s: download failed, record skipped', $slug)); continue; } } } // --- record $existingUid = $this->findBySlug($slug); if ($existingUid > 0 && !$force) { $existing++; continue; } $data = [ 'pid' => $pid, 'title' => (string)$r['title'], 'slug' => $slug, 'description' => (string)($r['description'] ?? ''), 'fileprefix' => (string)($r['fileprefix'] ?? ''), 'filepath' => '/' . self::TARGET_REL_DIR . '/' . $filename, ]; $ft = $this->firstValue($r, 'filetype'); if ($ft !== '' && isset($filetypeMap[$ft])) { $data['categories'] = (string)$filetypeMap[$ft]; } $ty = $this->firstValue($r, 'type'); if ($ty !== '' && isset($typeMap[$ty])) { $data['type'] = (string)$typeMap[$ty]; } if ($dryRun) { $output->writeln(sprintf( ' %-8s %-45s cat=%s type=%s %s', $existingUid > 0 ? 'UPDATE' : 'CREATE', substr($slug, 0, 45), $ft !== '' ? $ft : '-', $ty !== '' ? $ty : '-', $filename )); $existingUid > 0 ? $updated++ : $created++; continue; } $id = $existingUid > 0 ? (string)$existingUid : 'NEW' . md5($slug); if ($existingUid > 0) { unset($data['pid']); } $dataHandler = GeneralUtility::makeInstance(DataHandler::class); $dataHandler->start([self::TABLE => [$id => $data]], []); $dataHandler->process_datamap(); if ($dataHandler->errorLog !== []) { $output->writeln(sprintf(' %s: %s', $slug, implode(' | ', $dataHandler->errorLog))); continue; } $existingUid > 0 ? $updated++ : $created++; } // ------------------------------------------------------------ report $output->writeln(''); $output->writeln(sprintf( '%s: %d created, %d updated, %d already present (use --force to update)', $dryRun ? 'DRY-RUN' : 'Done', $created, $updated, $existing )); if (!$skipFiles) { $output->writeln(sprintf( 'Files: %d %s, %d already on disk, %d failed', $filesFetched, $dryRun ? 'to fetch' : 'fetched', $filesPresent, $fileErrors )); } if ($renamed !== []) { $output->writeln(sprintf( '%d file(s) renamed on fetch to match the version convention:', count($renamed) )); foreach ($renamed as $rn) { $output->writeln(' ~ ' . $rn); } } if ($nonConvention !== []) { $output->writeln(sprintf( '%d filename(s) do not match the ____- convention;' . ' runtime resolution falls back to filepath for these:', count($nonConvention) )); foreach ($nonConvention as $n) { $output->writeln(' ! ' . $n); } } return $fileErrors === 0 ? Command::SUCCESS : Command::FAILURE; } // ------------------------------------------------------------ loading private function loadRaw(InputInterface $input, OutputInterface $output): ?string { $file = trim((string)$input->getOption('file')); if ($file !== '') { if (!is_file($file)) { $output->writeln(sprintf('File not found: %s', $file)); return null; } return (string)file_get_contents($file); } $url = (string)$input->getOption('url'); try { $response = GeneralUtility::makeInstance(RequestFactory::class) ->request($url, 'GET', ['timeout' => 120]); return (string)$response->getBody(); } catch (\Throwable $e) { $output->writeln(sprintf('Fetch failed: %s', $e->getMessage())); return null; } } /** * The export page wraps the JSON in the full site template. Cut the * {"count":...} object out of the HTML by brace counting (string-safe). * * @return array>|null */ private function extractRows(string $raw, OutputInterface $output): ?array { $start = strpos($raw, '{"count"'); if ($start === false) { // maybe it is clean JSON already $decoded = json_decode($raw, true); if (is_array($decoded) && isset($decoded['downloads'])) { return $decoded['downloads']; } $output->writeln('No {"count" marker found in the response.'); return null; } $depth = 0; $inString = false; $escaped = false; $end = null; $len = strlen($raw); for ($i = $start; $i < $len; $i++) { $ch = $raw[$i]; if ($inString) { if ($escaped) { $escaped = false; } elseif ($ch === '\\') { $escaped = true; } elseif ($ch === '"') { $inString = false; } continue; } if ($ch === '"') { $inString = true; } elseif ($ch === '{') { $depth++; } elseif ($ch === '}') { $depth--; if ($depth === 0) { $end = $i + 1; break; } } } if ($end === null) { $output->writeln('Unbalanced JSON block in the response.'); return null; } $decoded = json_decode(html_entity_decode(substr($raw, $start, $end - $start), ENT_QUOTES | ENT_HTML5), true); if (!is_array($decoded) || !isset($decoded['downloads']) || !is_array($decoded['downloads'])) { $output->writeln('Extracted block is not the expected {count, downloads[]} object.'); return null; } return $decoded['downloads']; } // --------------------------------------------------------- categories /** @param array> $rows * @return array */ private function collectValues(array $rows, string $field): array { $values = []; foreach ($rows as $r) { $v = $this->firstValue($r, $field); if ($v !== '') { $values[$v] = true; } } return array_keys($values); } /** @param array $row */ private function firstValue(array $row, string $field): string { $v = $row[$field] ?? null; if (is_array($v)) { $v = $v[0] ?? ''; } return trim((string)$v); } /** * Map export values to sys_category uids under $parent. Match order: * custom column ($matchColumn), then title. Missing categories are * created (with $matchColumn set) unless dry-run. * * @param array $values * @return array */ private function resolveCategories(array $values, int $parent, string $matchColumn, bool $dryRun, OutputInterface $output): array { $map = []; $qb = GeneralUtility::makeInstance(ConnectionPool::class)->getQueryBuilderForTable(self::CATEGORY_TABLE); $cats = $qb ->select('uid', 'pid', 'title', $matchColumn) ->from(self::CATEGORY_TABLE) ->where( $qb->expr()->eq('parent', $qb->createNamedParameter($parent, ParameterType::INTEGER)), $qb->expr()->eq('deleted', 0) ) ->executeQuery() ->fetchAllAssociative(); $parentRow = $this->categoryRow($parent); $catPid = $parentRow !== null ? (int)$parentRow['pid'] : 0; foreach ($values as $value) { foreach ($cats as $c) { if (strcasecmp(trim((string)($c[$matchColumn] ?? '')), $value) === 0 || strcasecmp(trim((string)$c['title']), $value) === 0 ) { $map[$value] = (int)$c['uid']; continue 2; } } if ($dryRun) { $output->writeln(sprintf(' would create category "%s" (parent %d, %s)', $value, $parent, $matchColumn)); $map[$value] = 0; // placeholder so dry-run still reports assignment continue; } $uid = $this->createCategory($value, $parent, $catPid, $matchColumn); if ($uid > 0) { $output->writeln(sprintf(' created category "%s" -> uid %d (parent %d)', $value, $uid, $parent)); $map[$value] = $uid; } else { $output->writeln(sprintf(' could not create category "%s"', $value)); } } return $map; } private function findOrCreateTypeParent(int $categoryParent, bool $dryRun, OutputInterface $output): int { $qb = GeneralUtility::makeInstance(ConnectionPool::class)->getQueryBuilderForTable(self::CATEGORY_TABLE); $row = $qb ->select('uid') ->from(self::CATEGORY_TABLE) ->where( $qb->expr()->eq('title', $qb->createNamedParameter(self::TYPE_PARENT_TITLE, ParameterType::STRING)), $qb->expr()->eq('deleted', 0) ) ->setMaxResults(1) ->executeQuery() ->fetchAssociative(); if ($row) { return (int)$row['uid']; } $anchor = $this->categoryRow($categoryParent); $pid = $anchor !== null ? (int)$anchor['pid'] : 0; if ($dryRun) { $output->writeln(sprintf(' would create parent category "%s" (pid %d)', self::TYPE_PARENT_TITLE, $pid)); return 0; } $uid = $this->createCategory(self::TYPE_PARENT_TITLE, 0, $pid, ''); $output->writeln(sprintf(' created parent category "%s" -> uid %d', self::TYPE_PARENT_TITLE, $uid)); return $uid; } /** @return array|null */ private function categoryRow(int $uid): ?array { if ($uid <= 0) { return null; } $qb = GeneralUtility::makeInstance(ConnectionPool::class)->getQueryBuilderForTable(self::CATEGORY_TABLE); $row = $qb->select('uid', 'pid', 'title')->from(self::CATEGORY_TABLE) ->where($qb->expr()->eq('uid', $qb->createNamedParameter($uid, ParameterType::INTEGER))) ->executeQuery()->fetchAssociative(); return $row ?: null; } private function createCategory(string $title, int $parent, int $pid, string $matchColumn): int { $data = ['pid' => $pid, 'parent' => $parent, 'title' => $title]; if ($matchColumn !== '') { $data[$matchColumn] = $title; } $id = 'NEW' . md5('cat' . $parent . $title); $dataHandler = GeneralUtility::makeInstance(DataHandler::class); $dataHandler->start([self::CATEGORY_TABLE => [$id => $data]], []); $dataHandler->process_datamap(); return (int)($dataHandler->substNEWwithIDs[$id] ?? 0); } // -------------------------------------------------------------- misc /** * Bring legacy version segments onto the - convention: * ___NN -> __NN (collapsed underscores) * __NN_A -> __NN-A (underscore as version separator) * __NNA -> __NN-A (missing separator) * __NN -> __NN-A (no revision letter at all: -A = initial * revision, so a future -B supersedes it) * Verified against the full 2026-08 export: all 179 names match the * convention afterwards, none of the already-correct ones change. */ private function normalizeFilename(string $filename): string { $filename = (string)preg_replace('/_{3,}(\d)/', '__$1', $filename); $filename = (string)preg_replace('/__(\d+)_([A-Za-z]+)\.([A-Za-z0-9]+)$/', '__$1-$2.$3', $filename); $filename = (string)preg_replace('/__(\d+)([A-Za-z]+)\.([A-Za-z0-9]+)$/', '__$1-$2.$3', $filename); return (string)preg_replace('/__(\d+)\.([A-Za-z0-9]+)$/', '__$1-A.$2', $filename); } private function detectPid(): int { $qb = GeneralUtility::makeInstance(ConnectionPool::class)->getQueryBuilderForTable(self::TABLE); $row = $qb->select('pid')->from(self::TABLE) ->where($qb->expr()->eq('deleted', 0)) ->setMaxResults(1) ->executeQuery()->fetchAssociative(); return $row ? (int)$row['pid'] : 0; } private function findBySlug(string $slug): int { $qb = GeneralUtility::makeInstance(ConnectionPool::class)->getQueryBuilderForTable(self::TABLE); $row = $qb->select('uid')->from(self::TABLE) ->where( $qb->expr()->eq('slug', $qb->createNamedParameter($slug, ParameterType::STRING)), $qb->expr()->eq('deleted', 0) ) ->setMaxResults(1) ->executeQuery()->fetchAssociative(); return $row ? (int)$row['uid'] : 0; } private function fetchFile(string $url, string $targetFile, OutputInterface $output): bool { try { $response = GeneralUtility::makeInstance(RequestFactory::class) ->request($url, 'GET', ['timeout' => 300]); if ($response->getStatusCode() !== 200) { return false; } $tmp = $targetFile . '.part'; $fh = fopen($tmp, 'wb'); if ($fh === false) { return false; } $body = $response->getBody(); while (!$body->eof()) { fwrite($fh, $body->read(1048576)); } fclose($fh); if (filesize($tmp) === 0) { @unlink($tmp); return false; } rename($tmp, $targetFile); if ($output->isVerbose()) { $output->writeln(sprintf(' fetched %s (%.1f KB)', basename($targetFile), filesize($targetFile) / 1024)); } return true; } catch (\Throwable $e) { $output->writeln(sprintf(' %s: %s', basename($targetFile), $e->getMessage())); return false; } } }