From 4d44d1443c5fb245c3af64fd7a0745bb6bf52e2a Mon Sep 17 00:00:00 2001 From: Joshua Schumacher Date: Thu, 22 Dec 2022 15:01:52 +0100 Subject: [PATCH 1/2] Implemented the bolt content translations --- composer.json | 2 + src/Configuration/Config.php | 10 + .../Backend/ContentTranslationController.php | 618 ++++++++++++++++++ src/Menu/BackendMenuBuilder.php | 11 +- src/Repository/FieldRepository.php | 37 +- templates/translations/index.html.twig | 28 + translations/messages.de.xlf | 42 ++ translations/messages.en.xlf | 42 ++ translations/validators.de.xlf | 6 + translations/validators.en.xlf | 6 + 10 files changed, 793 insertions(+), 9 deletions(-) create mode 100644 src/Controller/Backend/ContentTranslationController.php create mode 100644 templates/translations/index.html.twig diff --git a/composer.json b/composer.json index b8d5f4028..ce34715b0 100644 --- a/composer.json +++ b/composer.json @@ -8,6 +8,7 @@ "ext-json": "*", "ext-mbstring": "*", "ext-pdo": "*", + "ext-zip": "*", "composer-runtime-api": "^2", "api-platform/core": "^2.6", "babdev/pagerfanta-bundle": "^2.11", @@ -47,6 +48,7 @@ "symfony/console": "^5.3", "symfony/debug-bundle": "^5.3", "symfony/dependency-injection": "^5.3", + "symfony/dom-crawler": "^5.2", "symfony/dotenv": "^5.3", "symfony/error-handler": "^5.3", "symfony/event-dispatcher": "^5.3", diff --git a/src/Configuration/Config.php b/src/Configuration/Config.php index afd6033c1..04d37c41d 100644 --- a/src/Configuration/Config.php +++ b/src/Configuration/Config.php @@ -268,6 +268,16 @@ public function getFileTypes(): Collection return new Collection($this->get('general/accept_file_types')); } + public function getLocales(): array + { + return explode('|', $this->locales); + } + + public function getDefaultLocale(): string + { + return $this->defaultLocale; + } + public function getMaxUpload(): int { return min( diff --git a/src/Controller/Backend/ContentTranslationController.php b/src/Controller/Backend/ContentTranslationController.php new file mode 100644 index 000000000..3d7f598e4 --- /dev/null +++ b/src/Controller/Backend/ContentTranslationController.php @@ -0,0 +1,618 @@ +config = $config; + $this->entityManager = $entityManager; + $this->fieldRepository = $fieldRepository; + $this->translator = $translator; + } + + /** + * @return Response + * + * @Route( + * "/content-translations", + * name="bolt_content_translations_index", + * methods={"GET"}, + * ) + */ + public function indexAction(): Response + { + $form = $this->getForm(); + + return $this->renderForm( + '@bolt/translations/index.html.twig', + [ + 'form' => $form, + ] + ); + } + + /** + * @param Request $request + * @return Response + * @throws Exception + * + * @Route( + * "/content-translations/export", + * name="bolt_content_translations_export", + * methods={"GET"}, + * ) + */ + public function exportAction(Request $request): Response + { + $this->currentHost = $request->getHttpHost(); + $this->locales = $this->config->getLocales(); + $this->defaultLocale = $this->config->getDefaultLocale(); + $catalogues = $this->generateTranslationCatalogues(); + + return $this->createZipFile(); + } + + /** + * @return void + * @throws Exception + */ + private function generateTranslationCatalogues() + { + $catalogues = $this->createCatalogues(); + $translations = $this->getExistingTranslations(); + $groupedTranslations = $this->groupTranslations($translations); + + $this->addTranslationsToCatalogues($catalogues, $groupedTranslations); + $this->dumpCatalogues($catalogues); + + return $catalogues; + } + + /** + * @param Request $request + * @return Response + * @throws Exception + * + * @Route( + * "/content-translations/import", + * name="bolt_content_translations_import", + * methods={"GET", "POST"}, + * ) + */ + public function importAction(Request $request): Response + { + $notFound = []; + $form = $this->getForm(); + + $form->handleRequest(); + + if ( + $form->isSubmitted() && + $form->isValid() + ) { + $form = $request->files->get('form'); + + if (isset($form['files'])) { + $files = $form['files']; + + foreach ($files as $file) { + if ($file instanceof UploadedFile) { + $notFound = [...$notFound, ...$this->processXlfTranslations($file)]; + } + } + + if (count($notFound)) { + foreach ($notFound as $locale => $fields) { + $notFoundValues = implode(',
', $fields); + + $request->getSession()->getFlashBag()->add( + 'warning', + $this->translator->trans( + 'import.not_found_warning', + [ + '{{ locale }}' => $locale, + '{{ fields }}' => $notFoundValues, + ], + 'validators' + ) + ); + } + } + } + + return new RedirectResponse($this->generateUrl('bolt_content_translations_index')); + } + + throw new Exception('Could not process the imported translation file.'); + } + + /** + * @return FormInterface + */ + private function getForm(): FormInterface + { + $formFactory = Forms::createFormFactory(); + $form = $formFactory->createBuilder() + ->setAction($this->generateUrl('bolt_content_translations_import')) + ->add( + 'files', + FileType::class, + [ + 'label' => $this->translator->trans('bolt_content_translations_files', [], 'messages'), + 'required' => true, + 'multiple' => true, + ] + ) + ->add( + 'save', + SubmitType::class, + [ + 'label' => $this->translator->trans('bolt_content_translations_send', [], 'messages'), + 'attr' => [ + 'class' => 'btn btn-secondary btn-small', + ], + ] + ) + ->getForm() + ; + + return $form; + } + + /** + * @return array + */ + private function createCatalogues(): array + { + $catalogues = []; + + foreach ($this->locales as $locale) { + $catalogues[$locale] = new MessageCatalogue($locale); + } + + return $catalogues; + } + + /** + * @return array + * @throws Exception + */ + private function getExistingTranslations(): array + { + $translations = []; + $allFields = $this->fieldRepository->findAll(); + + foreach ($allFields as $field) { + if ($field->isTranslatable()) { + $fieldTranslations = []; + + foreach ($field->getTranslations() as $translation) { + $values = $translation->getValue(); + + if (count($values) <= 1) { + $value = $values[0]; + + if ($value) { + $fieldTranslations[$translation->getLocale()] = [ + 'value' => $value, + 'type' => $field->getType(), + ]; + } + } else if (!in_array($field->getType(), self::NON_TEXT_FIELDS)) { + throw new Exception('Unknown field type give, please contact a bolt developer to check this seemingly special case.'); + } + } + + if (count($fieldTranslations)) { + $translations[$field->getId()] = $fieldTranslations; + } + } + } + + return $translations; + } + + /** + * @param array $translations + * @return array + */ + private function groupTranslations(array $translations): array + { + $groupedTranslations = []; + + foreach ($translations as $id => $translation) { + foreach ($this->locales as $locale) { + if (isset($translation[$this->defaultLocale])) { + if (!isset($groupedTranslations[$locale])) { + $groupedTranslations[$locale] = []; + } + + $value = $translation[$this->defaultLocale]['value']; + + // We encode the string to avoid errors with special characters that cannot be used as an array key + $key = base64_encode($value); + + if (!isset($groupedTranslations[$locale][$key])) { + $groupedTranslations[$locale][$key] = [ + 'text' => $translation[$locale]['value'] ?? $translation[$this->defaultLocale]['value'], + 'type' => $translation[$locale]['type'] ?? $translation[$this->defaultLocale]['type'], + 'ids' => [$id], + ]; + } else { + $groupedTranslations[$locale][$key]['ids'] = array_merge($groupedTranslations[$locale][$key]['ids'], [$id]); + } + } + } + } + + return $groupedTranslations; + } + + /** + * @param array $catalogues + * @param array $translations + * @return void + */ + private function addTranslationsToCatalogues(array $catalogues, array $translations): void + { + /** + * @var MessageCatalogue $catalogue + */ + foreach ($catalogues as $locale => $catalogue) { + foreach ($translations[$locale] as $key => $translation) { + $decodedKey = base64_decode($key); + $translationText = $translation['text']; + $translationType = $translation['type']; + $translationTextJson = json_decode($translationText); + + if (!$translationTextJson) { + $this->addTranslationToCatalogues($catalogue, $decodedKey, $translationText, $translation['ids'], $translationText, $translationType); + } else { + foreach ($translationTextJson as $key => $textParts) { + $this->addTranslationToCatalogues($catalogue, $textParts, $textParts, $translation['ids'], $translationText, $translationType, $key); + } + } + } + } + } + + /** + * @param MessageCatalogue $catalogue + * @param string $decodedKey + * @param string $translationText + * @param array $translationIds + * @param string|null $originalText + * @param string|null $translationType + * @param string|null $postfix + * @return void + */ + private function addTranslationToCatalogues(MessageCatalogue $catalogue, string $decodedKey, string $translationText, array $translationIds, string $originalText = null, string $translationType = null, string $postfix = null) + { + if ($translationText) { + $catalogue->add( + [ + $decodedKey => $translationText, + ], + $this->currentHost + ); + + $idNotes = []; + + foreach ($translationIds as $id) { + $idNotes[] = [ + 'category' => 'id', + 'content' => $id, + ]; + + $idNotes[] = [ + 'category' => 'type', + 'content' => $translationType, + ]; + + if ($postfix) { + $idNotes[] = [ + 'category' => 'original', + 'content' => $originalText, + ]; + + $idNotes[] = [ + 'category' => 'part', + 'content' => $postfix, + ]; + } + } + + $catalogue->setMetadata( + $decodedKey, + [ + 'notes' => $idNotes, + ], + $this->currentHost + ); + } + } + + /** + * @param array $catalogues + * @return void + */ + private function dumpCatalogues(array $catalogues): void + { + $dumper = new XliffFileDumper(); + + foreach ($catalogues as $catalogue) { + $dumper->dump( + $catalogue, + [ + 'path' => sys_get_temp_dir(), + 'xliff_version' => '2.0', + ] + ); + } + } + + /** + * @return Response + */ + private function createZipFile(): Response + { + $zip = new ZipArchive(); + $files = glob(sys_get_temp_dir() . '/' . $this->currentHost . '.*.xlf'); + $zipName = $this->currentHost . '.zip'; + + $zip->open( + $zipName, + ZipArchive::CREATE + ); + + foreach ($files as $file) { + $zip->addFromString(basename($file), file_get_contents($file)); + + @unlink($file); + } + + $zip->close(); + + $response = new Response(file_get_contents($zipName)); + + $response->headers->set('Content-Type', 'application/zip'); + $response->headers->set('Content-Disposition', 'attachment;filename="' . $zipName . '"'); + $response->headers->set('Content-length', filesize($zipName)); + + @unlink($zipName); + + return $response; + } + + /** + * @param UploadedFile $file + * @return array + */ + private function processXlfTranslations(UploadedFile $file): array + { + $fileContent = file_get_contents($file->getPathname()); + $crawler = new Crawler($fileContent); + $xliff = $crawler->filterXPath('//xliff'); + $targetLanguage = $xliff->attr('trgLang'); + $translations = $crawler->filterXPath('//xliff/file/unit')->each( + function (Crawler $parentCrawler) { + $results = []; + $source = $parentCrawler->filterXPath('node()/segment/source')->text(); + $original = $parentCrawler->filterXPath('node()/notes/note[@category="original"]'); + $part = $parentCrawler->filterXPath('node()/notes/note[@category="part"]'); + $ids = $parentCrawler->filterXPath('node()/notes/note[@category="id"]')->each( + function (Crawler $parentCrawler) { + return $parentCrawler->text(); + } + ); + + foreach ($ids as $id) { + $results['ids'][$id] = $parentCrawler->filterXPath('node()/segment/target')->text(); + } + + if ($original->count()) { + $results['original'] = $original->text(); + } + + if ($part->count()) { + $results['part'] = $part->text(); + } + + $results['source'] = $source; + + return $results; + } + ); + + $groupedTranslations = $this->groupXlfTranslations($translations); + + return $this->persistXlfTranslations($groupedTranslations, $targetLanguage); + } + + /** + * @param array $translations + * @return array + */ + private function groupXlfTranslations(array $translations): array + { + $groupedTranslations = []; + + foreach ($translations as $translation) { + if (isset($translation['original'])) { + $groupedTranslations[implode('__', array_keys($translation['ids']))][] = $translation; + } else { + $groupedTranslations[implode('__', array_keys($translation['ids']))] = $translation; + } + } + + foreach ($groupedTranslations as $key => $groupedTranslation) { + $mergedTranslation = [ + 'ids' => [], + ]; + + foreach ($groupedTranslation as $translationPart) { + if (isset($translationPart['original'])) { + $mergedTranslation['source'] = $translationPart['original']; + + foreach ($translationPart['ids'] as $id => $translation) { + if (!isset($mergedTranslation['ids'][$id])) { + $mergedTranslation['ids'][$id] = json_decode($translationPart['original']); + } + + $mergedTranslation['ids'][$id]->{$translationPart['part']} = $translation; + } + } else { + $mergedTranslation['source'] = $groupedTranslation['source']; + } + } + + if (count($mergedTranslation['ids'])) { + $groupedTranslations[$key] = $mergedTranslation; + } + } + + return $groupedTranslations; + } + + /** + * @param array $translations + * @param string $targetLanguage + * @return array + */ + private function persistXlfTranslations(array $translations, string $targetLanguage): array + { + $notFound = []; + + foreach ($translations as $translationGroup) { + $source = $translationGroup['source']; + + foreach ($translationGroup['ids'] as $id => $value) { + if (is_object($value)) { + $value = json_encode($value); + } + + $field = $this->fieldRepository->findOneById($id); + + if (!$field) { + $field = $this->fieldRepository->findOneByTranslationValue($source); + } + + if ($field) { + $fieldTranslation = $this->getFieldTranslation($field->getTranslations(), $targetLanguage); + + $fieldTranslation->setValue($value); + $fieldTranslation->setLocale($targetLanguage); + $field->addTranslation($fieldTranslation); + + $this->entityManager->persist($field); + } else { + $notFound[$targetLanguage][] = $value; + } + } + } + + $this->entityManager->flush(); + + return $notFound; + } + + /** + * @param Collection $translations + * @param string $locale + * @return FieldTranslation + */ + private function getFieldTranslation(Collection $translations, string $locale): FieldTranslation + { + $translation = array_values( + array_filter( + $translations->getValues(), + function (FieldTranslation $translation) use ($locale) { + return $translation->getLocale() === $locale; + } + ) + ); + + if ($translation) { + return $translation[0]; + } + + return new FieldTranslation(); + } +} diff --git a/src/Menu/BackendMenuBuilder.php b/src/Menu/BackendMenuBuilder.php index 2f6b68689..7272804fd 100644 --- a/src/Menu/BackendMenuBuilder.php +++ b/src/Menu/BackendMenuBuilder.php @@ -6,7 +6,6 @@ use Bolt\Configuration\Config; use Bolt\Configuration\Content\ContentType; -use Bolt\Entity\Content; use Bolt\Repository\ContentRepository; use Bolt\Security\ContentVoter; use Bolt\Twig\ContentExtension; @@ -261,6 +260,16 @@ private function createAdminMenu(): ItemInterface ]); } + if ($this->authorizationChecker->isGranted('translation')) { + $menu->getChild('Maintenance')->addChild('ContentTranslations', [ + 'uri' => $this->urlGenerator->generate('bolt_content_translations_index'), + 'extras' => [ + 'name' => $t->trans('caption.content_translations'), + 'icon' => 'fa-language', + ], + ]); + } + // Hide this menu item, unless we're on a "Git clone" install and user has 'kitchensink' permissions if (Version::installType() === 'Git clone' && $this->authorizationChecker->isGranted('kitchensink')) { $menu->getChild('Maintenance')->addChild('The Kitchensink', [ diff --git a/src/Repository/FieldRepository.php b/src/Repository/FieldRepository.php index 67534a75b..0aee1c4c8 100644 --- a/src/Repository/FieldRepository.php +++ b/src/Repository/FieldRepository.php @@ -42,7 +42,7 @@ private function getQueryBuilder(?QueryBuilder $qb = null) public function findOneBySlug(string $slug): ?Field { - $qb = $this->getQueryBuilder(); + $qb = $this->getQueryBuilder(); $connection = $qb->getEntityManager()->getConnection(); [$where, $slug] = JsonHelper::wrapJsonFunction('translations.value', $slug, $connection); @@ -56,12 +56,13 @@ public function findOneBySlug(string $slug): ?Field ->setParameter('type', 'slug') ->setMaxResults(1) ->getQuery() - ->getOneOrNullResult(); + ->getOneOrNullResult() + ; } public function findAllBySlug(string $slug): array { - $qb = $this->getQueryBuilder(); + $qb = $this->getQueryBuilder(); $connection = $qb->getEntityManager()->getConnection(); [$where, $slug] = JsonHelper::wrapJsonFunction('translations.value', $slug, $connection); @@ -74,12 +75,13 @@ public function findAllBySlug(string $slug): array ->andWhere('field INSTANCE OF :type') ->setParameter('type', 'slug') ->getQuery() - ->getResult(); + ->getResult() + ; } public function findAllByParent(Field $field): ?array { - if (! $field instanceof FieldParentInterface || ! $field->getId()) { + if (!$field instanceof FieldParentInterface || !$field->getId()) { return []; } @@ -90,12 +92,13 @@ public function findAllByParent(Field $field): ?array ->setParameter('parentId', $field->getId()) ->orderBy('field.sortorder', 'ASC') ->getQuery() - ->getResult(); + ->getResult() + ; } public static function factory(Collection $definition, string $name = '', string $label = ''): Field { - $type = $definition['type']; + $type = $definition['type']; $classname = self::getFieldClassname($type); if ($classname && class_exists($classname)) { @@ -136,7 +139,8 @@ public static function getFieldClassname(string $type): ?string $allFields = collect($classes) ->filter(function ($class) { return in_array('Bolt\\Entity\\FieldInterface', class_implements($class), true); - }); + }) + ; // Classnames that end with $classname $match = $allFields->filter(function ($class) use ($classname) { @@ -145,4 +149,21 @@ public static function getFieldClassname(string $type): ?string return $match->isNotEmpty() ? $match->first() : null; } + + public function findOneByTranslationValue(string $value) + { + $connection = $this->getEntityManager()->getConnection(); + + $result = $this->getQueryBuilder() + ->leftJoin('field.translations', 'field_translation') + ->where(JsonHelper::wrapJsonFunction('field_translation.value', null, $connection) . ' = :value') + ->setParameter('value', $value) + ->getQuery() + ->getResult() + ; + + if (isset($result[0])) { + return $result[0]; + } + } } diff --git a/templates/translations/index.html.twig b/templates/translations/index.html.twig new file mode 100644 index 000000000..08d604b06 --- /dev/null +++ b/templates/translations/index.html.twig @@ -0,0 +1,28 @@ +{% extends '@bolt/_base/layout.html.twig' %} +{% import '@bolt/_macro/_macro.html.twig' as macro %} + +{% block title %} + {{ macro.icon('language') }} {{ 'import_export_content_translations'|trans }} +{% endblock %} + +{% block main %} + +{% endblock %} diff --git a/translations/messages.de.xlf b/translations/messages.de.xlf index 20ecb2327..4e3e6f1b7 100644 --- a/translations/messages.de.xlf +++ b/translations/messages.de.xlf @@ -363,6 +363,48 @@ Usernamen oder Email eingeben + + + import_export_content_translations + Import/Export Inhalts-Übersetzungen + + + + + export_all_content_translations + Alle Inhalts-Übersetzungen exportieren + + + + + export + Exportieren + + + + + import + Importieren + + + + + bolt_content_translations_files + Übersetzungsdateien + + + + + bolt_content_translations_send + Inhalts-Übersetzungen importieren + + + + + caption.content_translations + Inhalts-Übersetzungen + + bolt-core/src/Form/LoginType.php:63 diff --git a/translations/messages.en.xlf b/translations/messages.en.xlf index 865108a0b..7fbe8d2c3 100644 --- a/translations/messages.en.xlf +++ b/translations/messages.en.xlf @@ -2725,6 +2725,48 @@ You have to login in order to access this page. + + + import_export_content_translations + Import/Export Content Translations + + + + + export_all_content_translations + Export All Content Translations + + + + + export + Export + + + + + import + Import + + + + + bolt_content_translations_files + Translation files + + + + + bolt_content_translations_send + Import translations + + + + + caption.content_translations + Content Translations + + modal.title.file_field diff --git a/translations/validators.de.xlf b/translations/validators.de.xlf index eeb62b0f9..c51e96c97 100644 --- a/translations/validators.de.xlf +++ b/translations/validators.de.xlf @@ -1339,5 +1339,11 @@ Zu viele Tags (höchstens {{ limit }} Tags sind erlaubt) + + + import.not_found_warning + {{ fields }}]]> + + diff --git a/translations/validators.en.xlf b/translations/validators.en.xlf index c2a9cf389..bd77dad50 100644 --- a/translations/validators.en.xlf +++ b/translations/validators.en.xlf @@ -673,5 +673,11 @@ Invalid display name + + + import.not_found_warning + {{ fields }}]]> + + From 75662864bb0766cb98dcc8c46407abc2ca633c1c Mon Sep 17 00:00:00 2001 From: Joshua Schumacher Date: Thu, 22 Dec 2022 15:25:28 +0100 Subject: [PATCH 2/2] Use array merge instead of the spread operator, which does not work with php 7 --- src/Controller/Backend/ContentTranslationController.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Controller/Backend/ContentTranslationController.php b/src/Controller/Backend/ContentTranslationController.php index 3d7f598e4..be4712426 100644 --- a/src/Controller/Backend/ContentTranslationController.php +++ b/src/Controller/Backend/ContentTranslationController.php @@ -174,7 +174,7 @@ public function importAction(Request $request): Response foreach ($files as $file) { if ($file instanceof UploadedFile) { - $notFound = [...$notFound, ...$this->processXlfTranslations($file)]; + $notFound = array_merge($notFound, $this->processXlfTranslations($file)); } }