From 5b6dd4e012a3c8b37f5bfec1b0bfc2acac014be2 Mon Sep 17 00:00:00 2001 From: thisismeonmounteverest Date: Sat, 28 Oct 2023 15:53:50 +0200 Subject: [PATCH] Turn geonames locale into rox locale and allow to use fallback locale in Manticore. --- src/Command/GeonamesUpdateFullCommand.php | 16 +++++++--- src/Command/ManticoreIndicesCreateCommand.php | 19 +++++++++++- src/Model/SuggestLocationModel.php | 29 ++++++++++++------- 3 files changed, 48 insertions(+), 16 deletions(-) diff --git a/src/Command/GeonamesUpdateFullCommand.php b/src/Command/GeonamesUpdateFullCommand.php index bf5a908f44..04d13af9e3 100644 --- a/src/Command/GeonamesUpdateFullCommand.php +++ b/src/Command/GeonamesUpdateFullCommand.php @@ -55,7 +55,7 @@ protected function configure() null, InputOption::VALUE_NONE, 'Fetches all data; truncates the database tables and imports the data. Sets all options ' . - '(except --continue-on-errors and --country). Does download the files if not explicitly forbidden.' + '(except --continue-on-errors, --country, and --update). Does download the files if not explicitly forbidden.' ) ->addOption( 'admin-units', @@ -87,6 +87,12 @@ protected function configure() InputOption::VALUE_NONE, 'Continue importing the alternate names even if an error happened while importing geonames.' ) + ->addOption( + 'update', + null, + InputOption::VALUE_NONE, + 'Update all rows instead of inserting new ones. If used with \'full\' will not truncate the tables.' + ) ->setHelp('Downloads geonames or alternatenames data dump and imports them') ; } @@ -112,6 +118,7 @@ protected function execute(InputInterface $input, OutputInterface $output): int $alternateNames = true; $adminUnits = true; } + $update = $input->getOption('update'); $this->output = $output; @@ -549,7 +556,7 @@ private function updateGeonamesInDatabase(SymfonyStyle $io, array $rows, Progres $connection->executeQuery('SET FOREIGN_KEY_CHECKS=0'); // Build the query from scratch $query = - 'INSERT IGNORE INTO geo__names (`geonameId`, `name`, `latitude`, `longitude`, `feature_class`, `feature_code`,' + 'INSERT INTO geo__names (`geonameId`, `name`, `latitude`, `longitude`, `feature_class`, `feature_code`,' . '`country_id`, `admin_1_id`, `admin_2_id`, `admin_3_id`, `admin_4_id`, `population`, `moddate`) ' . 'VALUES ' ; @@ -583,6 +590,7 @@ private function updateGeonamesInDatabase(SymfonyStyle $io, array $rows, Progres } } $query = substr($query, 0, -2); + $query .= " ON DUPLICATE KEY UPDATE"; $progressbar->setMessage('Executing query...', 'status'); $connection->executeQuery($query); $connection->executeQuery('SET FOREIGN_KEY_CHECKS=1'); @@ -615,10 +623,10 @@ private function updateAlternatenamesInDatabase(SymfonyStyle $io, array $rows, P // always use lower case for locale switch ($row[2]) { case 'zh-TW': - $row[2] = 'zh_hant'; + $row[2] = 'zh-hant'; break; case 'zh-CN': - $row[2] = 'zh_hans'; + $row[2] = 'zh-hans'; break; } diff --git a/src/Command/ManticoreIndicesCreateCommand.php b/src/Command/ManticoreIndicesCreateCommand.php index a163c87c8b..1bff631792 100644 --- a/src/Command/ManticoreIndicesCreateCommand.php +++ b/src/Command/ManticoreIndicesCreateCommand.php @@ -254,7 +254,7 @@ private function addGeonamesDocumentsToIndex(Index $index, NativeQuery $query, P 'isPlace' => $isPlace, 'isAdmin' => $isAdmin, 'isCountry' => $isCountry, - 'locale' => $location['locale'], + 'locale' => $this->adaptLocale($location['locale']), 'admin1' => $location['admin1'], 'admin2' => $location['admin2'], 'admin3' => $location['admin3'], @@ -305,4 +305,21 @@ private function getProgressBar(OutputInterface $output, $count): ProgressBar return $progressBar; } + + private function adaptLocale(string $locale) + { + switch ($locale) { + case "zh-TW": + $locale = "zh-hant"; + break; + case "zh-CN": + $locale = "zh-hans"; + break; + case "pt-BR": + $locale = "pt-br"; + break; + } + + return $locale; + } } diff --git a/src/Model/SuggestLocationModel.php b/src/Model/SuggestLocationModel.php index b4cc5c6db7..386ebd866d 100644 --- a/src/Model/SuggestLocationModel.php +++ b/src/Model/SuggestLocationModel.php @@ -79,7 +79,7 @@ private function getPlacesExact(string $term, int $limit, ?string $translationId $query = $this->getQueryForGeonamesRt(); $matchQuery = new MatchPhrase($parts[0], 'name'); - $locale = $this->adaptLocale($this->translator->getLocale()); + $locale = $this->translator->getLocale(); $localeQuery = new BoolQuery(); $localeQuery->should(new Equals('locale', '_geo')); @@ -188,9 +188,8 @@ private function getAdminUnits(string $term, int $limit = 10): array $query = $this->getQueryForGeonamesRt(); $matchQuery = new MatchQuery($parts[0] . '*', 'name'); - $localeQuery = new BoolQuery(); - $localeQuery->should(new Equals('locale', '_geo')); - $localeQuery->should(new Equals('locale', $this->translator->getLocale())); + $localeQuery = $this->getLocaleQuery(); + $filterElements = ['country', 'admin1', 'admin2', 'admin3', 'admin4']; $adminUnitFilterQuery = new BoolQuery(); foreach ($adminUnits as $adminUnit) { @@ -282,7 +281,7 @@ private function getCountryId(string $countryOrAdminUnit): ?string public function getLocationDetails(array $results, string $typeTranslationId = null): array { - $locale = $this->adaptLocale($this->translator->getLocale()); + $locale = $this->translator->getLocale(); $type = ''; if (null !== $typeTranslationId) { $type = $this->translator->trans($typeTranslationId); @@ -322,7 +321,7 @@ public function getLocationDetails(array $results, string $typeTranslationId = n private function getDetailsForId($id) { - $locale = $this->adaptLocale($this->translator->getLocale()); + $locale = $this->translator->getLocale(); $qb = $this->entityManager->createQueryBuilder(); $query = $qb ->select('l') @@ -438,19 +437,27 @@ private function getManticoreResults(Search $query, int $limit = 1000): array private function getQueryForGeonamesRt(): Search { + $locale = $this->translator->getLocale(); $config = ['host' => '127.0.0.1','port' => 9412]; $client = new Client($config); $query = new Search($client); $query - ->setIndex('geonames_rt') - ->orFilter('locale', 'equals', $this->translator->getLocale()) - ->orFilter('locale', 'equals', '_geo'); + ->setIndex('geonames_rt'); return $query; } - private function adaptLocale(string $locale) + private function getLocaleQuery(): BoolQuery { - return substr($locale, 0, 2); + $locale = $this->translator->getLocale(); + $localeQuery = new BoolQuery(); + $localeQuery->should(new Equals('locale', '_geo')); + $localeQuery->should(new Equals('locale', $locale)); + + if (strlen($locale) > 2) { + $localeQuery->should(new Equals('locale', substr($locale, 0, 2))); + } + + return $localeQuery; } }