Turn geonames locale into rox locale and allow to use fallback locale…

… in Manticore.

src/Command/GeonamesUpdateFullCommand.php

@@ -55,7 +55,7 @@ protected function configure()
                 null,
                 InputOption::VALUE_NONE,
                 'Fetches all data; truncates the database tables and imports the data. Sets all options ' .
-                '(except --continue-on-errors and --country). Does download the files if not explicitly forbidden.'
+                '(except --continue-on-errors, --country, and --update). Does download the files if not explicitly forbidden.'
             )
             ->addOption(
                 'admin-units',
@@ -87,6 +87,12 @@ protected function configure()
                 InputOption::VALUE_NONE,
                 'Continue importing the alternate names even if an error happened while importing geonames.'
             )
+            ->addOption(
+                'update',
+                null,
+                InputOption::VALUE_NONE,
+                'Update all rows instead of inserting new ones. If used with \'full\' will not truncate the tables.'
+            )
             ->setHelp('Downloads geonames or alternatenames data dump and imports them')
         ;
     }
@@ -112,6 +118,7 @@ protected function execute(InputInterface $input, OutputInterface $output): int
             $alternateNames = true;
             $adminUnits = true;
         }
+        $update = $input->getOption('update');
 
         $this->output = $output;
 
@@ -549,7 +556,7 @@ private function updateGeonamesInDatabase(SymfonyStyle $io, array $rows, Progres
         $connection->executeQuery('SET FOREIGN_KEY_CHECKS=0');
         // Build the query from scratch
         $query =
-            'INSERT IGNORE INTO geo__names (`geonameId`, `name`, `latitude`, `longitude`, `feature_class`, `feature_code`,'
+            'INSERT INTO geo__names (`geonameId`, `name`, `latitude`, `longitude`, `feature_class`, `feature_code`,'
             . '`country_id`, `admin_1_id`, `admin_2_id`, `admin_3_id`, `admin_4_id`, `population`, `moddate`) '
             . 'VALUES '
         ;
@@ -583,6 +590,7 @@ private function updateGeonamesInDatabase(SymfonyStyle $io, array $rows, Progres
             }
         }
         $query = substr($query, 0, -2);
+        $query .= " ON DUPLICATE KEY UPDATE";
         $progressbar->setMessage('Executing query...', 'status');
         $connection->executeQuery($query);
         $connection->executeQuery('SET FOREIGN_KEY_CHECKS=1');
@@ -615,10 +623,10 @@ private function updateAlternatenamesInDatabase(SymfonyStyle $io, array $rows, P
                 // always use lower case for locale
                 switch ($row[2]) {
                     case 'zh-TW':
-                        $row[2] = 'zh_hant';
+                        $row[2] = 'zh-hant';
                         break;
                     case 'zh-CN':
-                        $row[2] = 'zh_hans';
+                        $row[2] = 'zh-hans';
                         break;
                 }
 

src/Command/ManticoreIndicesCreateCommand.php

@@ -254,7 +254,7 @@ private function addGeonamesDocumentsToIndex(Index $index, NativeQuery $query, P
                 'isPlace' => $isPlace,
                 'isAdmin' => $isAdmin,
                 'isCountry' => $isCountry,
-                'locale' => $location['locale'],
+                'locale' => $this->adaptLocale($location['locale']),
                 'admin1' => $location['admin1'],
                 'admin2' => $location['admin2'],
                 'admin3' => $location['admin3'],
@@ -305,4 +305,21 @@ private function getProgressBar(OutputInterface $output, $count): ProgressBar
 
         return $progressBar;
     }
+
+    private function adaptLocale(string $locale)
+    {
+        switch ($locale) {
+            case "zh-TW":
+                $locale = "zh-hant";
+                break;
+            case "zh-CN":
+                $locale = "zh-hans";
+                break;
+            case "pt-BR":
+                $locale = "pt-br";
+                break;
+        }
+
+        return $locale;
+    }
 }

src/Model/SuggestLocationModel.php

@@ -79,7 +79,7 @@ private function getPlacesExact(string $term, int $limit, ?string $translationId
         $query = $this->getQueryForGeonamesRt();
         $matchQuery = new MatchPhrase($parts[0], 'name');
 
-        $locale = $this->adaptLocale($this->translator->getLocale());
+        $locale = $this->translator->getLocale();
 
         $localeQuery = new BoolQuery();
         $localeQuery->should(new Equals('locale', '_geo'));
@@ -188,9 +188,8 @@ private function getAdminUnits(string $term, int $limit = 10): array
 
         $query = $this->getQueryForGeonamesRt();
         $matchQuery = new MatchQuery($parts[0] . '*', 'name');
-        $localeQuery = new BoolQuery();
-        $localeQuery->should(new Equals('locale', '_geo'));
-        $localeQuery->should(new Equals('locale', $this->translator->getLocale()));
+        $localeQuery = $this->getLocaleQuery();
+
         $filterElements = ['country', 'admin1', 'admin2', 'admin3', 'admin4'];
         $adminUnitFilterQuery = new BoolQuery();
         foreach ($adminUnits as $adminUnit) {
@@ -282,7 +281,7 @@ private function getCountryId(string $countryOrAdminUnit): ?string
 
     public function getLocationDetails(array $results, string $typeTranslationId = null): array
     {
-        $locale = $this->adaptLocale($this->translator->getLocale());
+        $locale = $this->translator->getLocale();
         $type = '';
         if (null !== $typeTranslationId) {
             $type = $this->translator->trans($typeTranslationId);
@@ -322,7 +321,7 @@ public function getLocationDetails(array $results, string $typeTranslationId = n
 
     private function getDetailsForId($id)
     {
-        $locale = $this->adaptLocale($this->translator->getLocale());
+        $locale = $this->translator->getLocale();
         $qb = $this->entityManager->createQueryBuilder();
         $query = $qb
             ->select('l')
@@ -438,19 +437,27 @@ private function getManticoreResults(Search $query, int $limit = 1000): array
 
     private function getQueryForGeonamesRt(): Search
     {
+        $locale = $this->translator->getLocale();
         $config = ['host' => '127.0.0.1','port' => 9412];
         $client = new Client($config);
         $query = new Search($client);
         $query
-            ->setIndex('geonames_rt')
-            ->orFilter('locale', 'equals', $this->translator->getLocale())
-            ->orFilter('locale', 'equals', '_geo');
+            ->setIndex('geonames_rt');
 
         return $query;
     }
 
-    private function adaptLocale(string $locale)
+    private function getLocaleQuery(): BoolQuery
     {
-        return substr($locale, 0, 2);
+        $locale = $this->translator->getLocale();
+        $localeQuery = new BoolQuery();
+        $localeQuery->should(new Equals('locale', '_geo'));
+        $localeQuery->should(new Equals('locale', $locale));
+
+        if (strlen($locale) > 2) {
+            $localeQuery->should(new Equals('locale', substr($locale, 0, 2)));
+        }
+
+        return $localeQuery;
     }
 }