Merge pull request #24 from iknow/upload-default-language-first

Prevent parallel upload of default language

lib/phraseapp_updater/phraseapp_api.rb

@@ -4,6 +4,7 @@
 require 'phraseapp_updater/index_by'
 require 'uri'
 require 'phrase'
+require 'concurrent'
 require 'parallel'
 require 'tempfile'
 
@@ -96,38 +97,60 @@ def download_files(locales, skip_unverified:)
       end
     end
 
-    # Empirically, PhraseApp fails to parse the uploaded files when uploaded in
-    # parallel. Give it a better chance by uploading them one at a time.
     def upload_files(locale_files, default_locale:)
-      is_default = ->(l) { l.locale_name == default_locale }
+      locale_files = locale_files.sort_by(&:locale_name)
+      default_locale_file = locale_files.detect { |l| l.locale_name == default_locale }
+      locale_files.delete(default_locale_file) if default_locale_file
 
-      # Ensure the locales all exist
-      STDERR.puts('Creating locales')
       known_locales = fetch_locales.index_by(&:name)
+
+      # Phraseapp appears to use to use the first file uploaded to resolve conflicts
+      # between pluralized and non-pluralized keys. Upload and verify the canonical
+      # default locale first before uploading translated locales.
+      if default_locale_file
+        unless known_locales.has_key?(default_locale_file.locale_name)
+          STDERR.puts("Creating default locale (#{default_locale_file})")
+          create_locale(default_locale_file.locale_name, default: true)
+        end
+
+        STDERR.puts("Uploading default locale (#{default_locale_file})")
+        upload_id = upload_file(default_locale_file)
+
+        successful_default_upload = verify_uploads({ upload_id => default_locale_file })
+      else
+        STDERR.puts("No upload for default locale (#{default_locale})")
+      end
+
+      # Ensure the locales all exist
+      STDERR.puts('Creating translation locales')
       threaded_request(locale_files) do |locale_file|
         unless known_locales.has_key?(locale_file.locale_name)
-          create_locale(locale_file.locale_name, default: is_default.(locale_file))
+          create_locale(locale_file.locale_name, default: false)
         end
       end
 
-      # Upload the files in a stable order, ensuring the default locale is first.
-      locale_files.sort! do |a, b|
-        next -1 if is_default.(a)
-        next 1  if is_default.(b)
+      uploads = Concurrent::Hash.new
 
-        a.locale_name <=> b.locale_name
+      threaded_request(locale_files) do |locale_file|
+        STDERR.puts("Uploading #{locale_file}")
+        upload_id = upload_file(locale_file)
+        uploads[upload_id] = locale_file
       end
 
-      uploads = {}
+      successful_uploads = verify_uploads(uploads)
 
-      uploads = locale_files.to_h do |locale_file|
-        STDERR.puts("Uploading #{locale_file}")
-        upload_id = upload_file(locale_file)
-        [upload_id, locale_file]
+      if default_locale_file
+        successful_uploads = successful_uploads.merge(successful_default_upload)
       end
 
-      # Validate the uploads, retrying failures as necessary
-      successful_upload_ids = {}
+      successful_uploads
+    end
+
+    # Given a map of {upload_id => locale_file} pairs, use the upload_show
+    # API to verify that they're complete, and re-upload them if they failed.
+    # Return a map of locale name to upload id.
+    def verify_uploads(uploads)
+      successful_upload_ids = Concurrent::Hash.new
 
       STDERR.puts('Verifying uploads...')
       until uploads.empty?
@@ -157,6 +180,7 @@ def upload_files(locale_files, default_locale:)
       successful_upload_ids
     end
 
+
     def remove_keys_not_in_uploads(upload_ids)
       threaded_request(upload_ids) do |upload_id|
         STDERR.puts "Removing keys not in upload #{upload_id}"

lib/phraseapp_updater/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 class PhraseAppUpdater
-  VERSION = '3.3.0'
+  VERSION = '3.3.1'
 end

nix/gem/Gemfile

@@ -1,32 +1,32 @@
 source "https://rubygems.org/"
 source "https://rubygems.org/"
 
+gem "bigdecimal", "3.1.8"
 gem "byebug", "11.1.3"
 gem "coderay", "1.1.3"
+gem "concurrent-ruby", "1.0.5"
 gem "deep_merge", "1.2.2"
 gem "diff-lcs", "1.5.0"
 gem "ethon", "0.16.0"
-gem "ffi", "1.16.3"
+gem "ffi", "1.15.5"
 gem "hashdiff", "1.0.1"
 gem "io-console", "0.6.0"
-gem "irb", "1.10.0"
-gem "json", "2.7.0"
+gem "irb", "1.6.2"
+gem "json", "2.6.3"
 gem "link-header-parser", "4.0.0"
 gem "method_source", "1.0.0"
-gem "oj", "3.16.1"
-gem "parallel", "1.23.0"
+gem "oj", "3.16.6"
+gem "ostruct", "0.6.0"
+gem "parallel", "1.26.3"
 gem "phrase", "2.20.0"
 gem "pry", "0.14.2"
-gem "psych", "5.1.1.1"
-gem "rake", "13.1.0"
-gem "rdoc", "6.6.0"
-gem "reline", "0.4.1"
+gem "rake", "13.2.1"
+gem "reline", "0.3.2"
 gem "rspec", "3.12.0"
-gem "rspec-core", "3.12.2"
-gem "rspec-expectations", "3.12.3"
-gem "rspec-mocks", "3.12.6"
-gem "rspec-support", "3.12.1"
+gem "rspec-core", "3.12.1"
+gem "rspec-expectations", "3.12.2"
+gem "rspec-mocks", "3.12.3"
+gem "rspec-support", "3.12.0"
 gem "rspec_junit_formatter", "0.6.0"
-gem "stringio", "3.1.0"
-gem "thor", "1.3.0"
-gem "typhoeus", "1.4.1"
+gem "thor", "1.3.2"
+gem "typhoeus", "1.4.0"

nix/gem/Gemfile.lock

@@ -1,90 +1,90 @@
 GEM
   remote: https://rubygems.org/
   specs:
+    bigdecimal (3.1.8)
     byebug (11.1.3)
     coderay (1.1.3)
+    concurrent-ruby (1.0.5)
     deep_merge (1.2.2)
     diff-lcs (1.5.0)
     ethon (0.16.0)
       ffi (>= 1.15.0)
-    ffi (1.16.3)
+    ffi (1.15.5)
     hashdiff (1.0.1)
     io-console (0.6.0)
-    irb (1.10.0)
-      rdoc
-      reline (>= 0.3.8)
-    json (2.7.0)
+    irb (1.6.2)
+      reline (>= 0.3.0)
+    json (2.6.3)
     link-header-parser (4.0.0)
     method_source (1.0.0)
-    oj (3.16.1)
-    parallel (1.23.0)
+    oj (3.16.6)
+      bigdecimal (>= 3.0)
+      ostruct (>= 0.2)
+    ostruct (0.6.0)
+    parallel (1.26.3)
     phrase (2.20.0)
       json (~> 2.1, >= 2.1.0)
       link-header-parser (~> 4.0)
       typhoeus (~> 1.0, >= 1.0.1)
     pry (0.14.2)
       coderay (~> 1.1)
       method_source (~> 1.0)
-    psych (5.1.1.1)
-      stringio
-    rake (13.1.0)
-    rdoc (6.6.0)
-      psych (>= 4.0.0)
-    reline (0.4.1)
+    rake (13.2.1)
+    reline (0.3.2)
       io-console (~> 0.5)
     rspec (3.12.0)
       rspec-core (~> 3.12.0)
       rspec-expectations (~> 3.12.0)
       rspec-mocks (~> 3.12.0)
-    rspec-core (3.12.2)
+    rspec-core (3.12.1)
       rspec-support (~> 3.12.0)
-    rspec-expectations (3.12.3)
+    rspec-expectations (3.12.2)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.12.0)
-    rspec-mocks (3.12.6)
+    rspec-mocks (3.12.3)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.12.0)
-    rspec-support (3.12.1)
+    rspec-support (3.12.0)
     rspec_junit_formatter (0.6.0)
       rspec-core (>= 2, < 4, != 2.12.0)
-    stringio (3.1.0)
-    thor (1.3.0)
-    typhoeus (1.4.1)
+    thor (1.3.2)
+    typhoeus (1.4.0)
       ethon (>= 0.9.0)
 
 PLATFORMS
   ruby
+  x86_64-darwin-23
 
 DEPENDENCIES
+  bigdecimal (= 3.1.8)
   byebug (= 11.1.3)
   coderay (= 1.1.3)
+  concurrent-ruby (= 1.0.5)
   deep_merge (= 1.2.2)
   diff-lcs (= 1.5.0)
   ethon (= 0.16.0)
-  ffi (= 1.16.3)
+  ffi (= 1.15.5)
   hashdiff (= 1.0.1)
   io-console (= 0.6.0)
-  irb (= 1.10.0)
-  json (= 2.7.0)
+  irb (= 1.6.2)
+  json (= 2.6.3)
   link-header-parser (= 4.0.0)
   method_source (= 1.0.0)
-  oj (= 3.16.1)
-  parallel (= 1.23.0)
+  oj (= 3.16.6)
+  ostruct (= 0.6.0)
+  parallel (= 1.26.3)
   phrase (= 2.20.0)
   pry (= 0.14.2)
-  psych (= 5.1.1.1)
-  rake (= 13.1.0)
-  rdoc (= 6.6.0)
-  reline (= 0.4.1)
+  rake (= 13.2.1)
+  reline (= 0.3.2)
   rspec (= 3.12.0)
-  rspec-core (= 3.12.2)
-  rspec-expectations (= 3.12.3)
-  rspec-mocks (= 3.12.6)
-  rspec-support (= 3.12.1)
+  rspec-core (= 3.12.1)
+  rspec-expectations (= 3.12.2)
+  rspec-mocks (= 3.12.3)
+  rspec-support (= 3.12.0)
   rspec_junit_formatter (= 0.6.0)
-  stringio (= 3.1.0)
-  thor (= 1.3.0)
-  typhoeus (= 1.4.1)
+  thor (= 1.3.2)
+  typhoeus (= 1.4.0)
 
 BUNDLED WITH
-   2.4.22
+   2.5.16