From bcf90785a578e3269df1c05146c0eaea0f05788f Mon Sep 17 00:00:00 2001 From: Chris Andreae Date: Mon, 30 Sep 2024 15:36:40 +0900 Subject: [PATCH] Prevent parallel upload of default language Permitted keys of translation languages seem to depend on the default language, so upload the default language first. In addition, upload translation languages in parallel. --- lib/phraseapp_updater/phraseapp_api.rb | 46 ++++++---- lib/phraseapp_updater/version.rb | 2 +- nix/gem/Gemfile | 32 +++---- nix/gem/Gemfile.lock | 74 ++++++++-------- nix/gem/gemset.nix | 117 ++++++++++++------------- phraseapp_updater.gemspec | 1 + 6 files changed, 143 insertions(+), 129 deletions(-) diff --git a/lib/phraseapp_updater/phraseapp_api.rb b/lib/phraseapp_updater/phraseapp_api.rb index 61fb26f..2658762 100644 --- a/lib/phraseapp_updater/phraseapp_api.rb +++ b/lib/phraseapp_updater/phraseapp_api.rb @@ -99,35 +99,48 @@ def download_files(locales, skip_unverified:) # Empirically, PhraseApp fails to parse the uploaded files when uploaded in # parallel. Give it a better chance by uploading them one at a time. def upload_files(locale_files, default_locale:) - is_default = ->(l) { l.locale_name == default_locale } + locale_files = locale_files.sort_by(&:locale_name) + default_locale_file = locale_files.delete { l.locale_name == default_locale } + + if default_locale_file + STDERR.puts("Creating default locale (#{locale_file})") + unless known_locales.has_key?(default_locale_file.locale_name) + create_locale(default_locale_file.locale_name, default: true) + end + + STDERR.puts("Uploading default locale (#{locale_file})") + upload_id = upload_file(default_locale_file) + + verify_uploads({ upload_id => default_locale_file }) + else + STDERR.puts("No upload for default locale (#{default_locale})") + end # Ensure the locales all exist - STDERR.puts('Creating locales') + STDERR.puts('Creating translation locales') known_locales = fetch_locales.index_by(&:name) threaded_request(locale_files) do |locale_file| unless known_locales.has_key?(locale_file.locale_name) - create_locale(locale_file.locale_name, default: is_default.(locale_file)) + create_locale(locale_file.locale_name, default: false) end end - # Upload the files in a stable order, ensuring the default locale is first. - locale_files.sort! do |a, b| - next -1 if is_default.(a) - next 1 if is_default.(b) + uploads = Concurrent::Hash.new - a.locale_name <=> b.locale_name - end - - uploads = {} - - uploads = locale_files.to_h do |locale_file| + threaded_request(locale_files) do |locale_file| STDERR.puts("Uploading #{locale_file}") upload_id = upload_file(locale_file) - [upload_id, locale_file] + uploads[upload_id] = locale_file end - # Validate the uploads, retrying failures as necessary - successful_upload_ids = {} + verify_uploads(uploads) + end + + # Given a map of {upload_id => locale_file} pairs, use the upload_show + # API to verify that they're complete, and re-upload them if they failed. + # Return a map of locale name to upload id. + def verify_uploads(uploads) + successful_upload_ids = Concurrent::Hash.new STDERR.puts('Verifying uploads...') until uploads.empty? @@ -157,6 +170,7 @@ def upload_files(locale_files, default_locale:) successful_upload_ids end + def remove_keys_not_in_uploads(upload_ids) threaded_request(upload_ids) do |upload_id| STDERR.puts "Removing keys not in upload #{upload_id}" diff --git a/lib/phraseapp_updater/version.rb b/lib/phraseapp_updater/version.rb index 14b5e4a..e8c42a4 100644 --- a/lib/phraseapp_updater/version.rb +++ b/lib/phraseapp_updater/version.rb @@ -1,5 +1,5 @@ # frozen_string_literal: true class PhraseAppUpdater - VERSION = '3.3.0' + VERSION = '3.3.1' end diff --git a/nix/gem/Gemfile b/nix/gem/Gemfile index af29a76..d95b408 100644 --- a/nix/gem/Gemfile +++ b/nix/gem/Gemfile @@ -1,32 +1,32 @@ source "https://rubygems.org/" source "https://rubygems.org/" +gem "bigdecimal", "3.1.8" gem "byebug", "11.1.3" gem "coderay", "1.1.3" +gem "concurrent-ruby", "1.0.5" gem "deep_merge", "1.2.2" gem "diff-lcs", "1.5.0" gem "ethon", "0.16.0" -gem "ffi", "1.16.3" +gem "ffi", "1.15.5" gem "hashdiff", "1.0.1" gem "io-console", "0.6.0" -gem "irb", "1.10.0" -gem "json", "2.7.0" +gem "irb", "1.6.2" +gem "json", "2.6.3" gem "link-header-parser", "4.0.0" gem "method_source", "1.0.0" -gem "oj", "3.16.1" -gem "parallel", "1.23.0" +gem "oj", "3.16.6" +gem "ostruct", "0.6.0" +gem "parallel", "1.26.3" gem "phrase", "2.20.0" gem "pry", "0.14.2" -gem "psych", "5.1.1.1" -gem "rake", "13.1.0" -gem "rdoc", "6.6.0" -gem "reline", "0.4.1" +gem "rake", "13.2.1" +gem "reline", "0.3.2" gem "rspec", "3.12.0" -gem "rspec-core", "3.12.2" -gem "rspec-expectations", "3.12.3" -gem "rspec-mocks", "3.12.6" -gem "rspec-support", "3.12.1" +gem "rspec-core", "3.12.1" +gem "rspec-expectations", "3.12.2" +gem "rspec-mocks", "3.12.3" +gem "rspec-support", "3.12.0" gem "rspec_junit_formatter", "0.6.0" -gem "stringio", "3.1.0" -gem "thor", "1.3.0" -gem "typhoeus", "1.4.1" +gem "thor", "1.3.2" +gem "typhoeus", "1.4.0" diff --git a/nix/gem/Gemfile.lock b/nix/gem/Gemfile.lock index d4edc55..ffd923a 100644 --- a/nix/gem/Gemfile.lock +++ b/nix/gem/Gemfile.lock @@ -1,23 +1,27 @@ GEM remote: https://rubygems.org/ specs: + bigdecimal (3.1.8) byebug (11.1.3) coderay (1.1.3) + concurrent-ruby (1.0.5) deep_merge (1.2.2) diff-lcs (1.5.0) ethon (0.16.0) ffi (>= 1.15.0) - ffi (1.16.3) + ffi (1.15.5) hashdiff (1.0.1) io-console (0.6.0) - irb (1.10.0) - rdoc - reline (>= 0.3.8) - json (2.7.0) + irb (1.6.2) + reline (>= 0.3.0) + json (2.6.3) link-header-parser (4.0.0) method_source (1.0.0) - oj (3.16.1) - parallel (1.23.0) + oj (3.16.6) + bigdecimal (>= 3.0) + ostruct (>= 0.2) + ostruct (0.6.0) + parallel (1.26.3) phrase (2.20.0) json (~> 2.1, >= 2.1.0) link-header-parser (~> 4.0) @@ -25,66 +29,62 @@ GEM pry (0.14.2) coderay (~> 1.1) method_source (~> 1.0) - psych (5.1.1.1) - stringio - rake (13.1.0) - rdoc (6.6.0) - psych (>= 4.0.0) - reline (0.4.1) + rake (13.2.1) + reline (0.3.2) io-console (~> 0.5) rspec (3.12.0) rspec-core (~> 3.12.0) rspec-expectations (~> 3.12.0) rspec-mocks (~> 3.12.0) - rspec-core (3.12.2) + rspec-core (3.12.1) rspec-support (~> 3.12.0) - rspec-expectations (3.12.3) + rspec-expectations (3.12.2) diff-lcs (>= 1.2.0, < 2.0) rspec-support (~> 3.12.0) - rspec-mocks (3.12.6) + rspec-mocks (3.12.3) diff-lcs (>= 1.2.0, < 2.0) rspec-support (~> 3.12.0) - rspec-support (3.12.1) + rspec-support (3.12.0) rspec_junit_formatter (0.6.0) rspec-core (>= 2, < 4, != 2.12.0) - stringio (3.1.0) - thor (1.3.0) - typhoeus (1.4.1) + thor (1.3.2) + typhoeus (1.4.0) ethon (>= 0.9.0) PLATFORMS ruby + x86_64-darwin-23 DEPENDENCIES + bigdecimal (= 3.1.8) byebug (= 11.1.3) coderay (= 1.1.3) + concurrent-ruby (= 1.0.5) deep_merge (= 1.2.2) diff-lcs (= 1.5.0) ethon (= 0.16.0) - ffi (= 1.16.3) + ffi (= 1.15.5) hashdiff (= 1.0.1) io-console (= 0.6.0) - irb (= 1.10.0) - json (= 2.7.0) + irb (= 1.6.2) + json (= 2.6.3) link-header-parser (= 4.0.0) method_source (= 1.0.0) - oj (= 3.16.1) - parallel (= 1.23.0) + oj (= 3.16.6) + ostruct (= 0.6.0) + parallel (= 1.26.3) phrase (= 2.20.0) pry (= 0.14.2) - psych (= 5.1.1.1) - rake (= 13.1.0) - rdoc (= 6.6.0) - reline (= 0.4.1) + rake (= 13.2.1) + reline (= 0.3.2) rspec (= 3.12.0) - rspec-core (= 3.12.2) - rspec-expectations (= 3.12.3) - rspec-mocks (= 3.12.6) - rspec-support (= 3.12.1) + rspec-core (= 3.12.1) + rspec-expectations (= 3.12.2) + rspec-mocks (= 3.12.3) + rspec-support (= 3.12.0) rspec_junit_formatter (= 0.6.0) - stringio (= 3.1.0) - thor (= 1.3.0) - typhoeus (= 1.4.1) + thor (= 1.3.2) + typhoeus (= 1.4.0) BUNDLED WITH - 2.4.22 + 2.5.16 diff --git a/nix/gem/gemset.nix b/nix/gem/gemset.nix index b19d5ac..c303d1b 100644 --- a/nix/gem/gemset.nix +++ b/nix/gem/gemset.nix @@ -1,4 +1,14 @@ { + bigdecimal = { + groups = ["default"]; + platforms = []; + source = { + remotes = ["https://rubygems.org"]; + sha256 = "1gi7zqgmqwi5lizggs1jhc3zlwaqayy9rx2ah80sxy24bbnng558"; + type = "gem"; + }; + version = "3.1.8"; + }; byebug = { groups = ["default"]; platforms = []; @@ -19,6 +29,16 @@ }; version = "1.1.3"; }; + concurrent-ruby = { + groups = ["default"]; + platforms = []; + source = { + remotes = ["https://rubygems.org"]; + sha256 = "183lszf5gx84kcpb779v6a2y0mx9sssy8dgppng1z9a505nj1qcf"; + type = "gem"; + }; + version = "1.0.5"; + }; deep_merge = { groups = ["default"]; platforms = []; @@ -55,10 +75,10 @@ platforms = []; source = { remotes = ["https://rubygems.org"]; - sha256 = "1yvii03hcgqj30maavddqamqy50h7y6xcn2wcyq72wn823zl4ckd"; + sha256 = "1862ydmclzy1a0cjbvm8dz7847d9rch495ib0zb64y84d3xd4bkg"; type = "gem"; }; - version = "1.16.3"; + version = "1.15.5"; }; hashdiff = { groups = ["default"]; @@ -81,25 +101,25 @@ version = "0.6.0"; }; irb = { - dependencies = ["rdoc" "reline"]; + dependencies = ["reline"]; groups = ["default"]; platforms = []; source = { remotes = ["https://rubygems.org"]; - sha256 = "05ha3lanjjfb0q0lp6bl0cpwvf372ix3m3jiwmkr87xyiwqck4dn"; + sha256 = "10qvm0s8784lhmz98blqnxh36i7d7rzkk17znx17d46666z7czrf"; type = "gem"; }; - version = "1.10.0"; + version = "1.6.2"; }; json = { groups = ["default"]; platforms = []; source = { remotes = ["https://rubygems.org"]; - sha256 = "0wi7g6c8q0v1kahwp38mv8d526p1n2ddsr79ajx84idvih0c601i"; + sha256 = "0nalhin1gda4v8ybk6lq8f407cgfrj6qzn234yra4ipkmlbfmal6"; type = "gem"; }; - version = "2.7.0"; + version = "2.6.3"; }; link-header-parser = { groups = ["default"]; @@ -122,24 +142,35 @@ version = "1.0.0"; }; oj = { + dependencies = ["bigdecimal" "ostruct"]; + groups = ["default"]; + platforms = []; + source = { + remotes = ["https://rubygems.org"]; + sha256 = "1k2skb0n7mf2azznnbsa6irwghdxlmnhdxv9qs6jqg3gd0k2n4zx"; + type = "gem"; + }; + version = "3.16.6"; + }; + ostruct = { groups = ["default"]; platforms = []; source = { remotes = ["https://rubygems.org"]; - sha256 = "0m4vsd6i093kmyz9gckvzpnws997laldaiaf86hg5lza1ir82x7n"; + sha256 = "11dsv71gfbhy92yzj3xkckjzdai2bsz5a4fydgimv62dkz4kc5rv"; type = "gem"; }; - version = "3.16.1"; + version = "0.6.0"; }; parallel = { groups = ["default"]; platforms = []; source = { remotes = ["https://rubygems.org"]; - sha256 = "0jcc512l38c0c163ni3jgskvq1vc3mr8ly5pvjijzwvfml9lf597"; + sha256 = "1vy7sjs2pgz4i96v5yk9b7aafbffnvq7nn419fgvw55qlavsnsyq"; type = "gem"; }; - version = "1.23.0"; + version = "1.26.3"; }; phrase = { dependencies = ["json" "link-header-parser" "typhoeus"]; @@ -163,37 +194,15 @@ }; version = "0.14.2"; }; - psych = { - dependencies = ["stringio"]; - groups = ["default"]; - platforms = []; - source = { - remotes = ["https://rubygems.org"]; - sha256 = "0wjzrkssjfjpynij5dpycyflhqbjvi1gc2j73xgq3b196s1d3c24"; - type = "gem"; - }; - version = "5.1.1.1"; - }; rake = { groups = ["default"]; platforms = []; source = { remotes = ["https://rubygems.org"]; - sha256 = "1ilr853hawi09626axx0mps4rkkmxcs54mapz9jnqvpnlwd3wsmy"; + sha256 = "17850wcwkgi30p7yqh60960ypn7yibacjjha0av78zaxwvd3ijs6"; type = "gem"; }; - version = "13.1.0"; - }; - rdoc = { - dependencies = ["psych"]; - groups = ["default"]; - platforms = []; - source = { - remotes = ["https://rubygems.org"]; - sha256 = "1pnkgnk2vli1y8bbc25qbgv6z2al44dlgcm2mx3ssm34j7xz7gqh"; - type = "gem"; - }; - version = "6.6.0"; + version = "13.2.1"; }; reline = { dependencies = ["io-console"]; @@ -201,10 +210,10 @@ platforms = []; source = { remotes = ["https://rubygems.org"]; - sha256 = "1hi6zfj6zqzxcbamhjm9w9cswv62f76l8gsdfcnmhpw35cyxphh8"; + sha256 = "1vpsmij5mknpiqy4b835rzl87slshm5dkr08hm8wypfya3v4m6cp"; type = "gem"; }; - version = "0.4.1"; + version = "0.3.2"; }; rspec = { dependencies = ["rspec-core" "rspec-expectations" "rspec-mocks"]; @@ -223,10 +232,10 @@ platforms = []; source = { remotes = ["https://rubygems.org"]; - sha256 = "0l95bnjxdabrn79hwdhn2q1n7mn26pj7y1w5660v5qi81x458nqm"; + sha256 = "0da45cvllbv39sdbsl65vp5djb2xf5m10mxc9jm7rsqyyxjw4h1f"; type = "gem"; }; - version = "3.12.2"; + version = "3.12.1"; }; rspec-expectations = { dependencies = ["diff-lcs" "rspec-support"]; @@ -234,10 +243,10 @@ platforms = []; source = { remotes = ["https://rubygems.org"]; - sha256 = "05j44jfqlv7j2rpxb5vqzf9hfv7w8ba46wwgxwcwd8p0wzi1hg89"; + sha256 = "03ba3lfdsj9zl00v1yvwgcx87lbadf87livlfa5kgqssn9qdnll6"; type = "gem"; }; - version = "3.12.3"; + version = "3.12.2"; }; rspec-mocks = { dependencies = ["diff-lcs" "rspec-support"]; @@ -245,20 +254,20 @@ platforms = []; source = { remotes = ["https://rubygems.org"]; - sha256 = "1gq7gviwpck7fhp4y5ibljljvxgjklza18j62qf6zkm2icaa8lfy"; + sha256 = "0sq2cc9pm5gq411y7iwfvzbmgv3g91lyf7y7cqn1lr3yf1v122nc"; type = "gem"; }; - version = "3.12.6"; + version = "3.12.3"; }; rspec-support = { groups = ["default"]; platforms = []; source = { remotes = ["https://rubygems.org"]; - sha256 = "1ky86j3ksi26ng9ybd7j0qsdf1lpr8mzrmn98yy9gzv801fvhsgr"; + sha256 = "12y52zwwb3xr7h91dy9k3ndmyyhr3mjcayk0nnarnrzz8yr48kfx"; type = "gem"; }; - version = "3.12.1"; + version = "3.12.0"; }; rspec_junit_formatter = { dependencies = ["rspec-core"]; @@ -271,25 +280,15 @@ }; version = "0.6.0"; }; - stringio = { - groups = ["default"]; - platforms = []; - source = { - remotes = ["https://rubygems.org"]; - sha256 = "063psvsn1aq6digpznxfranhcpmi0sdv2jhra5g0459sw0x2dxn1"; - type = "gem"; - }; - version = "3.1.0"; - }; thor = { groups = ["default"]; platforms = []; source = { remotes = ["https://rubygems.org"]; - sha256 = "1hx77jxkrwi66yvs10wfxqa8s25ds25ywgrrf66acm9nbfg7zp0s"; + sha256 = "1nmymd86a0vb39pzj2cwv57avdrl6pl3lf5bsz58q594kqxjkw7f"; type = "gem"; }; - version = "1.3.0"; + version = "1.3.2"; }; typhoeus = { dependencies = ["ethon"]; @@ -297,9 +296,9 @@ platforms = []; source = { remotes = ["https://rubygems.org"]; - sha256 = "0z7gamf6s83wy0yqms3bi4srirn3fc0lc7n65lqanidxcj1xn5qw"; + sha256 = "1m22yrkmbj81rzhlny81j427qdvz57yk5wbcf3km0nf3bl6qiygz"; type = "gem"; }; - version = "1.4.1"; + version = "1.4.0"; }; } diff --git a/phraseapp_updater.gemspec b/phraseapp_updater.gemspec index 46f6732..a2e5531 100644 --- a/phraseapp_updater.gemspec +++ b/phraseapp_updater.gemspec @@ -25,6 +25,7 @@ Gem::Specification.new do |spec| spec.add_dependency "oj", "~> 3.16" spec.add_dependency "deep_merge", "~> 1.2" spec.add_dependency "parallel", "~> 1.23" + spec.add_dependency "concurrent-ruby", "~> 1.0.2" spec.add_development_dependency "bundler", "~> 2.2" spec.add_development_dependency "rake", "~> 13.1"