From 5cca70f4404b3a52366b92f3d687f8866594d85f Mon Sep 17 00:00:00 2001 From: ccaruceru Date: Tue, 22 Aug 2023 11:27:30 +0200 Subject: [PATCH 1/8] Add support for parallel GooglePlay uploads --- .../lib/fastlane_core/queue_worker.rb | 2 +- supply/lib/supply/uploader.rb | 35 +++++++++++++------ 2 files changed, 25 insertions(+), 12 deletions(-) diff --git a/fastlane_core/lib/fastlane_core/queue_worker.rb b/fastlane_core/lib/fastlane_core/queue_worker.rb index ab2a9069bad..869301efce6 100644 --- a/fastlane_core/lib/fastlane_core/queue_worker.rb +++ b/fastlane_core/lib/fastlane_core/queue_worker.rb @@ -6,7 +6,7 @@ module FastlaneCore # Use this when you have all the items that you'll process in advance. # Simply enqueue them to this and call `QueueWorker#start`. class QueueWorker - NUMBER_OF_THREADS = FastlaneCore::Helper.test? ? 1 : [(ENV["DELIVER_NUMBER_OF_THREADS"] || ENV.fetch("FL_NUMBER_OF_THREADS", 10)).to_i, 10].min + NUMBER_OF_THREADS = FastlaneCore::Helper.test? ? 1 : [(ENV["DELIVER_NUMBER_OF_THREADS"] || ENV.fetch("FL_NUMBER_OF_THREADS", 10)).to_i, 1].max # @param concurrency (Numeric) - A number of threads to be created # @param block (Proc) - A task you want to execute with enqueued items diff --git a/supply/lib/supply/uploader.rb b/supply/lib/supply/uploader.rb index 01345b05e29..cfdbc76c08f 100644 --- a/supply/lib/supply/uploader.rb +++ b/supply/lib/supply/uploader.rb @@ -1,6 +1,10 @@ +require 'fastlane_core' + module Supply # rubocop:disable Metrics/ClassLength class Uploader + UploadJob = Struct.new(:language, :version_code, :release_notes) + def perform_upload FastlaneCore::PrintTable.print_values(config: Supply.config, hide_keys: [:issuer], mask_keys: [:json_key_data], title: "Summary for supply #{Fastlane::VERSION}") @@ -81,6 +85,21 @@ def perform_upload_meta(version_codes, track_name) version_codes.to_s == "" end + upload_worker = FastlaneCore::QueueWorker.new do |job| + begin + UI.message("Preparing uploads for language '#{job.language}'...") + start_time = Time.now + listing = client.listing_for_language(job.language) + upload_metadata(job.language, listing) unless Supply.config[:skip_upload_metadata] + upload_images(job.language) unless Supply.config[:skip_upload_images] + upload_screenshots(job.language) unless Supply.config[:skip_upload_screenshots] + job.release_notes << upload_changelog(job.language, job.version_code) unless Supply.config[:skip_upload_changelogs] + UI.message("Uploaded all items for language '#{job.language}'... (#{Time.now - start_time} secs)") + rescue => error + UI.error("#{job.language} - #{error}") + end + end + version_codes.each do |version_code| UI.user_error!("Could not find folder #{metadata_path}") unless File.directory?(metadata_path) @@ -89,17 +108,11 @@ def perform_upload_meta(version_codes, track_name) UI.user_error!("Could not find release for version code '#{version_code}' to update changelog") unless release release_notes = [] - all_languages.each do |language| - next if language.start_with?('.') # e.g. . or .. or hidden folders - UI.message("Preparing to upload for language '#{language}'...") - - listing = client.listing_for_language(language) - - upload_metadata(language, listing) unless Supply.config[:skip_upload_metadata] - upload_images(language) unless Supply.config[:skip_upload_images] - upload_screenshots(language) unless Supply.config[:skip_upload_screenshots] - release_notes << upload_changelog(language, version_code) unless Supply.config[:skip_upload_changelogs] - end + upload_worker.batch_enqueue( + # skip . or .. or hidden folders + all_languages.reject { |lang| lang.start_with?('.') }.map { |lang| UploadJob.new(lang, version_code, release_notes) } + ) + upload_worker.start upload_changelogs(release_notes, release, track, track_name) unless release_notes.empty? end From 5d4c89abd6455f8f8827983b244936c8038449e1 Mon Sep 17 00:00:00 2001 From: Nobody Date: Fri, 15 Sep 2023 14:51:09 +0200 Subject: [PATCH 2/8] change NUMBER_OF_THREADS logic; move release_notes; remove unnecessary blocks --- .../lib/fastlane_core/queue_worker.rb | 2 +- supply/lib/supply/uploader.rb | 28 +++++++++---------- 2 files changed, 14 insertions(+), 16 deletions(-) diff --git a/fastlane_core/lib/fastlane_core/queue_worker.rb b/fastlane_core/lib/fastlane_core/queue_worker.rb index 869301efce6..795384e3e2d 100644 --- a/fastlane_core/lib/fastlane_core/queue_worker.rb +++ b/fastlane_core/lib/fastlane_core/queue_worker.rb @@ -6,7 +6,7 @@ module FastlaneCore # Use this when you have all the items that you'll process in advance. # Simply enqueue them to this and call `QueueWorker#start`. class QueueWorker - NUMBER_OF_THREADS = FastlaneCore::Helper.test? ? 1 : [(ENV["DELIVER_NUMBER_OF_THREADS"] || ENV.fetch("FL_NUMBER_OF_THREADS", 10)).to_i, 1].max + NUMBER_OF_THREADS = FastlaneCore::Helper.test? ? 1 : [ENV["DELIVER_NUMBER_OF_THREADS"], ENV["FL_NUMBER_OF_THREADS"], 10].map(&:to_i).find(&:positive?).clamp(1, ENV.fetch("FL_MAX_NUMBER_OF_THREADS", 10).to_i) # @param concurrency (Numeric) - A number of threads to be created # @param block (Proc) - A task you want to execute with enqueued items diff --git a/supply/lib/supply/uploader.rb b/supply/lib/supply/uploader.rb index cfdbc76c08f..800f9832b0c 100644 --- a/supply/lib/supply/uploader.rb +++ b/supply/lib/supply/uploader.rb @@ -3,7 +3,7 @@ module Supply # rubocop:disable Metrics/ClassLength class Uploader - UploadJob = Struct.new(:language, :version_code, :release_notes) + UploadJob = Struct.new(:language, :version_code) def perform_upload FastlaneCore::PrintTable.print_values(config: Supply.config, hide_keys: [:issuer], mask_keys: [:json_key_data], title: "Summary for supply #{Fastlane::VERSION}") @@ -85,19 +85,18 @@ def perform_upload_meta(version_codes, track_name) version_codes.to_s == "" end + release_notes = [] upload_worker = FastlaneCore::QueueWorker.new do |job| - begin - UI.message("Preparing uploads for language '#{job.language}'...") - start_time = Time.now - listing = client.listing_for_language(job.language) - upload_metadata(job.language, listing) unless Supply.config[:skip_upload_metadata] - upload_images(job.language) unless Supply.config[:skip_upload_images] - upload_screenshots(job.language) unless Supply.config[:skip_upload_screenshots] - job.release_notes << upload_changelog(job.language, job.version_code) unless Supply.config[:skip_upload_changelogs] - UI.message("Uploaded all items for language '#{job.language}'... (#{Time.now - start_time} secs)") - rescue => error - UI.error("#{job.language} - #{error}") - end + UI.message("Preparing uploads for language '#{job.language}'...") + start_time = Time.now + listing = client.listing_for_language(job.language) + upload_metadata(job.language, listing) unless Supply.config[:skip_upload_metadata] + upload_images(job.language) unless Supply.config[:skip_upload_images] + upload_screenshots(job.language) unless Supply.config[:skip_upload_screenshots] + release_notes << upload_changelog(job.language, job.version_code) unless Supply.config[:skip_upload_changelogs] + UI.message("Uploaded all items for language '#{job.language}'... (#{Time.now - start_time} secs)") + rescue => error + UI.abort_with_message!("#{job.language} - #{error}") end version_codes.each do |version_code| @@ -107,10 +106,9 @@ def perform_upload_meta(version_codes, track_name) UI.user_error!("Unable to find the requested track - '#{Supply.config[:track]}'") unless track UI.user_error!("Could not find release for version code '#{version_code}' to update changelog") unless release - release_notes = [] upload_worker.batch_enqueue( # skip . or .. or hidden folders - all_languages.reject { |lang| lang.start_with?('.') }.map { |lang| UploadJob.new(lang, version_code, release_notes) } + all_languages.reject { |lang| lang.start_with?('.') }.map { |lang| UploadJob.new(lang, version_code) } ) upload_worker.start From 548b293ce0257f07ccfc280ab6d338ae2f681ce6 Mon Sep 17 00:00:00 2001 From: ccaruceru Date: Tue, 19 Sep 2023 20:09:41 +0200 Subject: [PATCH 3/8] create_meta_upload_worker; add tests --- supply/lib/supply/uploader.rb | 36 +++++---- .../fixtures/metadata/android/en-US/.gitkeep | 0 .../metadata/android/en-US/changelogs/1.txt | 1 + .../metadata/android/en-US/changelogs/2.txt | 1 + .../android/en-US/changelogs/default.txt | 1 + .../android/en-US/full_description.txt | 1 + .../android/en-US/short_description.txt | 1 + .../fixtures/metadata/android/en-US/title.txt | 1 + .../fixtures/metadata/android/en-US/video.txt | 1 + .../fixtures/metadata/android/fr-FR/.gitkeep | 0 .../metadata/android/fr-FR/changelogs/1.txt | 1 + .../metadata/android/fr-FR/changelogs/2.txt | 1 + .../android/fr-FR/changelogs/default.txt | 1 + .../android/fr-FR/full_description.txt | 1 + .../android/fr-FR/short_description.txt | 1 + .../fixtures/metadata/android/fr-FR/title.txt | 1 + .../fixtures/metadata/android/fr-FR/video.txt | 1 + .../fixtures/metadata/android/ja-JP/.gitkeep | 0 .../metadata/android/ja-JP/changelogs/1.txt | 1 + .../metadata/android/ja-JP/changelogs/2.txt | 1 + .../android/ja-JP/changelogs/default.txt | 1 + .../android/ja-JP/full_description.txt | 1 + .../android/ja-JP/short_description.txt | 1 + .../fixtures/metadata/android/ja-JP/title.txt | 1 + .../fixtures/metadata/android/ja-JP/video.txt | 1 + supply/spec/uploader_spec.rb | 73 +++++++++++++++---- 26 files changed, 101 insertions(+), 29 deletions(-) delete mode 100644 supply/spec/fixtures/metadata/android/en-US/.gitkeep create mode 100644 supply/spec/fixtures/metadata/android/en-US/changelogs/1.txt create mode 100644 supply/spec/fixtures/metadata/android/en-US/changelogs/2.txt create mode 100644 supply/spec/fixtures/metadata/android/en-US/changelogs/default.txt create mode 100644 supply/spec/fixtures/metadata/android/en-US/full_description.txt create mode 100644 supply/spec/fixtures/metadata/android/en-US/short_description.txt create mode 100644 supply/spec/fixtures/metadata/android/en-US/title.txt create mode 100644 supply/spec/fixtures/metadata/android/en-US/video.txt delete mode 100644 supply/spec/fixtures/metadata/android/fr-FR/.gitkeep create mode 100644 supply/spec/fixtures/metadata/android/fr-FR/changelogs/1.txt create mode 100644 supply/spec/fixtures/metadata/android/fr-FR/changelogs/2.txt create mode 100644 supply/spec/fixtures/metadata/android/fr-FR/changelogs/default.txt create mode 100644 supply/spec/fixtures/metadata/android/fr-FR/full_description.txt create mode 100644 supply/spec/fixtures/metadata/android/fr-FR/short_description.txt create mode 100644 supply/spec/fixtures/metadata/android/fr-FR/title.txt create mode 100644 supply/spec/fixtures/metadata/android/fr-FR/video.txt delete mode 100644 supply/spec/fixtures/metadata/android/ja-JP/.gitkeep create mode 100644 supply/spec/fixtures/metadata/android/ja-JP/changelogs/1.txt create mode 100644 supply/spec/fixtures/metadata/android/ja-JP/changelogs/2.txt create mode 100644 supply/spec/fixtures/metadata/android/ja-JP/changelogs/default.txt create mode 100644 supply/spec/fixtures/metadata/android/ja-JP/full_description.txt create mode 100644 supply/spec/fixtures/metadata/android/ja-JP/short_description.txt create mode 100644 supply/spec/fixtures/metadata/android/ja-JP/title.txt create mode 100644 supply/spec/fixtures/metadata/android/ja-JP/video.txt diff --git a/supply/lib/supply/uploader.rb b/supply/lib/supply/uploader.rb index 800f9832b0c..940b82a261d 100644 --- a/supply/lib/supply/uploader.rb +++ b/supply/lib/supply/uploader.rb @@ -3,7 +3,7 @@ module Supply # rubocop:disable Metrics/ClassLength class Uploader - UploadJob = Struct.new(:language, :version_code) + UploadJob = Struct.new(:language, :version_code, :release_notes) def perform_upload FastlaneCore::PrintTable.print_values(config: Supply.config, hide_keys: [:issuer], mask_keys: [:json_key_data], title: "Summary for supply #{Fastlane::VERSION}") @@ -85,20 +85,6 @@ def perform_upload_meta(version_codes, track_name) version_codes.to_s == "" end - release_notes = [] - upload_worker = FastlaneCore::QueueWorker.new do |job| - UI.message("Preparing uploads for language '#{job.language}'...") - start_time = Time.now - listing = client.listing_for_language(job.language) - upload_metadata(job.language, listing) unless Supply.config[:skip_upload_metadata] - upload_images(job.language) unless Supply.config[:skip_upload_images] - upload_screenshots(job.language) unless Supply.config[:skip_upload_screenshots] - release_notes << upload_changelog(job.language, job.version_code) unless Supply.config[:skip_upload_changelogs] - UI.message("Uploaded all items for language '#{job.language}'... (#{Time.now - start_time} secs)") - rescue => error - UI.abort_with_message!("#{job.language} - #{error}") - end - version_codes.each do |version_code| UI.user_error!("Could not find folder #{metadata_path}") unless File.directory?(metadata_path) @@ -106,12 +92,15 @@ def perform_upload_meta(version_codes, track_name) UI.user_error!("Unable to find the requested track - '#{Supply.config[:track]}'") unless track UI.user_error!("Could not find release for version code '#{version_code}' to update changelog") unless release + release_notes = Queue.new + upload_worker = create_meta_upload_worker upload_worker.batch_enqueue( # skip . or .. or hidden folders - all_languages.reject { |lang| lang.start_with?('.') }.map { |lang| UploadJob.new(lang, version_code) } + all_languages.reject { |lang| lang.start_with?('.') }.map { |lang| UploadJob.new(lang, version_code, release_notes) } ) upload_worker.start + release_notes = Array.new(release_notes.size) { release_notes.pop } # Queue to Array upload_changelogs(release_notes, release, track, track_name) unless release_notes.empty? end end @@ -495,6 +484,21 @@ def obb_expansion_file_type(obb_file_path) 'patch' end end + + def create_meta_upload_worker + FastlaneCore::QueueWorker.new do |job| + UI.message("Preparing uploads for language '#{job.language}'...") + start_time = Time.now + listing = client.listing_for_language(job.language) + upload_metadata(job.language, listing) unless Supply.config[:skip_upload_metadata] + upload_images(job.language) unless Supply.config[:skip_upload_images] + upload_screenshots(job.language) unless Supply.config[:skip_upload_screenshots] + job.release_notes << upload_changelog(job.language, job.version_code) unless Supply.config[:skip_upload_changelogs] + UI.message("Uploaded all items for language '#{job.language}'... (#{Time.now - start_time} secs)") + rescue => error + UI.abort_with_message!("#{job.language} - #{error}") + end + end end # rubocop:enable Metrics/ClassLength end diff --git a/supply/spec/fixtures/metadata/android/en-US/.gitkeep b/supply/spec/fixtures/metadata/android/en-US/.gitkeep deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/supply/spec/fixtures/metadata/android/en-US/changelogs/1.txt b/supply/spec/fixtures/metadata/android/en-US/changelogs/1.txt new file mode 100644 index 00000000000..d7f424e4d34 --- /dev/null +++ b/supply/spec/fixtures/metadata/android/en-US/changelogs/1.txt @@ -0,0 +1 @@ +en-US changelog 1 \ No newline at end of file diff --git a/supply/spec/fixtures/metadata/android/en-US/changelogs/2.txt b/supply/spec/fixtures/metadata/android/en-US/changelogs/2.txt new file mode 100644 index 00000000000..7d580f59ec0 --- /dev/null +++ b/supply/spec/fixtures/metadata/android/en-US/changelogs/2.txt @@ -0,0 +1 @@ +en-US changelog 2 \ No newline at end of file diff --git a/supply/spec/fixtures/metadata/android/en-US/changelogs/default.txt b/supply/spec/fixtures/metadata/android/en-US/changelogs/default.txt new file mode 100644 index 00000000000..cdcc144e99a --- /dev/null +++ b/supply/spec/fixtures/metadata/android/en-US/changelogs/default.txt @@ -0,0 +1 @@ +en-US changelog -1 \ No newline at end of file diff --git a/supply/spec/fixtures/metadata/android/en-US/full_description.txt b/supply/spec/fixtures/metadata/android/en-US/full_description.txt new file mode 100644 index 00000000000..973a04dafbf --- /dev/null +++ b/supply/spec/fixtures/metadata/android/en-US/full_description.txt @@ -0,0 +1 @@ +en-US full description \ No newline at end of file diff --git a/supply/spec/fixtures/metadata/android/en-US/short_description.txt b/supply/spec/fixtures/metadata/android/en-US/short_description.txt new file mode 100644 index 00000000000..1dfa67f040a --- /dev/null +++ b/supply/spec/fixtures/metadata/android/en-US/short_description.txt @@ -0,0 +1 @@ +en-US short description \ No newline at end of file diff --git a/supply/spec/fixtures/metadata/android/en-US/title.txt b/supply/spec/fixtures/metadata/android/en-US/title.txt new file mode 100644 index 00000000000..3036e5464f0 --- /dev/null +++ b/supply/spec/fixtures/metadata/android/en-US/title.txt @@ -0,0 +1 @@ +en-US title \ No newline at end of file diff --git a/supply/spec/fixtures/metadata/android/en-US/video.txt b/supply/spec/fixtures/metadata/android/en-US/video.txt new file mode 100644 index 00000000000..a8f038b1447 --- /dev/null +++ b/supply/spec/fixtures/metadata/android/en-US/video.txt @@ -0,0 +1 @@ +en-US video \ No newline at end of file diff --git a/supply/spec/fixtures/metadata/android/fr-FR/.gitkeep b/supply/spec/fixtures/metadata/android/fr-FR/.gitkeep deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/supply/spec/fixtures/metadata/android/fr-FR/changelogs/1.txt b/supply/spec/fixtures/metadata/android/fr-FR/changelogs/1.txt new file mode 100644 index 00000000000..9c3fa90dcbb --- /dev/null +++ b/supply/spec/fixtures/metadata/android/fr-FR/changelogs/1.txt @@ -0,0 +1 @@ +fr-FR changelog 1 \ No newline at end of file diff --git a/supply/spec/fixtures/metadata/android/fr-FR/changelogs/2.txt b/supply/spec/fixtures/metadata/android/fr-FR/changelogs/2.txt new file mode 100644 index 00000000000..9a324144c13 --- /dev/null +++ b/supply/spec/fixtures/metadata/android/fr-FR/changelogs/2.txt @@ -0,0 +1 @@ +fr-FR changelog 2 \ No newline at end of file diff --git a/supply/spec/fixtures/metadata/android/fr-FR/changelogs/default.txt b/supply/spec/fixtures/metadata/android/fr-FR/changelogs/default.txt new file mode 100644 index 00000000000..7772c685b1b --- /dev/null +++ b/supply/spec/fixtures/metadata/android/fr-FR/changelogs/default.txt @@ -0,0 +1 @@ +fr-FR changelog -1 \ No newline at end of file diff --git a/supply/spec/fixtures/metadata/android/fr-FR/full_description.txt b/supply/spec/fixtures/metadata/android/fr-FR/full_description.txt new file mode 100644 index 00000000000..44ae5e97ffe --- /dev/null +++ b/supply/spec/fixtures/metadata/android/fr-FR/full_description.txt @@ -0,0 +1 @@ +fr-FR full description \ No newline at end of file diff --git a/supply/spec/fixtures/metadata/android/fr-FR/short_description.txt b/supply/spec/fixtures/metadata/android/fr-FR/short_description.txt new file mode 100644 index 00000000000..927ce14ad3a --- /dev/null +++ b/supply/spec/fixtures/metadata/android/fr-FR/short_description.txt @@ -0,0 +1 @@ +fr-FR short description \ No newline at end of file diff --git a/supply/spec/fixtures/metadata/android/fr-FR/title.txt b/supply/spec/fixtures/metadata/android/fr-FR/title.txt new file mode 100644 index 00000000000..904a6905c00 --- /dev/null +++ b/supply/spec/fixtures/metadata/android/fr-FR/title.txt @@ -0,0 +1 @@ +fr-FR title \ No newline at end of file diff --git a/supply/spec/fixtures/metadata/android/fr-FR/video.txt b/supply/spec/fixtures/metadata/android/fr-FR/video.txt new file mode 100644 index 00000000000..34c823195f5 --- /dev/null +++ b/supply/spec/fixtures/metadata/android/fr-FR/video.txt @@ -0,0 +1 @@ +fr-FR video \ No newline at end of file diff --git a/supply/spec/fixtures/metadata/android/ja-JP/.gitkeep b/supply/spec/fixtures/metadata/android/ja-JP/.gitkeep deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/supply/spec/fixtures/metadata/android/ja-JP/changelogs/1.txt b/supply/spec/fixtures/metadata/android/ja-JP/changelogs/1.txt new file mode 100644 index 00000000000..0204d556d53 --- /dev/null +++ b/supply/spec/fixtures/metadata/android/ja-JP/changelogs/1.txt @@ -0,0 +1 @@ +ja-JP changelog 1 \ No newline at end of file diff --git a/supply/spec/fixtures/metadata/android/ja-JP/changelogs/2.txt b/supply/spec/fixtures/metadata/android/ja-JP/changelogs/2.txt new file mode 100644 index 00000000000..388a1939c25 --- /dev/null +++ b/supply/spec/fixtures/metadata/android/ja-JP/changelogs/2.txt @@ -0,0 +1 @@ +ja-JP changelog 2 \ No newline at end of file diff --git a/supply/spec/fixtures/metadata/android/ja-JP/changelogs/default.txt b/supply/spec/fixtures/metadata/android/ja-JP/changelogs/default.txt new file mode 100644 index 00000000000..7c30573ea3e --- /dev/null +++ b/supply/spec/fixtures/metadata/android/ja-JP/changelogs/default.txt @@ -0,0 +1 @@ +ja-JP changelog -1 \ No newline at end of file diff --git a/supply/spec/fixtures/metadata/android/ja-JP/full_description.txt b/supply/spec/fixtures/metadata/android/ja-JP/full_description.txt new file mode 100644 index 00000000000..d36af775953 --- /dev/null +++ b/supply/spec/fixtures/metadata/android/ja-JP/full_description.txt @@ -0,0 +1 @@ +ja-JP full description \ No newline at end of file diff --git a/supply/spec/fixtures/metadata/android/ja-JP/short_description.txt b/supply/spec/fixtures/metadata/android/ja-JP/short_description.txt new file mode 100644 index 00000000000..22259367b25 --- /dev/null +++ b/supply/spec/fixtures/metadata/android/ja-JP/short_description.txt @@ -0,0 +1 @@ +ja-JP short description \ No newline at end of file diff --git a/supply/spec/fixtures/metadata/android/ja-JP/title.txt b/supply/spec/fixtures/metadata/android/ja-JP/title.txt new file mode 100644 index 00000000000..5fb03495028 --- /dev/null +++ b/supply/spec/fixtures/metadata/android/ja-JP/title.txt @@ -0,0 +1 @@ +ja-JP title \ No newline at end of file diff --git a/supply/spec/fixtures/metadata/android/ja-JP/video.txt b/supply/spec/fixtures/metadata/android/ja-JP/video.txt new file mode 100644 index 00000000000..f61eab02c20 --- /dev/null +++ b/supply/spec/fixtures/metadata/android/ja-JP/video.txt @@ -0,0 +1 @@ +ja-JP video \ No newline at end of file diff --git a/supply/spec/uploader_spec.rb b/supply/spec/uploader_spec.rb index 57b8d1e0f18..3ae0355f973 100644 --- a/supply/spec/uploader_spec.rb +++ b/supply/spec/uploader_spec.rb @@ -203,22 +203,69 @@ def find_obbs end end - describe '#perform_upload' do - let(:client) { double('client') } - let(:config) { { apk: 'some/path/app.apk' } } + RSpec::Matchers.define(:same_localizedtext_as) do |expected_arr| + match do |actual_arr| + actual_arr.sort_by(&:language).zip(expected_arr.sort_by(&:language)).map do |actual_localization, expected_localization| + actual_localization.language == expected_localization.language && actual_localization.text == expected_localization.text + end.reduce(:&) + end + end - before do - Supply.config = config - allow(Supply::Client).to receive(:make_from_config).and_return(client) - allow(client).to receive(:upload_apk).with(config[:apk]).and_return(1) # newly uploaded version code - allow(client).to receive(:begin_edit).and_return(nil) - allow(client).to receive(:commit_current_edit!).and_return(nil) + shared_examples 'run supply to upload metadata' do + describe '#perform_upload' do + subject(:release) { OpenStruct.new({ version_codes: version_codes }) } + let(:client) { double('client') } + let(:config) { { apk_paths: version_codes.map { |v_code| "some/path/app-v#{v_code}.apk" }, metadata_path: 'supply/spec/fixtures/metadata/android', track: 'track-name' } } + + before do + Supply.config = config + allow(Supply::Client).to receive(:make_from_config).and_return(client) + version_codes.each do |version_code| + allow(client).to receive(:upload_apk).with("some/path/app-v#{version_code}.apk").and_return(version_code) # newly uploaded version code + end + allow(client).to receive(:upload_changelogs).and_return(nil) + allow(client).to receive(:tracks).with('track-name').and_return([OpenStruct.new({ releases: [ release ] })]) + languages.each do |lang| + allow(client).to receive(:listing_for_language).with(lang).and_return(Supply::Listing.new(client, lang)) + allow(client).to receive(:update_listing_for_language).with({ language: lang, full_description: "#{lang} full description", short_description: "#{lang} short description", title: "#{lang} title", video: "#{lang} video" }) + end + allow(client).to receive(:begin_edit).and_return(nil) + allow(client).to receive(:commit_current_edit!).and_return(nil) + end + + it 'should update track with correct version codes and optional changelog' do + uploader = Supply::Uploader.new + expect(uploader).to receive(:update_track).with(version_codes).once + version_codes.each do |version_code| + expected_notes = [] + languages.each do |lang| + expected_notes << AndroidPublisher::LocalizedText.new( + language: lang, + text: "#{lang} changelog #{with_explicit_changelogs ? version_code : -1}" + ) + end + # check if at least one of the assignments of release_notes is what we expect + expect(release).to receive(:release_notes=).with(same_localizedtext_as(expected_notes)) + end + + uploader.perform_upload + end + end + end + + describe '#peform_upload with metadata and explicit changelogs' do + it_behaves_like 'run supply to upload metadata' do + let(:version_codes) { [1, 2] } + let(:languages) { ['en-US', 'fr-FR', 'ja-JP'] } + let(:with_explicit_changelogs) { true } end + end - it 'should update track with correct version codes' do - uploader = Supply::Uploader.new - expect(uploader).to receive(:update_track).with([1]).once - uploader.perform_upload + describe '#peform_upload with metadata and default changelogs' do + it_behaves_like 'run supply to upload metadata' do + let(:version_codes) { [3] } + let(:languages) { ['en-US', 'fr-FR', 'ja-JP'] } + let(:with_explicit_changelogs) { false } end end end From f5b6c1bc5fe95b4494fd8486bb9c163b4f8e5983 Mon Sep 17 00:00:00 2001 From: ccaruceru Date: Wed, 20 Sep 2023 10:16:32 +0200 Subject: [PATCH 4/8] update docs --- .../lib/fastlane/actions/docs/upload_to_play_store.md | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/fastlane/lib/fastlane/actions/docs/upload_to_play_store.md b/fastlane/lib/fastlane/actions/docs/upload_to_play_store.md index c7e57ec0162..55b41c57749 100644 --- a/fastlane/lib/fastlane/actions/docs/upload_to_play_store.md +++ b/fastlane/lib/fastlane/actions/docs/upload_to_play_store.md @@ -179,6 +179,12 @@ This can be done using the `--track_promote_to` parameter. The `--track_promote_ Before performing a new APK upload you may want to check existing track version codes or release names, or you may simply want to provide an informational lane that displays the currently promoted version codes or release name for the production track. You can use the `google_play_track_version_codes` action to retrieve existing version codes for a package and track. You can use the `google_play_track_release_names` action to retrieve existing release names for a package and track. For more information, see the `fastlane action google_play_track_version_codes` and `fastlane action google_play_track_release_names` help output. +## Parallel uploads + +By default _supply_ will spawn 10 threads to upload the metadata concurrently (_images, screenshots, texts_). If you want to change this, set either `DELIVER_NUMBER_OF_THREADS` or `FL_NUMBER_OF_THREADS` environment variable to any value between 1 and 10. + +If you want _supply_ to upload with more than 10 threads in parallel then you need to **additionally** set `FL_MAX_NUMBER_OF_THREADS` environment variable to the max number of parallel upload threads you wish to have (**Warning ⚠️** use this at your own risk!). + ## Migration from AndroidPublisherV2 to AndroidPublisherV3 in _fastlane_ 2.135.0 ### New Options @@ -200,5 +206,3 @@ For more information, see the `fastlane action google_play_track_version_codes` - Google Play will automatically remove releases that are superseded now - `:deactivate_on_promote` - Google Play will automatically deactivate a release from its previous track on promote - -: From e5a4030e5d2a18853dcc5e4bc6a9e7f0c231e428 Mon Sep 17 00:00:00 2001 From: Nobody Date: Fri, 13 Oct 2023 20:18:01 +0200 Subject: [PATCH 5/8] rename release_notes_queue; update tests --- supply/lib/supply/uploader.rb | 10 ++-- supply/spec/uploader_spec.rb | 86 ++++++++++++++++------------------- 2 files changed, 43 insertions(+), 53 deletions(-) diff --git a/supply/lib/supply/uploader.rb b/supply/lib/supply/uploader.rb index 322f36fe02b..622a73ac15e 100644 --- a/supply/lib/supply/uploader.rb +++ b/supply/lib/supply/uploader.rb @@ -3,7 +3,7 @@ module Supply # rubocop:disable Metrics/ClassLength class Uploader - UploadJob = Struct.new(:language, :version_code, :release_notes) + UploadJob = Struct.new(:language, :version_code, :release_notes_queue) def perform_upload FastlaneCore::PrintTable.print_values(config: Supply.config, hide_keys: [:issuer], mask_keys: [:json_key_data], title: "Summary for supply #{Fastlane::VERSION}") @@ -92,15 +92,15 @@ def perform_upload_meta(version_codes, track_name) UI.user_error!("Unable to find the requested track - '#{Supply.config[:track]}'") unless track UI.user_error!("Could not find release for version code '#{version_code}' to update changelog") unless release - release_notes = Queue.new + release_notes_queue = Queue.new upload_worker = create_meta_upload_worker upload_worker.batch_enqueue( # skip . or .. or hidden folders - all_languages.reject { |lang| lang.start_with?('.') }.map { |lang| UploadJob.new(lang, version_code, release_notes) } + all_languages.reject { |lang| lang.start_with?('.') }.map { |lang| UploadJob.new(lang, version_code, release_notes_queue) } ) upload_worker.start - release_notes = Array.new(release_notes.size) { release_notes.pop } # Queue to Array + release_notes = Array.new(release_notes_queue.size) { release_notes_queue.pop } # Queue to Array upload_changelogs(release_notes, release, track, track_name) unless release_notes.empty? end end @@ -520,7 +520,7 @@ def create_meta_upload_worker upload_metadata(job.language, listing) unless Supply.config[:skip_upload_metadata] upload_images(job.language) unless Supply.config[:skip_upload_images] upload_screenshots(job.language) unless Supply.config[:skip_upload_screenshots] - job.release_notes << upload_changelog(job.language, job.version_code) unless Supply.config[:skip_upload_changelogs] + job.release_notes_queue << upload_changelog(job.language, job.version_code) unless Supply.config[:skip_upload_changelogs] UI.message("Uploaded all items for language '#{job.language}'... (#{Time.now - start_time} secs)") rescue => error UI.abort_with_message!("#{job.language} - #{error}") diff --git a/supply/spec/uploader_spec.rb b/supply/spec/uploader_spec.rb index 4e18d793319..8a14e050dc0 100644 --- a/supply/spec/uploader_spec.rb +++ b/supply/spec/uploader_spec.rb @@ -211,62 +211,52 @@ def find_obbs end end - shared_examples 'run supply to upload metadata' do - describe '#perform_upload' do - subject(:release) { OpenStruct.new({ version_codes: version_codes }) } - let(:client) { double('client') } - let(:config) { { apk_paths: version_codes.map { |v_code| "some/path/app-v#{v_code}.apk" }, metadata_path: 'supply/spec/fixtures/metadata/android', track: 'track-name' } } - - before do - Supply.config = config - allow(Supply::Client).to receive(:make_from_config).and_return(client) - version_codes.each do |version_code| - allow(client).to receive(:upload_apk).with("some/path/app-v#{version_code}.apk").and_return(version_code) # newly uploaded version code - end - allow(client).to receive(:upload_changelogs).and_return(nil) - allow(client).to receive(:tracks).with('track-name').and_return([OpenStruct.new({ releases: [ release ] })]) - languages.each do |lang| - allow(client).to receive(:listing_for_language).with(lang).and_return(Supply::Listing.new(client, lang)) - allow(client).to receive(:update_listing_for_language).with({ language: lang, full_description: "#{lang} full description", short_description: "#{lang} short description", title: "#{lang} title", video: "#{lang} video" }) - end - allow(client).to receive(:begin_edit).and_return(nil) - allow(client).to receive(:commit_current_edit!).and_return(nil) + shared_examples 'run supply to upload metadata' do |version_codes:, with_explicit_changelogs:| + let(:languages) { ['en-US', 'fr-FR', 'ja-JP'] } + subject(:release) { double('release', version_codes: version_codes) } + let(:client) { double('client') } + let(:config) { { apk_paths: version_codes.map { |v_code| "some/path/app-v#{v_code}.apk" }, metadata_path: 'supply/spec/fixtures/metadata/android', track: 'track-name' } } + + before do + Supply.config = config + allow(Supply::Client).to receive(:make_from_config).and_return(client) + version_codes.each do |version_code| + allow(client).to receive(:upload_apk).with("some/path/app-v#{version_code}.apk").and_return(version_code) # newly uploaded version code + end + allow(client).to receive(:upload_changelogs).and_return(nil) + allow(client).to receive(:tracks).with('track-name').and_return([double('tracks', releases: [ release ] )]) + languages.each do |lang| + allow(client).to receive(:listing_for_language).with(lang).and_return(Supply::Listing.new(client, lang)) end + allow(client).to receive(:begin_edit).and_return(nil) + allow(client).to receive(:commit_current_edit!).and_return(nil) + end - it 'should update track with correct version codes and optional changelog' do - uploader = Supply::Uploader.new - expect(uploader).to receive(:update_track).with(version_codes).once - version_codes.each do |version_code| - expected_notes = [] - languages.each do |lang| - expected_notes << AndroidPublisher::LocalizedText.new( - language: lang, - text: "#{lang} changelog #{with_explicit_changelogs ? version_code : -1}" - ) - end - # check if at least one of the assignments of release_notes is what we expect - expect(release).to receive(:release_notes=).with(same_localizedtext_as(expected_notes)) + it 'should update track with correct version codes and optional changelog' do + uploader = Supply::Uploader.new + expect(uploader).to receive(:update_track).with(version_codes).once + version_codes.each do |version_code| + expected_notes = languages.map do |lang| + AndroidPublisher::LocalizedText.new( + language: lang, + text: "#{lang} changelog #{with_explicit_changelogs ? version_code : -1}" + ) + end + # check if at least one of the assignments of release_notes is what we expect + expect(release).to receive(:release_notes=).with(same_localizedtext_as(expected_notes)) + # check if the listings are updated for each language with text data from disk + languages.each do |lang| + expect(client).to receive(:update_listing_for_language).with({ language: lang, full_description: "#{lang} full description", short_description: "#{lang} short description", title: "#{lang} title", video: "#{lang} video" }) end - - uploader.perform_upload end - end - end - describe '#peform_upload with metadata and explicit changelogs' do - it_behaves_like 'run supply to upload metadata' do - let(:version_codes) { [1, 2] } - let(:languages) { ['en-US', 'fr-FR', 'ja-JP'] } - let(:with_explicit_changelogs) { true } + uploader.perform_upload end end - describe '#peform_upload with metadata and default changelogs' do - it_behaves_like 'run supply to upload metadata' do - let(:version_codes) { [3] } - let(:languages) { ['en-US', 'fr-FR', 'ja-JP'] } - let(:with_explicit_changelogs) { false } - end + describe '#peform_upload with metadata' do + it_behaves_like 'run supply to upload metadata', version_codes: [1, 2], with_explicit_changelogs: true + it_behaves_like 'run supply to upload metadata', version_codes: [3], with_explicit_changelogs: false end context 'when sync_image_upload is set' do From ed3b77dde2f265b31c331f52469af906147dfe8a Mon Sep 17 00:00:00 2001 From: Nobody Date: Fri, 13 Oct 2023 20:30:48 +0200 Subject: [PATCH 6/8] fix rubocop --- supply/spec/uploader_spec.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/supply/spec/uploader_spec.rb b/supply/spec/uploader_spec.rb index 8a14e050dc0..db6d4339838 100644 --- a/supply/spec/uploader_spec.rb +++ b/supply/spec/uploader_spec.rb @@ -224,7 +224,7 @@ def find_obbs allow(client).to receive(:upload_apk).with("some/path/app-v#{version_code}.apk").and_return(version_code) # newly uploaded version code end allow(client).to receive(:upload_changelogs).and_return(nil) - allow(client).to receive(:tracks).with('track-name').and_return([double('tracks', releases: [ release ] )]) + allow(client).to receive(:tracks).with('track-name').and_return([double('tracks', releases: [ release ])]) languages.each do |lang| allow(client).to receive(:listing_for_language).with(lang).and_return(Supply::Listing.new(client, lang)) end From 31e9866a053e7977e4639b7e329819ff0b1097a7 Mon Sep 17 00:00:00 2001 From: Nobody Date: Sun, 15 Oct 2023 11:39:52 +0200 Subject: [PATCH 7/8] open class and remove matcher --- supply/spec/uploader_spec.rb | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/supply/spec/uploader_spec.rb b/supply/spec/uploader_spec.rb index db6d4339838..27f297aadb6 100644 --- a/supply/spec/uploader_spec.rb +++ b/supply/spec/uploader_spec.rb @@ -203,11 +203,10 @@ def find_obbs end end - RSpec::Matchers.define(:same_localizedtext_as) do |expected_arr| - match do |actual_arr| - actual_arr.sort_by(&:language).zip(expected_arr.sort_by(&:language)).map do |actual_localization, expected_localization| - actual_localization.language == expected_localization.language && actual_localization.text == expected_localization.text - end.reduce(:&) + # add basic == functionality to LocalizedText class for testing purpose + class AndroidPublisher::LocalizedText + def ==(other) + self.language == other.language && self.text == other.text end end @@ -243,7 +242,7 @@ def find_obbs ) end # check if at least one of the assignments of release_notes is what we expect - expect(release).to receive(:release_notes=).with(same_localizedtext_as(expected_notes)) + expect(release).to receive(:release_notes=).with(match_array(expected_notes)) # check if the listings are updated for each language with text data from disk languages.each do |lang| expect(client).to receive(:update_listing_for_language).with({ language: lang, full_description: "#{lang} full description", short_description: "#{lang} short description", title: "#{lang} title", video: "#{lang} video" }) From fd7568c219763d061aafc594df2f42e62e085eb5 Mon Sep 17 00:00:00 2001 From: Cristian Caruceru <90720255+ccaruceru@users.noreply.github.com> Date: Mon, 16 Oct 2023 12:10:26 +0300 Subject: [PATCH 8/8] uniq for expected notes Co-authored-by: Olivier Halligon --- supply/spec/uploader_spec.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/supply/spec/uploader_spec.rb b/supply/spec/uploader_spec.rb index 27f297aadb6..82ab0d19506 100644 --- a/supply/spec/uploader_spec.rb +++ b/supply/spec/uploader_spec.rb @@ -240,7 +240,7 @@ def ==(other) language: lang, text: "#{lang} changelog #{with_explicit_changelogs ? version_code : -1}" ) - end + end.uniq # check if at least one of the assignments of release_notes is what we expect expect(release).to receive(:release_notes=).with(match_array(expected_notes)) # check if the listings are updated for each language with text data from disk