BigQuery: Allow loading data from a list of URLs

google-cloud-bigquery/acceptance/bigquery/dataset_reference_test.rb

@@ -159,6 +159,51 @@
     result.must_equal true
   end
 
+  it "imports data from a list of files in your bucket with load_job" do
+    begin
+      more_data = rows.map { |row| JSON.generate row }.join("\n")
+      bucket = Google::Cloud.storage.create_bucket "#{prefix}_bucket"
+      file1 = bucket.create_file local_file
+      file2 = bucket.create_file StringIO.new(more_data),
+                                 "more-kitten-test-data.json"
+      gs_url = "gs://#{file2.bucket}/#{file2.name}"
+
+      # Test both by file object and URL as string
+      job = dataset.load_job table_id, [file1, gs_url]
+      job.wait_until_done!
+      job.wont_be :failed?
+      job.input_files.must_equal 2
+      job.output_rows.must_equal 6
+    ensure
+      post_bucket = Google::Cloud.storage.bucket "#{prefix}_bucket"
+      if post_bucket
+        post_bucket.files.map &:delete
+        post_bucket.delete
+      end
+    end
+  end
+
+  it "imports data from a list of files in your bucket with load" do
+    begin
+      more_data = rows.map { |row| JSON.generate row }.join("\n")
+      bucket = Google::Cloud.storage.create_bucket "#{prefix}_bucket"
+      file1 = bucket.create_file local_file
+      file2 = bucket.create_file StringIO.new(more_data),
+                                 "more-kitten-test-data.json"
+      gs_url = "gs://#{file2.bucket}/#{file2.name}"
+
+      # Test both by file object and URL as string
+      result = dataset.load table_id, [file1, gs_url]
+      result.must_equal true
+    ensure
+      post_bucket = Google::Cloud.storage.bucket "#{prefix}_bucket"
+      if post_bucket
+        post_bucket.files.map &:delete
+        post_bucket.delete
+      end
+    end
+  end
+
   it "adds an access entry with specifying user scope" do
     dataset.access do |acl|
       acl.add_reader_user user_val

google-cloud-bigquery/acceptance/bigquery/dataset_test.rb

@@ -192,6 +192,30 @@
     job.output_rows.must_equal 3
   end
 
+  it "imports data from a list of files in your bucket with load_job" do
+    begin
+      more_data = rows.map { |row| JSON.generate row }.join("\n")
+      bucket = Google::Cloud.storage.create_bucket "#{prefix}_bucket"
+      file1 = bucket.create_file local_file
+      file2 = bucket.create_file StringIO.new(more_data),
+                                 "more-kitten-test-data.json"
+      gs_url = "gs://#{file2.bucket}/#{file2.name}"
+
+      # Test both by file object and URL as string
+      job = dataset.load_job table_with_schema.table_id, [file1, gs_url]
+      job.wait_until_done!
+      job.wont_be :failed?
+      job.input_files.must_equal 2
+      job.output_rows.must_equal 6
+    ensure
+      post_bucket = Google::Cloud.storage.bucket "#{prefix}_bucket"
+      if post_bucket
+        post_bucket.files.map &:delete
+        post_bucket.delete
+      end
+    end
+  end
+
   it "imports data from a local file and creates a new table with specified schema in a block with load" do
     result = dataset.load "local_file_table", local_file do |schema|
       schema.integer   "id",    description: "id description",    mode: :required
@@ -219,6 +243,27 @@
     result.must_equal true
   end
 
+  it "imports data from a list of files in your bucket with load" do
+    begin
+      more_data = rows.map { |row| JSON.generate row }.join("\n")
+      bucket = Google::Cloud.storage.create_bucket "#{prefix}_bucket"
+      file1 = bucket.create_file local_file
+      file2 = bucket.create_file StringIO.new(more_data),
+                                 "more-kitten-test-data.json"
+      gs_url = "gs://#{file2.bucket}/#{file2.name}"
+
+      # Test both by file object and URL as string
+      result = dataset.load table_with_schema.table_id, [file1, gs_url]
+      result.must_equal true
+    ensure
+      post_bucket = Google::Cloud.storage.bucket "#{prefix}_bucket"
+      if post_bucket
+        post_bucket.files.map &:delete
+        post_bucket.delete
+      end
+    end
+  end
+
   it "inserts rows directly and gets its data" do
     insert_response = dataset.insert table_with_schema.table_id, rows
     insert_response.must_be :success?

google-cloud-bigquery/acceptance/bigquery/table_reference_test.rb

@@ -245,11 +245,89 @@
     job.output_rows.must_equal 3
   end
 
+  it "imports data from a file in your bucket with load_job" do
+    begin
+      bucket = Google::Cloud.storage.create_bucket "#{prefix}_bucket"
+      file = bucket.create_file local_file
+
+      job = table.load_job file
+      job.wait_until_done!
+      job.wont_be :failed?
+    ensure
+      post_bucket = Google::Cloud.storage.bucket "#{prefix}_bucket"
+      if post_bucket
+        post_bucket.files.map &:delete
+        post_bucket.delete
+      end
+    end
+  end
+
+  it "imports data from a list of files in your bucket with load_job" do
+    begin
+      more_data = rows.map { |row| JSON.generate row }.join("\n")
+      bucket = Google::Cloud.storage.create_bucket "#{prefix}_bucket"
+      file1 = bucket.create_file local_file
+      file2 = bucket.create_file StringIO.new(more_data),
+                                 "more-kitten-test-data.json"
+      gs_url = "gs://#{file2.bucket}/#{file2.name}"
+
+      # Test both by file object and URL as string
+      job = table.load_job [file1, gs_url]
+      job.wait_until_done!
+      job.wont_be :failed?
+      job.input_files.must_equal 2
+      job.output_rows.must_equal 6
+    ensure
+      post_bucket = Google::Cloud.storage.bucket "#{prefix}_bucket"
+      if post_bucket
+        post_bucket.files.map &:delete
+        post_bucket.delete
+      end
+    end
+  end
+
   it "imports data from a local file with load" do
     result = table.load local_file
     result.must_equal true
   end
 
+  it "imports data from a file in your bucket with load" do
+    begin
+      bucket = Google::Cloud.storage.create_bucket "#{prefix}_bucket"
+      file = bucket.create_file local_file
+
+      result = table.load file
+      result.must_equal true
+    ensure
+      post_bucket = Google::Cloud.storage.bucket "#{prefix}_bucket"
+      if post_bucket
+        post_bucket.files.map &:delete
+        post_bucket.delete
+      end
+    end
+  end
+
+  it "imports data from a list of files in your bucket with load" do
+    begin
+      more_data = rows.map { |row| JSON.generate row }.join("\n")
+      bucket = Google::Cloud.storage.create_bucket "#{prefix}_bucket"
+      file1 = bucket.create_file local_file
+      file2 = bucket.create_file StringIO.new(more_data),
+                                 "more-kitten-test-data.json"
+      gs_url = "gs://#{file2.bucket}/#{file2.name}"
+
+      # Test both by file object and URL as string
+      result = table.load [file1, gs_url]
+      result.must_equal true
+    ensure
+      post_bucket = Google::Cloud.storage.bucket "#{prefix}_bucket"
+      if post_bucket
+        post_bucket.files.map &:delete
+        post_bucket.delete
+      end
+    end
+  end
+
   it "copies itself to another table with copy_job" do
     job_id = "test_job_#{SecureRandom.urlsafe_base64(21)}" # client-generated
     copy_job = table.copy_job target_table_id, create: :needed, write: :empty, job_id: job_id, labels: labels

google-cloud-bigquery/acceptance/bigquery/table_test.rb

@@ -476,6 +476,30 @@
     end
   end
 
+  it "imports data from a list of files in your bucket with load_job" do
+    begin
+      more_data = rows.map { |row| JSON.generate row }.join("\n")
+      bucket = Google::Cloud.storage.create_bucket "#{prefix}_bucket"
+      file1 = bucket.create_file local_file
+      file2 = bucket.create_file StringIO.new(more_data),
+                                 "more-kitten-test-data.json"
+      gs_url = "gs://#{file2.bucket}/#{file2.name}"
+
+      # Test both by file object and URL as string
+      job = table.load_job [file1, gs_url]
+      job.wait_until_done!
+      job.wont_be :failed?
+      job.input_files.must_equal 2
+      job.output_rows.must_equal 6
+    ensure
+      post_bucket = Google::Cloud.storage.bucket "#{prefix}_bucket"
+      if post_bucket
+        post_bucket.files.map &:delete
+        post_bucket.delete
+      end
+    end
+  end
+
   it "imports data from a local file with load" do
     result = table.load local_file
     result.must_equal true
@@ -497,6 +521,27 @@
     end
   end
 
+  it "imports data from a list of files in your bucket with load" do
+    begin
+      more_data = rows.map { |row| JSON.generate row }.join("\n")
+      bucket = Google::Cloud.storage.create_bucket "#{prefix}_bucket"
+      file1 = bucket.create_file local_file
+      file2 = bucket.create_file StringIO.new(more_data),
+                                 "more-kitten-test-data.json"
+      gs_url = "gs://#{file2.bucket}/#{file2.name}"
+
+      # Test both by file object and URL as string
+      result = table.load [file1, gs_url]
+      result.must_equal true
+    ensure
+      post_bucket = Google::Cloud.storage.bucket "#{prefix}_bucket"
+      if post_bucket
+        post_bucket.files.map &:delete
+        post_bucket.delete
+      end
+    end
+  end
+
   it "copies itself to another table with copy_job" do
     job_id = "test_job_#{SecureRandom.urlsafe_base64(21)}" # client-generated
     copy_job = table.copy_job target_table_id, create: :needed, write: :empty, job_id: job_id, labels: labels