Permalink
Browse files

first

  • Loading branch information...
0 parents commit 5f527d1777fb02a59e3f386d1ff7f67174e89e1f @tagomoris tagomoris committed Dec 17, 2013
Showing with 187 additions and 0 deletions.
  1. +18 −0 .gitignore
  2. +4 −0 Gemfile
  3. +22 −0 LICENSE.txt
  4. +29 −0 README.md
  5. +1 −0 Rakefile
  6. +22 −0 fluent-plugin-bigquery.gemspec
  7. +91 −0 lib/fluent/plugin/out_bigquery.rb
@@ -0,0 +1,18 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp
+script/
@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in fluent-plugin-bigquery.gemspec
+gemspec
@@ -0,0 +1,22 @@
+Copyright (c) 2013 TAGOMORI Satoshi
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
@@ -0,0 +1,29 @@
+# Fluent::Plugin::Bigquery
+
+TODO: Write a gem description
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+ gem 'fluent-plugin-bigquery'
+
+And then execute:
+
+ $ bundle
+
+Or install it yourself as:
+
+ $ gem install fluent-plugin-bigquery
+
+## Usage
+
+TODO: Write usage instructions here
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request
@@ -0,0 +1 @@
+require "bundler/gem_tasks"
@@ -0,0 +1,22 @@
+# coding: utf-8
+Gem::Specification.new do |spec|
+ spec.name = "fluent-plugin-bigquery"
+ spec.version = "0.0.1"
+ spec.authors = ["TAGOMORI Satoshi"]
+ spec.email = ["tagomoris@gmail.com"]
+ spec.description = %q{TODO: Write a gem description}
+ spec.summary = %q{TODO: Write a gem summary}
+ spec.homepage = ""
+ spec.license = "APLv2"
+
+ spec.files = `git ls-files`.split($/)
+ spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+ spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
+ spec.require_paths = ["lib"]
+
+ spec.add_development_dependency "rake"
+ gem.add_runtime_dependency "fluentd"
+ gem.add_runtime_dependency "fluent-mixin-plaintextformatter", '>= 0.2.1'
+ gem.add_runtime_dependency "fluent-mixin-config-placeholders", ">= 0.2.0"
+ gem.add_runtime_dependency "google-api-client", "~> 0.6.4"
+end
@@ -0,0 +1,91 @@
+# -*- coding: utf-8 -*-
+
+require 'fluent/mixin/config_placeholders'
+require 'fluent/mixin/plaintextformatter'
+
+module Fluent
+ class BigQueryOutput < BufferedOutput
+ Fluent::Plugin.register_output('bigquery', self)
+
+ # https://developers.google.com/bigquery/browser-tool-quickstart
+ # https://developers.google.com/bigquery/bigquery-api-quickstart
+
+
+ # dataset_name
+ # The name can be up to 1,024 characters long, and consist of A-Z, a-z, 0-9, and the underscore,
+ # but it cannot start with a number or underscore, or have spaces.
+
+ # table_id
+ # In Table ID, enter a name for your new table. Naming rules are the same as for your dataset.
+
+ # see as simple reference
+ # https://github.com/abronte/BigQuery/blob/master/lib/bigquery.rb
+
+ # https://developers.google.com/bigquery/loading-data-into-bigquery
+ # Maximum File Sizes:
+ # File Type Compressed Uncompressed
+ # CSV 1 GB With new-lines in strings: 4 GB
+ # Without new-lines in strings: 1 TB
+ # JSON 1 GB 1 TB
+ config_set_default :buffer_type, 'memory'
+ config_set_default :flush_interval, 1800 # 30min => 48 imports/day
+ config_set_default :buffer_chunk_limit, 1000*1000*1000*1000 # 1.0*10^12 < 1TB (1024^4)
+
+ ### OAuth credential
+ # config_param :client_id, :string
+ # config_param :client_secret, :string
+
+ ### Service Account credential
+ config_param :email, :string
+ config_param :private_key_path, :string
+ config_param :private_key_passphrase, :string, :default => 'notasecret'
+
+ config_param :project_id, :string
+
+ def initialize
+ require 'google/api_client'
+ require 'google/api_client/client_secrets'
+ require 'google/api_client/auth/installed_app'
+ end
+
+ def client
+ @bq = client.discovered_api("bigquery", "v2")
+
+ client = Google::APIClient.new
+ key = Google::APIClient::PKCS12.load_key( @private_key_path, @private_key_passphrase )
+ asserter = Google::APIClient::JWTAsserter.new(
+ @email,
+ "https://www.googleapis.com/auth/bigquery",
+ key
+ )
+ # refresh_auth
+ client.authorization = asserter.authorize
+ client
+ end
+
+ def client_oauth # not implemented
+ raise NotImplementedError, "OAuth needs browser authentication..."
+
+ client = Google::APIClient.new(
+ :application_name => 'Example Ruby application',
+ :application_version => '1.0.0'
+ )
+ bigquery = client.discovered_api('bigquery', 'v2')
+ flow = Google::APIClient::InstalledAppFlow.new(
+ :client_id => @client_id
+ :client_secret => @client_secret
+ :scope => ['https://www.googleapis.com/auth/bigquery']
+ )
+ client.authorization = flow.authorize # browser authentication !
+ client
+ end
+
+ def get_datasets_list
+ client().execute(@bq.datasets.list)
+ end
+
+ def execute(method)
+ client().execute( :api_method => method, :parameters => {'projectId' => @project_id} )
+ end
+ end
+end

0 comments on commit 5f527d1

Please sign in to comment.