Permalink
Browse files

first

  • Loading branch information...
tagomoris committed Dec 17, 2013
0 parents commit 5f527d1777fb02a59e3f386d1ff7f67174e89e1f
Showing with 187 additions and 0 deletions.
  1. +18 −0 .gitignore
  2. +4 −0 Gemfile
  3. +22 −0 LICENSE.txt
  4. +29 −0 README.md
  5. +1 −0 Rakefile
  6. +22 −0 fluent-plugin-bigquery.gemspec
  7. +91 −0 lib/fluent/plugin/out_bigquery.rb
@@ -0,0 +1,18 @@
*.gem
*.rbc
.bundle
.config
.yardoc
Gemfile.lock
InstalledFiles
_yardoc
coverage
doc/
lib/bundler/man
pkg
rdoc
spec/reports
test/tmp
test/version_tmp
tmp
script/
@@ -0,0 +1,4 @@
source 'https://rubygems.org'
# Specify your gem's dependencies in fluent-plugin-bigquery.gemspec
gemspec
@@ -0,0 +1,22 @@
Copyright (c) 2013 TAGOMORI Satoshi
MIT License
Permission is hereby granted, free of charge, to any person obtaining
a copy of this software and associated documentation files (the
"Software"), to deal in the Software without restriction, including
without limitation the rights to use, copy, modify, merge, publish,
distribute, sublicense, and/or sell copies of the Software, and to
permit persons to whom the Software is furnished to do so, subject to
the following conditions:
The above copyright notice and this permission notice shall be
included in all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
@@ -0,0 +1,29 @@
# Fluent::Plugin::Bigquery
TODO: Write a gem description
## Installation
Add this line to your application's Gemfile:
gem 'fluent-plugin-bigquery'
And then execute:
$ bundle
Or install it yourself as:
$ gem install fluent-plugin-bigquery
## Usage
TODO: Write usage instructions here
## Contributing
1. Fork it
2. Create your feature branch (`git checkout -b my-new-feature`)
3. Commit your changes (`git commit -am 'Add some feature'`)
4. Push to the branch (`git push origin my-new-feature`)
5. Create new Pull Request
@@ -0,0 +1 @@
require "bundler/gem_tasks"
@@ -0,0 +1,22 @@
# coding: utf-8
Gem::Specification.new do |spec|
spec.name = "fluent-plugin-bigquery"
spec.version = "0.0.1"
spec.authors = ["TAGOMORI Satoshi"]
spec.email = ["tagomoris@gmail.com"]
spec.description = %q{TODO: Write a gem description}
spec.summary = %q{TODO: Write a gem summary}
spec.homepage = ""
spec.license = "APLv2"
spec.files = `git ls-files`.split($/)
spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
spec.require_paths = ["lib"]
spec.add_development_dependency "rake"
gem.add_runtime_dependency "fluentd"
gem.add_runtime_dependency "fluent-mixin-plaintextformatter", '>= 0.2.1'
gem.add_runtime_dependency "fluent-mixin-config-placeholders", ">= 0.2.0"
gem.add_runtime_dependency "google-api-client", "~> 0.6.4"
end
@@ -0,0 +1,91 @@
# -*- coding: utf-8 -*-
require 'fluent/mixin/config_placeholders'
require 'fluent/mixin/plaintextformatter'
module Fluent
class BigQueryOutput < BufferedOutput
Fluent::Plugin.register_output('bigquery', self)
# https://developers.google.com/bigquery/browser-tool-quickstart
# https://developers.google.com/bigquery/bigquery-api-quickstart
# dataset_name
# The name can be up to 1,024 characters long, and consist of A-Z, a-z, 0-9, and the underscore,
# but it cannot start with a number or underscore, or have spaces.
# table_id
# In Table ID, enter a name for your new table. Naming rules are the same as for your dataset.
# see as simple reference
# https://github.com/abronte/BigQuery/blob/master/lib/bigquery.rb
# https://developers.google.com/bigquery/loading-data-into-bigquery
# Maximum File Sizes:
# File Type Compressed Uncompressed
# CSV 1 GB With new-lines in strings: 4 GB
# Without new-lines in strings: 1 TB
# JSON 1 GB 1 TB
config_set_default :buffer_type, 'memory'
config_set_default :flush_interval, 1800 # 30min => 48 imports/day
config_set_default :buffer_chunk_limit, 1000*1000*1000*1000 # 1.0*10^12 < 1TB (1024^4)
### OAuth credential
# config_param :client_id, :string
# config_param :client_secret, :string
### Service Account credential
config_param :email, :string
config_param :private_key_path, :string
config_param :private_key_passphrase, :string, :default => 'notasecret'
config_param :project_id, :string
def initialize
require 'google/api_client'
require 'google/api_client/client_secrets'
require 'google/api_client/auth/installed_app'
end
def client
@bq = client.discovered_api("bigquery", "v2")
client = Google::APIClient.new
key = Google::APIClient::PKCS12.load_key( @private_key_path, @private_key_passphrase )
asserter = Google::APIClient::JWTAsserter.new(
@email,
"https://www.googleapis.com/auth/bigquery",
key
)
# refresh_auth
client.authorization = asserter.authorize
client
end
def client_oauth # not implemented
raise NotImplementedError, "OAuth needs browser authentication..."
client = Google::APIClient.new(
:application_name => 'Example Ruby application',
:application_version => '1.0.0'
)
bigquery = client.discovered_api('bigquery', 'v2')
flow = Google::APIClient::InstalledAppFlow.new(
:client_id => @client_id
:client_secret => @client_secret
:scope => ['https://www.googleapis.com/auth/bigquery']
)
client.authorization = flow.authorize # browser authentication !
client
end
def get_datasets_list
client().execute(@bq.datasets.list)
end
def execute(method)
client().execute( :api_method => method, :parameters => {'projectId' => @project_id} )
end
end
end

0 comments on commit 5f527d1

Please sign in to comment.