fluent-plugin-github-activities.conf

# requires fluent-plugin-github-activities

<source>
  type github-activities
  # Authentication is strongly recommended, because there is a rate limit: 60requests/hour by default.
  # With authentication, it becomes 5000requests/hour (means about 80requests/minute).
  # See following the GitHub help about access_token
  #   https://help.github.com/articles/creating-an-access-token-for-command-line-use/
  access_token access-token-of-github
  users ashie,cosmo0920,kenhys,kou,min-shin,myokoym,okkez,piroor
  clients 4
  interval 1
  base_tag github-activity.
  pos_file /var/log/td-agent/github-activities.json
</source>


# copy record for event, tag, and actor (requires fluent-plugin-record-reformer)

<match github-activity.**>
  type copy

  # record for the Events table
  <store>
    type        record_reformer
    enable_ruby true
    tag         event.${tag}
    # fill common fields if possible
    <record>
      type        ${tag_suffix[-1]}
      actor       ${actor && actor["login"]}
      source_icon https://github.com/favicon.ico
      timestamp   ${created_at}
      created_at  ${time}
      _repository ${repo && repo["name"]}
    </record>
  </store>

  # record for the Actors table
  <store>
    type         record_reformer
    enable_ruby  true
    renew_record true
    tag          sharetary.actor
    <record>
      _key  ${(actor || committer)["login"]}
      uri   https://github.com/${(actor || committer)["login"]}/
      icon  ${self["$github-activities-related-avatar"]}
      class major
    </record>
  </store>

  # record for the Tags table
  <store>
    type         record_reformer
    enable_ruby  true
    renew_record true
    tag          sharetary.tag
    <record>
      _key  ${tag.end_with?(".commit") ? url.match(/repos\/([^\/]+\/[^\/]+)\/commits/)[1] : tag.end_with?(".fork") ? payload["forkee"]["full_name"] : repo["name"]}
      icon  ${self["$github-activities-related-organization-logo"]}
    </record>
  </store>
</match>



# complete event record from activity (requires fluent-plugin-record-reformer)


## commit and push

<match event.github-activity.push>
  type         record_reformer
  enable_ruby  true
  renew_record true
  keep_keys    type,actor,source_icon,timestamp,created_at,_repository
  tag          sharetary.event
  <record>
    _key        https://github.com/${repo["name"]}/compare/${payload["before"]}...${payload["head"]}
    class       minor
    title       Push
    description Pushed ${payload["size"].to_s} commits to ${repo["name"]}
    uri         https://github.com/${repo["name"]}/compare/${payload["before"]}...${payload["head"]}
    reply_uri   https://github.com/${repo["name"]}/compare/${payload["before"]}...${payload["head"]}#commits_bucket
  </record>
</match>

<match event.github-activity.commit>
  type         record_reformer
  enable_ruby  true
  renew_record true
  keep_keys    type,source_icon,created_at
  tag          sharetary.event
  <record>
    _key        ${html_url}
    class       normal
    _repository ${url.match(/repos\/([^\/]+\/[^\/]+)\/commits/)[1]}
    title       Commit ${stats["total"].to_s} changes
    description ${commit["message"]}
    extra_description ${files.collect{|file| "#{file["filename"]} (#{file["status"]})" }.join("\n, ")}\n\n${files.collect{|file| file["patch"] }.join("\n\n")}
    actor       ${committer["login"]}
    uri         ${html_url}
    reply_uri   ${html_url}#new_commit_comment_field
    timestamp   ${commit["committer"]["date"]}
    parent      ${self["$github-activities-related-event"] && "https://github.com/#{self["$github-activities-related-event"]["repo"]["name"]}/compare/#{self["$github-activities-related-event"]["payload"]["before"]}...#{self["$github-activities-related-event"]["payload"]["head"]}" || "" }
  </record>
</match>


## comments on commits

<match event.github-activity.commit-comment>
  type         record_reformer
  enable_ruby  true
  renew_record true
  keep_keys    type,actor,source_icon,timestamp,created_at,_repository
  tag          sharetary.event
  <record>
    _key        ${payload["comment"]["html_url"]}
    class       major
    title       Commented to the commit ${repo["name"]}/${payload["comment"]["commit_id"]}
    description ${payload["comment"]["body"]}
    uri         ${payload["comment"]["html_url"]}
    reply_uri   ${payload["comment"]["html_url"].split("#").first}#new_commit_comment_field
    parent      ${payload["comment"]["html_url"].split("#").first}
  </record>
</match>


## issues

<match event.github-activity.issue-open>
  type         record_reformer
  enable_ruby  true
  renew_record true
  keep_keys    type,actor,source_icon,timestamp,created_at,_repository
  tag          sharetary.event
  <record>
    _key        ${payload["issue"]["html_url"]}
    class       major
    title       Open
    description Opened new issue: ${"#"}${payload["issue"]["number"].to_s} ${payload["issue"]["title"]}: ${(payload["issue"]["body"] || "")}
    uri         ${payload["issue"]["html_url"]}
    reply_uri   ${payload["issue"]["html_url"].split("#").first}#new_comment_field
  </record>
</match>

<match event.github-activity.issue-close>
  type         record_reformer
  enable_ruby  true
  renew_record true
  keep_keys    type,actor,source_icon,timestamp,created_at,_repository
  tag          sharetary.event
  <record>
    _key        ${payload["issue"]["html_url"]}#${payload["issue"]["updated_at"]}
    class       major
    title       Close
    description Closed the issue: ${"#"}${payload["issue"]["number"].to_s} ${payload["issue"]["title"]}
    uri         ${payload["issue"]["html_url"]}
    reply_uri   ${payload["issue"]["html_url"].split("#").first}#new_comment_field
    parent      ${payload["issue"]["html_url"]}
  </record>
</match>

<match event.github-activity.issue-reopen>
  type         record_reformer
  enable_ruby  true
  renew_record true
  keep_keys    type,actor,source_icon,timestamp,created_at,_repository
  tag          sharetary.event
  <record>
    _key        ${payload["issue"]["html_url"]}#${payload["issue"]["updated_at"]}
    class       major
    title       Reopen
    description Reopened the issue: ${"#"}${payload["issue"]["number"].to_s} ${payload["issue"]["title"]}
    uri         ${payload["issue"]["html_url"]}
    reply_uri   ${payload["issue"]["html_url"].split("#").first}#new_comment_field
    parent      ${payload["issue"]["html_url"]}
  </record>
</match>

<match event.github-activity.issue-comment>
  type         record_reformer
  enable_ruby  true
  renew_record true
  keep_keys    type,actor,source_icon,timestamp,created_at,_repository
  tag          sharetary.event
  <record>
    _key        ${payload["comment"]["html_url"]}
    class       major
    title       Comment to ${"#"}${payload["issue"]["number"].to_s} ${payload["issue"]["title"]}
    description ${payload["comment"]["body"]}
    uri         ${payload["comment"]["html_url"]}
    reply_uri   ${payload["comment"]["html_url"].split("#").first}#new_comment_field
    parent      ${payload["issue"]["html_url"]}
  </record>
</match>


## pull requests

<match event.github-activity.pull-request>
  type         record_reformer
  enable_ruby  true
  renew_record true
  keep_keys    type,actor,source_icon,timestamp,created_at,_repository
  tag          sharetary.event
  <record>
    _key        ${payload["pull_request"]["html_url"]}
    class       important
    title       PR
    description Created new pull-request: ${"#"}${payload["pull_request"]["number"].to_s} ${payload["pull_request"]["title"]}: ${(payload["pull_request"]["body"] || "")}
    uri         ${payload["pull_request"]["html_url"]}
    reply_uri   ${payload["pull_request"]["html_url"].split("#").first}#new_comment_field
  </record>
</match>

<match event.github-activity.pull-request-reopen>
  type         record_reformer
  enable_ruby  true
  renew_record true
  keep_keys    type,actor,source_icon,timestamp,created_at,_repository
  tag          sharetary.event
  <record>
    _key        ${payload["pull_request"]["html_url"]}#${payload["pull_request"]["updated_at"]}
    class       important
    title       PR Reopen
    description Reopened the pull-request ${"#"}${payload["pull_request"]["number"].to_s} ${payload["pull_request"]["title"]}
    uri         ${payload["pull_request"]["html_url"]}
    reply_uri   ${payload["pull_request"]["html_url"].split("#").first}#new_comment_field
    parent      ${payload["pull_request"]["html_url"]}
  </record>
</match>

<match event.github-activity.pull-request-merged>
  type         record_reformer
  enable_ruby  true
  renew_record true
  keep_keys    type,actor,source_icon,timestamp,created_at,_repository
  tag          sharetary.event
  <record>
    _key        ${payload["pull_request"]["html_url"]}#${payload["pull_request"]["updated_at"]}
    class       important
    title       PR Merge
    description Merged the pull-request ${"#"}${payload["pull_request"]["number"].to_s} ${payload["pull_request"]["title"]}
    uri         ${payload["pull_request"]["html_url"]}
    reply_uri   ${payload["pull_request"]["html_url"].split("#").first}#new_comment_field
    parent      ${payload["pull_request"]["html_url"]}
  </record>
</match>

<match event.github-activity.pull-request-cancelled>
  type         record_reformer
  enable_ruby  true
  renew_record true
  keep_keys    type,actor,source_icon,timestamp,created_at,_repository
  tag          sharetary.event
  <record>
    _key        ${payload["pull_request"]["html_url"]}#${payload["pull_request"]["updated_at"]}
    class       important
    title       PR Cancel
    description Cancelled the pull-request ${"#"}${payload["pull_request"]["number"].to_s} ${payload["pull_request"]["title"]}
    uri         ${payload["pull_request"]["html_url"]}
    reply_uri   ${payload["pull_request"]["html_url"].split("#").first}#new_comment_field
    parent      ${payload["pull_request"]["html_url"]}
  </record>
</match>

<match event.github-activity.pull-request-comment>
  type         record_reformer
  enable_ruby  true
  renew_record true
  keep_keys    type,actor,source_icon,created_at,_repository
  tag          sharetary.event
  <record>
    _key        ${payload["comment"]["html_url"]}
    class       important
    title       PR Comment ${"#"}${payload["issue"]["number"].to_s} ${payload["issue"]["title"]}
    description ${payload["comment"]["body"]}
    uri         ${payload["comment"]["html_url"]}
    reply_uri   ${payload["comment"]["html_url"].split("#").first}#new_comment_field
    timestamp   ${payload["comment"]["created_at"]}
    parent      ${payload["issue"]["pull_request"]["html_url"]}
  </record>
</match>


## fork

<match event.github-activity.fork>
  type         record_reformer
  enable_ruby  true
  renew_record true
  keep_keys    type,actor,source_icon,timestamp,created_at
  tag          sharetary.event
  <record>
    _key        ${payload["forkee"]["html_url"]}#forked
    class       normal
    title       Fork
    description Forked the repository ${repo["name"]} to ${payload["forkee"]["full_name"]}
    uri         ${payload["forkee"]["html_url"]}#forked
    _repository ${payload["forkee"]["full_name"]}
  </record>
</match>


## branch, tag

<match event.github-activity.branch>
  type         record_reformer
  enable_ruby  true
  renew_record true
  keep_keys    type,actor,source_icon,timestamp,created_at,_repository
  tag          sharetary.event
  <record>
    _key        https://github.com/${repo["name"]}/tree/${payload["ref"]}
    class       normal
    title       Branch
    description Created new branch ${payload["ref"]} at the repository ${repo["name"]}
    uri         https://github.com/${repo["name"]}/tree/${payload["ref"]}
  </record>
</match>

<match event.github-activity.tag>
  type         record_reformer
  enable_ruby  true
  renew_record true
  keep_keys    type,actor,source_icon,timestamp,created_at,_repository
  tag          sharetary.event
  <record>
    _key        https://github.com/${repo["name"]}/tree/${payload["ref"]}
    class       normal
    title       Tag
    description Created new tag ${payload["ref"]} at the repository ${repo["name"]}
    uri         https://github.com/${repo["name"]}/tree/${payload["ref"]}
  </record>
</match>


# finalize array type `tags` field (requires fluent-plugin-record-reformer)

<filter sharetary.event>
  type         record_transformer
  enable_ruby  false
  remove_keys  _repository
  <record>
    tags ["${_repository}"]
    # tags ["${_repository}", "sample-extra-tag"] # if needed you can set extra tags.
  </record>
</filter>


# convert datetime string fields to unixtime integer fields (requires fluent-plugin-filter_typecast)

<filter sharetary.event>
  type  typecast
  types created_at:time,timestamp:time
</filter>



# storing event into Groonga's database (requires fluent-plugin-groonga)

<match sharetary.event>
  type groonga
  store_table Events

  protocol http
  host localhost

  buffer_type file
  buffer_path /var/spool/td-agent/buffer/groonga-sharetary-event
  flush_interval 1s

  <table>
    name Events
    flags TABLE_HASH_KEY
    key_type ShortText
  </table>

  <table>
    name Timestamps
    flags TABLE_PAT_KEY
    key_type Time
  </table>

  <table>
    name Tags
    flags TABLE_HASH_KEY
    key_type ShortText
  </table>

  <table>
    name Actors
    flags TABLE_HASH_KEY
    key_type ShortText
  </table>

  <table>
    name Terms
    flags TABLE_PAT_KEY
    key_type ShortText
    default_tokenizer TokenBigram
    normalizer NormalizerAuto
  </table>

  <mapping>
    name type
    type ShortText
  </mapping>

  <mapping>
    name class
    type ShortText
  </mapping>

  <mapping>
    name title
    type ShortText
    <index>
      table Terms
      name Events_title_index
      flags WITH_POSITION
    </index>
  </mapping>

  <mapping>
    name description
    type Text
    <index>
      table Terms
      name Events_description_index
      flags WITH_POSITION
    </index>
  </mapping>

  <mapping>
    name extra_description
    type Text
    <index>
      table Terms
      name Events_extra_description_index
      flags WITH_POSITION
    </index>
  </mapping>

  <mapping>
    name tags
    type Tags
    <index>
      table Tags
      name Events_tags_index
    </index>
  </mapping>

  <mapping>
    name uri
    type ShortText
    <index>
      table Terms
      name Events_uri_index
      flags WITH_POSITION
    </index>
  </mapping>

  <mapping>
    name source_icon
    type ShortText
  </mapping>

  <mapping>
    name reply_uri
    type ShortText
  </mapping>

  <mapping>
    name actor
    type Actors
    <index>
      table Actors
      name Events_actor_index
    </index>
  </mapping>

  <mapping>
    name timestamp
    type Time
    <index>
      table Timestamps
      name Events_timestamp_index
    </index>
  </mapping>

  <mapping>
    name created_at
    type Time
    <index>
      table Timestamps
      name Events_created_at_index
    </index>
  </mapping>

  <mapping>
    name parent
    type ShortText
    <index>
      table Terms
      name Events_parent_index
      flags WITH_POSITION
    </index>
  </mapping>
</match>



# tags

<match sharetary.tag>
  type groonga
  store_table Tags

  protocol http
  host localhost

  buffer_type file
  buffer_path /var/spool/td-agent/buffer/groonga-sharetary-tag
  flush_interval 1s

  <table>
    name Tags
    flags TABLE_HASH_KEY
    key_type ShortText
  </table>

  <mapping>
    name icon
    type ShortText
  </mapping>
</match>



# actor

<match sharetary.actor>
  type groonga
  store_table Actors

  protocol http
  host localhost

  buffer_type file
  buffer_path /var/spool/td-agent/buffer/groonga-sharetary-actor
  flush_interval 1s

  <table>
    name Actors
    flags TABLE_HASH_KEY
    key_type ShortText
  </table>

  <mapping>
    name uri
    type ShortText
  </mapping>

  <mapping>
    name icon
    type ShortText
  </mapping>

  <mapping>
    name class
    type ShortText
  </mapping>
</match>