Permalink
Browse files

Added the so far developped project. Also created .gitignore with loc…

…al app files ignored
  • Loading branch information...
1 parent a876a0e commit 5bf1aaa9ad4f4679b4fd0d2ed150c92f23368964 Emre Berge Ergenekon committed Nov 1, 2011
View
@@ -0,0 +1,17 @@
+# OSX noise
+.DS_Store
+
+# Compiled python files
+*.pyc
+
+# Eclipse project files
+.project
+.pydevproject
+
+# Local stuff
+local/
+local.meta
+lib/
+
+# Git repositories folder
+git-repositories/
No changes.
View
@@ -0,0 +1,69 @@
+# Copyright 2011 Splunk, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
+import httplib2
+import re
+
+class GithubAPI(object):
+ '''
+ Author: Emre Berge Ergenekon
+ Contains functions for making some of the Github v3 API requests.
+ '''
+ _base_url = ''
+
+ def __init__(self, user_name, repo_name):
+ self._base_url = 'https://api.github.com/repos/{0}/{1}'.format(user_name, repo_name)
+
+ def watchers(self):
+ return self.make_request('watchers')
+
+ def open_issues(self):
+ return self._issues('open')
+
+ def closed_issues(self):
+ return self._issues('closed')
+
+ def _issues(self, state='open'):
+ return self.make_request('issues?state={0}'.format(state))
+
+ def forks(self):
+ return self.make_request('forks')
+
+ def make_request(self, partial_url):
+ http = httplib2.Http()
+ next_url = self._create_full_api_url(partial_url)
+ last_url = ''
+ all_responses = []
+ while True :
+ response, content = http.request(next_url)
+ if response.status != 200: break
+ all_responses[len(all_responses):] = json.loads(content)
+ if next_url == last_url or not self._has_link_header(response): break
+ next_url = self._link_header_value_for_reletion(response, 'next')
+ last_url = self._link_header_value_for_reletion(response, 'last')
+ return all_responses
+
+ def _create_full_api_url(self, partial_url):
+ return '{base_url}/{partial_url}{parameter_separator}per_page=100'.format(base_url=self._base_url, partial_url=partial_url, parameter_separator=('&' if '?' in partial_url else '?'))
+
+ def _link_header_value_for_reletion(self, response, rel):
+ linkHeader = response['link']
+ pattern = '(?<=\<)[^>]+?(?=\>; rel="{0}")'.format(rel)
+ value = re.search(pattern, linkHeader).group(0)
+ return value
+
+ def _has_link_header(self, response):
+ return 'link' in response
+
View
@@ -0,0 +1,78 @@
+#!/bin/bash
+
+# Copyright 2011 Splunk, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This script polls a git remote repo for changes and prints the results to standart out in a format splunk easily can read.
+# Author: Petter Eriksson, Emre Berge Ergenekon
+
+#Global variables
+APP_HOME=$SPLUNK_HOME/etc/apps/Splunkgit
+GIT_REPO=`splunk cmd python $APP_HOME/bin/splunkgit_settings.py`
+GIT_REPO_FOLDER=`echo $GIT_REPO | sed 's/.*\///'`
+GIT_REPOS_HOME=$APP_HOME/git-repositories
+chosen_repository=$GIT_REPOS_HOME/$GIT_REPO_FOLDER
+
+#echo $chosen_repository
+
+NUMBER_OF_COMMITS_TO_SKIP=`splunk search "index=splunkgit | stats dc(commit_hash) as commitCount" -auth admin:changeme -app Splunkgit | grep -o -P '[0-9]+'`
+
+main ()
+{
+if [ -d "$chosen_repository" ]; then
+ print_hashes_and_git_log_numstat
+ else
+ #TODO Handle this
+ echo "repository does not exist!" 1>&2
+ fetch_git_repository
+ fi
+}
+
+#Not safe to run this method parallel from the same directory, since the $numstat_file is touched, written to and deleted.
+#TODO: Figure out a way to remove the $numstat_file logic.
+print_hashes_and_git_log_numstat ()
+{
+ numstat_file=git-commit-formatted.out #temporary gather git-diff-tree output
+ cd $chosen_repository
+ git fetch 1>&2
+
+#for each commit in the git history
+ for commit in `git log --pretty=format:'%H' --topo-order --all --no-color --no-renames --no-merges --skip=$NUMBER_OF_COMMITS_TO_SKIP`
+ do
+ touch $numstat_file
+ git --no-pager diff-tree $commit --pretty=format:'[%ci] author_name="%an" author_mail="%ae" commit_hash="%H" parrent_hash="%P" tree_hash="%T"' --numstat | sed '/^$/d' | awk -F \t -v FIRST_LINE=1 -v REPO="$GIT_REPO" '{if (FIRST_LINE==1) {FIRST_LINE=0;COMMIT_INFO=$0} else {print COMMIT_INFO" insertions=\""$1"\" deletions=\""$2"\" path=\""$3"\" file_type=\"---/"$3"---\" repository=\""REPO"\""}}' | perl -pe 's|---.*/(.+?)---|---\.\1---|' | perl -pe 's|---.*\.(.+?)---|\1|' | tee $numstat_file
+
+ if [ ! -s $numstat_file ]; then #if there was no numstat output, just print the commit_info
+ git --no-pager show $commit --pretty=format:'[%ci] author_name="%an" author_mail="%ae" commit_hash="%H" parrent_hash="%P" tree_hash="%T"' --quiet
+ echo ;
+ fi
+ rm $numstat_file #clean up
+ done
+}
+
+fetch_git_repository ()
+{
+ error_output=err.out
+ touch $error_output
+ mkdir -p $GIT_REPOS_HOME
+ git clone --mirror $GIT_REPO $chosen_repository &> $error_output
+ if [ ! -s $error_output ]; then #if there are no errors from git clone.
+ print_hashes_and_git_log_numstat # try again
+ else
+ cat $error_output 1>&2
+ fi
+ rm $error_output
+}
+
+main
View
@@ -0,0 +1,81 @@
+# Copyright 2011 Splunk, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+'''
+This script polls usefull github data from github
+Author: Emre Berge Ergenekon, Petter Eriksson
+'''
+
+#Import from std lib
+from time import localtime, strftime
+import os
+import sys
+
+#import from 3rd party lib
+LIB_PATH = os.path.abspath(os.path.join(__file__, '..', '..', 'lib'))
+sys.path.insert(0, LIB_PATH)
+from joblib import Parallel, delayed
+
+#import own classes
+from GithubAPI import GithubAPI
+from splunkgit_settings import github_repo_name, github_user_login_name
+
+GITHUB_USER = github_user_login_name()
+GITHUB_REPO = github_repo_name()
+
+def get_total_fork_count(forks, run_as_single_job=False):
+ total_number_of_forks = 1
+ total_number_of_forks += _count_forks_in_all_depths(forks, run_as_single_job)
+ return total_number_of_forks
+
+def _count_forks_in_all_depths(forks, run_as_single_job=False):
+ if run_as_single_job:
+ return _count_forks_in_all_depths_on_single_thread(forks)
+ else:
+ return _count_forks_in_all_depths_on_multiple_threads(forks)
+
+def _count_forks_in_all_depths_on_single_thread(forks):
+ total_number_of_forks = 0
+ for fork in forks:
+ total_number_of_forks += _get_total_fork_count_inner_loop(fork)
+ return total_number_of_forks
+
+def _count_forks_in_all_depths_on_multiple_threads(forks):
+ fork_counts = Parallel(n_jobs=16)(delayed(_get_total_fork_count_inner_loop)(forks[i]) for i in range(len(forks)))
+ return sum(fork_counts)
+
+def _get_total_fork_count_inner_loop(fork):
+ if fork['private'] == True:
+ return 0
+ else:
+ return _get_total_fork_count_for_public_fork(fork)
+
+def _get_total_fork_count_for_public_fork(fork):
+ github_user = fork['owner']['login']
+ github_fork = fork['name']
+ github_api = _get_github_api_for(github_user, github_fork)
+ return get_total_fork_count(github_api.forks(), True)
+
+def _get_github_api_for(github_user, github_repo):
+ return GithubAPI(github_user, github_repo)
+
+if __name__ == '__main__':
+ github_api = _get_github_api_for(GITHUB_USER, GITHUB_REPO)
+ time_stamp = strftime("%Y-%m-%d %H:%M:%S %z", localtime())
+
+ print '[{0}] github_watcher_count={1}'.format(time_stamp, len(github_api.watchers()))
+ all_issues = github_api.closed_issues() + github_api.open_issues()
+ for issue in all_issues :
+ print u'[{0}] github_issue_number={1} github_issue_state="{2}" github_issue_comment_count={3} github_issue_reporter="{4}" github_issue_title="{5}" github_issue_close_time="{6}" github_issue_update_time="{7}" github_issue_creation_time="{8}"'.format(time_stamp, issue['number'], issue['state'], issue['comments'], issue['user']['login'], issue['title'], issue['closed_at'], issue['updated_at'], issue['created_at'])
+ print '[{0}] github_forks_count={1}'.format(time_stamp, get_total_fork_count(github_api.forks()))
View
@@ -0,0 +1,34 @@
+# Copyright 2011 Splunk, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import splunk.clilib.cli_common
+
+'''
+Functions for retriveing settigs from splunkgit conf file.
+Author: Emre Berge Ergenekon, Petter Eriksson
+'''
+SPLUNKGIT_GIT_SETTINGS = splunk.clilib.cli_common.getConfStanza('splunkgit','git')
+SPLUNKGIT_GITHUB_SETTINGS = splunk.clilib.cli_common.getConfStanza('splunkgit','github')
+
+def git_repo_address():
+ return SPLUNKGIT_GIT_SETTINGS['repo_address']
+
+def github_user_login_name():
+ return SPLUNKGIT_GITHUB_SETTINGS['user_login_name']
+
+def github_repo_name():
+ return SPLUNKGIT_GITHUB_SETTINGS['repo_name']
+
+if __name__ == '__main__':
+ print git_repo_address()
View
@@ -0,0 +1,22 @@
+# Copyright 2011 Splunk, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+[ui]
+is_visible=true
+label=Splunkgit
+
+[launcher]
+author=Emre Berge Ergenekon & Petter Eriksson
+description=Retrives a chosen git repo and generates statistics about the repo.
+version=0.0.1
@@ -0,0 +1,26 @@
+<?xml version='1.0' encoding='utf-8'?>
+<!--
+Copyright 2011 Splunk, Inc.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+
+<nav>
+ <view name="flashtimeline" default="true"/>
+ <collection label="Github">
+ <view source="all" match="github_"/>
+ </collection>
+ <collection label="Git">
+ <view source="all" match="git_"/>
+ </collection>
+</nav>
Oops, something went wrong.

0 comments on commit 5bf1aaa

Please sign in to comment.