From 6efa627d29c951f495dc4617baaba8bbe349c551 Mon Sep 17 00:00:00 2001 From: Anatoly Kislov Date: Tue, 19 Sep 2017 18:01:12 +0300 Subject: [PATCH 01/10] wip: single observable for traversing commits --- src/main/kotlin/app/hashers/CommitHasher.kt | 137 ++++---------------- src/main/kotlin/app/hashers/RepoHasher.kt | 112 ++++++++++++++++ src/main/kotlin/app/model/Commit.kt | 1 + src/main/kotlin/app/model/DiffEdit.kt | 16 --- src/main/kotlin/app/model/DiffFile.kt | 2 + 5 files changed, 137 insertions(+), 131 deletions(-) delete mode 100644 src/main/kotlin/app/model/DiffEdit.kt diff --git a/src/main/kotlin/app/hashers/CommitHasher.kt b/src/main/kotlin/app/hashers/CommitHasher.kt index 326cd2bc..cbcef535 100644 --- a/src/main/kotlin/app/hashers/CommitHasher.kt +++ b/src/main/kotlin/app/hashers/CommitHasher.kt @@ -9,26 +9,12 @@ import app.api.Api import app.extractors.Extractor import app.model.Author import app.model.Commit -import app.model.DiffContent -import app.model.DiffFile -import app.model.DiffRange import app.model.Fact import app.model.LocalRepo import app.model.Repo -import app.utils.RepoHelper import io.reactivex.Observable import io.reactivex.schedulers.Schedulers import org.eclipse.jgit.api.Git -import org.eclipse.jgit.diff.DiffEntry -import org.eclipse.jgit.lib.Repository -import org.eclipse.jgit.revwalk.RevWalk -import java.nio.charset.Charset -import org.eclipse.jgit.diff.DiffFormatter -import org.eclipse.jgit.diff.RawText -import org.eclipse.jgit.lib.ObjectId -import org.eclipse.jgit.errors.MissingObjectException -import org.eclipse.jgit.revwalk.RevCommit -import org.eclipse.jgit.util.io.DisabledOutputStream import java.time.LocalDateTime import java.time.ZoneOffset import java.util.concurrent.TimeUnit @@ -37,33 +23,29 @@ import java.util.concurrent.TimeUnit * CommitHasher hashes repository and uploads stats to server. */ class CommitHasher(private val localRepo: LocalRepo, - private val repo: Repo = Repo(), + private val serverRepo: Repo = Repo(), private val api: Api, private val git: Git) { - private val gitRepo: Repository = git.repository - private fun findFirstOverlappingCommit(): Commit? { - val serverHistoryCommits = repo.commits.toHashSet() + val serverHistoryCommits = serverRepo.commits.toHashSet() return getCommitsAsObservable() .skipWhile { commit -> !serverHistoryCommits.contains(commit) } .blockingFirst(null) } private fun hashAndSendCommits() { - val lastKnownCommit = repo.commits.lastOrNull() - val knownCommits = repo.commits.toHashSet() + val lastKnownCommit = serverRepo.commits.lastOrNull() + val knownCommits = serverRepo.commits.toHashSet() val factsDayWeek = hashMapOf>() val factsDayTime = hashMapOf>() - // Commits are combined in pairs, an empty commit concatenated to - // calculate the diff of the initial commit. - Observable.concat(getCommitsAsObservable() + .doOnNext { commit -> Logger.debug("Commit: ${commit.raw?.name ?: ""}: " + commit.raw?.shortMessage) - commit.repo = repo + commit.repo = serverRepo // Calculate facts. val author = commit.author @@ -78,27 +60,24 @@ class CommitHasher(private val localRepo: LocalRepo, factDayTime[dateTime.hour] += 1 factsDayWeek[author] = factDayWeek factsDayTime[author] = factDayTime - }, Observable.just(Commit())) - .pairWithNext() // Pair commits to get diff. - .takeWhile { (new, _) -> // Hash until last known commit. - new.rehash != lastKnownCommit?.rehash } - .filter { (new, _) -> knownCommits.isEmpty() // Don't hash known. - || !knownCommits.contains(new) } - .filter { (new, _) -> emailFilter(new) } // Email filtering. - .map { (new, old) -> // Mapping and stats extraction. - val diffFiles = getDiffFiles(new, old) - Logger.debug("Diff: ${diffFiles.size} entries") - new.stats = Extractor().extract(diffFiles) - Logger.debug("Stats: ${new.stats.size} entries") + } + .takeWhile { commit -> // Hash until last known commit. + commit.rehash != lastKnownCommit?.rehash } + .filter { commit -> knownCommits.isEmpty() // Don't hash known. + || !knownCommits.contains(commit) } + .filter { commit -> emailFilter(commit) } // Email filtering. + .map { commit -> // Mapping and stats extraction. + commit.stats = Extractor().extract(commit.diffs) + Logger.debug("Stats: ${commit.stats.size} entries") // Count lines on all non-binary files. This is additional // statistics to CommitStats because not all file extensions // may be supported. - new.numLinesAdded = diffFiles.fold(0) { total, file -> + commit.numLinesAdded = commit.diffs.fold(0) { total, file -> total + file.getAllAdded().size } - new.numLinesDeleted = diffFiles.fold(0) { total, file -> + commit.numLinesDeleted = commit.diffs.fold(0) { total, file -> total + file.getAllDeleted().size } - new + commit } .observeOn(Schedulers.io()) // Different thread for data sending. .buffer(20, TimeUnit.SECONDS) // Group ready commits by time. @@ -111,7 +90,7 @@ class CommitHasher(private val localRepo: LocalRepo, factsDayTime.map { (author, list) -> list.forEachIndexed { hour, count -> if (count > 0) { - facts.add(Fact(repo, FactKey.COMMITS_DAY_TIME + + facts.add(Fact(serverRepo, FactKey.COMMITS_DAY_TIME + hour, count.toDouble(), author)) } } @@ -119,7 +98,7 @@ class CommitHasher(private val localRepo: LocalRepo, factsDayWeek.map { (author, list) -> list.forEachIndexed { day, count -> if (count > 0) { - facts.add(Fact(repo, FactKey.COMMITS_DAY_WEEK + + facts.add(Fact(serverRepo, FactKey.COMMITS_DAY_WEEK + day, count.toDouble(), author)) } } @@ -128,53 +107,6 @@ class CommitHasher(private val localRepo: LocalRepo, }) } - private fun getDiffFiles(commitNew: Commit, - commitOld: Commit): List { - val revCommitNew:RevCommit? = commitNew.raw - val revCommitOld:RevCommit? = commitOld.raw - - return DiffFormatter(DisabledOutputStream.INSTANCE).use { formatter -> - formatter.setRepository(gitRepo) - formatter.setDetectRenames(true) - formatter.scan(revCommitOld?.tree, revCommitNew?.tree) - // RENAME change type doesn't change file content. - .filter { it.changeType != DiffEntry.ChangeType.RENAME } - // Skip binary files. - .filter { - val id = if (it.changeType == DiffEntry.ChangeType.DELETE) { - it.oldId.toObjectId() - } else { - it.newId.toObjectId() - } - !RawText.isBinary(gitRepo.open(id).openStream()) - } - .map { diff -> - val new = getContentByObjectId(diff.newId.toObjectId()) - val old = getContentByObjectId(diff.oldId.toObjectId()) - - val edits = formatter.toFileHeader(diff).toEditList() - val path = when (diff.changeType) { - DiffEntry.ChangeType.DELETE -> diff.oldPath - else -> diff.newPath - } - DiffFile(path = path, - old = DiffContent(old, edits.map { edit -> - DiffRange(edit.beginA, edit.endA) }), - new = DiffContent(new, edits.map { edit -> - DiffRange(edit.beginB, edit.endB) })) - } - } - } - - private fun getContentByObjectId(objectId: ObjectId): List { - return try { - gitRepo.open(objectId).bytes.toString(Charset.defaultCharset()) - .split('\n') - } catch (e: MissingObjectException) { - listOf() - } - } - private fun postCommitsToServer(commits: List) { if (commits.isNotEmpty()) { api.postCommits(commits) @@ -196,35 +128,10 @@ class CommitHasher(private val localRepo: LocalRepo, } } - private fun getCommitsAsObservable(): Observable = - Observable.create { subscriber -> - try { - val revWalk = RevWalk(gitRepo) - val commitId = gitRepo.resolve(RepoHelper.MASTER_BRANCH) - revWalk.markStart(revWalk.parseCommit(commitId)) - for (revCommit in revWalk) { - subscriber.onNext(Commit(revCommit)) - } - } catch (e: Exception) { - Logger.error("Commit producing error", e) - subscriber.onError(e) - } - subscriber.onComplete() - } - private val emailFilter: (Commit) -> Boolean = { val email = it.author.email localRepo.hashAllContributors || (email == localRepo.author.email || - repo.emails.contains(email)) - } - - private fun Observable.pairWithNext(): Observable> { - return this.map { emit -> Pair(emit, emit) } - // Accumulate emits by prev-next pair. - .scan { pairAccumulated, pairNext -> - Pair(pairAccumulated.second, pairNext.second) - } - .skip(1) // Skip initial not paired emit. + serverRepo.emails.contains(email)) } fun update() { @@ -232,7 +139,7 @@ class CommitHasher(private val localRepo: LocalRepo, // common commit then next commits are not deleted because hash of a // commit calculated including hashes of its parents. val firstOverlapCommit = findFirstOverlappingCommit() - val deletedCommits = repo.commits + val deletedCommits = serverRepo.commits .takeWhile { it.rehash != firstOverlapCommit?.rehash } deleteCommitsOnServer(deletedCommits) diff --git a/src/main/kotlin/app/hashers/RepoHasher.kt b/src/main/kotlin/app/hashers/RepoHasher.kt index f5ad7b2d..d7fca06a 100644 --- a/src/main/kotlin/app/hashers/RepoHasher.kt +++ b/src/main/kotlin/app/hashers/RepoHasher.kt @@ -3,17 +3,33 @@ package app.hashers +import app.FactKey import app.Logger import app.api.Api import app.config.Configurator +import app.extractors.Extractor +import app.model.Commit +import app.model.DiffContent +import app.model.DiffFile +import app.model.DiffRange +import app.model.Fact import app.model.LocalRepo import app.model.Repo import app.utils.RepoHelper +import io.reactivex.Observable +import io.reactivex.schedulers.Schedulers import org.eclipse.jgit.api.Git +import org.eclipse.jgit.diff.DiffEntry +import org.eclipse.jgit.diff.DiffFormatter +import org.eclipse.jgit.diff.RawText +import org.eclipse.jgit.errors.MissingObjectException +import org.eclipse.jgit.lib.ObjectId import org.eclipse.jgit.revwalk.RevCommit import org.eclipse.jgit.revwalk.RevWalk +import org.eclipse.jgit.util.io.DisabledOutputStream import java.io.File import java.io.IOException +import java.nio.charset.Charset class RepoHasher(private val localRepo: LocalRepo, private val api: Api, private val configurator: Configurator) { @@ -95,4 +111,100 @@ class RepoHasher(private val localRepo: LocalRepo, private val api: Api, val initialCommit = revWalk.last() return initialCommit.id.name } + + private fun hashAndSendCommits(git: Git) { + // Commits are combined in pairs, an empty commit concatenated to + // calculate the diff of the initial commit. + Observable.concat(getCommitsObservable(git), Observable.just(Commit())) + .pairWithNext() // Pair commits to get diff. + .map { (new, old) -> // Mapping and stats extraction. + Logger.debug("Commit: ${new.raw?.name ?: ""}: " + + new.raw?.shortMessage) + new.repo = repo + new.diffs = getDiffFiles(git, new, old) + Logger.debug("Diff: ${new.diffs.size} entries") + new + } + .blockingSubscribe({ commit -> // OnNext. + + }, { e -> // OnError. + Logger.error("Error while hashing: $e") + }) + } + + private fun getCommitsObservable(git: Git): Observable = + Observable.create { subscriber -> + try { + val revWalk = RevWalk(git.repository) + val commitId = git.repository.resolve(RepoHelper.MASTER_BRANCH) + revWalk.markStart(revWalk.parseCommit(commitId)) + for (revCommit in revWalk) { + subscriber.onNext(Commit(revCommit)) + } + } catch (e: Exception) { + Logger.error("Commit producing error", e) + subscriber.onError(e) + } + subscriber.onComplete() + } + + private fun getDiffFiles(git: Git, + commitNew: Commit, + commitOld: Commit): List { + val revCommitNew:RevCommit? = commitNew.raw + val revCommitOld:RevCommit? = commitOld.raw + + return DiffFormatter(DisabledOutputStream.INSTANCE).use { formatter -> + formatter.setRepository(git.repository) + formatter.setDetectRenames(true) + formatter.scan(revCommitOld?.tree, revCommitNew?.tree) + // RENAME change type doesn't change file content. + .filter { it.changeType != DiffEntry.ChangeType.RENAME } + // Skip binary files. + .filter { + val id = if (it.changeType == DiffEntry.ChangeType.DELETE) { + it.oldId.toObjectId() + } else { + it.newId.toObjectId() + } + !RawText.isBinary(git.repository.open(id).openStream()) + } + .map { diff -> + val new = getContentByObjectId(git, diff.newId.toObjectId()) + val old = getContentByObjectId(git, diff.oldId.toObjectId()) + + val edits = formatter.toFileHeader(diff).toEditList() + val path = when (diff.changeType) { + DiffEntry.ChangeType.DELETE -> diff.oldPath + else -> diff.newPath + } + DiffFile(path = path, + changeType = diff.changeType, + old = DiffContent(old, edits.map { edit -> + DiffRange(edit.beginA, edit.endA) }), + new = DiffContent(new, edits.map { edit -> + DiffRange(edit.beginB, edit.endB) })) + } + } + } + + private fun getContentByObjectId(git: Git, + objectId: ObjectId): List { + return try { + git.repository + .open(objectId).bytes.toString(Charset.defaultCharset()) + .split('\n') + } catch (e: MissingObjectException) { + listOf() + } + } + + private fun Observable.pairWithNext(): Observable> { + return this.map { emit -> Pair(emit, emit) } + // Accumulate emits by prev-next pair. + .scan { pairAccumulated, pairNext -> + Pair(pairAccumulated.second, pairNext.second) + } + .skip(1) // Skip initial not paired emit. + } } diff --git a/src/main/kotlin/app/model/Commit.kt b/src/main/kotlin/app/model/Commit.kt index 33da1dc3..9810a77a 100644 --- a/src/main/kotlin/app/model/Commit.kt +++ b/src/main/kotlin/app/model/Commit.kt @@ -27,6 +27,7 @@ data class Commit( ) { // Wrapping JGit's RevCommit. var raw: RevCommit? = null // Not sent to sever. + var diffs: List = listOf() constructor(revCommit: RevCommit) : this() { raw = revCommit diff --git a/src/main/kotlin/app/model/DiffEdit.kt b/src/main/kotlin/app/model/DiffEdit.kt deleted file mode 100644 index 33bab111..00000000 --- a/src/main/kotlin/app/model/DiffEdit.kt +++ /dev/null @@ -1,16 +0,0 @@ -// Copyright 2017 Sourcerer Inc. All Rights Reserved. -// Author: Anatoly Kislov (anatoly@sourcerer.io) - -package app.model - -import org.eclipse.jgit.diff.Edit - -/** - * Edit is partial change of file. [del] specifies range of deleted lines in old - * content, [add] specifies range of added lines instead of deleted lines. - * Made to decouple statistics classes from JGit. - */ -data class DiffEdit(val del: DiffRange, val add: DiffRange) { - constructor(edit: Edit) : this(DiffRange(edit.beginA, edit.endA), - DiffRange(edit.beginB, edit.endB)) -} diff --git a/src/main/kotlin/app/model/DiffFile.kt b/src/main/kotlin/app/model/DiffFile.kt index 66a06a17..ea8a49aa 100644 --- a/src/main/kotlin/app/model/DiffFile.kt +++ b/src/main/kotlin/app/model/DiffFile.kt @@ -4,9 +4,11 @@ package app.model import app.utils.FileHelper +import org.eclipse.jgit.diff.DiffEntry class DiffFile( val path: String = "", + val changeType: DiffEntry.ChangeType, val old: DiffContent = DiffContent(), val new: DiffContent = DiffContent(), var language: String = "" From 7c615f3925275658a6c606ca0830f6440d731f35 Mon Sep 17 00:00:00 2001 From: Anatoly Kislov Date: Wed, 20 Sep 2017 03:04:49 +0300 Subject: [PATCH 02/10] wip: trying to switch to observables (APP-100) --- src/main/kotlin/app/hashers/CommitHasher.kt | 150 +++++++++++--------- src/main/kotlin/app/hashers/RepoHasher.kt | 132 +++++++++-------- 2 files changed, 155 insertions(+), 127 deletions(-) diff --git a/src/main/kotlin/app/hashers/CommitHasher.kt b/src/main/kotlin/app/hashers/CommitHasher.kt index cbcef535..0946bfc6 100644 --- a/src/main/kotlin/app/hashers/CommitHasher.kt +++ b/src/main/kotlin/app/hashers/CommitHasher.kt @@ -14,10 +14,10 @@ import app.model.LocalRepo import app.model.Repo import io.reactivex.Observable import io.reactivex.schedulers.Schedulers -import org.eclipse.jgit.api.Git import java.time.LocalDateTime import java.time.ZoneOffset import java.util.concurrent.TimeUnit +import java.util.LinkedList /** * CommitHasher hashes repository and uploads stats to server. @@ -25,28 +25,64 @@ import java.util.concurrent.TimeUnit class CommitHasher(private val localRepo: LocalRepo, private val serverRepo: Repo = Repo(), private val api: Api, - private val git: Git) { + private val rehashes: LinkedList) { - private fun findFirstOverlappingCommit(): Commit? { - val serverHistoryCommits = serverRepo.commits.toHashSet() - return getCommitsAsObservable() - .skipWhile { commit -> !serverHistoryCommits.contains(commit) } - .blockingFirst(null) + init { + // Delete locally missing commits from server. If found at least one + // common commit then next commits are not deleted because hash of a + // commit calculated including hashes of its parents. + val firstOverlapCommitRehash = findFirstOverlappingCommitRehash() + val deletedCommits = serverRepo.commits + .takeWhile { it.rehash != firstOverlapCommitRehash } + deleteCommitsOnServer(deletedCommits) + } + + private fun findFirstOverlappingCommitRehash(): String? { + + val serverHistoryRehashes = serverRepo.commits + .map { commit -> commit.rehash } + .toHashSet() + return rehashes.firstOrNull { rehash -> + !serverHistoryRehashes.contains(rehash) } + } + + private fun postCommitsToServer(commits: List) { + if (commits.isNotEmpty()) { + api.postCommits(commits) + Logger.debug("Sent ${commits.size} added commits to server") + } } - private fun hashAndSendCommits() { + private fun postFactsToServer(facts: List) { + if (facts.isNotEmpty()) { + api.postFacts(facts) + Logger.debug("Sent ${facts.size} facts to server") + } + } + + private fun deleteCommitsOnServer(commits: List) { + if (commits.isNotEmpty()) { + api.deleteCommits(commits) + Logger.debug("Sent ${commits.size} deleted commits to server") + } + } + + private val emailFilter: (Commit) -> Boolean = { + val email = it.author.email + localRepo.hashAllContributors || (email == localRepo.author.email || + serverRepo.emails.contains(email)) + } + + // Hash added and missing server commits and send them to server. + fun updateFromObservable(observable: Observable) { val lastKnownCommit = serverRepo.commits.lastOrNull() val knownCommits = serverRepo.commits.toHashSet() val factsDayWeek = hashMapOf>() val factsDayTime = hashMapOf>() - + observable .doOnNext { commit -> - Logger.debug("Commit: ${commit.raw?.name ?: ""}: " - + commit.raw?.shortMessage) - commit.repo = serverRepo - // Calculate facts. val author = commit.author val factDayWeek = factsDayWeek[author] ?: Array(7) { 0 } @@ -61,12 +97,17 @@ class CommitHasher(private val localRepo: LocalRepo, factsDayWeek[author] = factDayWeek factsDayTime[author] = factDayTime } - .takeWhile { commit -> // Hash until last known commit. - commit.rehash != lastKnownCommit?.rehash } - .filter { commit -> knownCommits.isEmpty() // Don't hash known. - || !knownCommits.contains(commit) } + .takeWhile { commit -> + // Hash until last known commit. + commit.rehash != lastKnownCommit?.rehash + } + .filter { commit -> + knownCommits.isEmpty() // Don't hash known. + || !knownCommits.contains(commit) + } .filter { commit -> emailFilter(commit) } // Email filtering. - .map { commit -> // Mapping and stats extraction. + .map { commit -> + // Mapping and stats extraction. commit.stats = Extractor().extract(commit.diffs) Logger.debug("Stats: ${commit.stats.size} entries") @@ -74,76 +115,43 @@ class CommitHasher(private val localRepo: LocalRepo, // statistics to CommitStats because not all file extensions // may be supported. commit.numLinesAdded = commit.diffs.fold(0) { total, file -> - total + file.getAllAdded().size } + total + file.getAllAdded().size + } commit.numLinesDeleted = commit.diffs.fold(0) { total, file -> - total + file.getAllDeleted().size } + total + file.getAllDeleted().size + } commit } .observeOn(Schedulers.io()) // Different thread for data sending. .buffer(20, TimeUnit.SECONDS) // Group ready commits by time. - .blockingSubscribe({ commitsBundle -> // OnNext. - postCommitsToServer(commitsBundle) // Send ready commits. - }, { e -> // OnError. - Logger.error("Error while hashing: $e") - }, { // OnComplete. + // Send ready commits. + .doOnNext { commitsBundle -> postCommitsToServer(commitsBundle) } + .doOnComplete { val facts = mutableListOf() factsDayTime.map { (author, list) -> list.forEachIndexed { hour, count -> if (count > 0) { - facts.add(Fact(serverRepo, FactKey.COMMITS_DAY_TIME + - hour, count.toDouble(), author)) + facts.add(Fact(serverRepo, + FactKey.COMMITS_DAY_TIME + hour, + count.toDouble(), author)) } } } factsDayWeek.map { (author, list) -> list.forEachIndexed { day, count -> if (count > 0) { - facts.add(Fact(serverRepo, FactKey.COMMITS_DAY_WEEK + - day, count.toDouble(), author)) + facts.add(Fact(serverRepo, + FactKey.COMMITS_DAY_WEEK + day, + count.toDouble(), author)) } } } - postFactsToServer(facts) - }) - } - - private fun postCommitsToServer(commits: List) { - if (commits.isNotEmpty()) { - api.postCommits(commits) - Logger.debug("Sent ${commits.size} added commits to server") - } - } - - private fun postFactsToServer(facts: List) { - if (facts.isNotEmpty()) { - api.postFacts(facts) - Logger.debug("Sent ${facts.size} facts to server") - } - } - - private fun deleteCommitsOnServer(commits: List) { - if (commits.isNotEmpty()) { - api.deleteCommits(commits) - Logger.debug("Sent ${commits.size} deleted commits to server") - } - } - - private val emailFilter: (Commit) -> Boolean = { - val email = it.author.email - localRepo.hashAllContributors || (email == localRepo.author.email || - serverRepo.emails.contains(email)) - } - - fun update() { - // Delete locally missing commits from server. If found at least one - // common commit then next commits are not deleted because hash of a - // commit calculated including hashes of its parents. - val firstOverlapCommit = findFirstOverlappingCommit() - val deletedCommits = serverRepo.commits - .takeWhile { it.rehash != firstOverlapCommit?.rehash } - deleteCommitsOnServer(deletedCommits) - - // Hash added and missing server commits and send them to server. - hashAndSendCommits() + try { + postFactsToServer(facts) + } catch (e: Exception) { + Logger.error("", e) + } + } + .subscribe() } } diff --git a/src/main/kotlin/app/hashers/RepoHasher.kt b/src/main/kotlin/app/hashers/RepoHasher.kt index d7fca06a..56587a96 100644 --- a/src/main/kotlin/app/hashers/RepoHasher.kt +++ b/src/main/kotlin/app/hashers/RepoHasher.kt @@ -3,21 +3,19 @@ package app.hashers -import app.FactKey import app.Logger import app.api.Api import app.config.Configurator -import app.extractors.Extractor +import app.model.Author import app.model.Commit import app.model.DiffContent import app.model.DiffFile import app.model.DiffRange -import app.model.Fact import app.model.LocalRepo import app.model.Repo import app.utils.RepoHelper import io.reactivex.Observable -import io.reactivex.schedulers.Schedulers +import org.apache.commons.codec.digest.DigestUtils import org.eclipse.jgit.api.Git import org.eclipse.jgit.diff.DiffEntry import org.eclipse.jgit.diff.DiffFormatter @@ -30,10 +28,11 @@ import org.eclipse.jgit.util.io.DisabledOutputStream import java.io.File import java.io.IOException import java.nio.charset.Charset +import java.util.* class RepoHasher(private val localRepo: LocalRepo, private val api: Api, private val configurator: Configurator) { - var repo: Repo = Repo() + var serverRepo: Repo = Repo() init { if (!RepoHelper.isValidRepo(localRepo.path)) { @@ -43,8 +42,14 @@ class RepoHasher(private val localRepo: LocalRepo, private val api: Api, println("Hashing $localRepo...") val git = loadGit(localRepo.path) try { + val (rehashes, authors) = fetchRehashesAndAuthors(git) + localRepo.parseGitConfig(git.repository.config) - initializeRepo(git) + if (localRepo.author.email.isBlank()) { + throw IllegalStateException("Can't load email from Git config") + } + + initServerRepo(rehashes.last) if (!isKnownRepo()) { // Notify server about new contributor and his email. @@ -54,8 +59,14 @@ class RepoHasher(private val localRepo: LocalRepo, private val api: Api, getRepoFromServer() // Hash by all plugins. - CommitHasher(localRepo, repo, api, git).update() - CodeLongevity(localRepo, repo, api, git).update() + val observable = getCommitsObservable(git).publish() + CommitHasher(localRepo, serverRepo, api, rehashes) + .updateFromObservable(observable) + //CodeLongevity(localRepo, serverRepo, api, git) + // .updateFromObservable(observable) + // WRONG: Synchronously wait for all subscribers to complete tasks. + // TODO(anatoly): Somehow wait for subscribers, its for obsrvbl now. + observable.connect() // Confirm hashing completion. postRepoToServer() @@ -81,59 +92,28 @@ class RepoHasher(private val localRepo: LocalRepo, private val api: Api, private fun isKnownRepo(): Boolean { return configurator.getRepos() - .find { it.rehash == repo.rehash } != null + .find { it.rehash == serverRepo.rehash } != null } private fun getRepoFromServer() { - repo = api.getRepo(repo.rehash) - Logger.debug("Received repo from server with ${repo.commits.size} " + - "commits") + serverRepo = api.getRepo(serverRepo.rehash) + Logger.debug("Received repo from server with " + + serverRepo.commits.size + " commits") } private fun postRepoToServer() { - api.postRepo(repo) + api.postRepo(serverRepo) } - private fun initializeRepo(git: Git) { - repo = Repo(userEmail = localRepo.author.email) - repo.initialCommitRehash = getInitialCommitRehash(git) - repo.rehash = RepoHelper.calculateRepoRehash(repo.initialCommitRehash, - localRepo) - } - - private fun getInitialCommitRehash(git: Git): String { - val head: RevCommit = RevWalk(git.repository) - .parseCommit(git.repository.resolve(RepoHelper.MASTER_BRANCH)) - - val revWalk = RevWalk(git.repository) - revWalk.markStart(head) - - val initialCommit = revWalk.last() - return initialCommit.id.name + private fun initServerRepo(initCommitRehash: String) { + serverRepo = Repo(userEmail = localRepo.author.email) + serverRepo.initialCommitRehash = initCommitRehash + serverRepo.rehash = RepoHelper.calculateRepoRehash( + serverRepo.initialCommitRehash, localRepo) } - private fun hashAndSendCommits(git: Git) { - // Commits are combined in pairs, an empty commit concatenated to - // calculate the diff of the initial commit. - Observable.concat(getCommitsObservable(git), Observable.just(Commit())) - .pairWithNext() // Pair commits to get diff. - .map { (new, old) -> // Mapping and stats extraction. - Logger.debug("Commit: ${new.raw?.name ?: ""}: " - + new.raw?.shortMessage) - new.repo = repo - new.diffs = getDiffFiles(git, new, old) - Logger.debug("Diff: ${new.diffs.size} entries") - new - } - .blockingSubscribe({ commit -> // OnNext. - - }, { e -> // OnError. - Logger.error("Error while hashing: $e") - }) - } - - private fun getCommitsObservable(git: Git): Observable = - Observable.create { subscriber -> + private fun getCommitsObservable(git: Git): Observable = Observable + .create { subscriber -> try { val revWalk = RevWalk(git.repository) val commitId = git.repository.resolve(RepoHelper.MASTER_BRANCH) @@ -141,11 +121,24 @@ class RepoHasher(private val localRepo: LocalRepo, private val api: Api, for (revCommit in revWalk) { subscriber.onNext(Commit(revCommit)) } + // Commits are combined in pairs, an empty commit concatenated + // to calculate the diff of the initial commit. + subscriber.onNext(Commit()) } catch (e: Exception) { Logger.error("Commit producing error", e) subscriber.onError(e) } subscriber.onComplete() + } // TODO(anatoly): Rewrite diff calculation in non-weird way. + .pairWithNext() // Pair commits to get diff. + .map { (new, old) -> + // Mapping and stats extraction. + Logger.debug("Commit: ${new.raw?.name ?: ""}: " + + new.raw?.shortMessage) + new.repo = serverRepo + new.diffs = getDiffFiles(git, new, old) + Logger.debug("Diff: ${new.diffs.size} entries") + new } private fun getDiffFiles(git: Git, @@ -179,11 +172,11 @@ class RepoHasher(private val localRepo: LocalRepo, private val api: Api, else -> diff.newPath } DiffFile(path = path, - changeType = diff.changeType, - old = DiffContent(old, edits.map { edit -> - DiffRange(edit.beginA, edit.endA) }), - new = DiffContent(new, edits.map { edit -> - DiffRange(edit.beginB, edit.endB) })) + changeType = diff.changeType, + old = DiffContent(old, edits.map { edit -> + DiffRange(edit.beginA, edit.endA) }), + new = DiffContent(new, edits.map { edit -> + DiffRange(edit.beginB, edit.endB) })) } } } @@ -199,6 +192,33 @@ class RepoHasher(private val localRepo: LocalRepo, private val api: Api, } } + private fun fetchRehashesAndAuthors(git: Git): + Pair, HashMap> { + val head: RevCommit = RevWalk(git.repository) + .parseCommit(git.repository.resolve(RepoHelper.MASTER_BRANCH)) + + val revWalk = RevWalk(git.repository) + revWalk.markStart(head) + + val commitsRehashes = LinkedList() + val contributors = hashMapOf() + + var commit: RevCommit? = revWalk.next() + while (commit != null) { + commitsRehashes.add(DigestUtils.sha256Hex(commit.name)) + if (!contributors.containsKey(commit.authorIdent.emailAddress)) { + val author = Author(commit.authorIdent.name, + commit.authorIdent.emailAddress) + contributors.put(commit.authorIdent.emailAddress, author) + } + commit.disposeBody() + commit = revWalk.next() + } + revWalk.dispose() + + return Pair(commitsRehashes, contributors) + } + private fun Observable.pairWithNext(): Observable> { return this.map { emit -> Pair(emit, emit) } // Accumulate emits by prev-next pair. From 154a0a25232e32d106783a8695406120ebcf6f5d Mon Sep 17 00:00:00 2001 From: Anatoly Kislov Date: Wed, 20 Sep 2017 05:00:16 +0300 Subject: [PATCH 03/10] feat: add common observable for commits with diffs --- src/main/kotlin/app/hashers/CommitHasher.kt | 60 +++++++++++---------- src/main/kotlin/app/hashers/RepoHasher.kt | 4 +- 2 files changed, 34 insertions(+), 30 deletions(-) diff --git a/src/main/kotlin/app/hashers/CommitHasher.kt b/src/main/kotlin/app/hashers/CommitHasher.kt index 0946bfc6..bda3e21e 100644 --- a/src/main/kotlin/app/hashers/CommitHasher.kt +++ b/src/main/kotlin/app/hashers/CommitHasher.kt @@ -81,6 +81,9 @@ class CommitHasher(private val localRepo: LocalRepo, val factsDayWeek = hashMapOf>() val factsDayTime = hashMapOf>() + var skipAfterLastKnown = false + val throwables = mutableListOf() + observable .doOnNext { commit -> // Calculate facts. @@ -96,11 +99,12 @@ class CommitHasher(private val localRepo: LocalRepo, factDayTime[dateTime.hour] += 1 factsDayWeek[author] = factDayWeek factsDayTime[author] = factDayTime + + if (commit.rehash == lastKnownCommit?.rehash) { + skipAfterLastKnown = true + } } - .takeWhile { commit -> - // Hash until last known commit. - commit.rehash != lastKnownCommit?.rehash - } + .filter { !skipAfterLastKnown } // Hash until last known commit. .filter { commit -> knownCommits.isEmpty() // Don't hash known. || !knownCommits.contains(commit) @@ -122,36 +126,36 @@ class CommitHasher(private val localRepo: LocalRepo, } commit } - .observeOn(Schedulers.io()) // Different thread for data sending. .buffer(20, TimeUnit.SECONDS) // Group ready commits by time. - // Send ready commits. - .doOnNext { commitsBundle -> postCommitsToServer(commitsBundle) } - .doOnComplete { - val facts = mutableListOf() - factsDayTime.map { (author, list) -> - list.forEachIndexed { hour, count -> - if (count > 0) { - facts.add(Fact(serverRepo, - FactKey.COMMITS_DAY_TIME + hour, - count.toDouble(), author)) + .subscribe({ commitsBundle -> // OnNext. + postCommitsToServer(commitsBundle) // Send ready commits. + }, { e -> // OnError. + throwables.add(e) // TODO(anatoly): Top-class handling errors. + }, { // OnComplete. + try { + val facts = mutableListOf() + factsDayTime.map { (author, list) -> + list.forEachIndexed { hour, count -> + if (count > 0) { + facts.add(Fact(serverRepo, + FactKey.COMMITS_DAY_TIME + hour, + count.toDouble(), author)) + } } } - } - factsDayWeek.map { (author, list) -> - list.forEachIndexed { day, count -> - if (count > 0) { - facts.add(Fact(serverRepo, - FactKey.COMMITS_DAY_WEEK + day, - count.toDouble(), author)) + factsDayWeek.map { (author, list) -> + list.forEachIndexed { day, count -> + if (count > 0) { + facts.add(Fact(serverRepo, + FactKey.COMMITS_DAY_WEEK + day, + count.toDouble(), author)) + } } } - } - try { postFactsToServer(facts) - } catch (e: Exception) { - Logger.error("", e) + } catch (e: Throwable) { + throwables.add(e) } - } - .subscribe() + }) } } diff --git a/src/main/kotlin/app/hashers/RepoHasher.kt b/src/main/kotlin/app/hashers/RepoHasher.kt index 56587a96..7331cb41 100644 --- a/src/main/kotlin/app/hashers/RepoHasher.kt +++ b/src/main/kotlin/app/hashers/RepoHasher.kt @@ -60,12 +60,12 @@ class RepoHasher(private val localRepo: LocalRepo, private val api: Api, // Hash by all plugins. val observable = getCommitsObservable(git).publish() + CommitHasher(localRepo, serverRepo, api, rehashes) .updateFromObservable(observable) //CodeLongevity(localRepo, serverRepo, api, git) // .updateFromObservable(observable) - // WRONG: Synchronously wait for all subscribers to complete tasks. - // TODO(anatoly): Somehow wait for subscribers, its for obsrvbl now. + // Synchronously wait for all subscribers to complete tasks. observable.connect() // Confirm hashing completion. From 439a2f3495bfe8fe2eb2d48f3152dc262ded6f46 Mon Sep 17 00:00:00 2001 From: Anatoly Kislov Date: Wed, 20 Sep 2017 15:28:22 +0300 Subject: [PATCH 04/10] feat: move fact logic to FactHasher --- src/main/kotlin/app/hashers/CommitHasher.kt | 59 +-------------- src/main/kotlin/app/hashers/FactHasher.kt | 82 +++++++++++++++++++++ src/main/kotlin/app/hashers/RepoHasher.kt | 2 + 3 files changed, 88 insertions(+), 55 deletions(-) create mode 100644 src/main/kotlin/app/hashers/FactHasher.kt diff --git a/src/main/kotlin/app/hashers/CommitHasher.kt b/src/main/kotlin/app/hashers/CommitHasher.kt index bda3e21e..7893365b 100644 --- a/src/main/kotlin/app/hashers/CommitHasher.kt +++ b/src/main/kotlin/app/hashers/CommitHasher.kt @@ -53,13 +53,6 @@ class CommitHasher(private val localRepo: LocalRepo, } } - private fun postFactsToServer(facts: List) { - if (facts.isNotEmpty()) { - api.postFacts(facts) - Logger.debug("Sent ${facts.size} facts to server") - } - } - private fun deleteCommitsOnServer(commits: List) { if (commits.isNotEmpty()) { api.deleteCommits(commits) @@ -81,33 +74,14 @@ class CommitHasher(private val localRepo: LocalRepo, val factsDayWeek = hashMapOf>() val factsDayTime = hashMapOf>() - var skipAfterLastKnown = false val throwables = mutableListOf() observable - .doOnNext { commit -> - // Calculate facts. - val author = commit.author - val factDayWeek = factsDayWeek[author] ?: Array(7) { 0 } - val factDayTime = factsDayTime[author] ?: Array(24) { 0 } - val timestamp = commit.dateTimestamp - val dateTime = LocalDateTime.ofEpochSecond(timestamp, 0, - ZoneOffset.ofTotalSeconds(commit.dateTimeZoneOffset * 60)) - // The value is numbered from 1 (Monday) to 7 (Sunday). - factDayWeek[dateTime.dayOfWeek.value - 1] += 1 - // Hour from 0 to 23. - factDayTime[dateTime.hour] += 1 - factsDayWeek[author] = factDayWeek - factsDayTime[author] = factDayTime - - if (commit.rehash == lastKnownCommit?.rehash) { - skipAfterLastKnown = true - } + .takeWhile { new -> // Hash until last known commit. + new.rehash != lastKnownCommit?.rehash } - .filter { !skipAfterLastKnown } // Hash until last known commit. - .filter { commit -> - knownCommits.isEmpty() // Don't hash known. - || !knownCommits.contains(commit) + .filter { commit -> // Don't hash known. + knownCommits.isEmpty() || !knownCommits.contains(commit) } .filter { commit -> emailFilter(commit) } // Email filtering. .map { commit -> @@ -131,31 +105,6 @@ class CommitHasher(private val localRepo: LocalRepo, postCommitsToServer(commitsBundle) // Send ready commits. }, { e -> // OnError. throwables.add(e) // TODO(anatoly): Top-class handling errors. - }, { // OnComplete. - try { - val facts = mutableListOf() - factsDayTime.map { (author, list) -> - list.forEachIndexed { hour, count -> - if (count > 0) { - facts.add(Fact(serverRepo, - FactKey.COMMITS_DAY_TIME + hour, - count.toDouble(), author)) - } - } - } - factsDayWeek.map { (author, list) -> - list.forEachIndexed { day, count -> - if (count > 0) { - facts.add(Fact(serverRepo, - FactKey.COMMITS_DAY_WEEK + day, - count.toDouble(), author)) - } - } - } - postFactsToServer(facts) - } catch (e: Throwable) { - throwables.add(e) - } }) } } diff --git a/src/main/kotlin/app/hashers/FactHasher.kt b/src/main/kotlin/app/hashers/FactHasher.kt new file mode 100644 index 00000000..08bd3c57 --- /dev/null +++ b/src/main/kotlin/app/hashers/FactHasher.kt @@ -0,0 +1,82 @@ +package app.hashers + +import app.FactKey +import app.Logger +import app.api.Api +import app.extractors.Extractor +import app.model.Author +import app.model.Commit +import app.model.Fact +import app.model.LocalRepo +import app.model.Repo +import io.reactivex.Observable +import java.time.LocalDateTime +import java.time.ZoneOffset +import java.util.* +import java.util.concurrent.TimeUnit + +/** + * CommitHasher hashes repository and uploads stats to server. + */ +class FactHasher(private val localRepo: LocalRepo, + private val serverRepo: Repo = Repo(), + private val api: Api) { + + private fun postFactsToServer(facts: List) { + if (facts.isNotEmpty()) { + api.postFacts(facts) + Logger.debug("Sent ${facts.size} facts to server") + } + } + + fun updateFromObservable(observable: Observable) { + val factsDayWeek = hashMapOf>() + val factsDayTime = hashMapOf>() + + val throwables = mutableListOf() + + observable + .subscribe({ commit -> // OnNext. + // Calculate facts. + val author = commit.author + val factDayWeek = factsDayWeek[author] ?: Array(7) { 0 } + val factDayTime = factsDayTime[author] ?: Array(24) { 0 } + val timestamp = commit.dateTimestamp + val dateTime = LocalDateTime.ofEpochSecond(timestamp, 0, + ZoneOffset.ofTotalSeconds(commit.dateTimeZoneOffset * 60)) + // The value is numbered from 1 (Monday) to 7 (Sunday). + factDayWeek[dateTime.dayOfWeek.value - 1] += 1 + // Hour from 0 to 23. + factDayTime[dateTime.hour] += 1 + factsDayWeek[author] = factDayWeek + factsDayTime[author] = factDayTime + }, { e -> // OnError. + throwables.add(e) // TODO(anatoly): Top-class handling errors. + }, { // OnComplete. + try { + val facts = mutableListOf() + factsDayTime.map { (author, list) -> + list.forEachIndexed { hour, count -> + if (count > 0) { + facts.add(Fact(serverRepo, + FactKey.COMMITS_DAY_TIME + hour, + count.toDouble(), author)) + } + } + } + factsDayWeek.map { (author, list) -> + list.forEachIndexed { day, count -> + if (count > 0) { + facts.add(Fact(serverRepo, + FactKey.COMMITS_DAY_WEEK + day, + count.toDouble(), author)) + } + } + } + postFactsToServer(facts) + } catch (e: Throwable) { + throwables.add(e) + } + }) + } +} diff --git a/src/main/kotlin/app/hashers/RepoHasher.kt b/src/main/kotlin/app/hashers/RepoHasher.kt index 7331cb41..de7ac292 100644 --- a/src/main/kotlin/app/hashers/RepoHasher.kt +++ b/src/main/kotlin/app/hashers/RepoHasher.kt @@ -63,6 +63,8 @@ class RepoHasher(private val localRepo: LocalRepo, private val api: Api, CommitHasher(localRepo, serverRepo, api, rehashes) .updateFromObservable(observable) + FactHasher(localRepo, serverRepo, api) + .updateFromObservable(observable) //CodeLongevity(localRepo, serverRepo, api, git) // .updateFromObservable(observable) // Synchronously wait for all subscribers to complete tasks. From 3540e558a714daa287c41d9619791dc74adc1361 Mon Sep 17 00:00:00 2001 From: Anatoly Kislov Date: Wed, 20 Sep 2017 15:29:52 +0300 Subject: [PATCH 05/10] chore: remove excess imports, add author --- src/main/kotlin/app/hashers/CommitHasher.kt | 5 ----- src/main/kotlin/app/hashers/FactHasher.kt | 6 +++--- 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/src/main/kotlin/app/hashers/CommitHasher.kt b/src/main/kotlin/app/hashers/CommitHasher.kt index 7893365b..d7869af4 100644 --- a/src/main/kotlin/app/hashers/CommitHasher.kt +++ b/src/main/kotlin/app/hashers/CommitHasher.kt @@ -3,19 +3,14 @@ package app.hashers -import app.FactKey import app.Logger import app.api.Api import app.extractors.Extractor import app.model.Author import app.model.Commit -import app.model.Fact import app.model.LocalRepo import app.model.Repo import io.reactivex.Observable -import io.reactivex.schedulers.Schedulers -import java.time.LocalDateTime -import java.time.ZoneOffset import java.util.concurrent.TimeUnit import java.util.LinkedList diff --git a/src/main/kotlin/app/hashers/FactHasher.kt b/src/main/kotlin/app/hashers/FactHasher.kt index 08bd3c57..04cf2426 100644 --- a/src/main/kotlin/app/hashers/FactHasher.kt +++ b/src/main/kotlin/app/hashers/FactHasher.kt @@ -1,9 +1,11 @@ +// Copyright 2017 Sourcerer Inc. All Rights Reserved. +// Author: Anatoly Kislov (anatoly@sourcerer.io) + package app.hashers import app.FactKey import app.Logger import app.api.Api -import app.extractors.Extractor import app.model.Author import app.model.Commit import app.model.Fact @@ -12,8 +14,6 @@ import app.model.Repo import io.reactivex.Observable import java.time.LocalDateTime import java.time.ZoneOffset -import java.util.* -import java.util.concurrent.TimeUnit /** * CommitHasher hashes repository and uploads stats to server. From f405038de8e8c97fe903a1f487b3ad192c68a11a Mon Sep 17 00:00:00 2001 From: Anatoly Kislov Date: Wed, 20 Sep 2017 16:03:14 +0300 Subject: [PATCH 06/10] fix: pinch --- src/main/kotlin/app/hashers/CommitHasher.kt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/kotlin/app/hashers/CommitHasher.kt b/src/main/kotlin/app/hashers/CommitHasher.kt index d7869af4..7e91d2f9 100644 --- a/src/main/kotlin/app/hashers/CommitHasher.kt +++ b/src/main/kotlin/app/hashers/CommitHasher.kt @@ -38,7 +38,8 @@ class CommitHasher(private val localRepo: LocalRepo, .map { commit -> commit.rehash } .toHashSet() return rehashes.firstOrNull { rehash -> - !serverHistoryRehashes.contains(rehash) } + serverHistoryRehashes.contains(rehash) + } } private fun postCommitsToServer(commits: List) { From 894b0c69ecf022a9e046856044339f4c8a50dc1c Mon Sep 17 00:00:00 2001 From: Anatoly Kislov Date: Wed, 20 Sep 2017 16:42:59 +0300 Subject: [PATCH 07/10] feat: update tests, add FactHasherTest, add CommitObservable, minor fixes --- src/main/kotlin/app/hashers/CommitHasher.kt | 6 +- .../kotlin/app/hashers/CommitObservable.kt | 111 ++++++++++++++++++ src/main/kotlin/app/hashers/RepoHasher.kt | 95 +-------------- .../test/tests/hashers/CommitHasherTest.kt | 94 +++------------ .../test/tests/hashers/FactHasherTest.kt | 101 ++++++++++++++++ 5 files changed, 233 insertions(+), 174 deletions(-) create mode 100644 src/main/kotlin/app/hashers/CommitObservable.kt create mode 100644 src/test/kotlin/test/tests/hashers/FactHasherTest.kt diff --git a/src/main/kotlin/app/hashers/CommitHasher.kt b/src/main/kotlin/app/hashers/CommitHasher.kt index 7e91d2f9..0dff2274 100644 --- a/src/main/kotlin/app/hashers/CommitHasher.kt +++ b/src/main/kotlin/app/hashers/CommitHasher.kt @@ -6,7 +6,6 @@ package app.hashers import app.Logger import app.api.Api import app.extractors.Extractor -import app.model.Author import app.model.Commit import app.model.LocalRepo import app.model.Repo @@ -20,7 +19,7 @@ import java.util.LinkedList class CommitHasher(private val localRepo: LocalRepo, private val serverRepo: Repo = Repo(), private val api: Api, - private val rehashes: LinkedList) { + private val rehashes: List) { init { // Delete locally missing commits from server. If found at least one @@ -67,9 +66,6 @@ class CommitHasher(private val localRepo: LocalRepo, val lastKnownCommit = serverRepo.commits.lastOrNull() val knownCommits = serverRepo.commits.toHashSet() - val factsDayWeek = hashMapOf>() - val factsDayTime = hashMapOf>() - val throwables = mutableListOf() observable diff --git a/src/main/kotlin/app/hashers/CommitObservable.kt b/src/main/kotlin/app/hashers/CommitObservable.kt new file mode 100644 index 00000000..9e0c261a --- /dev/null +++ b/src/main/kotlin/app/hashers/CommitObservable.kt @@ -0,0 +1,111 @@ +package app.hashers + +import app.Logger +import app.model.Commit +import app.model.DiffContent +import app.model.DiffFile +import app.model.DiffRange +import app.model.Repo +import app.utils.RepoHelper +import io.reactivex.Observable +import org.eclipse.jgit.api.Git +import org.eclipse.jgit.diff.DiffEntry +import org.eclipse.jgit.diff.DiffFormatter +import org.eclipse.jgit.diff.RawText +import org.eclipse.jgit.errors.MissingObjectException +import org.eclipse.jgit.lib.ObjectId +import org.eclipse.jgit.revwalk.RevCommit +import org.eclipse.jgit.revwalk.RevWalk +import org.eclipse.jgit.util.io.DisabledOutputStream +import java.nio.charset.Charset + +object CommitObservable { + fun getObservable(git: Git, repo: Repo) = Observable + .create { subscriber -> + try { + val revWalk = RevWalk(git.repository) + val commitId = git.repository.resolve(RepoHelper.MASTER_BRANCH) + revWalk.markStart(revWalk.parseCommit(commitId)) + for (revCommit in revWalk) { + subscriber.onNext(Commit(revCommit)) + } + // Commits are combined in pairs, an empty commit concatenated + // to calculate the diff of the initial commit. + subscriber.onNext(Commit()) + } catch (e: Exception) { + Logger.error("Commit producing error", e) + subscriber.onError(e) + } + subscriber.onComplete() + } // TODO(anatoly): Rewrite diff calculation in non-weird way. + .pairWithNext() // Pair commits to get diff. + .map { (new, old) -> + // Mapping and stats extraction. + Logger.debug("Commit: ${new.raw?.name ?: ""}: " + + new.raw?.shortMessage) + new.diffs = getDiffFiles(git, new, old) + Logger.debug("Diff: ${new.diffs.size} entries") + new.repo = repo + new + } + + private fun getDiffFiles(git: Git, + commitNew: Commit, + commitOld: Commit): List { + val revCommitNew: RevCommit? = commitNew.raw + val revCommitOld: RevCommit? = commitOld.raw + + return DiffFormatter(DisabledOutputStream.INSTANCE).use { formatter -> + formatter.setRepository(git.repository) + formatter.setDetectRenames(true) + formatter.scan(revCommitOld?.tree, revCommitNew?.tree) + // RENAME change type doesn't change file content. + .filter { it.changeType != DiffEntry.ChangeType.RENAME } + // Skip binary files. + .filter { + val id = if (it.changeType == DiffEntry.ChangeType.DELETE) { + it.oldId.toObjectId() + } else { + it.newId.toObjectId() + } + !RawText.isBinary(git.repository.open(id).openStream()) + } + .map { diff -> + val new = getContentByObjectId(git, diff.newId.toObjectId()) + val old = getContentByObjectId(git, diff.oldId.toObjectId()) + + val edits = formatter.toFileHeader(diff).toEditList() + val path = when (diff.changeType) { + DiffEntry.ChangeType.DELETE -> diff.oldPath + else -> diff.newPath + } + DiffFile(path = path, + changeType = diff.changeType, + old = DiffContent(old, edits.map { edit -> + DiffRange(edit.beginA, edit.endA) }), + new = DiffContent(new, edits.map { edit -> + DiffRange(edit.beginB, edit.endB) })) + } + } + } + + private fun getContentByObjectId(git: Git, + objectId: ObjectId): List { + return try { + git.repository + .open(objectId).bytes.toString(Charset.defaultCharset()) + .split('\n') + } catch (e: MissingObjectException) { + listOf() + } + } + + private fun Observable.pairWithNext(): Observable> { + return this.map { emit -> Pair(emit, emit) } + // Accumulate emits by prev-next pair. + .scan { pairAccumulated, pairNext -> + Pair(pairAccumulated.second, pairNext.second) + } + .skip(1) // Skip initial not paired emit. + } +} diff --git a/src/main/kotlin/app/hashers/RepoHasher.kt b/src/main/kotlin/app/hashers/RepoHasher.kt index de7ac292..986ece1a 100644 --- a/src/main/kotlin/app/hashers/RepoHasher.kt +++ b/src/main/kotlin/app/hashers/RepoHasher.kt @@ -59,15 +59,15 @@ class RepoHasher(private val localRepo: LocalRepo, private val api: Api, getRepoFromServer() // Hash by all plugins. - val observable = getCommitsObservable(git).publish() - + val observable = CommitObservable.getObservable(git, serverRepo) + .publish() CommitHasher(localRepo, serverRepo, api, rehashes) .updateFromObservable(observable) FactHasher(localRepo, serverRepo, api) .updateFromObservable(observable) //CodeLongevity(localRepo, serverRepo, api, git) // .updateFromObservable(observable) - // Synchronously wait for all subscribers to complete tasks. + // Start and synchronously wait until all subscribers complete. observable.connect() // Confirm hashing completion. @@ -114,86 +114,6 @@ class RepoHasher(private val localRepo: LocalRepo, private val api: Api, serverRepo.initialCommitRehash, localRepo) } - private fun getCommitsObservable(git: Git): Observable = Observable - .create { subscriber -> - try { - val revWalk = RevWalk(git.repository) - val commitId = git.repository.resolve(RepoHelper.MASTER_BRANCH) - revWalk.markStart(revWalk.parseCommit(commitId)) - for (revCommit in revWalk) { - subscriber.onNext(Commit(revCommit)) - } - // Commits are combined in pairs, an empty commit concatenated - // to calculate the diff of the initial commit. - subscriber.onNext(Commit()) - } catch (e: Exception) { - Logger.error("Commit producing error", e) - subscriber.onError(e) - } - subscriber.onComplete() - } // TODO(anatoly): Rewrite diff calculation in non-weird way. - .pairWithNext() // Pair commits to get diff. - .map { (new, old) -> - // Mapping and stats extraction. - Logger.debug("Commit: ${new.raw?.name ?: ""}: " - + new.raw?.shortMessage) - new.repo = serverRepo - new.diffs = getDiffFiles(git, new, old) - Logger.debug("Diff: ${new.diffs.size} entries") - new - } - - private fun getDiffFiles(git: Git, - commitNew: Commit, - commitOld: Commit): List { - val revCommitNew:RevCommit? = commitNew.raw - val revCommitOld:RevCommit? = commitOld.raw - - return DiffFormatter(DisabledOutputStream.INSTANCE).use { formatter -> - formatter.setRepository(git.repository) - formatter.setDetectRenames(true) - formatter.scan(revCommitOld?.tree, revCommitNew?.tree) - // RENAME change type doesn't change file content. - .filter { it.changeType != DiffEntry.ChangeType.RENAME } - // Skip binary files. - .filter { - val id = if (it.changeType == DiffEntry.ChangeType.DELETE) { - it.oldId.toObjectId() - } else { - it.newId.toObjectId() - } - !RawText.isBinary(git.repository.open(id).openStream()) - } - .map { diff -> - val new = getContentByObjectId(git, diff.newId.toObjectId()) - val old = getContentByObjectId(git, diff.oldId.toObjectId()) - - val edits = formatter.toFileHeader(diff).toEditList() - val path = when (diff.changeType) { - DiffEntry.ChangeType.DELETE -> diff.oldPath - else -> diff.newPath - } - DiffFile(path = path, - changeType = diff.changeType, - old = DiffContent(old, edits.map { edit -> - DiffRange(edit.beginA, edit.endA) }), - new = DiffContent(new, edits.map { edit -> - DiffRange(edit.beginB, edit.endB) })) - } - } - } - - private fun getContentByObjectId(git: Git, - objectId: ObjectId): List { - return try { - git.repository - .open(objectId).bytes.toString(Charset.defaultCharset()) - .split('\n') - } catch (e: MissingObjectException) { - listOf() - } - } - private fun fetchRehashesAndAuthors(git: Git): Pair, HashMap> { val head: RevCommit = RevWalk(git.repository) @@ -220,13 +140,4 @@ class RepoHasher(private val localRepo: LocalRepo, private val api: Api, return Pair(commitsRehashes, contributors) } - - private fun Observable.pairWithNext(): Observable> { - return this.map { emit -> Pair(emit, emit) } - // Accumulate emits by prev-next pair. - .scan { pairAccumulated, pairNext -> - Pair(pairAccumulated.second, pairNext.second) - } - .skip(1) // Skip initial not paired emit. - } } diff --git a/src/test/kotlin/test/tests/hashers/CommitHasherTest.kt b/src/test/kotlin/test/tests/hashers/CommitHasherTest.kt index 0e8efdfd..a74c83d6 100644 --- a/src/test/kotlin/test/tests/hashers/CommitHasherTest.kt +++ b/src/test/kotlin/test/tests/hashers/CommitHasherTest.kt @@ -7,6 +7,8 @@ package test.tests.hashers import app.FactKey import app.api.MockApi import app.hashers.CommitHasher +import app.hashers.CommitObservable +import app.hashers.FactHasher import app.model.* import app.utils.RepoHelper import org.eclipse.jgit.api.Git @@ -60,19 +62,13 @@ class CommitHasherTest : Spek({ return lastCommit } - fun createDate(year: Int = 2017, month: Int = 1, day: Int = 1, - hour: Int = 0, minute: Int = 0, seconds: Int = 0): Date { - val cal = Calendar.getInstance() - // Month in calendar is 0-based. - cal.set(year, month - 1, day, hour, minute, seconds) - return cal.time - } - given("repo with initial commit and no history") { repo.commits = listOf() val mockApi = MockApi(mockRepo = repo) - CommitHasher(localRepo, repo, mockApi, gitHasher).update() + val observable = CommitObservable.getObservable(gitHasher, repo) + CommitHasher(localRepo, repo, mockApi, repo.commits.map {it.rehash}) + .updateFromObservable(observable) it("send added commits") { assertEquals(1, mockApi.receivedAddedCommits.size) @@ -87,7 +83,9 @@ class CommitHasherTest : Spek({ repo.commits = listOf(getLastCommit(git)) val mockApi = MockApi(mockRepo = repo) - CommitHasher(localRepo, repo, mockApi, gitHasher).update() + val observable = CommitObservable.getObservable(gitHasher, repo) + CommitHasher(localRepo, repo, mockApi, repo.commits.map {it.rehash}) + .updateFromObservable(observable) it("doesn't send added commits") { assertEquals(0, mockApi.receivedAddedCommits.size) @@ -105,7 +103,9 @@ class CommitHasherTest : Spek({ val revCommit = git.commit().setMessage("Second commit.").call() val addedCommit = Commit(revCommit) - CommitHasher(localRepo, repo, mockApi, gitHasher).update() + val observable = CommitObservable.getObservable(gitHasher, repo) + CommitHasher(localRepo, repo, mockApi, repo.commits.map {it.rehash}) + .updateFromObservable(observable) it("doesn't send deleted commits") { assertEquals(0, mockApi.receivedDeletedCommits.size) @@ -139,7 +139,9 @@ class CommitHasherTest : Spek({ val revCommit = git.commit().setMessage(message).call() authorCommits.add(Commit(revCommit)) } - CommitHasher(localRepo, repo, mockApi, gitHasher).update() + val observable = CommitObservable.getObservable(gitHasher, repo) + CommitHasher(localRepo, repo, mockApi, repo.commits.map {it.rehash}) + .updateFromObservable(observable) it("posts five commits as added") { assertEquals(5, mockApi.receivedAddedCommits.size) @@ -199,7 +201,9 @@ class CommitHasherTest : Spek({ val removedCommit = addedCommits.removeAt(1) repo.commits = addedCommits.toList().asReversed() - CommitHasher(localRepo, repo, mockApi, gitHasher).update() + val observable = CommitObservable.getObservable(gitHasher, repo) + CommitHasher(localRepo, repo, mockApi, repo.commits.map {it.rehash}) + .updateFromObservable(observable) it("adds posts one commit as added and received commit is lost one") { assertEquals(1, mockApi.receivedAddedCommits.size) @@ -211,69 +215,5 @@ class CommitHasherTest : Spek({ } } - given("test of commits date facts") { - val testRepoPath = "../testrepo1" - val testRepo = TestRepo(testRepoPath) - val author1 = Author("Test1", "test@domain.com") - val author2 = Author("Test2", "test@domain.com") - - val repo = Repo(rehash = "rehash", commits = listOf()) - val mockApi = MockApi(mockRepo = repo) - val facts = mockApi.receivedFacts - - afterEachTest { - facts.clear() - } - - it("send two facts") { - testRepo.createFile("test1.txt", listOf("line1", "line2")) - testRepo.commit(message = "initial commit", - author = author1, - // Sunday. - date = createDate(year = 2017, month = 1, day = 1, - hour = 13, minute = 0, seconds = 0)) - - CommitHasher(localRepo, repo, mockApi, testRepo.git).update() - - assertEquals(2, facts.size) - assertTrue(facts.contains(Fact(repo, FactKey.COMMITS_DAY_TIME + 13, - 1.0, author1))) - assertTrue(facts.contains(Fact(repo, FactKey.COMMITS_DAY_WEEK + 6, - 1.0, author1))) - } - - it("send more facts") { - testRepo.createFile("test2.txt", listOf("line1", "line2")) - testRepo.commit(message = "second commit", - author = author2, - // Monday. - date = createDate(year=2017, month = 1, day = 2, - hour = 18, minute = 0, seconds = 0)) - - testRepo.createFile("test3.txt", listOf("line1", "line2")) - testRepo.commit(message = "third commit", - author = author1, - // Monday. - date = createDate(month = 1, day = 2, - hour = 13, minute = 0, seconds = 0)) - - CommitHasher(localRepo, repo, mockApi, testRepo.git).update() - - assertEquals(5, facts.size) - assertTrue(facts.contains(Fact(repo, FactKey.COMMITS_DAY_TIME + 18, - 1.0, author2))) - assertTrue(facts.contains(Fact(repo, FactKey.COMMITS_DAY_WEEK + 0, - 1.0, author2))) - assertTrue(facts.contains(Fact(repo, FactKey.COMMITS_DAY_TIME + 13, - 2.0, author1))) - assertTrue(facts.contains(Fact(repo, FactKey.COMMITS_DAY_WEEK + 0, - 1.0, author1))) - } - - afterGroup { - testRepo.destroy() - } - } - Runtime.getRuntime().exec("src/test/delete_repo.sh").waitFor() }) diff --git a/src/test/kotlin/test/tests/hashers/FactHasherTest.kt b/src/test/kotlin/test/tests/hashers/FactHasherTest.kt new file mode 100644 index 00000000..f161ccb1 --- /dev/null +++ b/src/test/kotlin/test/tests/hashers/FactHasherTest.kt @@ -0,0 +1,101 @@ +// Copyright 2017 Sourcerer Inc. All Rights Reserved. +// Author: Anatoly Kislov (anatoly@sourcerer.io) + +package test.tests.hashers + +import app.FactKey +import app.api.MockApi +import app.hashers.CommitObservable +import app.hashers.FactHasher +import app.model.Author +import app.model.Fact +import app.model.LocalRepo +import app.model.Repo +import org.jetbrains.spek.api.Spek +import org.jetbrains.spek.api.dsl.given +import org.jetbrains.spek.api.dsl.it +import test.utils.TestRepo +import java.util.* +import kotlin.test.assertEquals +import kotlin.test.assertTrue + +class FactHasherTest : Spek({ + val userName = "Contributor" + val userEmail = "test@domain.com" + + val repoPath = "../testrepo-fact-hasher-" + val localRepo = LocalRepo(repoPath) + localRepo.author = Author(userName, userEmail) + + given("test of commits date facts") { + val testRepo = TestRepo(repoPath + "date-facts") + val author1 = Author("Test1", "test@domain.com") + val author2 = Author("Test2", "test@domain.com") + + val repo = Repo(rehash = "rehash", commits = listOf()) + val mockApi = MockApi(mockRepo = repo) + val facts = mockApi.receivedFacts + + fun createDate(year: Int = 2017, month: Int = 1, day: Int = 1, + hour: Int = 0, minute: Int = 0, seconds: Int = 0): Date { + val cal = Calendar.getInstance() + // Month in calendar is 0-based. + cal.set(year, month - 1, day, hour, minute, seconds) + return cal.time + } + + afterEachTest { + facts.clear() + } + + it("send two facts") { + testRepo.createFile("test1.txt", listOf("line1", "line2")) + testRepo.commit(message = "initial commit", + author = author1, + date = createDate(year = 2017, month = 1, day = 1, // Sunday. + hour = 13, minute = 0, seconds = 0)) + + val observable = CommitObservable.getObservable(testRepo.git, repo) + FactHasher(localRepo, repo, mockApi) + .updateFromObservable(observable) + + assertEquals(2, facts.size) + assertTrue(facts.contains(Fact(repo, FactKey.COMMITS_DAY_TIME + 13, + 1.0, author1))) + assertTrue(facts.contains(Fact(repo, FactKey.COMMITS_DAY_WEEK + 6, + 1.0, author1))) + } + + it("send more facts") { + testRepo.createFile("test2.txt", listOf("line1", "line2")) + testRepo.commit(message = "second commit", + author = author2, + date = createDate(year=2017, month = 1, day = 2, // Monday. + hour = 18, minute = 0, seconds = 0)) + + testRepo.createFile("test3.txt", listOf("line1", "line2")) + testRepo.commit(message = "third commit", + author = author1, + date = createDate(year=2017, month = 1, day = 2, // Monday. + hour = 13, minute = 0, seconds = 0)) + + val observable = CommitObservable.getObservable(testRepo.git, repo) + FactHasher(localRepo, repo, mockApi) + .updateFromObservable(observable) + + assertEquals(5, facts.size) + assertTrue(facts.contains(Fact(repo, FactKey.COMMITS_DAY_TIME + 18, + 1.0, author2))) + assertTrue(facts.contains(Fact(repo, FactKey.COMMITS_DAY_WEEK + 0, + 1.0, author2))) + assertTrue(facts.contains(Fact(repo, FactKey.COMMITS_DAY_TIME + 13, + 2.0, author1))) + assertTrue(facts.contains(Fact(repo, FactKey.COMMITS_DAY_WEEK + 0, + 1.0, author1))) + } + + afterGroup { + testRepo.destroy() + } + } +}) From 3cf3ba44326e71542d1a33e7e60992644700a6d8 Mon Sep 17 00:00:00 2001 From: Anatoly Kislov Date: Wed, 20 Sep 2017 16:47:03 +0300 Subject: [PATCH 08/10] chore: author --- src/main/kotlin/app/hashers/CommitObservable.kt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/main/kotlin/app/hashers/CommitObservable.kt b/src/main/kotlin/app/hashers/CommitObservable.kt index 9e0c261a..b42921a9 100644 --- a/src/main/kotlin/app/hashers/CommitObservable.kt +++ b/src/main/kotlin/app/hashers/CommitObservable.kt @@ -1,3 +1,6 @@ +// Copyright 2017 Sourcerer Inc. All Rights Reserved. +// Author: Anatoly Kislov (anatoly@sourcerer.io) + package app.hashers import app.Logger From 457468dae15953d6220312f006de5429fef75844 Mon Sep 17 00:00:00 2001 From: Anatoly Kislov Date: Wed, 20 Sep 2017 17:08:32 +0300 Subject: [PATCH 09/10] chore: remove excess imports --- src/main/kotlin/app/hashers/CommitHasher.kt | 1 - src/main/kotlin/app/hashers/RepoHasher.kt | 12 ------------ 2 files changed, 13 deletions(-) diff --git a/src/main/kotlin/app/hashers/CommitHasher.kt b/src/main/kotlin/app/hashers/CommitHasher.kt index 0dff2274..457ffaed 100644 --- a/src/main/kotlin/app/hashers/CommitHasher.kt +++ b/src/main/kotlin/app/hashers/CommitHasher.kt @@ -11,7 +11,6 @@ import app.model.LocalRepo import app.model.Repo import io.reactivex.Observable import java.util.concurrent.TimeUnit -import java.util.LinkedList /** * CommitHasher hashes repository and uploads stats to server. diff --git a/src/main/kotlin/app/hashers/RepoHasher.kt b/src/main/kotlin/app/hashers/RepoHasher.kt index 986ece1a..b69f794d 100644 --- a/src/main/kotlin/app/hashers/RepoHasher.kt +++ b/src/main/kotlin/app/hashers/RepoHasher.kt @@ -7,27 +7,15 @@ import app.Logger import app.api.Api import app.config.Configurator import app.model.Author -import app.model.Commit -import app.model.DiffContent -import app.model.DiffFile -import app.model.DiffRange import app.model.LocalRepo import app.model.Repo import app.utils.RepoHelper -import io.reactivex.Observable import org.apache.commons.codec.digest.DigestUtils import org.eclipse.jgit.api.Git -import org.eclipse.jgit.diff.DiffEntry -import org.eclipse.jgit.diff.DiffFormatter -import org.eclipse.jgit.diff.RawText -import org.eclipse.jgit.errors.MissingObjectException -import org.eclipse.jgit.lib.ObjectId import org.eclipse.jgit.revwalk.RevCommit import org.eclipse.jgit.revwalk.RevWalk -import org.eclipse.jgit.util.io.DisabledOutputStream import java.io.File import java.io.IOException -import java.nio.charset.Charset import java.util.* class RepoHasher(private val localRepo: LocalRepo, private val api: Api, From 3a9cc56ea4ae35535242f39559fbe423aac3272f Mon Sep 17 00:00:00 2001 From: Anatoly Kislov Date: Mon, 25 Sep 2017 18:05:04 +0300 Subject: [PATCH 10/10] wip: fix PR --- .../{CommitObservable.kt => CommitCrawler.kt} | 12 +++++++----- src/main/kotlin/app/hashers/CommitHasher.kt | 4 ++-- src/main/kotlin/app/hashers/RepoHasher.kt | 7 ++++--- .../test/tests/hashers/CommitHasherTest.kt | 17 ++++++----------- .../kotlin/test/tests/hashers/FactHasherTest.kt | 6 +++--- 5 files changed, 22 insertions(+), 24 deletions(-) rename src/main/kotlin/app/hashers/{CommitObservable.kt => CommitCrawler.kt} (93%) diff --git a/src/main/kotlin/app/hashers/CommitObservable.kt b/src/main/kotlin/app/hashers/CommitCrawler.kt similarity index 93% rename from src/main/kotlin/app/hashers/CommitObservable.kt rename to src/main/kotlin/app/hashers/CommitCrawler.kt index b42921a9..d6dbf329 100644 --- a/src/main/kotlin/app/hashers/CommitObservable.kt +++ b/src/main/kotlin/app/hashers/CommitCrawler.kt @@ -20,9 +20,8 @@ import org.eclipse.jgit.lib.ObjectId import org.eclipse.jgit.revwalk.RevCommit import org.eclipse.jgit.revwalk.RevWalk import org.eclipse.jgit.util.io.DisabledOutputStream -import java.nio.charset.Charset -object CommitObservable { +object CommitCrawler { fun getObservable(git: Git, repo: Repo) = Observable .create { subscriber -> try { @@ -95,9 +94,12 @@ object CommitObservable { private fun getContentByObjectId(git: Git, objectId: ObjectId): List { return try { - git.repository - .open(objectId).bytes.toString(Charset.defaultCharset()) - .split('\n') + val rawText = RawText(git.repository.open(objectId).bytes) + val content = ArrayList(rawText.size()) + for (i in 0..(rawText.size() - 1)) { + content.add(rawText.getString(i)) + } + return content } catch (e: MissingObjectException) { listOf() } diff --git a/src/main/kotlin/app/hashers/CommitHasher.kt b/src/main/kotlin/app/hashers/CommitHasher.kt index 457ffaed..f3a1a05f 100644 --- a/src/main/kotlin/app/hashers/CommitHasher.kt +++ b/src/main/kotlin/app/hashers/CommitHasher.kt @@ -22,8 +22,8 @@ class CommitHasher(private val localRepo: LocalRepo, init { // Delete locally missing commits from server. If found at least one - // common commit then next commits are not deleted because hash of a - // commit calculated including hashes of its parents. + // common commit then preceding commits are not deleted because hash of + // a commit calculated including hashes of its parents. val firstOverlapCommitRehash = findFirstOverlappingCommitRehash() val deletedCommits = serverRepo.commits .takeWhile { it.rehash != firstOverlapCommitRehash } diff --git a/src/main/kotlin/app/hashers/RepoHasher.kt b/src/main/kotlin/app/hashers/RepoHasher.kt index b69f794d..975e2f44 100644 --- a/src/main/kotlin/app/hashers/RepoHasher.kt +++ b/src/main/kotlin/app/hashers/RepoHasher.kt @@ -47,17 +47,18 @@ class RepoHasher(private val localRepo: LocalRepo, private val api: Api, getRepoFromServer() // Hash by all plugins. - val observable = CommitObservable.getObservable(git, serverRepo) + val observable = CommitCrawler.getObservable(git, serverRepo) .publish() CommitHasher(localRepo, serverRepo, api, rehashes) .updateFromObservable(observable) FactHasher(localRepo, serverRepo, api) .updateFromObservable(observable) - //CodeLongevity(localRepo, serverRepo, api, git) - // .updateFromObservable(observable) // Start and synchronously wait until all subscribers complete. observable.connect() + // TODO(anatoly): CodeLongevity hash from observable. + CodeLongevity(localRepo, serverRepo, api, git) + // Confirm hashing completion. postRepoToServer() } diff --git a/src/test/kotlin/test/tests/hashers/CommitHasherTest.kt b/src/test/kotlin/test/tests/hashers/CommitHasherTest.kt index a74c83d6..875d2798 100644 --- a/src/test/kotlin/test/tests/hashers/CommitHasherTest.kt +++ b/src/test/kotlin/test/tests/hashers/CommitHasherTest.kt @@ -4,25 +4,20 @@ package test.tests.hashers -import app.FactKey import app.api.MockApi import app.hashers.CommitHasher -import app.hashers.CommitObservable -import app.hashers.FactHasher +import app.hashers.CommitCrawler import app.model.* import app.utils.RepoHelper import org.eclipse.jgit.api.Git import org.jetbrains.spek.api.Spek import org.jetbrains.spek.api.dsl.given import org.jetbrains.spek.api.dsl.it -import test.utils.TestRepo import java.io.File -import java.util.* import java.util.stream.StreamSupport.stream import kotlin.streams.toList import kotlin.test.assertEquals import kotlin.test.assertNotEquals -import kotlin.test.assertTrue class CommitHasherTest : Spek({ val userName = "Contributor" @@ -66,7 +61,7 @@ class CommitHasherTest : Spek({ repo.commits = listOf() val mockApi = MockApi(mockRepo = repo) - val observable = CommitObservable.getObservable(gitHasher, repo) + val observable = CommitCrawler.getObservable(gitHasher, repo) CommitHasher(localRepo, repo, mockApi, repo.commits.map {it.rehash}) .updateFromObservable(observable) @@ -83,7 +78,7 @@ class CommitHasherTest : Spek({ repo.commits = listOf(getLastCommit(git)) val mockApi = MockApi(mockRepo = repo) - val observable = CommitObservable.getObservable(gitHasher, repo) + val observable = CommitCrawler.getObservable(gitHasher, repo) CommitHasher(localRepo, repo, mockApi, repo.commits.map {it.rehash}) .updateFromObservable(observable) @@ -103,7 +98,7 @@ class CommitHasherTest : Spek({ val revCommit = git.commit().setMessage("Second commit.").call() val addedCommit = Commit(revCommit) - val observable = CommitObservable.getObservable(gitHasher, repo) + val observable = CommitCrawler.getObservable(gitHasher, repo) CommitHasher(localRepo, repo, mockApi, repo.commits.map {it.rehash}) .updateFromObservable(observable) @@ -139,7 +134,7 @@ class CommitHasherTest : Spek({ val revCommit = git.commit().setMessage(message).call() authorCommits.add(Commit(revCommit)) } - val observable = CommitObservable.getObservable(gitHasher, repo) + val observable = CommitCrawler.getObservable(gitHasher, repo) CommitHasher(localRepo, repo, mockApi, repo.commits.map {it.rehash}) .updateFromObservable(observable) @@ -201,7 +196,7 @@ class CommitHasherTest : Spek({ val removedCommit = addedCommits.removeAt(1) repo.commits = addedCommits.toList().asReversed() - val observable = CommitObservable.getObservable(gitHasher, repo) + val observable = CommitCrawler.getObservable(gitHasher, repo) CommitHasher(localRepo, repo, mockApi, repo.commits.map {it.rehash}) .updateFromObservable(observable) diff --git a/src/test/kotlin/test/tests/hashers/FactHasherTest.kt b/src/test/kotlin/test/tests/hashers/FactHasherTest.kt index f161ccb1..3da42935 100644 --- a/src/test/kotlin/test/tests/hashers/FactHasherTest.kt +++ b/src/test/kotlin/test/tests/hashers/FactHasherTest.kt @@ -5,7 +5,7 @@ package test.tests.hashers import app.FactKey import app.api.MockApi -import app.hashers.CommitObservable +import app.hashers.CommitCrawler import app.hashers.FactHasher import app.model.Author import app.model.Fact @@ -55,7 +55,7 @@ class FactHasherTest : Spek({ date = createDate(year = 2017, month = 1, day = 1, // Sunday. hour = 13, minute = 0, seconds = 0)) - val observable = CommitObservable.getObservable(testRepo.git, repo) + val observable = CommitCrawler.getObservable(testRepo.git, repo) FactHasher(localRepo, repo, mockApi) .updateFromObservable(observable) @@ -79,7 +79,7 @@ class FactHasherTest : Spek({ date = createDate(year=2017, month = 1, day = 2, // Monday. hour = 13, minute = 0, seconds = 0)) - val observable = CommitObservable.getObservable(testRepo.git, repo) + val observable = CommitCrawler.getObservable(testRepo.git, repo) FactHasher(localRepo, repo, mockApi) .updateFromObservable(observable)