Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP
Browse files

not fully merged yet, but i need a stopping point

  • Loading branch information...
commit 30ba2284cbd952fa43a11a56a0559b07ab4625eb 1 parent 7c14230
Nick Kallen authored
Showing with 535 additions and 28 deletions.
  1. +10 −0 .git-scala-build/COMMIT_EDITMSG
  2. +1 −0  .git-scala-build/HEAD
  3. +11 −0 .git-scala-build/config
  4. +1 −0  .git-scala-build/description
  5. +15 −0 .git-scala-build/hooks/applypatch-msg
  6. +24 −0 .git-scala-build/hooks/commit-msg
  7. +8 −0 .git-scala-build/hooks/post-commit
  8. +16 −0 .git-scala-build/hooks/post-receive
  9. +8 −0 .git-scala-build/hooks/post-update
  10. +14 −0 .git-scala-build/hooks/pre-applypatch
  11. +70 −0 .git-scala-build/hooks/pre-commit
  12. +150 −0 .git-scala-build/hooks/pre-rebase
  13. +36 −0 .git-scala-build/hooks/prepare-commit-msg
  14. +107 −0 .git-scala-build/hooks/update
  15. BIN  .git-scala-build/index
  16. +6 −0 .git-scala-build/info/exclude
  17. +1 −0  .git-scala-build/logs/HEAD
  18. +1 −0  .git-scala-build/logs/refs/heads/master
  19. +1 −0  .git-scala-build/logs/refs/remotes/origin/HEAD
  20. +1 −0  .git-scala-build/logs/refs/remotes/origin/master
  21. BIN  .git-scala-build/objects/pack/pack-aa56c6a61ae1b1d9ce6497cd8c3dd18d8e775041.idx
  22. BIN  .git-scala-build/objects/pack/pack-aa56c6a61ae1b1d9ce6497cd8c3dd18d8e775041.pack
  23. +1 −0  .git-scala-build/refs/heads/master
  24. +1 −0  .git-scala-build/refs/remotes/origin/HEAD
  25. +1 −0  .git-scala-build/refs/remotes/origin/master
  26. BIN  libs/gizzard-1.0.jar
  27. +0 −11 src/main/scala/com/twitter/rowz/CopyManager.scala
  28. +4 −4 src/main/scala/com/twitter/rowz/Main.scala
  29. +3 −0  src/main/scala/com/twitter/rowz/ReadWriteShardAdapter.scala
  30. +1 −1  src/main/scala/com/twitter/rowz/Row.scala
  31. +1 −2  src/main/scala/com/twitter/rowz/Rowz.scala
  32. +9 −0 src/main/scala/com/twitter/rowz/Shard.scala
  33. +10 −6 src/main/scala/com/twitter/rowz/SqlShard.scala
  34. +17 −0 src/main/scala/com/twitter/rowz/jobs/Copy.scala
  35. +1 −1  src/main/scala/com/twitter/rowz/jobs/Create.scala
  36. +2 −2 src/main/scala/com/twitter/rowz/thrift/conversions/Row.scala
  37. +2 −0  src/main/thrift/Rowz.thrift
  38. +1 −1  src/test/scala/com/twitter/rowz/SqlShardSpec.scala
View
10 .git-scala-build/COMMIT_EDITMSG
@@ -0,0 +1,10 @@
+
+# Please enter the commit message for your changes.
+# (Comment lines starting with '#' will not be included)
+#
+# Committer: Nick Kallen <pivotal@Toemang-2.local>
+#
+# On branch master
+#
+# Initial commit
+#
View
1  .git-scala-build/HEAD
@@ -0,0 +1 @@
+ref: refs/heads/master
View
11 .git-scala-build/config
@@ -0,0 +1,11 @@
+[core]
+ repositoryformatversion = 0
+ filemode = true
+ bare = false
+ logallrefupdates = true
+[remote "origin"]
+ url = git://github.com/robey/scala-build.git
+ fetch = +refs/heads/*:refs/remotes/origin/*
+[branch "master"]
+ remote = origin
+ merge = refs/heads/master
View
1  .git-scala-build/description
@@ -0,0 +1 @@
+Unnamed repository; edit this file to name it for gitweb.
View
15 .git-scala-build/hooks/applypatch-msg
@@ -0,0 +1,15 @@
+#!/bin/sh
+#
+# An example hook script to check the commit log message taken by
+# applypatch from an e-mail message.
+#
+# The hook should exit with non-zero status after issuing an
+# appropriate message if it wants to stop the commit. The hook is
+# allowed to edit the commit message file.
+#
+# To enable this hook, make this file executable.
+
+. git-sh-setup
+test -x "$GIT_DIR/hooks/commit-msg" &&
+ exec "$GIT_DIR/hooks/commit-msg" ${1+"$@"}
+:
View
24 .git-scala-build/hooks/commit-msg
@@ -0,0 +1,24 @@
+#!/bin/sh
+#
+# An example hook script to check the commit log message.
+# Called by git-commit with one argument, the name of the file
+# that has the commit message. The hook should exit with non-zero
+# status after issuing an appropriate message if it wants to stop the
+# commit. The hook is allowed to edit the commit message file.
+#
+# To enable this hook, make this file executable.
+
+# Uncomment the below to add a Signed-off-by line to the message.
+# Doing this in a hook is a bad idea in general, but the prepare-commit-msg
+# hook is more suited to it.
+#
+# SOB=$(git var GIT_AUTHOR_IDENT | sed -n 's/^\(.*>\).*$/Signed-off-by: \1/p')
+# grep -qs "^$SOB" "$1" || echo "$SOB" >> "$1"
+
+# This example catches duplicate Signed-off-by lines.
+
+test "" = "$(grep '^Signed-off-by: ' "$1" |
+ sort | uniq -c | sed -e '/^[ ]*1[ ]/d')" || {
+ echo >&2 Duplicate Signed-off-by lines.
+ exit 1
+}
View
8 .git-scala-build/hooks/post-commit
@@ -0,0 +1,8 @@
+#!/bin/sh
+#
+# An example hook script that is called after a successful
+# commit is made.
+#
+# To enable this hook, make this file executable.
+
+: Nothing
View
16 .git-scala-build/hooks/post-receive
@@ -0,0 +1,16 @@
+#!/bin/sh
+#
+# An example hook script for the post-receive event
+#
+# This script is run after receive-pack has accepted a pack and the
+# repository has been updated. It is passed arguments in through stdin
+# in the form
+# <oldrev> <newrev> <refname>
+# For example:
+# aa453216d1b3e49e7f6f98441fa56946ddcd6a20 68f7abf4e6f922807889f52bc043ecd31b79f814 refs/heads/master
+#
+# see contrib/hooks/ for an sample, or uncomment the next line (on debian)
+#
+
+
+#. /usr/share/doc/git-core/contrib/hooks/post-receive-email
View
8 .git-scala-build/hooks/post-update
@@ -0,0 +1,8 @@
+#!/bin/sh
+#
+# An example hook script to prepare a packed repository for use over
+# dumb transports.
+#
+# To enable this hook, make this file executable by "chmod +x post-update".
+
+exec git-update-server-info
View
14 .git-scala-build/hooks/pre-applypatch
@@ -0,0 +1,14 @@
+#!/bin/sh
+#
+# An example hook script to verify what is about to be committed
+# by applypatch from an e-mail message.
+#
+# The hook should exit with non-zero status after issuing an
+# appropriate message if it wants to stop the commit.
+#
+# To enable this hook, make this file executable.
+
+. git-sh-setup
+test -x "$GIT_DIR/hooks/pre-commit" &&
+ exec "$GIT_DIR/hooks/pre-commit" ${1+"$@"}
+:
View
70 .git-scala-build/hooks/pre-commit
@@ -0,0 +1,70 @@
+#!/bin/sh
+#
+# An example hook script to verify what is about to be committed.
+# Called by git-commit with no arguments. The hook should
+# exit with non-zero status after issuing an appropriate message if
+# it wants to stop the commit.
+#
+# To enable this hook, make this file executable.
+
+# This is slightly modified from Andrew Morton's Perfect Patch.
+# Lines you introduce should not have trailing whitespace.
+# Also check for an indentation that has SP before a TAB.
+
+if git-rev-parse --verify HEAD 2>/dev/null
+then
+ git-diff-index -p -M --cached HEAD --
+else
+ # NEEDSWORK: we should produce a diff with an empty tree here
+ # if we want to do the same verification for the initial import.
+ :
+fi |
+perl -e '
+ my $found_bad = 0;
+ my $filename;
+ my $reported_filename = "";
+ my $lineno;
+ sub bad_line {
+ my ($why, $line) = @_;
+ if (!$found_bad) {
+ print STDERR "*\n";
+ print STDERR "* You have some suspicious patch lines:\n";
+ print STDERR "*\n";
+ $found_bad = 1;
+ }
+ if ($reported_filename ne $filename) {
+ print STDERR "* In $filename\n";
+ $reported_filename = $filename;
+ }
+ print STDERR "* $why (line $lineno)\n";
+ print STDERR "$filename:$lineno:$line\n";
+ }
+ while (<>) {
+ if (m|^diff --git a/(.*) b/\1$|) {
+ $filename = $1;
+ next;
+ }
+ if (/^@@ -\S+ \+(\d+)/) {
+ $lineno = $1 - 1;
+ next;
+ }
+ if (/^ /) {
+ $lineno++;
+ next;
+ }
+ if (s/^\+//) {
+ $lineno++;
+ chomp;
+ if (/\s$/) {
+ bad_line("trailing whitespace", $_);
+ }
+ if (/^\s* \t/) {
+ bad_line("indent SP followed by a TAB", $_);
+ }
+ if (/^([<>])\1{6} |^={7}$/) {
+ bad_line("unresolved merge conflict", $_);
+ }
+ }
+ }
+ exit($found_bad);
+'
View
150 .git-scala-build/hooks/pre-rebase
@@ -0,0 +1,150 @@
+#!/bin/sh
+#
+# Copyright (c) 2006 Junio C Hamano
+#
+
+publish=next
+basebranch="$1"
+if test "$#" = 2
+then
+ topic="refs/heads/$2"
+else
+ topic=`git symbolic-ref HEAD`
+fi
+
+case "$basebranch,$topic" in
+master,refs/heads/??/*)
+ ;;
+*)
+ exit 0 ;# we do not interrupt others.
+ ;;
+esac
+
+# Now we are dealing with a topic branch being rebased
+# on top of master. Is it OK to rebase it?
+
+# Is topic fully merged to master?
+not_in_master=`git-rev-list --pretty=oneline ^master "$topic"`
+if test -z "$not_in_master"
+then
+ echo >&2 "$topic is fully merged to master; better remove it."
+ exit 1 ;# we could allow it, but there is no point.
+fi
+
+# Is topic ever merged to next? If so you should not be rebasing it.
+only_next_1=`git-rev-list ^master "^$topic" ${publish} | sort`
+only_next_2=`git-rev-list ^master ${publish} | sort`
+if test "$only_next_1" = "$only_next_2"
+then
+ not_in_topic=`git-rev-list "^$topic" master`
+ if test -z "$not_in_topic"
+ then
+ echo >&2 "$topic is already up-to-date with master"
+ exit 1 ;# we could allow it, but there is no point.
+ else
+ exit 0
+ fi
+else
+ not_in_next=`git-rev-list --pretty=oneline ^${publish} "$topic"`
+ perl -e '
+ my $topic = $ARGV[0];
+ my $msg = "* $topic has commits already merged to public branch:\n";
+ my (%not_in_next) = map {
+ /^([0-9a-f]+) /;
+ ($1 => 1);
+ } split(/\n/, $ARGV[1]);
+ for my $elem (map {
+ /^([0-9a-f]+) (.*)$/;
+ [$1 => $2];
+ } split(/\n/, $ARGV[2])) {
+ if (!exists $not_in_next{$elem->[0]}) {
+ if ($msg) {
+ print STDERR $msg;
+ undef $msg;
+ }
+ print STDERR " $elem->[1]\n";
+ }
+ }
+ ' "$topic" "$not_in_next" "$not_in_master"
+ exit 1
+fi
+
+exit 0
+
+################################################################
+
+This sample hook safeguards topic branches that have been
+published from being rewound.
+
+The workflow assumed here is:
+
+ * Once a topic branch forks from "master", "master" is never
+ merged into it again (either directly or indirectly).
+
+ * Once a topic branch is fully cooked and merged into "master",
+ it is deleted. If you need to build on top of it to correct
+ earlier mistakes, a new topic branch is created by forking at
+ the tip of the "master". This is not strictly necessary, but
+ it makes it easier to keep your history simple.
+
+ * Whenever you need to test or publish your changes to topic
+ branches, merge them into "next" branch.
+
+The script, being an example, hardcodes the publish branch name
+to be "next", but it is trivial to make it configurable via
+$GIT_DIR/config mechanism.
+
+With this workflow, you would want to know:
+
+(1) ... if a topic branch has ever been merged to "next". Young
+ topic branches can have stupid mistakes you would rather
+ clean up before publishing, and things that have not been
+ merged into other branches can be easily rebased without
+ affecting other people. But once it is published, you would
+ not want to rewind it.
+
+(2) ... if a topic branch has been fully merged to "master".
+ Then you can delete it. More importantly, you should not
+ build on top of it -- other people may already want to
+ change things related to the topic as patches against your
+ "master", so if you need further changes, it is better to
+ fork the topic (perhaps with the same name) afresh from the
+ tip of "master".
+
+Let's look at this example:
+
+ o---o---o---o---o---o---o---o---o---o "next"
+ / / / /
+ / a---a---b A / /
+ / / / /
+ / / c---c---c---c B /
+ / / / \ /
+ / / / b---b C \ /
+ / / / / \ /
+ ---o---o---o---o---o---o---o---o---o---o---o "master"
+
+
+A, B and C are topic branches.
+
+ * A has one fix since it was merged up to "next".
+
+ * B has finished. It has been fully merged up to "master" and "next",
+ and is ready to be deleted.
+
+ * C has not merged to "next" at all.
+
+We would want to allow C to be rebased, refuse A, and encourage
+B to be deleted.
+
+To compute (1):
+
+ git-rev-list ^master ^topic next
+ git-rev-list ^master next
+
+ if these match, topic has not merged in next at all.
+
+To compute (2):
+
+ git-rev-list master..topic
+
+ if this is empty, it is fully merged to "master".
View
36 .git-scala-build/hooks/prepare-commit-msg
@@ -0,0 +1,36 @@
+#!/bin/sh
+#
+# An example hook script to prepare the commit log message.
+# Called by git-commit with the name of the file that has the
+# commit message, followed by the description of the commit
+# message's source. The hook's purpose is to edit the commit
+# message file. If the hook fails with a non-zero status,
+# the commit is aborted.
+#
+# To enable this hook, make this file executable.
+
+# This hook includes three examples. The first comments out the
+# "Conflicts:" part of a merge commit.
+#
+# The second includes the output of "git diff --name-status -r"
+# into the message, just before the "git status" output. It is
+# commented because it doesn't cope with --amend or with squashed
+# commits.
+#
+# The third example adds a Signed-off-by line to the message, that can
+# still be edited. This is rarely a good idea.
+
+case "$2,$3" in
+ merge,)
+ perl -i -ne 's/^/# /, s/^# #/#/ if /^Conflicts/ .. /#/; print' "$1" ;;
+
+# ,|template,)
+# perl -i -pe '
+# print "\n" . `git diff --cached --name-status -r`
+# if /^#/ && $first++ == 0' "$1" ;;
+
+ *) ;;
+esac
+
+# SOB=$(git var GIT_AUTHOR_IDENT | sed -n 's/^\(.*>\).*$/Signed-off-by: \1/p')
+# grep -qs "^$SOB" "$1" || echo "$SOB" >> "$1"
View
107 .git-scala-build/hooks/update
@@ -0,0 +1,107 @@
+#!/bin/sh
+#
+# An example hook script to blocks unannotated tags from entering.
+# Called by git-receive-pack with arguments: refname sha1-old sha1-new
+#
+# To enable this hook, make this file executable by "chmod +x update".
+#
+# Config
+# ------
+# hooks.allowunannotated
+# This boolean sets whether unannotated tags will be allowed into the
+# repository. By default they won't be.
+# hooks.allowdeletetag
+# This boolean sets whether deleting tags will be allowed in the
+# repository. By default they won't be.
+# hooks.allowdeletebranch
+# This boolean sets whether deleting branches will be allowed in the
+# repository. By default they won't be.
+#
+
+# --- Command line
+refname="$1"
+oldrev="$2"
+newrev="$3"
+
+# --- Safety check
+if [ -z "$GIT_DIR" ]; then
+ echo "Don't run this script from the command line." >&2
+ echo " (if you want, you could supply GIT_DIR then run" >&2
+ echo " $0 <ref> <oldrev> <newrev>)" >&2
+ exit 1
+fi
+
+if [ -z "$refname" -o -z "$oldrev" -o -z "$newrev" ]; then
+ echo "Usage: $0 <ref> <oldrev> <newrev>" >&2
+ exit 1
+fi
+
+# --- Config
+allowunannotated=$(git config --bool hooks.allowunannotated)
+allowdeletebranch=$(git config --bool hooks.allowdeletebranch)
+allowdeletetag=$(git config --bool hooks.allowdeletetag)
+
+# check for no description
+projectdesc=$(sed -e '1q' "$GIT_DIR/description")
+if [ -z "$projectdesc" -o "$projectdesc" = "Unnamed repository; edit this file to name it for gitweb." ]; then
+ echo "*** Project description file hasn't been set" >&2
+ exit 1
+fi
+
+# --- Check types
+# if $newrev is 0000...0000, it's a commit to delete a ref.
+if [ "$newrev" = "0000000000000000000000000000000000000000" ]; then
+ newrev_type=delete
+else
+ newrev_type=$(git-cat-file -t $newrev)
+fi
+
+case "$refname","$newrev_type" in
+ refs/tags/*,commit)
+ # un-annotated tag
+ short_refname=${refname##refs/tags/}
+ if [ "$allowunannotated" != "true" ]; then
+ echo "*** The un-annotated tag, $short_refname, is not allowed in this repository" >&2
+ echo "*** Use 'git tag [ -a | -s ]' for tags you want to propagate." >&2
+ exit 1
+ fi
+ ;;
+ refs/tags/*,delete)
+ # delete tag
+ if [ "$allowdeletetag" != "true" ]; then
+ echo "*** Deleting a tag is not allowed in this repository" >&2
+ exit 1
+ fi
+ ;;
+ refs/tags/*,tag)
+ # annotated tag
+ ;;
+ refs/heads/*,commit)
+ # branch
+ ;;
+ refs/heads/*,delete)
+ # delete branch
+ if [ "$allowdeletebranch" != "true" ]; then
+ echo "*** Deleting a branch is not allowed in this repository" >&2
+ exit 1
+ fi
+ ;;
+ refs/remotes/*,commit)
+ # tracking branch
+ ;;
+ refs/remotes/*,delete)
+ # delete tracking branch
+ if [ "$allowdeletebranch" != "true" ]; then
+ echo "*** Deleting a tracking branch is not allowed in this repository" >&2
+ exit 1
+ fi
+ ;;
+ *)
+ # Anything else (is there anything else?)
+ echo "*** Update hook: unknown type of update to ref $refname of type $newrev_type" >&2
+ exit 1
+ ;;
+esac
+
+# --- Finished
+exit 0
View
BIN  .git-scala-build/index
Binary file not shown
View
6 .git-scala-build/info/exclude
@@ -0,0 +1,6 @@
+# git-ls-files --others --exclude-from=.git/info/exclude
+# Lines that start with '#' are comments.
+# For a project mostly in C, the following would be a good set of
+# exclude patterns (uncomment them if you want to use them):
+# *.[oa]
+# *~
View
1  .git-scala-build/logs/HEAD
@@ -0,0 +1 @@
+0000000000000000000000000000000000000000 984abd3cf968586e2ac897e16d7b2e0a9d032162 Nick Kallen <pivotal@Toemang-2.local> 1269636414 -0700 clone: from git://github.com/robey/scala-build.git
View
1  .git-scala-build/logs/refs/heads/master
@@ -0,0 +1 @@
+0000000000000000000000000000000000000000 984abd3cf968586e2ac897e16d7b2e0a9d032162 Nick Kallen <pivotal@Toemang-2.local> 1269636414 -0700 clone: from git://github.com/robey/scala-build.git
View
1  .git-scala-build/logs/refs/remotes/origin/HEAD
@@ -0,0 +1 @@
+0000000000000000000000000000000000000000 984abd3cf968586e2ac897e16d7b2e0a9d032162 Nick Kallen <pivotal@Toemang-2.local> 1269636414 -0700 clone: from git://github.com/robey/scala-build.git
View
1  .git-scala-build/logs/refs/remotes/origin/master
@@ -0,0 +1 @@
+0000000000000000000000000000000000000000 984abd3cf968586e2ac897e16d7b2e0a9d032162 Nick Kallen <pivotal@Toemang-2.local> 1269636414 -0700 clone: from git://github.com/robey/scala-build.git
View
BIN  .git-scala-build/objects/pack/pack-aa56c6a61ae1b1d9ce6497cd8c3dd18d8e775041.idx
Binary file not shown
View
BIN  .git-scala-build/objects/pack/pack-aa56c6a61ae1b1d9ce6497cd8c3dd18d8e775041.pack
Binary file not shown
View
1  .git-scala-build/refs/heads/master
@@ -0,0 +1 @@
+984abd3cf968586e2ac897e16d7b2e0a9d032162
View
1  .git-scala-build/refs/remotes/origin/HEAD
@@ -0,0 +1 @@
+ref: refs/remotes/origin/master
View
1  .git-scala-build/refs/remotes/origin/master
@@ -0,0 +1 @@
+984abd3cf968586e2ac897e16d7b2e0a9d032162
View
BIN  libs/gizzard-1.0.jar
Binary file not shown
View
11 src/main/scala/com/twitter/rowz/CopyManager.scala
@@ -1,11 +0,0 @@
-package com.twitter.rowz
-
-import com.twitter.gizzard.nameserver
-import com.twitter.gizzard.scheduler.JobScheduler
-
-
-class CopyManager(val scheduler: JobScheduler) extends nameserver.CopyManager[Shard] {
- def newCopyJob(sourceShardId: Int, destinationShardId: Int) = null
-
- def newMigrateJob(migration: nameserver.ShardMigration) = null
-}
View
8 src/main/scala/com/twitter/rowz/Main.scala
@@ -2,7 +2,7 @@ package com.twitter.rowz
import net.lag.configgy.Configgy
import net.lag.logging.Logger
-import com.twitter.gizzard.nameserver.NameServer
+import com.twitter.gizzard.nameserver.{NameServer, Copier}
import com.twitter.gizzard.scheduler.PrioritizingJobScheduler
import com.twitter.gizzard.thrift.{TSelectorServer, JobManager, JobManagerService, ShardManager, ShardManagerService}
import com.facebook.thrift.server.{TServer, TThreadPoolServer}
@@ -16,7 +16,7 @@ object Main {
var rowzService: RowzService = null
var nameServer: NameServer[Shard] = null
var scheduler: PrioritizingJobScheduler = null
- var copyManager: CopyManager = null
+ var copier: Copier[Shard] = null
var rowzServer: TSelectorServer = null
var jobServer: TSelectorServer = null
@@ -40,7 +40,7 @@ object Main {
rowzService = state._1
nameServer = state._2
scheduler = state._3
- copyManager = state._4
+ copier = state._4
startThrift()
}
@@ -55,7 +55,7 @@ object Main {
val jobProcessor = new JobManager.Processor(LoggingProxy[JobManager.Iface](Stats, Main.w3c, "RowzJobs", jobService))
jobServer = TSelectorServer("rowz-jobs", config("rowz.job_server_port").toInt, jobProcessor, executor, timeout)
- val shardService = new ShardManagerService(nameServer, copyManager)
+ val shardService = new ShardManagerService(nameServer, copier, scheduler(Priority.Low.id))
val shardProcessor = new ShardManager.Processor(ExceptionWrappingProxy(LoggingProxy[ShardManager.Iface](Stats, Main.w3c, "RowzShards", shardService)))
shardServer = TSelectorServer("rowz-shards", config("rowz.shard_server_port").toInt, shardProcessor, executor, timeout)
View
3  src/main/scala/com/twitter/rowz/ReadWriteShardAdapter.scala
@@ -3,6 +3,7 @@ package com.twitter.rowz
import com.twitter.gizzard.shards
import com.twitter.gizzard.shards.ReadWriteShard
import com.twitter.xrayspecs.Time
+import Shard.Cursor
class ReadWriteShardAdapter(shard: ReadWriteShard[Shard])
@@ -10,6 +11,8 @@ class ReadWriteShardAdapter(shard: ReadWriteShard[Shard])
def create(id: Long, name: String, at: Time) = shard.writeOperation(_.create(id, name, at))
def destroy(row: Row, at: Time) = shard.writeOperation(_.destroy(row, at))
+ def write(rows: Seq[Row]) = shard.writeOperation(_.write(rows))
def read(id: Long) = shard.readOperation(_.read(id))
+ def selectAll(cursor: Cursor, count: Int) = shard.readOperation(_.selectAll(cursor, count))
}
View
2  src/main/scala/com/twitter/rowz/Row.scala
@@ -3,4 +3,4 @@ package com.twitter.rowz
import com.twitter.xrayspecs.Time
-case class Row(id: Long, name: String, createdAt: Time)
+case class Row(id: Long, name: String, createdAt: Time, updatedAt: Time, state: State.Value)
View
3  src/main/scala/com/twitter/rowz/Rowz.scala
@@ -64,7 +64,6 @@ object Rowz {
schedulerMap(priority.id) = scheduler
}
val prioritizingScheduler = new PrioritizingJobScheduler(schedulerMap)
- val copyManager = new CopyManager(prioritizingScheduler(Priority.Low.id))
val copyJobParser = new BoundJobParser((nameServer, prioritizingScheduler(Priority.Low.id)))
val rowzJobParser = new BoundJobParser(forwardingManager)
@@ -76,6 +75,6 @@ object Rowz {
nameServer.reload()
prioritizingScheduler.start()
- (rowzService, nameServer, prioritizingScheduler, copyManager)
+ (rowzService, nameServer, prioritizingScheduler, new jobs.Copy(_, _, Shard.CursorStart))
}
}
View
9 src/main/scala/com/twitter/rowz/Shard.scala
@@ -4,8 +4,17 @@ import com.twitter.gizzard.shards
import com.twitter.xrayspecs.Time
+object Shard {
+ type Cursor = Int
+ val CursorStart = -1
+}
+
trait Shard extends shards.Shard {
+ import Shard.Cursor
+
def create(id: Long, name: String, at: Time)
def destroy(row: Row, at: Time)
def read(id: Long): Option[Row]
+ def selectAll(cursor: Cursor, count: Int): (Seq[Row], Option[Cursor])
+ def write(rows: Seq[Row])
}
View
16 src/main/scala/com/twitter/rowz/SqlShard.scala
@@ -9,6 +9,7 @@ import java.sql.SQLException
import com.twitter.gizzard.proxy.SqlExceptionWrappingProxy
import com.twitter.xrayspecs.Time
import com.twitter.xrayspecs.TimeConversions._
+import Shard.Cursor
class SqlShardFactory(queryEvaluatorFactory: QueryEvaluatorFactory, config: ConfigMap)
@@ -52,22 +53,25 @@ class SqlShard(private val queryEvaluator: QueryEvaluator, val shardInfo: shards
private val table = shardInfo.tablePrefix + "_rowz"
- def create(id: Long, name: String, at: Time) = write(id, name, at, State.Normal, at)
- def destroy(row: Row, at: Time) = write(row.id, row.name, row.createdAt, State.Destroyed, at)
+ def create(id: Long, name: String, at: Time) = write(new Row(id, name, at, at, State.Normal))
+ def destroy(row: Row, at: Time) = write(new Row(row.id, row.name, row.createdAt, at, State.Destroyed))
def read(id: Long) = {
queryEvaluator.selectOne("SELECT * FROM " + table + " WHERE id = ? AND state = ?", id, State.Normal.id) { row =>
- new Row(row.getLong("id"), row.getString("name"), Time(row.getLong("created_at").seconds))
+ new Row(row.getLong("id"), row.getString("name"), Time(row.getLong("created_at").seconds), Time(row.getLong("updated_at").seconds), State(row.getInt("state")))
}
}
- private def write(id: Long, name: String, createdAt: Time, state: State.Value, at: Time) = {
+ def selectAll(cursor: Cursor, count: Int) = null
+
+ def write(row: Row) = {
+ val Row(id, name, createdAt, updatedAt, state) = row
insertOrUpdate {
queryEvaluator.execute("INSERT INTO " + table + " (id, name, created_at, updated_at, state) VALUES (?, ?, ?, ?, ?)",
- id, name, createdAt.inSeconds, at.inSeconds, state.id)
+ id, name, createdAt.inSeconds, updatedAt.inSeconds, state.id)
} {
queryEvaluator.execute("UPDATE " + table + " SET id = ?, name = ?, created_at = ?, updated_at = ?, state = ? WHERE updated_at < ?",
- id, name, createdAt.inSeconds, at.inSeconds, state.id, at.inSeconds)
+ id, name, createdAt.inSeconds, updatedAt.inSeconds, state.id, updatedAt.inSeconds)
}
}
View
17 src/main/scala/com/twitter/rowz/jobs/Copy.scala
@@ -0,0 +1,17 @@
+package com.twitter.rowz.jobs
+
+import com.twitter.gizzard
+import Shard.Cursor
+
+
+object Copy {
+ val COUNT = 500
+}
+
+case class Copy(sourceShardId: Int, destinationShardId: Int, cursor: Cursor) extends gizzard.jobs.Copy[Shard](sourceShardId, destinationShardId, Copy.COUNT) {
+ protected def copyPage(sourceShard: Shard, destinationShard: Shard, count: Int) = {
+ val (items, nextCursor) = sourceShard.selectAll(cursor, count)
+ destinationShard.write(items)
+ nextCursor.map(new Copy(sourceShardId, destinationShardId, _))
+ }
+}
View
2  src/main/scala/com/twitter/rowz/jobs/Create.scala
@@ -10,6 +10,6 @@ case class Create(id: Long, name: String, at: Time) extends UnboundJob[Forwardin
}
def apply(forwardingManager: ForwardingManager) = {
- forwardingManager(row.id).create(row.id, row.name, at)
+ forwardingManager(id).create(id, name, at)
}
}
View
4 src/main/scala/com/twitter/rowz/thrift/conversions/Row.scala
@@ -6,12 +6,12 @@ import com.twitter.xrayspecs.TimeConversions._
object Row {
class RichShardingRow(row: rowz.Row) {
- def toThrift = new thrift.Row(row.id, row.name, row.createdAt.inSeconds)
+ def toThrift = new thrift.Row(row.id, row.name, row.createdAt.inSeconds, row.updatedAt.inSeconds, row.state.id)
}
implicit def shardingRowToRichShardingRow(row: rowz.Row) = new RichShardingRow(row)
class RichThriftRow(row: thrift.Row) {
- def fromThrift = new rowz.Row(row.id, row.name, Time(row.created_at.seconds))
+ def fromThrift = new rowz.Row(row.id, row.name, Time(row.created_at.seconds), Time(row.updated_at.seconds), State(row.state))
}
implicit def thriftRowToRichThriftRow(row: thrift.Row) = new RichThriftRow(row)
}
View
2  src/main/thrift/Rowz.thrift
@@ -5,6 +5,8 @@ struct Row {
1: i64 id
2: string name
3: i32 created_at
+ 4: i32 updated_at
+ 5: i32 state
}
exception RowzException {
View
2  src/test/scala/com/twitter/rowz/SqlShardSpec.scala
@@ -16,7 +16,7 @@ object SqlShard extends Specification with JMocker with ClassMocker {
"table_001", "localhost", "INT UNSIGNED", "INT UNSIGNED", Busy.Normal, 1)
val sqlShard = shardFactory.instantiate(shardInfo, 1, List[Shard]())
val queryEvaluator = queryEvaluatorFactory(shardInfo.hostname, null, config("rowz.db.username"), config("rowz.db.password"))
- val row = new Row(1, "a row", Time.now)
+ val row = new Row(1, "a row", Time.now, Time.now, State.Normal.id)
doBefore {
queryEvaluator.execute("DROP DATABASE IF EXISTS " + config("rowz.db.name"))
Please sign in to comment.
Something went wrong with that request. Please try again.