Skip to content
This repository has been archived by the owner on Sep 18, 2021. It is now read-only.

Commit

Permalink
Merge branch 'master' of github.com:twitter/gizzmo
Browse files Browse the repository at this point in the history
  • Loading branch information
@freels
freels committed Feb 8, 2012
2 parents b653b12 + e869c65 commit 3f0777c
Show file tree
Hide file tree
Showing 7 changed files with 107 additions and 94 deletions.
6 changes: 5 additions & 1 deletion lib/gizzard/thrift.rb
View file
Expand Up @@ -32,7 +32,11 @@ def <=>(o)
alias_method :to_unix, :inspect

def self.parse(string)
new(*string.match("(.*)/(.*)").values_at(1, 2))
if groups = string.match(/(.*)\/([^_]+_\d+_\d{3,}.*)/)
new(*groups.values_at(1, 2))
else
raise "Invalid shard id: " + string
end
end
end

Expand Down
2 changes: 1 addition & 1 deletion test/expected/blocked-transform-shard.txt
View file
@@ -1 +1 @@
Aborting due to blocked shard localhost/table_a_3_blocked
Aborting due to blocked shard localhost/base_13_003_a_blocked
2 changes: 1 addition & 1 deletion test/expected/busy-transform-shard.txt
View file
@@ -1 +1 @@
Aborting due to busy shard localhost/table_a_3
Aborting due to busy shard localhost/base_13_003_a
40 changes: 20 additions & 20 deletions test/expected/find-only-sql-shard-type.txt
View file
@@ -1,20 +1,20 @@
localhost/table_a_0
localhost/table_a_1
localhost/table_a_2
localhost/table_a_3
localhost/table_a_4
localhost/table_a_5
localhost/table_a_6
localhost/table_a_7
localhost/table_a_8
localhost/table_a_9
localhost/table_b_0
localhost/table_b_1
localhost/table_b_2
localhost/table_b_3
localhost/table_b_4
localhost/table_b_5
localhost/table_b_6
localhost/table_b_7
localhost/table_b_8
localhost/table_b_9
localhost/base_13_000_a
localhost/base_13_000_b
localhost/base_13_001_a
localhost/base_13_001_b
localhost/base_13_002_a
localhost/base_13_002_b
localhost/base_13_003_a
localhost/base_13_003_b
localhost/base_13_004_a
localhost/base_13_004_b
localhost/base_13_005_a
localhost/base_13_005_b
localhost/base_13_006_a
localhost/base_13_006_b
localhost/base_13_007_a
localhost/base_13_007_b
localhost/base_13_008_a
localhost/base_13_008_b
localhost/base_13_009_a
localhost/base_13_009_b
60 changes: 30 additions & 30 deletions test/expected/info.txt
View file
@@ -1,30 +1,30 @@
localhost/table_a_0 TestShard ok
localhost/table_a_1 TestShard ok
localhost/table_a_2 TestShard ok
localhost/table_a_3 TestShard ok
localhost/table_a_4 TestShard ok
localhost/table_a_5 TestShard ok
localhost/table_a_6 TestShard ok
localhost/table_a_7 TestShard ok
localhost/table_a_8 TestShard ok
localhost/table_a_9 TestShard ok
localhost/table_b_0 TestShard ok
localhost/table_b_1 TestShard ok
localhost/table_b_2 TestShard ok
localhost/table_b_3 TestShard ok
localhost/table_b_4 TestShard ok
localhost/table_b_5 TestShard ok
localhost/table_b_6 TestShard ok
localhost/table_b_7 TestShard ok
localhost/table_b_8 TestShard ok
localhost/table_b_9 TestShard ok
localhost/table_repl_0 com.twitter.gizzard.shards.ReplicatingShard ok
localhost/table_repl_1 com.twitter.gizzard.shards.ReplicatingShard ok
localhost/table_repl_2 com.twitter.gizzard.shards.ReplicatingShard ok
localhost/table_repl_3 com.twitter.gizzard.shards.ReplicatingShard ok
localhost/table_repl_4 com.twitter.gizzard.shards.ReplicatingShard ok
localhost/table_repl_5 com.twitter.gizzard.shards.ReplicatingShard ok
localhost/table_repl_6 com.twitter.gizzard.shards.ReplicatingShard ok
localhost/table_repl_7 com.twitter.gizzard.shards.ReplicatingShard ok
localhost/table_repl_8 com.twitter.gizzard.shards.ReplicatingShard ok
localhost/table_repl_9 com.twitter.gizzard.shards.ReplicatingShard ok
localhost/base_13_000_a TestShard ok
localhost/base_13_000_b TestShard ok
localhost/base_13_000_replicating com.twitter.gizzard.shards.ReplicatingShard ok
localhost/base_13_001_a TestShard ok
localhost/base_13_001_b TestShard ok
localhost/base_13_001_replicating com.twitter.gizzard.shards.ReplicatingShard ok
localhost/base_13_002_a TestShard ok
localhost/base_13_002_b TestShard ok
localhost/base_13_002_replicating com.twitter.gizzard.shards.ReplicatingShard ok
localhost/base_13_003_a TestShard ok
localhost/base_13_003_b TestShard ok
localhost/base_13_003_replicating com.twitter.gizzard.shards.ReplicatingShard ok
localhost/base_13_004_a TestShard ok
localhost/base_13_004_b TestShard ok
localhost/base_13_004_replicating com.twitter.gizzard.shards.ReplicatingShard ok
localhost/base_13_005_a TestShard ok
localhost/base_13_005_b TestShard ok
localhost/base_13_005_replicating com.twitter.gizzard.shards.ReplicatingShard ok
localhost/base_13_006_a TestShard ok
localhost/base_13_006_b TestShard ok
localhost/base_13_006_replicating com.twitter.gizzard.shards.ReplicatingShard ok
localhost/base_13_007_a TestShard ok
localhost/base_13_007_b TestShard ok
localhost/base_13_007_replicating com.twitter.gizzard.shards.ReplicatingShard ok
localhost/base_13_008_a TestShard ok
localhost/base_13_008_b TestShard ok
localhost/base_13_008_replicating com.twitter.gizzard.shards.ReplicatingShard ok
localhost/base_13_009_a TestShard ok
localhost/base_13_009_b TestShard ok
localhost/base_13_009_replicating com.twitter.gizzard.shards.ReplicatingShard ok
60 changes: 30 additions & 30 deletions test/expected/original-find.txt
View file
@@ -1,30 +1,30 @@
localhost/table_a_0
localhost/table_a_1
localhost/table_a_2
localhost/table_a_3
localhost/table_a_4
localhost/table_a_5
localhost/table_a_6
localhost/table_a_7
localhost/table_a_8
localhost/table_a_9
localhost/table_b_0
localhost/table_b_1
localhost/table_b_2
localhost/table_b_3
localhost/table_b_4
localhost/table_b_5
localhost/table_b_6
localhost/table_b_7
localhost/table_b_8
localhost/table_b_9
localhost/table_repl_0
localhost/table_repl_1
localhost/table_repl_2
localhost/table_repl_3
localhost/table_repl_4
localhost/table_repl_5
localhost/table_repl_6
localhost/table_repl_7
localhost/table_repl_8
localhost/table_repl_9
localhost/base_13_000_a
localhost/base_13_000_b
localhost/base_13_000_replicating
localhost/base_13_001_a
localhost/base_13_001_b
localhost/base_13_001_replicating
localhost/base_13_002_a
localhost/base_13_002_b
localhost/base_13_002_replicating
localhost/base_13_003_a
localhost/base_13_003_b
localhost/base_13_003_replicating
localhost/base_13_004_a
localhost/base_13_004_b
localhost/base_13_004_replicating
localhost/base_13_005_a
localhost/base_13_005_b
localhost/base_13_005_replicating
localhost/base_13_006_a
localhost/base_13_006_b
localhost/base_13_006_replicating
localhost/base_13_007_a
localhost/base_13_007_b
localhost/base_13_007_replicating
localhost/base_13_008_a
localhost/base_13_008_b
localhost/base_13_008_replicating
localhost/base_13_009_a
localhost/base_13_009_b
localhost/base_13_009_replicating
31 changes: 20 additions & 11 deletions test/test.sh
View file
Expand Up @@ -9,7 +9,7 @@ function g_silent {
}

function expect {
diff -u - "expected/$1" && echo -e " success\t$1." || (echo -e " failed\t$1." && exit 1)
diff -U3 "expected/$1" - && echo -e " success\t$1." || (echo -e " failed\t$1." && exit 1)
}

function expect-string {
Expand All @@ -21,6 +21,10 @@ TABLE=13
REPLICATING_SHARD_CLASS="com.twitter.gizzard.shards.ReplicatingShard"
BLOCKED_SHARD_CLASS="com.twitter.gizzard.shards.BlockedShard"

function shard_id {
printf "localhost/base_%d_%03d_%s" $TABLE $1 $2
}

function cleanup {
for shard in `g find -hlocalhost`; do
# links
Expand All @@ -35,12 +39,12 @@ function cleanup {

function initialize {
for i in {0..9}; do
g_silent create $REPLICATING_SHARD_CLASS localhost/table_repl_$i
g_silent create TestShard localhost/table_a_$i --source-type="INT UNSIGNED" --destination-type="INT UNSIGNED"
g_silent create TestShard localhost/table_b_$i --source-type="INT UNSIGNED" --destination-type="INT UNSIGNED"
REPLICATING_SHARD="localhost/table_repl_$i"
g_silent addlink $REPLICATING_SHARD "localhost/table_a_$i" 2
g_silent addlink $REPLICATING_SHARD "localhost/table_b_$i" 1
REPLICATING_SHARD=$(shard_id $i "replicating")
g_silent create $REPLICATING_SHARD_CLASS $REPLICATING_SHARD
g_silent create TestShard $(shard_id $i "a") --source-type="INT UNSIGNED" --destination-type="INT UNSIGNED"
g_silent create TestShard $(shard_id $i "b") --source-type="INT UNSIGNED" --destination-type="INT UNSIGNED"
g_silent addlink $REPLICATING_SHARD $(shard_id $i "a") 2
g_silent addlink $REPLICATING_SHARD $(shard_id $i "b") 1
g_silent addforwarding $TABLE `date +%s` $REPLICATING_SHARD
done
}
Expand All @@ -62,17 +66,22 @@ function simple_transform {
}

{ # test-busy-transform
g_silent markbusy localhost/table_a_3
g_silent markbusy $(shard_id 3 "a")
simple_transform | expect busy-transform-shard.txt
g_silent markunbusy localhost/table_a_3
g_silent markunbusy $(shard_id 3 "a")
}

{ # test-blocked-transform
g_silent wrap $BLOCKED_SHARD_CLASS localhost/table_a_3
g_silent wrap $BLOCKED_SHARD_CLASS $(shard_id 3 "a")
simple_transform | expect blocked-transform-shard.txt
g_silent unwrap localhost/table_a_3_blocked
g_silent unwrap $(shard_id 3 "a_blocked")
}

# FIXME: remaining tests are out of date: see DATASERV-83
###############################################################################
exit
###############################################################################

g -D wrap com.twitter.gizzard.shards.ReplicatingShard localhost/table_b_0 | expect dry-wrap-table_b_0.txt
g wrap com.twitter.gizzard.shards.ReplicatingShard localhost/table_b_0 | expect wrap-table_b_0.txt
g wrap com.twitter.gizzard.shards.ReplicatingShard localhost/table_b_0 | expect wrap-table_b_0.txt
Expand Down

0 comments on commit 3f0777c

Please sign in to comment.