afr: add checks for allowing lookups

Problem:
In an arbiter volume, lookup was being served from one of the sink
bricks (source brick was down). shard uses the iatt values from lookup cbk
to calculate the size and block count, which in this case were incorrect
values. shard_local_t->last_block was thus initialised to -1, resulting
in an infinite while loop in shard_common_resolve_shards().

Fix:
Use client quorum logic to allow or fail the lookups from afr if there
are no readable subvolumes. So in replica-3 or arbiter vols, if there is
no good copy or if quorum is not met, fail lookup with ENOTCONN.

With this fix, we are also removing support for quorum-reads xlator
option. So if quorum is not met, neither read nor write txns are allowed
and we fail the fop with ENOTCONN.

Change-Id: Ic65c00c24f77ece007328b421494eee62a505fa0
BUG: 1467250
Signed-off-by: Ravishankar N <ravishankar@redhat.com>

tests/basic/afr/quorum.t

@@ -31,11 +31,7 @@ TEST $CLI volume set $V0 cluster.quorum-count 2
 TEST test_write
 TEST kill_brick $V0 $H0 $B0/${V0}1
 TEST ! test_write
-EXPECT "abc" cat $M0/b
-TEST $CLI volume set $V0 cluster.quorum-reads on
-EXPECT_WITHIN $CONFIG_UPDATE_TIMEOUT "1" mount_get_option_value $M0 $V0-replicate-0 quorum-reads
 TEST ! cat $M0/b
-TEST $CLI volume reset $V0 cluster.quorum-reads
 
 TEST $CLI volume set $V0 cluster.quorum-type auto
 EXPECT auto volume_option $V0 cluster.quorum-type
@@ -44,11 +40,7 @@ EXPECT_WITHIN $PROCESS_UP_TIMEOUT "1" afr_child_up_status $V0 0
 TEST test_write
 TEST kill_brick $V0 $H0 $B0/${V0}1
 TEST ! test_write
-EXPECT "abc" cat $M0/b
-TEST $CLI volume set $V0 cluster.quorum-reads on
-EXPECT_WITHIN $CONFIG_UPDATE_TIMEOUT "1" mount_get_option_value $M0 $V0-replicate-0 quorum-reads
 TEST ! cat $M0/b
-TEST $CLI volume reset $V0 cluster.quorum-reads
 
 TEST $CLI volume set $V0 cluster.quorum-type none
 EXPECT none volume_option $V0 cluster.quorum-type
@@ -57,11 +49,6 @@ TEST test_write
 TEST $CLI volume reset $V0 cluster.quorum-type
 TEST test_write
 EXPECT "abc" cat $M0/b
-TEST $CLI volume set $V0 cluster.quorum-reads on
-EXPECT_WITHIN $CONFIG_UPDATE_TIMEOUT "1" mount_get_option_value $M0 $V0-replicate-0 quorum-reads
-EXPECT "abc" cat $M0/b
-TEST $CLI volume reset $V0 cluster.quorum-reads
-
 
 cleanup;
 TEST glusterd;
@@ -86,24 +73,14 @@ TEST $CLI volume set $V0 cluster.quorum-count 3
 TEST test_write
 TEST kill_brick $V0 $H0 $B0/${V0}1
 TEST ! test_write
-EXPECT "abc" cat $M0/b
-TEST $CLI volume set $V0 cluster.quorum-reads on
-EXPECT_WITHIN $CONFIG_UPDATE_TIMEOUT "1" mount_get_option_value $M0 $V0-replicate-0 quorum-reads
 TEST ! cat $M0/b
-TEST $CLI volume reset $V0 cluster.quorum-reads
-
 
 TEST $CLI volume set $V0 cluster.quorum-type auto
 EXPECT auto volume_option $V0 cluster.quorum-type
 TEST test_write
 TEST kill_brick $V0 $H0 $B0/${V0}3
 TEST ! test_write
-EXPECT "abc" cat $M0/b
-TEST $CLI volume set $V0 cluster.quorum-reads on
-EXPECT_WITHIN $CONFIG_UPDATE_TIMEOUT "1" mount_get_option_value $M0 $V0-replicate-0 quorum-reads
 TEST ! cat $M0/b
-TEST $CLI volume reset $V0 cluster.quorum-reads
-
 
 TEST $CLI volume set $V0 cluster.quorum-type none
 EXPECT none volume_option $V0 cluster.quorum-type

tests/bugs/replicate/bug-977797.t

@@ -53,6 +53,8 @@ TEST chmod 757 $M0/a/file
 TEST $CLI volume start $V0 force
 EXPECT_WITHIN $CHILD_UP_TIMEOUT "1" afr_child_up_status $V0 1;
 
+#Trigger entry heal of $M0/a
+getfattr -n user.nosuchattr $M0/a
 dd if=$M0/a/file of=/dev/null bs=1024k
 #read fails, but heal is triggered.
 TEST [ $? -ne 0 ]