Add additional state TXG_STATE_WAIT_FOR_SYNC for txg.

In several cases when digging into kstats we can found two txgs
in SYNC state, e.g.

txg     birth            state  nreserved  nread      nwritten ...
985452  258127184872561  C      0          373948416  2376272384 ...
985453  258129016180616  C      0          378173440  28793344 ...
985454  258129016271523  S      0          0          0 ...
985455  258130864245986  S      0          0          0 ...
985456  258130867458851  O      0          0          0 ...

However only first txg (985454) is really syncing at this moment.
The other one (985455) marked as SYNCED is actually in a post-QUIESCED
state and waiting to start sync.   So, the new TXG_STATE_WAIT_FOR_SYNC
state between TXG_STATE_QUIESCED and TXG_STATE_SYNCED was added to
reveal this situation.

txg     birth            state  nreserved  nread      nwritten ...
1086896 235261068743969  C      0          163577856  8437248 ...
1086897 235262870830801  C      0          280625152  822594048 ...
1086898 235264172219064  S      0          0          0 ...
1086899 235264936134407  W      0          0          0 ...
1086900 235264936296156  O      0          0          0 ...

Signed-off-by: Igor Lvovsky <ilvovsky@gmail.com>
Signed-off-by: Brian Behlendorf <behlendorf1@llnl.gov>
Issue #2075

include/sys/spa.h

@@ -557,8 +557,9 @@ typedef enum txg_state {
 	TXG_STATE_BIRTH		= 0,
 	TXG_STATE_OPEN		= 1,
 	TXG_STATE_QUIESCED	= 2,
-	TXG_STATE_SYNCED	= 3,
-	TXG_STATE_COMMITTED	= 4,
+	TXG_STATE_WAIT_FOR_SYNC	= 3,
+	TXG_STATE_SYNCED	= 4,
+	TXG_STATE_COMMITTED	= 5,
 } txg_state_t;
 
 extern void spa_stats_init(spa_t *spa);

module/zfs/spa_stats.c

@@ -258,9 +258,9 @@ static int
 spa_txg_history_headers(char *buf, size_t size)
 {
 	size = snprintf(buf, size - 1, "%-8s %-16s %-5s %-12s %-12s %-12s "
-	    "%-8s %-8s %-12s %-12s %-12s\n", "txg", "birth", "state",
+	    "%-8s %-8s %-12s %-12s %-12s %-12s\n", "txg", "birth", "state",
 	    "nreserved", "nread", "nwritten", "reads", "writes",
-	    "otime", "qtime", "stime");
+	    "otime", "qtime", "wtime", "stime");
 	buf[size] = '\0';
 
 	return (0);
@@ -270,13 +270,14 @@ static int
 spa_txg_history_data(char *buf, size_t size, void *data)
 {
 	spa_txg_history_t *sth = (spa_txg_history_t *)data;
-	uint64_t open = 0, quiesce = 0, sync = 0;
+	uint64_t open = 0, quiesce = 0, wait = 0, sync = 0;
 	char state;
 
 	switch (sth->state) {
 		case TXG_STATE_BIRTH:		state = 'B';	break;
 		case TXG_STATE_OPEN:		state = 'O';	break;
 		case TXG_STATE_QUIESCED:	state = 'Q';	break;
+		case TXG_STATE_WAIT_FOR_SYNC:	state = 'W';	break;
 		case TXG_STATE_SYNCED:		state = 'S';	break;
 		case TXG_STATE_COMMITTED:	state = 'C';	break;
 		default:			state = '?';	break;
@@ -290,17 +291,22 @@ spa_txg_history_data(char *buf, size_t size, void *data)
 		quiesce = sth->times[TXG_STATE_QUIESCED] -
 		    sth->times[TXG_STATE_OPEN];
 
+	if (sth->times[TXG_STATE_WAIT_FOR_SYNC])
+		wait = sth->times[TXG_STATE_WAIT_FOR_SYNC] -
+		    sth->times[TXG_STATE_QUIESCED];
+
 	if (sth->times[TXG_STATE_SYNCED])
 		sync = sth->times[TXG_STATE_SYNCED] -
-		    sth->times[TXG_STATE_QUIESCED];
+		    sth->times[TXG_STATE_WAIT_FOR_SYNC];
 
 	size = snprintf(buf, size - 1, "%-8llu %-16llu %-5c %-12llu "
-	    "%-12llu %-12llu %-8llu %-8llu %-12llu %-12llu %-12llu\n",
+	    "%-12llu %-12llu %-8llu %-8llu %-12llu %-12llu %-12llu %-12llu\n",
 	    (longlong_t)sth->txg, sth->times[TXG_STATE_BIRTH], state,
 	    (u_longlong_t)sth->nreserved,
 	    (u_longlong_t)sth->nread, (u_longlong_t)sth->nwritten,
 	    (u_longlong_t)sth->reads, (u_longlong_t)sth->writes,
-	    (u_longlong_t)open, (u_longlong_t)quiesce, (u_longlong_t)sync);
+	    (u_longlong_t)open, (u_longlong_t)quiesce, (u_longlong_t)wait,
+	    (u_longlong_t)sync);
 	buf[size] = '\0';
 
 	return (0);

module/zfs/txg.c

@@ -555,6 +555,9 @@ txg_sync_thread(dsl_pool_t *dp)
 		    txg, tx->tx_quiesce_txg_waiting, tx->tx_sync_txg_waiting);
 		mutex_exit(&tx->tx_sync_lock);
 
+		spa_txg_history_set(spa, txg, TXG_STATE_WAIT_FOR_SYNC,
+		    gethrtime());
+
 		start = ddi_get_lbolt();
 		spa_sync(spa, txg);
 		delta = ddi_get_lbolt() - start;