Fix flush of clunk or remove to destroy fids

Also add a flush debug flag and set it by default for now, so we
see what type of ops are getting flushed.

diod/diod.c

@@ -585,6 +585,7 @@ _service_run (srvmode_t mode, int rfdno, int wfdno)
     umask (0);
 
     flags |= SRV_FLAGS_DEBUG_FIDPOOL; /* XXX temporary */
+    flags |= SRV_FLAGS_DEBUG_FLUSH;   /* XXX temporary */
 
     flags |= SRV_FLAGS_AUTHCONN;
     flags |= SRV_FLAGS_FLUSHSIG;

libnpfs/fcall.c

@@ -256,20 +256,29 @@ np_flush(Npreq *req, Npfcall *tc)
 	Npreq *creq;
 	Npfcall *ret;
 	Nptpool *tp;
+	Npsrv *srv = req->conn->srv;
 
 	xpthread_mutex_lock(&req->conn->srv->lock);
 	for (tp = req->conn->srv->tpool; tp != NULL; tp = tp->next) {
 		for(creq = tp->reqs_first; creq != NULL; creq = creq->next) {
 			if (!(creq->conn==req->conn && creq->tag==oldtag))
 				continue;
+			if ((srv->flags & SRV_FLAGS_DEBUG_FLUSH)) {
+				np_logmsg (srv, "flush(early): req type %d",
+					   creq->tcall->type);
+			}
 			np_srv_remove_req(tp, creq);
 			np_req_unref(creq);
 			goto done;
 		}
 		for(creq = tp->workreqs; creq != NULL; creq = creq->next) {
 			if (!(creq->conn==req->conn && creq->tag==oldtag))
 				continue;
-			creq->flushed = 1;
+			if ((srv->flags & SRV_FLAGS_DEBUG_FLUSH)) {
+				np_logmsg (srv, "flush(late): req type %d",
+					   creq->tcall->type);
+			}
+			creq->flushed = 1; /* prevents response */
 			if (req->conn->srv->flags & SRV_FLAGS_FLUSHSIG)
 				pthread_kill (creq->wthread->thread, SIGUSR2);
 			goto done;

libnpfs/npfs.h

@@ -235,6 +235,7 @@ enum {
 	SRV_FLAGS_DEBUG_9PTRACE	=0x00000001,
 	SRV_FLAGS_DEBUG_USER    =0x00000002,
 	SRV_FLAGS_DEBUG_FIDPOOL =0x00000004,
+	SRV_FLAGS_DEBUG_FLUSH   =0x00000008,
 
 	/* features */
 	SRV_FLAGS_SETFSID	=0x00010000,

libnpfs/srv.c

@@ -815,6 +815,14 @@ np_req_unref(Npreq *req)
 	xpthread_mutex_unlock(&req->lock);
 
 	if (req->fid) {
+		/* We expect to get here with valid fid if request is flushed.
+ 		 * Special case: need to free fid on flushed clunk or remove,
+ 		 * or client reuse of the fid will cause an error (issue 81).
+ 		 */
+		if (req->tcall && (req->tcall->type == P9_TCLUNK ||
+				   req->tcall->type == P9_TREMOVE)) {
+			np_fid_decref (req->fid, req->tcall->type);
+		}
 		np_fid_decref (req->fid, req->tcall ? req->tcall->type : 0);
 		req->fid = NULL;
 	}