flaky test fix trial (pytorch#11391)

Summary:
Add a barrier() to wait for all PG created before destroy
Pull Request resolved: pytorch#11391

Differential Revision: D9727383

Pulled By: teng-li

fbshipit-source-id: 689d62c978e642b68f4949dcf29982e34869ada4

Author: teng-li

test/test_distributed.py

@@ -241,6 +241,7 @@ def test_destroy_group(self):
         else:
             group = [0, 1]
         group_id = dist.new_group(group)
+        self._barrier()
         dist.destroy_process_group(group_id)
 
     # Test get rank and size of group
@@ -260,6 +261,7 @@ def test_get_rank_size_group(self):
     # Test destroy full groups
     def test_destroy_full_group(self):
         _, group_id, _ = self._init_full_group_test()
+        self._barrier()
         dist.destroy_process_group(group_id)
 
     # Test get rank and size of full group
@@ -1272,6 +1274,7 @@ def _run(self, rank):
             # self.id() == e.g. '__main__.TestDistributed.test_get_rank'
             # We're retreiving a corresponding test and executing it.
             getattr(self, self.id().split(".")[2])()
+            self._barrier()
             dist.destroy_process_group()
             sys.exit(0)