diff --git a/plan/logical_plan_test.go b/plan/logical_plan_test.go
index f1445fe5379d4..ae30579b8a390 100644
--- a/plan/logical_plan_test.go
+++ b/plan/logical_plan_test.go
@@ -1578,6 +1578,11 @@ func (s *testPlanSuite) TestTopNPushDown(c *C) {
 			sql:  "select * from t union all (select * from t s order by a) limit 5",
 			best: "UnionAll{DataScan(t)->Limit->Projection->DataScan(s)->TopN([s.a],0,5)->Projection}->Limit",
 		},
+		// Test `ByItem` containing column from both sides.
+		{
+			sql:  "select ifnull(t1.b, t2.a) from t t1 left join t t2 on t1.e=t2.e order by ifnull(t1.b, t2.a) limit 5",
+			best: "Join{DataScan(t1)->DataScan(t2)}(t1.e,t2.e)->TopN([ifnull(t1.b, t2.a)],0,5)->Projection->Projection",
+		},
 	}
 	for i, tt := range tests {
 		comment := Commentf("case:%v sql:%s", i, tt.sql)
diff --git a/plan/topn_push_down.go b/plan/topn_push_down.go
index 84f6994ce8f2c..a0f6306f481f2 100644
--- a/plan/topn_push_down.go
+++ b/plan/topn_push_down.go
@@ -109,8 +109,10 @@ func (p *LogicalJoin) pushDownTopNToChild(topN *LogicalTopN, idx int) LogicalPla
 
 	for _, by := range topN.ByItems {
 		cols := expression.ExtractColumns(by.Expr)
-		if len(p.children[1-idx].Schema().ColumnsIndices(cols)) != 0 {
-			return p.children[idx].pushDownTopN(nil)
+		for _, col := range cols {
+			if p.children[1-idx].Schema().Contains(col) {
+				return p.children[idx].pushDownTopN(nil)
+			}
 		}
 	}