diff --git a/activerecord/lib/active_record/associations/collection_association.rb b/activerecord/lib/active_record/associations/collection_association.rb
index 129505cc7d37c..7b29143009949 100644
--- a/activerecord/lib/active_record/associations/collection_association.rb
+++ b/activerecord/lib/active_record/associations/collection_association.rb
@@ -75,7 +75,7 @@ def ids_writer(ids)
       def reset
         super
         @target = []
-        @replaced_targets = Set.new
+        @replaced_or_added_targets = Set.new
         @association_ids = nil
       end
 
@@ -447,7 +447,7 @@ def concat_records(records, raise = false)
         end
 
         def replace_on_target(record, skip_callbacks, replace:, inversing: false)
-          if replace && (!record.new_record? || @replaced_targets.include?(record))
+          if replace && (!record.new_record? || @replaced_or_added_targets.include?(record))
             index = @target.index(record)
           end
 
@@ -461,7 +461,7 @@ def replace_on_target(record, skip_callbacks, replace:, inversing: false)
 
           yield(record) if block_given?
 
-          @replaced_targets << record if inversing || index
+          @replaced_or_added_targets << record if inversing || index || record.new_record?
 
           if index
             target[index] = record
diff --git a/activerecord/test/cases/associations/belongs_to_associations_test.rb b/activerecord/test/cases/associations/belongs_to_associations_test.rb
index 3f1d8f402e8d7..d3f5dd18219c1 100644
--- a/activerecord/test/cases/associations/belongs_to_associations_test.rb
+++ b/activerecord/test/cases/associations/belongs_to_associations_test.rb
@@ -1469,6 +1469,16 @@ def test_multiple_counter_cache_with_after_create_update
       end
     end
   end
+
+  test "assigning an association doesn't result in duplicate objects" do
+    post = Post.create!(title: "title", body: "body")
+    post.comments = [post.comments.build(body: "body")]
+    post.save!
+
+    assert_equal 1, post.comments.size
+    assert_equal 1, Comment.where(post_id: post.id).count
+    assert_equal post.id, Comment.last.post.id
+  end
 end
 
 class BelongsToWithForeignKeyTest < ActiveRecord::TestCase