Cannot .populate() does not correctly work from base when using discriminators

[aforty]

mongoose: 4.4.12

Thought this was related to #2719 but perhaps not. Figured I'd start a new issue.

I'm still having an issue with this, or perhaps I'm just doing something wrong. I also have the context of a social app with Activity, and then specific cases like ActivityLiked and ActivityCommentMentioned.

When I find() and then populate(), it will not populate some fields. I can't give more detail than that as I haven't really found it to be predictable in any way. It does seem like it's the field that is not present in another, fields that overlap seem to always populate fine.

Activity

var ActivitySchema = new mongoose.Schema(
  {
    user: { type: Number, ref: 'User', required: true, index: true },
    activityAt: { type: Date, required: true, default: Date.now },
  },
  {
    collection: 'Activities',
    discriminatorKey: 'type',
  }
);

ActivityLiked

let ActivityLiked = Activity.discriminator('Liked', new mongoose.Schema({
  from: { type: Number, ref: 'User', required: true },
  item: { type: Number, ref: 'Item', required: true },
}, { discriminatorKey: 'type' }));

ActivityCommentMentioned

let ActivityCommentMentioned = Activity.discriminator('CommentMentioned', new mongoose.Schema({
  from: { type: Number, ref: 'User', required: true },
  comment: { type: Number, ref: 'ItemComment', required: true },
  item: { type: Number, ref: 'Item', required: true },
}, { discriminatorKey: 'type' }));

Then I find() all the activity for a user and attempt to populate it:

let items = await Activity
  .find({ user: req.user._id })
  .sort({ activityAt: -1 })
  .limit(30)
  .populate('user from item comment');

Then the comment field will sometimes not be populated on CommentMentioned types. Everything else is populated as expected (maybe because they overlap between both discriminators?)

[dciccale]

I think that you have to specify the correct type on the fields you want to populate, in which case would be something like

from: {type: Schema.Types.ObjectId, ref: 'User', required: true},
comment: {type: Schema.Types.ObjectId, ref: 'ItemComment', required: true},
item: {type: Schema.Types.ObjectId, ref: 'Item', required: true}

[vkarpov15]

I wouldn't be surprised if there was a subtle bug in population's handling of discriminators, but until we can repro the issue consistently we'll just be taking shots in the dark. Can you perhaps share a data set where this issue happens? The below script executes without error for me with mongoose 4.4.12.

'use strict';

var assert = require('assert');
var mongoose = require('mongoose');
var Schema = mongoose.Schema;

mongoose.connect('mongodb://localhost/gh4073');
mongoose.set('debug', true);

var ActivitySchema = new mongoose.Schema(
  {
    user: { type: Number, ref: 'User', required: true, index: true },
  },
  {
    collection: 'Activities',
    discriminatorKey: 'type',
  }
);

const Activity = mongoose.model('Activity', ActivitySchema);

let ActivityLiked = Activity.discriminator('Liked', new mongoose.Schema({
  from: { type: Number, ref: 'User', required: true },
  item: { type: Number, ref: 'Item', required: true },
}, { discriminatorKey: 'type' }));

let ActivityCommentMentioned = Activity.discriminator('CommentMentioned', new mongoose.Schema({
  from: { type: Number, ref: 'User', required: true },
  comment: { type: Number, ref: 'ItemComment', required: true },
  item: { type: Number, ref: 'Item', required: true },
}, { discriminatorKey: 'type' }));

const User = mongoose.model('User', { _id: Number, name: String });
const Item = mongoose.model('Item', { _id: Number, title: String });
const ItemComment = mongoose.model('ItemComment', { _id: Number, comment: String });

User.create({ _id: 1, name: 'Val' }, function(error, user) {
  assert.ifError(error);
  Item.create({ _id: 2, title: 'test' }, function(error, item) {
    assert.ifError(error);
    ItemComment.create({ _id: 3, comment: 'xyz' }, function(error, itemComment) {
      assert.ifError(error);
      ActivityLiked.create({ user: 1, from: 1, item: 2 }, function(error, activityLiked) {
        assert.ifError(error);
        const acms = [
          { user: 1, from: 1, comment: 3, item: 2 },
          { user: 1, from: 1, comment: 3, item: 2 },
          { user: 1, from: 1, comment: 3, item: 2 },
          { user: 1, from: 1, comment: 3, item: 2 },
          { user: 1, from: 1, comment: 3, item: 2 },
          { user: 1, from: 1, comment: 3, item: 2 },
          { user: 1, from: 1, comment: 3, item: 2 },
          { user: 1, from: 1, comment: 3, item: 2 },
          { user: 1, from: 1, comment: 3, item: 2 },
          { user: 1, from: 1, comment: 3, item: 2 }
        ];
        ActivityCommentMentioned.create(acms, function(error, acm) {
          assert.ifError(error);
          test();
        });
      });
    });
  });
});

function test() {
  Activity.
    find({}).
    sort({ _id: 1 }).
    limit(30).
    populate('user from item comment').
    exec(function(error, docs) {
      assert.ifError(error);
      assert.equal(docs.length, 11);
      for (let i = 1; i < docs.length; ++i) {
        assert.equal(docs[i].comment.comment, 'xyz');
      }
      console.log('done');
      process.exit(0);
    });
}

[aforty]

I wrote a test that demonstrates the issue. Run it with mocha from the source directory.

/**
 * Test dependencies.
 */

var start = require('./common');
var assert = require('power-assert');
var mongoose = start.mongoose;
var utils = require('../lib/utils');
var random = utils.random;
var Schema = mongoose.Schema;
var ObjectId = Schema.ObjectId;
var async = require('async');

/**
 * Setup.
 */

/**
 * User schema.
 */

var UserSchema = new Schema({
  name: String
});

/**
 * Comment subdocument schema.
 */

var CommentSchema = new Schema({
  content: String
});

/**
 * Blog post schema.
 */

var BlogPostSchema = new Schema({
  title: String
});

/**
 * Event schema.
 */

var EventSchema = new Schema({
  name: String,
  createdAt: { type: Date, default: Date.now }
});

/**
 * User event schema.
 */

var UserEventSchema = new Schema({
  user: { type: ObjectId, ref: 'RefUser' }
});

/**
 * Comment event schema.
 */

var CommentEventSchema = new Schema({
  comment: { type: ObjectId, ref: 'RefComment' }
});

/**
 * Blog post event schema.
 */

var BlogPostEventSchema = new Schema({
  blogpost: { type: ObjectId, ref: 'RefBlogPost' }
});

describe('model', function() {
  describe('discriminator populating', function() {
    var db, User, Comment, BlogPost, Event, UserEvent, CommentEvent, BlogPostEvent;

    before(function() {
      db = start();
      User = db.model('RefUser', UserSchema, 'model-discriminator-populating-user-' + random());
      Comment = db.model('RefComment', CommentSchema, 'model-discriminator-populating-comment-' + random());
      BlogPost = db.model('RefBlogPost', BlogPostSchema, 'model-discriminator-populating-blogpost-' + random());

      Event = db.model('model-discriminator-populating-event', EventSchema, 'model-discriminator-querying-' + random());
      UserEvent = Event.discriminator('model-discriminator-populating-user', UserEventSchema);
      CommentEvent = Event.discriminator('model-discriminator-populating-comment', CommentEventSchema);
      BlogPostEvent = Event.discriminator('model-discriminator-populating-blogpost', BlogPostEventSchema);
    });

    afterEach(function(done) {
      async.series(
        [
          function(next) { Event.remove(next); },
          function(next) { User.remove(next); },
          function(next) { Comment.remove(next); },
          function(next) { BlogPost.remove(next); }
        ],
        done
      );
    });

    after(function(done) {
      db.close(done);
    });

    it('should populate all fields indicated', function(done) {
      var u1 = new User({ name: 'user 1' });
      var u2 = new User({ name: 'user 2' });
      var u3 = new User({ name: 'user 3' });
      var c1 = new Comment({ content: 'comment 1' });
      var c2 = new Comment({ content: 'comment 2' });
      var c3 = new Comment({ content: 'comment 3' });
      var b1 = new BlogPost({ title: 'blog post 1' });
      var b2 = new BlogPost({ title: 'blog post 2' });
      var b3 = new BlogPost({ title: 'blog post 3' });
      var ue1 = new UserEvent({ user: u1 });
      var ue2 = new UserEvent({ user: u2 });
      var ue3 = new UserEvent({ user: u3 });
      var ce1 = new CommentEvent({ comment: c1 });
      var ce2 = new CommentEvent({ comment: c2 });
      var ce3 = new CommentEvent({ comment: c3 });
      var be1 = new BlogPostEvent({ blogpost: b1 });
      var be2 = new BlogPostEvent({ blogpost: b2 });
      var be3 = new BlogPostEvent({ blogpost: b3 });

      async.series(
        [
          u1.save,
          u2.save,
          u3.save,

          c1.save,
          c2.save,
          c3.save,

          b1.save,
          b2.save,
          b3.save,

          ce1.save,
          ue1.save,
          be1.save,

          ce2.save,
          ue2.save,
          be2.save,

          ce3.save,
          ue3.save,
          be3.save,

          function(next) {
            Event
              .find({})
              .populate('user comment blogpost')
              .exec(function(err, docs) {
                console.log(docs);

                docs.forEach(function(doc) {
                  if (doc.__t === 'model-discriminator-populating-user') {
                    assert.ok(doc.user !== null, 'user is null');
                  }
                  else if (doc.__t === 'model-discriminator-populating-comment') {
                    assert.ok(doc.comment !== null, 'comment is null');
                  }
                  else if (doc.__t === 'model-discriminator-populating-blogpost') {
                    assert.ok(doc.blogpost !== null, 'blogpost is null');
                  }
                });
                next();
              });
          }
        ],
        done
      );
    });
  });
});

[vkarpov15]

Yep turns out it was a pretty subtle bug in how we ordered documents when populating. Should be fixed with the above commit, I'll push a release on monday.

[aforty]

That's amazing, thanks for the quick fix and push!