dynamic populate with refPath in sub-document

[iammola]

Prerequisites

• I have written a descriptive issue title
• I have searched existing issues to ensure the bug has not already been reported

Mongoose version

6.4.3

Node.js version

18.4.0

MongoDB server version

5.0

Description

This is most likely a bug, but I'm not sure.

I have a path in my model that is a sub-document, and I want to dynamically populate another path in that sub-document using the refPath option.

import mongoose from "mongoose";

const subSchema = new mongoose.Schema({
  by: {
    type: mongoose.Schema.Types.ObjectId,
    refPath: "user_type", // path to model
    required: true,
  },
  user_type: {
    type: String,
    required: true,
    // the `by` field should be populated on one of these models
    enum: ["Parent", "Child"],
  },
});

const RecordSchema = new mongoose.Schema({
  title: {
    type: String,
    required: true,
  },
  created: {
    type: subSchema,
    required: true
  },
});

const ParentSchema = new mongoose.Schema({
  name: {
    type: String,
    required: true,
  },
  job: { type: String }
});

const ChildSchema = new mongoose.Schema({
  name: {
    type: String,
    required: true
  },
  siblings: [mongoose.Schema.Types.ObjectId]
});

I then query the database for a document and populate the created.by field, but the path isn't replaced with null or the foreign document.

import mongoose from "mongoose";

async function run() {
  const [parent, child] = await Promise.all([
    ParentModel.create({ name:"Lionel Messi", job: "Footballer" }),
    ChildModel.create({ name: "Thiago Messi" })
  ]);

  await RecordModel.create([
    { title: "Heart of a Lio", created: { by: parent._id, user_type: "Parent" } },
    { title: "My day with Hulk", created: { by: child._id, user_type: "Child" } },
    { title: "How to be the GOAT", created: { by: parent._id, user_type: "Parent" } },
  ]);

  return await RecordModel.find({}).populate("created.by").lean().exec();
}

const RecordModel = mongoose.model('Record', RecordSchema);
const ParentModel = mongoose.model('Parent', ParentSchema);
const ChildModel = mongoose.model('Child', ChildSchema);

run().then(console.log);

// [{
//   title: "Heart of a Lio",
//   created: {
//     by: "ParentID",
//     user_type: "Parent"
//   }
// }, { ... }, { ... }]

Steps to Reproduce

• Create two schema for a model. One as a sub-document path to the other.
• On the schema to be used as a sub-document, add a path that will hold the foreign document's _id_ and another to specify what model to populate on.
• Make a model with the top-level schema
• Query the model and populate on the path in the sub-document that leads to the foreign document's _id.

Expected Behavior

For the sub-document path to be populated with the dynamically specified model.

[iammola]

Note
I don't know enough about Mongoose's complete architecture, there could be a reason this hasn't been done before or why it was done this way. But I got the documents to be populated by making changes to an internal file. I didn't test this with any other use cases, so there may be other ways this will break people's code.

When I wasn't sure what the issue was, I used a function as the refPath and console.traced my way to this file.

mongoose/lib/helpers/populate/getModelsMapForPopulate.js

Lines 231 to 247 in a1d88e5

	if (typeof refPath === 'function') {
	const subdocPath = options.path.slice(0, options.path.length - schema.path.length - 1);
	const vals = mpath.get(subdocPath, doc, lookupLocalFields);
	const subdocsBeingPopulated = Array.isArray(vals) ?
	utils.array.flatten(vals) :
	(vals ? [vals] : []);
	modelNames = new Set();
	for (const subdoc of subdocsBeingPopulated) {
	refPath = refPath.call(subdoc, subdoc, options.path);
	modelNamesFromRefPath(refPath, doc, options.path, modelSchema, options._queryProjection).
	forEach(name => modelNames.add(name));
	}
	modelNames = Array.from(modelNames);
	} else {
	modelNames = modelNamesFromRefPath(refPath, doc, options.path, modelSchema, options._queryProjection);
	}

I realized that the resolved modelNames from modelNamesFromRefPath is [undefined] because you try to get the model refPath leads relative to the top document rather than to the sub-document. Which is undefined when the refPath on the top document doesn't exist.

mongoose/lib/helpers/populate/modelNamesFromRefPath.js

Line 56 in a1d88e5

const refValue = mpath.get(refPath, doc, lookupLocalFields);

So I got the documents to be populated in two ways

• By passing the subdoc to the modelNamesFromRefPath function

const subdocPath = options.path.slice(0, options.path.length - schema.path.length - 1);
const vals = mpath.get(subdocPath, doc, lookupLocalFields);
const subdocsBeingPopulated = Array.isArray(vals) ? utils.array.flatten(vals) : (vals ? [vals] : []);

for (const subdoc of subdocsBeingPopulated) {
  // !! ^^ This was only passed to the refPath if it was a function
  if (typeof refPath === 'function') {
    modelNames = new Set();
    refPath = refPath.call(subdoc, subdoc, options.path);
    modelNamesFromRefPath(refPath, subdoc, options.path, modelSchema, options._queryProjection).forEach(name => modelNames.add(name));
    //                            !! ^^ doc to subdoc
    modelNames = Array.from(modelNames);
  } else {
    modelNames = modelNamesFromRefPath(refPath, subdoc, options.path, modelSchema, options._queryProjection);
    //                            !! ^^ doc to subdoc
  }
}

• By passing the path the refPath is referring to relative to the top-level document

const subdocPath = options.path.slice(0, options.path.length - schema.path.length - 1);

if (typeof refPath === 'function') {
  const vals = mpath.get(subdocPath, doc, lookupLocalFields);
  const subdocsBeingPopulated = Array.isArray(vals) ?
    utils.array.flatten(vals) :
    (vals ? [vals] : []);

  modelNames = new Set();
  for (const subdoc of subdocsBeingPopulated) {
    refPath = refPath.call(subdoc, subdoc, options.path);

    modelNamesFromRefPath(`${subdocPath}.${refPath}`, doc, options.path, modelSchema, options._queryProjection).forEach(name => modelNames.add(name));
    //                         !! ^^ new path 
  }
  modelNames = Array.from(modelNames);
} else {
  modelNames = modelNamesFromRefPath(`${subdocPath}.${refPath}`, doc, options.path, modelSchema, options._queryProjection);
  //                                   !! ^^ new path 
}

[IslandRhythms]

All the enum field does is ensure that the property can only be one of the defined values specified in enum. If you want to only populate the by field in created when the user_type is either Parent or Child, your repro script doesn't even make an attempt. Your blindly populating the field on the model. You would most likely need to check the model name and if it is Parent or Child, perform the query with the populate, otherwise omit the populate.

[iammola]

Yeah @IslandRhythms. I forgot to make the schema definitions for the other two models. I've updated my repro script with it and dummy content.

[IslandRhythms]

const mongoose = require('mongoose');

const subSchema = new mongoose.Schema({
  by: {
    type: mongoose.Schema.Types.ObjectId,
    refPath: "user_type", // path to model
    required: true,
  },
  user_type: {
    type: String,
    required: true,
    // the `by` field should be populated on one of these models
    enum: ["Parent", "Child"],
  },
});

const RecordSchema = new mongoose.Schema({
  title: {
    type: String,
    required: true,
  },
  created: {
    type: subSchema,
    required: true
  },
});

const ParentSchema = new mongoose.Schema({
  name: {
    type: String,
    required: true,
  },
  job: { type: String }
});

const ChildSchema = new mongoose.Schema({
  name: {
    type: String,
    required: true
  },
  siblings: [mongoose.Schema.ObjectId]
});

const Record = mongoose.model('Record', RecordSchema);
const Parent = mongoose.model('Parent', ParentSchema);
const Child = mongoose.model('Child', ChildSchema);

async function run() {
  await mongoose.connect('mongodb://localhost:27017');
  await mongoose.connection.dropDatabase();

  const [parent, child] = await Promise.all([
    Parent.create({ name:"Lionel Messi", job: "Footballer" }),
    Child.create({ name: "Thiago Messi" })
  ]);

  await Record.create([
    { title: "Heart of a Lio", created: { by: parent._id, user_type: "Parent" } },
    { title: "My day with Hulk", created: { by: child._id, user_type: "Child" } },
    { title: "How to be the GOAT", created: { by: parent._id, user_type: "Parent" } },
  ]);

  console.log(await Record.find().populate('created.by').lean());
}

run();

[vkarpov15]

@iammola unfortunately, refPath requires you pass the full path relative to the top-level document, not the path relative to the subdocument. So, in this case, you would need to do:

const subSchema = new mongoose.Schema({
  by: {
    type: mongoose.Schema.Types.ObjectId,
    refPath: "created.user_type", // <-- add `created.` here
    required: true,
  },
  user_type: {
    type: String,
    required: true,
    // the `by` field should be populated on one of these models
    enum: ["Parent", "Child"],
  },
});

This is a design decision to make it easier to define a refPath that isn't tied to the subdocument, like if you actually wanted user_type as a top-level property in the document.

If you prefer to specify the path relative to the subdocument, rather than the path relative to the top-level document, it is easier to use ref as a function as follows:

const subSchema = new mongoose.Schema({
  by: {
    type: mongoose.Schema.Types.ObjectId,
    ref: subdoc => subdoc.user_type, // <-- `subdoc` is the subdoc being populated
    required: true,
  },
  user_type: {
    type: String,
    required: true,
    // the `by` field should be populated on one of these models
    enum: ["Parent", "Child"],
  },
});