This commit is contained in:
Jonasz Bigda
2023-03-25 21:51:42 +01:00
parent 0db1d5117e
commit b332e9ceb0
1044 changed files with 37502 additions and 63938 deletions

View File

@@ -8,7 +8,9 @@ const Readable = require('stream').Readable;
const promiseOrCallback = require('../helpers/promiseOrCallback');
const eachAsync = require('../helpers/cursor/eachAsync');
const helpers = require('../queryhelpers');
const immediate = require('../helpers/immediate');
const util = require('util');
const utils = require('../../lib/utils');
/**
* A QueryCursor is a concurrency primitive for processing query results
@@ -16,8 +18,8 @@ const util = require('util');
* in addition to several other mechanisms for loading documents from MongoDB
* one at a time.
*
* QueryCursors execute the model's pre find hooks, but **not** the model's
* post find hooks.
* QueryCursors execute the model's pre `find` hooks before loading any documents
* from MongoDB, and the model's post `find` hooks after loading each document.
*
* Unless you're an advanced user, do **not** instantiate this class directly.
* Use [`Query#cursor()`](/docs/api.html#query_Query-cursor) instead.
@@ -33,7 +35,14 @@ const util = require('util');
*/
function QueryCursor(query, options) {
Readable.call(this, { objectMode: true });
const streamOpts = { objectMode: true };
// for node < 12 we will emit 'close' event after 'end'
if (utils.nodeMajorVersion() >= 12) {
// set autoDestroy=true because on node 12 it's by default false
// gh-10902 need autoDestroy to destroy correctly and emit 'close' event for node >= 12
streamOpts.autoDestroy = true;
}
Readable.call(this, streamOpts);
this.cursor = null;
this.query = query;
@@ -54,11 +63,18 @@ function QueryCursor(query, options) {
if (this.options.batchSize) {
this.options.cursor = options.cursor || {};
this.options.cursor.batchSize = options.batchSize;
// Max out the number of documents we'll populate in parallel at 5000.
this.options._populateBatchSize = Math.min(this.options.batchSize, 5000);
}
model.collection.find(query._conditions, this.options, function(err, cursor) {
if (_this._error) {
cursor.close(function() {});
if (cursor != null) {
cursor.close(function() {});
}
_this.emit('cursor', null);
_this.listeners('error').length > 0 && _this.emit('error', _this._error);
return;
}
if (err) {
return _this.emit('error', err);
@@ -87,13 +103,10 @@ QueryCursor.prototype._read = function() {
if (error) {
return _this.emit('error', error);
}
setTimeout(function() {
// on node >= 14 streams close automatically (gh-8834)
const isNotClosedAutomatically = !_this.destroyed;
if (isNotClosedAutomatically) {
_this.emit('close');
}
}, 0);
// for node >= 12 the autoDestroy will emit the 'close' event
if (utils.nodeMajorVersion() < 12) {
_this.on('end', () => _this.emit('close'));
}
});
return;
}
@@ -118,7 +131,7 @@ QueryCursor.prototype._read = function() {
* on('data', function(doc) { console.log(doc.foo); });
*
* // Or map documents returned by `.next()`
* var cursor = Thing.find({ name: /^hello/ }).
* const cursor = Thing.find({ name: /^hello/ }).
* cursor().
* map(function (doc) {
* doc.foo = "bar";
@@ -270,6 +283,61 @@ QueryCursor.prototype.transformNull = function(val) {
return this;
};
/*!
* ignore
*/
QueryCursor.prototype._transformForAsyncIterator = function() {
if (this._transforms.indexOf(_transformForAsyncIterator) === -1) {
this.map(_transformForAsyncIterator);
}
return this;
};
/**
* Returns an asyncIterator for use with [`for/await/of` loops](https://thecodebarbarian.com/getting-started-with-async-iterators-in-node-js).
* You do not need to call this function explicitly, the JavaScript runtime
* will call it for you.
*
* ####Example
*
* // Works without using `cursor()`
* for await (const doc of Model.find([{ $sort: { name: 1 } }])) {
* console.log(doc.name);
* }
*
* // Can also use `cursor()`
* for await (const doc of Model.find([{ $sort: { name: 1 } }]).cursor()) {
* console.log(doc.name);
* }
*
* Node.js 10.x supports async iterators natively without any flags. You can
* enable async iterators in Node.js 8.x using the [`--harmony_async_iteration` flag](https://github.com/tc39/proposal-async-iteration/issues/117#issuecomment-346695187).
*
* **Note:** This function is not if `Symbol.asyncIterator` is undefined. If
* `Symbol.asyncIterator` is undefined, that means your Node.js version does not
* support async iterators.
*
* @method Symbol.asyncIterator
* @memberOf Query
* @instance
* @api public
*/
if (Symbol.asyncIterator != null) {
QueryCursor.prototype[Symbol.asyncIterator] = function() {
return this.transformNull()._transformForAsyncIterator();
};
}
/*!
* ignore
*/
function _transformForAsyncIterator(doc) {
return doc == null ? { done: true } : { value: doc, done: false };
}
/*!
* Get the next doc from the underlying cursor and mongooseify it
* (populate, etc.)
@@ -289,46 +357,132 @@ function _next(ctx, cb) {
}
if (ctx._error) {
return process.nextTick(function() {
return immediate(function() {
callback(ctx._error);
});
}
if (ctx.cursor) {
return ctx.cursor.next(function(error, doc) {
if (error) {
return callback(error);
}
if (!doc) {
return callback(null, null);
}
const opts = ctx.query._mongooseOptions;
if (!opts.populate) {
return opts.lean ?
callback(null, doc) :
_create(ctx, doc, null, callback);
}
const pop = helpers.preparePopulationOptionsMQ(ctx.query,
if (ctx.query._mongooseOptions.populate && !ctx._pop) {
ctx._pop = helpers.preparePopulationOptionsMQ(ctx.query,
ctx.query._mongooseOptions);
pop.__noPromise = true;
ctx.query.model.populate(doc, pop, function(err, doc) {
if (err) {
return callback(err);
ctx._pop.__noPromise = true;
}
if (ctx.query._mongooseOptions.populate && ctx.options._populateBatchSize > 1) {
if (ctx._batchDocs && ctx._batchDocs.length) {
// Return a cached populated doc
return _nextDoc(ctx, ctx._batchDocs.shift(), ctx._pop, callback);
} else if (ctx._batchExhausted) {
// Internal cursor reported no more docs. Act the same here
return callback(null, null);
} else {
// Request as many docs as batchSize, to populate them also in batch
ctx._batchDocs = [];
return ctx.cursor.next(_onNext.bind({ ctx, callback }));
}
} else {
return ctx.cursor.next(function(error, doc) {
if (error) {
return callback(error);
}
return opts.lean ?
callback(null, doc) :
_create(ctx, doc, pop, callback);
if (!doc) {
return callback(null, null);
}
if (!ctx.query._mongooseOptions.populate) {
return _nextDoc(ctx, doc, null, callback);
}
ctx.query.model.populate(doc, ctx._pop, function(err, doc) {
if (err) {
return callback(err);
}
return _nextDoc(ctx, doc, ctx._pop, callback);
});
});
});
}
} else {
ctx.once('cursor', function() {
ctx.once('cursor', function(cursor) {
if (cursor == null) {
return;
}
_next(ctx, cb);
});
}
}
/*!
* ignore
*/
function _onNext(error, doc) {
if (error) {
return this.callback(error);
}
if (!doc) {
this.ctx._batchExhausted = true;
return _populateBatch.call(this);
}
this.ctx._batchDocs.push(doc);
if (this.ctx._batchDocs.length < this.ctx.options._populateBatchSize) {
// If both `batchSize` and `_populateBatchSize` are huge, calling `next()` repeatedly may
// cause a stack overflow. So make sure we clear the stack regularly.
if (this.ctx._batchDocs.length > 0 && this.ctx._batchDocs.length % 1000 === 0) {
return immediate(() => this.ctx.cursor.next(_onNext.bind(this)));
}
this.ctx.cursor.next(_onNext.bind(this));
} else {
_populateBatch.call(this);
}
}
/*!
* ignore
*/
function _populateBatch() {
if (!this.ctx._batchDocs.length) {
return this.callback(null, null);
}
const _this = this;
this.ctx.query.model.populate(this.ctx._batchDocs, this.ctx._pop, function(err) {
if (err) {
return _this.callback(err);
}
_nextDoc(_this.ctx, _this.ctx._batchDocs.shift(), _this.ctx._pop, _this.callback);
});
}
/*!
* ignore
*/
function _nextDoc(ctx, doc, pop, callback) {
if (ctx.query._mongooseOptions.lean) {
return ctx.model.hooks.execPost('find', ctx.query, [[doc]], err => {
if (err != null) {
return callback(err);
}
callback(null, doc);
});
}
_create(ctx, doc, pop, (err, doc) => {
if (err != null) {
return callback(err);
}
ctx.model.hooks.execPost('find', ctx.query, [[doc]], err => {
if (err != null) {
return callback(err);
}
callback(null, doc);
});
});
}
/*!
* ignore
*/
@@ -337,7 +491,10 @@ function _waitForCursor(ctx, cb) {
if (ctx.cursor) {
return cb();
}
ctx.once('cursor', function() {
ctx.once('cursor', function(cursor) {
if (cursor == null) {
return;
}
cb();
});
}