Increasing the "through2" highWaterMark

[mdurchholz]

I am using your plugin to loop through a set of SCSS files and compile them into beautified CSS for documentation purposes. Anyways, I am running into an issue where the amount of loop iterations is maxing out at 16 files. After doing some research, I see that there is a property in "through2" called "highWaterMark" which is set to 16 by default.

gulpjs/gulp#716

Is there a way to update this number through your plugin?

[mariusGundersen]

Do you have some code? I suspect you are not returning the stream inside the flatMap callback. As the comment in your link says, if nothing is consuming the stream then it won't continue processing.

[mdurchholz]

I return stream once to avoid my "style.scss" file on the level of my sourceLib.

var sourceLib = "path/to/sources/";

//  librarySources should return a length 25
var librarySources = [
    sourceLib+'**/*.scss', // Search for all SCSS files
    '!'+sourceLib          // but exclude the first level
];

gulp.task('createExampleCSS', function(done){

    gulp.src(librarySources)
        .pipe(plugins.flatmap(function(stream, file)
        {
            var basePath = file.path.replace(__dirname+'/'+sourceLib, '');

            // Ignore any files on the first level (Ex. the main style.scss that is unrelated to this gulp function) 
            if( basePath.split('/').length == 1 ) return stream;

            // Only returns 16 paths
            console.log(basePath);

            // Run custom gulp on individual file that creates a beautified CSS file of the SCSS file
            return gulp
            .src([ ... ])
            .pipe( ... )
            .pipe( gulp.dest("path/to/production/") );
        }));

    done();

});

[mdurchholz]

I can actually update my snippet to the following and get the same result without return stream:

var sourceLib = "path/to/sources/";

// librarySources should return 25 files
var librarySources = [
    sourceLib+'**/*.scss',     // Search for all SCSS files
    '!'+sourceLib+'style.scss' // but exclude the style.scss
];

gulp.task('example', function(done){

    gulp.src(librarySources)
        .pipe(plugins.flatmap(function(stream, file)
        {
            var basePath = file.path.replace(__dirname+'/'+sourceLib, '');

            // Only returns 16 paths
            console.log(basePath);

            // Run gulp on individual file
            return gulp
            .src([ ... ])
            .pipe( ... );
        }));

    done();

});

[mdurchholz]

Ahhh, I see what needs to be done now from that post I shared regarding the "consumption". I am all set. Thanks for the quick responses!

var browserSync = require('browser-sync').create();

var sourceLib = "path/to/sources/";

// librarySources should return 25 files
var librarySources = [
    sourceLib+'**/*.scss',     // Search for all SCSS files
    '!'+sourceLib+'style.scss' // but exclude the style.scss
];

gulp.task('example', function(done){

    gulp.src(librarySources)
        .pipe(plugins.flatmap(function(stream, file)
        {
            var basePath = file.path.replace(__dirname+'/'+sourceLib, '');

            // Now returns everything!
            console.log(basePath);

            // Run gulp on individual file
            return gulp
            .src([ ... ])
            .pipe( ... )
            .pipe( gulp.dest("path/to/production/") );
        }))

       // "Consume" the stream
       .pipe( browserSync.stream() );

    done();

});

[mariusGundersen]

Here is how I would do it:

gulp.task('example', function(){
    return gulp.src(librarySources)
        .pipe(plugins.flatmap(function(stream, file)
        {
            var basePath = file.path.replace(__dirname+'/'+sourceLib, '');

            console.log(basePath);

            // Run gulp on individual file
            return gulp
            .src([ ... ])
            .pipe( ... );
        }))
        .pipe( gulp.dest("path/to/production/") );
});

This way the task isn't marked as done until it is actually done processing (by returning the stream) and you consume the entire stream (by having gulp.dest() outside of flatMap().