How to batch similar gulp tasks to reduce code rep

Question

I have written a gulp file that watches over several directories for changes, and then create concatenation to multiple specified destination.

Here is a simplified version of my project structure:

I have 2 site folders: one/ and two/

Each site have two branch folders: a/ and b/

Inside each branch, there are three folders: inner/, outer/ and web/

My task is to grab similar part files from the inner and outer folders, and concatenate them into relative web folders. Below is a simple example of desired output.

-- inner/
 |-- color1
 |-- color2
 |-- fruit1
 |-- fruit2
-- outer/
 |-- color1
 |-- color2
 |-- fruit1
 |-- fruit2
-- web/
 |-- colors.txt
 |-- fruits.txt

I have created a config.json file to hold site specific configuration. Currently only using it to customize site paths. Here is the config.json

{
  "sites": {
    "one": {
      "a": "/path/to/one/a/",
      "b": "/path/to/one/b/"
    },
    "two": {
      "a": "/path/to/two/a/",
      "b": "/path/to/two/b/"
    }
  }
}

And finally here is the gulpfile.js

// Include local Gulp
var gulp = require("gulp");

// Get data from config.json
var sites = require("./config.json").sites;

// Include Gulp specific plugins
var gConcat = require("gulp-concat");
var gHeader = require("gulp-header");
var gUtil = require("gulp-util");
var gNotify = require("gulp-notify");

// Setup directories
var outer = "outer/";
var inner = "inner/";
var web = "web/";

// Misc
var alertMessage = "# GENERATED FILE - DO NOT MODIFY\n\n";

// 8 total tasks for concatenation

// Concatenate to colors.txt - 4 tasks
// Color task 1: [ Site => one ] [ Branch => a ]
gulp.task("one_a_color", function() {
    return gulp.src([sites.one.a + outer + "color?", sites.one.a + inner + "color?"])
        .pipe(gConcat("colors.txt"))
        .pipe(gHeader(alertMessage))
        .pipe(gulp.dest(sites.one.a + web))
        .pipe(gNotify());
});

// Color task 2: [ Site => one ] [ Branch => b ]
gulp.task("one_b_color", function() {
    return gulp.src([sites.one.b + outer + "color?", sites.one.b + inner + "color?"])
        .pipe(gConcat("colors.txt"))
        .pipe(gHeader(alertMessage))
        .pipe(gulp.dest(sites.one.b + web))
        .pipe(gNotify());
});

// Color task 3: [ Site => two ] [ Branch => a ]
gulp.task("two_a_color", function() {
    return gulp.src([sites.two.a + outer + "color?", sites.two.a + inner + "color?"])
        .pipe(gConcat("colors.txt"))
        .pipe(gHeader(alertMessage))
        .pipe(gulp.dest(sites.two.a + web))
        .pipe(gNotify());
});

// Color task 4: [ Site => two ] [ Branch => b ]
gulp.task("two_b_color", function() {
    return gulp.src([sites.two.b + outer + "color?", sites.two.b + inner + "color?"])
        .pipe(gConcat("colors.txt"))
        .pipe(gHeader(alertMessage))
        .pipe(gulp.dest(sites.two.b + web))
        .pipe(gNotify());
});

// Concatenate to fruits.txt - 4 tasks
// Fruit task 1: [ Site => one ] [ Branch => a ]
gulp.task("one_a_fruit", function() {
    return gulp.src([sites.one.a + outer + "fruit?", sites.one.a + inner + "fruit?"])
        .pipe(gConcat("fruits.txt"))
        .pipe(gHeader(alertMessage))
        .pipe(gulp.dest(sites.one.a + web))
        .pipe(gNotify());
});

// Fruit task 2: [ Site => one ] [ Branch => b ]
gulp.task("one_b_fruit", function() {
    return gulp.src([sites.one.b + outer + "fruit?", sites.one.b + inner + "fruit?"])
        .pipe(gConcat("fruits.txt"))
        .pipe(gHeader(alertMessage))
        .pipe(gulp.dest(sites.one.b + web))
        .pipe(gNotify());
});

// Fruit task 3: [ Site => two ] [ Branch => a ]
gulp.task("two_a_fruit", function() {
    return gulp.src([sites.two.a + outer + "fruit?", sites.two.a + inner + "fruit?"])
        .pipe(gConcat("fruits.txt"))
        .pipe(gHeader(alertMessage))
        .pipe(gulp.dest(sites.two.a + web))
        .pipe(gNotify());
});

// Fruit task 4: [ Site => two ] [ Branch => b ]
gulp.task("two_b_fruit", function() {
    return gulp.src([sites.two.b + outer + "fruit?", sites.two.b + inner + "fruit?"])
        .pipe(gConcat("fruits.txt"))
        .pipe(gHeader(alertMessage))
        .pipe(gulp.dest(sites.two.b + web))
        .pipe(gNotify());
});

// Watch for all events in specified {directories}/{files}, then trigger appropriate task
// 8 total watch jobs
gulp.task("watch", function () {
    // Color related watch jobs - Total 4
    // Color watch 1: [ Site => one ] [ Branch => a ]
    gulp.watch([sites.one.a + outer + "**/color?", sites.one.a + inner + "**/color?"], function(event) {
        gUtil.log(event.path.split("/").pop(), "=>", event.type);
        gulp.start("one_a_color");
    });

    // Color watch 2: [ Site => one ] [ Branch => b ]
    gulp.watch([sites.one.b + outer + "**/color?", sites.one.b + inner + "**/color?"], function(event) {
        gUtil.log(event.path.split("/").pop(), "=>", event.type);
        gulp.start("one_b_color");
    });

    // Color watch 3: [ Site => two ] [ Branch => a ]
    gulp.watch([sites.two.a + outer + "**/color?", sites.two.a + inner + "**/color?"], function(event) {
        gUtil.log(event.path.split("/").pop(), "=>", event.type);
        gulp.start("two_a_color");
    });

    // Color watch 4: [ Site => two ] [ Branch => b ]
    gulp.watch([sites.one.b + outer + "**/color?", sites.one.b + inner + "**/color?"], function(event) {
        gUtil.log(event.path.split("/").pop(), "=>", event.type);
        gulp.start("two_b_color");
    });

    // Fruit related watch jobs - Total 4
    // Fruit watch 1: [ Site => one ] [ Branch => a ]
    gulp.watch([sites.one.a + outer + "**/fruit?", sites.one.a + inner + "**/fruit?"], function(event) {
        gUtil.log(event.path.split("/").pop(), "=>", event.type);
        gulp.start("one_a_fruit");
    });

    // Fruit watch 2: [ Site => one ] [ Branch => b ]
    gulp.watch([sites.one.b + outer + "**/fruit?", sites.one.b + inner + "**/fruit?"], function(event) {
        gUtil.log(event.path.split("/").pop(), "=>", event.type);
        gulp.start("one_b_fruit");
    });

    // Fruit watch 3: [ Site => two ] [ Branch => a ]
    gulp.watch([sites.two.a + outer + "**/fruit?", sites.two.a + inner + "**/fruit?"], function(event) {
        gUtil.log(event.path.split("/").pop(), "=>", event.type);
        gulp.start("two_a_fruit");
    });

    // Fruit watch 4: [ Site => two ] [ Branch => b ]
    gulp.watch([sites.one.b + outer + "**/fruit?", sites.one.b + inner + "**/fruit?"], function(event) {
        gUtil.log(event.path.split("/").pop(), "=>", event.type);
        gulp.start("two_b_fruit");
    });
});

// Run all tasks
gulp.task("background",
    [
        "one_a_color", "one_b_color", "two_a_color", "two_b_color",
        "one_a_fruit", "one_b_fruit", "two_a_fruit", "two_b_fruit",
        "watch"
    ]
);

The above gulp file works and does the job. However, as you can see, most of the codes are repeated, only part that changes are the gulp.src and gulp.dest, along with the task names.

My question is. Would it be possible to simplify this gulp file, so instead of repeating codes for every tasks, maybe similar tasks can be batched together.

Answer 1

Not that easy a task, but let's see if we can optimise that. Gulp and Globs greatly deal with arrays, that's why we have to convert your paths to an array first:

var gulp = require('gulp');
var concat = require('gulp-concat');
var es = require('event-stream');

var sites = require('./config.json').sites;

var toArray = function(conf) {
    var arr = [];
    for(var key in conf) {
        if(typeof conf[key] === 'object') {
            arr = arr.concat(toArray(conf[key]));
        } else {
            arr.push(conf[key]);
        }
    }
    return arr;
};

var sites = toArray(sites);

Now that we have the paths, we create the globs for fruits and colors.

var globs = [];
sites.forEach(function(data) {
    globs.push(data + '**/color*');
    globs.push(data + '**/fruit*');
});

With your current config, you get an array of 8 entries. Next, let us define the concat-task. Here is what you mean with "batched" together, we need a so called stream array (I wrote about that here). It's a simple mapping of an existing array to many gulp streams, which are merged at the end via the event-stream module. With the color/fruit thing going on, we need to be a little creative with our concat names and dest names. Note that I use the changed plugin to prevent useless builds.

gulp.task('concat', function() {
    var tasks = globs.map(function(glob) {
        var file = glob.indexOf('color') >= 0 ? 'col' : 'fru';
        var dest = glob.replace('**/color*','').replace('**/fruit*','') + 'web';
        return gulp.src(glob)
            .pipe(concat(file + '.txt'))
            .pipe(gulp.dest(dest))
    });

    return es.merge.apply(null, tasks);
});

This task now does everything we need, and incrementally so. So our watch process is rather straightforward.

gulp.task('watch', ['concat'], function() {
    gulp.watch(globs, ['concat']);
});

Hope this helps!

Update

Alright, I made some adaptations, which should prevent having your whole project rebuilt.

First, I extracted the concatStream to a function. This is actually the one thing you already did with your own sample:

var concatStream = function(glob) {
    var file = glob.indexOf('color') >= 0 ? 'farbe' : 'frucht';
    var dest = glob.replace('**/color*','').replace('**/fruit*','') + 'web';
    return gulp.src(glob)
        .pipe(concat(file + '.txt'))
        .pipe(header(alertMessage))
        .pipe(notify())
        .pipe(gulp.dest(dest))
};

Depending on the Glob (the file pattern we select either colors or fruits from our directories), we define a new output (file, is 'col' when 'color' is in our search string, 'fru' otherwise) and a new destination (which is just the old folder without the colors/fruits search pattern). gulp.task('concat') does now the following:

gulp.task('concat', function() {
    var tasks = globs.map(concatStream);
    return es.merge.apply(null, tasks);
});

Each of our globs (console.log them, if you want to know what's in there) gets mapped to the concatStream, then the new array of streams gets merged and executed.

The watch task is now new... we do kinda the same as with our 'concat' task:

gulp.task('watch', ['concat'], function() {
    globs.map(function(glob) {
        gulp.watch(glob, function() {
           return concatStream(glob);
        })
    })
});

For each glob, we create a new watcher, which just calls the concatStream again.

---

Update

Small change

Inside glob, changing the wildcard (*) to an optional single character match (?), will allow us to use the same name for output file (ex. color and fruit).

var globs = [];
    sites.forEach(function(data) {
        globs.push(data + '**/color?');
        globs.push(data + '**/fruit?');
    });

And this as well...

var concatStream = function(glob) {
    var file = glob.indexOf('color') >= 0 ? 'color' : 'fruit';
    var dest = glob.replace('**/color?','').replace('**/fruit?','') + 'web';
    return gulp.src(glob)
        .pipe(concat(file + '.txt'))
        .pipe(header(alertMessage))
        .pipe(notify())
        .pipe(gulp.dest(dest))
};

Now I can keep the names of color and fruit for my output file, without worrying bout glob matching the name and adding its existing content back onto the file