Remove duplicates values in array Google Apps Scri

Question

I would like to know how to remove duplicates values from 2 arrays, combined into one main array.

This values must NOT be removed from the sheets or document, just in the array, thats why I didnt use clear() or clearContents() built in functions;

Ive also tried to modify the removeDuplicates() function from the GAS tutorials, but it throws me rows inside columns from A to Z, instead filtered rows...a total mess.

Notes:

Parameters from getClients() are from others functions, and works ok.

newClients list clients from the sheet 'Users' and newUsers list users from another sheet called 'Data'.

Boths sheets belongs to the same spreadsheet.

newClients and newUsers: both arrays only contains strings (usernames), and in both there are duplicated values.

So the goal is identified and remove those values, the original and the duplicate.

Should be easier I think, but Im new in JS, so everything Ive been tried, didnt worked.

Thanks

The Code

function getAllClientsFromData(body,project){

  var ss = SpreadsheetApp.getActiveSpreadsheet();

  // Active the Sheets
  var sheet= ss.getSheets()[1,3];

  // Access data in Sheet "Data"
  var rangeC = ss.getRangeByName("clients"); //A:2:C273  
  var rangeU = ss.getRangeByName("names"); //A5:A

  var clients = rangeC.getValues();
  var users   = rangeU.getValues();

  var lastRow = sheet.getLastRow()-2;

  body += "<h2>Total of " + lastRow + " clients in " + project + "</h2>" 
  body += "<table style=" + STYLE.TABLE + ">";
  body += getClients(ss,clients,users,project);
  body += "</table>";

  return body;  

}

function getClients(ss,clients,users,project){

  var body = "";

  var newClients = [];

  for( var g = 0; g < clients.length; g++ ){

    var currentProject = clients[g][2];

    if( clients[g]!= "" ){

       newClients.push(clients[g][0]);

    } 

  } // end for

  var newUsers = []; 

  for( var u = 0; u < users.length; u++ ){

    if( users[u] != "" ){    

      newUsers.push(users[u][0]);

    }

  } // end for

  var allData = newUsers.concat(newClients);

  var uniqueData = allData.sort();

  body += "<tr><td style=" + STYLE.TD + ">" + uniqueData.join("</td><tr><td style=" + STYLE.TD + ">") + "</td></tr></tr>";

  return body;

}

UPDATES!

The answers works great filtering, but Im getting the same result as on my previous tries: displaying the filtered results. I need to remove them from the array. example:

var array = ['aa','bb','aa','ff','pp', 'pp'];
filtering code...
var array = ['bb','ff'];

I try to add splice() js method but the params I pass, does not working ok.

Answer 1

The array you are working on is not a 2D array anymore since you extracted the fields before sorting... so you can use a very simple duplicate removal function as shown below with an example and some added Logger.log to see how it works.

function test(){
  var array = ['aa','bb','cc','aa','dd','cc']
  Logger.log(removeDups(array));
}

function removeDups(array) {
  var outArray = [];
  array.sort():
  outArray.push(array[0]);
  for(var n in array){
    Logger.log(outArray[outArray.length-1]+'  =  '+array[n]+' ?');
    if(outArray[outArray.length-1]!=array[n]){
      outArray.push(array[n]);
    }
  }
  return outArray;
}

in your code this would replace the line

var uniqueData = allData.sort();

that would become :

var uniqueData = removeDups(allData);

---

EDIT :

If letter case is an issue, you can modify this code to ignore it. You should change the condition and the sort function so that they both ignore the case in your names but preferably keep the original letter case.

This could be achieved with the code below :

function test(){
  var array = ['aa','bb','Cc','AA','dd','CC'];// an example with Upper and Lower case
  Logger.log(removeDups(array));
}

function removeDups(array) {
  var outArray = [];
  array.sort(lowerCase);
  function lowerCase(a,b){
    return a.toLowerCase()>b.toLowerCase() ? 1 : -1;// sort function that does not "see" letter case
  }
  outArray.push(array[0]);
  for(var n in array){
    Logger.log(outArray[outArray.length-1]+'  =  '+array[n]+' ?');
    if(outArray[outArray.length-1].toLowerCase()!=array[n].toLowerCase()){
      outArray.push(array[n]);
    }
  }
  return outArray;
}

Logger result :

---

EDIT 2 :

Here is another version that keeps only unique values (I didn't understand correctly your request in the first version as it kept one element from the duplicates...)

I simply added an else if condition to remove the elements that were part of a group of duplicates.(I kept the case insensitive version but you can remove it easily)

function test(){
  var array = ['aa','dd','hh','aa','bb','Cc','cc','cc','bb','nn','bb','AA','dd','CC'];// an example with Upper and Lower case
  Logger.log('original array = '+array);
  Logger.log('unique result = '+removeDups2(array));
}


function removeDups2(array) {
  var uniqueArray = []
  array.sort(lowerCase);
  function lowerCase(a,b){
    return a.toLowerCase()>b.toLowerCase() ? 1 : -1;// sort function that does not "see" letter case
  }
  var temp = array[0];
  for(var n=1 ;n<array.length ; n++){
    Logger.log(temp+'  =  '+array[n]+' ?');
    if(temp.toLowerCase()!=array[n].toLowerCase()){
      uniqueArray.push(array[n]);
      temp = array[n];
    }else if(uniqueArray[uniqueArray.length-1]==temp){
      uniqueArray.pop();// remove it from result if one of the duplicate values
    }
  }
  return uniqueArray;
}

Logger result new code :

Answer 2

In your code you are not doing anything to filter the duplicate values. This line will just sort the data and won't give you unique data.

var uniqueData = allData.sort(); 

You can do something like this on your merged array, after you 'installing' 2DArray lib: https://sites.google.com/site/scriptsexamples/custom-methods/2d-arrays-library

var uniqueData = unique(allData);

Another option is to create a loop and check for duplicate values, but you should remember to transform all the values of the string to lowercase before you do these matches.