const array = [
  [
    { name: 'A', age: 10 },
    { name: 'B', age: 10 },
    { name: 'A', age: 10 },
    { name: 'B', age: 10 },
  ],
  [
    { name: 'C', age: 10 },
    { name: 'D', age: 10 },
    { name: 'C', age: 10 },
    { name: 'D', age: 10 },
  ],
];
const groupBy = (collection, extractKey) => {
  const cache = new Map();
  
  return collection.reduce((accumulator, entry) => {
    const key = extractKey(entry);
    if (!cache.has(key)) {
      const group = [];
      cache.set(key, group);
      accumulator.push(group);
    }
    cache.get(key).push(entry);
    return accumulator;
  }, []);
};
const newArray = array.map(entry => groupBy(entry, item => item.name));
console.log(newArray);
/*
[
  [
    [ { name: 'A', age: 10 }, { name: 'A', age: 10 } ],
    [ { name: 'B', age: 10 }, { name: 'B', age: 10 } ]
  ],
  [
    [ { name: 'C', age: 10 }, { name: 'C', age: 10 } ],
    [ { name: 'D', age: 10 }, { name: 'D', age: 10 } ]
  ]
]
*/
Также провел тесты, нашел самый шустрый вариант.
const groupBy = (collection, extractKey) => {
  const cache = new Map();
  
  for (const entry of collection) {
    const key = extractKey(entry);
    if (!cache.has(key)) {
      cache.set(key, []);
    }
    cache.get(key).push(entry);
  }
  
  return [...cache.values()];
};