You may use
/[?&]filter\[([^\].]+\.[^\]]+)]=/g
See the regex demo
Details
[?&] - a ? or &
filter\[ - a filter[ substring
([^\].]+\.[^\]]+) - Capturing group 1:
[^\].]+ - 1 or more chars other than ] and .
\. - a dot
[^\]]+ - 1 or more chars other than ]
]= - a ]= substring
JS demo:
var s = '/domain.com?filter[a.b.c]=value1&filter[a.b.d]=value2';
var rx = /[?&]filter\[([^\].]+\.[^\]]+)]=/g;
var m, res=[];
while(m=rx.exec(s)) {
res.push(m[1]);
}
console.log(res);
Note that in case & is never present as part of the query param value, you may add it to the negated character classes, [^\].]+ => [^\]&.]+, to make sure the regex does not overmatch across param values.
Since you need to extract text inside outer square brackets that may contain consecutive [...] substrings with at least 1 dot inside one of them, you may use a simpler regex with a bit more code:
var strs = ['/domain.com?filter[a.b.c]=value1&filter[a.b.d]=value2',
'/domain.com?filter[a.b.c]=value1&filter[a.b.d]=value2&filter[a][b.e]=value3',
'/domain.com?filter[a.b.c]=value1&filter[b][a.b.d][d]=value2&filter[a][b.e]=value3'];
var rx = /[?&]filter((?:\[[^\][]*])+)=/g;
for (var s of strs) {
var m, res=[];
console.log(s);
while(m=rx.exec(s)) {
if (m[1].indexOf('.') > -1) {
res.push(m[1].substring(1,m[1].length-1));
}
}
console.log(res);
console.log("--- NEXT STRING ----");
}