There is no way to filter transactions based upon which function is called on a given program. The best you can do is to subscribe to logs involving a program, then to filter the logs for any mention of your target instruction.
const PROGRAM_ADDRESS = '...';
const FOREIGN_PROGRAM_INVOKE_LOG_REGEX = new RegExp(
`^Program ((?!${PROGRAM_ADDRESS})\\w)+ invoke \\[\\d+\\]$`,
);
const TARGET_PROGRAM_INVOKE_LOG_REGEX = new RegExp(
`^Program ${PROGRAM_ADDRESS} invoke \\[\\d+\\]$`,
);
connection.onLogs(new PublicKey(PROGRAM_ADDRESS), ({logs}) => {
let inTargetProgramLogSection = false;
for (const line in logs) {
if (inTargetProgramLogSection) {
if (FOREIGN_PROGRAM_INVOKE_LOG_REGEX.test(line)) {
inTargetProgramLogSection = false;
} else if (line === 'Program log: Instruction: MyInstruction') {
// Found the target instruction. Run your code here.
break;
}
} else if (TARGET_PROGRAM_INVOKE_LOG_REGEX.test(line)) {
inTargetProgramLogSection === true;
}
}
});
Note that this is predicated upon the program outputting a log when your target instruction is invoked.