Trigger callback from external component react testing library

For your example, there is actually no need to mock the MyModal component unless the MyModal component has many external dependencies. See testing-recipes.html#mocking-modules.

In order to trigger the onClose function, you still need to trigger the onClick function on MyModal component.

Besides, You did not mock the MyModal component correctly. Here is a working example:

ParentComp.tsx:

import React, { useState } from 'react';
import { MyModal } from './MyModal';

export const ParentComp = () => {
  const [openModal, setOpenModal] = useState(false);
  return (
    <>
      <MyModal showModal={openModal} onClose={() => setOpenModal(false)} />
      <button data-testid="open-modal-button" onClick={() => setOpenModal(true)}>
        Test
      </button>
    </>
  );
};

MyModal.tsx:

import React from 'react';

export const MyModal = ({ showModal, onClose }) => {
  return (
    showModal && (
      <>
        <div>This is a modal</div>
        <button data-testid="close-modal-button" onClick={onClose}>
          Close
        </button>
      </>
    )
  );
};

ParentComp.test.tsx:

import { fireEvent, render } from '@testing-library/react';
import React from 'react';
import { ParentComp } from './ParentComp';

function MockedMyModal({ onClose, showModal }) {
  return (
    showModal && (
      <>
        <div>This is a mocked modal</div>
        <button data-testid="my-modal" onClick={onClose}>
          Close
        </button>
      </>
    )
  );
}

jest.mock('./MyModal', () => {
  return { MyModal: MockedMyModal };
});

describe('65038548', () => {
  afterAll(() => {
    jest.resetAllMocks();
  });
  it('should open the modal', () => {
    const { getByTestId } = render(<ParentComp></ParentComp>);
    const openModalButton = getByTestId('open-modal-button');

    fireEvent.click(openModalButton);
    const myModal = getByTestId('my-modal');

    expect(myModal).toBeDefined();
  });

  it('should close the modal', () => {
    const { getByTestId, queryByText } = render(<ParentComp></ParentComp>);
    const openModalButton = getByTestId('open-modal-button');
    fireEvent.click(openModalButton);

    const closeModalButton = getByTestId('my-modal');
    expect(closeModalButton).toBeDefined();

    fireEvent.click(closeModalButton);

    expect(queryByText('This is a mocked modal')).toBeNull();
  });
});

Why use queryByText instead of getByTestId, see Asserting elements are not present

unit test result:

 PASS  examples/65038548/ParentComp.test.tsx
  65038548
    ✓ should pass (34 ms)

----------------|---------|----------|---------|---------|-------------------
File            | % Stmts | % Branch | % Funcs | % Lines | Uncovered Line #s 
----------------|---------|----------|---------|---------|-------------------
All files       |    87.5 |      100 |   66.67 |   85.71 |                   
 ParentComp.tsx |    87.5 |      100 |   66.67 |   85.71 | 8                 
----------------|---------|----------|---------|---------|-------------------
Test Suites: 1 passed, 1 total
Tests:       1 passed, 1 total
Snapshots:   0 total
Time:        5.848 s

source code: https://github.com/mrdulin/jest-v26-codelab/tree/main/examples/65038548