AngularJS $destroy dynamically inserted component

I am building a modal that provides a service to toggle it on and off. That modal has a small controller which controls the close button, and the $compile of the template that goes into the content of the modal.

That template is a component, and of course, that component has a controller.

How do I destroy that component, after hiding the modal? Technically, when ng-if takes care of removing my modal from the DOM, it still has that component loaded into it.

Here is the code:

modal.controller.js

class ModalController {
  constructor($scope, modalService, $compile, $timeout) {
    this.$scope = $scope;
    this.modalService = modalService;
    this.$compile = $compile;
    this.$timeout = $timeout;
  }

  $onInit() {
    this.$scope.$watch(this.modalService.getConfig(), (newVal) => {
      this.config = newVal.isDisplayed
        ? angular.extend(this.config, newVal)
        : {};

      // I wait for ng-if to put the modal into the DOM then I 
      // compile the component.
      this.$timeout(() => {
        if (this.config.template) {
          const component = this.$compile(this.config.template)(this.$scope);
          angular.element('#modal-content').html(component);
          this.config.isRendered = true;
        }
      }, 0);
    }, true);
  }

  close() {
    this.modalService.close();
  }
}

ModalController.$inject = [
  '$scope',
  'modalService',
  '$compile',
  '$timeout',
];

export default ModalController;

modal.service.js

class ModalService {
  constructor($timeout) {
    this.config = {};
    this.$timeout = $timeout;
  }

  open(newConfig) {
    this.config = newConfig;
    this.config.isDisplayed = true;
  }

  close() {
    this.config.template = null;
    this.config.isRendered = false;

    // the reason there is timeout here is to run my CSS animation
    // before ng-if removes the modal from the DOM.
    this.$timeout(() => {
      this.config.isDisplayed = false;
      this.config = {};
    }, 310);
  }

  getConfig() {
    return () => this.config;
  }
}

ModalService.$inject = [
  '$timeout',
];

export default ModalService;

BarController where I call the modal:

class BarController {
  constructor(modalService) {
    this.modalService = modalService;
  }

  openModal() {
    this.modalService.open({
      title: 'Add a document',
      template: '<foo-component></foo-component>',
    });
  }
}

BarController.$inject = [
  'modalService',
];

export default BarController;