I got where I ended to go, in the end, mostly through a lot of trial and error - my understanding is still pretty shallow. I found scrapping the UI templates and starting from scratch to be the least confounding option, as I never really figured out how to 'remove' a dialog added by the templates, but this probably wouldn't be true if I knew what I was doing.
As the other answers suggest, there are possibly more conventional and powerful UI options available these days, but a basic approach suits my needs here.
This minimal project does work and demonstrates an untemplated UI making use of both custom and standard dialogs, with images. I've tried to take out anything redundant. Uninstall flow works but is completely standard. Requires the WixToolset.UI.wixext extension, either specified in a .wixproj for a dotnet built or supplied as an -ext argument to a direct wix build (ext needs to be manually installed first?)
Product.wxs:
<?xml version="1.0" encoding="UTF-8"?>
<Wix xmlns="http://wixtoolset.org/schemas/v4/wxs"
xmlns:ui="http://wixtoolset.org/schemas/v4/wxs/ui">
<Package Name="MyApp"
Version="1.0.0.0"
Manufacturer="My Company"
UpgradeCode="12345678-1234-1234-1234-123456789012">
<!-- UI Resources -->
<!-- Primarily seems to serve to make the image references in builtin dialogs work -->
<UIRef Id="WixUI_Common" />
<!-- Font Definitions -->
<!-- I'm sure there's some redundancy between these and the TextStyles in the UI,
but I'm not going to chase this down immediately. -->
<Property Id="DefaultUIFont" Value="WixUI_Font_Normal" />
<WixVariable Id="WixUIFont" Value="Tahoma" Overridable="yes" />
<WixVariable Id="WixUIFontBig" Value="Tahoma" Overridable="yes" />
<WixVariable Id="WixUIFontTitle" Value="Tahoma" Overridable="yes" />
<WixVariable Id="WixUIFontSize" Value="8" Overridable="yes" />
<!-- Image Resources -->
<!-- I don't think these actually have to be bmps but too late now -->
<Binary Id="WixUIBanner" SourceFile="banner.bmp" />
<!-- Image Properties -->
<!-- The property points at a binary resource for our custom dialog -->
<Property Id="BannerBitmap" Value="WixUIBanner" />
<!-- The WixVariables work for builtin dialogs provided we're referencing common ui-->
<WixVariable Id="WixUIBannerBmp" Value="banner.bmp" />
<WixVariable Id="WixUIDialogBmp" Value="dialog.bmp" />
<!-- Progress Dialog Properties -->
<Property Id="ActionText" Value="Installing MyApp" />
<Property Id="PROGRESSTEXT" Value="Installation Progress..." />
<Property Id="StatusLabel" Value="Please wait..." />
<!-- Application Properties -->
<!-- These disable repair/modify in Add/Remove Programs -->
<Property Id="ARPNOREPAIR" Value="1" />
<Property Id="ARPNOMODIFY" Value="1" />
<!-- Exit Dialog Properties -->
<Property Id="WIXUI_EXITDIALOGOPTIONALCHECKBOX" Value="0"/>
<Property Id="WIXUI_EXITDIALOGOPTIONALTEXT" Value="Goodbye world!"/>
<!-- Custom Dialog Properties -->
<Property Id="CUSTOMTEXT1" Secure="yes" />
<Property Id="CUSTOMTEXT2" Secure="yes" />
<!-- Installation Directory -->
<StandardDirectory Id="ProgramFilesFolder">
<Directory Id="INSTALLFOLDER" Name="MyApp" />
</StandardDirectory>
<!-- Components and Features -->
<Component Id="MainComponent" Directory="INSTALLFOLDER">
<File Source="myexe.exe" />
</Component>
<Feature Id="MainFeature" Title="Main Feature">
<ComponentRef Id="MainComponent" />
</Feature>
<!-- UI Definition -->
<UI>
<TextStyle Id="WixUI_Font_Normal" FaceName="Tahoma" Size="8" />
<TextStyle Id="WixUI_Font_Bigger" FaceName="Tahoma" Size="12" />
<TextStyle Id="WixUI_Font_Title" FaceName="Tahoma" Size="9" Bold="yes" />
<!-- Standard Dialog References -->
<DialogRef Id="CancelDlg" />
<DialogRef Id="ErrorDlg" />
<DialogRef Id="FatalError" />
<DialogRef Id="FilesInUse" />
<DialogRef Id="UserExit" />
<DialogRef Id="ProgressDlg" />
<DialogRef Id="ExitDialog"/>
<!-- Validation Error Dialog -->
<Dialog Id="ValidationError" Width="260" Height="85" Title="Validation Error">
<Control Id="Return" Type="PushButton" X="100" Y="57" Width="56" Height="17" Default="yes" Cancel="yes" Text="OK">
<Publish Event="EndDialog" Value="Return" />
</Control>
<Control Id="Text" Type="Text" X="48" Y="15" Width="194" Height="30" Text="Please fill in all required fields before continuing." />
</Dialog>
<!-- Custom Dialog -->
<Dialog Id="CustomDlg" Width="370" Height="270" Title="Custom Dialog">
<Control Id="Title" Type="Text" X="15" Y="6" Width="200" Height="15" Transparent="yes" NoPrefix="yes">
<Text Value="{\WixUI_Font_Title}Custom Dialog Title" />
</Control>
<Control Id="Description" Type="Text" X="25" Y="23" Width="280" Height="15" Transparent="yes" NoPrefix="yes">
<Text Value="Custom dialog description" />
</Control>
<Control Id="Text1Label" Type="Text" X="25" Y="70" Width="100" Height="15" Text="Field 1:" />
<Control Id="Text1" Type="Edit" X="25" Y="85" Width="320" Height="18" Property="CUSTOMTEXT1" />
<Control Id="Text2Label" Type="Text" X="25" Y="110" Width="100" Height="15" Text="Field 2:" />
<Control Id="Text2" Type="Edit" X="25" Y="125" Width="320" Height="18" Property="CUSTOMTEXT2" />
<Control Id="Next" Type="PushButton" X="236" Y="243" Width="56" Height="17" Default="yes" Text="Next">
<Publish Event="SpawnDialog" Value="ValidationError" Condition="NOT CUSTOMTEXT1 OR NOT CUSTOMTEXT2" />
<Publish Event="EndDialog" Value="Return" Condition="CUSTOMTEXT1 AND CUSTOMTEXT2" />
</Control>
<Control Id="Cancel" Type="PushButton" X="304" Y="243" Width="56" Height="17" Cancel="yes" Text="Cancel">
<Publish Event="SpawnDialog" Value="CancelDlg" />
</Control>
<Control Id="BannerBitmap" Type="Bitmap" X="0" Y="0" Width="370" Height="44" TabSkip="no" Text="[BannerBitmap]" />
<Control Id="BannerLine" Type="Line" X="0" Y="44" Width="370" Height="0" />
<Control Id="BottomLine" Type="Line" X="0" Y="234" Width="370" Height="0" />
</Dialog>
<!-- Dialog Sequences -->
<!-- Other referenced dialogs insert themselves? Not sure they can be overriden. -->
<InstallUISequence>
<Show Dialog="CustomDlg" Before="ExecuteAction" Condition="NOT REMOVE" />
</InstallUISequence>
<!-- Exit Dialog Handling -->
<!-- (Without this, the exit dialog cannot be closed) -->
<Publish Dialog="ExitDialog" Control="Finish" Event="EndDialog" Value="Return" />
</UI>
</Package>
</Wix>