Try restyling the ItemsContainer.
You'll find VisualStates like UnselectedNormal, UnselectedPointerOver.
Here's the entire style from the generic.xaml. Note that you might need to remove some unnecessary namespaces like controls:.
<Style TargetType="ItemContainer">
<Setter Property="Background" Value="{ThemeResource ItemContainerBackground}" />
<Setter Property="UseSystemFocusVisuals" Value="True" />
<Setter Property="FocusVisualPrimaryThickness" Value="2" />
<Setter Property="CornerRadius" Value="{ThemeResource ControlCornerRadius}" />
<Setter Property="Template">
<Setter.Value>
<ControlTemplate TargetType="ItemContainer">
<Grid x:Name="PART_ContainerRoot" Background="{TemplateBinding Background}" CornerRadius="{TemplateBinding CornerRadius}">
<VisualStateManager.VisualStateGroups>
<VisualStateGroup x:Name="CombinedStates">
<VisualState x:Name="UnselectedNormal" />
<VisualState x:Name="UnselectedPointerOver">
<Storyboard>
<ObjectAnimationUsingKeyFrames Storyboard.TargetName="PART_CommonVisual" Storyboard.TargetProperty="Fill">
<DiscreteObjectKeyFrame KeyTime="0" Value="{ThemeResource ItemContainerPointerOverBackground}" />
</ObjectAnimationUsingKeyFrames>
<ObjectAnimationUsingKeyFrames Storyboard.TargetName="PART_CommonVisual" Storyboard.TargetProperty="Stroke">
<DiscreteObjectKeyFrame KeyTime="0" Value="{ThemeResource ItemContainerPointerOverBorderBrush}" />
</ObjectAnimationUsingKeyFrames>
</Storyboard>
</VisualState>
<VisualState x:Name="UnselectedPressed">
<Storyboard>
<ObjectAnimationUsingKeyFrames Storyboard.TargetName="PART_CommonVisual" Storyboard.TargetProperty="Fill">
<DiscreteObjectKeyFrame KeyTime="0" Value="{ThemeResource ItemContainerPressedBackground}" />
</ObjectAnimationUsingKeyFrames>
<ObjectAnimationUsingKeyFrames Storyboard.TargetName="PART_CommonVisual" Storyboard.TargetProperty="Stroke">
<DiscreteObjectKeyFrame KeyTime="0" Value="{ThemeResource ItemContainerPressedBorderBrush}" />
</ObjectAnimationUsingKeyFrames>
</Storyboard>
</VisualState>
<VisualState x:Name="SelectedNormal">
<Storyboard>
<DoubleAnimation Storyboard.TargetName="PART_SelectionVisual" Storyboard.TargetProperty="Opacity" Duration="0" To="1" />
<ObjectAnimationUsingKeyFrames Storyboard.TargetName="PART_SelectionVisual" Storyboard.TargetProperty="BorderBrush">
<DiscreteObjectKeyFrame KeyTime="0" Value="{ThemeResource ItemContainerSelectionVisualBackground}" />
</ObjectAnimationUsingKeyFrames>
<ObjectAnimationUsingKeyFrames Storyboard.TargetName="PART_CommonVisual" Storyboard.TargetProperty="Fill">
<DiscreteObjectKeyFrame KeyTime="0" Value="{ThemeResource ItemContainerSelectedBackground}" />
</ObjectAnimationUsingKeyFrames>
<ObjectAnimationUsingKeyFrames Storyboard.TargetName="PART_CommonVisual" Storyboard.TargetProperty="Stroke">
<DiscreteObjectKeyFrame KeyTime="0" Value="{ThemeResource ItemContainerSelectedInnerBorderBrush}" />
</ObjectAnimationUsingKeyFrames>
<ObjectAnimationUsingKeyFrames Storyboard.TargetName="PART_CommonVisual" Storyboard.TargetProperty="Margin">
<DiscreteObjectKeyFrame KeyTime="0" Value="{ThemeResource ItemContainerSelectedInnerMargin}" />
</ObjectAnimationUsingKeyFrames>
</Storyboard>
</VisualState>
<VisualState x:Name="SelectedPointerOver">
<Storyboard>
<DoubleAnimationUsingKeyFrames Storyboard.TargetName="PART_SelectionVisual" Storyboard.TargetProperty="Opacity">
<SplineDoubleKeyFrame KeyTime="{StaticResource ControlFastAnimationDuration}" Value="1" KeySpline="0,0,0,1" />
</DoubleAnimationUsingKeyFrames>
<ObjectAnimationUsingKeyFrames Storyboard.TargetName="PART_CommonVisual" Storyboard.TargetProperty="Fill">
<DiscreteObjectKeyFrame KeyTime="0" Value="{ThemeResource ItemContainerSelectedPointerOverBackground}" />
</ObjectAnimationUsingKeyFrames>
<ObjectAnimationUsingKeyFrames Storyboard.TargetName="PART_SelectionVisual" Storyboard.TargetProperty="BorderBrush">
<DiscreteObjectKeyFrame KeyTime="0" Value="{ThemeResource ItemContainerSelectionVisualPointerOverBackground}" />
</ObjectAnimationUsingKeyFrames>
<ObjectAnimationUsingKeyFrames Storyboard.TargetName="PART_CommonVisual" Storyboard.TargetProperty="Stroke">
<DiscreteObjectKeyFrame KeyTime="0" Value="{ThemeResource ItemContainerSelectedInnerBorderBrush}" />
</ObjectAnimationUsingKeyFrames>
<ObjectAnimationUsingKeyFrames Storyboard.TargetName="PART_CommonVisual" Storyboard.TargetProperty="Margin">
<DiscreteObjectKeyFrame KeyTime="0" Value="{ThemeResource ItemContainerSelectedInnerMargin}" />
</ObjectAnimationUsingKeyFrames>
</Storyboard>
</VisualState>
<VisualState x:Name="SelectedPressed">
<Storyboard>
<DoubleAnimationUsingKeyFrames Storyboard.TargetName="PART_SelectionVisual" Storyboard.TargetProperty="Opacity">
<SplineDoubleKeyFrame KeyTime="{StaticResource ControlFastAnimationDuration}" Value="1" KeySpline="0,0,0,1" />
</DoubleAnimationUsingKeyFrames>
<ObjectAnimationUsingKeyFrames Storyboard.TargetName="PART_CommonVisual" Storyboard.TargetProperty="Fill">
<DiscreteObjectKeyFrame KeyTime="0" Value="{ThemeResource ItemContainerSelectedPressedBackground}" />
</ObjectAnimationUsingKeyFrames>
<ObjectAnimationUsingKeyFrames Storyboard.TargetName="PART_SelectionVisual" Storyboard.TargetProperty="BorderBrush">
<DiscreteObjectKeyFrame KeyTime="0" Value="{ThemeResource ItemContainerSelectionVisualPressedBackground}" />
</ObjectAnimationUsingKeyFrames>
<ObjectAnimationUsingKeyFrames Storyboard.TargetName="PART_CommonVisual" Storyboard.TargetProperty="Stroke">
<DiscreteObjectKeyFrame KeyTime="0" Value="{ThemeResource ItemContainerSelectedInnerBorderBrush}" />
</ObjectAnimationUsingKeyFrames>
<ObjectAnimationUsingKeyFrames Storyboard.TargetName="PART_CommonVisual" Storyboard.TargetProperty="Margin">
<DiscreteObjectKeyFrame KeyTime="0" Value="{ThemeResource ItemContainerSelectedInnerMargin}" />
</ObjectAnimationUsingKeyFrames>
</Storyboard>
</VisualState>
</VisualStateGroup>
<VisualStateGroup x:Name="MultiSelectStates">
<VisualState x:Name="Single" />
<VisualState x:Name="Multiple">
<Storyboard>
<DoubleAnimationUsingKeyFrames Storyboard.TargetName="PART_SelectionCheckbox" Storyboard.TargetProperty="Opacity">
<SplineDoubleKeyFrame KeyTime="{StaticResource ControlFastAnimationDuration}" Value="1" KeySpline="0,0,0,1" />
</DoubleAnimationUsingKeyFrames>
</Storyboard>
</VisualState>
</VisualStateGroup>
<VisualStateGroup x:Name="DisabledStates">
<VisualState x:Name="Enabled" />
<VisualState x:Name="Disabled">
<Storyboard>
<DoubleAnimationUsingKeyFrames Storyboard.TargetName="PART_ContainerRoot" Storyboard.TargetProperty="Opacity">
<SplineDoubleKeyFrame KeyTime="{StaticResource ControlFastAnimationDuration}" Value="{StaticResource ItemContainerDisabledOpacity}" KeySpline="0,0,0,1" />
</DoubleAnimationUsingKeyFrames>
<ObjectAnimationUsingKeyFrames Storyboard.TargetName="PART_SelectionVisual" Storyboard.TargetProperty="Visibility">
<DiscreteObjectKeyFrame KeyTime="0" Value="Collapsed" />
</ObjectAnimationUsingKeyFrames>
</Storyboard>
</VisualState>
</VisualStateGroup>
</VisualStateManager.VisualStateGroups>
<!--Placeholder for child -->
<Grid x:Name="PART_SelectionVisual" x:DeferLoadStrategy="Lazy" BorderBrush="Transparent" BorderThickness="3" Background="Transparent" CornerRadius="{TemplateBinding CornerRadius}" HorizontalAlignment="Stretch" VerticalAlignment="Stretch" IsHitTestVisible="False" Opacity="0" />
<Rectangle x:Name="PART_CommonVisual" Stroke="{ThemeResource ItemContainerBorderBrush}" StrokeThickness="{ThemeResource ItemContainerSelectedInnerThickness}" HorizontalAlignment="Stretch" VerticalAlignment="Stretch" IsHitTestVisible="False" RadiusX="{Binding CornerRadius, RelativeSource={RelativeSource TemplatedParent}, Converter={StaticResource TopLeftCornerRadiusDoubleValueConverter}}" RadiusY="{Binding CornerRadius, RelativeSource={RelativeSource TemplatedParent}, Converter={StaticResource BottomRightCornerRadiusDoubleValueConverter}}" />
<CheckBox x:Name="PART_SelectionCheckbox" Style="{StaticResource ItemContainerSelectionCheckboxStyle}" x:DeferLoadStrategy="Lazy" Opacity="0">
<CheckBox.Resources>
<StaticResource x:Key="CheckBoxCheckBackgroundFillUnchecked" ResourceKey="ControlOnImageFillColorDefaultBrush" />
</CheckBox.Resources>
</CheckBox>
</Grid>
</ControlTemplate>
</Setter.Value>
</Setter>
</Style>