.Net Core Custom Authentication using API Keys with Identity Server 4

I have a .NET Core 2.2 Web API that authenticates with JWT tokens. Tokens are generated by Identity Server 4 on a separate API.

All the authentication and authorisation works as expected with JWT tokens. But I need to extend this to allow usage of API keys. If an API key is supplied, I want to load up the claims of that particular user, add it to the request and let Authorize attribute deal with the set policies.

Here is what I have done so far following suggestions from here. My error is exactly the same as the linked post and it works for me as well using GenericPrincipal with a set of roles but I am using AuthorisationPolicies and I always get 401 error with my current implementation, giving me errors similar to the link above.

Startup.cs

public void ConfigureServices(IServiceCollection services)
    {
        services.AddMvcCore(options =>
        {
            options.Filters.Add(new RequireHttpsAttribute());
            options.Filters.Add(new AuthorizeFilter());
            options.Filters.Add(typeof(ValidateModelStateAttribute));
            options.AllowEmptyInputInBodyModelBinding = true;
        })
        .AddAuthorization(options =>
        {
            options.AddPolicies();
        })
        .AddJsonFormatters();

        services.AddAuthentication(IdentityServerAuthenticationDefaults.AuthenticationScheme)
            .AddIdentityServerAuthentication(options =>
            {
                options.Authority = Configuration["Authentication:Authority"];
                options.RequireHttpsMetadata = true;
                options.ApiName = Configuration["Authentication:ApiName"];
            });
        services.AddCors();
    }

public void Configure(IApplicationBuilder app, IHostingEnvironment env)
    {
        if (env.IsDevelopment())
        {
            app.UseDeveloperExceptionPage();
        }
        else
        {
            // The default HSTS value is 30 days. You may want to change this for production scenarios, see https://aka.ms/aspnetcore-hsts.
            app.UseHsts();
        }

        app.UseCors(policy =>
        {
            policy.AllowAnyHeader();
            policy.AllowAnyMethod();
            policy.AllowAnyOrigin();
        });

        app.UseHttpsRedirection();
        app.UseMiddleware<ApiKeyMiddleware>();
        app.UseAuthentication();
        app.UseMvc();
    }

AuthorizationPolicies.cs

public static class AuthorizationPolicies
{
    public const string ReadUsersPolicy = "ReadUsers";
    public const string EditUsersPolicy = "EditUsers";

    public static void AddPolicies(this AuthorizationOptions options)
    {
        options.AddPolicy(ReadUsersPolicy, policy => policy.RequireClaim(Foo.Permission, Foo.CanReadUsers));
        options.AddPolicy(EditUsersPolicy, policy => policy.RequireClaim(Foo.Permission, Foo.CanEditUsers));
    }
}

ApiKeyMiddleware

public class ApiKeyMiddleware
{
    public ApiKeyMiddleware(RequestDelegate next)
    {
        _next = next;
    }
    private readonly RequestDelegate _next;

    public async Task Invoke(HttpContext context)
    {
        if (context.Request.Path.StartsWithSegments(new PathString("/api")))
        {
            if (context.Request.Headers.Keys.Contains("ApiKey", StringComparer.InvariantCultureIgnoreCase))
            {
                var headerKey = context.Request.Headers["ApiKey"].FirstOrDefault();
                await ValidateApiKey(context, _next, headerKey);
            }
            else
            {
                await _next.Invoke(context);
            }
        }
        else
        {
            await _next.Invoke(context);
        }
    }

    private async Task ValidateApiKey(HttpContext context, RequestDelegate next, string key)
    {
        var userClaimsService = context.RequestServices.GetService<IUserClaimsService>();
        List<string> permissions = (await userClaimsService.GetAllPermissionsForApiKey(key))?.ToList();
        if (permissions == null)
        {
            context.Response.StatusCode = (int)HttpStatusCode.Unauthorized;
            await context.Response.WriteAsync("Invalid Api Key");
            return;
        }

        ICollection<Claim> claims = permissions.Select(x => new Claim(FooClaimTypes.Permission, x)).ToList();
        var identity = new ClaimsIdentity(claims);
        var principal = new ClaimsPrincipal(identity);
        context.User = principal;
        await next.Invoke(context);
    }
}

UsersController.cs

[Authorize(AuthorizationPolicies.EditUsersPolicy)]
    public async Task<IActionResult> Put([FromBody] UserUpdateDto userUpdateDto)
    {
        ...
    }