HttpClient header getting nulled when using in async methods

Question

I am using .NET Framework 4.6.1.

I have a controller in my web api, where I have static HttpClient to handle all the http requests. After I hosted my app on IIS, approximately once in a month, I get the following exception for all the incoming request to my app:

System.ArgumentNullException: Value cannot be null.
   at System.Threading.Monitor.Enter(Object obj)
   at System.Net.Http.Headers.HttpHeaders.ParseRawHeaderValues(String name, HeaderStoreItemInfo info, Boolean removeEmptyHeader)
   at System.Net.Http.Headers.HttpHeaders.AddHeaders(HttpHeaders sourceHeaders)
   at System.Net.Http.Headers.HttpRequestHeaders.AddHeaders(HttpHeaders sourceHeaders)
   at System.Net.Http.HttpClient.PrepareRequestMessage(HttpRequestMessage request)
   at System.Net.Http.HttpClient.SendAsync(HttpRequestMessage request, HttpCompletionOption completionOption, CancellationToken cancellationToken)
   at System.Net.Http.HttpClient.SendAsync(HttpRequestMessage request, CancellationToken cancellationToken)
   at System.Net.Http.HttpClient.PutAsync(Uri requestUri, HttpContent content, CancellationToken cancellationToken)
   at Attributes.Controllers.AttributesBaseController.<UpdateAttributes>d__6.MoveNext() in D:\Git\PortalSystem\Attributes\Controllers\AttributesBaseController.cs:line 42

If I restart the app pool on IIS, everything starts to work fine again. Here is the code that I have:

public class AttributesBaseController : ApiController
{
    [Inject]
    public IPortalsRepository PortalsRepository { get; set; }

    private static HttpClient Client = new HttpClient(new HttpClientHandler { Proxy = null, UseProxy = false })
                                                                            { Timeout = TimeSpan.FromSeconds(double.Parse(WebConfigurationManager.AppSettings["httpTimeout"])) };
    private static readonly Logger logger = LogManager.GetCurrentClassLogger();

    protected async Task UpdateAttributes(int clientId, int? updateAttrId = null)
    {
        try
        {
            Client.DefaultRequestHeaders.Accept.Clear();
            Client.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json"));

            #region Update Client Dossier !!! BELOW IS LINE 42 !!!!          
            using (var response = await Client.PutAsync(new Uri(WebConfigurationManager.AppSettings["dossier"] + "api/dossier?clientId=" + clientId), null))
            {
                if (!response.IsSuccessStatusCode)
                {
                    logger.Error($"Dossier update failed");
                }
            }
            #endregion

            #region Gather Initial Info
            var checkSystems = PortalsRepository.GetCheckSystems(clientId);
            var currentAttributes = PortalsRepository.GetCurrentAttributes(clientId, checkSystems);
            #endregion

            List<Task> tasks = new List<Task>();
            #region Initialize Tasks
            foreach (var cs in checkSystems)
            {
                if (!string.IsNullOrEmpty(cs.KeyValue))
                {
                    tasks.Add(Task.Run(async () =>
                    {
                            var passedAttributes = currentAttributes.Where(ca => ca.SystemId == cs.SystemId && ca.AttributeId == cs.AttributeId && 
                            (ca.SysClientId == cs.KeyValue || ca.OwnerSysClientId == cs.KeyValue)).ToList();

                            if (cs.AttributeId == 2 && (updateAttrId == null || updateAttrId == 2))
                            {
                                await UpdateOpenWayIndividualCardsInfo(passedAttributes, cs, clientId);
                            }
                            else if (cs.AttributeId == 3 && (updateAttrId == null || updateAttrId == 3))
                            {
                                await UpdateEquationAccountsInfo(passedAttributes, cs, clientId);
                            }
                            else if (cs.AttributeId == 8 && (updateAttrId == null || updateAttrId == 8))
                            {
                                await UpdateOpenWayCorporateInfo(passedAttributes, cs, clientId);
                            }
                            else if (cs.AttributeId == 9 && (updateAttrId == null || updateAttrId == 9))
                            {
                                await UpdateEquationDealsInfo(passedAttributes, cs, clientId);
                            }
                            else if (cs.AttributeId == 10 && (updateAttrId == null || updateAttrId == 10))
                            {
                                await UpdateOpenWayIndividualCardDepositsInfo(passedAttributes, cs, clientId);
                            }
                            else if (cs.AttributeId == 16 && (updateAttrId == null || updateAttrId == 16))
                            {
                                await UpdateOpenWayBonusInfo(passedAttributes, cs, clientId);
                            }
                            else if (cs.AttributeId == 17 && (/*updateAttrId == null ||*/ updateAttrId == 17))
                            {
                                await UpdateExternalCardsInfo(passedAttributes, cs, clientId);
                            }
                            if (cs.AttributeId == 18 && (updateAttrId == null || updateAttrId == 18))
                            {
                                await UpdateCRSInfo(passedAttributes, cs, clientId);
                            }
                            else if (cs.AttributeId == 22 && (updateAttrId == null || updateAttrId == 22))
                            {
                                await UpdateCardInsuranceInfo(passedAttributes, cs, clientId);
                            }
                    }));
                }
            }
            #endregion

            // Run all tasks
            await Task.WhenAny(Task.WhenAll(tasks.ToArray()), Task.Delay(TimeSpan.FromSeconds(double.Parse(WebConfigurationManager.AppSettings["taskWaitTime"]))));
        }
        catch (Exception ex)
        {
            logger.Error(ex);
        }
    }
}

Can anyone give me advice/help to figure out the problem? I just don't know whether the problem is in the way I am using HttpClient with tasks or something bad happens on IIS.

"...or something bad happens on IIS" -- not completely impossible, but extremely unlikely. But, without a good minimal reproducible example that reliably reproduces the problem, there's not anything the Stack Overflow community can offer in the way of a good, specific answer. — Peter Duniho
– Peter Duniho, Commented Dec 26, 2017 at 3:24
@PeterDuniho The problem is that I can't reproduce the problem if I do requests manually. Maybe you can give me some advice where to start investigating the problem? — Nomad
– Nomad, Commented Dec 26, 2017 at 3:44
First step is the same as for any other problem: simplify the scenario as much as you can. Since you say it happens only once a month, it may take awhile going that route. Another standard technique is to add logging; in this case, that may be your best bet. Assuming you have at least some idea of the general area of code that's causing a problem, add logging to record all of the state, so that you can get an idea of what state is required to reproduce the issue. ... — Peter Duniho
– Peter Duniho, Commented Dec 26, 2017 at 3:51
... Initially this might just be variables and class members, but you might also want to log things like number of clients, identities, thread pool state, and the like. — Peter Duniho
– Peter Duniho, Commented Dec 26, 2017 at 3:51
@PeterDuniho Also, I forgot to mention that I gave 3 worker processes for the app pool. When this exception happens, all worker processes don't fail at once, they fail one by one. — Nomad
– Nomad, Commented Dec 26, 2017 at 3:58

Kevin Gosse · Accepted Answer · 2017-12-26 09:57:53Z

Looking at the implementation of DefaultRequestHeaders, we can see that it uses a simple Dictionary to store the headers:

private Dictionary<string, HttpHeaders.HeaderStoreItemInfo> headerStore;

DefaultRequestHeaders.Accept.Clear just removes the key from the dictionary, without any kind of synchronization:

public bool Remove(string name)
{
  this.CheckHeaderName(name);
  if (this.headerStore == null)
    return false;
  return this.headerStore.Remove(name);
}

Dictionary.Remove isn't thread-safe, unpredictable behavior can happen if you access the dictionary during this operation.

Now if we look at the ParseRawHeaderValues method in your stacktrace:

private bool ParseRawHeaderValues(string name, HttpHeaders.HeaderStoreItemInfo info, bool removeEmptyHeader)
{
  lock (info)
  {
    // stuff
  }
  return true;
}

We can see that the error would be cause by info to be null. Now looking at the caller:

internal virtual void AddHeaders(HttpHeaders sourceHeaders)
{
  if (sourceHeaders.headerStore == null)
    return;
  List<string> stringList = (List<string>) null;
  foreach (KeyValuePair<string, HttpHeaders.HeaderStoreItemInfo> keyValuePair in sourceHeaders.headerStore)
  {
    if (this.headerStore == null || !this.headerStore.ContainsKey(keyValuePair.Key))
    {
      HttpHeaders.HeaderStoreItemInfo headerStoreItemInfo = keyValuePair.Value;
      if (!sourceHeaders.ParseRawHeaderValues(keyValuePair.Key, headerStoreItemInfo, false))
      {
        if (stringList == null)
          stringList = new List<string>();
        stringList.Add(keyValuePair.Key);
      }
      else
        this.AddHeaderInfo(keyValuePair.Key, headerStoreItemInfo);
    }
  }
  if (stringList == null)
    return;
  foreach (string key in stringList)
    sourceHeaders.headerStore.Remove(key);
}

Long story short, we iterate the dictionary in DefaultRequestHeaders (that's sourceHeaders.headerStore) and copy the headers into the request.

Summing it up, at the same time we have a thread iterating the contents of the dictionary, and another adding/removing elements. This can lead to the behavior you're seeing.

To fix this, you have two solutions:

Initialize DefaultRequestHeaders in a static constructor, then never change it:

static AttributesBaseController 
{
    Client = new HttpClient(new HttpClientHandler { Proxy = null, UseProxy = false })
    {
        Timeout = TimeSpan.FromSeconds(double.Parse(WebConfigurationManager.AppSettings["httpTimeout"]))
    };

    Client.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json"));
}

Use SendAsync with your custom headers instead of PutAsync:

var message = new HttpRequestMessage(HttpMethod.Put, new Uri(WebConfigurationManager.AppSettings["dossier"] + "api/dossier?clientId=" + clientId));
message.Headers.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json"));
using (var response = await Client.SendAsync(message))
{
     // ...
}

Just for fun, a small repro:

var client = new HttpClient();

client.DefaultRequestHeaders.Accept.Clear();
client.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json"));

var storeField = typeof(HttpHeaders).GetField("headerStore", BindingFlags.Instance | BindingFlags.NonPublic);

FieldInfo valueField = null;

var store = (IEnumerable)storeField.GetValue(client.DefaultRequestHeaders);

foreach (var item in store)
{
    valueField = item.GetType().GetField("value", BindingFlags.Instance | BindingFlags.NonPublic);

    Console.WriteLine(valueField.GetValue(item));
}

for (int i = 0; i < 8; i++)
{
    Task.Run(() =>
    {
        int iteration = 0;

        while (true)
        {
            iteration++;

            try
            {
                foreach (var item in store)
                {
                    var value = valueField.GetValue(item);

                    if (value == null)
                    {
                        Console.WriteLine("Iteration {0}, value is null", iteration);
                    }

                    break;
                }

                client.DefaultRequestHeaders.Accept.Clear();
                client.DefaultRequestHeaders.Accept.Add(new Net.Http.Headers.MediaTypeWithQualityHeaderValue("application/json"));
            }
            catch (Exception) { }
        }
    });
}

Console.ReadLine();

Output:

System.Net.Http.Headers.HttpHeaders+HeaderStoreItemInfo

Iteration 137, value is null

Reproducing the issue may take a few tries because threads tend to get stuck in an infinite loop when accessing the dictionary concurrently (if it happens on your webserver, ASP.NET will abort the thread after the timeout elapses).

@Nomad - option 2 (and optionally option 1 as well) of Kevin's answer is what you need to do. You should not be altering DefaultRequestHeaders for each request; they are intended as they say on the tin; i.e. defaults that apply to all messages that that instance of HttpClient sends. You're just creating a race condition across threads by altering them. Per-request headers should be added to the HttpRequestMessage, not the HttpClient.
Thank you for the detailed explanation. I changed my code and updated the app on a server. Now, I will monitor how it behaves.

Collectives™ on Stack Overflow

HttpClient header getting nulled when using in async methods

1 Answer 1

3 Comments

Your Answer

Linked

Hot Network Questions

Collectives™ on Stack Overflow

1 Answer 1

3 Comments

Your Answer

Sign up or log in

Post as a guest

Linked

Related