How to implement SSL with HttpWebRequest?

Question

I'm going to get information from https web pages. But the downloaded data is coded and illegible.

\u001f�\b\0\0\0\0\0\0\0�iw۸�0�Y�\u0015\u0018��(y�z��,v��rܝ\u0019�v�N/�s��P\",3�H]��r��\u007f\u007fQ�\b��N��\u0011Il�B��\0\u0014�^�@�ŋ\u0017��\u007f��Gt��\u0003t��S\u001f9k\u001b\u001b\u007fl�76\u000e/\u000fџ�^~>F[��2v��O�(t��'\u000er��t��qww�~��\u001eţ��{�k\v\n�ǵT)��}�\u001eR�\a�^�E��\u0018�.��?��\a�,vGc�A�(Lq�~p�hm�\u000e��a�߇��>�\u001dGA�b\u001f�'~�\u0013��\u0016�\n\bB�/��\K\u001f&X)��t\u0003z��n��×ˣ��\u000e\"�YkjOh\u001f۲��\0�]\��\rY?\u001b\u0015u��%��\0w�o��{��\u000fo�u��>8��\u001b�$�\b\"��_\lv6�v6�ެ�o\u000e�q��I҇\0'�

How to get the SSL certificate into the HttpWebRequest class?

class MyHttpWebRequest : IDisposable
{
    private static int Count = 0;
    private HttpWebRequest request;
    private Stream dataStream;
    private string Status { get; set; }

    private string userAgent = "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36";
    private string accept = "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8";
    private string acceptLanguage = "Accept-Language: en-US,en;q=0.9";
    private string acceptEncoding = "Accept-Encoding: gzip, deflate, br";
    private string host;
    private X509Certificate2 clientCertificates;
    private String referer { get; set; } = "";
    private CookieCollection cookieCollection { get; set; } = null;

    public MyHttpWebRequest()
    {
        Count++;
        clientCertificates = new X509Certificate2(@"mahan.cer");
        //clientCertificates = X509Certificate.CreateFromCertFile(@"mahan.cer");//(@"LocalAuthority.crt");
        //ServicePointManager.Expect100Continue = true;
        //ServicePointManager.SecurityProtocol = SecurityProtocolType.Ssl3;
        //ServicePointManager.Expect100Continue = false;
        //ServicePointManager.SecurityProtocol = SecurityProtocolType.Ssl3 | SecurityProtocolType.Tls | SecurityProtocolType.Tls11 | SecurityProtocolType.Tls12;
        //WebRequestHandler handler = new WebRequestHandler();
        //X509Certificate2 certificate = GetMyX509Certificate();
        //handler.ClientCertificates.Add(certificate);
        //HttpClient client = new HttpClient(handler);
    }
    public MyHttpWebRequest(string host) : this()
    {
        this.host = host;
    }
    ~MyHttpWebRequest()
    {
        Count--;
    }
    public string GetResponse()
    {
        try
        {
            HttpWebResponse response = (HttpWebResponse)request.GetResponse();
            if (cookieCollection == null) cookieCollection = new CookieCollection();
            cookieCollection.Add(response.Cookies);
            this.Status = response.StatusDescription;

            dataStream = response.GetResponseStream();

            StreamReader reader = new StreamReader(dataStream);

            string responseFromServer = reader.ReadToEnd();

            // Clean up the streams.
            reader.Close();
            //dataStream.Close();
            response.Close();

            //clientCertificates = new X509Certificate2(request.ServicePoint.Certificate);

            this.Status = "Successful";
            return responseFromServer;
        }
        catch (Exception ex)
        {
            this.Status = ex.Message;
            return ex.ToString();
        }

    }
    public string SetRequest(string url)
    {
        try
        {
            request = (HttpWebRequest)WebRequest.Create(url);
            request.Method = "GET";
            request.UserAgent = userAgent;
            request.Accept = accept;
            request.Headers.Add(acceptLanguage);
            request.Headers.Add(acceptEncoding);
            request.KeepAlive = true;
            request.CookieContainer = new CookieContainer();
            if (cookieCollection != null)
                request.CookieContainer.Add(cookieCollection);
            request.ProtocolVersion = HttpVersion.Version11;
            request.AllowAutoRedirect = false;
            request.ContentType = "application/x-www-form-urlencoded";
            request.Referer = referer;
            referer = url;
            request.Host = host;
            //request.ClientCertificates.Clear();
            request.ClientCertificates.Add(clientCertificates);
            request.PreAuthenticate = true;

            this.Status = "Successful";
        }
        catch (Exception ex)
        {
            this.Status = ex.ToString();
        }
        return this.ToString();
    }
    public string SetRequest(string url, string method)
    {
        try
        {
            request = (HttpWebRequest)WebRequest.Create(url);
            if (method.Equals("GET") || method.Equals("POST"))
            {
                request.Method = method;
            }
            else
            {
                throw new Exception("Invalid Method Type");
            }
            request.UserAgent = userAgent;
            request.Accept = accept;
            request.Headers.Add(acceptLanguage);
            request.Headers.Add(acceptEncoding);
            request.KeepAlive = true;
            request.CookieContainer = new CookieContainer();
            if (cookieCollection != null)
                request.CookieContainer.Add(cookieCollection);
            request.ProtocolVersion = HttpVersion.Version11;
            request.AllowAutoRedirect = false;
            request.ContentType = "application/x-www-form-urlencoded";
            request.Referer = referer;
            referer = url;
            request.Host = host;
            //request.ClientCertificates.Clear();
            request.ClientCertificates.Add(clientCertificates);
            request.PreAuthenticate = true;

            this.Status = "Successful";
        }
        catch (Exception ex)
        {
            this.Status = ex.ToString();
        }
        return this.ToString();
    }
    public string SetRequest(string url, string method, string data)
    {
        try
        {
            request = (HttpWebRequest)WebRequest.Create(url);
            if (method.Equals("GET") || method.Equals("POST"))
            {
                request.Method = method;
            }
            else
            {
                throw new Exception("Invalid Method Type");
            }
            request.UserAgent = userAgent;
            request.Accept = accept;
            request.Headers.Add(acceptLanguage);
            request.Headers.Add(acceptEncoding);
            request.KeepAlive = true;
            request.CookieContainer = new CookieContainer();
            if (cookieCollection != null)
                request.CookieContainer.Add(cookieCollection);
            request.ProtocolVersion = HttpVersion.Version11;
            request.AllowAutoRedirect = false;
            request.ContentType = "application/x-www-form-urlencoded";
            request.Referer = referer;
            referer = url;
            request.Host = host;
            //request.ClientCertificates.Clear();
            request.ClientCertificates.Add(clientCertificates);
            request.PreAuthenticate = true;

            byte[] byteArray = Encoding.UTF8.GetBytes(data);
            request.ContentLength = byteArray.Length;
            dataStream = request.GetRequestStream();
            dataStream.Write(byteArray, 0, byteArray.Length);
            dataStream.Close();

            this.Status = "Successful";
        }
        catch (Exception ex)
        {
            this.Status = ex.Message;
        }
        return this.Status;
    }
    public string SetRequest(string url, string method, string data, string contentType)
    {
        try
        {
            request = (HttpWebRequest)WebRequest.Create(url);
            if (method.Equals("GET") || method.Equals("POST"))
            {
                request.Method = method;
            }
            else
            {
                throw new Exception("Invalid Method Type");
            }
            request.UserAgent = userAgent;
            request.Accept = accept;
            request.Headers.Add(acceptLanguage);
            request.Headers.Add(acceptEncoding);
            request.KeepAlive = true;
            request.CookieContainer = new CookieContainer();
            if (cookieCollection != null)
                request.CookieContainer.Add(cookieCollection);
            request.ProtocolVersion = HttpVersion.Version11;
            request.AllowAutoRedirect = false;
            request.ContentType = contentType;
            request.Referer = referer;
            referer = url;
            request.Host = host;
            //request.ClientCertificates.Clear();
            request.ClientCertificates.Add(clientCertificates);
            request.PreAuthenticate = true;

            byte[] byteArray = Encoding.UTF8.GetBytes(data);
            request.ContentLength = byteArray.Length;
            dataStream = request.GetRequestStream();
            dataStream.Write(byteArray, 0, byteArray.Length);
            dataStream.Close();

            this.Status = "Successful";
        }
        catch (Exception ex)
        {
            this.Status = ex.Message;
        }
        return this.Status;
    }
    public void Dispose()
    {
        request.Abort();
        request = null;
        dataStream.Close();
        dataStream.Dispose();
        dataStream = null;
    }
}

File may be GZIP or data could be Unicode. Use sniffer like wireshark or fiddler to determine encoding. — jdweng
– jdweng, Commented Feb 4, 2018 at 9:08
The data does not look like http data. Any certificate or encoded data on http you be ascii characters. You have binary character like \u0013 which is a unicode. So I suspect something else is wrong. — jdweng
– jdweng, Commented Feb 4, 2018 at 9:22

Steffen Ullrich · Accepted Answer · 2018-02-04 09:38:55Z

4

How to get the SSL certificate into the HttpWebRequest class?

Your problem is not related to SSL at all.

private string acceptEncoding = "Accept-Encoding: gzip, deflate, br";
...
        request.Headers.Add(acceptEncoding);

With this code you explicitly tell the server that you'll support various content compression algorithms. Only, this is a lie since you don't deal with compression when reading the response.

\u001f�\b\0\0\0\0\0\0\0

This looks the beginning of a gzip data stream. It starts with hex \x1f\x8b but you interpret this wrongly as UTF-8 which it is not.

The easiest way to fix this part is to remove the Accept-Encoding from your request or use Accept-Encoding: identity to signal that you don't accept any compression. This way a well behaving server will send you the body without any compression.

edited Feb 4, 2018 at 9:38

answered Feb 4, 2018 at 9:33

Steffen Ullrich

125k11 gold badges155 silver badges194 bronze badges

Sign up to request clarification or add additional context in comments.

Comments

xav-stargate · Accepted Answer · 2018-02-04 09:14:27Z

0

HttpWebRequest request = (HttpWebRequest)WebRequest.Create("https://yourwebsite.nld");
HttpWebResponse response = (HttpWebResponse)request.GetResponse();
response.Close();


X509Certificate certV1 = request.ServicePoint.Certificate;

Certificate is a X509Certificate not a X509Certificate2. To convert add

X509Certificate2 certV2 = new X509Certificate2(certV1);

answered Feb 4, 2018 at 9:14

xav-stargate

7371 gold badge5 silver badges16 bronze badges

1 Comment

Arash Irani Over a year ago

what is certV2 doing? Where is it used?

Collectives™ on Stack Overflow

How to implement SSL with HttpWebRequest?

2 Answers 2

Comments

1 Comment

Your Answer

Hot Network Questions

Collectives™ on Stack Overflow

2 Answers 2

Comments

1 Comment

Your Answer

Sign up or log in

Post as a guest

Related