Small HTTP client/SDK created using libcurl in C language

Question

I created a SDK in C for my Rust project (a key value store in Rust). It should give access to the HTTP routes in a simple manner. (I never worked with C outside of university.)

I used:

libcurl to call the HTTP endpoints from C, because it is a well known library and works on almost any system.
CMake to make it easier to build and link the library and the example.
clang-tidy for static analysis/linting and clang-format for formatting
for the result struct, I wanted a Rust Result feeling, and took inspiration from this gist: https://gist.github.com/f0rki/9b2c2b73d46ccdad2b39ab79b3a5517f

The lib code is pasted here, but I have a PR open on GitHub, with CMake, header files and a example executable. I accept reviews there as well. Thanks!

https://github.com/auyer/MemoryKV/pull/2

#include <curl/curl.h>
#include <stdbool.h>
#include <stdlib.h>
#include <string.h>

// string_carrier struct to store the body results from curl with easy reallocation
struct string_carrier {
    char *ptr;
    size_t len;
    bool error_flag;
};

// init_string_carrier initializes the string_carrier struct to store the body results from curl
void init_string_carrier(struct string_carrier *s) {
    s->len = 0;
    s->ptr = malloc(s->len + 1);
    s->error_flag = false;
    if (s->ptr == NULL) {
        // TODO: figure out a better way to handle errors in C
        fprintf(stderr, "memkv client: Error in Callback, malloc() failed\n");
        // skip exit and return error instead
        // exit(EXIT_FAILURE);
        s->error_flag = true;
    } else {
        s->ptr[0] = '\0';
    }
}

// string_carrier_writefunc is the callback function to read the body from libcurl into a string_carrier
size_t string_carrier_writefunc(void *ptr, size_t size, size_t nmemb, struct string_carrier *s) {
    // new length to realocate response chunks from libcurl
    size_t new_len = s->len + size * nmemb;
    s->ptr = realloc(s->ptr, new_len + 1);
    if (s->ptr == NULL) {
        // TODO: figure out a better way to handle errors in C
        fprintf(stderr, "memkv client: Error in string_carrier_writefunc Callback, realloc() failed\n");
        // skip exit and return error instead
        // exit(EXIT_FAILURE);
        s->error_flag = true;
    } else {
        memcpy(s->ptr + s->len, ptr, size * nmemb);
        s->ptr[new_len] = '\0';
        s->len = new_len;
    }

    return size * nmemb;
}

// memkv_client
typedef struct memkv_client {
    char *host;
} memkv_client;

void memkv_init_client(struct memkv_client *client, char *host) {
    curl_global_init(CURL_GLOBAL_ALL);
    client->host = host;
}

memkv_client *memkv_client_new(char *host) {
    memkv_client *client = malloc(sizeof(memkv_client));
    memkv_init_client(client, host);
    return client;
}

char *build_url(const char *host, const char *params) {
    // snprintf returns the number of characters that would have been written if called with NULL
    unsigned long sizeneeded = snprintf(NULL, 0, "%s/%s", host, params);
    // use that number to allocate a buffer of the right size
    char *url = malloc(sizeneeded + 1);
    // write the string_carrier to the buffer
    sprintf(url, "%s/%s", host, params);
    return url;
}

typedef struct {
    bool success;
    union {
        char *result;
        char *error;
    };
} memkv_result;

static const char base_curl_error[] = "Error from curl";
static const char unknown_error_msg[] = "unknown error";

void make_curl_error(memkv_result *r, const char *err) {
    r->success = false;

    if (strlen(err) == 0) {
        r->error = malloc(sizeof(unknown_error_msg));
        strcpy(r->error, unknown_error_msg);
        return;
    }

    unsigned long sizeneeded = snprintf(NULL, 0, "%s : %s", base_curl_error, err);

    r->error = malloc(sizeneeded + 1);

    snprintf(r->error, sizeneeded + 1, "%s : %s", base_curl_error, err);
}

// init_memkv_result initializes the memkv_result struct with success as false
memkv_result *init_memkv_result() {

    memkv_result *r = malloc(sizeof(memkv_result));

    r->result = malloc(sizeof(char));
    if (r->result == NULL) {
        free(r);
        printf("Error in init_memkv_result");
        return NULL;
    }

    r->success = false;
    return r;
}

// the functions that make the HTTP calls start here
// they are very similar, and have some code repetition that maybe can be put in a function. 

memkv_result *memkv_get_key(struct memkv_client *client, const char *key) {
    memkv_result *r = init_memkv_result();

    CURL *curl;
    CURLcode res;

    curl = curl_easy_init();
    if (!curl) {
        make_curl_error(r, "Failed startig curl");
        return r;
    }

    struct string_carrier s;
    init_string_carrier(&s);
    if (s.error_flag) {
        make_curl_error(r, "Failed to initialize string_carrier");
        return r;
    }

    char *url = build_url(client->host, key);
    curl_easy_setopt(curl, CURLOPT_URL, url);
    curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L);
    curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, string_carrier_writefunc);
    curl_easy_setopt(curl, CURLOPT_WRITEDATA, &s);

    res = curl_easy_perform(curl);

    /* always cleanup */
    curl_easy_cleanup(curl);
    free(url);

    if (res != CURLE_OK) {
        const char *err = curl_easy_strerror(res);
        make_curl_error(r, err);
        return r;
    }
    if (s.error_flag) {
        make_curl_error(r, "Failed to get results from server");
        return r;
    }
    r->success = true;
    r->result = malloc(strlen(s.ptr) + 1);

    strcpy(r->result, s.ptr);
    return r;
}

memkv_result *memkv_delete_key(struct memkv_client *client, const char *key) {
    memkv_result *r = init_memkv_result();

    CURL *curl;
    CURLcode res;

    curl = curl_easy_init();
    if (!curl) {
        make_curl_error(r, "Failed startig curl");
        return r;
    }

    struct string_carrier s;
    init_string_carrier(&s);
    if (s.error_flag) {
        make_curl_error(r, "Failed to initialize string_carrier");
        return r;
    }

    char *url = build_url(client->host, key);
    curl_easy_setopt(curl, CURLOPT_URL, url);
    curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L);

    // set custom request to delete
    curl_easy_setopt(curl, CURLOPT_CUSTOMREQUEST, "DELETE");
    curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L);
    curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, string_carrier_writefunc);
    curl_easy_setopt(curl, CURLOPT_WRITEDATA, &s);

    /* Perform the request, res will get the return code */
    res = curl_easy_perform(curl);

    /* always cleanup */
    curl_easy_cleanup(curl);
    free(url);

    /* Check for errors */
    if (res != CURLE_OK) {
        const char *err = curl_easy_strerror(res);
        make_curl_error(r, err);
        return r;
    }

    if (s.error_flag) {
        make_curl_error(r, "Failed to get results from server");
        return r;
    }
    r->success = true;
    r->result = malloc(strlen(s.ptr) + 1);

    strcpy(r->result, s.ptr);
    return r;
}

memkv_result *memkv_put_key(struct memkv_client *client, const char *key, const char *put_body) {
    memkv_result *r = init_memkv_result();
    CURL *curl;
    CURLcode res;
    // start the easy interface in curl
    curl = curl_easy_init();
    if (!curl) {
        make_curl_error(r, "Failed startig curl");
        return r;
    }
    struct string_carrier s;
    init_string_carrier(&s);
    if (s.error_flag) {
        make_curl_error(r, "Failed to initialize string_carrier");
        return r;
    }

    char *url = build_url(client->host, key);
    curl_easy_setopt(curl, CURLOPT_URL, url);

    struct curl_slist *headers = NULL;

    /* default type with postfields is application/x-www-form-urlencoded,
       change it if you want */
    headers = curl_slist_append(headers, "Content-Type: application/octet-stream");
    curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers);

    /* pass on content in request body. When CURLOPT_POSTFIELDSIZE is not used,
    curl does strlen to get the size. */
    curl_easy_setopt(curl, CURLOPT_POSTFIELDS, put_body);
    curl_easy_setopt(curl, CURLOPT_CUSTOMREQUEST, "PUT");
    curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L);
    curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, string_carrier_writefunc);
    curl_easy_setopt(curl, CURLOPT_WRITEDATA, &s);

    /* Perform the request, res will get the return code */
    res = curl_easy_perform(curl);

    /* always cleanup */
    curl_easy_cleanup(curl);
    free(url);
    /* free headers */
    curl_slist_free_all(headers);

    /* Check for errors */
    if (res != CURLE_OK) {
        const char *err = curl_easy_strerror(res);
        make_curl_error(r, err);
        return r;
    }

    if (s.error_flag) {
        make_curl_error(r, "Failed to get results from server");
        return r;
    }
    r->success = true;
    r->result = malloc(strlen(s.ptr) + 1);

    strcpy(r->result, s.ptr);
    return r;
}

memkv_result *memkv_list_keys(struct memkv_client *client) {
    memkv_result *r = init_memkv_result();

    CURL *curl;
    CURLcode res;

    // start the easy interface in curl
    curl = curl_easy_init();
    if (!curl) {
        make_curl_error(r, "Failed startig curl");
        return r;
    }
    struct string_carrier s;
    init_string_carrier(&s);
    if (s.error_flag) {
        make_curl_error(r, "Failed to initialize string_carrier");
        return r;
    }

    const char *key = "keys";
    char *url = build_url(client->host, key);

    // set custom request to delete
    curl_easy_setopt(curl, CURLOPT_URL, url);
    curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L);
    curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, string_carrier_writefunc);
    curl_easy_setopt(curl, CURLOPT_WRITEDATA, &s);

    /* Perform the request, res will get the return code */
    res = curl_easy_perform(curl);

    /* always cleanup */
    curl_easy_cleanup(curl);
    free(url);

    /* Check for errors */
    if (res != CURLE_OK) {
        r->success = false;
        const char *err = curl_easy_strerror(res);
        make_curl_error(r, err);
        return r;
    }

    if (s.error_flag) {
        make_curl_error(r, "Failed to get results from server");
        return r;
    }
    r->success = true;
    r->result = malloc(strlen(s.ptr) + 1);

    strcpy(r->result, s.ptr);
    return r;
}

memkv_result *memkv_list_keys_with_prefix(struct memkv_client *client, const char *key_prefix) {
    memkv_result *r = init_memkv_result();

    CURL *curl;
    CURLcode res;

    // start the easy interface in curl
    curl = curl_easy_init();
    if (!curl) {
        make_curl_error(r, "Failed startig curl");
        return r;
    }
    struct string_carrier s;
    init_string_carrier(&s);
    if (s.error_flag) {
        make_curl_error(r, "Failed to initialize string_carrier");
        return r;
    }

    const char *key = "keys";
    char *url_part1 = build_url(client->host, key);
    // TODO: build URL in one call
    char *url = build_url(url_part1, key_prefix);
    free(url_part1);

    curl_easy_setopt(curl, CURLOPT_URL, url);
    curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L);
    curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, string_carrier_writefunc);
    curl_easy_setopt(curl, CURLOPT_WRITEDATA, &s);

    /* Perform the request, res will get the return code */
    res = curl_easy_perform(curl);

    /* always cleanup */
    curl_easy_cleanup(curl);
    free(url);

    /* Check for errors */
    if (res != CURLE_OK) {
        r->success = false;
        const char *err = curl_easy_strerror(res);
        make_curl_error(r, err);
        return r;
    }

    if (s.error_flag) {
        make_curl_error(r, "Failed to get results from server");
        return r;
    }
    r->success = true;
    r->result = malloc(strlen(s.ptr) + 1);

    strcpy(r->result, s.ptr);
    return r;
}

memkv_result *memkv_delete_keys_with_prefix(struct memkv_client *client, const char *key_prefix) {
    memkv_result *r = init_memkv_result();

    CURL *curl;
    CURLcode res;

    curl = curl_easy_init();
    if (!curl) {
        make_curl_error(r, "Failed startig curl");
        return r;
    }

    struct string_carrier s;
    init_string_carrier(&s);
    if (s.error_flag) {
        make_curl_error(r, "Failed to initialize string_carrier");
        return r;
    }

    const char *key = "keys";
    char *url_part1 = build_url(client->host, key);
    // TODO: build URL in one call
    char *url = build_url(url_part1, key_prefix);
    free(url_part1);

    curl_easy_setopt(curl, CURLOPT_URL, url);
    curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L);

    // set custom request to delete
    curl_easy_setopt(curl, CURLOPT_CUSTOMREQUEST, "DELETE");
    curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L);
    curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, string_carrier_writefunc);
    curl_easy_setopt(curl, CURLOPT_WRITEDATA, &s);

    /* Perform the request, res will get the return code */
    res = curl_easy_perform(curl);

    /* always cleanup */
    curl_easy_cleanup(curl);
    free(url);

    /* Check for errors */
    if (res != CURLE_OK) {
        const char *err = curl_easy_strerror(res);
        make_curl_error(r, err);
        return r;
    }

    if (s.error_flag) {
        make_curl_error(r, "Failed to get results from server");
        return r;
    }
    r->success = true;
    r->result = malloc(strlen(s.ptr) + 1);

    strcpy(r->result, s.ptr);
    return r;
}

memkv_result *memkv_delete_all_keys(struct memkv_client *client) {
    memkv_result *r = init_memkv_result();

    CURL *curl;
    CURLcode res;

    // start the easy interface in curl
    curl = curl_easy_init();
    if (!curl) {
        make_curl_error(r, "Failed startig curl");
        return r;
    }
    struct string_carrier s;
    init_string_carrier(&s);
    if (s.error_flag) {
        make_curl_error(r, "Failed to initialize string_carrier");
        return r;
    }

    const char *key = "keys";
    char *url = build_url(client->host, key);

    curl_easy_setopt(curl, CURLOPT_CUSTOMREQUEST, "DELETE");
    curl_easy_setopt(curl, CURLOPT_URL, url);
    curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L);
    curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, string_carrier_writefunc);
    curl_easy_setopt(curl, CURLOPT_WRITEDATA, &s);

    /* Perform the request, res will get the return code */
    res = curl_easy_perform(curl);

    /* always cleanup */
    curl_easy_cleanup(curl);
    free(url);

    /* Check for errors */
    if (res != CURLE_OK) {
        r->success = false;
        const char *err = curl_easy_strerror(res);
        make_curl_error(r, err);
        return r;
    }

    if (s.error_flag) {
        make_curl_error(r, "Failed to get results from server");
        return r;
    }
    r->success = true;
    r->result = malloc(strlen(s.ptr) + 1);

    strcpy(r->result, s.ptr);
    return r;
}
```

Madagascar · Accepted Answer · 2023-10-15 09:51:29Z

Memory Leak:

s->ptr = realloc(s->ptr, new_len + 1);

If realloc() returned NULL, we'd lose access to the original memory and suffer a memory leak. Use a temporary pointer instead:

void *const tmp = realloc(s->ptr, new_len + 1);

if (!tmp) {
    /* realloc() failed to reallocate memory. The original memory is still
     * there and needs to be freed.
     */
     handle_failure();
} else {
     /* Now s->ptr points to the new chunk of memory. */
     s->ptr = tmp;
     ...
}

Reduce scope:

Unless I am mistaken, base_curl_error and unknown_error_msg are not used outside of make_url_error(), so there is no point in defining them outside of the function.

Simplify initialization:

In init_string_carrier():

# if 0
    s->len = 0;
    s->ptr = malloc(s->len + 1);
    s->error_flag = false;
#else
    memset(s, 0, sizeof *s);

Check the return value of library functions:

We forego checking the result of malloc() in memkv_client_new(), build_url(), make_curl_error(), init_memkv_result(), memkv_get_key(), et cetera. Good job on not casting the return value albeit.

The return value of curl_global_init() in memkv_init_client() goes unchecked. According to the documentation:

If this function returns non-zero, something went wrong and you cannot use the other curl functions.

But it may not be necessary to call it at all, since the documentation for curl_easy_init() states:

If you did not already call curl_global_init(3), curl_easy_init(3) does it automatically.

Stylistic choice:

Why has struct memkv been typedefed, but struct string_carrier has not?

Redundant comment:

// memkv_client
typedef struct memkv_client {
   ..

It should be removed, or replaced with one that describes the struct.

snprintf() returns an int:

 // unsigned long sizeneeded = snprintf(NULL, 0, "%s/%s", host, params);
  int sizeneeded = snprint(NULL, 0, "%s/%s", host, params);

Is there any specific reason why snake-case has been disregarded whilst naming the above identifier, unlike the rest of the code?

Reduce duplication:

There's a significant amount of code duplication in the functions for making HTTP requests in your code. If we introduce a helper like so:

// NOTE: This is untested code. Change it as you may. 
memkv_result *memkv_execute_request(
    const char *url,
    const char *custom_req, 
    const char *content_type,
    const char *post_data
) { 
    memkv_result *r = init_memkv_result();

    CURL *curl = curl_easy_init();
    
    if (!curl) {
        make_curl_error(r, "Failed starting curl.");
        return r;
    }

    CURLCODE res;

    struct string_carrier s;
    init_string_carrier(&s);

    if (s.error_flag) {
        make_curl_error(r, "Failed to initialize string_carrier");
        return r;
    }

    curl_easy_setopt(curl, CURLOPT_URL, url);

    if (custom_req) {
        curl_easy_setopt(curl, CURLOPT_CUSTOMREQUEST, custom_req);
    }

    struct curl_slist *headers = NULL;

    if (content_type) {
        /* default type with postfields is application/x-www-form-urlencoded,
           change it if you want. */
        headers = curl_slist_append(headers, content_type);
        curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers);
    }   
    
    if (post_data) {
        curl_easy_setopt(curl, CURLOPT_POSTFIELDS, post_data);
    }

    curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L);
    curl_easy_setopt(curl, CURLOPT_WRITE_FUNCIION, string_carrier_writefunc);
    curl_easy_setopt(curl, CUROPT_WRITEDATA, &s);

    res = curl_easy_perform(curl);

    curl_easy_cleanup(curl);
    free(url);
    
    if (headers) {
        curl_slist_free_all(headers);
    }

    if (res != CURLE_OK) {
        const char *err = curl_easy_strerror(res);
        make_curl_error(r, err);
        return r;
    }

    if (s.error_flag) {
        make_curl_error(r, "Failed to get results from server.");
        return r;
    }
    
    r->success = true;
    r->result = malloc(strlen(s.ptr) + 1);
    
    strcpy(r->result, s.ptr);
    return r;
}

we get left with just:

memkv_result *memkv_get_key(struct memkv_client *client, const char *key) {
    char *url = build_url(client->host, key);
    return memkv_execute_request(url, NULL, NULL, NULL);

memkv_result *memkv_delete_key(struct memkv_client *client, const char *key) {
    char *url = build_url(client->host, key);
    return memkv_execute_request(url, "DELETE", NULL, NULL);

memkv_result *memkv_put_key(struct memkv_client *client, const char *key, const char *put_body) {
    char *url = build_url(client->host, key);
    return memkv_execute_request(url, "PUT", "Content-Type: application/octet-stream", put_body);
}

memkv_result *memkv_list_keys(struct memkv_client *client) {
    char *url = build_url(client->host, "keys");
    return memkv_execute_request(url, NULL, NULL, NULL);
}
    
memkv_result *memkv_list_keys_with_prefix(struct memkv_client *client, const char *key_prefix) {
    char *url_part1 = build_url(client->host, "keys");
    // TODO: build URL in one call
    char *url = build_url(url_part1, key_prefix);
    free(url_part1);

    return memkv_execute_request(url, NULL, NULL, NULL);
}

memkv_result *memkv_delete_keys_with_prefix(struct memkv_client *client, const char *key_prefix) {
    char *url_part1 = build_url(client->host, "keys");
    // TODO: build URL in one call
    char *url = build_url(url_part1, key_prefix);
    free(url_part1);
    return memkv_execute_request(url, "DELETE", NULL, NULL);
}
    
memkv_result *memkv_delete_all_keys(struct memkv_client *client) {
    char *url = build_url(client->host, "keys");
    return memkv_execute_request(url, "DELETE", NULL, NULL);
}

Others:

The functions that are not used outside of this translation unit should be marked as having internal linkage. (with the static keyword)
The parameters that are not changed anywhere in the function should be const-qualified.
Don't declare more variables than you need. The key variable in all the HTTP functions is not required.
Consider initializing variables as you declare them.
Use more const.
Fix the spelling mistakes in the error strings.
sizeof(char) is defined by the C standard to be 1.

Allocate to the referenced type:

// memkv_result *r = malloc(sizeof(memkv_result));
memkv_result *r = malloc(sizeof *r);

Consider rewriting the build_url() function to take a pointer to a pointer as the the second parameter, and optionally, a third argument to specify the number of items like so:

static char *build_url(const char *host, const char **params, size_t num_params) {
    ...
}

Now we can call it like this:

#if 0
    char *url_part1 = build_url(client->host, "keys");
    // TODO: build URL in one call
    char *url = build_url(url_part1, key_prefix);
#else 
    const char *const url = build_url(client->host, {"keys", key_prefix}, 2);

Thanks a lot! I have some further questions/comments! --- About the handle_failure(); in the "Memory Leak" topic: What should be done to handle this failure ? Should s->ptr be freed ? I was afraid curl would fail during the write process and give me an unexpected result. --- About snprintf() returns an int. clang-tidy reported this to me (and its the reason I used unsigned long) libMemoryKV.c:71:21: error: either cast from 'int' to 'unsigned long' is ineffective, or there is loss of precision before the conversion — Auyer
– Auyer, Commented Oct 15, 2023 at 1:28
1) In case of a realloc() failure, s->ptr does indeed remain valid and needs to be freed then, or at some other point in the code. 2) This error message is telling you that there is an issue with converting an int to an unsigned long. As snprint() returns an int, its result should be stored in an int, and not an unsigned long, or it could result in a loss of information due to a difference in data sizes. If an output error is encountered, snprintf() returns a negative value, but an unsigned type can not hold negative values. — Madagascar
– Madagascar, Commented Oct 15, 2023 at 9:43
I see. But to clarify, the message I get in the case "2" is when I change it back to int, not the other way around. Maybe that's a difference in C standards ? I have configured c23 in this project. — Auyer
– Auyer, Commented Oct 15, 2023 at 14:42

Stack Exchange Network

Small HTTP client/SDK created using libcurl in C language

1 Answer 1

You must log in to answer this question.

Hot Network Questions

Small HTTP client/SDK created using libcurl in C language

1 Answer 1

You must log in to answer this question.

Related

Hot Network Questions