Segmentation fault when trying to copy memory from Gst buffer to torch tensor using torch::from_blob::clone()

I'm getting a segmentation fault when I'm trying to copy memory from gst buffer to torch tensor. Here is code. I'll mark where the segfault originates from:

static GstPadProbeReturn on_probe(
    GstPad *pad, GstPadProbeInfo *info, gpointer user_data
) {
    GstBuffer *buffer = GST_PAD_PROBE_INFO_BUFFER(info);
    GstMapInfo map;
    FrameInfo *fr = static_cast<FrameInfo*>(user_data);
    std::ostringstream file_name;
    file_name << "image";

    int expected_size = 1920 * 2560 * 3;

    std::cout << "Expected size: " << expected_size;
    std::cout << " Got size: " << map.size << std::endl;

    if (gst_buffer_map(buffer, &map, GST_MAP_READ)) {
        cv::Mat norm_frame;
        cv::Mat frame(1920, 2560, CV_8UC3, map.data);
        cv::cvtColor(frame, frame, cv::COLOR_RGB2BGR);
        frame.convertTo(norm_frame, CV_32FC3, 1.0f / 255.0f);

        file_name << fr->frame_no << ".png";
        bool success = cv::imwrite(file_name.str(), frame);
        if (!success) { 
            std::cerr << "Failed to write image";
        }

        at::Tensor tensor = torch::from_blob(
            norm_frame.data, {1, norm_frame.rows, norm_frame.cols, 3}, at::kFloat
        ).clone(); // <- SEGFAULT here
        tensor = tensor.permute({0, 3, 1, 2});

        torch::jit::IValue output = fr->model.forward({tensor});
        // at::Tensor output = fr->model.forward({tensor}).toTensor();
        // std::cout << tensor.sizes() << "\n";

        fr->frame_no++;
        gst_buffer_unmap(buffer, &map);
    }   

    if (buffer) {
        GstClockTime pts = GST_BUFFER_PTS(buffer);
        std::cout << "Buffer PTS: " << GST_TIME_AS_MSECONDS(pts) << " ms" << std::endl;
    }
    return GST_PAD_PROBE_OK;
}

This probe function is attached to the autovideosink of a gstreamer pipeline. The pipeline looks like this:

filesrc location=/path/to/video.mp4 ! decodebin ! videoconvert ! video/x-raw,format=RGB,width=2560,height=1920 ! autovideosink name=sink"

I've tried saving each frame using OpenCV, that worked. If I don't use the clone method and directly pass it to the model, it segfaults there as well. The model is a torchscript model with 1920x2560 input dimensions.