1

server side extracting frame from video using ffmpeg library. Here is code:

    AVFormatContext *pFormatContext = avformat_alloc_context();
    avformat_open_input(&pFormatContext, "VID_20230219_171331_300.mp4.mov", NULL, NULL);
    avformat_find_stream_info(pFormatContext,  NULL);

    const AVCodec *pCodec = NULL;
    AVCodecParameters *pCodecParameters =  NULL;
    int video_stream_index = -1;

    // loop though all the streams and print its main information
    for (int i = 0; i < pFormatContext->nb_streams; i++)
    {
        AVCodecParameters *pLocalCodecParameters =  NULL;
        pLocalCodecParameters = pFormatContext->streams[i]->codecpar;
        const AVCodec *pLocalCodec = NULL;
        pLocalCodec = avcodec_find_decoder(pLocalCodecParameters->codec_id);

        if(pLocalCodec==NULL)
            continue;

        // when the stream is a video we store its index, codec parameters and codec
        if(pLocalCodecParameters->codec_type == AVMEDIA_TYPE_VIDEO)
        {
            if(video_stream_index == -1)
            {
                video_stream_index = i;
                pCodec = pLocalCodec;
                pCodecParameters = pLocalCodecParameters;

                const char *buffer = reinterpret_cast <const char*> (pFormatContext->streams[i]->codecpar->extradata);
                QFile file("extradata.txt"); //!!!!!!!!!!!!!!!! storing extradata for browser visualization
                file.open(QIODevice::WriteOnly);
                QDataStream out(&file);
                out.writeRawData(buffer, pFormatContext->streams[i]->codecpar->extradata_size);
                file.flush();
                file.close();
            }
        }
    }

    AVCodecContext *pCodecContext = avcodec_alloc_context3(pCodec);
    avcodec_parameters_to_context(pCodecContext, pCodecParameters);
    avcodec_open2(pCodecContext, pCodec, NULL);

    AVFrame *pFrame = av_frame_alloc();
    AVPacket *pPacket = av_packet_alloc();

    int response = 0;
    int how_many_packets_to_process = 800;

    int frame_c = 0, counter = 0;
    while(true)
    {
        frame_c = av_read_frame(pFormatContext, pPacket);

        if(frame_c < 0)
            break;

        if(pPacket->stream_index == video_stream_index)
        {
            const char *buffer = reinterpret_cast <const char*> (pPacket->data);
            response = decode_packet(pPacket, pCodecContext, pFrame);

            if(response < 0)
                break;

            counter++;
            if(--how_many_packets_to_process <= 0) break;

            if(counter > 1)
            {
                QFile file("frame.txt"); // !!!!!!!!!!!!!!!! storing single frame for browser visualization
                file.open(QIODevice::WriteOnly);
                QDataStream out(&file);
                out.writeRawData(buffer, arr.size());
                file.flush();
                file.close();
                break;
            }
        }

        av_packet_unref(pPacket);
    }

As the result i stored 2 txt files: 1 with extradata of codec and 2 with frame pixel data. The next step is to decode this data in browser and display picture. Here is how i try to do it:

<input type="file" id="file-input" />
<h3>Contents of the file:</h3>
<pre id="file-content"></pre>

<input type="file" id="file-input2" />
<h3>Contents of the file:</h3>
<pre id="file-content2"></pre>

<canvas id="myCanvas" width="720" height="1280"
style="border:1px solid #d3d3d3;">
Your browser does not support the HTML5 canvas tag.
</canvas>

<script>

let pendingFrames = [];
let extraData = new Uint8Array;

const init = {
  output: handleFrame,
  error: (e) => {
  console.log(e.message);
  },
};

const config = {
  codec: "avc1.64001E",
  codedWidth:  720,
  codedHeight: 1280,
  description: extraData.buffer,
};

function handleFrame(frame) {
  const canvas = document.getElementById("myCanvas");
  const ctx = canvas.getContext("2d");
  ctx.drawImage(frame, 0, 0);
  frame.close();
}

function readSingleFile(e) {
  var file = e.target.files[0];
  if (!file) {
    return;
  }
  var reader = new FileReader();
  reader.onload = function(e) {
    var contents = e.target.result;
    displayContents(contents);
  };
  reader.readAsText(file);
} //end

function displayContents(contents) {
  var element = document.getElementById('file-content');
  element.textContent = contents;
} // end

const readFile = e => {
  var file = e.target.files[0];
  if (!file) {
    return;
  }

  let reader = new FileReader();

  reader.onload = function(e) {
      let arrayBuffer = new Uint8Array(reader.result);
      console.log(arrayBuffer);

        const decoder = new VideoDecoder(init);
        decoder.configure(config);
        
        const chunk = new EncodedVideoChunk({
          timestamp: 20,
          type: "key",
          data: arrayBuffer,
        });

        decoder.decode(chunk);
  }

  reader.readAsArrayBuffer(file);
} //end

const readFile2 = e => {
  var file = e.target.files[0];
  if (!file) {
    return;
  }

  let reader = new FileReader();

  reader.onload = function(e) {
      extraData = new Uint8Array(reader.result);
      console.log(extraData);
  }

  reader.readAsArrayBuffer(file);
} //end

document.getElementById('file-input')
  .addEventListener('change', readFile, false);

document.getElementById('file-input2')
  .addEventListener('change', readFile2, false);

</script>

file-input2 - is for extradata and file-input - for frame data.

After i run this code in browser i got following error message: ---> Failed to execute 'configure' on 'VideoDecoder': description is detached. Also i had another type of error messages such as something wrong with codec config (codec description)

Please help to find what im doing wrong =) Thanks in advance.

6
  • 1
    (1) Try replacing description: extraData.buffer with just description: extraData since a Uint8Array does not have a .buffer property. (2) Also make sure you save binary data (why use .txt instead of using .h264 to signify to a reader that it's not really a "text" file?). (3) Use a hex editor to check your bytes structure (for next step). Make sure your frame data begins with a start code (not size) and that the SPS and PPS metadata bytes exist in one of your .txt files. Commented Mar 25, 2023 at 0:51
  • 1
    (4) For quicker testing... Make sure your H264 frame is a keyframe data (for immediate picture viewing). Make sure the H264 data is in AnnexB format (for setup simplicity, does not need extraData etc). Then your code should work (you still need to add SPS and PPS once with first keyframe then onwards can just feed in any other frame types to the decoder). PS: See if my Answer about decoding helps you. Commented Mar 25, 2023 at 0:54
  • Thank you so much. Your links helped me a lot. I have dataset of videos for testing. One of them is coded with AV_CODEC_ID_H264 codec (annex b format). Im able to reproduce frames in browser using webcodecs without specifying any codec description. But other videos giving me list of problems with displaying theit frames. I see two solutions: 1) convert each frame to annex b format before sending to browser 2) get extradata of codec in ffmpeg and send it to browser for webcodec descriprion Im new to ffmpeg and reading literature for few days didnt give good results. Commented Mar 28, 2023 at 22:05
  • Can i ask you please point me how to realise both scenarious. The seond one sims to be much simpler and have better performance, but i didnt find any good example or description how to put extradata to webcodec description. Thanks in advance! Commented Mar 28, 2023 at 22:08
  • 1
    today i did some tests and decided to simplify my task. Decided to reject all files encoded not with h264 codec. For annex B transformation i decided to use ffmpeg.exe with command "ffmpeg -i input.mp4 -codec copy -bsf:v h264_mp4toannexb output.ts". Works perfect. Maybe will attach code samples after finish my codeing. Commented Mar 29, 2023 at 16:12

0

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service and acknowledge you have read our privacy policy.

Start asking to get answers

Find the answer to your question by asking.

Ask question

Explore related questions

See similar questions with these tags.