0

I'm working on an Android application where I'm implementing live text recognition using CameraX and ML Kit. The recognized text is displayed with bounding boxes on the camera preview, but I'm facing an issue where these bounding boxes are not aligning correctly with the text in the live feed.

Problem Description When running the application:

  • The camera preview displays the live feed correctly.
  • The ML Kit Text Recognition processes the image and identifies text blocks.
  • Bounding boxes are drawn around detected text elements.
  • However, these bounding boxes do not align with the actual text in the camera preview. They appear displaced or incorrectly scaled.

Main Activity Snippets

package com.aviskaarlab.booksnap.ui.views.home

import android.Manifest
import android.content.pm.PackageManager
import android.os.Build
import android.os.Bundle
import android.util.Log
import androidx.appcompat.app.AppCompatActivity
import androidx.camera.core.AspectRatio
import androidx.camera.core.CameraSelector
import androidx.camera.core.ImageAnalysis
import androidx.camera.core.Preview
import androidx.camera.core.resolutionselector.AspectRatioStrategy
import androidx.camera.core.resolutionselector.ResolutionSelector
import androidx.camera.lifecycle.ProcessCameraProvider
import androidx.camera.view.PreviewView
import androidx.core.content.ContextCompat
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors

class MainActivity : AppCompatActivity() {
    private lateinit var viewBinding: ActivityMainBinding
    private lateinit var cameraExecutor: ExecutorService
    private lateinit var textOverlay: TextOverlay
    private lateinit var viewFinder: PreviewView

    override fun onCreate(savedInstanceState: Bundle?) {
        super.onCreate(savedInstanceState)
        viewBinding = ActivityMainBinding.inflate(layoutInflater)
        setContentView(viewBinding.root)

        viewFinder = viewBinding.previewView
        textOverlay = viewBinding.textOverlay

        // Initialize camera and start preview
        startCamera()

        cameraExecutor = Executors.newSingleThreadExecutor()
    }

    private fun startCamera() {
        val cameraProviderFuture = ProcessCameraProvider.getInstance(this)

        cameraProviderFuture.addListener({
            val resolutionSelector = ResolutionSelector.Builder()
                .setAspectRatioStrategy(AspectRatioStrategy.RATIO_4_3_FALLBACK_AUTO_STRATEGY)
                .build()

            val rotation = viewFinder.display.rotation
            val cameraProvider: ProcessCameraProvider = cameraProviderFuture.get()

            // Preview
            val preview = Preview.Builder()
                .setResolutionSelector(resolutionSelector)
                .setTargetRotation(rotation)
                .build()
                .also {
                    it.setSurfaceProvider(viewBinding.previewView.surfaceProvider)
                }

            // Image Analysis
            val imageAnalysis = ImageAnalysis.Builder()
                .setResolutionSelector(resolutionSelector)
                .setTargetRotation(rotation)
                .build()
                .also {
                    it.setAnalyzer(
                        cameraExecutor,
                        BookSnapWordAnalyzer(
                            textOverlay,
                            viewBinding.previewView,
                        )
                    )
                }

            // Select back camera as default
            val cameraSelector = CameraSelector.DEFAULT_BACK_CAMERA

            try {
                cameraProvider.unbindAll()
                cameraProvider.bindToLifecycle(
                    this, cameraSelector, preview, imageAnalysis
                )

                preview.setSurfaceProvider(viewFinder.surfaceProvider)
            } catch (exc: Exception) {
                Log.e(TAG, "Use case binding failed", exc)
            }

        }, ContextCompat.getMainExecutor(this))
    }

    override fun onDestroy() {
        super.onDestroy()
        cameraExecutor.shutdown()
    }

    companion object {
        private const val TAG = "CameraXApp"
    }
}

BookSnapWordAnalyzer Code Snippet

package com.aviskaarlab.booksnap.ui.views.home

import android.graphics.Matrix
import android.graphics.Rect
import android.graphics.RectF
import android.util.Log
import androidx.camera.core.ExperimentalGetImage
import androidx.camera.core.ImageAnalysis
import androidx.camera.core.ImageProxy
import androidx.camera.view.PreviewView
import com.google.mlkit.vision.common.InputImage
import com.google.mlkit.vision.text.Text
import com.google.mlkit.vision.text.TextRecognition
import com.google.mlkit.vision.text.latin.TextRecognizerOptions

internal class BookSnapWordAnalyzer(
    private val overlay: TextOverlay,
    private val previewView: PreviewView,
) : ImageAnalysis.Analyzer {

    companion object {
        private const val TAG = "BookSnapWordAnalyzer"
        private const val WORD_LENGTH = 4
    }

    private val recognizer = TextRecognition.getClient(TextRecognizerOptions.DEFAULT_OPTIONS)
    private lateinit var visionText: Text
    private lateinit var matrix: Matrix
    private var rotationDegrees: Int = 0

    @OptIn(ExperimentalGetImage::class)
    override fun analyze(imageProxy: ImageProxy) {
        val mediaImage = imageProxy.image ?: return
        rotationDegrees = imageProxy.imageInfo.rotationDegrees
        matrix = getCorrectionMatrix(imageProxy, previewView)

        val image =
            InputImage.fromMediaImage(mediaImage, imageProxy.imageInfo.rotationDegrees)

        recognizer.process(image)
            .addOnSuccessListener { visionText ->
                this.visionText = visionText
                val boxes = mutableListOf<CustomRect>()
                for (block in visionText.textBlocks) {
                    for (line in block.lines) {
                        for (element in line.elements) {
                            val elementText = element.text
                            val boundingBox = element.boundingBox
                            if (elementText.length >= WORD_LENGTH && boundingBox != null) {
                                boxes.add(
                                    CustomRect(
                                        adjustBoundingBox(boundingBox, imageProxy, previewView),
                                        elementText
                                    )
                                )
                            }
                        }
                    }
                }
                overlay.updateBoundingBoxes(boxes)
            }
            .addOnFailureListener { e ->
                Log.e(TAG, "Text recognition failed", e)
            }.addOnCompleteListener {
                imageProxy.close()
            }
    }

    private fun adjustBoundingBox(
        rect: Rect,
        imageProxy: ImageProxy,
        previewView: PreviewView
    ): RectF {
        val cropRect = imageProxy.cropRect
        val imageWidth = cropRect.width()
        val imageHeight = cropRect.height()

        val previewWidth = previewView.width
        val previewHeight = previewView.height

        val scaleX = previewWidth.toFloat() / imageWidth
        val scaleY = previewHeight.toFloat() / imageHeight

        val verticalOffset = (previewHeight - imageHeight * scaleY) / 2
        val horizontalOffset = (previewWidth - imageWidth * scaleX) / 2

        val left = rect.left * scaleX + horizontalOffset
        val top = rect.top * scaleY + verticalOffset
        val right = rect.right * scaleX + horizontalOffset
        val bottom = rect.bottom * scaleY + verticalOffset

        return RectF(left, top, right, bottom)
    }

    private fun getCorrectionMatrix(
        imageProxy: ImageProxy,
        previewView: PreviewView,
    ): Matrix {
        val cropRect = imageProxy.cropRect
        val matrix = Matrix()

        val source = floatArrayOf(
            cropRect.left.toFloat(), cropRect.top.toFloat(),
            cropRect.right.toFloat(), cropRect.top.toFloat(),
            cropRect.right.toFloat(), cropRect.bottom.toFloat(),
            cropRect.left.toFloat(), cropRect.bottom.toFloat()
        )

        val destination = floatArrayOf(
            0f, 0f,
            previewView.width.toFloat(), 0f,
            previewView.width.toFloat(), previewView.height.toFloat(),
            0f, previewView.height.toFloat()
        )

        matrix.setPolyToPoly(source, 0, destination, 0, 4)
        return matrix
    }
}

TextOverlay Code Snippet

package com.aviskaarlab.booksnap.ui.views.home

import android.content.Context
import android.graphics.Canvas
import android.graphics.Color
import android.graphics.Paint
import android.graphics.RectF
import android.util.AttributeSet
import android.view.MotionEvent
import android.view.View

class TextOverlay @JvmOverloads constructor(
    context: Context, attrs: AttributeSet? = null, defStyleAttr: Int = 0
) : View(context, attrs, defStyleAttr) {

    private val paint = Paint().apply {
        color = Color.RED
        style = Paint.Style.STROKE
        strokeWidth = 2.0f
    }

    private val boundingBoxes = mutableListOf<CustomRect>()
    private var clickListener: ((String) -> Unit)? = null

    fun setOnRectangleClickListener(listener: (String) -> Unit) {
        clickListener = listener
    }

    fun updateBoundingBoxes(newBoundingBoxes: List<CustomRect>) {
        boundingBoxes.clear()
        boundingBoxes.addAll(newBoundingBoxes)
        invalidate() // Redraw the view
    }

    override fun onDraw(canvas: Canvas) {
        super.onDraw(canvas)
        for (box in boundingBoxes) {
            canvas.drawRect(box.rect, paint)
        }
    }

    override fun onTouchEvent(event: MotionEvent): Boolean {
        if (event.action == MotionEvent.ACTION_UP) {
            val x = event.x
            val y = event.y
            for (box in boundingBoxes) {
                if (box.rect.contains(x, y)) {
                    clickListener?.invoke("Clicked on rectangle ${box.text}")
                    return true
                }
            }
        }
        return true // Event handled
    }
}

Issue The rectangles drawn around recognized text do not align with the text in the camera preview. How can I adjust the bounding box coordinates so that they correctly overlay on the text in the live camera feed?(Please see image attached)

This is the issue, boxes are not aligned with the text

Improve this question
7
  • 1
    CameraX image frames use a physical camera coordinate system which is based on sensor resolution , while ML Kit operate on a normalized coordinate system (typically 0.0 to 1.0 for width and height). This can lead to misalignment if the conversion between these systems isn't handled correctly. Commented Jun 25, 2024 at 9:18
  • 1
    Thanks @BobSmith for this information, Is it possible for you to share some documentation link related to this. I want to go through it, and do these conversions. It'll help me a lot. Commented Jun 26, 2024 at 8:26
  • 1
    @BobSmith Sample code will help a lot, if you can provide. Commented Jun 26, 2024 at 8:53
  • 1
    I will try to provide solution after some time, but that won't be tested build and +1 for your effort. Due to lack of proper documentation by google its always hard time for developers to make it work properly . Commented Jun 26, 2024 at 10:47
  • 1
    Thanks a ton @BobSmith any code sample would be very helpful, And Yes definitely its happening because of the lack of the docs. I'm not able to figure out where should i look for the behaviour of the apis. Commented Jun 26, 2024 at 11:04

3 Answers 3

Reset to default
2
+50

If you are using MLKit and PreviewView, use MLKitAnalyzer to detect image while setting the target coordinate system to COORDINATE_SYSTEM_VIEW_REFERENCED. The output coordinates will be in the PreviewView coordinate system.

Improve this answer
Sign up to request clarification or add additional context in comments.

Comments

1

The CameraX image frames use a physical camera coordinate system which is based on sensor resolution while ML Kit operate on a normalized coordinate system (typically 0.0 to 1.0 for width and height). This can lead to misalignment if the conversion between these systems isn't handled correctly.

So, the difference you are seeing in the bounding boxes in your BookSnapWordAnalyzer is most likely due to coordinate system difference between the ML kit and camera preview.

Things to consider :

  1. Aspect Ratio and Scaling (between camera image and preview)

  2. Transformation Matrix (camera image is rotated)

  3. Bounding Box Adjustment (handing different aspect ratios properly)

  4. Aspect Ratio Consistency (between image analysis and preview)

  5. Transformation Matrix Application (ensure correct alignment)

According to my understanding, I think we need to remove updateBoundingBoxes() and add new function called drawBoundingBoxes() and use it in addOnSuccessListener().

public void drawBoundingBoxes(List<CustomRect> boxes) {
    Canvas canvas = getHolder().lockCanvas(); // Assuming the overlay uses a SurfaceView
    if (canvas != null) {
        for (CustomRect box : boxes) {
            RectF adjustedBox = box.getBoundingBox();
            // Apply the correction matrix to the bounding box coordinates
            matrix.mapRect(adjustedBox);
            // Draw a red rectangle using adjustedBox coordinates and red paint
            paint.setColor(Color.RED);
            canvas.drawRect(adjustedBox, paint);
        }
        getHolder().unlockCanvasAndPost(canvas);
    }
}

The frames processing and the UI updating in order to draw boxes takes too much time Subsequently, try to use a thread for drawing. It can help avoid situations when even navigating within a UI you get lags or the application window freezes.

And call it like :

.addOnSuccessListener { visionText ->
      this.visionText = visionText;
      val boxes = mutableListOf<CustomRect>()
      this.visionText = visionText
        val boxes = mutableListOf<CustomRect>()
        for (block in visionText.textBlocks) {
            for (line in block.lines) {
                for (element in line.elements) {
                    val elementText = element.text
                    val boundingBox = element.boundingBox
                    if (elementText.length >= WORD_LENGTH && boundingBox != null) {
                        boxes.add(
                            CustomRect(
                                adjustBoundingBox(boundingBox, imageProxy, previewView),
                                elementText
                            )
                        )
                    }
                }
            }
        }

      val handler = Handler(Looper.getMainLooper())
      val runnable = Runnable {
          overlay.drawBoundingBoxes(boxes);
      }
      handler.post(runnable)
  }

Notes :

Aligning the bounding boxes accurately requires careful handling of aspect ratios and transformations.

Reference links :

Improve this answer

Comments

1

After going through different trials and errors, Finally this helped me to come to a proper code solution. I'm posting my modified answer, may be it could help someone in future.

Sometimes Keeping it simple is the best way

private fun startCamera() {
        val previewView: PreviewView = viewBinding.previewView
        val cameraController = LifecycleCameraController(baseContext)
        cameraController.bindToLifecycle(this)
        cameraController.cameraSelector = CameraSelector.DEFAULT_BACK_CAMERA
        previewView.controller = cameraController

        val textRecognizer = TextRecognition.getClient(TextRecognizerOptions.DEFAULT_OPTIONS)

        cameraController.setImageAnalysisAnalyzer(
            ContextCompat.getMainExecutor(this),
            MlKitAnalyzer(
                listOf(textRecognizer),
                COORDINATE_SYSTEM_VIEW_REFERENCED,
                ContextCompat.getMainExecutor(this)
            ) { result: MlKitAnalyzer.Result? ->
                // The value of result.getResult can be used directly for drawing UI overlay.
                val visionText = result?.getValue(textRecognizer)
                if (visionText != null) {
                    // Process the texts in your desired method
                    processText(visionText)
                }
            })

    }
Improve this answer

Comments

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service and acknowledge you have read our privacy policy.

Start asking to get answers

Find the answer to your question by asking.

Ask question

Explore related questions

See similar questions with these tags.