1

I've followed a good tutorial to build up a simple AR video overlay over a series of target images. It works pretty well, however, especially in the first moment the image is recognised for the first time, the augmented object rendering shows some issue, such it's tilting very fast over the target image, instead of being properly anchored.

Here a video which shows the poor performance which is even worse on some actual mobile phone. https://www.youtube.com/watch?v=QZH6eLiFaUs

And here some of the used code, in Kotlin.

This part handles the anchors

import android.util.Pair
import com.google.ar.core.Anchor
import com.google.ar.sceneform.AnchorNode
import com.google.ar.sceneform.Node
import com.google.ar.sceneform.math.Quaternion
import com.google.ar.sceneform.math.Vector3
import com.google.ar.sceneform.rendering.Renderable
import com.shliama.augmentedvideotutorial.VideoScaleType.*
import android.util.Log


class VideoAnchorNode : AnchorNode() {


private val videoNode = Node().also { it.setParent(this) }

override fun setRenderable(renderable: Renderable?) {
    videoNode.renderable = renderable
}

fun setVideoProperties(
    videoWidth: Float, videoHeight: Float, videoRotation: Float,
    imageWidth: Float, imageHeight: Float,
    videoScaleType: VideoScaleType
) {
    Log.i("DEBUGDATA", "videoWidth: " + videoWidth + "/n" + "videoHeight: " + videoHeight + "/n imageWidth")

    scaleNode(videoWidth, videoHeight, imageWidth, imageHeight, videoScaleType)
    rotateNode(videoRotation)
}

private fun scaleNode(
    videoWidth: Float, videoHeight: Float,
    imageWidth: Float, imageHeight: Float,
    videoScaleType: VideoScaleType
) {
    videoNode.localScale = when (videoScaleType) {
        FitXY -> scaleFitXY(imageWidth, imageHeight)
        CenterCrop -> scaleCenterCrop(videoWidth, videoHeight, imageWidth, imageHeight)
        CenterInside -> scaleCenterInside(videoWidth, videoHeight, imageWidth, imageHeight)
    }
}

private fun rotateNode(videoRotation: Float) {
    videoNode.localRotation = Quaternion.axisAngle(Vector3(0.0f, -1.0f, 0.0f), videoRotation)
}

private fun scaleFitXY(imageWidth: Float, imageHeight: Float) = Vector3(imageWidth, 1.0f, imageHeight)

private fun scaleCenterCrop(videoWidth: Float, videoHeight: Float, imageWidth: Float, imageHeight: Float): Vector3 {
    val isVideoVertical = videoHeight > videoWidth
    val videoAspectRatio = if (isVideoVertical) videoHeight / videoWidth else videoWidth / videoHeight
    val imageAspectRatio = if (isVideoVertical) imageHeight / imageWidth else imageWidth / imageHeight

    return if (isVideoVertical) {
        if (videoAspectRatio > imageAspectRatio) {
            Vector3(imageWidth, 1.0f, imageWidth * videoAspectRatio)
        } else {
            Vector3(imageHeight / videoAspectRatio, 1.0f, imageHeight)
        }
    } else {
        if (videoAspectRatio > imageAspectRatio) {
            Vector3(imageHeight * videoAspectRatio, 1.0f, imageHeight)
        } else {
            Vector3(imageWidth, 1.0f, imageWidth / videoAspectRatio)
        }
    }
}

private fun scaleCenterInside(videoWidth: Float, videoHeight: Float, imageWidth: Float, imageHeight: Float): Vector3 {
    val isVideoVertical = videoHeight > videoWidth
    val videoAspectRatio = if (isVideoVertical) videoHeight / videoWidth else videoWidth / videoHeight
    val imageAspectRatio = if (isVideoVertical) imageHeight / imageWidth else imageWidth / imageHeight

    return if (isVideoVertical) {
        if (videoAspectRatio < imageAspectRatio) {
            Vector3(imageWidth, 1.0f, imageWidth * videoAspectRatio)
        } else {
            Vector3(imageHeight / videoAspectRatio, 1.0f, imageHeight)
        }
    } else {
        if (videoAspectRatio < imageAspectRatio) {
            Vector3(imageHeight * videoAspectRatio, 1.0f, imageHeight)
        } else {
            if (videoAspectRatio == 1.0f){
                Vector3(imageWidth, 1.0f, imageWidth)
            }else{
                Vector3(imageWidth, 1.0f, imageWidth / videoAspectRatio)
            }
        }
    }
}

}

This part triggers the rendering, after the target matching with the inner database (.imgdb)

onUpdate

 override fun onUpdate(frameTime: FrameTime) {
    val frame = arSceneView.arFrame ?: return

    val updatedAugmentedImages = frame.getUpdatedTrackables(AugmentedImage::class.java)

    // If current active augmented image isn't tracked anymore and video playback is started - pause video playback
    val nonFullTrackingImages = updatedAugmentedImages.filter { it.trackingMethod != AugmentedImage.TrackingMethod.FULL_TRACKING }
    activeAugmentedImage?.let { activeAugmentedImage ->
        if (isArVideoPlaying() && nonFullTrackingImages.any { it.index == activeAugmentedImage.index }) {
            pauseArVideo()
        }
    }

    val fullTrackingImages = updatedAugmentedImages.filter { it.trackingMethod == AugmentedImage.TrackingMethod.FULL_TRACKING }
    if (fullTrackingImages.isEmpty()) return //here it stops

    // If current active augmented image is tracked but video playback is paused - resume video playback
    activeAugmentedImage?.let { activeAugmentedImage ->
        if (fullTrackingImages.any { it.index == activeAugmentedImage.index }) {
            if (!isArVideoPlaying()) {
                resumeArVideo()
            }
            return
        }
    }

    // Otherwise - make the first tracked image active and start video playback
    fullTrackingImages.firstOrNull()?.let { augmentedImage ->
        try {
            playbackArVideo(augmentedImage)
        } catch (e: Exception) {
            Log.e(TAG, "Could not play video [${augmentedImage.name}]", e)
        }
    }

Here the "playback" part

 private fun playbackArVideo(augmentedImage: AugmentedImage) {
    Log.d(TAG, "playbackVideo = ${augmentedImage.name}")

    requireContext().assets.openFd(augmentedImage.name)
        .use { descriptor ->

            val metadataRetriever = MediaMetadataRetriever()
            metadataRetriever.setDataSource(
                descriptor.fileDescriptor,
                descriptor.startOffset,
                descriptor.length
            )

            val videoWidth = metadataRetriever.extractMetadata(METADATA_KEY_VIDEO_WIDTH).toFloatOrNull() ?: 0f
            val videoHeight = metadataRetriever.extractMetadata(METADATA_KEY_VIDEO_HEIGHT).toFloatOrNull() ?: 0f
            val videoRotation = metadataRetriever.extractMetadata(METADATA_KEY_VIDEO_ROTATION).toFloatOrNull() ?: 0f

            // Account for video rotation, so that scale logic math works properly
            val imageSize = RectF(0f, 0f, augmentedImage.extentX, augmentedImage.extentZ)
                .transform(rotationMatrix(videoRotation))

            val videoScaleType = VideoScaleType.CenterCrop

            videoAnchorNode.setVideoProperties(
                videoWidth = videoWidth, videoHeight = videoHeight, videoRotation = videoRotation,
                imageWidth = imageSize.width(), imageHeight = imageSize.height(),
                videoScaleType = videoScaleType
            )

            // Update the material parameters
            videoRenderable.material.setFloat2(MATERIAL_IMAGE_SIZE, imageSize.width(), imageSize.height())
            videoRenderable.material.setFloat2(MATERIAL_VIDEO_SIZE, videoWidth, videoHeight)
            videoRenderable.material.setBoolean(MATERIAL_VIDEO_CROP, VIDEO_CROP_ENABLED)

            mediaPlayer.reset()
            mediaPlayer.setDataSource(descriptor)
        }.also {
            mediaPlayer.isLooping = true
            mediaPlayer.prepare()
            //mediaPlayer.start()
        }


    videoAnchorNode.anchor?.detach()
    videoAnchorNode.anchor = augmentedImage.createAnchor(augmentedImage.centerPose)

    activeAugmentedImage = augmentedImage

    externalTexture.surfaceTexture.setOnFrameAvailableListener {
        it.setOnFrameAvailableListener(null)
        fadeInVideo()
    }

Here the fade-in part (optional)

       private fun fadeInVideo() {
       ValueAnimator.ofFloat(0f, 1f).apply {
        duration = 400L
        interpolator = LinearInterpolator()
        addUpdateListener { v ->
            videoRenderable.material.setFloat(MATERIAL_VIDEO_ALPHA, v.animatedValue as Float)
        }
        doOnStart { videoAnchorNode.renderable = videoRenderable }
        start()
    }

I figured one possible issue is a fast stop and resume of the video, a behaviour which depends of the target being hardly recognised, maybe. However the image is in HD and the valuation of the target itself scored 100/100... Another possible cause could be the rendering starting too soon compared with the anchors.

In both cases, which could be a good solution?

Thank you in advance

0

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service and acknowledge you have read our privacy policy.