How to have similar mechanism of center-crop on Ex

Question

Background

We record a video of the user's face, and usually the face is located at the upper half of the video.

Later we wish to view the video, but the aspect ratio of the PlayerView might be different than the one of the video, so there needs to be some scaling and cropping.

The problem

The only way I've found to scale the PlayerView so that it will be shown in the entire space it has but keeping the aspect ratio (which will result in cropping, of course) , is by using app:resize_mode="zoom" . But this is only for the center, meaning it takes a point of 0.5x0.5 of the video, and scale-crops from that point. This causes many cases of losing the important content of the video.

For example, if we have a video that was taken in portrait, and we have a square PlayerView, this is the part that will be visible:

What I've tried

I've tried searching over the Internet, StackOverflow (here) and on Github, but I couldn't find how to do it. The only clue I've found is about AspectRatioFrameLayout and AspectRatioTextureView, but I didn't find how to use them for this task, if it's even possible.

I was told (here) that I should use a normal TextureView , and provide it directly to SimpleExoPlayer using SimpleExoPlayer.setVideoTextureView. And to set a special transformation to it using TextureView.setTransform.

After a lot of trying what is best to use (and looking at this Github repo and this one too), I've published a working sample that seems to show the video correctly, but I'm still not sure about it, as I also use an ImageView that's shown on top of it before it starts playing (to have a nicer transition instead of black content).

Here's the current code:

class MainActivity : AppCompatActivity() {
    private val imageResId = R.drawable.test
    private val videoResId = R.raw.test
    private val percentageY = 0.2f
    private var player: SimpleExoPlayer? = null


    override fun onCreate(savedInstanceState: Bundle?) {
        window.setBackgroundDrawable(ColorDrawable(0xff000000.toInt()))
        super.onCreate(savedInstanceState)
        if (cache == null) {
            cache = SimpleCache(File(cacheDir, "media"), LeastRecentlyUsedCacheEvictor(MAX_PREVIEW_CACHE_SIZE_IN_BYTES))
        }
        setContentView(R.layout.activity_main)
//        imageView.visibility = View.INVISIBLE
        imageView.setImageResource(imageResId)
        imageView.doOnPreDraw {
            imageView.imageMatrix = prepareMatrixForImageView(imageView, imageView.drawable.intrinsicWidth.toFloat(), imageView.drawable.intrinsicHeight.toFloat())
//            imageView.imageMatrix = prepareMatrix(imageView, imageView.drawable.intrinsicWidth.toFloat(), imageView.drawable.intrinsicHeight.toFloat())
//            imageView.visibility = View.VISIBLE
        }
    }

    override fun onStart() {
        super.onStart()
        playVideo()
    }

    private fun prepareMatrix(view: View, contentWidth: Float, contentHeight: Float): Matrix {
        var scaleX = 1.0f
        var scaleY = 1.0f
        val viewWidth = view.measuredWidth.toFloat()
        val viewHeight = view.measuredHeight.toFloat()
        Log.d("AppLog", "viewWidth $viewWidth viewHeight $viewHeight contentWidth:$contentWidth contentHeight:$contentHeight")
        if (contentWidth > viewWidth && contentHeight > viewHeight) {
            scaleX = contentWidth / viewWidth
            scaleY = contentHeight / viewHeight
        } else if (contentWidth < viewWidth && contentHeight < viewHeight) {
            scaleY = viewWidth / contentWidth
            scaleX = viewHeight / contentHeight
        } else if (viewWidth > contentWidth)
            scaleY = viewWidth / contentWidth / (viewHeight / contentHeight)
        else if (viewHeight > contentHeight)
            scaleX = viewHeight / contentHeight / (viewWidth / contentWidth)
        val matrix = Matrix()
        val pivotPercentageX = 0.5f
        val pivotPercentageY = percentageY

        matrix.setScale(scaleX, scaleY, viewWidth * pivotPercentageX, viewHeight * pivotPercentageY)
        return matrix
    }

    private fun prepareMatrixForVideo(view: View, contentWidth: Float, contentHeight: Float): Matrix {
        val msWidth = view.measuredWidth
        val msHeight = view.measuredHeight
        val matrix = Matrix()
        matrix.setScale(1f, (contentHeight / contentWidth) * (msWidth.toFloat() / msHeight), msWidth / 2f, percentageY * msHeight) /*,msWidth/2f,msHeight/2f*/
        return matrix
    }

    private fun prepareMatrixForImageView(view: View, contentWidth: Float, contentHeight: Float): Matrix {
        val dw = contentWidth
        val dh = contentHeight
        val msWidth = view.measuredWidth
        val msHeight = view.measuredHeight
//        Log.d("AppLog", "viewWidth $msWidth viewHeight $msHeight contentWidth:$contentWidth contentHeight:$contentHeight")
        val scalew = msWidth.toFloat() / dw
        val theoryh = (dh * scalew).toInt()
        val scaleh = msHeight.toFloat() / dh
        val theoryw = (dw * scaleh).toInt()
        val scale: Float
        var dx = 0
        var dy = 0
        if (scalew > scaleh) { // fit width
            scale = scalew
//            dy = ((msHeight - theoryh) * 0.0f + 0.5f).toInt() // + 0.5f for rounding
        } else {
            scale = scaleh
            dx = ((msWidth - theoryw) * 0.5f + 0.5f).toInt() // + 0.5f for rounding
        }
        dy = ((msHeight - theoryh) * percentageY + 0.5f).toInt() // + 0.5f for rounding
        val matrix = Matrix()
//        Log.d("AppLog", "scale:$scale dx:$dx dy:$dy")
        matrix.setScale(scale, scale)
        matrix.postTranslate(dx.toFloat(), dy.toFloat())
        return matrix
    }

    private fun playVideo() {
        player = ExoPlayerFactory.newSimpleInstance(this@MainActivity, DefaultTrackSelector())
        player!!.setVideoTextureView(textureView)
        player!!.addVideoListener(object : VideoListener {
            override fun onVideoSizeChanged(width: Int, height: Int, unappliedRotationDegrees: Int, pixelWidthHeightRatio: Float) {
                super.onVideoSizeChanged(width, height, unappliedRotationDegrees, pixelWidthHeightRatio)
                Log.d("AppLog", "onVideoSizeChanged: $width $height")
                val videoWidth = if (unappliedRotationDegrees % 180 == 0) width else height
                val videoHeight = if (unappliedRotationDegrees % 180 == 0) height else width
                val matrix = prepareMatrixForVideo(textureView, videoWidth.toFloat(), videoHeight.toFloat())
                textureView.setTransform(matrix)
            }

            override fun onRenderedFirstFrame() {
                Log.d("AppLog", "onRenderedFirstFrame")
                player!!.removeVideoListener(this)
//                imageView.animate().alpha(0f).setDuration(5000).start()
                imageView.visibility = View.INVISIBLE
            }
        })
        player!!.volume = 0f
        player!!.repeatMode = Player.REPEAT_MODE_ALL
        player!!.playRawVideo(this, videoResId)
        player!!.playWhenReady = true
        //        player!!.playVideoFromUrl(this, "https://sample-videos.com/video123/mkv/240/big_buck_bunny_240p_20mb.mkv", cache!!)
        //        player!!.playVideoFromUrl(this, "https://sample-videos.com/video123/mkv/720/big_buck_bunny_720p_1mb.mkv", cache!!)
        //        player!!.playVideoFromUrl(this@MainActivity, "https://sample-videos.com/video123/mkv/720/big_buck_bunny_720p_1mb.mkv")
    }

    override fun onStop() {
        super.onStop()
        player!!.setVideoTextureView(null)
        //        playerView.player = null
        player!!.release()
        player = null
    }

    companion object {
        const val MAX_PREVIEW_CACHE_SIZE_IN_BYTES = 20L * 1024L * 1024L
        var cache: com.google.android.exoplayer2.upstream.cache.Cache? = null

        @JvmStatic
        fun getUserAgent(context: Context): String {
            val packageManager = context.packageManager
            val info = packageManager.getPackageInfo(context.packageName, 0)
            val appName = info.applicationInfo.loadLabel(packageManager).toString()
            return Util.getUserAgent(context, appName)
        }
    }

    fun SimpleExoPlayer.playRawVideo(context: Context, @RawRes rawVideoRes: Int) {
        val dataSpec = DataSpec(RawResourceDataSource.buildRawResourceUri(rawVideoRes))
        val rawResourceDataSource = RawResourceDataSource(context)
        rawResourceDataSource.open(dataSpec)
        val factory: DataSource.Factory = DataSource.Factory { rawResourceDataSource }
        prepare(LoopingMediaSource(ExtractorMediaSource.Factory(factory).createMediaSource(rawResourceDataSource.uri)))
    }

    fun SimpleExoPlayer.playVideoFromUrl(context: Context, url: String, cache: Cache? = null) = playVideoFromUri(context, Uri.parse(url), cache)

    fun SimpleExoPlayer.playVideoFile(context: Context, file: File) = playVideoFromUri(context, Uri.fromFile(file))

    fun SimpleExoPlayer.playVideoFromUri(context: Context, uri: Uri, cache: Cache? = null) {
        val factory = if (cache != null)
            CacheDataSourceFactory(cache, DefaultHttpDataSourceFactory(getUserAgent(context)))
        else
            DefaultDataSourceFactory(context, MainActivity.getUserAgent(context))
        val mediaSource = ExtractorMediaSource.Factory(factory).createMediaSource(uri)
        prepare(mediaSource)
    }
}

I had various issues on trying this till I got to the current situation, and I've updated this question multiple times accordingly. Now it even works with the percentageY I talked about, so I could set it to be from 20% of the top of the video, if I wish. However, I still think that it has a big chance that something is wrong, because when I tried to set it to 50% , I've noticed that the content might not fit the entire View.

I even looked at the source code of ImageView (here), to see how center-crop is used. When applied to the ImageView, it still worked as center-crop, but when I used the same technique on the video, it gave me a very wrong result.

The questions

My goal here was to show both ImageView and the video so that it will smoothly transition from a static image to a video. All that while having both have the top-scale-crop of 20% from the top (for example). I've published a sample project here to try it out and share people of what I've found.

So now my questions are around why this doesn't seem to work well for the imageView and/or video :

1. As it turns out, none of the matrix creations that I've tried work well for either ImageView or the video. What's wrong with it exactly? How can I change it for them to look the same? To scale-crop from the top 20%, for example?

2. I tried to use the exact matrix for both, but it seems each need it differently, even though both have the exact same size and content size. Why would I need a different matrix for each?

Answer 1

Rewrite

This is a demo of the effect that is desired. The demo has an ImageView overlaying a TextureVieweach embedded in a CardView. The CardView for the ImageView overlays the CardView for the TextureView. When the image view is clicked, the image view is made GONE and the video starts.

This is not exactly the implementation you are looking for, but it does demonstrate appropriate scaling and cropping. See the code in the main activity for the matrix operations performed on the TextureView and the ImageView. This demo also takes into account vertical translation of the image and video.

The still image and video were taken as-is from https://coverr.co/tags/vertical

MainActivity.kt

class MainActivity : AppCompatActivity() {
    private var player: SimpleExoPlayer? = null
    val additionalTranslationY = -300f
    // Still and video from https://coverr.co/videos/Maple%20Wing
    val stillResourceId = R.drawable.maple_wing
    val videoResourceId = R.raw.maple_wing

    override fun onCreate(savedInstanceState: Bundle?) {
        window.setBackgroundDrawable(ColorDrawable(0xff000000.toInt()))
        super.onCreate(savedInstanceState)
        if (cache == null) {
            cache = SimpleCache(File(cacheDir, "media"), LeastRecentlyUsedCacheEvictor(MAX_PREVIEW_CACHE_SIZE_IN_BYTES))
        }
        setContentView(R.layout.activity_main)

        preview.post {
            preview.scaleType = ImageView.ScaleType.MATRIX
            val matrix = Matrix()
            val imageWidth: Float = ContextCompat.getDrawable(this, stillResourceId)!!.intrinsicWidth.toFloat()
            val viewWidth: Float = preview.width.toFloat()
            val scaleFactor = viewWidth / imageWidth
            matrix.postScale(scaleFactor, scaleFactor, 0f, 0f)
            matrix.postTranslate(0f, additionalTranslationY)
            preview.imageMatrix = matrix
            preview.setImageResource(stillResourceId)
        }
    }

    fun playTheVideo(view: View) {
        playVideo()
    }

    private fun playVideo() {
        player = ExoPlayerFactory.newSimpleInstance(this@MainActivity, DefaultTrackSelector())
        player!!.setVideoTextureView(textureView)
        player!!.addVideoListener(object : VideoListener {
            override fun onVideoSizeChanged(width: Int, height: Int, unappliedRotationDegrees: Int, pixelWidthHeightRatio: Float) {
                super.onVideoSizeChanged(width, height, unappliedRotationDegrees, pixelWidthHeightRatio)
                val matrix = Matrix()
                val viewWidth = textureView.width.toFloat()
                val viewHeight = textureView.height.toFloat()
                val videoWidth = width.toFloat()
                val videoHeight = height.toFloat()
                val scaleX = 1.0f
                val scaleY: Float = (viewWidth / viewHeight) * (videoHeight / videoWidth)
                matrix.postScale(scaleX, scaleY, 0f, 0f)
                matrix.postTranslate(0f, additionalTranslationY)
                textureView.setTransform(matrix)
            }

            override fun onRenderedFirstFrame() {
                player!!.removeVideoListener(this)
                imageCard.visibility = View.GONE
            }
        })
        player!!.volume = 0f
        player!!.repeatMode = Player.REPEAT_MODE_ALL
        player!!.playRawVideo(this, videoResourceId)
        player!!.playWhenReady = true
    }

    override fun onStop() {
        super.onStop()
        player!!.setVideoTextureView(null)
        player!!.release()
        player = null
    }

    companion object {
        const val MAX_PREVIEW_CACHE_SIZE_IN_BYTES = 20L * 1024L * 1024L
        var cache: com.google.android.exoplayer2.upstream.cache.Cache? = null

        @JvmStatic
        fun getUserAgent(context: Context): String {
            val packageManager = context.packageManager
            val info = packageManager.getPackageInfo(context.packageName, 0)
            val appName = info.applicationInfo.loadLabel(packageManager).toString()
            return Util.getUserAgent(context, appName)
        }
    }

    fun SimpleExoPlayer.playRawVideo(context: Context, @RawRes rawVideoRes: Int) {
        val dataSpec = DataSpec(RawResourceDataSource.buildRawResourceUri(rawVideoRes))
        val rawResourceDataSource = RawResourceDataSource(context)
        rawResourceDataSource.open(dataSpec)
        val factory: DataSource.Factory = DataSource.Factory { rawResourceDataSource }
        prepare(LoopingMediaSource(ExtractorMediaSource.Factory(factory).createMediaSource(rawResourceDataSource.uri)))
    }
}

actvity_main.xml

<androidx.constraintlayout.widget.ConstraintLayout 
    android:id="@+id/noClipsView"
    android:layout_width="match_parent"
    android:layout_height="match_parent"
    android:background="#fff"
    tools:context=".MainActivity">

    <androidx.cardview.widget.CardView
        android:id="@+id/videoCard"
        android:layout_width="@dimen/card_width"
        android:layout_height="@dimen/card_height"
        app:cardCornerRadius="16dp"
        app:layout_constraintBottom_toBottomOf="parent"
        app:layout_constraintEnd_toEndOf="parent"
        app:layout_constraintStart_toStartOf="parent"
        app:layout_constraintTop_toTopOf="parent">

        <TextureView
            android:id="@+id/textureView"
            android:layout_width="match_parent"
            android:layout_height="match_parent"
            tools:foreground="#33ff0000" />
    </androidx.cardview.widget.CardView>

    <androidx.cardview.widget.CardView
        android:id="@+id/imageCard"
        android:layout_width="0dp"
        android:layout_height="0dp"
        app:cardCornerRadius="16dp"
        app:layout_constraintBottom_toBottomOf="@id/videoCard"
        app:layout_constraintEnd_toEndOf="@id/videoCard"
        app:layout_constraintStart_toStartOf="@id/videoCard"
        app:layout_constraintTop_toTopOf="@id/videoCard">

        <ImageView
            android:id="@+id/preview"
            android:layout_width="match_parent"
            android:layout_height="match_parent"
            android:onClick="playTheVideo"
            android:scaleType="matrix" />
    </androidx.cardview.widget.CardView>
</androidx.constraintlayout.widget.ConstraintLayout>

dimens.xml

<resources>  
    <dimen name="card_width">200dp</dimen>  
    <dimen name="card_height">200dp</dimen>  
</resources>

Working off the sample project, try the following. If shiftUpPercentage is set to zero, it should work like center crop.

MainActivity.kt

class MainActivity : AppCompatActivity() {
    private val imageResId = R.drawable.test
    private val videoResId = R.raw.test
    private val percentageY = 0.2f
    private var player: SimpleExoPlayer? = null


    override fun onCreate(savedInstanceState: Bundle?) {
        window.setBackgroundDrawable(ColorDrawable(0xff000000.toInt()))
        super.onCreate(savedInstanceState)
        if (cache == null) {
            cache = SimpleCache(File(cacheDir, "media"), LeastRecentlyUsedCacheEvictor(MAX_PREVIEW_CACHE_SIZE_IN_BYTES))
        }
        setContentView(R.layout.activity_main)
//        imageView.visibility = View.INVISIBLE
        imageView.setImageResource(imageResId)
        imageView.doOnPreDraw {
            imageView.scaleType = ImageView.ScaleType.MATRIX
            val imageWidth: Float = ContextCompat.getDrawable(this, imageResId)!!.intrinsicWidth.toFloat()
            val imageHeight: Float = ContextCompat.getDrawable(this, imageResId)!!.intrinsicHeight.toFloat()
            imageView.imageMatrix = prepareMatrix(imageView, imageWidth, imageHeight, 0.2f, false)
            imageView.setImageResource(imageResId)
        }
    }

    override fun onStart() {
        super.onStart()
        playVideo()
    }

    private fun prepareMatrix(targetView: View, mediaWidth: Float, mediaHeight: Float,
                              shiftUpPercentage: Float, isVideo: Boolean): Matrix {
        val matrix = Matrix()
        val viewWidth = targetView.width - targetView.paddingStart - targetView.paddingEnd
        val viewHeight = targetView.height - targetView.paddingTop - targetView.paddingBottom
        val translateY = -viewHeight * shiftUpPercentage
        val scaleFactorX = (1 + shiftUpPercentage) * viewWidth / mediaWidth
        val scaleFactorY = (1 + shiftUpPercentage) * viewHeight / mediaHeight
        if (isVideo) {
            matrix.setScale(mediaWidth / viewWidth, mediaHeight / viewHeight)
        }
        if (scaleFactorY > scaleFactorX) {
            // Height will fit
            matrix.postScale(scaleFactorY, scaleFactorY)
            matrix.postTranslate(-(mediaWidth * scaleFactorY - viewWidth) / 2, translateY)
        } else {
            // Width will fit
            matrix.postScale(viewWidth / mediaWidth, viewWidth / mediaWidth)
            matrix.postTranslate(0f, translateY)
        }
        return matrix
    }

    private fun playVideo() {
        player = ExoPlayerFactory.newSimpleInstance(this@MainActivity, DefaultTrackSelector())
        player!!.setVideoTextureView(textureView)
        player!!.addVideoListener(object : VideoListener {
            override fun onVideoSizeChanged(width: Int, height: Int, unappliedRotationDegrees: Int, pixelWidthHeightRatio: Float) {
                super.onVideoSizeChanged(width, height, unappliedRotationDegrees, pixelWidthHeightRatio)
                textureView.setTransform(prepareMatrix(textureView, width.toFloat(), height.toFloat(), 0.2f, true))
            }

            override fun onRenderedFirstFrame() {
                Log.d("AppLog", "onRenderedFirstFrame")
                player!!.removeVideoListener(this)
//                imageView.animate().alpha(0f).setDuration(5000).start()
                imageView.visibility = View.INVISIBLE
            }
        })
        player!!.volume = 0f
        player!!.repeatMode = Player.REPEAT_MODE_ALL
        player!!.playRawVideo(this, videoResId)
        player!!.playWhenReady = true
        //        player!!.playVideoFromUrl(this, "https://sample-videos.com/video123/mkv/240/big_buck_bunny_240p_20mb.mkv", cache!!)
        //        player!!.playVideoFromUrl(this, "https://sample-videos.com/video123/mkv/720/big_buck_bunny_720p_1mb.mkv", cache!!)
        //        player!!.playVideoFromUrl(this@MainActivity, "https://sample-videos.com/video123/mkv/720/big_buck_bunny_720p_1mb.mkv")
    }

    override fun onStop() {
        super.onStop()
        player!!.setVideoTextureView(null)
        //        playerView.player = null
        player!!.release()
        player = null
    }

    companion object {
        const val MAX_PREVIEW_CACHE_SIZE_IN_BYTES = 20L * 1024L * 1024L
        var cache: com.google.android.exoplayer2.upstream.cache.Cache? = null

        @JvmStatic
        fun getUserAgent(context: Context): String {
            val packageManager = context.packageManager
            val info = packageManager.getPackageInfo(context.packageName, 0)
            val appName = info.applicationInfo.loadLabel(packageManager).toString()
            return Util.getUserAgent(context, appName)
        }
    }

    fun SimpleExoPlayer.playRawVideo(context: Context, @RawRes rawVideoRes: Int) {
        val dataSpec = DataSpec(RawResourceDataSource.buildRawResourceUri(rawVideoRes))
        val rawResourceDataSource = RawResourceDataSource(context)
        rawResourceDataSource.open(dataSpec)
        val factory: DataSource.Factory = DataSource.Factory { rawResourceDataSource }
        prepare(LoopingMediaSource(ExtractorMediaSource.Factory(factory).createMediaSource(rawResourceDataSource.uri)))
    }

    fun SimpleExoPlayer.playVideoFromUrl(context: Context, url: String, cache: Cache? = null) = playVideoFromUri(context, Uri.parse(url), cache)

    fun SimpleExoPlayer.playVideoFile(context: Context, file: File) = playVideoFromUri(context, Uri.fromFile(file))

    fun SimpleExoPlayer.playVideoFromUri(context: Context, uri: Uri, cache: Cache? = null) {
        val factory = if (cache != null)
            CacheDataSourceFactory(cache, DefaultHttpDataSourceFactory(getUserAgent(context)))
        else
            DefaultDataSourceFactory(context, MainActivity.getUserAgent(context))
        val mediaSource = ExtractorMediaSource.Factory(factory).createMediaSource(uri)
        prepare(mediaSource)
    }
}