Fragment within CollapsingToolbarLayout with ViewP

I have an activity with a CollapsingToolbarLayout and a TabLayout. When I slide right and left it moves perfectly between fragments. However when I try to scroll down (red arrow in the screenshot) it ignores it. I tried adding a ScrollView to the fragment but it did not make a different. Any ideas why?

BTW - Somehow on the second fragment, a RecycleView, the sliding down works. This is seen on the right Screenshot:

MainActivity's XML:

    <android.support.design.widget.CoordinatorLayout
        android:id="@+id/root_coordinator"
        android:layout_width="match_parent"
        android:layout_height="match_parent">

        <android.support.design.widget.AppBarLayout
            android:id="@+id/app_bar_layout"
            android:layout_width="match_parent"
            android:layout_height="wrap_content">

            <android.support.design.widget.CollapsingToolbarLayout
                android:id="@+id/collapsing_toolbar_layout"
                android:layout_width="match_parent"
                android:layout_height="wrap_content"
                app:contentScrim="?attr/colorPrimary"
                app:layout_scrollFlags="scroll|enterAlways">

                <ImageView
                    android:layout_width="match_parent"
                    android:layout_height="122dp"
                    android:scaleType="centerCrop"
                    android:src="@drawable/rsz_bg_cover"
                    app:layout_collapseMode="parallax" />

                <android.support.v7.widget.Toolbar
                    android:id="@+id/app_bar"
                    android:layout_width="match_parent"
                    android:layout_height="?attr/actionBarSize"
                    android:theme="@style/ThemeOverlay.AppCompat.Dark.ActionBar"
                    app:layout_collapseMode="pin" />

            </android.support.design.widget.CollapsingToolbarLayout>

            <android.support.design.widget.CollapsingToolbarLayout
                android:layout_width="match_parent"
                android:layout_height="wrap_content"
                app:layout_scrollFlags="scroll|enterAlways">

                <android.support.design.widget.TabLayout
                    android:id="@+id/tab_layout"
                    android:layout_width="match_parent"
                    android:layout_height="wrap_content"
                    android:background="@color/colorAccent"
                    app:layout_collapseMode="pin"
                    app:tabIndicatorColor="@color/colorPrimary"
                    app:tabSelectedTextColor="@android:color/white"
                    app:tabTextColor="#EEE" />
            </android.support.design.widget.CollapsingToolbarLayout>

        </android.support.design.widget.AppBarLayout>

        <android.support.v4.view.ViewPager
            android:id="@+id/view_pager"
            android:layout_width="match_parent"
            android:layout_height="match_parent"
            app:layout_behavior="@string/appbar_scrolling_view_behavior" />


    </android.support.design.widget.CoordinatorLayout>

    <android.support.design.widget.NavigationView
        android:id="@+id/navigation_drawer"
        android:layout_width="wrap_content"
        android:layout_height="match_parent"
        android:layout_gravity="start"
        app:menu="@menu/menu_drawer" />
</android.support.v4.widget.DrawerLayout>

Try this layout. I have added TabLayout in AppBarLayout, I believe it should work same as you want. But if you want, you can keep two CollapsingToolbarLayout to achieve your desired behaviour. And make sure fitsSystemWindows should be same either true or false in all layouts else you might not see the expected behaviour.

<?xml version="1.0" encoding="utf-8"?>
<android.support.design.widget.CoordinatorLayout xmlns:android="http://schemas.android.com/apk/res/android"
    xmlns:app="http://schemas.android.com/apk/res-auto"
    android:id="@+id/root_coordinator"
    android:layout_width="match_parent"
    android:layout_height="match_parent"
    android:fitsSystemWindows="true">

    <android.support.design.widget.AppBarLayout
        android:id="@+id/app_bar_layout"
        android:layout_width="match_parent"
        android:layout_height="wrap_content"
        android:fitsSystemWindows="true">

        <android.support.design.widget.CollapsingToolbarLayout
            android:id="@+id/collapsing_toolbar_layout"
            android:layout_width="match_parent"
            android:layout_height="wrap_content"
            app:contentScrim="?attr/colorPrimary"
            android:fitsSystemWindows="true"
            app:layout_scrollFlags="scroll|enterAlways">

            <ImageView
                android:layout_width="match_parent"
                android:layout_height="122dp"
                android:scaleType="centerCrop"
                android:src="@drawable/rsz_bg_cover"
                app:layout_collapseMode="parallax" />

            <android.support.v7.widget.Toolbar
                android:id="@+id/toolbar"
                android:layout_width="match_parent"
                android:layout_height="?attr/actionBarSize"
                android:theme="@style/ThemeOverlay.AppCompat.Dark.ActionBar"
                app:layout_collapseMode="pin" />

        </android.support.design.widget.CollapsingToolbarLayout>

        <android.support.design.widget.TabLayout
            android:id="@+id/tab_layout"
            android:layout_width="match_parent"
            android:layout_height="wrap_content"
            android:background="@color/colorAccent"
            app:layout_collapseMode="pin"
            app:tabIndicatorColor="@color/colorPrimary"
            app:tabSelectedTextColor="@android:color/white"
            app:tabTextColor="#EEE" />
    </android.support.design.widget.AppBarLayout>

    <android.support.v4.view.ViewPager
        android:id="@+id/view_pager"
        android:layout_width="match_parent"
        android:layout_height="match_parent"
        app:layout_behavior="@string/appbar_scrolling_view_behavior" />

</android.support.design.widget.CoordinatorLayout>

And in Fragment layout, Use NestedScrollView and add layout_behavior in it.

<?xml version="1.0" encoding="utf-8"?>
<android.support.v4.widget.NestedScrollView
    xmlns:android="http://schemas.android.com/apk/res/android"
    xmlns:app="http://schemas.android.com/apk/res-auto"
    xmlns:tools="http://schemas.android.com/tools"
    android:layout_width="match_parent"
    android:layout_height="match_parent"
    android:background="@color/white"
    app:layout_behavior="@string/appbar_scrolling_view_behavior">

        // Your Layout

</android.support.v4.widget.NestedScrollView>