Enable `--multi-dex` option in ant for Android

Question

It's easy to enable multi-dex option for gradle build system, but I haven't found example how I can enable this option for ant building. How can archive this?

Answer 1

I also added multidex to the ANT build, but in a slightly different fashion that is more flexible in the classes*.dex support.

In any case, I did this in two phases: 1) get DX to output multidex then 2) use aapt to include the multidex classes. There is an optional step for producing the main classes list at the end of this posting but it is not necessary for the basic implementation.

Here's the implementation in build.xml, with comments to follow:

... your build script ...
<!-- version-tag: custom -->
<!-- (1) Override -package target to add additional JAR to the apk -->
<property name="multidex-secondary-classes.jar" value="classes-secondary.jar" />
<target name="-package" depends="-dex, -package-resources, -create-multidex-secondary-classes-jar">
    <!-- Copied from SDK/tools/ant/build.xml -->
    <!-- only package apk if *not* a library project -->
    <do-only-if-not-library elseText="Library project: do not package apk..." >
        <if condition="${build.is.instrumented}">
            <then>
                <package-helper>
                    <extra-jars>
                        <!-- Injected from external file -->
                        <jarfile path="${emma.dir}/emma_device.jar" />
                    </extra-jars>
                </package-helper>
            </then>
            <else>
                <!-- We can finesse apkbuilder by putting secondary classes file(s) in a jar file -->
                <if condition="${build.is.multidex}">
                    <then>
                        <package-helper>
                            <extra-jars>
                                <jarfile path="${out.dir}/${multidex-secondary-classes.jar}" />
                            </extra-jars>
                        </package-helper>
                    </then>
                    <else>
                        <package-helper>
                            <extra-jars />
                        </package-helper>
                    </else>
                 </if>
            </else>
        </if>
    </do-only-if-not-library>
</target>

<!-- (2) Create a JAR file of the secondary classes*.dex files -->
<target name="-create-multidex-secondary-classes-jar" if="${build.is.multidex}">
    <jar destfile="${out.dir}/${multidex-secondary-classes.jar}"
         basedir="${out.dir}"
         includes="classes*.dex"
         excludes="classes.dex"
         filesonly="true"
         />
</target>

<!-- Standard import of Android build.xml -->
<import file="${sdk.dir}/tools/ant/build.xml" />

<!-- (3) Replacement of "dex-helper" to support multidex -->
<macrodef name="dex-helper">
    <element name="external-libs" optional="yes" />
    <attribute name="nolocals" default="false" />
    <sequential>
        <!-- sets the primary input for dex. If a pre-dex task sets it to
             something else this has no effect -->
        <property name="out.dex.input.absolute.dir" value="${out.classes.absolute.dir}" />

        <!-- set the secondary dx input: the project (and library) jar files
             If a pre-dex task sets it to something else this has no effect -->
        <if>
            <condition>
                <isreference refid="out.dex.jar.input.ref" />
            </condition>
            <else>
                <path id="out.dex.jar.input.ref">
                    <path refid="project.all.jars.path" />
                </path>
            </else>
        </if>

        <if condition="${build.is.multidex}" >
            <then>
                <if condition="${dex.force.jumbo}" >
                    <else>
                        <fail message="The following assumes dex.force.jumbo is true" />
                    </else>
                </if>
                <apply executable="${dx}" failonerror="true" parallel="true">
                    <arg value="--dex" />
                    <arg value="--force-jumbo" />

                    <!-- Specify a multi-dex APK -->
                    <arg value="--multi-dex" />

                    <!-- For multidex output to a folder -->
                    <arg value="--output" />
                    <arg value="${out.dir}" />

                    <path path="${out.dex.input.absolute.dir}" />
                </apply>
            </then>
            <else>
                <!-- The value from SDK/tools/ant/build.xml -->
                <dex executable="${dx}"
                        output="${intermediate.dex.file}"
                        dexedlibs="${out.dexed.absolute.dir}"
                        nolocals="@{nolocals}"
                        forceJumbo="${dex.force.jumbo}"
                        disableDexMerger="${dex.disable.merger}"
                        verbose="${verbose}">
                    <path path="${out.dex.input.absolute.dir}"/>
                    <path refid="out.dex.jar.input.ref" />
                    <external-libs />
                </dex>
            </else>
        </if>
    </sequential>
</macrodef>

Modification (1) replaces the -package target to allow extra jars to be passed to ApkBuilder. It turns out ApkBuilder just copies the contents of the JAR files into the APK, so if we put our classes[1-N].dex into a JAR we can get ApkBuilder to package those extra jars into the APK.

(2) Builds that extra JAR file with all classes*.dex except classes.dex, so supports any number of extra classes.dex files

(3) Is a work around for that fact that the "dex" AntTask does not support any way to pass --multi-dex to "dx.bat", which knows how to use it. I looked at what dx.bat was invoking and added that directly here (Amended: michaelbrz has a better implementation in his script so I borrowed it. Thank you)

BTW, we always run Proguard (either obfuscated or unobfuscated) to get class and method shrinkage. This is handy for generating a "main class" list. In our case, we added Google Play Services and that blew our DEX file method limit, so decided to put those classes and methods in the secondary dex. Generating that class list from the Proguard output is a simple matter of filtering dump.txt:

<property name="multidex-main-dex-list.file"
          value="bin/multidex-main-dex-list.txt" />

<target name="-create-multidex-main-list" if="${build.is.multidex}">
    <delete file="${multidex-main-dex-list.file}" />
    <copy file="${out.dir}/proguard/dump.txt"
          tofile="${multidex-main-dex-list.file}" >
        <filterchain>

            <!-- Convert classes to the right format -->
            <tokenfilter>
                <containsregex
                    pattern="^..Program class..(.*)"
                    replace="\1.class"/>
            </tokenfilter>

            <!-- Exclude Google Play Services -->
            <linecontains negate="true">
                <contains value="com/google/android/gms/"/>
            </linecontains>

        </filterchain>
    </copy>
</target>

Answer 2

We have 2 options:

1. Change DexExecTask [introduce new parameter for multi dex], compile ant.jar, use this jar for building. I don't like this option, coz we have to provide updated ant.jar for all team members.
2. Modify project build.xml file. I have found awesome ant build file, that has all modifications for support multi-dex: https://github.com/ruboto/ruboto-irb/blob/master/build.xml

Hope, that it helps.

Answer 3

I introduced multi-dex support in application using ant based on Ruboto's build.xml (kudos to them!). So that's the path I would recommend to take. In that case you don't have to create custom ant jars, just edit your build.xml and apply all non-gradle steps from https://developer.android.com/tools/building/multidex.html

As for build.xml you would have to take a look if all paths play well with each other, you might have changed some of them and need to adjust ant steps.

In my case I had to also generate list of classes that must be placed in first generated dex (default one), so the app could even start as it loads additional dexes in Application's onCreate. In order to do that you need to run script mainDexClasses, which you will find in your build-tools:

mainDexClasses [--output <output file>] <application path>

Alex Lipov wrote nice post about it.

When you have list of classes generated, you have to point to this list in dx's parameter by adding

<arg value="--main-dex-list="class_list_path" />

where dx is executed.

It's worth remembering that Ruboto's script assumes that only one additional dex will be created, which is not always case. I had two additional dexes generated so I implemented adding if exists:

            <if>
                <condition>
                    <available file="${third_dex_path}"/>
                </condition>
                <then>
                    <exec executable="${aapt}" dir="${out.absolute.dir}" failonerror="true">
                        <arg line='add -v "${out.absolute.dir}/${resource.package.file.name}" ${third_dex}'/>
                    </exec> 
                </then>
            </if>

And multidex support in Ant is on! It's not as easy as with gradle, but it's possible.

UPDATE: my build.xml (apart from not answer-related parts):

<!-- Packages the application. Overriden in order to add  "-post-package-resources"" dependency-->
    <target name="-package" depends="-dex, -package-resources, -post-package-resources">
        <!-- only package apk if *not* a library project -->
        <do-only-if-not-library elseText="Library project: do not package apk..." >
            <if condition="${build.is.instrumented}">
                <then>
                    <package-helper>
                        <extra-jars>
                            <!-- Injected from external file -->
                            <jarfile path="${emma.dir}/emma_device.jar" />
                        </extra-jars>
                    </package-helper>
                </then>
                <else>
                    <package-helper />
                </else>
            </if>
        </do-only-if-not-library>
    </target>

    <target name="-post-package-resources">
        <property name="second_dex" value="classes2.dex" />
        <property name="third_dex" value="classes3.dex" />
        <property name="second_dex_path" value="${out.absolute.dir}/${second_dex}" />
        <if>
            <condition>
              <and>
                <available file="${second_dex_path}" />
                <or>
                  <not>
                    <uptodate srcfile="${second_dex_path}" targetfile="${out.absolute.dir}/${resource.package.file.name}" />
                  </not>
                  <uptodate srcfile="${out.absolute.dir}/${resource.package.file.name}" targetfile="${out.absolute.dir}/${resource.package.file.name}.d" />
                </or>
              </and>
            </condition>
            <then>
                <echo>Adding classes2.dex to ${resource.package.file.name}</echo>
                <exec executable="${aapt}" dir="${out.absolute.dir}" failonerror="true">
                    <arg line='add -v "${out.absolute.dir}/${resource.package.file.name}" ${second_dex}'/>
                </exec>
                <if>
                    <condition>
                        <available file="${out.absolute.dir}/classes3.dex"/>
                    </condition>
                    <then>
                        <echo>Adding classes3.dex to ${resource.package.file.name}</echo>
                        <exec executable="${aapt}" dir="${out.absolute.dir}" failonerror="true">
                            <arg line='add -v "${out.absolute.dir}/${resource.package.file.name}" ${third_dex}'/>
                        </exec> 
                    </then>
                </if>
            </then>
        </if>
    </target> 

<!-- builds dex in regular way and if it fails, switches to multidex-->
    <macrodef name="dex-helper">
        <element name="external-libs" optional="yes" />
        <attribute name="nolocals" default="false" />
        <sequential>
            <condition property="verbose.option" value="--verbose" else="">
                <istrue value="${verbose}" />
            </condition>
            <condition property="jumbo.option" value="--force-jumbo" else="">
                <istrue value="${dex.force.jumbo}" />
            </condition>

            <!-- Regular DEX process.  We would prefer to use the Android SDK
                 ANT target, but we need to detect the "use multidex" error.
                 https://android.googlesource.com/platform/sdk/+/tools_r21.1/anttasks/src/com/android/ant/DexExecTask.java
            -->
            <mapper id="pre-dex-mapper" type="glob" from="libs/*.jar" to="bin/dexedLibs/*-dexed.jar"/>

            <apply executable="${dx}" failonerror="true" parallel="false" dest="${out.dexed.absolute.dir}" relative="true">
                        <arg value="--dex" />
                        <arg value="--output" />
                        <targetfile/>
                        <arg line="${jumbo.option}" />
                        <arg line="${verbose.option}" />
                        <fileset dir="." includes="libs/*" />
                        <external-libs />
                        <mapper refid="pre-dex-mapper"/>
            </apply>

            <apply executable="${dx}" resultproperty="dex.merge.result" outputproperty="dex.merge.output" parallel="true">
                <arg value="--dex" />
                <arg value="--output=${intermediate.dex.file}" />
                <arg line="${jumbo.option}" />
                <arg line="${verbose.option}" />
                <path path="${out.dex.input.absolute.dir}"/>
                <path refid="out.dex.jar.input.ref" />
                <external-libs />
            </apply>

            <if>
                <condition>
                    <or>
                        <contains string="${dex.merge.output}" substring="Too many field references"/>
                        <contains string="${dex.merge.output}" substring="Too many method references"/>
                    </or>
                </condition>
                <then>
                    <echo message="Number of field or method references is too big.  Switching to multi-dex build." />
                    <multi-dex-helper>
                        <external-libs>
                        <external-libs/>
                        </external-libs>
                    </multi-dex-helper>
                </then>
                <else>
                    <echo message="${dex.merge.output}"/>
                    <fail status="${dex.merge.result}">
                        <condition>
                            <not>
                                <equals arg1="${dex.merge.result}" arg2="0"/>
                            </not>
                        </condition>
                    </fail>
                </else>
            </if>

      </sequential>
    </macrodef>

    <macrodef name="multi-dex-helper">
        <element name="external-libs" optional="yes" />
        <sequential>
            <property name="mainDexClasses" location="${android.build.tools.dir}/mainDexClasses" />

            <exec executable="${mainDexClasses}" failonerror="true" >
                <arg line="--output ${out.absolute.dir}/classes_to_kepp_in_main_dex"/>
                <arg file="${out.absolute.dir}/proguard/obfuscated.jar"/>
            </exec>
            <echo>Converting compiled files and external libraries into ${out.absolute.dir} (multi-dex)</echo>
            <echo>Dexing ${out.classes.absolute.dir} and ${toString:out.dex.jar.input.ref}</echo>
            <apply executable="${dx}" failonerror="true" parallel="true">
                <arg value="--dex" />
                <arg value="--multi-dex" />
                <arg value="--main-dex-list=${out.absolute.dir}/classes_to_kepp_in_main_dex" />
                <arg value="--output=${out.absolute.dir}" />

                <arg line="${jumbo.option}" />
                <arg line="${verbose.option}" />
                <arg path="${out.absolute.dir}/proguard/obfuscated.jar" />
                <path refid="out.dex.jar.input.ref" />
                <external-libs />
            </apply>
        </sequential>
    </macrodef>