Apache HTTPClient DigestAuth doesn't forward “

I'm trying to use Digest authentication with HTTP Client against a 3rd-party web service that I don't control.

I started out with the sample code from here:

http://hc.apache.org/httpcomponents-client-4.5.x/httpclient/examples/org/apache/http/examples/client/ClientPreemptiveDigestAuthentication.java

I got it working against httpbin.org, before attempting the next step described below.

It appears that the target 3rd-party service that I'm using requires the opaque value to be copied from the WWW-Authentication header on the initial response to the Authorization header on the next request, as described here:

https://security.stackexchange.com/questions/24425/what-is-the-opaque-field-in-http-digest-access-authentication-used-for

However, I have turned on wire-logging and stepped through the code (again this is really just the sample code linked above, no need to copy/paste it here) and I see that the opaque is NOT copied.

Any ideas what prevents it from being copied?

I even tried overriding the processChallenge method:

    DigestScheme digestAuth = new DigestScheme() {
        @Override
        public void processChallenge(
                Header header) throws MalformedChallengeException {

but it appears that any value introduced into the Parameters at this point is ignored in the next request.

Finally fixed by overriding the Authorize header explicitly, instead of relying on the internals of HttpClient to do it automatically:

package [...];

import java.util.Arrays;
import java.util.Map;
import java.util.stream.Collectors;

import org.apache.commons.io.IOUtils;
import org.apache.http.*;
import org.apache.http.auth.AuthScope;
import org.apache.http.auth.UsernamePasswordCredentials;
import org.apache.http.client.*;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.protocol.HttpClientContext;
import org.apache.http.impl.auth.DigestScheme;
import org.apache.http.impl.client.*;
import org.testng.Assert;

public class DigestTest {

    private static final String URL
            = "https://...";

    private static final String PASSWORD = ...;

    private static final String USER = ...;

    public static void main(String[] args) throws Exception {

        new DigestTest().run();
    }

    public void run() throws Exception {

        HttpGet httpget = new HttpGet(URL);

        HttpHost target
                = new HttpHost(httpget.getURI().getHost(), 443, "https");
        CredentialsProvider credsProvider = new BasicCredentialsProvider();

        UsernamePasswordCredentials credentials
                = new UsernamePasswordCredentials(USER, PASSWORD);
        credsProvider.setCredentials(
                new AuthScope(target.getHostName(), target.getPort()),
                credentials);

        CookieStore cookieStore = new BasicCookieStore();

        CloseableHttpClient httpclient
                = HttpClients.custom().setDefaultCookieStore(cookieStore)
                        .setDefaultCredentialsProvider(credsProvider).build();

        try {

            DigestScheme digestAuth = new DigestScheme();

            digestAuth.overrideParamter("qop", "auth");
            digestAuth.overrideParamter("nc", "0");
            digestAuth.overrideParamter("cnonce", DigestScheme.createCnonce());

            AuthCache authCache = new BasicAuthCache();
            authCache.put(target, digestAuth);

            HttpClientContext localContext = HttpClientContext.create();
            localContext.setAuthCache(authCache);

            CloseableHttpResponse response;

            response = httpclient.execute(target, httpget, localContext);
            Map<String, String> wwwAuth = Arrays
                    .stream(response.getHeaders("WWW-Authenticate")[0]
                            .getElements())
                    .collect(Collectors.toMap(HeaderElement::getName,
                            HeaderElement::getValue));

            // the first call ALWAYS fails with a 401
            Assert.assertEquals(response.getStatusLine().getStatusCode(), 401);

            digestAuth.overrideParamter("opaque", wwwAuth.get("opaque"));
            digestAuth.overrideParamter("nonce", wwwAuth.get("nonce"));
            digestAuth.overrideParamter("realm", wwwAuth.get("Digest realm"));
            Header authenticate = digestAuth.authenticate(credentials, httpget,
                    localContext);
            httpget.addHeader(authenticate);

            response = httpclient.execute(target, httpget, localContext);

            // the 2nd call is the real deal
            Assert.assertEquals(response.getStatusLine().getStatusCode(), 200);

            System.out.println(IOUtils
                    .toString(response.getEntity().getContent(), "utf-8"));

        } finally {
            httpclient.close();
        }
    }

}