Wildfly-Swarm and connection to external activemq

So I've already spent considerable time on this, but now have come to a point where I'm completely at my wits end...

The requirement: I'm trying to connect a wildfly 10.1.0 based message driven bean to an external activemq 5.15.0 server (the 'old' activemq, not artemis mq!). For this I'm deploying the resource adapter and tweaking the configuration. In standard deployment of wildfly this works ok. I'm using the following script to setup the container:

# Disable the artemis messaging completely
/subsystem=messaging-activemq/server=default:remove

# Deploy the resource adapter
deploy ${project.build.directory}/activemq-rar-5.15.0.rar

/subsystem=resource-adapters/resource-adapter=activemq-rar.rar:add(archive=activemq-rar-5.15.0.rar,transaction-support=LocalTransaction)
/subsystem=resource-adapters/resource-adapter=activemq-rar.rar/config-properties=ServerUrl:add(value="${activemq.broker}")
/subsystem=resource-adapters/resource-adapter=activemq-rar.rar/config-properties=UserName:add(value="${jboss.user}")
/subsystem=resource-adapters/resource-adapter=activemq-rar.rar/config-properties=Password:add(value="${jboss.password}")
/subsystem=resource-adapters/resource-adapter=activemq-rar.rar/config-properties=UseInboundSession:add(value="false")
/subsystem=resource-adapters/resource-adapter=activemq-rar.rar/connection-definitions=AMQConnectionFactory:add(jndi-name=ConnectionFactory,class-name=org.apache.activemq.ra.ActiveMQManagedConnectionFactory,enabled=true,min-pool-size=1,max-pool-size=20,pool-prefill=false,same-rm-override=false,use-java-context=true)
/subsystem=resource-adapters/resource-adapter=activemq-rar.rar/admin-objects=REQUESTQUEUE:add(class-name=org.apache.activemq.command.ActiveMQQueue,jndi-name=queues/request,use-java-context=true)
/subsystem=resource-adapters/resource-adapter=activemq-rar.rar/admin-objects=REPLYQUEUE:add(class-name=org.apache.activemq.command.ActiveMQQueue,jndi-name=queues/reply,use-java-context=true)
/subsystem=ejb3:write-attribute(name=default-resource-adapter-name,value=activemq-rar.rar)
/subsystem=ejb3:write-attribute(name=default-mdb-instance-pool,value=mdb-strict-max-pool)
/subsystem=ee/service=default-bindings:write-attribute(name=jms-connection-factory,value=java:/ConnectionFactory)

reload

With this configuration I can launch the container with the standalone-full configuration (to get the JMS related classes as well) and it works as intended.

But if I attempt to achieve the same with wildfly swarm, the same test mdb.jar cannot be deployed, during startup of the container I get the following exception:

"WFLYCTL0412: Required services that are not installed:" => ["jboss.ra.activemq-rar"],
"WFLYCTL0180: Services with missing/unavailable dependencies" => ["jboss.deployment.unit.\"mdbtest.jar\".component.TestMDB.CREATE is missing [jboss.ra.activemq-rar]"]

So it seems, that the resource adapter is not available, however it is visible in the administration console I deployed as well.

To setup the swarm container I use the following project-defaults.xml:

swarm:

  resource-adapters:
    resource-adapters:
      activemq-rar:
        archive: activemq-rar-15.5.0.rar
        transaction-support: LocalTransaction
        config-properties:
          ServerUrl:
            value: failover:tcp://localhost:61616
          UserName:
            value: admin
          Password:
            value: admin
          UseInboundSession:
            value: false

        connection-definitions:
            AMQConnectionFactory:
                jndi-name: ConnectionFactory
                class-name: org.apache.activemq.ra.ActiveMQManagedConnectionFactory
                enabled: true
                min-pool-size: 1
                max-pool-size: 20
                pool-prefill: false
                same-rm-override: false
                use-java-context: true
                same-rm-override: false

        admin-objects:
            REQUESTQUEUE:
                class-name: org.apache.activemq.command.ActiveMQQueue
                jndi-name: queues/request
                use-java-context: true
            REPLEYQUEUE:
                class-name: org.apache.activemq.command.ActiveMQQueue
                jndi-name: queues/reply
                use-java-context: true

  ejb3:
    # Switch the MDB default to the resource adapter defined above
    default-resource-adapter-name: activemq-rar
    default-mdb-instance-pool: mdb-strict-max-pool

  ee:
      default-bindings:
          jms-connection-factory: java:/ConnectionFactory


  management:
    security-realms:
      ManagementRealm:
        in-memory-authentication:
          users:
            admin:    
              password: admin                            
    http-interface-management-interface:
      allowed-origins:
      - http://localhost:8080
      security-realm: ManagementRealm

  messaging-activemq:
    servers:
      default:
#        active: false
#        connection-factories:
#          InVmConnectionFactory:
#            client-id: blahblabla
#            block-on-acknowledge: true
#            entries:
#            - "java:/ArtemisConnectionFactory"
#        pooled-connection-factories:
#          activemq-ra:
#            entries:
#            connectors: in-vm
#            transaction: xa

  jca:
    archive-validation:
      enabled: false


  datasources:
    jdbc-drivers:
      org.postgresql:
        driver-class-name: org.postgresql.Driver
        xa-datasource-class-name: org.postgresql.xa.PGXADataSource
        driver-module-name: org.postgresql

    data-sources:
      myDS:
        connection-url: jdbc:postgresql://localhost:5432/mydb
        user-name: dbuser
        password: dbpassword
        driver-name: postgresql
        jndi-name: java:jboss/datasources/myDS
        min-pool-size: 4
        max-pool-size: 64
        use-ccm: false


  deployment:
    org.apache.activemq:activemq-rar.rar:
    com.oneworldsync.mdb:mdbtest.jar:
    org.postgresql:postgresql.jar:

  logging:
    loggers:
      org.jboss:
        level: warn
      org.wildfly:
        level: warn

The relevant part of the pom looks like this:

<build>
  <plugins>
    <plugin>
      <groupId>org.wildfly.swarm</groupId>
      <artifactId>wildfly-swarm-plugin</artifactId> 
      <version>2017.4.0</version>
      <executions>
        <execution>
          <id>package</id>
          <goals>
            <goal>package</goal>
          </goals>
          <phase>package</phase>
        </execution>
      </executions>
    </plugin>
  </plugins>
</build>


<dependencies>
  <dependency>
    <groupId>com.oneworldsync.mdb</groupId>
    <artifactId>mdbtest</artifactId>
    <version>1.0.0-SNAPSHOT</version>
  </dependency>
  <dependency>
    <groupId>org.wildfly.swarm</groupId>
    <artifactId>logging</artifactId>
  </dependency>
  <dependency>
    <groupId>org.wildfly.swarm</groupId>
    <artifactId>datasources</artifactId>
  </dependency>
  <dependency>
    <groupId>org.wildfly.swarm</groupId>
    <artifactId>undertow</artifactId>
  </dependency>
  <dependency>
    <groupId>org.postgresql</groupId>
    <artifactId>postgresql</artifactId>
    <version>42.1.4</version>
  </dependency>
  <dependency>
    <groupId>org.wildfly.swarm</groupId>
    <artifactId>cdi</artifactId>
  </dependency>
  <dependency>
    <groupId>org.wildfly.swarm</groupId>
    <artifactId>jpa</artifactId>
  </dependency>
  <dependency>
    <groupId>org.wildfly.swarm</groupId>
    <artifactId>msc</artifactId>
  </dependency>
  <dependency>
    <groupId>org.wildfly.swarm</groupId>
    <artifactId>infinispan</artifactId>
  </dependency>
  <dependency>
    <groupId>org.wildfly.swarm</groupId>
    <artifactId>messaging</artifactId>
  </dependency>
  <dependency>
    <groupId>org.wildfly.swarm</groupId>
    <artifactId>jca</artifactId>
  </dependency>
  <dependency>
    <groupId>org.wildfly.swarm</groupId>
    <artifactId>resource-adapters</artifactId>
  </dependency>
  <dependency>
    <groupId>org.apache.activemq</groupId>
    <artifactId>activemq-rar</artifactId>
    <version>${activemq.version}</version>
    <type>rar</type>
  </dependency>
  <dependency>
    <groupId>org.wildfly.swarm</groupId>
    <artifactId>management</artifactId>
  </dependency>
  <dependency>
    <groupId>org.wildfly.swarm</groupId>
    <artifactId>management-console</artifactId>
  </dependency>
  <dependency>
    <groupId>org.wildfly.swarm</groupId>
    <artifactId>cli</artifactId>
    <version>${jboss.version}</version>
  </dependency>
  <dependency>
    <groupId>org.wildfly.swarm</groupId>
    <artifactId>ejb</artifactId>
  </dependency>
  <dependency>
    <groupId>org.wildfly.swarm</groupId>
    <artifactId>connector</artifactId>
  </dependency>
  <dependency>
    <groupId>org.wildfly.swarm</groupId>
    <artifactId>jaxrs</artifactId>
  </dependency>
  <dependency>
    <groupId>org.wildfly.swarm</groupId>
    <artifactId>remoting</artifactId>
  </dependency>
</dependencies>

The testing MDB is straightforward:

@MessageDriven(activationConfig = {
    @ActivationConfigProperty(propertyName = "acknowledgeMode",
            propertyValue = "Auto-acknowledge"),
    @ActivationConfigProperty(propertyName = "destination",
            propertyValue = "request"),
    @ActivationConfigProperty(propertyName = "destinationType",
            propertyValue = "javax.jms.Queue")
})
public class TestMDB implements MessageListener {

    @Resource(name = "java:/ConnectionFactory")
    private QueueConnectionFactory queueConnectionFactory;


    @Override
    public void onMessage(Message message) {
        System.out.println(message);
    }
}

So, I'm looking for a hint why the MDB cannot access the resource adapter, even if it has been deployed, is visible and uses the appropriate name. I would expect, it is something simple at this point, but I cannot find it.

Many thanks!

PS: The external activemq server is a requirement. Using a JMS bridge would probably work, but has some drawbacks which are not acceptable (performance impact due to additional 'hop', jms-reply-to destination don't work over jms bridge).

Edit 1

Arg, after posting I found at least one problem in my pom: I used a mismatching version of the swarm plugin: The swarm fractions are 2017.10.0, but the plugin is 2017.4.0. Correcting this changes the error symptoms a bit:

2017-10-26 09:21:07,731 ERROR [stderr] (main) Caused by: java.lang.RuntimeException: org.wildfly.swarm.container.DeploymentException: org.wildfly.swarm.container.DeploymentException: WFSWARM0004: Deployment failed: {"WFLYCTL0412: Required services that are not installed:" => ["jboss.ra.activemq-rar"],"WFLYCTL0180: Services with missing/unavailable dependencies" => ["jboss.deployment.unit.\"mdbtest.jar\".component.TestMDB.CREATE is missing [jboss.ra.activemq-rar]"]}
2017-10-26 09:21:07,732 ERROR [stderr] (main)   at org.wildfly.swarm.spi.api.ClassLoading.withTCCL(ClassLoading.java:45)
2017-10-26 09:21:07,732 ERROR [stderr] (main)   at org.wildfly.swarm.container.runtime.ServerBootstrapImpl.bootstrap(ServerBootstrapImpl.java:114)
2017-10-26 09:21:07,732 ERROR [stderr] (main)   at org.wildfly.swarm.Swarm.start(Swarm.java:386)
2017-10-26 09:21:07,732 ERROR [stderr] (main)   at org.wildfly.swarm.Swarm.main(Swarm.java:720)
2017-10-26 09:21:07,732 ERROR [stderr] (main)   ... 6 more

So there seems to be a classloading issue underneath. Will investigate this, but would still be glad for further input

Edit 2

The stacktrace above seems to be irrelevant: There is actually a difference how the swarmified application is started:

Start with java -jar something.app: No stacktrace
Start with mv nwildfly-swarm:run: The above stacktrace.

However, the basic cause is the same (dependencies not found). I interpret it that the maven plugin adds an additional layer to the invocation, that logs the additional stacktrace, but the issue is still the same.

Finally, after many hours I've solved the problem myself. As expected, the solution was simple but not obvious...

One important hint I took from here. After some serious analysis and debugging sessions I've found, that the resource-adapter was not correctly deployed: while a suitable deployment unit was mentioned in the installed services (the JMX console is very helpful to find those), the resource adapter itself was still not available. The solution to all this was the name of the deployment: To deploy the adapter correctly, the following snippet has to be used:

swarm:
  deployment:
    org.apache.activemq:activemq-rar.rar:

Note that the maven version number has to be omitted and just the maven artifactId has to be used (compare with my original snippet above). As the swarm plugin is tightly coupled to the maven dependency management, it figures out the physical name itself. In a standalone wildfly this is different: It doesn't know anything about maven artifacts, so the physical archive name has to be used. In the standalone-full.xml it would look like this:

<deployments>
   <deployment name="activemq-rar-5.15.0.rar" runtime-name="activemq-rar-5.15.0.rar">
       ...

When the name of the archive is corrected, the deployment works. Only thing left is a conflict in the JNDI due to the existence of the ArtemisMQ adapter (using the YAML approach of swarm it cannot be removed). So this has to be adapted as well.

So for completeness: Here is my finally working project-defaults.yaml:

swarm:
  deployment:
    org.apache.activemq:activemq-rar.rar:

  resource-adapters:
    resource-adapters:
      activemq-rar:
        # This is not the physical jar name, but the maven coordinates (without version!)
        archive: activemq-rar.rar
        transaction-support: LocalTransaction
        config-properties:
          ServerUrl:
            value: failover:tcp://localhost:61616
          UserName:
            value: admin
          Password:
            value: admin
          UseInboundSession:
            value: false

        connection-definitions:
            AMQConnectionFactory:
                jndi-name: ConnectionFactory
                class-name: org.apache.activemq.ra.ActiveMQManagedConnectionFactory
                enabled: true
                min-pool-size: 1
                max-pool-size: 20
                pool-prefill: false
                same-rm-override: false
                use-java-context: true

        admin-objects:
            REQUESTQUEUE:
                class-name: org.apache.activemq.command.ActiveMQQueue
                jndi-name: queues/request
                use-java-context: true
                config-properties:
                  PhysicalName: demo.request


  ejb3:
    default-resource-adapter-name: activemq-rar.rar
    default-mdb-instance-pool: mdb-strict-max-pool

  ee:
      default-bindings:
          jms-connection-factory: java:/ConnectionFactory


  management:
    security-realms:
      ManagementRealm:
        in-memory-authentication:
          users:
            admin:    
              password: admin                            
    http-interface-management-interface:
      allowed-origins: http://localhost:8080
      security-realm: ManagementRealm
      console-enabled: true

  messaging-activemq:
    servers:
      default:
        active: false
        connection-factories:
          InVmConnectionFactory:
            block-on-acknowledge: true
            entries:
            - "java:/ArtemisConnectionFactory"

  jca:
    archive-validation:
      enabled: false

  logging:
    loggers:
      org.jboss:
        level: INFO
      org.wildfly:
        level: INFO