How can I combine my customize module with Omnet++

As all the simple modules' behavior like 80211 mac layer has been defined in the INET module. If I want to add a customize layer between mac layer and network layer to handle network coding. How can I combine the customize module and INET's simple module ?

To add a new module between network layer and MAC layer I suggest creating a modified host in a new project. For OMNeT++ 4.6 and INET 3.2.4 do the following:

Create a new OMNeT++ empty project with src and simulation directories.
In the new project open Properties | Project References and select inet.
Right click on src and select New | Simple module. Call it DummyLayer.ned.

Open DummyLayer.ned and add:

@namespace(inet);
import inet.linklayer.contract.INic;

simple DummyLayer like INic {
parameters:
    @display("i=block/buffer");
    // here you can add others parameter
gates:
    input ifIn;
    output ifOut;
    input upperLayerIn;
    output upperLayerOut;
}

Modify DummyLayer.h and DummyLayer.cc (this module just passes every message from up and down, as well as increments counters):

// DummyLayer.h
#include <omnetpp.h>
namespace inet {
class DummyLayer: public cSimpleModule {
protected:
    virtual void initialize();
    virtual void handleMessage(cMessage *msg);
private:
    int upNumber;
    int downNumber;
};
} //namespace

//----------------------------------------------
// DummyLayer.cc
#include "DummyLayer.h"
namespace inet {
Define_Module(DummyLayer);

void DummyLayer::initialize() {
   upNumber = 0;
   downNumber = 0;
}

void DummyLayer::handleMessage(cMessage *msg) {
  if (msg->arrivedOn("upperLayerIn")) {
      send(msg, "ifOut");
      downNumber++;
  } else if (msg->arrivedOn("ifIn")) {
      send(msg, "upperLayerOut");
      upNumber++;
  } else {
      error("Incorrect gate");
  }
  char buf[128];
  sprintf(buf, "up: %d, down: %d", upNumber, downNumber);
  getDisplayString().setTagArg("t", 0, buf);
  }
} //namespace

Create a new compound module for own host, call it WirelessHostEx.ned:

import inet.common.lifecycle.NodeStatus;
import inet.linklayer.contract.IWiredNic;
import inet.linklayer.contract.IWirelessNic;
import inet.linklayer.loopback.LoopbackInterface;
import inet.mobility.contract.IMobility;
import inet.networklayer.contract.IRoutingTable;
import inet.networklayer.common.InterfaceTable;
import inet.networklayer.contract.INetworkLayer;
import inet.power.contract.IEnergyStorage;
import inet.power.contract.IEnergyGenerator;
import inet.applications.contract.IPingApp;
import inet.applications.contract.ISCTPApp;
import inet.applications.contract.ITCPApp;
import inet.applications.contract.IUDPApp;
import inet.transportlayer.contract.ISCTP;
import inet.transportlayer.contract.ITCP;
import inet.transportlayer.contract.IUDP;
import inet.node.inet.INetworkNode;

module WirelessHostEx like INetworkNode
{
    parameters:
        @networkNode;
        @display("i=device/wifilaptop");
        @labels(wireless-node);
        bool hasStatus = default(false);
        int numExtInterfaces = default(0);
        int numRadios = 1;
        int numTunInterfaces = default(0);
        string mobilityType = default(numRadios > 0 ? "StationaryMobility" : "");
        string networkLayerType = default("IPv4NetworkLayer");
        string routingTableType = default("IPv4RoutingTable");
        bool forwarding = default(true);
        bool multicastForwarding = default(false);
        string energyStorageType = default("");
        string energyGeneratorType = default("");
        routingTable.forwarding = forwarding;
        routingTable.multicastForwarding = multicastForwarding;  
        *.interfaceTableModule = default(absPath(".interfaceTable"));
        *.routingTableModule = default(routingTableType != "" ? absPath(".routingTable") : "");
        *.energySourceModule = default(energyStorageType != "" ? absPath(".energyStorage") : "");
        *.mobilityModule = default(mobilityType != "" ? absPath(".mobility") : "");
        int numTcpApps = default(0);
        int numUdpApps = default(0); 
        int numPingApps = default(0); 
        bool hasTcp = default(numTcpApps > 0);
        bool hasUdp = default(numUdpApps > 0);
        string tcpType = default(firstAvailableOrEmpty("TCP", "TCP_lwIP", "TCP_NSC"));  // tcp implementation (e.g. ~TCP, ~TCP_lwIP, ~TCP_NSC) or ~TCPSpoof
        string udpType = default(firstAvailableOrEmpty("UDP"));
        forwarding = default(false);  // disable routing by default
        networkLayer.proxyARP = default(false);
    gates:
        input radioIn[numRadios] @directIn;
        inout pppg[] @labels(PPPFrame-conn);
        inout ethg[] @labels(EtherFrame-conn);
    submodules:
        status: NodeStatus if hasStatus {
            @display("p=50,50");
        }
        energyStorage: <energyStorageType> like IEnergyStorage if energyStorageType != "" {
            parameters:
                @display("p=50,100;i=block/plug;is=s");
        }
        energyGenerator: <energyGeneratorType> like IEnergyGenerator if energyGeneratorType != "" {
            parameters:
                @display("p=50,150;i=block/plug;is=s");
        }
        mobility: <mobilityType> like IMobility if mobilityType != "" {
            parameters:
                @display("p=53,200");
        }
        networkLayer: <networkLayerType> like INetworkLayer {
            parameters:
                @display("p=329,287;q=queue");
        }
        routingTable: <routingTableType> like IRoutingTable if routingTableType != "" {
            parameters:
                @display("p=53,250;is=s");
        }
        interfaceTable: InterfaceTable {
            parameters:
                @display("p=53,300;is=s");
        }
        lo0: LoopbackInterface {
            @display("p=78,406");
        }
        wlan[numRadios]: <default("Ieee80211Nic")> like IWirelessNic {
            parameters:
                @display("p=216,406,row,60;q=queue");
        }
        eth[sizeof(ethg)]: <default("EthernetInterface")> like IWiredNic {
            parameters:
                @display("p=368,406,row,60;q=txQueue");
        }
        ppp[sizeof(pppg)]: <default("PPPInterface")> like IWiredNic {
            parameters:
                @display("p=558,406,row,60;q=txQueue");
        }
        tcpApp[numTcpApps]: <> like ITCPApp {
            parameters:
                @display("p=147,54,row,60");
        }
        tcp: <tcpType> like ITCP if hasTcp {
            parameters:
                @display("p=147,141");
        }
        udpApp[numUdpApps]: <> like IUDPApp {
            parameters:
                @display("p=329,54,row,60");
        }
        udp: <udpType> like IUDP if hasUdp {
            parameters:
                @display("p=329,141");
        }
        pingApp[numPingApps]: <default("PingApp")> like IPingApp {
            parameters:
                @display("p=635,141,row,60");
        }
        dummy: DummyLayer {
            @display("p=273,350");
        }
    connections allowunconnected:
        radioIn[0] --> { @display("m=s"); } --> wlan[0].radioIn;
        // the order of connections is important here
        wlan[0].upperLayerOut --> dummy.ifIn;
        dummy.upperLayerOut --> networkLayer.ifIn++;
        wlan[0].upperLayerIn <-- dummy.ifOut;
        dummy.upperLayerIn <-- networkLayer.ifOut++;
        networkLayer.ifOut++ --> lo0.upperLayerIn;
        lo0.upperLayerOut --> networkLayer.ifIn++;
        for i=0..sizeof(ethg)-1 {
            ethg[i] <--> { @display("m=s"); } <--> eth[i].phys;
            eth[i].upperLayerOut --> networkLayer.ifIn++;
            eth[i].upperLayerIn <-- networkLayer.ifOut++;
        }
        for i=0..sizeof(pppg)-1 {
            pppg[i] <--> { @display("m=s"); } <--> ppp[i].phys;
            ppp[i].upperLayerOut --> networkLayer.ifIn++;
            ppp[i].upperLayerIn <-- networkLayer.ifOut++;
        }
        for i=0..numTcpApps-1 {
            tcpApp[i].tcpOut --> tcp.appIn++;
            tcpApp[i].tcpIn <-- tcp.appOut++;
        }
        tcp.ipOut --> networkLayer.transportIn++ if hasTcp;
        tcp.ipIn <-- networkLayer.transportOut++ if hasTcp;
        for i=0..numUdpApps-1 {
            udpApp[i].udpOut --> udp.appIn++;
            udpApp[i].udpIn <-- udp.appOut++;
        }
        udp.ipOut --> networkLayer.transportIn++ if hasUdp;
        udp.ipIn <-- networkLayer.transportOut++ if hasUdp;
        for i=0..numPingApps-1 {
            networkLayer.pingOut++ --> pingApp[i].pingIn;
            networkLayer.pingIn++ <-- pingApp[i].pingOut;
        }
}

An own host module is necessary because StandardHost from INET creates connection between MAC and network layer automatically and it is not possible to add own module between these layers.

Create a network (for test):

import inet.networklayer.configurator.ipv4.IPv4NetworkConfigurator;
import inet.physicallayer.ieee80211.packetlevel.Ieee80211ScalarRadioMedium;
import inet.node.wireless.AccessPoint;

network WirelessNetwork {
    submodules:
        configurator: IPv4NetworkConfigurator {
            @display("p=33,81");
        }
        radioMedium: Ieee80211ScalarRadioMedium {
            @display("p=33,30");
        }
        node0: WirelessHostEx {
            @display("p=128,121");
        }
        node1: WirelessHostEx {
            @display("p=384,115");
        }
        ap: AccessPoint {
            @display("p=273,54");
        }
}

Modify omnetpp.ini:

[General]
network = WirelessNetwork

// node0 will send ping to node1
**.node0.numPingApps = 1
**.node0.pingApp[0].destAddr = "node1" // using IP address here is allowed too

After starting the simulation one can see that in each host dummyLayer send messages forward.