Use threading to process file chunk by chunk

问题:

I have a arraylist which consists of 5000 IP Addresses. For each IP Address, I want to execute a SNMPGet request and a FTPDownload command. I want to implement it in a fashion, wherein at a time 2 different threads run simultaneously for the first five IP Addresses. After the execution of these IP Addresses, next 2 IP Addresses will be executed on these threads. Can anyone help how to do it?

Here, connection is a class which extends the thread and the work to be implemented is written in its run() method. Please help.

Connection newConnection =new Connection(0);
Connection newConnection1 =new Connection(1);

for(int i = 0; i < NE_list.getRowCount(); i=i+2)
{
if(NE_list.getValueAt(i, 0).toString().equals("true")) //Some condition here for the IP Address
{

            newConnection.i=i;
            newConnection1.i=i+1;
            newConnection.runprogram();
            newConnection1.runprogram();
 }


    } 

class Connection extends Thread{
int  i;
Connection(int val){
  i=val;
}
void runprogram(){
start();
}
public void run(){
//SNMP and FTP Code here for IP Address in index i of NE_list
}
}

回答1:

Executor Framework will be best suit for your solution. I have created one example here. You can increase the number of threads as per your requirement.

import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;

class SomeRunnable implements Runnable {
    int threadNo = -1 ;
    List<String> list = new ArrayList<String>();
    public SomeRunnable(List list, int threadNo ) {
        this.list.addAll(list);
        this.threadNo =threadNo;
    }
    @Override
    public void run() {
        for (String element : list) {
            System.out.println("By Thread:" + threadNo+", Processed Element:" +element);
        }
    }

}

public class ExecutorDemo {
    public static void main(String[] args) {
        List<String> list = new ArrayList<String>();
        for (int i = 0; i < 100; i++) {
            list.add("Elem:"+i);
        }
        // Divide list 
        int divideIndex = list.size()/2;
        //Create objects of Runnable
        SomeRunnable obj1 = new SomeRunnable(list.subList(0, divideIndex),1);
        SomeRunnable obj2 = new SomeRunnable(list.subList(divideIndex,list.size()),2);

        //Create fixed Thread pool, here pool of 2 thread will created
        ExecutorService pool = Executors.newFixedThreadPool(2);

        pool.execute(obj1);
        pool.execute(obj2);

        pool.shutdown();
    }

}

回答2:

Herewith adding working example having 5 threads. Just put the test.txt in your CLASS_PATH of the application.

class MyRunnable implements Runnable {
List<List<String>> records;
MyRunnable(List<List<String>> records){
    this.records = records;
}
public void run(){
    for(List list : records){
        System.out.println(Thread.currentThread().getName() + " : "+list.toString());
    }
}}

Main Class -

public class FileProcessThreads {
public List<List<String>> process(String fileName) throws IOException {
    List<List<String>> records = new ArrayList<>();
    BufferedReader br = new BufferedReader(new FileReader(fileName));
    String line = null;
    while((line = br.readLine()) != null){
        List<String> listValues = Arrays.asList(line.split(" "));
        records.add(listValues);
    }
    System.out.println(records.size());
    return records;
}
public static void main(String[] args) throws IOException {
    FileProcessThreads fp = new FileProcessThreads();
    List<List<String>> records = fp.process("test.txt");
    ExecutorService es = Executors.newFixedThreadPool(5);
    int recordsInEachThread = (int) (records.size()/5);
    System.out.println(recordsInEachThread);

    MyRunnable my1 = new MyRunnable(records.subList(0, recordsInEachThread));
    MyRunnable my2 = new MyRunnable(records.subList(recordsInEachThread+1, recordsInEachThread*2));
    MyRunnable my3 = new MyRunnable(records.subList(recordsInEachThread*2 + 1, recordsInEachThread*3));
    MyRunnable my4 = new MyRunnable(records.subList(recordsInEachThread*3 + 1, recordsInEachThread*4));
    MyRunnable my5 = new MyRunnable(records.subList(recordsInEachThread*4 + 1, records.size() - 1));
    es.execute(my1);
    es.execute(my2);
    es.execute(my3);
    es.execute(my4);
    es.execute(my5);
    es.shutdown();
}}