How to dynamically remove a JPanel?

I have a a GUI looks as follow.

I want to dynamically add/remove a panel. I use ArrayList to keep trace of JPanel objects. And now I could add panel dynamically, but when I want to delete a panel, I could not get its attribute so that I can not remove it.

Here is my code:

public class Main implements ActionListener{
    private List <myPanel> mpList;
    private JPanel btnPanel;
    private JButton jbtAdd,jbtDelete;
    private JFrame jf;
    private JPanel jp;
    private JScrollPane js;
    private myPanel mp;
    private static int size=0;
    private int selectedId=-1;
    //private
    public Main(){
        mpList=new ArrayList<myPanel>();
        btnPanel=new JPanel();
        jbtAdd=new JButton("addJpanel");
        jbtDelete=new JButton("delJpanel");
        btnPanel.setLayout(new FlowLayout(FlowLayout.LEFT,1,1));
        btnPanel.add(jbtAdd);
        btnPanel.add(jbtDelete);
        jf=new JFrame("hello");
        jp=new JPanel();
        js=new JScrollPane(JScrollPane.VERTICAL_SCROLLBAR_AS_NEEDED, JScrollPane.HORIZONTAL_SCROLLBAR_AS_NEEDED);   
        //jp.setLayout(new FlowLayout(FlowLayout.LEFT,1,1));
        jf.setLayout(new BorderLayout(10,10));
        jp.setLayout(new GridLayout(0,1,1,1));
        jp.setPreferredSize(new Dimension(500, 82*6));
        for(int i=0;i<6;i++){
            myPanel mp=new myPanel();
            //mp.setSize(400, 82);
            //mp.setBounds(0,82*i,480,82);
            mp.getFileTypeIconLabel().setText(String.valueOf(i));
            mp.setIndexId(size);
            size++;
            mpList.add(0,mp);
            mp.addMouseListener(new MouseListener(){
                @Override
                public void mouseClicked(MouseEvent e) {
                // TODO Auto-generated method stub
                    if(2==e.getClickCount()){
                        System.out.println("indexInd is"+mpList.get(0).getIndexId());
                        //System.out.println(index);
                        mpList.get(0).setBackground(Color.yellow);
                    }
                }

                @Override
                public void mouseEntered(MouseEvent e) {
                    // TODO Auto-generated method stub

                }

                @Override
                public void mouseExited(MouseEvent e) {
                    // TODO Auto-generated method stub

                }

                @Override
                public void mousePressed(MouseEvent e) {
                    // TODO Auto-generated method stub

                }

                @Override
                public void mouseReleased(MouseEvent e) {
                    // TODO Auto-generated method stub

                }});
            jp.add(mpList.get(0));
        }
        js.setViewportView(jp);
        jf.setSize(600, 500);
        jf.add(btnPanel,BorderLayout.NORTH);
        jf.add(js);
        jbtAdd.addActionListener(this);
        jbtDelete.addActionListener(this);
        jf.setVisible(true);
        jf.setDefaultCloseOperation(JFrame.EXIT_ON_CLOSE);

    }
    public static void main(String args[]){
        new Main();
    }
    @Override
    public void actionPerformed(ActionEvent e) {
        // TODO Auto-generated method stub
        if(e.getSource()==jbtAdd){
            addMyPanel();
        }
        else if(e.getSource()==jbtDelete){
            delMyPanel();
        }
    }
    public void addMyPanel(){
        System.out.println("ok");
        mp=new myPanel();
        mp.getFileTypeIconLabel().setText(String.valueOf(mpList.size()));
        mp.setIndexId(size);
        System.out.println(size);
        size++;
        mpList.add(0, mp);
        mp.addMouseListener(new MouseListener(){
            @Override
            public void mouseClicked(MouseEvent e) {
            // TODO Auto-generated method stub
                if(2==e.getClickCount()){
                    System.out.println("indexInd is"+mpList.get(0).getIndexId());
                    //System.out.println(index);
                    mpList.get(0).setBackground(Color.yellow);
                }
            }

            @Override
            public void mouseEntered(MouseEvent e) {
                // TODO Auto-generated method stub

            }

            @Override
            public void mouseExited(MouseEvent e) {
                // TODO Auto-generated method stub

            }

            @Override
            public void mousePressed(MouseEvent e) {
                // TODO Auto-generated method stub

            }

            @Override
            public void mouseReleased(MouseEvent e) {
                // TODO Auto-generated method stub

            }});
        jp.add(mpList.get(0));
        jp.setPreferredSize(new Dimension(500, 82*mpList.size()));
        //jp.validate();
        js.validate();
        //jp.repaint();
        js.repaint();
    }
    public void delMyPanel(){
        selectedId=getIndexOfMyPanelById();
        if(selectedId>=0){
            int indexId=getIndexOfMyPanelById();
            jp.remove(mpList.get(indexId));
            mpList.remove(indexId);
            jp.setPreferredSize(new Dimension(500,82*mpList.size()));
            jp.repaint();
        }
    }
    protected int getIndexOfMyPanelById(){
        Iterator<myPanel> it=mpList.iterator();
        for(int id=0;it.hasNext();id++){
            myPanel mp;
            mp=it.next();
            if(mp.isSelected()){
                return id;
            }
        }
        return -1;
    }
}

Here is code of myPanel

class myPanel extends JPanel{
    private static final long serialVersionUID = 1L;
    private JProgressBar downloadProgress;
    private JLabel fileTypeIconLabel,fileNameLabel,downloadInfoLabel,freeLabel;
    private int indexId;
    private boolean isSelected=false;
    protected myPanel(){
        setLayout(null);
        downloadProgress=new JProgressBar(0,100);
        fileTypeIconLabel=new JLabel("test");
        fileNameLabel=new JLabel("fileNameLabel");
        downloadInfoLabel=new JLabel("downloadInfoLabel");
        freeLabel=new JLabel("freeLabel");
        downloadProgress.setBounds(80, 44, 400, 18);
        downloadProgress.setStringPainted(true);
        //downloadProgress.setString("88%");
        fileTypeIconLabel.setBounds(0, 0, 80, 80);
        fileTypeIconLabel.setBackground(Color.cyan);
        fileTypeIconLabel.setOpaque(true);
        fileNameLabel.setBounds(80,0,400,22);
        fileNameLabel.setBackground(Color.black);
        fileNameLabel.setOpaque(true);
        downloadInfoLabel.setBounds(80, 22, 400, 22);
        downloadInfoLabel.setBackground(Color.red);
        downloadInfoLabel.setOpaque(true);
        //downloadProgress.setValue(50);
        freeLabel.setBounds(80, 62, 400, 18);
        freeLabel.setBackground(Color.lightGray);
        freeLabel.setOpaque(true);
        add(downloadProgress);
        add(fileTypeIconLabel);
        add(fileNameLabel);
        add(downloadInfoLabel);
        add(freeLabel); 

    }
    protected JLabel getFileTypeIconLabel() {
        return fileTypeIconLabel;
    }
    protected int getIndexId() {
        return indexId;
    }
    protected void setIndexId(int indexId) {
        this.indexId = indexId;
    }
    protected boolean isSelected() {
        return isSelected;
    }
    protected void setSelected(boolean isSelected) {
        this.isSelected = isSelected;
    }
}

For many components in one space, use a CardLayout as seen in this short example.

Tips

jp.setPreferredSize(new Dimension(500, 82*6));

See Should I avoid the use of set(Preferred|Maximum|Minimum)Size methods in Java Swing? (Yes.)

downloadProgress.setBounds(80, 44, 400, 18);

Java GUIs might have to work on a number of platforms, on different screen resolutions & using different PLAFs. As such they are not conducive to exact placement of components. To organize the components for a robust GUI, instead use layout managers, or combinations of them¹, along with layout padding & borders for white space².

Edit 1

As an aside, the screen-shot screams JList with custom renderer to me. E.G.

import java.awt.*;
import java.awt.event.ActionEvent;
import javax.swing.*;
import javax.swing.border.EmptyBorder;

public class DymanicDownloadList {

    // the GUI as seen by the user (without frame)
    JPanel gui = new JPanel(new BorderLayout(2, 2));

    public void initGUI() {
        gui.setBorder(new EmptyBorder(2, 3, 2, 3));

        JToolBar tb = new JToolBar();
        gui.add(tb, BorderLayout.PAGE_START);

        final DefaultListModel<Download> listModel =
                new DefaultListModel<Download>();
        final JList<Download> list = new JList<Download>(listModel);
        list.setCellRenderer(new DownloadListCellRenderer());
        list.setVisibleRowCount(3);
        gui.add(new JScrollPane(list), BorderLayout.CENTER);

        Action add = new AbstractAction("Add Download") {

            @Override
            public void actionPerformed(ActionEvent e) {
                listModel.addElement(new Download());
            }
        };

        Action delete = new AbstractAction("Delete Download") {

            @Override
            public void actionPerformed(ActionEvent e) {
                int index = list.getSelectedIndex();
                if (index < 0) {
                    JOptionPane.showMessageDialog(
                            list,
                            "Select a download to delete!",
                            "Select Download",
                            JOptionPane.ERROR_MESSAGE);
                } else {
                    listModel.removeElementAt(index);
                }
            }
        };

        tb.add(add);
        tb.addSeparator();
        tb.add(delete);

        for (int ii = 0; ii < 2; ii++) {
            listModel.addElement(new Download());
        }
    }

    public JComponent getGUI() {
        return gui;
    }

    public static void main(String[] args) {
        Runnable r = new Runnable() {

            @Override
            public void run() {
                DymanicDownloadList ddl = new DymanicDownloadList();
                ddl.initGUI();

                JFrame f = new JFrame("Dynamic LIST");
                f.add(ddl.getGUI());
                // Ensures JVM closes after frame(s) closed and
                // all non-daemon threads are finished
                f.setDefaultCloseOperation(JFrame.DISPOSE_ON_CLOSE);
                // See https://stackoverflow.com/a/7143398/418556 for demo.
                f.setLocationByPlatform(true);

                // ensures the frame is the minimum size it needs to be
                // in order display the components within it
                f.pack();
                // should be done last, to avoid flickering, moving,
                // resizing artifacts.
                f.setVisible(true);
            }
        };
        // Swing GUIs should be created and updated on the EDT
        // http://docs.oracle.com/javase/tutorial/uiswing/concurrency
        SwingUtilities.invokeLater(r);
    }
}

class Download {

    Download() {
    }
}

class DownloadListCellRenderer extends DefaultListCellRenderer {

    JPanel downloadPanel = new JPanel(new BorderLayout(10, 10));
    JPanel labelStack = new JPanel(new GridLayout(0, 1, 2, 2));
    JLabel number = new JLabel("", SwingConstants.CENTER);
    JLabel source = new JLabel("File Name Label", SwingConstants.CENTER);
    JLabel info = new JLabel("Download Info Label", SwingConstants.CENTER);
    JLabel free = new JLabel("Free Label", SwingConstants.CENTER);
    JProgressBar progress = new JProgressBar() {
        @Override
        public Dimension getPreferredSize() {
            Dimension d = super.getPreferredSize();

            return new Dimension(400,d.height);
        }
    };

    DownloadListCellRenderer() {
        downloadPanel.add(labelStack, BorderLayout.CENTER);
        labelStack.setOpaque(false);
        number.setFont(number.getFont().deriveFont(40f));
        labelStack.add(source);
        labelStack.add(info);
        labelStack.add(progress);
        labelStack.add(free);
        downloadPanel.add(labelStack, BorderLayout.CENTER);
        downloadPanel.add(number, BorderLayout.LINE_START);
    }

    @Override
    public Component getListCellRendererComponent(
            JList list,
            Object value,
            int index,
            boolean isSelected,
            boolean cellHasFocus) {
        JLabel l = (JLabel) super.getListCellRendererComponent(
                list, value, index, isSelected, cellHasFocus);
        downloadPanel.setBackground(l.getBackground());
        number.setText("" + (index + 1));

        return downloadPanel;
    }
}