Invisible components still take up space JPanel

I have a series of components underneath each other in a JPanel set as a GridLayout. I need to temporarily hide the components but setVisible(false) doesn't cut it, because there is still an empty gap where the components were.

Is there a quick and easy way to do this? Or do I have to stay saving the state of the JPanel, removing the components, then restoring it?

SSCCE:

[GridLayout2.java]

import java.awt.Component;
import java.awt.Container;
import java.awt.Dimension;
import java.awt.GridLayout;
import java.awt.Insets;

public class GridLayout2 extends GridLayout 
{
  public GridLayout2() {
    this(1, 0, 0, 0);
  }

  public GridLayout2(int rows, int cols) {
    this(rows, cols, 0, 0);
  }

  public GridLayout2(int rows, int cols, int hgap, int vgap) {
    super(rows, cols, hgap, vgap);
  }

  public Dimension preferredLayoutSize(Container parent) {
    //System.err.println("preferredLayoutSize");
    synchronized (parent.getTreeLock()) {
      Insets insets = parent.getInsets();
      int ncomponents = parent.getComponentCount();
      int nrows = getRows();
      int ncols = getColumns();
      if (nrows > 0) {
        ncols = (ncomponents + nrows - 1) / nrows;
      } 
      else {
        nrows = (ncomponents + ncols - 1) / ncols;
      }
      int[] w = new int[ncols];
      int[] h = new int[nrows];
      for (int i = 0; i < ncomponents; i ++) {
        int r = i / ncols;
        int c = i % ncols;
        Component comp = parent.getComponent(i);
        Dimension d = comp.getPreferredSize();
        if (w[c] < d.width) {
          w[c] = d.width;
        }
        if (h[r] < d.height) {
          h[r] = d.height;
        }
      }
      int nw = 0;
      for (int j = 0; j < ncols; j ++) {
        nw += w[j];
      }
      int nh = 0;
      for (int i = 0; i < nrows; i ++) {
        nh += h[i];
      }
      return new Dimension(insets.left + insets.right + nw + (ncols