Undecorated JFrame freezes when setting background to transparent at runtime in java swing, gnome desktop

Trying to improve my response, I just encountered some strange (not to say erroneous) behavior from Swing. Let's run the following example:

public class Gui extends JFrame {

   boolean setBackgroundA = true;
   Color opaque = new Color( 255, 0, 0, 255 ), transparent = new Color( 255, 0, 0, 50 );

   public Gui() {
      setUndecorated( true );
      setDefaultCloseOperation( javax.swing.WindowConstants.EXIT_ON_CLOSE );
      setBackground( new Color( 255, 0, 0, 50 ) );
      setBounds( 10, 10, 500, 500 );
      setVisible( true );
      addMouseListener( new java.awt.event.MouseAdapter() {
         @Override
         public void mouseClicked( java.awt.event.MouseEvent evt ) {
            if( setBackgroundA ) {
               setBackground( opaque );
            }
            else {
               setBackground( transparent );
            }
            setBackgroundA =  ! setBackgroundA;
         }
      } );
   }

   public static void main( String[] args ) {
      SwingUtilities.invokeLater( () -> {
         new Gui();
      } );
   }
}

When it starts, it shows us the transparent frame. When we click on it, it becomes completely opaque, and if we click again, something strange happens: it is painted in the default color and without transparency (then it will continue to alternate between those two states). However, if we change the value from opaque to new Color( 255, 0, 0, 254 ), the transition between opaque and transparent works correctly... so a solution to part of your problem could be to use an almost completely opaque setting.

Let's start with a small implementation that reproduces the behavior you need (at least part of it).

public class ShadowWindow {

   JFrame frame;
   Decorate decorator;
   BasicPanel panel;

     // We use these variables to control when the "JFrame" can be moved.
   boolean focused = true;
   private MouseEvent myEvent;

     // used to store the position of the frame
   private Point location;

   public ShadowWindow() {
      EventQueue.invokeLater( new Runnable() {
         @Override
         public void run() {
            try {
               UIManager.setLookAndFeel( UIManager.getSystemLookAndFeelClassName() );
            }
            catch( ClassNotFoundException
                    | InstantiationException
                    | IllegalAccessException
                    | UnsupportedLookAndFeelException ex ) {
            }
            frame = new JFrame();
            frame.setLayout( null );
            frame.setUndecorated( true );
            frame.setLocationRelativeTo( null );
            frame.setBounds( 100, 100, 500, 300 );
            frame.setBackground( new Color( 0, 0, 0, 0 ) );
            frame.setDefaultCloseOperation( JFrame.EXIT_ON_CLOSE );

            Rectangle aux = frame.getBounds();

            decorator = new Decorate();
            decorator.setBounds( 0, 0, aux.width, aux.height );

              // "set" sets the layout of the panel and creates 
              // the two images we will use
            decorator.set( null, new Color( 100, 100, 255 ) );

              // We instantiate the labels that will display 
              // the window icon and controls.
            decorator.setJLabels();

              // It is essential to set the panel this way to 
              // prevent transparency from accumulating.
            decorator.setOpaque( false );

            panel = new BasicPanel();
            panel.setFocusable( true );
            panel.setBounds( 4, 25, aux.width - 8, aux.height - 29 );
            panel.set( new GridBagLayout(), new Color( 180, 180, 180 ) );
            panel.setOpaque( false );

            panel.add( new JLabel( "Look ma, no hands" ) );
            decorator.add( panel );
            frame.add( decorator );
            frame.setVisible( true );
            addMouse( panel );
            addMouse( decorator );
         }
      } );
   }

   void addMouse( Decorate decorator) {
      decorator.addMouseListener( new MouseAdapter() {
         @Override
         public void mousePressed( MouseEvent evt ) {
            skipRepetitiveCode( false, evt );
         }

         @Override
         public void mouseReleased( MouseEvent evt ) {
            skipRepetitiveCode( true, evt );
         }
      } );

      decorator.addMouseMotionListener( new MouseAdapter() {
         @Override
         public void mouseDragged( MouseEvent evt ) {
            if( focused ) {
               location = frame.getLocation( location );
               int x = location.x - myEvent.getX() + evt.getX();
               int y = location.y - myEvent.getY() + evt.getY();
               frame.setLocation( x, y );
            }
         }
      } );
   }

   void skipRepetitiveCode( boolean bool, MouseEvent evt ) {

        // We set the variable “isOpaque” so that the method “paintComponent()” 
        // uses it to choose which image to paint.
      panel.isOpaque = bool;
      decorator.isOpaque = bool;

        // we force the repainting
      panel.repaint();
      decorator.repaint();

        // we set the captured event
      myEvent = evt;
   }

   void addMouse( BasicPanel panel ) {
      panel.addMouseListener( new MouseAdapter() {

           // The value assigned to “focused” will prevent dragging when 
           // the mouse is over the internal panel; dragging will only 
           // occur when the mouse is over the external panel.
         @Override
         public void mouseEntered( MouseEvent e ) {
            focused = false;
         }

         @Override
         public void mouseExited( MouseEvent e ) {
            focused = true;
         }
      } );
   }

   public static void main( String[] args ) {
      new ShadowWindow();
   }
}

public class BasicPanel extends JPanel {

   boolean isOpaque = true;
   BufferedImage imageOpaque, imageTranslucid;

   public void set( LayoutManager2 lay, Color color ) {
      setLayout( lay );

        // we create the most opaque image
      imageOpaque = new BufferedImage( getWidth(), getHeight(), 2 );
      Graphics2D g2d = ( Graphics2D ) imageOpaque.getGraphics();

       // set the transparency
      AlphaComposite ac = AlphaComposite.getInstance( AlphaComposite.SRC_OVER, .9f );
      g2d.setComposite( ac );

      g2d.setColor( color );
      g2d.fillRect( 0, 0, getWidth(), getHeight() );

      imageTranslucid = new BufferedImage( getWidth(), getHeight(), 2 );
      g2d = ( Graphics2D ) imageTranslucid.getGraphics();
      ac = AlphaComposite.getInstance( AlphaComposite.SRC_OVER, .2f );
      g2d.setComposite( ac );
      g2d.setColor( new Color( 180, 180, 180 ) );
      g2d.fillRect( 0, 0, getWidth(), getHeight() );
   }

   @Override
   public void paintComponent( Graphics g ) {
      super.paintComponent( g );
      Graphics2D g2d = ( Graphics2D ) g;

        // we choose which image to draw
      if( isOpaque ) {
         g2d.drawImage( imageOpaque, 0, 0, null );
      }
      else {
         g2d.drawImage( imageTranslucid, 0, 0, null );
      }
   }
}

public class Decorate extends BasicPanel {

   String letters[] = { " X", " 0", "  -", " W" };
   JLabel close, maximize, minimize, icon;   
   Color colors[] = {
      new Color( 100, 100, 50, 100 ),
      new Color( 100, 100, 50, 100 ),
      new Color( 100, 100, 50, 100 ),
      new Color( 0, 100, 50, 100 )
   };
   int bounds[][] = {
      { 480, 5, 18, 18 },
      { 460, 5, 18, 18 },
      { 440, 5, 18, 18 },
      { 5, 5, 18, 18 }
   };  

   public void setJLabels() {
      setLabels( close, maximize, minimize, icon );
   }  

   void setLabels( JLabel... comps ) {
      int i = 0;
      for( JLabel com: comps ) {
         com = new JLabel( letters[ i ] );
         com.setOpaque( true );
         com.setBounds( bounds[ i ][ 0 ], bounds[ i ][ 1 ],
                 bounds[ i ][ 2 ], bounds[ i ][ 3 ] );
         com.setBackground( colors[ i ] );
         add( com );
         i ++;
      }
   }
}

As I said in the comments, Java does not generate shadows for JFrame; this operation is performed by the OS window manager. In general (or in my limited experience), they are set not to show shadows when the frame has no decoration, meaning that the only way to show them is to modify the configuration (if this option is available).

Note A: I think it goes without saying that the monstrosity I implemented with the icon and controls needs to be replaced with what you have implemented.

Note B: You have made a substantial change to the code in your question. I based my answer on the previous code, and I am not going to adapt it to the new code because I believe you are skilled enough to do so yourself.

Final edition:
Your code has many problems. Let's go through it step by step:

public class mcve extends JFrame {

   private boolean isDragging = false;
   private int pressX = 0, pressY = 0;

   public mcve() {
      setDefaultCloseOperation( EXIT_ON_CLOSE );
      setLocationRelativeTo( null );
      setSize( 400, 180 );
      setLayout( null );
      setUndecorated( true );

        // To avoid the problems you mention, you should start by 
        // setting the frame as transparent.
      setBackground( new Color( 0, 0, 0, 0 ) );

      // Frame border (background panel)
      JLabel frameBorder = new JLabel();

        // Since the frame is transparent, we need the *JLabel* to be opaque.
      frameBorder.setOpaque( true );
      frameBorder.setBackground( new Color( 230, 230, 230 ) );
      frameBorder.setLayout( null );

        // You may want to replace "setLocation()" and "setSize()" 
        // with "setBounds( 0, 0, 400, 180 );" 
      frameBorder.setLocation( 0, 0 );
      frameBorder.setSize( 400, 180 );
      add( frameBorder );

      // Title label
      JLabel title = new JLabel( "Window Title" );
      title.setFont( new Font( "Segoe UI", Font.BOLD, 14 ) );
      title.setForeground( Color.DARK_GRAY );
      title.setLocation( 10, 0 );
      title.setSize( 330, 36 );
      frameBorder.add( title );

      JButton closeButton = new JButton( "X" );
      closeButton.setForeground( Color.WHITE );
      closeButton.setBackground( new Color( 200, 0, 0 ) );
      closeButton.setBorder( new LineBorder( Color.DARK_GRAY ) );
      closeButton.setFocusable( false );
      closeButton.setLocation( 371, 6 );
      closeButton.setSize( 23, 23 );
      frameBorder.add( closeButton );

        // I have removed the button listeners as they are not 
        // part of the problem.

      // Minimize button
      JButton minButton = new JButton( "-" );
      minButton.setForeground( Color.WHITE );
      minButton.setBackground( new Color( 130, 130, 130 ) );
      minButton.setBorder( new LineBorder( Color.DARK_GRAY ) );
      minButton.setFocusable( false );
      minButton.setLocation( 345, 6 );
      minButton.setSize( 23, 23 );
      frameBorder.add( minButton );

      // Inner content
      JLabel textContainer = new JLabel();
      textContainer.setLayout( null );
      textContainer.setBorder( new LineBorder( Color.GRAY ) );
      textContainer.setLocation( 5, 35 );
      textContainer.setSize( 390, 140 );
      frameBorder.add( textContainer );

      JLabel message = new JLabel( "<html>This is a dialog text. The quick brown fox jumps over the lazy dog.</html>" );
      message.setLocation( 20, 20 );
      message.setSize( 350, 80 );
      textContainer.add( message );

      title.addMouseListener( new java.awt.event.MouseAdapter() {
         public void mousePressed( java.awt.event.MouseEvent e ) {
            isDragging = true;
            pressX = e.getX();
            pressY = e.getY();
            textContainer.setVisible( false );

              // we make the buttons invisible
            minButton.setVisible( false );
            closeButton.setVisible( false );

              // You have added a listener to this component. When you set it 
              // with "setVisible( false )", the listener stops working, so we
              // replace it with the next line
            // title.setVisible( false );
            title.setForeground( new Color( 0, 0, 0, 0 ) );
            
            frameBorder.setBorder( new LineBorder( Color.black, 2 ) );
            frameBorder.setCursor( Cursor.getPredefinedCursor( Cursor.MOVE_CURSOR ) );
            title.setForeground( new Color( 0, 0, 0, 0 ) );
            frameBorder.setBackground( new Color( 0, 0, 0, 0 ) );
            setBackground( new Color( 0, 0, 0, 0 ) );

              // delete the nexts lines for being redundant 
            // getRootPane().setBackground(new Color(0,0,0,0));
            // getContentPane().setBackground(new Color(0, 0, 0, 0));
         }

         public void mouseReleased( java.awt.event.MouseEvent e ) {
            isDragging = false;
            frameBorder.setCursor( Cursor.getPredefinedCursor( Cursor.DEFAULT_CURSOR ) );
            frameBorder.setBorder( new LineBorder( Color.GRAY, 0 ) );

              // We restore the original color (with its “alpha”).
            title.setForeground( Color.darkGray );
            textContainer.setVisible( true );

              // we make the buttons visible
            minButton.setVisible( true );
            closeButton.setVisible( true );

              // we make the "frameBorder" and "frame" opaque
            frameBorder.setBackground( new Color( 230, 230, 230 ) );
            setBackground( new Color( 0, 0, 0, 255 ) );
         }
      } );

      title.addMouseMotionListener( new java.awt.event.MouseMotionAdapter() {
         public void mouseDragged( java.awt.event.MouseEvent e ) {
            if( isDragging ) {
               setLocation( e.getXOnScreen() - pressX, e.getYOnScreen() - pressY );
            }
         }
      } );
   }

   public static void main( String[] args ) {
      SwingUtilities.invokeLater( () -> new mcve().setVisible( true ) );
   }
}

Beyond having made it work, I still think that the previous example is the way to go.