Adding varnish server to nginx using hostname rather than IP address

Trying to add varnish to nginx using a docker container name rather than IP address.

I've tried adding it directly set_real_ip_from site-varnish but that doesn't work.

Tried adding an upstream (below) and tried set_real_ip_from varnish_backend with no luck

upstream varnish_backend {
    server site-varnish;
}

Any help would be appreciated. I've added below the current working conf for reference.

upstream fastcgi_backend {
    server site-fpm;
}
server {
    listen 80;
    listen 443 ssl;
    server_name localhost;
    location = /ping {
        set_real_ip_from      192.168.176.2;
        real_ip_header        X-Forwarded-For;

        access_log            off;
        allow                 127.0.0.1;
        deny                  all;


        fastcgi_param         SCRIPT_FILENAME  $document_root$fastcgi_script_name;
        include               fastcgi_params;
        fastcgi_pass          fastcgi_backend;
    }
}

docker-compose.yml

version: "2"
services:
  site-varnish:
    build:
      context: ./etc/varnish/
    ports:
      - 80
    networks:
      - frontend

  site-web:
    build:
      context: ./etc/nginx/
    volumes_from:
      - site-appdata
    env_file:
      - ./global.env
    restart: always
    networks:
      - backend
      - frontend

  site-fpm:
    build:
      context: ./etc/7.2-fpm/
    ports:
      - 9000
    volumes_from:
      - site-appdata
    env_file:
      - ./global.env
    networks: 
      - backend

  site-appdata:
    image: tianon/true
    volumes:
      - ./html:/var/www/html

networks:
  frontend:
    external:
      name: webproxy
  backend:
    external:
      name: backbone