Golang project and postgres image with docker compose up doesn't fail, but doesn't work either

not sure what I did wrong here. I'm trying to make a golang/postgres dockerized project with a persistent db. Below are the files. When I run go run main.go then curl http://localhost:8081/ I get the expected output, but when I try this with docker compose up I'm having issues, but everything seems to be working because I don't see any error messages postgres-1 | 2022-08-29 05:31:59.703 UTC [1] LOG: database system is ready to accept connections, but when I try curl http://localhost:8081/ I'm getting an error curl: (56) Recv failure: Connection reset by peer. I tried removing the postgres part entirely and I'm still getting the same problem. I can see that docker is up and running and the port is listening

sudo lsof -i -P -n | grep 8081
docker-pr 592069            root    4u  IPv4 1760430      0t0  TCP *:8081 (LISTEN)

I'm using this on Ubuntu 22.04

main.go:

package main

import (
    "fmt"
    "log"
    "net/http"
)

func main() {

    http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
        fmt.Fprintf(w, "this can be anything")
    })

    http.HandleFunc("/try-it", func(w http.ResponseWriter, r *http.Request) {
        fmt.Fprintf(w, "it worked!")
    })

    log.Fatal(http.ListenAndServe(":8081", nil))

}

Dockerfile:

FROM golang:1.19-alpine3.16

WORKDIR /app

COPY go.mod ./
RUN go mod download

COPY . .

RUN go build -o main .

RUN go build -v -o /usr/local/bin/app ./...

EXPOSE 8081
CMD [ "./main" ]

docker-compose.yml:

version: '3.9'
services:
  api:
    restart: unless-stopped
    build:
      context: .
      dockerfile: Dockerfile
    ports:
      - "8081:8080"
    depends_on:
      - postgres
    networks:
      - backend
  postgres:
    image: postgres
    restart: unless-stopped
    ports:
      - "5001:5432"
    volumes:
      - psqlVolume:/var/lib/postgresql/data
    environment:
      - POSTGRES_USER=${POSTGRES_USER}
      - POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
      - POSTGRES_DB=${POSTGRES_DB}
    networks:
      - backend

networks:
  backend:

volumes:
  psqlVolume: