TCP socket server hangs randomly and rsa Decryption error raised randomly

I'm trying to make a server for a password manager I'm building, but the server seems to hang randomly and it seems to randomly throw rsa.pkcs1.DecryptionErrors randomly too. I did recently switch from sending an END message to the server that then just made the server stop expecting stuff, to having a test script I wrote close the connection and then the server checks if socket.fileno() == -1. Maybe that's throwing a wrench in things. But it was doing this even before I switched. Right now, most of the time its throwing a fit trying to decode the password. Here is the server code:

import json

import rsa
import socket
import os.path
import netifaces
import json_repair

def get_connection_data():
    if os.path.exists("data/data.json"):
        connection_data = tuple(json_repair.from_file("data/data.json"))

    else:
        interface = netifaces.interfaces()[int(input(f"Enter the index of the interface you want to use. Eg, to use {netifaces.interfaces()[0]} enter 1\n{netifaces.interfaces()}\n")) - 1]
        port = input("What port do you want to user? Press enter for the default \n")

        if port == "":
            port = 9000

        connection_data = (
            netifaces.ifaddresses(interface)[netifaces.AF_INET][0]["addr"],
            port
        )

        if not os.path.exists("data"):
            os.mkdir("data")

        with open("data/data.json", mode="w") as connection_file:
            json.dump(connection_data, connection_file, indent=4)

    return connection_data

def register_user(user_client, device_address: float):
    print("register_user called")

    # Make sure data path exists
    if not os.path.exists("data"):
        os.mkdir("data")

    if not os.path.exists(f"data/{device_address}"):
        os.mkdir(f"data/{device_address}")

    all_passwords = {}

    # receive all passwords
    while True:
        service = rsa.decrypt(user_client.recv(1024), priv_key=private_key).decode()

        print("Service is:" + service)


        while service != "END":
            user_client.send("READY".encode())

            username = rsa.decrypt(user_client.recv(1024), priv_key=private_key).decode()

            print("Username is: " + username)

            user_client.send("READY".encode())

            print("Sent ready after receiving username")

            password = rsa.decrypt(user_client.recv(1024), priv_key=private_key).decode()
            print(password)
            user_client.send("READY".encode())

            print("Sent ready after receiving password")

            key = rsa.decrypt(user_client.recv(1024), priv_key=private_key)
            user_client.send("READY".encode())

            print("Sent ready after receiving key")

            if user_client.fileno() == -1:
                break

        if user_client.fileno() == -1:
            break

    all_passwords[service] = {
        username: {
            "password": password,
            "key": key.decode()
        }
    }

    with open(f"data/{device_address}/.passwords.json", mode="w") as passwords_file:
        json.dump(all_passwords, passwords_file, indent=4)

public_key, private_key = rsa.newkeys(1024)
client_public_key = None

server = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
server.bind(get_connection_data())
server.listen()

print(f"Listening on {get_connection_data()[0]}:{get_connection_data()[1]}")

while True:
    client, ip_address = server.accept()

    print("Connection accepted")

    client.send(public_key.save_pkcs1("PEM"))
    client_public_key = rsa.PublicKey.load_pkcs1(client.recv(1024))

    print("Received and sent Public keys")

    action = client.recv(1024)# .decode("utf-8")
    # print("Received Action")

    # print(action)

    client.send("READY".encode())
    print("Sent ready message")


    if action.decode().lower() == "register user":
        print("Inside if statement")
        register_user(user_client=client, device_address=ip_address[0])

And here is the test script I put together:

import socket
import time

import rsa

IP_ADDRESS = input("Enter server IP: ")
PORT = 9000

pub_key, priv_key = rsa.newkeys(1024)
server_pub_key = None

server = socket.socket(socket.AF_INET, socket.SOCK_STREAM)

print("Connecting to server")

server.connect((IP_ADDRESS, PORT))

print("Receiving server public key and sending public key")

server_pub_key = rsa.PublicKey.load_pkcs1(server.recv(1024))
server.send(pub_key.save_pkcs1("PEM"))

message = "register user"

server.send(message.encode("utf-8"))
print("Sent action")

time.sleep(1)

print("Waiting for ready message")
server.recv(1024)
print("Received ready message")

all_passwords = {
        "service": {
            "username": {
                "password": "password",
                "key": "key"
                },
            "username2": {
                "password": "password2",
                "key": "key2",
                }
            },

            "service2": {
                "username": {
                    "password": "password",
                    "key": "key"
                },

                "username2": {
                    "password": "password2",
                    "key": "key2",
                }
            }
        }

for service in all_passwords:
    service_data = all_passwords[service]

    service = service

    print("Sending service")

    server.send(rsa.encrypt(service.encode(), pub_key=server_pub_key))
    server.recv(1024)

    for username in service_data.keys():
        username = username
        password = service_data[username]["password"]
        key = service_data[username]["key"]

        print(f"Service: {service} \nUsername: {username} \nPassword: {password} \nKey: {key}")

        server.send(rsa.encrypt(username.encode(), pub_key=server_pub_key))
        server.recv(1024)

        print("Received ready after sending username")

        server.send(rsa.encrypt(password.encode(), pub_key=server_pub_key))
        server.recv(1024)

        print("Received ready after sending password")

        server.send(rsa.encrypt(key.encode(), pub_key=server_pub_key))
        server.recv(1024)

        print("received ready after sending key")

server.close()

If anyone knows any better ways to do this, please let me know

I've checked the public and private keys probably 4 times by now, and I've tried going line by line through my code, but nothing caught my eye. I also did some brief searches on brave, found a thread where this was happening in cpython, clicked on it, and immediately went back cause it went straight over my head. Also, something I noticed is when I print the decoded password, using this code:

print(rsa.decrypt(user_client.recv(1024), priv_key=private_key).decode())
return

It runs without a problem, properly decrypting the password. Most of the time. Every once in a while, it also will give me a fit about decrypting the password

EDIT: I also tried sending an END message from the test script and then having the server close the connection, but that didn't change anything