path: root/src/DirectConnection.cpp

#include "../include/sibs/DirectConnection.hpp"
#include "../include/Log.hpp"
#include <cstdio>
#include <cstring>
#include <random>
#include <sibs/SafeSerializer.hpp>
#include <sibs/SafeDeserializer.hpp>

#ifndef WIN32
   #include <arpa/inet.h>
   #include <netdb.h>
#else
   #include <winsock2.h>
   #include <ws2tcpip.h>
#endif
#include <udt/udt.h>

namespace sibs
{
    static std::random_device rd;
    static std::mt19937 gen(rd());

    static u32 generateRandomNumber(u32 start, u32 end)
    {
        std::uniform_int_distribution<> dis(start, end);
        return dis(gen);
    }

    // Max received data size allowed when receiving regular data, receive data as file to receive more data
    const int MAX_RECEIVED_DATA_SIZE = 1024 * 1024 * 1; // 1Mb

    bool DirectConnectionPeer::operator == (const DirectConnectionPeer &other) const
    {
        return socket->udtSocket == other.socket->udtSocket;
    }

    bool DirectConnectionPeer::operator != (const DirectConnectionPeer &other) const
    {
        return !(*this == other);
    }
    
    DirectConnections::DirectConnections(u16 _port) : 
        port(_port == 0 ? (u16)generateRandomNumber(2000, 32000) : _port),
        alive(true),
        removeDisconnectedPeerCallback(nullptr)
    {
        UDT::startup();
        eid = UDT::epoll_create();
        receiveDataThread = std::thread(&DirectConnections::receiveData, this);
    }
    
    DirectConnections::~DirectConnections()
    {
        alive = false;
        peers.clear();
        UDT::epoll_release(eid);
        UDT::cleanup();
        receiveDataThread.join();
    }
    
    std::unique_ptr<Socket> DirectConnections::createSocket(const Ipv4 &addressToBind, bool rendezvous, bool reuseAddr, bool bind)
    {
        Log::debug("UDT: Creating socket for ipv4 address %s, port: %d", addressToBind.getAddress().c_str(), addressToBind.getPort());
        UDTSOCKET udtSocket = UDT::socket(AF_INET, SOCK_STREAM, 0);
        if(udtSocket == UDT::INVALID_SOCK)
        {
            std::string errMsg = "UDT: Failed to create socket, error: ";
            errMsg += UDT::getlasterror_desc();
            throw SocketCreateException(errMsg);
        }
        
        auto socket = std::make_unique<Socket>(udtSocket);
        UDT::setsockopt(udtSocket, 0, UDT_RENDEZVOUS, &rendezvous, sizeof(bool));
        UDT::setsockopt(udtSocket, 0, UDT_REUSEADDR, &reuseAddr, sizeof(bool));
        
        // Windows UDP issue
        // For better performance, modify HKLM\System\CurrentControlSet\Services\Afd\Parameters\FastSendDatagramThreshold
        #ifdef WIN32
        int mss = 1052;
        UDT::setsockopt(socket, 0, UDT_MSS, &mss, sizeof(mss));
        #endif

        if(rendezvous || bind)
        {
            if(reuseAddr)
            {
                /*
                if(UDT::bind(socket, (sockaddr*)&addressToBind.address, sizeof(addressToBind.address)) == UDT::ERROR)
                {
                    std::string errMsg = "UDT: Failed to bind, error: ";
                    errMsg += UDT::getlasterror_desc();
                    throw SocketCreateException(errMsg);
                }
                */
                Ipv4 myAddr = addressToBind;
                for(int i = 0; i < 2000; ++i)
                {
                    if(UDT::bind(udtSocket, (sockaddr*)&myAddr.address, sizeof(myAddr.address)) == UDT::ERROR)
                    {
                        u16 newPort = (u16)generateRandomNumber(2000, 32000);
                        Log::warn("DirectConnections: failed to bind socket to port %d, trying port %d. Fail reason: %s", port, newPort, UDT::getlasterror_desc());
                        port = newPort;
                        myAddr.address.sin_port = htons(port);
                    }
                    else
                        return socket;
                }
                throw SocketCreateException("UDT: Failed to bind after 2000 tries");
            }
            else
            {
                Ipv4 myAddr = addressToBind;
                for(int i = 0; i < 2000; ++i)
                {
                    if(UDT::bind(udtSocket, (sockaddr*)&myAddr.address, sizeof(myAddr.address)) == UDT::ERROR)
                    {
                        port = (u16)generateRandomNumber(2000, 32000);
                        myAddr.address.sin_port = htons(port);
                    }
                    else
                        return socket;
                }
                throw SocketCreateException("UDT: Failed to bind after 2000 tries");
            }
        }
        
        return socket;
    }
    
    void DirectConnections::connectServer(const Ipv4 &address, PubSubConnectCallback connectCallbackFunc, PubSubReceiveDataCallback receiveDataCallbackFunc)
    {
        connect(address, false, true, connectCallbackFunc, receiveDataCallbackFunc, true);
    }
    
    void DirectConnections::connect(const Ipv4 &address, PubSubConnectCallback connectCallbackFunc, PubSubReceiveDataCallback receiveDataCallbackFunc)
    {
        connect(address, true, true, connectCallbackFunc, receiveDataCallbackFunc, true);
    }
    
    void DirectConnections::connect(const Ipv4 &address, bool rendezvous, bool reuseAddr, PubSubConnectCallback connectCallbackFunc, PubSubReceiveDataCallback receiveDataCallbackFunc, bool bind)
    {
        std::thread([this, address, rendezvous, reuseAddr, connectCallbackFunc, receiveDataCallbackFunc, bind]()
        {
            std::shared_ptr<DirectConnectionPeer> peer = getPeerByAddress(address);
            if(peer)
            {
                // this doesn't really matter, we always call connect with same callback function
                peer->receiveDataCallbackFunc = receiveDataCallbackFunc;
                if(connectCallbackFunc)
                    connectCallbackFunc(peer, PubSubResult::OK, "");
                return;
            }
            else
            {
                peer = std::make_shared<DirectConnectionPeer>();
                peerByAddressMap[address] = peer;
            }
            
            std::unique_ptr<Socket> socket;
            try
            {
                socket = createSocket(Ipv4(nullptr, port), rendezvous, reuseAddr, bind);
            }
            catch(SocketCreateException &e)
            {
                if(connectCallbackFunc)
                    connectCallbackFunc(peer, PubSubResult::ERROR, e.what());
                return;
            }
            
            Log::debug("DirectConnections: Connecting to peer (ip: %s, port: %d, rendezvous: %s)", address.getAddress().c_str(), address.getPort(), rendezvous ? "yes" : "no");
            if(UDT::connect(socket->udtSocket, (sockaddr*)&address.address, sizeof(address.address)) == UDT::ERROR)
            {
                if(connectCallbackFunc)
                    connectCallbackFunc(peer, PubSubResult::ERROR, UDT::getlasterror_desc());
                return;
            }
            
            UDT::epoll_add_usock(eid, socket->udtSocket);
            socket->eid = eid;
            peersMutex.lock();
            peers[socket->udtSocket] = peer;
            peer->socket = std::move(socket);
            peer->address = address;
            peer->receiveDataCallbackFunc = receiveDataCallbackFunc;
            peersMutex.unlock();
            
            if(connectCallbackFunc)
                connectCallbackFunc(peer, PubSubResult::OK, "");
        }).detach();
    }
    
    bool DirectConnections::send(const std::shared_ptr<DirectConnectionPeer> peer, std::shared_ptr<Message> data, PubSubSendDataCallback sendDataCallbackFunc)
    {
        if(data->getDataSize() == 0)
            return true;

        if(data->getDataSize() > 819200) // 800kb
            return false;

        // TODO: Replace this with light-weight threads (fibers)?
        std::thread([peer, data, sendDataCallbackFunc]()
        {
            int sentSize = UDT::send(peer->socket->udtSocket, (char*)data->data(), data->getRawSize(), 0);
            if(sentSize == UDT::ERROR)
            {
                if(sendDataCallbackFunc)
                    sendDataCallbackFunc(PubSubResult::ERROR, UDT::getlasterror_desc());
            }
            else if(sendDataCallbackFunc)
                sendDataCallbackFunc(PubSubResult::OK, "");
        }).detach();
        return true;
    }

    void DirectConnections::onRemoveDisconnectedPeer(PubSubOnRemoveDisconnectedPeerCallback callbackFunc)
    {
        removeDisconnectedPeerCallback = callbackFunc;
    }
    
    bool DirectConnections::removePeer(int peerSocket)
    {
        bool wasRemoved = false;
        std::lock_guard<std::mutex> lock(peersMutex);
        auto peerIt = peers.find(peerSocket);
        if(peerIt != peers.end())
        {
            if(removeDisconnectedPeerCallback)
                removeDisconnectedPeerCallback(peerIt->second);
            peers.erase(peerIt);
            wasRemoved = true;
        }
        return wasRemoved;
    }

    std::vector<std::shared_ptr<DirectConnectionPeer>> DirectConnections::getPeers()
    {
        std::vector<std::shared_ptr<DirectConnectionPeer>> result;
        result.reserve(peers.size());
        std::lock_guard<std::mutex> lock(peersMutex);
        for(auto &it : peers)
        {
            result.push_back(it.second);
        }
        return result;
    }
    
    void DirectConnections::removeDisconnectedPeers()
    {
        peersMutex.lock();
        for(std::unordered_map<int, std::shared_ptr<DirectConnectionPeer>>::iterator it = peers.begin(); it != peers.end(); )
        {
            int socket = it->first;
            UDTSTATUS peerSocketStatus = UDT::getsockstate(socket);
            if(peerSocketStatus == UDTSTATUS::BROKEN || peerSocketStatus == UDTSTATUS::CLOSING || peerSocketStatus == UDTSTATUS::CLOSED || peerSocketStatus == UDTSTATUS::NONEXIST)
            {
                if(removeDisconnectedPeerCallback)
                    removeDisconnectedPeerCallback(it->second);

                Log::debug("UDT: Connection was broken to socket %d (peer most likely disconnected), removing peer", socket);
                it = peers.erase(it);
                Log::debug("UDT: Removed peer socket %d", socket);
            }
            else
                ++it;
        }
        peersMutex.unlock();
    }

    std::shared_ptr<DirectConnectionPeer> DirectConnections::getPeerByAddress(const Ipv4 &address) const
    {
        auto it = peerByAddressMap.find(address);
        if(it != peerByAddressMap.end())
            return it->second;
        return nullptr;
    }
    
    void DirectConnections::receiveData()
    {
        std::vector<char> data;
        data.resize(MAX_RECEIVED_DATA_SIZE);
        
        Log::debug("DirectConnections::receiveData(): waiting for events");
        std::set<UDTSOCKET> readfds;
        while(alive)
        {
            removeDisconnectedPeers();
            int numfsReady = UDT::epoll_wait(eid, &readfds, nullptr, 1000);
            if(numfsReady == 0)
            {
                continue;
            }
            else if(numfsReady == -1)
            {
                if(!alive)
                    continue;
                    
                if(UDT::getlasterror_code() == UDT::ERRORINFO::ETIMEOUT)
                {
                    continue;
                }
                else
                {
                    Log::error("UDT: Stop receiving data, got error: %s", UDT::getlasterror_desc());
                    return;
                }
            }
            
            for(UDTSOCKET receivedDataFromPeer : readfds)
            {
                bool peerDisconnected = false;
                usize receivedTotalSize = 0;
                int receivedDataStatus = receiveDataFromPeer(receivedDataFromPeer, data.data(), &receivedTotalSize);
                if(receivedDataStatus == 0)
                {
                    peersMutex.lock();
                    auto peer = peers[receivedDataFromPeer];
                    peersMutex.unlock();
                    try
                    {
                        Log::debug("DirectConnection: Received data from peer: (ip: %s, port: %d)", peer->address.getAddress().c_str(), peer->address.getPort());
                        if(peer->receiveDataCallbackFunc && receivedTotalSize > 0)
                        {
                            static_assert(sizeof(MessageType) == sizeof(u8), "");
                            MessageType messageType = (MessageType)data[0];
                            peer->receiveDataCallbackFunc(peer, messageType, data.data() + 1, receivedTotalSize - 1);
                        }
                    }
                    catch(std::exception &e)
                    {
                        Log::error("UDT: Receive callback function threw exception: %s, ignoring...", e.what());
                    }
                }
                else if(receivedDataStatus == CUDTException::EINVSOCK)
                {
                    Log::debug("UDT: Invalid socket %d, did remote peer disconnect?", receivedDataFromPeer);
                    peerDisconnected = true;
                }
                else if(receivedDataStatus == CUDTException::ECONNLOST)
                {
                    Log::debug("UDT: Connection was broken to socket %d (peer most likely disconnected), removing peer", receivedDataFromPeer);
                    peerDisconnected = true;
                }
                
                if(peerDisconnected)
                {
                    if(removePeer(receivedDataFromPeer))
                        Log::debug("UDT: Removed peer socket %d", receivedDataFromPeer);
                    else
                        Log::error("UDT: Failed to remove peer socket %d, system said we got data from it but we are not connected to it", receivedDataFromPeer);
                }
            }
            readfds.clear();
        }
    }
    
    int DirectConnections::receiveDataFromPeer(const int socket, char *output, usize *receivedTotalSize)
    {
        *receivedTotalSize = 0;
        while(*receivedTotalSize < MAX_RECEIVED_DATA_SIZE)
        {
            int dataAvailableSize;
            int receiveSizeDataTypeSize = sizeof(dataAvailableSize);
            if(UDT::getsockopt(socket, 0, UDT_RCVDATA, &dataAvailableSize, &receiveSizeDataTypeSize) == UDT::ERROR)
            {
                Log::error("UDT: Failed to receive data available size, error: %s (%d)", UDT::getlasterror_desc(), UDT::getlasterror_code());
                return UDT::getlasterror_code();
            }
            
            if(dataAvailableSize == 0)
                return 0;
            
            int receivedSize = UDT::recv(socket, &output[*receivedTotalSize], MAX_RECEIVED_DATA_SIZE - *receivedTotalSize, 0);
            if(receivedSize == UDT::ERROR)
            {
                Log::error("UDT: Failed to receive data, error: %s (%d)", UDT::getlasterror_desc(), UDT::getlasterror_code());
                return UDT::getlasterror_code();
            }
            
            if(receivedSize == 0)
            {
                return 0;
            }
            
            (*receivedTotalSize) += receivedSize;
        }
        
        Log::error("UDT: Received too much data, ignoring...");
        return 0;
    }

    std::vector<u8> DirectConnectionsUtils::serializePeers(const std::vector<std::shared_ptr<DirectConnectionPeer>> &peers)
    {
        sibs::SafeSerializer serializer;
        for(const auto &it : peers)
        {
            serializer.add((u32)it->address.address.sin_family);
            serializer.add((u32)it->address.address.sin_addr.s_addr);
            serializer.add((u16)it->address.address.sin_port);
        }
        return serializer.getBuffer();
    }

    std::vector<std::shared_ptr<DirectConnectionPeer>> DirectConnectionsUtils::deserializePeers(const u8 *data, const usize size)
    {
        std::vector<std::shared_ptr<DirectConnectionPeer>> result;
        sibs::SafeDeserializer deserializer(data, size);
        while(!deserializer.empty())
        {
            std::shared_ptr<DirectConnectionPeer> peer = std::make_shared<DirectConnectionPeer>();
            peer->address.address.sin_family = deserializer.extract<u32>();
            peer->address.address.sin_addr.s_addr = deserializer.extract<u32>();
            peer->address.address.sin_port = deserializer.extract<u32>();
            result.push_back(peer);
        }
        return result;
    }
}