monero/contrib/epee/include/net/abstract_tcp_server_cp.inl

// Copyright (c) 2006-2013, Andrey N. Sabelnikov, www.sabelnikov.net
// All rights reserved.
// 
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are met:
// * Redistributions of source code must retain the above copyright
// notice, this list of conditions and the following disclaimer.
// * Redistributions in binary form must reproduce the above copyright
// notice, this list of conditions and the following disclaimer in the
// documentation and/or other materials provided with the distribution.
// * Neither the name of the Andrey N. Sabelnikov nor the
// names of its contributors may be used to endorse or promote products
// derived from this software without specific prior written permission.
// 
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER  BE LIABLE FOR ANY
// DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
// 


#pragma comment(lib, "Ws2_32.lib")

#undef MONERO_DEFAULT_LOG_CATEGORY
#define MONERO_DEFAULT_LOG_CATEGORY "net"

namespace epee
{
namespace net_utils
{
template<class TProtocol> 
cp_server_impl<TProtocol>::cp_server_impl():
									m_port(0), m_stop(false),
									m_worker_thread_counter(0), m_listen_socket(INVALID_SOCKET)
{
}
//-------------------------------------------------------------
template<class TProtocol>
cp_server_impl<TProtocol>::~cp_server_impl()
{
	deinit_server();
}
//-------------------------------------------------------------
template<class TProtocol> 
bool cp_server_impl<TProtocol>::init_server(int port_no)
{
	m_port = port_no;
	
	WSADATA wsad = {0};
	int err = ::WSAStartup(MAKEWORD(2,2), &wsad);
	if ( err != 0  || LOBYTE( wsad.wVersion ) != 2 || HIBYTE( wsad.wVersion ) != 2 )
	{
		LOG_ERROR("Could not find a usable WinSock DLL, err = " << err << " \"" << socket_errors::get_socket_error_text(err) <<"\"");
		return false;
	}
	
	m_initialized = true;

	m_listen_socket = ::WSASocket(AF_INET, SOCK_STREAM, IPPROTO_TCP, NULL, 0, WSA_FLAG_OVERLAPPED);
	if(INVALID_SOCKET == m_listen_socket)
	{
		err = ::WSAGetLastError();
		LOG_ERROR("Failed to create socket, err = " << err << " \"" << socket_errors::get_socket_error_text(err) <<"\"");
		return false;
	}


	int opt = 1;
	err = setsockopt (m_listen_socket, SOL_SOCKET,SO_REUSEADDR, reinterpret_cast<char*>(&opt), sizeof(int));
	if(SOCKET_ERROR == err )
	{
		err = ::WSAGetLastError();
		LOG_PRINT("Failed to setsockopt(SO_REUSEADDR), err = " << err << " \"" << socket_errors::get_socket_error_text(err) <<"\"", LOG_LEVEL_1);
		deinit_server();
		return false;
	}


	sockaddr_in adr = {0};
	adr.sin_family = AF_INET;
	adr.sin_addr.s_addr = htonl(INADDR_ANY);
	adr.sin_port = (u_short)htons(m_port);

	//binding
	err = bind(m_listen_socket, (const sockaddr*)&adr, sizeof(adr ));
	if(SOCKET_ERROR == err )
	{
		err = ::WSAGetLastError();
		LOG_PRINT("Failed to Bind, err = " << err << " \"" << socket_errors::get_socket_error_text(err) <<"\"", LOG_LEVEL_1);
		deinit_server();
		return false;
	}


	m_completion_port = ::CreateIoCompletionPort(INVALID_HANDLE_VALUE, NULL, 0, 0);
	if(INVALID_HANDLE_VALUE == m_completion_port)
	{
		err = ::WSAGetLastError();
		LOG_PRINT("Failed to CreateIoCompletionPort, err = " << err << " \"" << socket_errors::get_socket_error_text(err) <<"\"", LOG_LEVEL_1);
		deinit_server();
		return false;
	}


	return true;
}
//-------------------------------------------------------------

//-------------------------------------------------------------
static int CALLBACK CPConditionFunc(
						   IN LPWSABUF lpCallerId,
						   IN LPWSABUF lpCallerData,
						   IN OUT LPQOS lpSQOS,
						   IN OUT LPQOS lpGQOS,
						   IN LPWSABUF lpCalleeId,
						   OUT LPWSABUF lpCalleeData,
						   OUT GROUP FAR *g,
						   IN DWORD_PTR dwCallbackData
						   )
{

	/*cp_server_impl* pthis = (cp_server_impl*)dwCallbackData;
	if(!pthis)
		return CF_REJECT;*/
	/*if(pthis->get_active_connections_num()>=FD_SETSIZE-1)
	{
		LOG_PRINT("Maximum connections count overfull.", LOG_LEVEL_2);
		return CF_REJECT;
	}*/
	
	return  CF_ACCEPT;
}
//-------------------------------------------------------------
template<class TProtocol>
size_t cp_server_impl<TProtocol>::get_active_connections_num()
{
	return m_connections.size();
}
//-------------------------------------------------------------
template<class TProtocol>
unsigned CALLBACK cp_server_impl<TProtocol>::worker_thread(void* param)
{
	if(!param)
		return 0;

	cp_server_impl<TProtocol>* pthis = (cp_server_impl<TProtocol>*)param;
	pthis->worker_thread_member();
	return 1;
}
//-------------------------------------------------------------
template<class TProtocol> 
bool cp_server_impl<TProtocol>::worker_thread_member()
{
	LOG_PRINT("Worker thread STARTED", LOG_LEVEL_1);
	bool stop_handling = false;
	while(!stop_handling)
	{
		PROFILE_FUNC("[worker_thread]Worker Loop");
		DWORD bytes_transfered = 0;
		connection<TProtocol>* pconnection = 0;
		io_data_base* pio_data = 0;
		
		{
			PROFILE_FUNC("[worker_thread]GetQueuedCompletionStatus");
			BOOL res = ::GetQueuedCompletionStatus (m_completion_port, &bytes_transfered ,	(PULONG_PTR)&pconnection, (LPOVERLAPPED *)&pio_data, INFINITE);
			if (res == 0)
			{
				// check return code for error
				int err = GetLastError();
				LOG_PRINT("GetQueuedCompletionStatus failed with error " << err << " " << log_space::get_win32_err_descr(err), LOG_LEVEL_1);

				if(pio_data)
					::InterlockedExchange(&pio_data->m_is_in_use, 0);


				continue;
			}
		}

		if(pio_data)
			::InterlockedExchange(&pio_data->m_is_in_use, 0);



		if(!bytes_transfered && !pconnection && !pio_data)
		{
			//signal to stop
			break;
		}
		if(!pconnection || !pio_data)
		{
			LOG_PRINT("BIG FAIL: pconnection or pio_data is empty: pconnection=" << pconnection << " pio_data=" << pio_data, LOG_LEVEL_0);
			break;
		}



		if(::InterlockedCompareExchange(&pconnection->m_connection_shutwoned, 0, 0))
		{
			LOG_ERROR("InterlockedCompareExchange(&pconnection->m_connection_shutwoned, 0, 0)");
			//DebugBreak();
		}

		if(pio_data->m_op_type == op_type_stop)
		{
			if(!pconnection)
			{
				LOG_ERROR("op_type=op_type_stop, but pconnection is empty!!!");
				continue;
			}
			shutdown_connection(pconnection);
			continue;//
		}
		else if(pio_data->m_op_type == op_type_send)
		{
			continue;
			//do nothing, just queuing request
		}else if(pio_data->m_op_type == op_type_recv)
		{
			PROFILE_FUNC("[worker_thread]m_tprotocol_handler.handle_recv");
			if(bytes_transfered)
			{
				bool res = pconnection->m_tprotocol_handler.handle_recv(pio_data->Buffer, bytes_transfered);
				if(!res)
					pconnection->query_shutdown();
			}
			else
			{
				pconnection->query_shutdown();
				continue;
			}

		}

		//preparing new request, 

		{
			PROFILE_FUNC("[worker_thread]RECV Request small loop");
			int res = 0;
			while(true)
			{
				LOG_PRINT("Prepearing data for WSARecv....", LOG_LEVEL_3);
				ZeroMemory(&pio_data->m_overlapped, sizeof(OVERLAPPED));
				pio_data->DataBuf.len = pio_data->TotalBuffBytes;
				pio_data->DataBuf.buf = pio_data->Buffer;
				pio_data->m_op_type = op_type_recv;
				//calling WSARecv() and go to completion waiting
				DWORD bytes_recvd = 0;
				DWORD flags = 0;

				LOG_PRINT("Calling WSARecv....", LOG_LEVEL_3);
				::InterlockedExchange(&pio_data->m_is_in_use, 1);
				res = WSARecv(pconnection->m_sock, &(pio_data->DataBuf), 1, &bytes_recvd , &flags, &(pio_data->m_overlapped), NULL);
				if(res == SOCKET_ERROR )
				{
					int err = ::WSAGetLastError();
					if(WSA_IO_PENDING == err )
					{//go pending, ok
						LOG_PRINT("WSARecv return WSA_IO_PENDING", LOG_LEVEL_3);
						break;
					}
					LOG_ERROR("BIG FAIL: WSARecv error code not correct, res=" << res << " last_err=" << err);
					::InterlockedExchange(&pio_data->m_is_in_use, 0);
					pconnection->query_shutdown();
					break;
				}
				break;
				/*else if(0 == res)
				{
					if(!bytes_recvd)
					{
						::InterlockedExchange(&pio_data->m_is_in_use, 0);
						LOG_PRINT("WSARecv return 0, bytes_recvd=0, graceful close.", LOG_LEVEL_3);
						int err = ::WSAGetLastError();
						//LOG_ERROR("BIG FAIL: WSARecv error code not correct, res=" << res << " last_err=" << err);
						//pconnection->query_shutdown();
						break;
					}else
					{
						LOG_PRINT("WSARecv return immediatily 0, bytes_recvd=" << bytes_recvd, LOG_LEVEL_3);
						//pconnection->m_tprotocol_handler.handle_recv(pio_data->Buffer, bytes_recvd);
					}
				}*/
			}
		}
	}

	
	LOG_PRINT("Worker thread STOPED", LOG_LEVEL_1);
	::InterlockedDecrement(&m_worker_thread_counter);
	return true;
}
//-------------------------------------------------------------
template<class TProtocol> 
bool cp_server_impl<TProtocol>::shutdown_connection(connection<TProtocol>* pconn)
{
	PROFILE_FUNC("[shutdown_connection]");
	
	if(!pconn)
	{
		LOG_ERROR("Attempt to remove null pptr connection!");
		return false;
	}
	else 
	{
		LOG_PRINT("Shutting down connection ("<< pconn << ")", LOG_LEVEL_3);
	}
	m_connections_lock.lock();
	connections_container::iterator it = m_connections.find(pconn->m_sock);
	m_connections_lock.unlock();
	if(it == m_connections.end())
	{
		LOG_ERROR("Failed to find closing socket=" << pconn->m_sock);
		return false;
	}
	SOCKET sock = it->second->m_sock;
	{
		PROFILE_FUNC("[shutdown_connection] shutdown, close");
		::shutdown(it->second->m_sock,  SD_SEND );
	}
	size_t close_sock_wait_count = 0;
	{
		LOG_PRINT("Entered to 'in_use wait zone'", LOG_LEVEL_3);
		PROFILE_FUNC("[shutdown_connection] wait for in_use");
		while(::InterlockedCompareExchange(&it->second->m_precv_data->m_is_in_use, 1, 1))
		{

			Sleep(100);
			close_sock_wait_count++;
		}
		LOG_PRINT("First step to 'in_use wait zone'", LOG_LEVEL_3);


		while(::InterlockedCompareExchange(&it->second->m_psend_data->m_is_in_use, 1, 1))
		{
			Sleep(100);
			close_sock_wait_count++;
		}
		LOG_PRINT("Leaved 'in_use wait zone'", LOG_LEVEL_3);
	}

	::closesocket(it->second->m_sock);

	::InterlockedExchange(&it->second->m_connection_shutwoned, 1);
	m_connections_lock.lock();
	m_connections.erase(it);
	m_connections_lock.unlock();
	LOG_PRINT("Socked " << sock << " closed, wait_count=" << close_sock_wait_count, LOG_LEVEL_2);
	return true;
}
//-------------------------------------------------------------
template<class TProtocol> 
bool cp_server_impl<TProtocol>::run_server(int threads_count = 0)
{
	int err = listen(m_listen_socket, 100);
	if(SOCKET_ERROR == err )
	{
		err = ::WSAGetLastError();
		LOG_ERROR("Failed to listen, err = " << err << " \"" << socket_errors::get_socket_error_text(err) <<"\"");
		return false;
	}

	if(!threads_count)
	{
		SYSTEM_INFO si = {0};
		::GetSystemInfo(&si);
		threads_count = si.dwNumberOfProcessors + 2;
	}
	for(int i = 0; i != threads_count; i++)
	{
		boost::thread(boost::bind(&cp_server_impl::worker_thread_member, this));
		//HANDLE h_thread = threads_helper::create_thread(worker_thread, this);
		InterlockedIncrement(&m_worker_thread_counter);
		//::CloseHandle(h_thread);
	}

	LOG_PRINT("Numbers of worker threads started: " << threads_count, LOG_LEVEL_1);

	m_stop = false;
	while(!m_stop)
	{
		PROFILE_FUNC("[run_server] main_loop");
		TIMEVAL tv = {0};
		tv.tv_sec = 0;
		tv.tv_usec = 100;
		fd_set sock_set;
		sock_set.fd_count = 1;
		sock_set.fd_array[0] = m_listen_socket;
		int select_res = 0;
		{
			PROFILE_FUNC("[run_server] select");
			select_res = select(0, &sock_set, &sock_set, NULL, &tv);
		}
		
		if(SOCKET_ERROR == select_res)
		{
			err = ::WSAGetLastError();
			LOG_ERROR("Failed to select, err = " << err << " \"" << socket_errors::get_socket_error_text(err) <<"\"");
			return false;
		}
		if(!select_res)
		{
			on_net_idle();
			continue;
		}
		else
		{
			sockaddr_in adr_from = {0};
			int adr_len = sizeof(adr_from);
			SOCKET new_sock = INVALID_SOCKET;
			{
				PROFILE_FUNC("[run_server] WSAAccept");
				new_sock = ::WSAAccept(m_listen_socket, (sockaddr *)&adr_from, &adr_len, CPConditionFunc, (DWORD_PTR)this);
			}
			
			if(INVALID_SOCKET == new_sock)
			{
				if(m_stop)
					break;
				int err = ::WSAGetLastError();
				LOG_PRINT("Failed to WSAAccept, err = " << err << " \"" << socket_errors::get_socket_error_text(err) <<"\"", LOG_LEVEL_2);
				continue;
			}
			LOG_PRINT("Accepted connection (new socket=" << new_sock << ")", LOG_LEVEL_2);
			{
				PROFILE_FUNC("[run_server] Add new connection");
				add_new_connection(new_sock, adr_from.sin_addr.s_addr, adr_from.sin_port);
			}
			
		}

	}
	LOG_PRINT("Closing connections("<< m_connections.size() << ") and waiting...", LOG_LEVEL_2);
	m_connections_lock.lock();
	for(connections_container::iterator it = m_connections.begin(); it != m_connections.end(); it++)
	{
		::shutdown(it->second->m_sock, SD_BOTH);
		::closesocket(it->second->m_sock);
	}
	m_connections_lock.unlock();
	size_t wait_count = 0;
	while(m_connections.size() && wait_count < 100)
	{
		::Sleep(100);
		wait_count++;
	}
	LOG_PRINT("Connections closed OK (wait_count=" << wait_count << ")", LOG_LEVEL_2);


	LOG_PRINT("Stopping worker threads("<< m_worker_thread_counter << ").", LOG_LEVEL_2);
	for(int i = 0; i<m_worker_thread_counter; i++)
	{
		::PostQueuedCompletionStatus(m_completion_port, 0, 0, 0);
	}

	wait_count = 0;
	while(InterlockedCompareExchange(&m_worker_thread_counter, 0, 0) && wait_count < 100)
	{
		Sleep(100);
		wait_count++;
	}

	LOG_PRINT("Net Server STOPPED, wait_count = " << wait_count, LOG_LEVEL_1);
	return true;
}
//-------------------------------------------------------------
template<class TProtocol>
bool cp_server_impl<TProtocol>::add_new_connection(SOCKET new_sock, long ip_from, int port_from)
{
	PROFILE_FUNC("[add_new_connection]");
	
	LOG_PRINT("Add new connection zone: entering lock", LOG_LEVEL_3);
	m_connections_lock.lock();

	boost::shared_ptr<connection<TProtocol> > ptr;
	ptr.reset(new connection<TProtocol>(m_config));

	connection<TProtocol>& conn = *ptr.get();
	m_connections[new_sock] = ptr;
	LOG_PRINT("Add new connection zone: leaving lock", LOG_LEVEL_3);
	m_connections_lock.unlock();
	conn.init_buffers();
	conn.m_sock = new_sock;
	conn.context.m_remote_ip = ip_from;
	conn.context.m_remote_port = port_from;
	conn.m_completion_port = m_completion_port;
	{
		PROFILE_FUNC("[add_new_connection] CreateIoCompletionPort");
		::CreateIoCompletionPort((HANDLE)new_sock, m_completion_port, (ULONG_PTR)&conn, 0);
	}

	//if(NULL == ::CreateIoCompletionPort((HANDLE)new_sock, m_completion_port, (ULONG_PTR)&conn, 0))
	//{
    //	int err = ::GetLastError();
	//	LOG_PRINT("Failed to CreateIoCompletionPort(associate socket and completion port), err = " << err << " \"" << socket_errors::get_socket_error_text(err) <<"\"", LOG_LEVEL_2);
	//	return false;
	//}

	conn.m_tprotocol_handler.after_init_connection();
	{
		PROFILE_FUNC("[add_new_connection] starting loop");
		int res = 0;
		while(true)//res!=SOCKET_ERROR)
		{
			PROFILE_FUNC("[add_new_connection] in loop time");
			conn.m_precv_data->TotalBuffBytes = LEVIN_DEFAULT_DATA_BUFF_SIZE;
			ZeroMemory(&conn.m_precv_data->m_overlapped, sizeof(OVERLAPPED));
			conn.m_precv_data->DataBuf.len = conn.m_precv_data->TotalBuffBytes;
			conn.m_precv_data->DataBuf.buf = conn.m_precv_data->Buffer;
			conn.m_precv_data->m_op_type = op_type_recv;
			InterlockedExchange(&conn.m_precv_data->m_is_in_use, 1);
			DWORD bytes_recvd = 0;
			DWORD flags = 0;

			::InterlockedExchange(&conn.m_precv_data->m_is_in_use, 1);
			{
				PROFILE_FUNC("[add_new_connection] ::WSARecv");
				res = ::WSARecv(conn.m_sock, &(conn.m_precv_data->DataBuf), 1, &bytes_recvd , &flags, &(conn.m_precv_data->m_overlapped), NULL);
			}
			if(res == SOCKET_ERROR )
			{
				int err = ::WSAGetLastError();
				if(WSA_IO_PENDING == err )
				{
					break;
				}
				LOG_ERROR("BIG FAIL: WSARecv error code not correct, res=" << res << " last_err=" << err << " " << log_space::get_win32_err_descr(err));
				::InterlockedExchange(&conn.m_precv_data->m_is_in_use, 0);
				conn.query_shutdown();
				//shutdown_connection(&conn);
				break;
			}
			

			break;
			/*else if(0 == res)
			{
				if(!bytes_recvd)
				{
					PROFILE_FUNC("[add_new_connection] shutdown_connection");
					::InterlockedExchange(&conn.m_precv_data->m_is_in_use, 0);
					conn.query_shutdown();
					//shutdown_connection(&conn);
					break;
				}else
				{
					PROFILE_FUNC("[add_new_connection] handle_recv");
				}
			}*/
		}
	}



	return true;
}
//-------------------------------------------------------------
template<class TProtocol>
bool cp_server_impl<TProtocol>::deinit_server()
{
	if(!m_initialized)
		return true;

	if(INVALID_SOCKET != m_listen_socket)
	{
		shutdown(m_listen_socket, SD_BOTH);
		int res = closesocket(m_listen_socket);
		if(SOCKET_ERROR == res)
		{
			int err = ::WSAGetLastError();
			LOG_ERROR("Failed to closesocket(), err = " << err << " \"" << socket_errors::get_socket_error_text(err) <<"\"");
		}
		m_listen_socket = INVALID_SOCKET;
	}

	int res = ::WSACleanup();
	if(SOCKET_ERROR == res)
	{
		int err = ::WSAGetLastError();
		LOG_ERROR("Failed to WSACleanup(), err = " << err << " \"" << socket_errors::get_socket_error_text(err) <<"\"");
	}
	m_initialized = false;

	return true;
}

//-------------------------------------------------------------
template<class TProtocol>
bool cp_server_impl<TProtocol>::send_stop_signal()
{
	::InterlockedExchange(&m_stop, 1);
	return true;
}
//-------------------------------------------------------------
template<class TProtocol>
bool cp_server_impl<TProtocol>::is_stop_signal()
{
	return m_stop?true:false;	
}
//-------------------------------------------------------------
}
}