added a memory management class for small objects.

git-svn-id: http://svn.code.sf.net/p/retroshare/code/trunk@4046 b45a01b8-16f6-495d-af2f-9b41ad6348cc
This commit is contained in:
csoler 2011-02-20 21:36:21 +00:00
parent db19db7e60
commit 827dbd8cf3
8 changed files with 538 additions and 4 deletions

View File

@ -26,7 +26,6 @@
/******
* #define FT_DEBUG 1
*****/
#define FT_DEBUG 1
#include "retroshare/rsturtle.h"
#include "fttransfermodule.h"

View File

@ -405,6 +405,7 @@ HEADERS += upnp/upnphandler.h
HEADERS += util/folderiterator.h \
util/rsdebug.h \
util/smallobject.h \
util/rsdir.h \
util/rsdiscspace.h \
util/rsnet.h \
@ -525,6 +526,7 @@ SOURCES += upnp/upnphandler.cc
SOURCES += util/folderiterator.cc \
util/rsdebug.cc \
util/smallobject.cc \
util/rsdir.cc \
util/rsdiscspace.cc \
util/rsnet.cc \

View File

@ -26,8 +26,11 @@
#include "serialiser/rsbaseserial.h"
#include "serialiser/rsserial.h"
#include "util/rsthreads.h"
#include <math.h>
#include <map>
#include <vector>
#include <iostream>
/***
@ -44,6 +47,68 @@ RsItem::RsItem(uint32_t t)
return;
}
#ifdef DO_STATISTICS
class Counter
{
public:
Counter(int i): _i(i) {}
Counter(): _i(0) {}
int v() const { return _i ; }
int& v() { return _i ; }
private:
int _i ;
};
static RsMutex smtx ;
static std::map<int,Counter> size_hits ;
static int nb_rsitem_creations = 0 ;
static int total_rsitem_mallocs = 0 ;
static int total_rsitem_frees = 0 ;
static int total_rsitem_freed = 0 ;
static time_t last_time = 0 ;
void *RsItem::operator new(size_t s)
{
// std::cerr << "New RsItem: s=" << s << std::endl;
RsStackMutex m(smtx) ;
++size_hits[ s ].v() ;
time_t now = time(NULL);
++nb_rsitem_creations ;
total_rsitem_mallocs += s ;
if(last_time + 20 < now)
{
std::cerr << "Memory statistics:" << std::endl;
std::cerr << " Total RsItem memory: " << total_rsitem_mallocs << std::endl;
std::cerr << " Total RsItem creations: " << nb_rsitem_creations << std::endl;
std::cerr << " Total RsItem freed memory: " << total_rsitem_freed << std::endl;
std::cerr << " Total RsItem deletions: " << total_rsitem_frees << std::endl;
std::cerr << "Now printing histogram:" << std::endl;
for(std::map<int,Counter>::const_iterator it(size_hits.begin());it!=size_hits.end();++it)
std::cerr << it->first << " " << it->second.v() << std::endl;
last_time = now ;
}
RsItem *a = static_cast<RsItem*>(::operator new(s)) ;
return a ;
}
void RsItem::operator delete(void *p,size_t s)
{
// std::cerr << "Delete RsItem: s=" << s << std::endl;
RsStackMutex m(smtx) ;
total_rsitem_freed += s ;
++total_rsitem_frees ;
::operator delete(p) ;
}
#endif
RsItem::RsItem(uint8_t ver, uint8_t cls, uint8_t t, uint8_t subtype)
{
type = (ver << 24) + (cls << 16) + (t << 8) + subtype;

View File

@ -56,6 +56,8 @@
* 8 bits: SubType
******************************************************************/
#include <util/smallobject.h>
const uint8_t RS_PKT_VERSION1 = 0x01;
const uint8_t RS_PKT_VERSION_SERVICE = 0x02;
@ -65,11 +67,15 @@ const uint8_t RS_PKT_CLASS_CONFIG = 0x02;
const uint8_t RS_PKT_SUBTYPE_DEFAULT = 0x01; /* if only one subtype */
class RsItem
class RsItem: public RsMemoryManagement::SmallObject
{
public:
RsItem(uint32_t t);
RsItem(uint8_t ver, uint8_t cls, uint8_t t, uint8_t subtype);
#ifdef DO_STATISTICS
void *operator new(size_t s) ;
void operator delete(void *,size_t s) ;
#endif
virtual ~RsItem();
virtual void clear() = 0;

View File

@ -7,8 +7,8 @@ RS_TOP_DIR = ../..
include $(RS_TOP_DIR)/tests/scripts/config.mk
###############################################################
TESTOBJ = netsetup_test.o random_test.o
TESTS = netsetup_test random_test
TESTOBJ = netsetup_test.o random_test.o memory_management_test.o
TESTS = netsetup_test random_test memory_management_test
all: tests
@ -18,6 +18,9 @@ netsetup_test: netsetup_test.o
random_test: random_test.o
$(CC) $(CFLAGS) -o random_test random_test.o $(LIBS)
memory_management_test: memory_management_test.o
$(CC) $(CFLAGS) -o memory_management_test memory_management_test.o $(LIBS)
###############################################################
include $(RS_TOP_DIR)/tests/scripts/rules.mk
###############################################################

View File

@ -0,0 +1,61 @@
#ifdef LINUX
#include <fenv.h>
#endif
#include <iostream>
#include <vector>
#include <stdlib.h>
#include <math.h>
#include "util/utest.h"
#include "serialiser/rsserial.h"
INITTEST();
// Make a fake class of size n.
//
template<int n> class RsTestItem: public RsItem
{
public:
unsigned char buff[n] ;
RsTestItem(uint32_t s) : RsItem(s) {}
virtual void clear() {}
virtual std::ostream& print(std::ostream& o, uint16_t) { return o ; }
};
int main(int argc, char **argv)
{
#ifdef LINUX
feenableexcept(FE_INVALID) ;
feenableexcept(FE_DIVBYZERO) ;
#endif
typedef RsTestItem<17> Test17 ;
typedef RsTestItem<31> Test31 ;
std::vector<RsItem*> v;
for(int i=0;i<300;++i)
v.push_back(new Test17(0)) ;
for(int i=0;i<700;++i)
v.push_back(new Test31(0)) ;
RsMemoryManagement::printStatistics() ;
// Now delete objects randomly.
//
for(int i=0;i<1000;++i)
{
int indx = lrand48()%(int)v.size() ;
delete v[indx] ;
v[indx] = v.back() ;
v.pop_back() ;
}
std::cerr << "After memory free: " << std::endl;
RsMemoryManagement::printStatistics() ;
FINALREPORT("memory_management_test");
exit(TESTRESULT());
}

View File

@ -0,0 +1,285 @@
#include <iostream>
#include "smallobject.h"
#include "util/rsthreads.h"
using namespace RsMemoryManagement ;
RsMutex SmallObject::_mtx ;
SmallObjectAllocator SmallObject::_allocator(RsMemoryManagement::MAX_SMALL_OBJECT_SIZE) ;
void Chunk::init(size_t blockSize,unsigned char blocks)
{
_data = new unsigned char[blockSize*blocks] ;
_firstAvailableBlock = 0 ;
_blocksAvailable = blocks ;
// Inits the first byte of each block to point to the next available block
//
unsigned char *p = _data ;
for(unsigned char i=0;i<blocks;p += blockSize)
*p = ++i ;
}
void Chunk::free()
{
delete[] _data ;
_data = NULL ;
}
void *Chunk::allocate(size_t blockSize)
{
if(!_blocksAvailable)
return NULL ;
assert(blockSize >= 2) ;
unsigned char *result = _data + _firstAvailableBlock*blockSize ;
// Grab id of next available block from the block being allocated.
// Always the case because the first available block is the first, so the next
// available block is given by *result.
//
_firstAvailableBlock = *result ;
--_blocksAvailable ;
return result ;
}
void Chunk::deallocate(void *p,size_t blockSize)
{
assert(p >= _data) ;
unsigned char *toRelease = static_cast<unsigned char *>(p) ;
// alignment check
assert( (toRelease - _data) % blockSize == 0 ) ;
*toRelease = _firstAvailableBlock ;
_firstAvailableBlock = static_cast<unsigned char>( (toRelease - _data)/blockSize) ;
// truncation check
assert(_firstAvailableBlock == (toRelease - _data)/blockSize);
++_blocksAvailable ;
}
void Chunk::printStatistics(int blockSize) const
{
std::cerr << " blocksAvailable : " << (int)_blocksAvailable << std::endl;
std::cerr << " firstBlockAvailable: " << (int)_firstAvailableBlock << std::endl;
std::cerr << " blocks : " << (void*)_data << " to " << (void*)(_data+BLOCKS_PER_CHUNK*blockSize) << std::endl;
}
FixedAllocator::FixedAllocator(size_t bytes)
{
_blockSize = bytes ;
_numBlocks = BLOCKS_PER_CHUNK ;
_allocChunk = -1 ;
_deallocChunk = -1 ;
}
FixedAllocator::~FixedAllocator()
{
for(uint32_t i=0;i<_chunks.size();++i)
{
_chunks[i]->free() ;
delete _chunks[i] ;
}
}
void *FixedAllocator::allocate()
{
if(_allocChunk < 0 || _chunks[_allocChunk]->_blocksAvailable == 0)
{
// find availabel memory in this chunk
//
uint32_t i ;
_allocChunk = -1 ;
for(i=0;i<_chunks.size();++i)
if(_chunks[i]->_blocksAvailable > 0) // found a chunk
{
_allocChunk = i ;
break ;
}
if( _allocChunk < 0 )
{
_chunks.reserve(_chunks.size()+1) ;
Chunk *newChunk = new Chunk;
if(newChunk == NULL)
{
std::cerr << "RsMemoryManagement: ran out of memory !" << std::endl;
exit(-1) ;
}
newChunk->init(_blockSize,_numBlocks) ;
_chunks.push_back(newChunk) ;
_allocChunk = _chunks.size()-1 ;
_deallocChunk = _chunks.size()-1 ;
}
}
assert(_chunks[_allocChunk] != NULL) ;
assert(_chunks[_allocChunk]->_blocksAvailable > 0) ;
return _chunks[_allocChunk]->allocate(_blockSize) ;
}
void FixedAllocator::deallocate(void *p)
{
if(_deallocChunk < 0 || !chunkOwnsPointer(*_chunks[_deallocChunk],p))
{
// find the chunk that contains this pointer. Perform a linear search.
_deallocChunk = -1 ;
for(uint32_t i=0;i<_chunks.size();++i)
if(chunkOwnsPointer(*_chunks[i],p))
{
_deallocChunk = i ;
break ;
}
}
assert(_chunks[_deallocChunk] != NULL) ;
_chunks[_deallocChunk]->deallocate(p,_blockSize) ;
if(_chunks[_deallocChunk]->_blocksAvailable == BLOCKS_PER_CHUNK)
{
_chunks[_deallocChunk]->free() ;
delete _chunks[_deallocChunk] ;
_chunks[_deallocChunk] = _chunks.back() ;
if(_allocChunk == _deallocChunk) _allocChunk = -1 ;
if(_allocChunk == ((int)_chunks.size())-1) _allocChunk = _deallocChunk ;
_deallocChunk = -1 ;
_chunks.pop_back();
}
}
void FixedAllocator::printStatistics() const
{
std::cerr << " numBLocks=" << (int)_numBlocks << std::endl;
std::cerr << " blockSize=" << (int)_blockSize << std::endl;
std::cerr << " Number of chunks: " << _chunks.size() << std::endl;
for(uint32_t i=0;i<_chunks.size();++i)
_chunks[i]->printStatistics(_blockSize) ;
}
SmallObjectAllocator::SmallObjectAllocator(size_t maxObjectSize)
: _maxObjectSize(maxObjectSize)
{
_lastAlloc = NULL ;
_lastDealloc = NULL ;
}
SmallObjectAllocator::~SmallObjectAllocator()
{
for(std::map<int,FixedAllocator*>::const_iterator it(_pool.begin());it!=_pool.end();++it)
delete it->second ;
}
void *SmallObjectAllocator::allocate(size_t bytes)
{
if(bytes > _maxObjectSize)
return malloc(bytes) ;
else if(_lastAlloc != NULL && _lastAlloc->blockSize() == bytes)
return _lastAlloc->allocate() ;
else
{
std::map<int,FixedAllocator*>::iterator it(_pool.find(bytes)) ;
if(it == _pool.end())
{
_pool[bytes] = new FixedAllocator(bytes) ;
it = _pool.find(bytes) ;
}
_lastAlloc = it->second ;
return it->second->allocate() ;
}
}
void SmallObjectAllocator::deallocate(void *p,size_t bytes)
{
if(bytes > _maxObjectSize)
free(p) ;
else if(_lastDealloc != NULL && _lastDealloc->blockSize() == bytes)
_lastDealloc->deallocate(p) ;
else
{
std::map<int,FixedAllocator*>::iterator it(_pool.find(bytes)) ;
if(it == _pool.end())
{
_pool[bytes] = new FixedAllocator(bytes) ;
it = _pool.find(bytes) ;
}
it->second->deallocate(p) ;
_lastDealloc = it->second ;
}
}
void SmallObjectAllocator::printStatistics() const
{
std::cerr << "RsMemoryManagement Statistics:" << std::endl;
std::cerr << " Total Fixed-size allocators: " << _pool.size() << std::endl;
std::cerr << "Pool" << std::endl;
for(std::map<int,FixedAllocator*>::const_iterator it(_pool.begin());it!=_pool.end();++it)
{
std::cerr << " Allocator for size " << it->first << " : " << std::endl;
std::cerr << " Last Alloc: " << _lastAlloc << std::endl;
std::cerr << " Last Dealloc: " << _lastDealloc << std::endl;
it->second->printStatistics() ;
}
}
void *SmallObject::operator new(size_t size)
{
#ifdef DEBUG_MEMORY
bool print=false ;
{
RsStackMutex m(_mtx) ;
static time_t last_time = 0 ;
time_t now = time(NULL) ;
if(now > last_time + 20)
{
last_time = now ;
print=true ;
}
}
if(print)
printStatistics() ;
#endif
RsStackMutex m(_mtx) ;
void *p = _allocator.allocate(size) ;
#ifdef DEBUG_MEMORY
std::cerr << "new RsItem: " << p << ", size=" << size << std::endl;
#endif
return p ;
}
void SmallObject::operator delete(void *p,size_t size)
{
RsStackMutex m(_mtx) ;
_allocator.deallocate(p,size) ;
#ifdef DEBUG_MEMORY
std::cerr << "del RsItem: " << p << ", size=" << size << std::endl;
#endif
}
void SmallObject::printStatistics()
{
RsStackMutex m(_mtx) ;
_allocator.printStatistics() ;
}
void RsMemoryManagement::printStatistics()
{
SmallObject::printStatistics();
}

View File

@ -0,0 +1,113 @@
/*
* libretroshare/src/util: smallobject.h
*
* Universal Networking Header for RetroShare.
*
* Copyright 2011 by Cyril Soler
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Library General Public
* License Version 2 as published by the Free Software Foundation.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Library General Public License for more details.
*
* You should have received a copy of the GNU Library General Public
* License along with this library; if not, write to the Free Software
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
* USA.
*
* Please report all bugs and problems to "cyril.soler@users.sourceforge.net".
*
*/
#include <stdlib.h>
#include <assert.h>
#include <vector>
#include <map>
#include <util/rsthreads.h>
namespace RsMemoryManagement
{
static const int MAX_SMALL_OBJECT_SIZE = 128 ;
static const unsigned char BLOCKS_PER_CHUNK = 255 ;
struct Chunk
{
void init(size_t blockSize,unsigned char blocks);
void free() ;
void *allocate(size_t);
void deallocate(void *p,size_t blockSize);
unsigned char *_data ;
unsigned char _firstAvailableBlock ;
unsigned char _blocksAvailable ;
void printStatistics(int blockSize) const ;
};
class FixedAllocator
{
public:
FixedAllocator(size_t bytes) ;
virtual ~FixedAllocator() ;
void *allocate();
void deallocate(void *p) ;
inline size_t blockSize() const { return _blockSize ; }
inline bool chunkOwnsPointer(const Chunk& c,void *p) const
{
return p >= c._data && (static_cast<unsigned char *>(p)-c._data)/_blockSize < _numBlocks ;
}
void printStatistics() const ;
private:
size_t _blockSize ;
unsigned char _numBlocks ;
std::vector<Chunk*> _chunks ;
int _allocChunk ; // last chunk that provided allocation. -1 if not inited
int _deallocChunk ; // last chunk that provided de-allocation. -1 if not inited
};
class SmallObjectAllocator
{
public:
SmallObjectAllocator(size_t maxObjectSize) ;
virtual ~SmallObjectAllocator() ;
void *allocate(size_t numBytes) ;
void deallocate(void *p,size_t size) ;
void printStatistics() const ;
private:
std::map<int,FixedAllocator*> _pool ;
FixedAllocator *_lastAlloc ;
FixedAllocator *_lastDealloc ;
size_t _maxObjectSize ;
};
class SmallObject
{
public:
static void *operator new(size_t size) ;
static void operator delete(void *p,size_t size) ;
static void printStatistics() ;
virtual ~SmallObject() {}
private:
static SmallObjectAllocator _allocator ;
static RsMutex _mtx;
};
extern void printStatistics() ;
}