From 120a27d9c713808a5d81852eda4686ccc4bea390 Mon Sep 17 00:00:00 2001 From: csoler Date: Sun, 6 Oct 2013 20:12:34 +0000 Subject: [PATCH] - added RsCompress with memory compression methods based on ZLib (already linked) - added test code for RsCompress methods - used compression to compress file lists so as to reduce bandwidth usage git-svn-id: http://svn.code.sf.net/p/retroshare/code/branches/v0.6-initdev@6809 b45a01b8-16f6-495d-af2f-9b41ad6348cc --- libretroshare/src/dbase/findex.cc | 112 ++++++--- libretroshare/src/libretroshare.pro | 2 + libretroshare/src/tests/util/Makefile | 6 +- libretroshare/src/tests/util/compress_test.cc | 217 ++++++++++++++++++ retroshare-gui/src/gui/MessagesDialog.cpp | 4 +- 5 files changed, 306 insertions(+), 35 deletions(-) create mode 100644 libretroshare/src/tests/util/compress_test.cc diff --git a/libretroshare/src/dbase/findex.cc b/libretroshare/src/dbase/findex.cc index 9fb7c63ab..f3ba81bde 100644 --- a/libretroshare/src/dbase/findex.cc +++ b/libretroshare/src/dbase/findex.cc @@ -26,6 +26,7 @@ #include "retroshare/rsexpr.h" #include "util/rsdir.h" #include "util/rsstring.h" +#include "util/rscompress.h" #include #include @@ -746,9 +747,10 @@ int FileIndex::printFileIndex(std::string &out) return 1; } -int FileIndex::loadIndex(const std::string& filename, const std::string& expectedHash, uint64_t /*size*/) +int FileIndex::loadIndex(const std::string& filename, const std::string& expectedHash, uint64_t size) { - std::ifstream file (filename.c_str(), std::ifstream::binary); + FILE *file = RsDirUtil::rs_fopen(filename.c_str(),"rb") ; + if (!file) { #ifdef FI_DEBUG @@ -759,28 +761,36 @@ int FileIndex::loadIndex(const std::string& filename, const std::string& expecte } /* load file into memory, close file */ - char ibuf[512]; - std::string s; - while(!file.eof()) - { - file.read(ibuf, 512); - s.append(ibuf, file.gcount()); - } - file.close(); + uint8_t *compressed_data = new uint8_t[size] ; - /* calculate hash */ - unsigned char sha_buf[SHA_DIGEST_LENGTH]; - SHA_CTX *sha_ctx = new SHA_CTX; - SHA1_Init(sha_ctx); - SHA1_Update(sha_ctx, s.c_str(), s.length()); - SHA1_Final(&sha_buf[0], sha_ctx); - delete sha_ctx; - - std::string tmpout; - for(int i = 0; i < SHA_DIGEST_LENGTH; i++) + if(compressed_data == NULL) { - rs_sprintf_append(tmpout, "%02x", (unsigned int) (sha_buf[i])); + std::cerr << "FileIndex::loadIndex(): can't allocate memory for " << size << " bytes." << std::endl; + return 0 ; } + int bytesread = 0 ; + if(size != (bytesread = fread(compressed_data,1,size,file))) + { + std::cerr << "FileIndex::loadIndex(): can't read " << size << " bytes from file " << filename << ". Only " << bytesread << " actually read." << std::endl; + return 0 ; + } + fclose(file) ; + + std::string tmpout = RsDirUtil::sha1sum((unsigned char *)(compressed_data),size).toStdString() ; + +// /* calculate hash */ +// unsigned char sha_buf[SHA_DIGEST_LENGTH]; +// SHA_CTX *sha_ctx = new SHA_CTX; +// SHA1_Init(sha_ctx); +// SHA1_Update(sha_ctx, s.c_str(), s.length()); +// SHA1_Final(&sha_buf[0], sha_ctx); +// delete sha_ctx; +// +// std::string tmpout; +// for(int i = 0; i < SHA_DIGEST_LENGTH; i++) +// { +// rs_sprintf_append(tmpout, "%02x", (unsigned int) (sha_buf[i])); +// } if (expectedHash != "" && expectedHash != tmpout) { @@ -791,6 +801,21 @@ int FileIndex::loadIndex(const std::string& filename, const std::string& expecte #endif return 0; } + // now uncompress the string + // + + uint8_t *uncompressed_data = NULL ; + unsigned int uncompressed_data_size = 0 ; + + if(!RsCompress::uncompress_memory_chunk(compressed_data,size,uncompressed_data,uncompressed_data_size)) + { + std::cerr << "FileIndex::loadIndex() Decompression failed! Fileindex can't be read." << std::endl; + return 0 ; + } + std::string s((char *)uncompressed_data,uncompressed_data_size) ; + + delete[] compressed_data ; + free(uncompressed_data) ; #define FIND_NEXT(s,start,end,c) end = s.find(c, start); if (end == std::string::npos) end = s.length(); @@ -966,7 +991,6 @@ int FileIndex::saveIndex(const std::string& filename, std::string &fileHash, uin std::string s; size = 0 ; - fileHash = "" ; /* print version and header */ s += "# FileIndex version 0.1\n"; @@ -1003,18 +1027,37 @@ int FileIndex::saveIndex(const std::string& filename, std::string &fileHash, uin /* signal to pop directory from stack in loadIndex() */ s += "-\n"; - /* calculate sha1 hash */ - SHA_CTX *sha_ctx = new SHA_CTX; - SHA1_Init(sha_ctx); - SHA1_Update(sha_ctx, s.c_str(), s.length()); - SHA1_Final(&sha_buf[0], sha_ctx); - delete sha_ctx; + // now compress the data. + + std::cerr << "FileIndex::saveIndex(): compressign data." << std::endl; - for(int i = 0; i < SHA_DIGEST_LENGTH; i++) + uint8_t *compressed_data = NULL ; + uint32_t compressed_data_size = 0 ; + + if(!RsCompress::compress_memory_chunk((unsigned char *)s.c_str(),s.length(),compressed_data,compressed_data_size)) { - rs_sprintf_append(fileHash, "%02x", (unsigned int) (sha_buf[i])); + std::cerr << "(EE) ERROR in file list compression ! file list can't be saved" << std::endl; + return false ; } + fileHash = RsDirUtil::sha1sum((unsigned char *)compressed_data,compressed_data_size).toStdString() ; + + std::cerr << " old size = " << s.length() << std::endl; + std::cerr << " new size = " << compressed_data_size << std::endl; + std::cerr << " hash = " << fileHash << std::endl; + +// /* calculate sha1 hash */ +// SHA_CTX *sha_ctx = new SHA_CTX; +// SHA1_Init(sha_ctx); +// SHA1_Update(sha_ctx, s.c_str(), s.length()); +// SHA1_Final(&sha_buf[0], sha_ctx); +// delete sha_ctx; +// +// for(int i = 0; i < SHA_DIGEST_LENGTH; i++) +// { +// rs_sprintf_append(fileHash, "%02x", (unsigned int) (sha_buf[i])); +// } + /* finally, save to file */ FILE *file = RsDirUtil::rs_fopen(filenametmp.c_str(), "wb"); @@ -1023,9 +1066,16 @@ int FileIndex::saveIndex(const std::string& filename, std::string &fileHash, uin std::cerr << "FileIndex::saveIndex error opening file for writting: " << filename << ". Giving up." << std::endl; return 0; } - fprintf(file,"%s",s.c_str()) ; + int outwritten ; + + if(compressed_data_size != (outwritten=fwrite(compressed_data,1,compressed_data_size,file))) + { + std::cerr << "FileIndex::saveIndex error. File not entirely written. Only " << outwritten << " bytes wrote out of " << compressed_data_size << " check for disk full, or disk quotas." << std::endl; + return 0; + } fclose(file); + free(compressed_data) ; // Use a temp file name so that the file is never half saved. // diff --git a/libretroshare/src/libretroshare.pro b/libretroshare/src/libretroshare.pro index d4058dc10..e3911f177 100644 --- a/libretroshare/src/libretroshare.pro +++ b/libretroshare/src/libretroshare.pro @@ -408,6 +408,7 @@ HEADERS += turtle/p3turtle.h \ HEADERS += util/folderiterator.h \ util/rsdebug.h \ + util/rscompress.h \ util/smallobject.h \ util/rsdir.h \ util/rsdiscspace.h \ @@ -542,6 +543,7 @@ SOURCES += turtle/p3turtle.cc \ SOURCES += util/folderiterator.cc \ util/rsdebug.cc \ + util/rscompress.cc \ util/smallobject.cc \ util/rsdir.cc \ util/rsdiscspace.cc \ diff --git a/libretroshare/src/tests/util/Makefile b/libretroshare/src/tests/util/Makefile index 9be844a06..8ec44d149 100644 --- a/libretroshare/src/tests/util/Makefile +++ b/libretroshare/src/tests/util/Makefile @@ -9,13 +9,15 @@ OPS_TOP_DIR = ../../../../openpgpsdk/src include $(RS_TOP_DIR)/tests/scripts/config.mk ############################################################### -TESTOBJ = dirtest.o sha1_test.o aes_test.o dchat_decrypt.o -TESTS = dirtest sha1_test aes_test dchat_decrypt +TESTOBJ = dirtest.o compress_test.o sha1_test.o aes_test.o dchat_decrypt.o +TESTS = dirtest sha1_test aes_test compress_test dchat_decrypt all: tests sha1_test: sha1_test.o $(CC) $(CFLAGS) -o sha1_test sha1_test.o $(LIBS) +compress_test: compress_test.o + $(CC) $(CFLAGS) -o compress_test compress_test.o $(LIBS) dirtest: dirtest.o $(CC) $(CFLAGS) -o dirtest dirtest.o $(LIBS) dirtest: aes_test.o diff --git a/libretroshare/src/tests/util/compress_test.cc b/libretroshare/src/tests/util/compress_test.cc new file mode 100644 index 000000000..07e24b2eb --- /dev/null +++ b/libretroshare/src/tests/util/compress_test.cc @@ -0,0 +1,217 @@ + +/* + * "$Id: compress.cc,v 1.1 2007-02-19 20:08:30 rmf24 Exp $" + * + * RetroShare C++ Interface. + * + * Copyright 2012-2012 by Cyril Soler + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License Version 2 as published by the Free Software Foundation. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 + * USA. + * + * Please report all bugs and problems to "retroshare@lunamutt.com". + * + */ + + +#include + +#include "util/rscompress.h" +#include "util/rsdir.h" +#include "util/utest.h" +#include "util/argstream.h" + +#include +#include +#include +#include + +void printHelp(int argc,char *argv[]) +{ + std::cerr << argv[0] << ": tests AES encryption/decryption functions." << std::endl; + std::cerr << "Usage: " << argv[0] << std::endl ; +} + +bool compareStrings(const std::string& s1,const std::string& s2) +{ + uint32_t L = std::min(s1.length(),s2.length()) ; + + for(int i=0;i>4) ] ; + std::cerr << outh[ data[j] & 0xf ] ; + } +} + +INITTEST() ; + +int main(int argc,char *argv[]) +{ + std::string inputfile ; + argstream as(argc,argv) ; + + as >> help() ; + + as.defaultErrorHandling() ; + + std::cerr << "Testing RsCompress" << std::endl; + + std::string source_string = "This is a very secret string, but ultimately it will always be decyphered \ + ar cqs libretroshare.a temp/linux-g++-64/obj/p3bitdht.o \ + temp/linux-g++-64/obj/p3bitdht_interface.o \ + temp/linux-g++-64/obj/p3bitdht_peers.o \ + temp/linux-g++-64/obj/p3bitdht_peernet.o \ + temp/linux-g++-64/obj/p3bitdht_relay.o \ + temp/linux-g++-64/obj/connectstatebox.o temp/linux-g++-64/obj/udppeer.o \ + temp/linux-g++-64/obj/tcppacket.o temp/linux-g++-64/obj/tcpstream.o \ + temp/linux-g++-64/obj/tou.o temp/linux-g++-64/obj/bss_tou.o \ + temp/linux-g++-64/obj/udpstunner.o temp/linux-g++-64/obj/udprelay.o \ + temp/linux-g++-64/obj/cachestrapper.o temp/linux-g++-64/obj/fimonitor.o \ + temp/linux-g++-64/obj/findex.o temp/linux-g++-64/obj/fistore.o \ + temp/linux-g++-64/obj/rsexpr.o temp/linux-g++-64/obj/ftchunkmap.o \ + temp/linux-g++-64/obj/ftcontroller.o \ + temp/linux-g++-64/obj/ftdatamultiplex.o temp/linux-g++-64/obj/ftdbase.o \ + temp/linux-g++-64/obj/ftextralist.o temp/linux-g++-64/obj/ftfilecreator.o \ + temp/linux-g++-64/obj/ftfileprovider.o \ + temp/linux-g++-64/obj/ftfilesearch.o temp/linux-g++-64/obj/ftserver.o \ + temp/linux-g++-64/obj/fttransfermodule.o \ + temp/linux-g++-64/obj/ftturtlefiletransferitem.o \ + temp/linux-g++-64/obj/authgpg.o temp/linux-g++-64/obj/authssl.o \ + temp/linux-g++-64/obj/pgphandler.o temp/linux-g++-64/obj/pgpkeyutil.o \ + temp/linux-g++-64/obj/rscertificate.o temp/linux-g++-64/obj/p3cfgmgr.o \ + temp/linux-g++-64/obj/p3peermgr.o temp/linux-g++-64/obj/p3linkmgr.o \ + temp/linux-g++-64/obj/p3netmgr.o temp/linux-g++-64/obj/p3notify.o \ + temp/linux-g++-64/obj/pqiqos.o temp/linux-g++-64/obj/pqiarchive.o \ + temp/linux-g++-64/obj/pqibin.o temp/linux-g++-64/obj/pqihandler.o \ + temp/linux-g++-64/obj/p3historymgr.o temp/linux-g++-64/obj/pqiipset.o \ + temp/linux-g++-64/obj/pqiloopback.o temp/linux-g++-64/obj/pqimonitor.o \ + temp/linux-g++-64/obj/pqinetwork.o temp/linux-g++-64/obj/pqiperson.o \ + temp/linux-g++-64/obj/pqipersongrp.o temp/linux-g++-64/obj/pqisecurity.o \ + temp/linux-g++-64/obj/pqiservice.o temp/linux-g++-64/obj/pqissl.o \ + temp/linux-g++-64/obj/pqissllistener.o \ + temp/linux-g++-64/obj/pqisslpersongrp.o temp/linux-g++-64/obj/pqissludp.o \ + temp/linux-g++-64/obj/pqisslproxy.o temp/linux-g++-64/obj/pqistore.o \ + temp/linux-g++-64/obj/pqistreamer.o \ + temp/linux-g++-64/obj/pqithreadstreamer.o \ + temp/linux-g++-64/obj/pqiqosstreamer.o temp/linux-g++-64/obj/sslfns.o \ + temp/linux-g++-64/obj/pqinetstatebox.o \ + temp/linux-g++-64/obj/p3face-config.o temp/linux-g++-64/obj/p3face-msgs.o \ + temp/linux-g++-64/obj/p3face-server.o temp/linux-g++-64/obj/p3history.o \ + temp/linux-g++-64/obj/p3msgs.o temp/linux-g++-64/obj/p3peers.o \ + temp/linux-g++-64/obj/p3status.o temp/linux-g++-64/obj/rsinit.o \ + temp/linux-g++-64/obj/rsloginhandler.o temp/linux-g++-64/obj/rstypes.o \ + temp/linux-g++-64/obj/p3serverconfig.o \ + temp/linux-g++-64/obj/pluginmanager.o temp/linux-g++-64/obj/dlfcn_win32.o \ + temp/linux-g++-64/obj/rspluginitems.o \ + temp/linux-g++-64/obj/rsbaseserial.o \ + temp/linux-g++-64/obj/rsfiletransferitems.o \ + temp/linux-g++-64/obj/rsserviceserialiser.o \ + temp/linux-g++-64/obj/rsconfigitems.o \ + temp/linux-g++-64/obj/rshistoryitems.o temp/linux-g++-64/obj/rsmsgitems.o \ + temp/linux-g++-64/obj/rsserial.o temp/linux-g++-64/obj/rsstatusitems.o \ + temp/linux-g++-64/obj/rstlvaddrs.o temp/linux-g++-64/obj/rstlvbase.o \ + temp/linux-g++-64/obj/rstlvfileitem.o temp/linux-g++-64/obj/rstlvimage.o \ + temp/linux-g++-64/obj/rstlvkeys.o temp/linux-g++-64/obj/rstlvkvwide.o \ + temp/linux-g++-64/obj/rstlvtypes.o temp/linux-g++-64/obj/rstlvutil.o \ + temp/linux-g++-64/obj/rstlvdsdv.o temp/linux-g++-64/obj/rsdsdvitems.o \ + temp/linux-g++-64/obj/rstlvbanlist.o \ + temp/linux-g++-64/obj/rsbanlistitems.o \ + temp/linux-g++-64/obj/rsbwctrlitems.o \ + temp/linux-g++-64/obj/rsdiscovery2items.o \ + temp/linux-g++-64/obj/rsheartbeatitems.o \ + temp/linux-g++-64/obj/rsrttitems.o temp/linux-g++-64/obj/p3chatservice.o \ + temp/linux-g++-64/obj/p3msgservice.o temp/linux-g++-64/obj/p3service.o \ + temp/linux-g++-64/obj/p3statusservice.o temp/linux-g++-64/obj/p3dsdv.o \ + temp/linux-g++-64/obj/p3banlist.o temp/linux-g++-64/obj/p3bwctrl.o \ + temp/linux-g++-64/obj/p3discovery2.o temp/linux-g++-64/obj/p3heartbeat.o \ + temp/linux-g++-64/obj/p3rtt.o temp/linux-g++-64/obj/p3turtle.o \ + temp/linux-g++-64/obj/rsturtleitem.o \ + temp/linux-g++-64/obj/folderiterator.o temp/linux-g++-64/obj/rsdebug.o \ + temp/linux-g++-64/obj/rscompress.o temp/linux-g++-64/obj/smallobject.o \ + temp/linux-g++-64/obj/rsdir.o temp/linux-g++-64/obj/rsdiscspace.o \ + temp/linux-g++-64/obj/rsnet.o temp/linux-g++-64/obj/rsnet_ss.o \ + temp/linux-g++-64/obj/extaddrfinder.o temp/linux-g++-64/obj/dnsresolver.o \ + temp/linux-g++-64/obj/rsprint.o temp/linux-g++-64/obj/rsstring.o \ + temp/linux-g++-64/obj/rsthreads.o temp/linux-g++-64/obj/rsversion.o \ + temp/linux-g++-64/obj/rswin.o temp/linux-g++-64/obj/rsaes.o \ + temp/linux-g++-64/obj/rsrandom.o temp/linux-g++-64/obj/rstickevent.o \ + temp/linux-g++-64/obj/UPnPBase.o \ + temp/linux-g++-64/obj/upnphandler_linux.o \ + temp/linux-g++-64/obj/rsnxsitems.o temp/linux-g++-64/obj/rsdataservice.o \ + temp/linux-g++-64/obj/rsgenexchange.o \ + temp/linux-g++-64/obj/rsgxsnetservice.o temp/linux-g++-64/obj/rsgxsdata.o \ + temp/linux-g++-64/obj/rsgxsitems.o \ + temp/linux-g++-64/obj/rsgxsdataaccess.o temp/linux-g++-64/obj/retrodb.o \ + temp/linux-g++-64/obj/contentvalue.o temp/linux-g++-64/obj/rsdbbind.o \ + temp/linux-g++-64/obj/gxssecurity.o temp/linux-g++-64/obj/gxstokenqueue.o \ + temp/linux-g++-64/obj/rsgxsnetutils.o temp/linux-g++-64/obj/rsgxsutil.o \ + temp/linux-g++-64/obj/p3idservice.o temp/linux-g++-64/obj/rsgxsiditems.o \ + temp/linux-g++-64/obj/p3gxscircles.o \ + temp/linux-g++-64/obj/rsgxscircleitems.o \ + temp/linux-g++-64/obj/p3gxsforums.o \ + temp/linux-g++-64/obj/rsgxsforumitems.o \ + temp/linux-g++-64/obj/p3gxschannels.o temp/linux-g++-64/obj/p3gxscommon.o \ + temp/linux-g++-64/obj/rsgxscommentitems.o \ + temp/linux-g++-64/obj/rsgxschannelitems.o temp/linux-g++-64/obj/p3wiki.o \ + temp/linux-g++-64/obj/rswikiitems.o temp/linux-g++-64/obj/p3wire.o \ + temp/linux-g++-64/obj/rswireitems.o temp/linux-g++-64/obj/p3posted.o \ + temp/linux-g++-64/obj/rsposteditems.o \ + temp/linux-g++-64/obj/p3photoservice.o \ + temp/linux-g++-64/obj/rsphotoitems.o" ; + + std::cerr << "Input string: length=" << source_string.length() << std::endl; + std::cerr << "Input string: hash =" << RsDirUtil::sha1sum((uint8_t*)source_string.c_str(),source_string.length()).toStdString() << std::endl; + + uint8_t *output_data ; + uint32_t output_length ; + + CHECK(RsCompress::compress_memory_chunk((uint8_t*)source_string.c_str(),source_string.length(),output_data,output_length)) ; + + std::cerr << "Compressed data: " << std::endl; + std::cerr << " Length = " << output_length << std::endl; + std::cerr << " hash = " << RsDirUtil::sha1sum(output_data,output_length).toStdString() << std::endl; + + std::cerr << "Uncompressing..." << std::endl; + + uint8_t *decomp_output_data=NULL ; + uint32_t decomp_output_length=0 ; + + CHECK(RsCompress::uncompress_memory_chunk(output_data,output_length,decomp_output_data,decomp_output_length)) ; + + std::cerr << "Decompressed data: size=" << decomp_output_length << std::endl; + std::cerr << "Decompressed data: hash=" << RsDirUtil::sha1sum(decomp_output_data,decomp_output_length).toStdString() << std::endl; + + std::string decompress_string((char *)decomp_output_data,decomp_output_length) ; + CHECK(compareStrings(decompress_string, source_string)) ; + + free(decomp_output_data) ; + free(output_data) ; + + FINALREPORT("RSCompress") ; + return TESTRESULT() ; +} + diff --git a/retroshare-gui/src/gui/MessagesDialog.cpp b/retroshare-gui/src/gui/MessagesDialog.cpp index 9c37d0e1b..d37b18f15 100644 --- a/retroshare-gui/src/gui/MessagesDialog.cpp +++ b/retroshare-gui/src/gui/MessagesDialog.cpp @@ -1138,7 +1138,7 @@ void MessagesDialog::insertMessages() if(it->msgflags & RS_MSG_ENCRYPTED) text = tr("Encrypted message. Right-click to decrypt it.") ; else - text = QString::fromStdString(it->title); + text = QString::fromUtf8(it->title.c_str()); item[COLUMN_SUBJECT]->setText(text); item[COLUMN_SUBJECT]->setData(text + dateString, ROLE_SORT); @@ -1206,7 +1206,7 @@ void MessagesDialog::insertMessages() if (gotInfo || rsMsgs->getMessage(it->msgId, msgInfo)) { gotInfo = true; QTextDocument doc; - doc.setHtml(QString::fromStdString(msgInfo.msg)); + doc.setHtml(QString::fromUtf8(msgInfo.msg.c_str())); item[COLUMN_CONTENT]->setText(doc.toPlainText().replace(QString("\n"), QString(" "))); } else { std::cerr << "MessagesDialog::insertMsgTxtAndFiles() Couldn't find Msg" << std::endl;