This patch allows Windows users to share files and folders with "exotic" characters.

The problem was that libretroshare handles files in UTF-8 but Windows's ANSI/POSIX C functions automatically assume that the char* parameters are encoded with the system's code page. There is no way to set that code page as UTF-8. So now under Windows the code translates the file name to UTF-16 before feeding it to one of the Unicode functions (they are usually prefixed or suffixed by 'w'). Please note that it is not very efficient.

Furthermore, Windows does not provide a Unicode version of opendir/readdir/closedir, so it was necessary to use FindFirstFileW/FindNextFileW/FindClose which has a different behaviour as well as different structures. The FolderIterator class was created in order to mimic the Unix way of traversing folders contents. Hence the algorithm is unchanged and the systems differences masked.

As it was necessary to use some functions from the Windows API, <windows.h> had to be included in a few files were it didn't appear before, creating macros and #define conflicts. In order to solve them, util/rswin.h must be included first in a file (if necessary). Otherwise the preprocessor will fail on purpose to avoid the code being compiled with different _WIN32_WINNT values.

As another side-effect, rstlvutil.h and rstlvutil.cc have been removed from libretroshare.pro file. They are only used by testing units and include util/utest.h which defines a macro FAILED that already exists in <windows.h>. I don't know if unit tests are still in use and I don't plan on coding often on Windows, so I'll leave that as an exercise (hot potato?) to a motivated fellow Windows programmer.

git-svn-id: http://svn.code.sf.net/p/retroshare/code/trunk@2924 b45a01b8-16f6-495d-af2f-9b41ad6348cc

libretroshare/src/dbase/fimonitor.cc

@@ -21,12 +21,16 @@
  *
  */
 
-#include <util/rswin.h>
+#ifdef WINDOWS_SYS
+#include "util/rswin.h"
+#endif
+
 #include "dbase/fimonitor.h"
 #include "util/rsdir.h"
 #include "serialiser/rsserviceids.h"
 #include "rsiface/rsiface.h"
 #include "rsiface/rsnotify.h"
+#include "util/folderiterator.h"
 #include <errno.h>
 
 #include <iostream>
@@ -357,8 +361,8 @@ void 	FileIndexMonitor::updateCycle()
 #endif
 
 		/* check for the dir existance */
-		DIR *dir = opendir(realpath.c_str());
-		if (!dir)
+                librs::util::FolderIterator dirIt(realpath);
+                if (!dirIt.isValid())
 		{
 #ifdef FIM_DEBUG
 			std::cerr << "FileIndexMonitor::updateCycle()";
@@ -397,23 +401,29 @@ void 	FileIndexMonitor::updateCycle()
 		 * files checked to see if they have changed. (rehashed)
 		 */
 
-		struct dirent *dent;
 		struct stat64 buf;
 
 		to_hash.push_back(DirContentToHash()) ;
 		to_hash.back().realpath = realpath ;
 		to_hash.back().dirpath = dirpath ;
 
-		while(NULL != (dent = readdir(dir)))
+                while(dirIt.readdir())
 		{
 			/* check entry type */
-			std::string fname = dent -> d_name;
+                        std::string fname;
+                        dirIt.d_name(fname);
 			std::string fullname = realpath + "/" + fname;
 #ifdef FIM_DEBUG
 			std::cerr << "calling stats on " << fullname <<std::endl;
 #endif
 
+#ifdef WINDOWS_SYS
+                        std::wstring wfullname;
+                        librs::util::ConvertUtf8ToUtf16(fullname, wfullname);
+                        if (-1 != _wstati64(wfullname.c_str(), &buf))
+#else
 			if (-1 != stat64(fullname.c_str(), &buf))
+#endif
 			{
 #ifdef FIM_DEBUG
 				std::cerr << "buf.st_mode: " << buf.st_mode <<std::endl;
@@ -508,7 +518,7 @@ void 	FileIndexMonitor::updateCycle()
 		olddir = NULL;
 
 		/* close directory */
-		closedir(dir);
+                dirIt.closedir();
 	}
 
 	// Now, hash all files at once.
@@ -968,7 +978,16 @@ bool FileIndexMonitor::hashFile(std::string fullpath, FileEntry& fent)
 #ifdef FIM_DEBUG
 	std::cerr << "File to hash = " << f_hash << std::endl;
 #endif
-	if (NULL == (fd = fopen64(f_hash.c_str(), "rb")))	return false;
+
+#ifdef WINDOWS_SYS
+        std::wstring wf_hash;
+        librs::util::ConvertUtf8ToUtf16(f_hash, wf_hash);
+        if (NULL == (fd = _wfopen(wf_hash.c_str(), L"rb")))
+            return false;
+#else
+        if (NULL == (fd = fopen64(f_hash.c_str(), "rb")))
+            return false;
+#endif
 
 	SHA1_Init(sha_ctx);
 	while((len = fread(gblBuf,1, 512, fd)) > 0)

libretroshare/src/ft/ftcontroller.cc

@@ -35,6 +35,10 @@
  *
  */
 
+#ifdef WINDOWS_SYS
+#include "util/rswin.h"
+#endif
+
 #include "ft/ftcontroller.h"
 
 #include "ft/ftfilecreator.h"
@@ -591,7 +595,17 @@ bool ftController::moveFile(const std::string& source,const std::string& dest)
 {
 	// First try a rename
 	//
+
+#ifdef WINDOWS_SYS
+	std::wstring sourceW;
+	std::wstring destW;
+	librs::util::ConvertUtf8ToUtf16(source,sourceW);
+	librs::util::ConvertUtf8ToUtf16(dest,destW);
+
+	if( 0 == MoveFileW(sourceW.c_str(), destW.c_str()))
+#else
 	if (0 == rename(source.c_str(), dest.c_str()))
+#endif
 	{
 #ifdef CONTROL_DEBUG
                 std::cerr << "ftController::completeFile() renaming to: ";
@@ -609,14 +623,22 @@ bool ftController::moveFile(const std::string& source,const std::string& dest)
 #endif
 	// We could not rename, probably because we're dealing with different file systems.
 	// Let's copy then.
-	
+
+#ifdef WINDOWS_SYS
+	if(CopyFileW(sourceW.c_str(), destW.c_str(), FALSE) == 0)
+#else
 	if(!copyFile(source,dest))
+#endif
 		return false ;
 
 	// copy was successfull, let's delete the original
 	std::cerr << "deleting original file " << source << std::endl ;
 
+#ifdef WINDOWS_SYS
+	if(0 == DeleteFileW(sourceW.c_str()))
+#else
 	if(0 == remove(source.c_str()))
+#endif
 		return true ;
 	else
 	{
@@ -627,8 +649,6 @@ bool ftController::moveFile(const std::string& source,const std::string& dest)
 
 bool ftController::copyFile(const std::string& source,const std::string& dest)
 {
-	std::string error ;
-
 	FILE *in = fopen(source.c_str(),"rb") ;
 
 	if(in == NULL)
@@ -967,8 +987,13 @@ bool 	ftController::FileRequest(std::string fname, std::string hash,
 			destination = dest + "/" + fname;
 
 		// create void file with the target name.
+#ifdef WINDOWS_SYS
+		std::wstring destinationW;
+		librs::util::ConvertUtf8ToUtf16(destination, destinationW);
+		FILE *f = _wfopen(destinationW.c_str(), L"w");
+#else
 		FILE *f = fopen(destination.c_str(),"w") ;
-
+#endif
 		if(f == NULL)
 			std::cerr << "Could not open file " << destination << " for writting." << std::endl ;
 		else

libretroshare/src/ft/ftextralist.cc

@@ -23,6 +23,10 @@
  *
  */
 
+#ifdef WINDOWS_SYS
+#include "util/rswin.h"
+#endif
+
 #include "ft/ftextralist.h"
 #include "serialiser/rsconfigitems.h"
 #include "util/rsdir.h"
@@ -423,7 +427,13 @@ bool    ftExtraList::loadList(std::list<RsItem *> load)
 		}
 
 		/* open file */
+#ifdef WINDOWS_SYS
+		std::wstring filepathW;
+		librs::util::ConvertUtf8ToUtf16(fi->file.path, filepathW);
+		FILE *fd = _wfopen(filepathW.c_str(), L"rb");
+#else
 		FILE *fd = fopen(fi->file.path.c_str(), "rb");
+#endif
 		if (fd == NULL)
 		{
 			delete (*it);

libretroshare/src/ft/ftfileprovider.cc

@@ -5,6 +5,11 @@
 #include <stdlib.h>
 #include <stdio.h>
 
+#ifdef WINDOWS_SYS
+#include "util/rswin.h"
+#endif // WINDOWS_SYS
+
+
 static const time_t UPLOAD_CHUNK_MAPS_TIME = 30 ;	// time to ask for a new chunkmap from uploaders in seconds.
 
 ftFileProvider::ftFileProvider(std::string path, uint64_t size, std::string
@@ -240,14 +245,24 @@ int ftFileProvider::initializeFileAttrs()
 	 * attempt to open file 
 	 */
 
+#ifdef WINDOWS_SYS
+	std::wstring wfile_name;
+	librs::util::ConvertUtf8ToUtf16(file_name, wfile_name);
+	fd = _wfopen(wfile_name.c_str(), L"r+b");
+#else
 	fd = fopen64(file_name.c_str(), "r+b");
+#endif
 	if (!fd)
 	{
 		std::cerr << "ftFileProvider::initializeFileAttrs() Failed to open (r+b): ";
 		std::cerr << file_name << std::endl;
 
 		/* try opening read only */
+#ifdef WINDOWS_SYS
+		fd = _wfopen(wfile_name.c_str(), L"rb");
+#else
 		fd = fopen64(file_name.c_str(), "rb");
+#endif
 		if (!fd)
 		{
 			std::cerr << "ftFileProvider::initializeFileAttrs() Failed to open (rb): ";

libretroshare/src/libretroshare.pro

@@ -268,7 +268,7 @@ HEADERS += dbase/cachestrapper.h \
            serialiser/rstlvkeys.h \
            serialiser/rstlvkvwide.h \
            serialiser/rstlvtypes.h \
-           serialiser/rstlvutil.h \
+#           serialiser/rstlvutil.h \
            services/p3channels.h \
            services/p3chatservice.h \
            services/p3disc.h \
@@ -296,6 +296,7 @@ HEADERS += dbase/cachestrapper.h \
            tcponudp/udplayer.h \
            tcponudp/udpsorter.h \
            upnp/upnphandler.h \
+           util/folderiterator.h \
 	   util/rsdebug.h \
            util/rsdir.h \
            util/rsnet.h \
@@ -397,7 +398,7 @@ SOURCES += \
 				serialiser/rsbaseitems.cc \
 				serialiser/rstlvkvwide.cc \
 				serialiser/rstlvimage.cc \
-				serialiser/rstlvutil.cc \
+#				serialiser/rstlvutil.cc \
 				serialiser/rstlvfileitem.cc \
 				serialiser/rstlvkeys.cc \
 				serialiser/rsbaseserial.cc \
@@ -412,11 +413,12 @@ SOURCES += \
 				tcponudp/udpsorter.cc \
 				tcponudp/tou_net.cc \
 				tcponudp/udplayer.cc \
+				util/folderiterator.cc \
 				util/rsdebug.cc \
 				util/rsdir.cc \
 				util/rsnet.cc \
 				util/rsprint.cc \
 				util/rsthreads.cc \
-				util/rsversion.cc 
-
+				util/rsversion.cc \
+				util/rswin.cc
 

libretroshare/src/pqi/pqinetwork.h

@@ -45,12 +45,7 @@
 
 #else
 
-/* This defines the platform to be WinXP or later...
- * and is needed for getaddrinfo.... (not used anymore)
- *
- */
-
-#define _WIN32_WINNT 0x0501
+#include "util/rswin.h"
 
 #include "util/rsnet.h" /* more generic networking header */
 

libretroshare/src/tcponudp/tou.h

@@ -40,6 +40,7 @@
 	#include <netinet/in.h>
 
 #else
+	#include "util/rswin.h"
 	#include <stdint.h>
 	#include <winsock2.h>
 	#include <ws2tcpip.h>

libretroshare/src/util/folderiterator.cc

@@ -0,0 +1,82 @@
+#include "folderiterator.h"
+#include "rswin.h"
+
+
+namespace librs { namespace util {
+
+
+FolderIterator::FolderIterator(const std::string& folderName)
+{
+#ifdef WINDOWS_SYS
+    std::wstring utf16Name;
+    if(! ConvertUtf8ToUtf16(folderName, utf16Name)) {
+        validity = false;
+        return;
+    }
+
+    utf16Name += L"\\*.*";
+
+    handle = FindFirstFileW(utf16Name.c_str(), &fileInfo);
+    validity = handle != INVALID_HANDLE_VALUE;
+    isFirstCall = true;
+#else
+    handle = opendir(folderName.c_str());
+    validity = handle != NULL;
+#endif
+}
+
+FolderIterator::~FolderIterator()
+{
+    closedir();
+}
+
+bool FolderIterator::readdir() {
+    if(!validity)
+        return false;
+
+#ifdef WINDOWS_SYS
+    if(isFirstCall) {
+        isFirstCall = false;
+        return true;
+    }
+    return FindNextFileW(handle, &fileInfo) != 0;
+#else
+    return readdir(handle) == 0;
+#endif
+}
+
+bool FolderIterator::d_name(std::string& dest)
+{
+    if(!validity)
+        return false;
+
+#ifdef WINDOWS_SYS
+    if(! ConvertUtf16ToUtf8(fileInfo.cFileName, dest)) {
+        validity = false;
+        return false;
+    }
+#else
+    dest = handle->d_name;
+#endif
+
+    return true;
+}
+
+bool FolderIterator::closedir()
+{
+    if(!validity)
+        return false;
+
+    validity = false;
+
+#ifdef WINDOWS_SYS
+    return FindClose(handle) != 0;
+#else
+    return closedir(handle) == 0;
+#endif
+}
+
+
+
+
+} } // librs::util

libretroshare/src/util/folderiterator.h

@@ -0,0 +1,53 @@
+#ifndef FOLDERITERATOR_H
+#define FOLDERITERATOR_H
+
+
+#include <iostream>
+#include <cstdio>
+
+#ifdef WINDOWS_SYS
+    #include "util/rswin.h"
+    #include <windows.h>
+    #include <tchar.h>
+    #include <stdio.h>
+    #include <string.h>
+#else
+    #include <dirent.h>
+#endif
+
+
+namespace librs { namespace util {
+
+
+class FolderIterator
+{
+public:
+    FolderIterator(const std::string& folderName);
+    ~FolderIterator();
+
+    bool isValid() const    { return validity; }
+
+    bool readdir();
+
+    bool d_name(std::string& dest);
+
+    bool closedir();
+
+private:
+    bool validity;
+
+#ifdef WINDOWS_SYS
+    HANDLE handle;
+    bool isFirstCall;
+    _WIN32_FIND_DATAW fileInfo;
+#else
+    DIR* handle;
+#endif
+
+};
+
+
+} } // librs::util
+
+
+#endif // FOLDERITERATOR_H

libretroshare/src/util/rsdir.cc

@@ -39,6 +39,7 @@
 #include <fstream>
 
 #if defined(WIN32) || defined(__CYGWIN__)
+#include "util/rswin.h"
 #include "wtypes.h"
 #include <winioctl.h>
 #else
@@ -345,8 +346,15 @@ bool RsDirUtil::getFileHash(std::string filepath,
 	unsigned char sha_buf[SHA_DIGEST_LENGTH];
 	unsigned char gblBuf[512];
 
+#ifdef WINDOWS_SYS
+	std::wstring filepathW;
+	librs::util::ConvertUtf8ToUtf16(filepath, filepathW);
+	if (NULL == (fd = _wfopen(filepathW.c_str(), L"rb")))
+		return false;
+#else
 	if (NULL == (fd = fopen(filepath.c_str(), "rb")))
 		return false;
+#endif
 
 	/* determine size */
  	fseek(fd, 0, SEEK_END);

libretroshare/src/util/rsnet.h

@@ -43,6 +43,8 @@
 
 #else
 
+#include "util/rswin.h"
+
 #include <winsock2.h>
 #include <ws2tcpip.h>
 

libretroshare/src/util/rsthreads.h

@@ -27,6 +27,10 @@
  */
 
 
+#ifdef WINDOWS_SYS
+#include "util/rswin.h"
+#endif
+
 #include <pthread.h>
 #include <inttypes.h>
 

libretroshare/src/util/rswin.cc

@@ -0,0 +1,49 @@
+#include "util/rswin.h"
+
+
+namespace librs { namespace util {
+
+
+bool ConvertUtf8ToUtf16(const std::string& source, std::wstring& dest) {
+    int nbChars = MultiByteToWideChar(CP_UTF8, 0,
+                                      source.c_str(), -1,
+                                      0, 0);
+    if(nbChars == 0)
+        return false;
+
+    wchar_t* utf16Name = new wchar_t[nbChars];
+    if( MultiByteToWideChar(CP_UTF8, 0,
+                            source.c_str(), -1,
+                            utf16Name, nbChars) == 0) {
+        return false;
+    }
+
+    dest = utf16Name;
+    delete[] utf16Name;
+
+    return true;
+}
+
+bool ConvertUtf16ToUtf8(const std::wstring& source, std::string& dest) {
+    int nbChars = WideCharToMultiByte(CP_UTF8, 0,
+                                      source.c_str(), -1,
+                                      0, 0,
+                                      0, 0);
+    if(nbChars == 0)
+        return false;
+
+    char* utf8Name = new char[nbChars];
+    if( WideCharToMultiByte(CP_UTF8, 0,
+                            source.c_str(), -1,
+                            utf8Name, nbChars,
+                            0, 0) == 0) {
+        return false;
+    }
+
+    dest = utf8Name;
+    delete[] utf8Name;
+    return true;
+}
+
+
+} } // librs::util

libretroshare/src/util/rswin.h

@@ -1,18 +1,63 @@
-#ifndef RS_UNIVERSAL_STUFF
-#define RS_UNIVERSAL_STUFF
+/****************************************************************
+ * This file is distributed under the following license:
+ *
+ * Copyright (c) 2010, Thomas Kister
+ *
+ *  This program is free software; you can redistribute it and/or
+ *  modify it under the terms of the GNU General Public License
+ *  as published by the Free Software Foundation; either version 2
+ *  of the License, or (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program; if not, write to the Free Software
+ *  Foundation, Inc., 51 Franklin Street, Fifth Floor, 
+ *  Boston, MA  02110-1301, USA.
+ ****************************************************************/
 
-#if defined(WIN32) || defined(MINGW) 
 
-#include <unistd.h>
-#define sleep(x) Sleep(1000 * x)
+/**
+ * This file provides helper functions for the windows environment
+ */
+
+
+#ifndef RSWIN_H_
+#define RSWIN_H_
+
+
+#ifdef WINDOWS_SYS
+
+#ifdef _WIN32_WINNT
+#error "Please include \"util/rswin.h\" *before* any other one as _WIN32_WINNT needs to predefined"
+#endif
+
+// This defines the platform to be WinXP or later and is needed for getaddrinfo
+// It must be declared before pthread.h includes windows.h
+#define _WIN32_WINNT 0x0501
+
+#include <windows.h>
+#include <string>
 
 // For win32 systems (tested on MingW+Ubuntu)
 #define stat64 _stati64 
 
 
-#endif
+namespace librs { namespace util {
 
 
+bool ConvertUtf8ToUtf16(const std::string& source, std::wstring& dest);
 
-#endif
-	
+bool ConvertUtf16ToUtf8(const std::wstring& source, std::string& dest);
+
+
+} } // librs::util
+
+
+#endif // WINDOWS_SYS
+
+
+#endif // RSWIN_H_