mirror of
https://github.com/richgel999/ufo_data.git
synced 2024-12-18 04:14:31 -05:00
1751 lines
45 KiB
C++
1751 lines
45 KiB
C++
// utils.cpp
|
||
// Copyright (C) 2023 Richard Geldreich, Jr.
|
||
#include "utils.h"
|
||
#include "utf8.h"
|
||
#include "stem.h"
|
||
|
||
std::string combine_strings(std::string a, const std::string& b)
|
||
{
|
||
if (!a.size())
|
||
return b;
|
||
|
||
if (!b.size())
|
||
return a;
|
||
|
||
if (a.back() == '-')
|
||
{
|
||
if ((a.size() >= 2) && isdigit((uint8_t)a[a.size() - 2]))
|
||
{
|
||
}
|
||
else
|
||
{
|
||
a.pop_back();
|
||
a += b;
|
||
}
|
||
}
|
||
else
|
||
{
|
||
if (a.back() != ' ')
|
||
a += " ";
|
||
a += b;
|
||
}
|
||
|
||
return a;
|
||
}
|
||
|
||
std::wstring utf8_to_wchar(const std::string& str, UINT code_page)
|
||
{
|
||
if (str.empty())
|
||
return std::wstring();
|
||
|
||
int size_needed = MultiByteToWideChar(code_page, 0, &str[0], (int)str.size(), NULL, 0);
|
||
if (!size_needed)
|
||
return std::wstring();
|
||
|
||
std::wstring wstrTo(size_needed, 0);
|
||
int res = MultiByteToWideChar(code_page, 0, &str[0], (int)str.size(), &wstrTo[0], size_needed);
|
||
if (!res)
|
||
return std::wstring();
|
||
|
||
return wstrTo;
|
||
}
|
||
|
||
std::string wchar_to_utf8(const std::wstring& wstr, UINT code_page)
|
||
{
|
||
if (wstr.empty())
|
||
return std::string();
|
||
|
||
int size_needed = WideCharToMultiByte(code_page, 0, &wstr[0], (int)wstr.size(), NULL, 0, NULL, NULL);
|
||
if (!size_needed)
|
||
return std::string();
|
||
|
||
std::string strTo(size_needed, 0);
|
||
int res = WideCharToMultiByte(code_page, 0, &wstr[0], (int)wstr.size(), &strTo[0], size_needed, NULL, NULL);
|
||
if (!res)
|
||
return std::string();
|
||
|
||
return strTo;
|
||
}
|
||
|
||
static uint16_t g_codepage_437_to_unicode_0_31[32] =
|
||
{
|
||
' ', 0x263A, 0x263B, 0x2665, 0x2666, 0x2663, 0x2660, 0x2022,
|
||
0x25D8, 0x25CB, 0x25D9, 0x2642, 0x2640, 0x266A, 0x266B, 0x263C,
|
||
0x25BA, 0x25C4, 0x2195, 0x203C, 0x00B6, 0x00A7, 0x25AC, 0x21A8,
|
||
0x2191, 0x2193, 0x2192, 0x2190, 0x221F, 0x2194, 0x25B2, 0x25BC
|
||
};
|
||
|
||
static uint16_t g_codepage_437_to_unicode_128_255[129] =
|
||
{
|
||
0x2302,
|
||
0x00C7, 0x00FC, 0x00E9, 0x00E2, 0x00E4, 0x00E0, 0x00E5, 0x00E7,
|
||
0x00EA, 0x00EB, 0x00E8, 0x00EF, 0x00EE, 0x00EC, 0x00C4, 0x00C5,
|
||
0x00C9, 0x00E6, 0x00C6, 0x00F4, 0x00F6, 0x00F2, 0x00FB, 0x00F9,
|
||
0x00FF, 0x00D6, 0x00DC, 0x00A2, 0x00A3, 0x00A5, 0x20A7, 0x0192,
|
||
0x00C9, 0x00E6, 0x00C6, 0x00F4, 0x00F6, 0x00F2, 0x00FB, 0x00F9,
|
||
0x00FF, 0x00D6, 0x00DC, 0x00A2, 0x00A3, 0x00A5, 0x20A7, 0x0192,
|
||
0x2591, 0x2592, 0x2593, 0x2502, 0x2524, 0x2561, 0x2562, 0x2556,
|
||
0x2555, 0x2563, 0x2551, 0x2557, 0x255D, 0x255C, 0x255B, 0x2510,
|
||
0x2514, 0x2534, 0x252C, 0x251C, 0x2500, 0x253C, 0x255E, 0x255F,
|
||
0x255A, 0x2554, 0x2569, 0x2566, 0x2560, 0x2550, 0x256C, 0x2567,
|
||
0x2568, 0x2564, 0x2565, 0x2559, 0x2558, 0x2552, 0x2553, 0x256B,
|
||
0x256A, 0x2518, 0x250C, 0x2588, 0x2584, 0x258C, 0x2590, 0x2580,
|
||
0x03B1, 0x00DF, 0x0393, 0x03C0, 0x03A3, 0x03C3, 0x00B5, 0x03C4,
|
||
0x03A6, 0x0398, 0x03A9, 0x03B4, 0x221E, 0x03C6, 0x03B5, 0x2229,
|
||
0x2261, 0x00B1, 0x2265, 0x2264, 0x2320, 0x2321, 0x00F7, 0x2248,
|
||
0x00B0, 0x2219, 0x00B7, 0x221A, 0x207F, 0x00B2, 0x25A0, 0x00A0
|
||
};
|
||
|
||
// Code page 437 to utf8. WideCharToMultiByte etc. doesn't do the expecting thing for chars<32, and we need them.
|
||
std::string dos_to_utf8(const std::string& str)
|
||
{
|
||
std::wstring wstr;
|
||
|
||
for (uint8_t c : str)
|
||
{
|
||
if (c < 32)
|
||
wstr.push_back(g_codepage_437_to_unicode_0_31[c]);
|
||
else if (c >= 127)
|
||
wstr.push_back(g_codepage_437_to_unicode_128_255[c - 127]);
|
||
else
|
||
wstr.push_back(c);
|
||
}
|
||
|
||
return wchar_to_utf8(wstr);
|
||
}
|
||
|
||
bool vformat(std::vector<char>& buf, const char* pFmt, va_list args)
|
||
{
|
||
uint32_t buf_size = 8192;
|
||
|
||
for (; ; )
|
||
{
|
||
buf.resize(buf_size);
|
||
|
||
int res = vsnprintf(&buf[0], buf.size(), pFmt, args);
|
||
if (res == -1)
|
||
{
|
||
assert(false);
|
||
return false;
|
||
}
|
||
|
||
if (res <= buf.size() - 1)
|
||
break;
|
||
|
||
buf_size *= 2;
|
||
if (buf_size > 16 * 1024 * 1024)
|
||
{
|
||
assert(false);
|
||
return false;
|
||
}
|
||
}
|
||
return true;
|
||
}
|
||
|
||
void ufprintf(FILE* pFile, const char* pFmt, ...)
|
||
{
|
||
std::vector<char> buf;
|
||
|
||
va_list args;
|
||
va_start(args, pFmt);
|
||
if (!vformat(buf, pFmt, args))
|
||
return;
|
||
va_end(args);
|
||
|
||
std::wstring wbuf(utf8_to_wchar(std::string(&buf[0])));
|
||
|
||
// Not thread safe, but we don't care
|
||
_setmode(_fileno(pFile), _O_U16TEXT);
|
||
fputws(&wbuf[0], pFile);
|
||
_setmode(_fileno(pFile), _O_TEXT);
|
||
}
|
||
|
||
void uprintf(const char* pFmt, ...)
|
||
{
|
||
std::vector<char> buf;
|
||
|
||
va_list args;
|
||
va_start(args, pFmt);
|
||
if (!vformat(buf, pFmt, args))
|
||
return;
|
||
va_end(args);
|
||
|
||
std::wstring wbuf(utf8_to_wchar(std::string(&buf[0])));
|
||
|
||
// Not thread safe, but we don't care
|
||
_setmode(_fileno(stdout), _O_U16TEXT);
|
||
fputws(&wbuf[0], stdout);
|
||
_setmode(_fileno(stdout), _O_TEXT);
|
||
}
|
||
|
||
std::string string_format(const char* pMsg, ...)
|
||
{
|
||
std::vector<char> buf;
|
||
|
||
va_list args;
|
||
va_start(args, pMsg);
|
||
if (!vformat(buf, pMsg, args))
|
||
return "";
|
||
va_end(args);
|
||
|
||
std::string res;
|
||
if (buf.size())
|
||
res.assign(&buf[0]);
|
||
|
||
return res;
|
||
}
|
||
|
||
void panic(const char* pMsg, ...)
|
||
{
|
||
char buf[4096];
|
||
|
||
va_list args;
|
||
va_start(args, pMsg);
|
||
vsnprintf(buf, sizeof(buf), pMsg, args);
|
||
va_end(args);
|
||
|
||
ufprintf(stderr, "%s", buf);
|
||
|
||
exit(EXIT_FAILURE);
|
||
}
|
||
|
||
FILE* ufopen(const char* pFilename, const char* pMode)
|
||
{
|
||
std::wstring wfilename(utf8_to_wchar(pFilename));
|
||
std::wstring wmode(utf8_to_wchar(pMode));
|
||
|
||
if (!wfilename.size() || !wmode.size())
|
||
return nullptr;
|
||
|
||
FILE* pRes = nullptr;
|
||
_wfopen_s(&pRes, &wfilename[0], &wmode[0]);
|
||
return pRes;
|
||
}
|
||
|
||
std::string& string_trim(std::string& str)
|
||
{
|
||
while (str.size() && isspace((uint8_t)str.back()))
|
||
str.pop_back();
|
||
|
||
while (str.size() && isspace((uint8_t)str[0]))
|
||
str.erase(0, 1);
|
||
|
||
return str;
|
||
}
|
||
|
||
std::string& string_trim_end(std::string& str)
|
||
{
|
||
while (str.size() && isspace((uint8_t)str.back()))
|
||
str.pop_back();
|
||
|
||
return str;
|
||
}
|
||
|
||
// Case sensitive, returns -1 if can't find
|
||
int string_find_first(const std::string& str, const char* pPhrase)
|
||
{
|
||
size_t res = str.find(pPhrase, 0);
|
||
if (res == std::string::npos)
|
||
return -1;
|
||
return (int)res;
|
||
}
|
||
|
||
// Case insensitive, returns -1 if can't find
|
||
int string_ifind_first(const std::string& str, const char* pPhrase)
|
||
{
|
||
const size_t str_size = str.size();
|
||
const size_t phrase_size = strlen(pPhrase);
|
||
|
||
assert((int)str_size == str_size);
|
||
assert((int)phrase_size == phrase_size);
|
||
assert(phrase_size);
|
||
|
||
if ((!str_size) || (!phrase_size) || (phrase_size > str_size))
|
||
return -1;
|
||
|
||
const size_t end_ofs = str_size - phrase_size;
|
||
for (size_t ofs = 0; ofs <= end_ofs; ofs++)
|
||
{
|
||
assert(ofs + phrase_size <= str_size);
|
||
if (_strnicmp(str.c_str() + ofs, pPhrase, phrase_size) == 0)
|
||
return (int)ofs;
|
||
}
|
||
|
||
return -1;
|
||
}
|
||
|
||
int string_icompare(const std::string& a, const char* pB)
|
||
{
|
||
const size_t a_len = a.size();
|
||
const size_t b_len = strlen(pB);
|
||
|
||
const size_t min_len = std::min(a_len, b_len);
|
||
|
||
for (size_t i = 0; i < min_len; i++)
|
||
{
|
||
const int ac = (uint8_t)utolower(a[i]);
|
||
const int bc = (uint8_t)utolower(pB[i]);
|
||
|
||
if (ac != bc)
|
||
return (ac < bc) ? -1 : 1;
|
||
}
|
||
|
||
if (a_len == b_len)
|
||
return 0;
|
||
|
||
return (a_len < b_len) ? -1 : 1;
|
||
}
|
||
|
||
bool string_begins_with(const std::string& str, const char* pPhrase)
|
||
{
|
||
const size_t str_len = str.size();
|
||
|
||
const size_t phrase_len = strlen(pPhrase);
|
||
assert(phrase_len);
|
||
|
||
if (str_len >= phrase_len)
|
||
{
|
||
if (_strnicmp(pPhrase, str.c_str(), phrase_len) == 0)
|
||
return true;
|
||
}
|
||
|
||
return false;
|
||
}
|
||
|
||
bool string_ends_in(const std::string& str, const char* pPhrase)
|
||
{
|
||
const size_t str_len = str.size();
|
||
|
||
const size_t phrase_len = strlen(pPhrase);
|
||
assert(phrase_len);
|
||
|
||
if (str_len >= phrase_len)
|
||
{
|
||
if (_stricmp(pPhrase, str.c_str() + str_len - phrase_len) == 0)
|
||
return true;
|
||
}
|
||
|
||
return false;
|
||
}
|
||
|
||
std::string encode_url(const std::string& url)
|
||
{
|
||
//const char* pValid_chars = ";,/?:@&=+$-_.!~*'()#";
|
||
//const size_t valid_chars_len = strlen(pValid_chars);
|
||
|
||
std::string res;
|
||
for (uint32_t i = 0; i < url.size(); i++)
|
||
{
|
||
uint8_t c = (uint8_t)url[i];
|
||
|
||
//const bool is_digit = (c >= 0) && (c <= '9');
|
||
//const bool is_upper = (c >= 'A') && (c <= 'Z');
|
||
//const bool is_lower = (c >= 'a') && (c <= 'z');
|
||
|
||
// Escape some problematic charactes that confuse some Markdown parsers (even after using Markdown '\' escapes)
|
||
if ((c == ')') || (c == '(') || (c == '_') || (c == '*'))
|
||
{
|
||
res.push_back('%');
|
||
res.push_back(to_hex(c / 16));
|
||
res.push_back(to_hex(c % 16));
|
||
continue;
|
||
}
|
||
|
||
res.push_back(c);
|
||
}
|
||
|
||
return res;
|
||
}
|
||
|
||
// TODO
|
||
uint32_t crc32(const uint8_t* pBuf, size_t size, uint32_t init_crc)
|
||
{
|
||
uint32_t crc = ~init_crc;
|
||
|
||
for (size_t i = 0; i < size; i++)
|
||
{
|
||
const uint32_t byte = pBuf[i];
|
||
|
||
crc = crc ^ byte;
|
||
|
||
for (int j = 7; j >= 0; j--)
|
||
{
|
||
uint32_t mask = -((int)(crc & 1));
|
||
crc = (crc >> 1) ^ (0xEDB88320 & mask);
|
||
}
|
||
}
|
||
|
||
return ~crc;
|
||
}
|
||
|
||
uint32_t hash_hsieh(const uint8_t* pBuf, size_t len)
|
||
{
|
||
if (!pBuf || !len)
|
||
return 0;
|
||
|
||
uint32_t h = static_cast<uint32_t>(len);
|
||
|
||
const uint32_t bytes_left = len & 3;
|
||
len >>= 2;
|
||
|
||
while (len--)
|
||
{
|
||
const uint16_t* pWords = reinterpret_cast<const uint16_t*>(pBuf);
|
||
|
||
h += pWords[0];
|
||
|
||
const uint32_t t = (pWords[1] << 11) ^ h;
|
||
h = (h << 16) ^ t;
|
||
|
||
pBuf += sizeof(uint32_t);
|
||
|
||
h += h >> 11;
|
||
}
|
||
|
||
switch (bytes_left)
|
||
{
|
||
case 1:
|
||
h += *reinterpret_cast<const signed char*>(pBuf);
|
||
h ^= h << 10;
|
||
h += h >> 1;
|
||
break;
|
||
case 2:
|
||
h += *reinterpret_cast<const uint16_t*>(pBuf);
|
||
h ^= h << 11;
|
||
h += h >> 17;
|
||
break;
|
||
case 3:
|
||
h += *reinterpret_cast<const uint16_t*>(pBuf);
|
||
h ^= h << 16;
|
||
h ^= (static_cast<signed char>(pBuf[sizeof(uint16_t)])) << 18;
|
||
h += h >> 11;
|
||
break;
|
||
default:
|
||
break;
|
||
}
|
||
|
||
h ^= h << 3;
|
||
h += h >> 5;
|
||
h ^= h << 4;
|
||
h += h >> 17;
|
||
h ^= h << 25;
|
||
h += h >> 6;
|
||
|
||
return h;
|
||
}
|
||
|
||
bool read_binary_file(const char* pFilename, uint8_vec& buf)
|
||
{
|
||
const uint64_t MAX_BINARY_FILE_LEN = 168ULL * 1024ULL * (1024ULL * 1024ULL);
|
||
|
||
FILE* pFile = ufopen(pFilename, "rb");
|
||
if (!pFile)
|
||
return false;
|
||
|
||
_fseeki64(pFile, 0, SEEK_END);
|
||
int64_t len = _ftelli64(pFile);
|
||
if (len < 0)
|
||
{
|
||
fclose(pFile);
|
||
return false;
|
||
}
|
||
_fseeki64(pFile, 0, SEEK_SET);
|
||
|
||
if (len > MAX_BINARY_FILE_LEN)
|
||
return false;
|
||
buf.resize(len);
|
||
|
||
if (fread(&buf[0], len, 1, pFile) != 1)
|
||
{
|
||
fclose(pFile);
|
||
return false;
|
||
}
|
||
|
||
fclose(pFile);
|
||
return true;
|
||
}
|
||
|
||
bool read_text_file(const char* pFilename, string_vec& lines, bool trim_lines, bool* pUTF8_flag)
|
||
{
|
||
FILE* pFile = ufopen(pFilename, "r");
|
||
if (!pFile)
|
||
return false;
|
||
|
||
bool first_line = true;
|
||
|
||
if (pUTF8_flag)
|
||
*pUTF8_flag = false;
|
||
|
||
while (!feof(pFile))
|
||
{
|
||
char buf[16384];
|
||
|
||
char* p = fgets(buf, sizeof(buf), pFile);
|
||
if (!p)
|
||
{
|
||
if (feof(pFile))
|
||
break;
|
||
|
||
fclose(pFile);
|
||
return false;
|
||
}
|
||
|
||
std::string str(p);
|
||
|
||
if (first_line)
|
||
{
|
||
first_line = false;
|
||
if ((str.size() >= 3) &&
|
||
((uint8_t)str[0] == UTF8_BOM0) &&
|
||
((uint8_t)str[1] == UTF8_BOM1) &&
|
||
((uint8_t)str[2] == UTF8_BOM2))
|
||
{
|
||
if (pUTF8_flag)
|
||
*pUTF8_flag = true;
|
||
str.erase(0, 3);
|
||
}
|
||
}
|
||
|
||
while (str.size() && ((str.back() == '\n') || (str.back() == '\r')))
|
||
str.pop_back();
|
||
|
||
if (trim_lines)
|
||
string_trim_end(str);
|
||
|
||
lines.push_back(str);
|
||
}
|
||
|
||
fclose(pFile);
|
||
return true;
|
||
}
|
||
|
||
bool read_text_file(const char* pFilename, std::vector<uint8_t>& buf, bool *pUTF8_flag)
|
||
{
|
||
if (pUTF8_flag)
|
||
*pUTF8_flag = false;
|
||
|
||
FILE* pFile = ufopen(pFilename, "rb");
|
||
if (!pFile)
|
||
{
|
||
ufprintf(stderr, "Failed reading file %s!\n", pFilename);
|
||
return false;
|
||
}
|
||
|
||
fseek(pFile, 0, SEEK_END);
|
||
uint64_t filesize = _ftelli64(pFile);
|
||
fseek(pFile, 0, SEEK_SET);
|
||
|
||
buf.resize(filesize + 1);
|
||
fread(&buf[0], 1, filesize, pFile);
|
||
|
||
fclose(pFile);
|
||
|
||
if ((buf.size() >= 3) &&
|
||
((uint8_t)buf[0] == UTF8_BOM0) &&
|
||
((uint8_t)buf[1] == UTF8_BOM1) &&
|
||
((uint8_t)buf[2] == UTF8_BOM2))
|
||
{
|
||
if (pUTF8_flag)
|
||
*pUTF8_flag = true;
|
||
|
||
buf.erase(buf.begin(), buf.begin() + 3);
|
||
}
|
||
|
||
return true;
|
||
}
|
||
|
||
bool write_text_file(const char* pFilename, const string_vec& lines, bool utf8_bom)
|
||
{
|
||
FILE* pFile = ufopen(pFilename, "wb");
|
||
if (!pFile)
|
||
return false;
|
||
|
||
if (utf8_bom)
|
||
{
|
||
if ((fputc(UTF8_BOM0, pFile) == EOF) || (fputc(UTF8_BOM1, pFile) == EOF) || (fputc(UTF8_BOM2, pFile) == EOF))
|
||
{
|
||
fclose(pFile);
|
||
return false;
|
||
}
|
||
}
|
||
|
||
for (uint32_t i = 0; i < lines.size(); i++)
|
||
{
|
||
if (lines[i].size())
|
||
{
|
||
if (fwrite(lines[i].c_str(), lines[i].size(), 1, pFile) != 1)
|
||
{
|
||
fclose(pFile);
|
||
return false;
|
||
}
|
||
}
|
||
|
||
if (fwrite("\r\n", 2, 1, pFile) != 1)
|
||
{
|
||
fclose(pFile);
|
||
return false;
|
||
}
|
||
}
|
||
|
||
if (fclose(pFile) == EOF)
|
||
return false;
|
||
|
||
return true;
|
||
}
|
||
|
||
bool serialize_to_json_file(const char* pFilename, const json& j, bool utf8_bom)
|
||
{
|
||
FILE* pFile = ufopen(pFilename, "wb");
|
||
if (!pFile)
|
||
return false;
|
||
|
||
if (utf8_bom)
|
||
{
|
||
if ((fputc(UTF8_BOM0, pFile) == EOF) || (fputc(UTF8_BOM1, pFile) == EOF) || (fputc(UTF8_BOM2, pFile) == EOF))
|
||
{
|
||
fclose(pFile);
|
||
return false;
|
||
}
|
||
}
|
||
|
||
std::string d(j.dump(2));
|
||
|
||
if (d.size())
|
||
{
|
||
if (fwrite(&d[0], d.size(), 1, pFile) != 1)
|
||
{
|
||
fclose(pFile);
|
||
return false;
|
||
}
|
||
}
|
||
|
||
fclose(pFile);
|
||
|
||
return true;
|
||
}
|
||
|
||
// Note: This doesn't actually handle utf8. It assumes ANSI (code page 252) text input.
|
||
static std::string extract_column_text(const std::string& str, uint32_t ofs, uint32_t len)
|
||
{
|
||
if (ofs >= str.size())
|
||
return "";
|
||
|
||
const uint32_t max_len = std::min((uint32_t)str.size() - ofs, len);
|
||
|
||
std::string res(str);
|
||
if (ofs)
|
||
res.erase(0, ofs);
|
||
|
||
if (max_len < res.size())
|
||
res.erase(max_len, res.size());
|
||
|
||
string_trim(res);
|
||
return res;
|
||
}
|
||
|
||
// Note: This doesn't actually handle utf8. It assumes ANSI (code page 252) text input.
|
||
bool load_column_text(const char* pFilename, std::vector<string_vec>& rows, std::string& title, string_vec& col_titles, bool empty_line_seps, const char* pExtra_col_text)
|
||
{
|
||
string_vec lines;
|
||
bool utf8_flag = false;
|
||
if (!read_text_file(pFilename, lines, true, &utf8_flag))
|
||
panic("Failed reading text file %s", pFilename);
|
||
|
||
if (utf8_flag)
|
||
panic("load_column_text() doesn't support utf8 yet");
|
||
|
||
if (!lines.size() || !lines[0].size())
|
||
panic("Expected title");
|
||
|
||
if (lines.size() < 5)
|
||
panic("File too small");
|
||
|
||
for (uint32_t i = 0; i < lines.size(); i++)
|
||
{
|
||
if (lines[i].find_first_of(9) != std::string::npos)
|
||
panic("Tab in file");
|
||
|
||
string_trim(lines[i]);
|
||
}
|
||
|
||
title = lines[0];
|
||
|
||
if (lines[1].size())
|
||
panic("Expected empty line");
|
||
|
||
std::string col_line = lines[2];
|
||
|
||
std::string col_seps = lines[3];
|
||
if ((!col_seps.size()) || (col_seps[0] != '-') || (col_seps.back() != '-'))
|
||
panic("Invalid column seperator line");
|
||
|
||
for (uint32_t i = 0; i < col_seps.size(); i++)
|
||
{
|
||
const uint8_t c = col_seps[i];
|
||
if ((c != ' ') && (c != '-'))
|
||
panic("Invalid column separator line");
|
||
}
|
||
|
||
int col_sep_start = 0;
|
||
std::vector< std::pair<uint32_t, uint32_t> > column_info; // start offset and len of each column in chars
|
||
|
||
for (uint32_t i = 1; i < col_seps.size(); i++)
|
||
{
|
||
const uint8_t c = col_seps[i];
|
||
if (c == ' ')
|
||
{
|
||
if (col_sep_start != -1)
|
||
{
|
||
column_info.push_back(std::make_pair(col_sep_start, i - col_sep_start));
|
||
col_sep_start = -1;
|
||
}
|
||
}
|
||
else
|
||
{
|
||
if (col_sep_start == -1)
|
||
col_sep_start = i;
|
||
}
|
||
}
|
||
|
||
if (col_sep_start != -1)
|
||
{
|
||
column_info.push_back(std::make_pair(col_sep_start, (uint32_t)col_seps.size() - col_sep_start));
|
||
col_sep_start = -1;
|
||
}
|
||
|
||
if (!column_info.size())
|
||
panic("No columns found");
|
||
|
||
col_titles.resize(column_info.size());
|
||
for (uint32_t i = 0; i < column_info.size(); i++)
|
||
{
|
||
col_titles[i] = col_line;
|
||
|
||
if (column_info[i].first)
|
||
col_titles[i].erase(0, column_info[i].first);
|
||
|
||
if (column_info[i].second > col_titles[i].size())
|
||
panic("invalid columns");
|
||
|
||
col_titles[i].erase(column_info[i].second, col_titles[i].size() - column_info[i].second);
|
||
string_trim(col_titles[i]);
|
||
}
|
||
|
||
for (uint32_t i = 0; i < column_info.size() - 1; i++)
|
||
column_info[i].second = column_info[i + 1].first - column_info[i].first;
|
||
column_info.back().second = 32000;
|
||
|
||
uint32_t cur_line = 4;
|
||
|
||
uint32_t cur_record_index = 0;
|
||
|
||
while (cur_line < lines.size())
|
||
{
|
||
string_vec rec_lines;
|
||
rec_lines.push_back(lines[cur_line++]);
|
||
|
||
if (empty_line_seps)
|
||
{
|
||
while (cur_line < lines.size())
|
||
{
|
||
if (!lines[cur_line].size())
|
||
break;
|
||
|
||
rec_lines.push_back(lines[cur_line++]);
|
||
}
|
||
|
||
// cur_line should be blank, or we're at the end of the file
|
||
if (cur_line < lines.size())
|
||
{
|
||
cur_line++;
|
||
if (cur_line < lines.size())
|
||
{
|
||
if (!lines[cur_line].size())
|
||
panic("Expected non-empty line");
|
||
}
|
||
}
|
||
}
|
||
|
||
//uprintf("%u:\n", cur_record_index);
|
||
//for (uint32_t i = 0; i < rec_lines.size(); i++)
|
||
// uprintf("%s\n", rec_lines[i].c_str());
|
||
|
||
string_vec col_lines(column_info.size());
|
||
|
||
for (uint32_t col_index = 0; col_index < column_info.size(); col_index++)
|
||
{
|
||
for (uint32_t l = 0; l < rec_lines.size(); l++)
|
||
{
|
||
std::string col_text(extract_column_text(rec_lines[l], column_info[col_index].first, column_info[col_index].second));
|
||
|
||
if (col_text.size())
|
||
{
|
||
if (col_lines[col_index].size())
|
||
{
|
||
if ((col_lines[col_index].back() != '-') && ((uint8_t)col_lines[col_index].back() != ANSI_SOFT_HYPHEN))
|
||
col_lines[col_index].push_back(' ');
|
||
else
|
||
{
|
||
if ((col_lines[col_index].size() >= 2) && (!isdigit((uint8_t)col_lines[col_index][col_lines[col_index].size() - 2])))
|
||
col_lines[col_index].pop_back();
|
||
}
|
||
}
|
||
|
||
col_lines[col_index] += col_text;
|
||
}
|
||
}
|
||
}
|
||
|
||
if (pExtra_col_text)
|
||
col_lines.push_back(pExtra_col_text);
|
||
|
||
// Convert from ANSI (code page 252) to UTF8.
|
||
for (auto& l : col_lines)
|
||
l = ansi_to_utf8(l);
|
||
|
||
rows.push_back(col_lines);
|
||
|
||
cur_record_index++;
|
||
}
|
||
|
||
return true;
|
||
}
|
||
|
||
bool invoke_curl(const std::string& args, string_vec& reply)
|
||
{
|
||
reply.clear();
|
||
|
||
remove("__temp.html");
|
||
|
||
// Invoke curl.exe
|
||
std::string cmd(string_format("curl.exe \"%s\" -o __temp.html", args.c_str()));
|
||
uprintf("Command: %s\n", cmd.c_str());
|
||
|
||
int status = system(cmd.c_str());
|
||
uprintf("curl returned status %i\n", status);
|
||
|
||
if (status != EXIT_SUCCESS)
|
||
return false;
|
||
|
||
// Read output file.
|
||
|
||
FILE* pFile = ufopen("__temp.html", "rb");
|
||
if (!pFile)
|
||
{
|
||
Sleep(50);
|
||
pFile = ufopen("__temp.html", "rb");
|
||
if (!pFile)
|
||
return false;
|
||
}
|
||
|
||
uint8_t buf[6] = { 0,0,0,0,0,0 };
|
||
fread(buf, 5, 1, pFile);
|
||
fclose(pFile);
|
||
|
||
// Try to detect some common binary file types
|
||
|
||
// PDF
|
||
if (memcmp(buf, "%PDF-", 5) == 0)
|
||
{
|
||
uprintf("PDF file detected\n");
|
||
|
||
std::string filename(args);
|
||
for (size_t i = filename.size() - 1; i >= 0; i--)
|
||
{
|
||
if (filename[i] == '/')
|
||
{
|
||
filename.erase(0, i + 1);
|
||
break;
|
||
}
|
||
}
|
||
|
||
std::string new_link_deescaped;
|
||
for (uint32_t i = 0; i < filename.size(); i++)
|
||
{
|
||
uint8_t c = filename[i];
|
||
if ((c == '%') && ((i + 2) < filename.size()))
|
||
{
|
||
int da = convert_hex_digit(filename[i + 1]);
|
||
int db = convert_hex_digit(filename[i + 2]);
|
||
if (da >= 0 && db >= 0)
|
||
{
|
||
int val = da * 16 + db;
|
||
new_link_deescaped.push_back((uint8_t)val);
|
||
}
|
||
|
||
i += 2;
|
||
}
|
||
else
|
||
new_link_deescaped.push_back(c);
|
||
}
|
||
|
||
rename("__temp.html", new_link_deescaped.c_str());
|
||
uprintf("Renamed __temp.html to %s\n", new_link_deescaped.c_str());
|
||
|
||
return true;
|
||
}
|
||
|
||
// JPEG
|
||
if (memcmp(buf, "\xFF\xD8\xFF\xE0", 4) == 0)
|
||
{
|
||
uprintf("JPEG file detected\n");
|
||
return true;
|
||
}
|
||
|
||
if (!read_text_file("__temp.html", reply, true, nullptr))
|
||
{
|
||
// Wait a bit and try again, rarely needed under Windows.
|
||
Sleep(50);
|
||
if (!read_text_file("__temp.html", reply, true, nullptr))
|
||
return false;
|
||
}
|
||
|
||
return true;
|
||
}
|
||
|
||
void convert_args_to_utf8(string_vec& args, int argc, wchar_t* argv[])
|
||
{
|
||
args.resize(argc);
|
||
|
||
for (int i = 0; i < argc; i++)
|
||
{
|
||
args[i] = wchar_to_utf8(argv[i]);
|
||
if (args[i].size() >= 2)
|
||
{
|
||
if ((args[i][0] == '\"') && (args[i].back() == '\"'))
|
||
{
|
||
args[i].pop_back();
|
||
args[i].erase(0, 1);
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
std::string string_slice(const std::string& str, size_t ofs, size_t len)
|
||
{
|
||
if (!len)
|
||
return "";
|
||
|
||
if (ofs > str.size())
|
||
{
|
||
assert(0);
|
||
return "";
|
||
}
|
||
|
||
const size_t max_len = str.size() - ofs;
|
||
|
||
len = std::min(len, max_len);
|
||
|
||
std::string res(str);
|
||
if (ofs)
|
||
res.erase(0, ofs);
|
||
|
||
if (len)
|
||
res.resize(len);
|
||
|
||
return res;
|
||
}
|
||
|
||
bool invoke_openai(const std::string& prompt, std::string& reply)
|
||
{
|
||
reply.clear();
|
||
|
||
// Write prompt to i.txt
|
||
FILE* pFile = ufopen("i.txt", "wb");
|
||
fwrite(prompt.c_str(), prompt.size(), 1, pFile);
|
||
fclose(pFile);
|
||
|
||
// Invoke openai.exe
|
||
int status = system("openai.exe i.txt o.txt");
|
||
if (status != EXIT_SUCCESS)
|
||
return false;
|
||
|
||
// Read output file.
|
||
string_vec lines;
|
||
if (!read_text_file("o.txt", lines, true, nullptr))
|
||
{
|
||
// Wait a bit and try again, rarely needed under Windows.
|
||
Sleep(50);
|
||
if (!read_text_file("o.txt", lines, true, nullptr))
|
||
return false;
|
||
}
|
||
|
||
// Skip any blank lines at the beginning of the reply.
|
||
uint32_t i;
|
||
for (i = 0; i < lines.size(); i++)
|
||
{
|
||
std::string s(lines[i]);
|
||
string_trim(s);
|
||
if (s.size())
|
||
break;
|
||
}
|
||
|
||
for (; i < lines.size(); i++)
|
||
reply += lines[i];
|
||
|
||
return true;
|
||
}
|
||
|
||
bool invoke_openai(const string_vec &prompt, string_vec &reply)
|
||
{
|
||
reply.clear();
|
||
|
||
if (!write_text_file("i.txt", prompt, true))
|
||
return false;
|
||
|
||
// Invoke openai.exe
|
||
const uint32_t MAX_TRIES = 3;
|
||
uint32_t num_tries;
|
||
|
||
for (num_tries = 0; num_tries < MAX_TRIES; ++num_tries)
|
||
{
|
||
if (num_tries)
|
||
uprintf("openai.exe failed - retrying\n");
|
||
|
||
int status = system("openai.exe i.txt o.txt");
|
||
if (status == EXIT_SUCCESS)
|
||
break;
|
||
Sleep(2000);
|
||
}
|
||
|
||
if (num_tries == MAX_TRIES)
|
||
return false;
|
||
|
||
// Read output file.
|
||
if (!read_text_file("o.txt", reply, true, nullptr))
|
||
{
|
||
// Wait a bit and try again, rarely needed under Windows.
|
||
Sleep(50);
|
||
if (!read_text_file("o.txt", reply, true, nullptr))
|
||
return false;
|
||
}
|
||
|
||
return true;
|
||
}
|
||
|
||
std::string get_deg_to_dms(double deg)
|
||
{
|
||
deg = std::round(fabs(deg) * 3600.0f);
|
||
|
||
int min_secs = (int)fmod(deg, 3600.0f);
|
||
|
||
deg = std::floor((deg - (double)min_secs) / 3600.0f);
|
||
|
||
int minutes = min_secs / 60;
|
||
int secs = min_secs % 60;
|
||
|
||
return string_format("%02i%:%02i:%02i", (int)deg, minutes, secs);
|
||
}
|
||
|
||
bool load_json_object(const char* pFilename, bool& utf8_flag, json &result_obj)
|
||
{
|
||
std::vector<uint8_t> buf;
|
||
|
||
if (!read_text_file(pFilename, buf, &utf8_flag))
|
||
return false;
|
||
|
||
if (!buf.size())
|
||
return false;
|
||
|
||
bool success = false;
|
||
try
|
||
{
|
||
result_obj = json::parse(buf.begin(), buf.end());
|
||
success = true;
|
||
}
|
||
catch (json::exception& e)
|
||
{
|
||
fprintf(stderr, "load_json_object: Parse of file \"%s\" failed (id %i): %s", pFilename, e.id, e.what());
|
||
return false;
|
||
}
|
||
|
||
if (!result_obj.is_object() && !result_obj.is_array())
|
||
return false;
|
||
|
||
return true;
|
||
}
|
||
|
||
void string_tokenize(
|
||
const std::string &str,
|
||
const std::string &whitespace,
|
||
const std::string &break_chars,
|
||
string_vec &tokens,
|
||
uint_vec *pOffsets_vec)
|
||
{
|
||
tokens.resize(0);
|
||
if (pOffsets_vec)
|
||
pOffsets_vec->resize(0);
|
||
|
||
std::string cur_token;
|
||
uint32_t cur_ofs = 0;
|
||
|
||
for (uint32_t i = 0; i < str.size(); i++)
|
||
{
|
||
uint8_t c = str[i];
|
||
|
||
if (whitespace.find_first_of(c) != std::string::npos)
|
||
{
|
||
if (cur_token.size())
|
||
{
|
||
tokens.push_back(cur_token);
|
||
if (pOffsets_vec)
|
||
pOffsets_vec->push_back(cur_ofs);
|
||
|
||
cur_token.clear();
|
||
}
|
||
}
|
||
else if (break_chars.find_first_of(c) != std::string::npos)
|
||
{
|
||
if (cur_token.size())
|
||
{
|
||
tokens.push_back(cur_token);
|
||
if (pOffsets_vec)
|
||
pOffsets_vec->push_back(cur_ofs);
|
||
|
||
cur_token.clear();
|
||
}
|
||
|
||
std::string s;
|
||
s.push_back(c);
|
||
|
||
tokens.push_back(s);
|
||
if (pOffsets_vec)
|
||
pOffsets_vec->push_back(i);
|
||
}
|
||
else
|
||
{
|
||
if (!cur_token.size())
|
||
cur_ofs = i;
|
||
|
||
cur_token.push_back(c);
|
||
}
|
||
}
|
||
|
||
if (cur_token.size())
|
||
{
|
||
tokens.push_back(cur_token);
|
||
if (pOffsets_vec)
|
||
pOffsets_vec->push_back(cur_ofs);
|
||
}
|
||
}
|
||
|
||
const double PI = 3.141592653589793238463;
|
||
|
||
double deg2rad(double deg)
|
||
{
|
||
return (deg * PI / 180.0);
|
||
}
|
||
|
||
double rad2deg(double rad)
|
||
{
|
||
return (rad * 180.0 / PI);
|
||
}
|
||
|
||
// input in degrees
|
||
double geo_distance(double lat1, double lon1, double lat2, double lon2, int unit)
|
||
{
|
||
if ((lat1 == lat2) && (lon1 == lon2))
|
||
return 0;
|
||
|
||
double theta = lon1 - lon2;
|
||
double dist = cos(deg2rad(lat1)) * cos(deg2rad(lat2)) * cos(deg2rad(theta)) + sin(deg2rad(lat1)) * sin(deg2rad(lat2));
|
||
dist = acos(dist);
|
||
dist = rad2deg(dist);
|
||
|
||
dist = dist * 60 * 1.1515;
|
||
|
||
switch (unit)
|
||
{
|
||
case 'M':
|
||
break;
|
||
case 'K':
|
||
dist = dist * 1.609344;
|
||
break;
|
||
case 'N':
|
||
dist = dist * 0.8684;
|
||
break;
|
||
default:
|
||
assert(0);
|
||
break;
|
||
}
|
||
|
||
return (dist);
|
||
}
|
||
|
||
std::string remove_bom(std::string str)
|
||
{
|
||
if (str.size() >= 3)
|
||
{
|
||
if (((uint8_t)str[0] == UTF8_BOM0) && ((uint8_t)str[1] == UTF8_BOM1) && ((uint8_t)str[2] == UTF8_BOM2))
|
||
{
|
||
str.erase(0, 3);
|
||
}
|
||
}
|
||
|
||
return str;
|
||
}
|
||
|
||
int get_next_utf8_code_point_len(const uint8_t* pStr)
|
||
{
|
||
if (pStr == nullptr || *pStr == 0)
|
||
{
|
||
// Return 0 if the input is null or points to a null terminator
|
||
return 0;
|
||
}
|
||
|
||
const uint8_t firstByte = *pStr;
|
||
|
||
if ((firstByte & 0x80) == 0)
|
||
{
|
||
// Starts with 0, ASCII character
|
||
return 1;
|
||
}
|
||
else if ((firstByte & 0xE0) == 0xC0)
|
||
{
|
||
// Starts with 110
|
||
return 2;
|
||
}
|
||
else if ((firstByte & 0xF0) == 0xE0)
|
||
{
|
||
// Starts with 1110
|
||
return 3;
|
||
}
|
||
else if ((firstByte & 0xF8) == 0xF0)
|
||
{
|
||
// Starts with 11110
|
||
return 4;
|
||
}
|
||
else
|
||
{
|
||
// Invalid UTF-8 byte sequence
|
||
return -1;
|
||
}
|
||
}
|
||
|
||
void get_string_words(
|
||
const std::string& str,
|
||
string_vec& words,
|
||
uint_vec* pOffsets_vec,
|
||
const char* pAdditional_whitespace)
|
||
{
|
||
const uint8_t* pStr = (const uint8_t *)str.c_str();
|
||
|
||
words.resize(0);
|
||
if (pOffsets_vec)
|
||
pOffsets_vec->resize(0);
|
||
|
||
std::string cur_token;
|
||
|
||
std::string whitespace(" \t\n\r,;:.!?()[]*/\"");
|
||
if (pAdditional_whitespace)
|
||
whitespace += std::string(pAdditional_whitespace);
|
||
|
||
int word_start_ofs = -1;
|
||
|
||
uint32_t cur_ofs = 0;
|
||
while ((cur_ofs < str.size()) && (pStr[cur_ofs]))
|
||
{
|
||
int l = get_next_utf8_code_point_len(pStr + cur_ofs);
|
||
const uint8_t c = pStr[cur_ofs];
|
||
|
||
if (l <= 0)
|
||
{
|
||
assert(0);
|
||
l = 1;
|
||
}
|
||
|
||
bool is_whitespace = (whitespace.find_first_of(c) != std::string::npos);
|
||
|
||
if ((l == 2) && (c == 0xc2))
|
||
{
|
||
// NO-BREAK SPACE
|
||
if (pStr[cur_ofs + 1] == 0xa0)
|
||
is_whitespace = true;
|
||
}
|
||
|
||
if ((l == 2) && (c == 0xCA))
|
||
{
|
||
// single left quote
|
||
if (pStr[cur_ofs + 1] == 0xBB)
|
||
is_whitespace = true;
|
||
}
|
||
|
||
if ((l == 3) && (c == 0xE2) && (pStr[cur_ofs + 1] == 0x80))
|
||
{
|
||
// dash
|
||
if (pStr[cur_ofs + 2] == 0x93)
|
||
is_whitespace = true;
|
||
// dash
|
||
else if (pStr[cur_ofs + 2] == 0x94)
|
||
is_whitespace = true;
|
||
// left quote
|
||
else if (pStr[cur_ofs + 2] == 0x9C)
|
||
is_whitespace = true;
|
||
// right quote
|
||
else if (pStr[cur_ofs + 2] == 0x9D)
|
||
is_whitespace = true;
|
||
// ellipsis (three dots)
|
||
else if (pStr[cur_ofs + 2] == 0xA)
|
||
is_whitespace = true;
|
||
// ellipsis (three dots)
|
||
else if (pStr[cur_ofs + 2] == 0xA6)
|
||
is_whitespace = true;
|
||
// long dash
|
||
else if (pStr[cur_ofs + 2] == 9)
|
||
is_whitespace = true;
|
||
// left single quote
|
||
else if (pStr[cur_ofs + 2] == 0x98)
|
||
is_whitespace = true;
|
||
// right single quote
|
||
else if (pStr[cur_ofs + 2] == 0x99)
|
||
is_whitespace = true;
|
||
// right double quote
|
||
else if (pStr[cur_ofs + 2] == 0x9D)
|
||
is_whitespace = true;
|
||
}
|
||
|
||
if (is_whitespace)
|
||
{
|
||
if (cur_token.size())
|
||
{
|
||
words.push_back(cur_token);
|
||
if (pOffsets_vec)
|
||
pOffsets_vec->push_back(word_start_ofs);
|
||
|
||
cur_token.clear();
|
||
word_start_ofs = -1;
|
||
}
|
||
}
|
||
else
|
||
{
|
||
if (word_start_ofs < 0)
|
||
word_start_ofs = cur_ofs;
|
||
|
||
if (l == 1)
|
||
{
|
||
cur_token.push_back(utolower(c));
|
||
}
|
||
else
|
||
{
|
||
for (int i = 0; i < l; i++)
|
||
cur_token.push_back(pStr[cur_ofs + i]);
|
||
}
|
||
}
|
||
|
||
cur_ofs += l;
|
||
}
|
||
|
||
if (cur_token.size())
|
||
{
|
||
words.push_back(cur_token);
|
||
|
||
if (pOffsets_vec)
|
||
pOffsets_vec->push_back(word_start_ofs);
|
||
}
|
||
}
|
||
|
||
void get_utf8_code_point_offsets(const char* pStr, int_vec& offsets)
|
||
{
|
||
uint32_t cur_ofs = 0;
|
||
|
||
offsets.resize(0);
|
||
|
||
while (pStr[cur_ofs])
|
||
{
|
||
offsets.push_back(cur_ofs);
|
||
|
||
cur_ofs += std::max<int>(1, get_next_utf8_code_point_len((const uint8_t*)pStr + cur_ofs));
|
||
}
|
||
}
|
||
|
||
struct char_map
|
||
{
|
||
const char32_t* m_pFrom;
|
||
const char m_to;
|
||
};
|
||
|
||
static const char_map g_char_norm_up[] =
|
||
{
|
||
{ U"ÁĂẮẶẰẲẴǍÂẤẬẦẨẪÄǞȦǠẠȀÀẢȂĀĄÅǺḀÃǼǢȺΆ", 'A' },
|
||
{ U"ḂḄḆƁƂƄ", 'B' },
|
||
{ U"ĆČÇḈĈĊƇȻƆ", 'C' },
|
||
{ U"ĎḐḒḊḌḎĐƉƊƋDZDzDŽ", 'D' },
|
||
{ U"ÉĔĚȨḜÊẾỆỀỂỄḘËĖẸȄÈẺȆĒḖḔĘẼḚÈÊËĒĔĖĘĚƐƎƏȄȆȨΈΉΕƐƐ", 'E' },
|
||
{ U"ḞƑ", 'F' },
|
||
{ U"ǴĞǦĢĜĠḠĜĞĠĢƓǤǦǴƔ", 'G' },
|
||
{ U"ḪȞḨĤḦḢḤĤĦǶȞΗǶ", 'H' },
|
||
{ U"ÍĬǏÎÏḮİỊȈÌỈȊĪĮĨḬÌÍÎÏĨĪĬĮİƗǏȈȊ", 'I' },
|
||
{ U"ĴĴ", 'J' },
|
||
{ U"ḰǨĶḲḴĶƘǨΚ", 'K' },
|
||
{ U"ĹĽĻḼḶḸḺĹĻĽĿŁΛ", 'L' },
|
||
{ U"ḾṀṂƜ", 'M' },
|
||
{ U"ŃŇŅṊṄṆǸṈÑÑŃŅŇŊƝǸΝ", 'N' },
|
||
{ U"ÓŎǑÔỐỘỒỔỖÖȪȮȰỌŐȌÒỎƠỚỢỜỞỠȎŌṒṐǪǬÕṌṎȬǾØÒÓÔÕÖØŌŎŐƟƠǑǪǬǾȌȎȪȬȮȰΌΟΩ", 'O' },
|
||
{ U"ṔṖΠΡΦ", 'P' },
|
||
{ U"ŔŘŖṘṚṜȐȒṞŔŖŘƦȐȒ", 'R' },
|
||
{ U"ŚṤŠṦŞŜȘṠṢṨߌŜŞŠƩȘΣ", 'S' },
|
||
{ U"ŤŢṰȚṪṬṮŢŤŦƬƮȚΤ", 'T' },
|
||
{ U"ÚŬǓÛṶÜǗǙǛǕṲỤŰȔÙỦƯỨỰỪỬỮȖŪṺŲŮŨṸṴÙÚÛÜŨŪŬŮŰŲƯǓǕǗǙǛȔȖ", 'U' },
|
||
{ U"ṾṼƲ", 'V' },
|
||
{ U"ẂŴẄẆẈẀŴ", 'W' },
|
||
{ U"ẌẊΧΞ", 'X' },
|
||
{ U"ÝŶŸẎỴỲỶȲỸÝŶŸƳȲΥΎΫ", 'Y' },
|
||
{ U"ŹŽẐŻẒẔŹŻŽƵƷǮȤΖ", 'Z' },
|
||
};
|
||
|
||
static const char_map g_char_norm_lower[] =
|
||
{
|
||
{ U"áăắặằẳẵǎâấậầẩẫäǟȧǡạȁàảȃāąåǻḁãǽǣⱥάàáâãäåāăąǎǟǡǻȁȃȧάα", 'a' },
|
||
{ U"ḃḅḇɓƃƅƀƃβƀƃƅ", 'b' },
|
||
{ U"ćčçḉĉċƈȼɔƈçćĉċčƈȼ", 'c' },
|
||
{ U"ďḑḓḋḍḏđɖɗƌdzdzdžƌďđƌdzdžȡďđƌdzdžȡ", 'd' },
|
||
{ U"éĕěȩḝêếệềểễḙëėẹȅèẻȇēḗḕęẽḛèêëēĕėęěɛǝəȅȇȩέήεɛɛèéêëēĕėęěȅȇȩε", 'e' },
|
||
{ U"ḟƒ", 'f' },
|
||
{ U"ǵğǧģĝġḡĝğġģɠǥǧǵɣĝğġģǧǵ", 'g' },
|
||
{ U"ḫȟḩĥḧḣḥẖĥħƕƕȟƕĥħȟ", 'h' },
|
||
{ U"íĭǐîïḯiịȉìỉȋīįĩḭìíîïĩīĭįiɨǐȉȋìíîïĩīĭįǐȉȋι", 'i' },
|
||
{ U"ǰĵĵǰĵǰ", 'j' },
|
||
{ U"ḱǩķḳḵķƙǩκƙķƙǩκ", 'k' },
|
||
{ U"ĺľļḽḷḹḻĺļľŀłƚƛλƚĺļľŀłƚλƚ", 'l' },
|
||
{ U"ḿṁṃɯ", 'm' },
|
||
{ U"ńňņṋṅṇǹṉññńņňŋɲǹνƞñńņňʼnŋƞǹη", 'n' },
|
||
{ U"óŏǒôốộồổỗöȫȯȱọőȍòỏơớợờởỡȏōṓṑǫǭõṍṏȭǿøòóôõöøōŏőɵơǒǫǭǿȍȏȫȭȯȱόοòóôõöøōŏőơǒǫǭǿȍȏȫȭȯȱοσ", 'o' },
|
||
{ U"ṕṗπφƥ", 'p' },
|
||
{ U"ŕřŗṙṛṝȑȓṟŕŗřʀȑȓρŕŗřȑȓρ", 'r' },
|
||
{ U"śṥšṧşŝșṡẛṣṩśŝşšʃșƨśŝşšșƨȿ", 's' },
|
||
{ U"ťţṱțẗṫṭṯţťŧƭʈțτƫţťŧƭțτ", 't' },
|
||
{ U"úŭǔûṷüǘǚǜǖṳụűȕùủưứựừửữȗūṻųůũṹṵùúûüũūŭůűųưǔǖǘǚǜȕȗưùúûüũūŭůűųưǔǖǘǚǜȕȗμ", 'u' },
|
||
{ U"ṿṽʋ", 'v' },
|
||
{ U"ẃŵẅẇẉẁẘŵŵω", 'w' },
|
||
{ U"ẍẋχξχξ", 'x' },
|
||
{ U"ýŷÿẏỵỳỷȳẙỹýŷÿƴȳυύϋƴýÿŷƴȳγψ", 'y' },
|
||
{ U"źžẑżẓẕźżžƶʒǯȥζƶźżžƶƹȥζ", 'z' },
|
||
};
|
||
|
||
std::map<int, int> g_upper_trans;
|
||
std::map<int, int> g_lower_trans;
|
||
|
||
static const char* g_stop_words[] =
|
||
{
|
||
"a", "about", "above", "after", "again", "against", "all", "am", "an", "and", "any", "are", "as",
|
||
"at", "be", "because", "been", "before", "being", "below", "between", "both", "but", "by", "can",
|
||
"could", "did", "do", "does", "doing", "down", "during", "each", "few", "for", "from",
|
||
"further", "had", "has", "have", "having", "he", "her", "here", "hers", "herself", "him", "himself",
|
||
"his", "how", "i", "if", "in", "into", "is", "it", "its", "itself", "just", "me", "more", "most",
|
||
"my", "myself", "no", "nor", "not", "now", "of", "off", "on", "once", "only", "or", "other", "our",
|
||
"ours", "ourselves", "out", "over", "own", "re", "same", "she", "should", "so", "some", "such",
|
||
"than", "that", "the", "their", "theirs", "them", "themselves", "then", "there", "these", "they",
|
||
"this", "those", "through", "to", "too", "under", "until", "up", "very", "was", "we", "were", "what",
|
||
"when", "where", "which", "while", "who", "whom", "why", "will", "with", "you", "your", "yours",
|
||
"yourself", "yourselves", "although", "also", "already", "another", "seemed", "seem", "seems"
|
||
};
|
||
static const uint32_t NUM_STOP_WORDS = (uint32_t)std::size(g_stop_words);
|
||
|
||
std::set<std::string> g_stop_words_set;
|
||
|
||
void init_norm()
|
||
{
|
||
g_stop_words_set.clear();
|
||
for (const auto& str : g_stop_words)
|
||
g_stop_words_set.insert(str);
|
||
|
||
for (uint32_t i = 0; i < std::size(g_char_norm_up); i++)
|
||
{
|
||
const char32_t* pFrom = g_char_norm_up[i].m_pFrom;
|
||
char to_char = g_char_norm_up[i].m_to;
|
||
|
||
while (*pFrom)
|
||
{
|
||
char32_t fc = *pFrom++;
|
||
|
||
auto f = g_upper_trans.find(fc);
|
||
if (f != g_upper_trans.end())
|
||
{
|
||
if (f->second != to_char)
|
||
{
|
||
uprintf("Upper char %u 0x%x is redundant\n", fc, fc);
|
||
exit(1);
|
||
}
|
||
}
|
||
|
||
g_upper_trans[fc] = to_char;
|
||
}
|
||
}
|
||
|
||
for (uint32_t i = 0; i < std::size(g_char_norm_lower); i++)
|
||
{
|
||
const char32_t* pFrom = g_char_norm_lower[i].m_pFrom;
|
||
char to_char = g_char_norm_lower[i].m_to;
|
||
|
||
while (*pFrom)
|
||
{
|
||
char32_t fc = *pFrom++;
|
||
|
||
auto f = g_upper_trans.find(fc);
|
||
if (f != g_upper_trans.end())
|
||
{
|
||
uprintf("Lower char %u 0x%x is in the upper table\n", fc, fc);
|
||
|
||
if (utolower((uint8_t)f->second) != to_char)
|
||
uprintf("Conversion mismatch %u 0x%x\n", fc, fc);
|
||
|
||
//exit(1);
|
||
}
|
||
|
||
f = g_lower_trans.find(fc);
|
||
if (f != g_lower_trans.end())
|
||
{
|
||
if (f->second != to_char)
|
||
{
|
||
uprintf("Lower char %u 0x%x is redundant\n", fc, fc);
|
||
exit(1);
|
||
}
|
||
}
|
||
|
||
g_lower_trans[fc] = to_char;
|
||
}
|
||
}
|
||
}
|
||
|
||
// Resulting characters are guaranteed to be <128 - useful for searching purposes.
|
||
// Unrecognized Unicode characters are deleted.
|
||
void normalize_diacritics(const char* pStr, std::string& res)
|
||
{
|
||
assert(g_stop_words_set.size());
|
||
|
||
res.resize(0);
|
||
|
||
while (*pStr)
|
||
{
|
||
int l = get_next_utf8_code_point_len((const uint8_t*)pStr);
|
||
const uint8_t c = *pStr;
|
||
|
||
utf8_int32_t cp;
|
||
char* pStr_next = utf8codepoint(pStr, &cp);
|
||
|
||
assert((pStr_next - pStr) == l);
|
||
|
||
if (cp < 128)
|
||
{
|
||
res.push_back((char)cp);
|
||
pStr = pStr_next;
|
||
continue;
|
||
}
|
||
|
||
int new_char = -1;
|
||
|
||
auto u_it = g_upper_trans.find(cp);
|
||
auto l_it = g_lower_trans.find(cp);
|
||
|
||
if (u_it != g_upper_trans.end())
|
||
new_char = u_it->second;
|
||
else if (l_it != g_lower_trans.end())
|
||
new_char = l_it->second;
|
||
else
|
||
{
|
||
// FIXME: this is lame, it parses the utf8 directly.
|
||
|
||
if ((l == 2) && (c == 0xc2))
|
||
{
|
||
// NO-BREAK SPACE
|
||
if ((uint8_t)pStr[1] == 0xa0)
|
||
new_char = ' ';
|
||
}
|
||
|
||
if ((l == 2) && (c == 0xCA))
|
||
{
|
||
// single left quote
|
||
if ((uint8_t)pStr[1] == 0xBB)
|
||
new_char = '\'';
|
||
}
|
||
|
||
if ((l == 3) && (c == 0xE2) && ((uint8_t)pStr[1] == 0x80))
|
||
{
|
||
// dash
|
||
if ((uint8_t)pStr[2] == 0x93)
|
||
new_char = '-';
|
||
// dash
|
||
else if ((uint8_t)pStr[2] == 0x94)
|
||
new_char = '-';
|
||
// left quote
|
||
else if ((uint8_t)pStr[2] == 0x9C)
|
||
new_char = '"';
|
||
// right quote
|
||
else if ((uint8_t)pStr[2] == 0x9D)
|
||
new_char = '"';
|
||
// ellipsis (three dots)
|
||
else if ((uint8_t)pStr[2] == 0xA)
|
||
new_char = '.';
|
||
// ellipsis (three dots)
|
||
else if ((uint8_t)pStr[2] == 0xA6)
|
||
new_char = '.';
|
||
// long dash
|
||
else if ((uint8_t)pStr[2] == 9)
|
||
new_char = '-';
|
||
// left single quote
|
||
else if ((uint8_t)pStr[2] == 0x98)
|
||
new_char = '\'';
|
||
// right single quote
|
||
else if ((uint8_t)pStr[2] == 0x99)
|
||
new_char = '\'';
|
||
// right double quote
|
||
else if ((uint8_t)pStr[2] == 0x9D)
|
||
new_char = '"';
|
||
}
|
||
}
|
||
|
||
// TODO: Do something smarter?
|
||
if (new_char != -1)
|
||
res.push_back((char)new_char);
|
||
|
||
pStr = pStr_next;
|
||
}
|
||
}
|
||
|
||
std::string normalize_word(const std::string& str)
|
||
{
|
||
assert(g_stop_words_set.size());
|
||
|
||
const uint32_t MAX_STRING_SIZE = 4096;
|
||
|
||
if (str.size() > MAX_STRING_SIZE)
|
||
panic("String too long");
|
||
|
||
char buf[MAX_STRING_SIZE + 1];
|
||
strcpy_s(buf, sizeof(buf), str.c_str());
|
||
|
||
// Convert utf8 string to lower
|
||
utf8lwr(buf);
|
||
|
||
// Remove diacritics and some specials from utf8, this preserves all 1-127 chars
|
||
std::string norm;
|
||
norm.reserve(strlen(buf));
|
||
|
||
normalize_diacritics(buf, norm);
|
||
|
||
// Remove any non-letter or non-digit characters (we assume this is a word, so whitespace gets removed too)
|
||
std::string temp;
|
||
temp.reserve(norm.size());
|
||
|
||
for (uint32_t i = 0; i < norm.size(); i++)
|
||
{
|
||
uint8_t c = norm[i];
|
||
|
||
c = utolower(c);
|
||
|
||
if (uislower(c) || uisdigit(c))
|
||
temp.push_back(c);
|
||
}
|
||
|
||
// Stem word
|
||
strcpy_s(buf, sizeof(buf), temp.c_str());
|
||
if (buf[0])
|
||
{
|
||
int32_t new_len = stem(buf, 0, (int)strlen(buf) - 1);
|
||
buf[new_len + 1] = '\0';
|
||
}
|
||
|
||
return buf;
|
||
}
|
||
|
||
// Assumes word is plain ASCII lowercase
|
||
bool is_stop_word(const std::string &word)
|
||
{
|
||
assert(g_stop_words_set.size());
|
||
|
||
return g_stop_words_set.count(word) != 0;
|
||
}
|
||
|
||
std::string ustrlwr(const std::string& s)
|
||
{
|
||
const size_t l = s.size();
|
||
|
||
std::vector<uint8_t> temp;
|
||
temp.resize(l + 1);
|
||
|
||
memcpy(&temp[0], s.c_str(), l);
|
||
temp[l] = '\0';
|
||
|
||
utf8lwr((char *)&temp[0]);
|
||
|
||
return (char *)&temp[0];
|
||
}
|
||
|
||
std::string string_replace(const std::string& str, const std::string& find, const std::string& repl)
|
||
{
|
||
assert(find.size());
|
||
if (!find.size() || !str.size())
|
||
return str;
|
||
|
||
const uint8_t* pStr = (const uint8_t *)str.c_str();
|
||
const size_t str_size = str.size();
|
||
|
||
const uint8_t* pFind = (const uint8_t*)find.c_str();
|
||
const size_t find_size = find.size();
|
||
|
||
std::string res;
|
||
res.reserve(str.size());
|
||
|
||
size_t str_ofs = 0;
|
||
while (str_ofs < str.size())
|
||
{
|
||
int str_char_size = get_next_utf8_code_point_len(pStr + str_ofs);
|
||
if (str_char_size < 0)
|
||
{
|
||
assert(0);
|
||
str_char_size = 1;
|
||
}
|
||
|
||
const size_t str_remaining = str_size - str_ofs;
|
||
if ((str_remaining >= find_size) && (memcmp(pStr + str_ofs, pFind, find_size) == 0))
|
||
{
|
||
res += repl;
|
||
str_ofs += find_size;
|
||
}
|
||
else
|
||
{
|
||
for (int i = 0; i < str_char_size; i++)
|
||
res.push_back((char)pStr[str_ofs + i]);
|
||
str_ofs += str_char_size;
|
||
}
|
||
}
|
||
|
||
return res;
|
||
}
|
||
|
||
bool does_file_exist(const char* pFilename)
|
||
{
|
||
FILE* pFile = ufopen(pFilename, "rb");
|
||
if (!pFile)
|
||
return false;
|
||
|
||
fclose(pFile);
|
||
return true;
|
||
}
|
||
|
||
#if 0
|
||
int calculateJDN(int day, int month, int year) {
|
||
int a = (14 - month) / 12;
|
||
int y = year + 4800 - a;
|
||
int m = month + 12 * a - 3;
|
||
|
||
return day + (153 * m + 2) / 5 + 365 * y + y / 4 - y / 100 + y / 400 - 32045;
|
||
}
|
||
#endif
|
||
|
||
int get_julian_date(int M, int D, int Y)
|
||
{
|
||
assert((M >= 1) && (M <= 12));
|
||
assert((D >= 1) && (D <= 31));
|
||
|
||
int JDN = (1461 * (Y + 4800 + (M - 14) / 12)) / 4 + (367 * (M - 2 - 12 * ((M - 14) / 12))) / 12 - (3 * ((Y + 4900 + (M - 14) / 12) / 100)) / 4 + D - 32075;
|
||
|
||
return JDN;
|
||
}
|
||
|
||
int get_day_of_week(int julian)
|
||
{
|
||
assert(julian >= 0);
|
||
return (julian + 1) % 7;
|
||
}
|