mirror of
https://github.com/monero-project/monero.git
synced 2024-10-01 11:49:47 -04:00
Adding libglim as an external library
libglim is an Apache-licensed C++ wrapper for lmdb, and rather than rolling our own it seems prudent to use it. Note: lmdb is not included in it, and unless something happens as did with libunbound, should be installed via each OS' package manager or equivalent.
This commit is contained in:
Thomas Winget
warptangent
parent
07733f98c0
commit
90d6f8bf62
62
external/glim/ChangeLog.darcs.txt
vendored
Normal file
62
external/glim/ChangeLog.darcs.txt
vendored
Normal file
@ -0,0 +1,62 @@
|
||||
Wed Dec 6 23:50:10 Russian Standard Time 2006 ArtemGr <artem@bizlink.ru>
|
||||
* Fixes discovered under FreeBSD.
|
||||
|
||||
Wed Dec 6 04:43:04 Russian Standard Time 2006 ArtemGr <artem@bizlink.ru>
|
||||
* Ported to Gentoo gcc 4.1.1
|
||||
|
||||
Fri Nov 24 14:17:08 Russian Standard Time 2006 ArtemGr <artem@bizlink.ru>
|
||||
* Documentation improvements for the release.
|
||||
|
||||
Fri Nov 24 13:13:50 Russian Standard Time 2006 ArtemGr <artem@bizlink.ru>
|
||||
* A few fixes.
|
||||
|
||||
Fri Nov 24 13:13:20 Russian Standard Time 2006 ArtemGr <artem@bizlink.ru>
|
||||
* Do not remove the "html" directory when cleaning.
|
||||
|
||||
Wed Nov 22 15:30:56 Russian Standard Time 2006 ArtemGr <artem@bizlink.ru>
|
||||
* Added a compatibility #pair method to bufstr_t.
|
||||
|
||||
Tue Nov 21 21:54:40 Russian Standard Time 2006 ArtemGr <artem@bizlink.ru>
|
||||
* Implemented the sugared way to querying.
|
||||
|
||||
Mon Oct 9 23:19:11 Russian Daylight Time 2006 ArtemGr <artem@bizlink.ru>
|
||||
* Fixed incorrect statement = NULL in #step. Implemented parameter bindings.
|
||||
|
||||
Mon Oct 9 20:09:17 Russian Daylight Time 2006 ArtemGr <artem@bizlink.ru>
|
||||
* SqliteQuery and SqliteParQuery classes (no binding support as of yet).
|
||||
|
||||
Fri Oct 6 12:11:43 Russian Daylight Time 2006 ArtemGr <artem@bizlink.ru>
|
||||
* typo
|
||||
|
||||
Thu Oct 5 23:26:21 Russian Daylight Time 2006 ArtemGr <artem@bizlink.ru>
|
||||
* Basic mutex operations in SqliteSession.
|
||||
|
||||
Sun Oct 1 23:19:36 Russian Daylight Time 2006 ArtemGr <artem@bizlink.ru>
|
||||
* Compatibility with std::string.
|
||||
|
||||
Fri Sep 29 11:42:09 Russian Daylight Time 2006 ArtemGr <artem@bizlink.ru>
|
||||
* Invented SqliteSession.
|
||||
|
||||
Fri Sep 29 01:23:31 Russian Daylight Time 2006 ArtemGr <artem@bizlink.ru>
|
||||
* SQLite wrapper: initial documentation; opening, closing.
|
||||
|
||||
Thu Sep 28 23:15:37 Russian Daylight Time 2006 ArtemGr <artem@bizlink.ru>
|
||||
* Apache version 2 license.
|
||||
|
||||
Thu Sep 28 23:12:41 Russian Daylight Time 2006 ArtemGr <artem@bizlink.ru>
|
||||
* Multiple source files for tests.
|
||||
|
||||
Thu Sep 28 01:05:21 Russian Daylight Time 2006 ArtemGr <artem@bizlink.ru>
|
||||
* Append from another bufstr_t and from a pair.
|
||||
|
||||
Thu Sep 28 01:04:46 Russian Daylight Time 2006 ArtemGr <artem@bizlink.ru>
|
||||
* Macro to construct pair from C character array.
|
||||
|
||||
Tue Sep 26 23:23:40 Russian Daylight Time 2006 ArtemGr <artem@bizlink.ru>
|
||||
* char const* instead of const char*
|
||||
|
||||
Tue Sep 26 20:56:07 Russian Daylight Time 2006 ArtemGr <artem@bizlink.ru>
|
||||
* Everything seems to work. The library is now headers-only.
|
||||
|
||||
Mon Sep 25 22:52:34 Russian Daylight Time 2006 ArtemGr <artem@bizlink.ru>
|
||||
* Initial revision, containing bufstr_t; compiles, but does not work.
|
||||
13
external/glim/LICENSE
vendored
Executable file
13
external/glim/LICENSE
vendored
Executable file
@ -0,0 +1,13 @@
|
||||
Copyright 2006-2012 Kozarezov Artem Aleksandrovich <artemciy@gmail.com>
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
51
external/glim/NsecTimer.hpp
vendored
Normal file
51
external/glim/NsecTimer.hpp
vendored
Normal file
@ -0,0 +1,51 @@
|
||||
#ifndef _NSEC_TIMER_H
|
||||
#define _NSEC_TIMER_H
|
||||
|
||||
#include <time.h> // clock_gettime, CLOCK_MONOTONIC
|
||||
#include <stdint.h>
|
||||
#include <string>
|
||||
#include <sstream>
|
||||
|
||||
namespace glim {
|
||||
|
||||
//! Safe nanoseconds timer.
|
||||
struct NsecTimer {
|
||||
timespec start;
|
||||
NsecTimer () {restart();}
|
||||
//! Nanoseconds since the creation or restart of the timer.
|
||||
int64_t operator()() const {
|
||||
timespec nsecStop; clock_gettime (CLOCK_MONOTONIC, &nsecStop);
|
||||
return (int64_t) (nsecStop.tv_sec - start.tv_sec) * 1000000000LL + (int64_t) (nsecStop.tv_nsec - start.tv_nsec);
|
||||
}
|
||||
/** Seconds since the creation or restart of the timer. */
|
||||
double sec() const {
|
||||
timespec nsecStop; clock_gettime (CLOCK_MONOTONIC, &nsecStop);
|
||||
double seconds = nsecStop.tv_sec - start.tv_sec;
|
||||
seconds += (double)(nsecStop.tv_nsec - start.tv_nsec) / 1000000000.0;
|
||||
return seconds;
|
||||
}
|
||||
//! Seconds since the creation or restart of the timer.
|
||||
std::string seconds (int precision = 9) const {
|
||||
// The trick is to avoid the scientific notation by printing integers.
|
||||
double sec = this->sec();
|
||||
std::ostringstream buf;
|
||||
int isec = (int) sec;
|
||||
buf << isec;
|
||||
|
||||
sec -= isec;
|
||||
for (int pc = precision; pc; --pc) sec *= 10.0;
|
||||
int ifrac = (int) sec;
|
||||
if (ifrac > 0) {
|
||||
buf << '.';
|
||||
buf.fill ('0'); buf.width (precision);
|
||||
buf << ifrac;
|
||||
}
|
||||
return buf.str();
|
||||
}
|
||||
void restart() {clock_gettime (CLOCK_MONOTONIC, &start);}
|
||||
int64_t getAndRestart() {int64_t tmp = operator()(); restart(); return tmp;}
|
||||
};
|
||||
|
||||
}
|
||||
|
||||
#endif // _NSEC_TIMER_H
|
||||
236
external/glim/SerializablePool.hpp
vendored
Normal file
236
external/glim/SerializablePool.hpp
vendored
Normal file
@ -0,0 +1,236 @@
|
||||
#ifndef _GLIM_SERIALIZABLEPOOL_HPP_INCLUDED
|
||||
#define _GLIM_SERIALIZABLEPOOL_HPP_INCLUDED
|
||||
|
||||
#include "gstring.hpp"
|
||||
#ifndef _SERIALIZABLEPOOL_NOLDB
|
||||
# include "ldb.hpp" // Reuse `ldbSerialize` and `ldbDeserialize` in the `with` method.
|
||||
#endif
|
||||
namespace glim {
|
||||
|
||||
namespace SerializablePoolHelpers {
|
||||
struct Impl {
|
||||
/**
|
||||
* Pool format: \code
|
||||
* uint32_t valuesStart; // Network byte order.
|
||||
* uint32_t value0Offset, value1Offset, ... valueNOffset; // Network byte order.
|
||||
* char value0[]; char zero; char value1[]; char zero; ... char valueN[]; char zero;
|
||||
* \endcode
|
||||
*/
|
||||
gstring _pool;
|
||||
std::vector<gstring> _changes;
|
||||
std::vector<bool> _changed;
|
||||
bool _readOnly = false;
|
||||
Impl() = default;
|
||||
Impl (const gstring& poolBytes): _pool (poolBytes), _readOnly (false) {}
|
||||
};
|
||||
|
||||
/// Can be used to avoid a deep copy of the pool and change vectors (pImpl idiom).
|
||||
/// Example: \code glim::SerializablePool pool; \endcode
|
||||
struct SharedPtr {
|
||||
std::shared_ptr<Impl> _impl;
|
||||
SharedPtr() = default;
|
||||
SharedPtr (const gstring& poolBytes): _impl (std::make_shared<Impl> (poolBytes)) {}
|
||||
Impl* get() const {return _impl.get();}
|
||||
Impl* operator->() const {return _impl.get();}
|
||||
Impl& operator*() const {return *_impl;}
|
||||
Impl* instance() {if (!_impl) _impl = std::make_shared<Impl>(); return _impl.get();}
|
||||
};
|
||||
|
||||
/// Can be used instead of SharedPtr to avoid the shared_ptr indirection when the SerializablePoolTpl isn't going to be copied.
|
||||
/// Example: \code glim::InlineSerializablePool temporaryPool (bytes); \endcode
|
||||
struct InlinePtr {
|
||||
Impl _impl;
|
||||
InlinePtr() = default;
|
||||
InlinePtr (const gstring& poolBytes): _impl (poolBytes) {}
|
||||
Impl* get() const {return const_cast<Impl*> (&_impl);}
|
||||
Impl* operator->() const {return const_cast<Impl*> (&_impl);}
|
||||
Impl& operator*() const {return const_cast<Impl&> (_impl);}
|
||||
Impl* instance() {return &_impl;}
|
||||
};
|
||||
}
|
||||
|
||||
/** Serialization with lazy parsing: fields are accessed without "unpacking" the byte array.
|
||||
* Changes are stored separately, allowing the user to know exactly what fields has been changed and compare the old values to the new ones. */
|
||||
template <typename PI>
|
||||
class SerializablePoolTpl {
|
||||
protected:
|
||||
using Impl = SerializablePoolHelpers::Impl;
|
||||
PI _impl;
|
||||
/** @param ref Return a zero-copy view. The view should not be used outside of the pool buffer's lifetime. */
|
||||
static gstring original (const gstring& pool, uint32_t num, bool ref = false) {
|
||||
uint32_t poolLength = pool.length(); if (poolLength < 4) return gstring();
|
||||
uint32_t valuesStart = ntohl (*(uint32_t*) pool.data());
|
||||
assert (valuesStart <= poolLength);
|
||||
uint32_t valueOffsetOffset = 4 + num * 4;
|
||||
if ((int) valuesStart - (int) valueOffsetOffset < 4) return gstring(); // num > size
|
||||
uint32_t valueOffset = ntohl (*(uint32_t*) (pool.data() + valueOffsetOffset));
|
||||
valueOffsetOffset += 4;
|
||||
uint32_t nextValueOffset = ((int) valuesStart - (int) valueOffsetOffset < 4)
|
||||
? poolLength
|
||||
: ntohl (*(uint32_t*) (pool.data() + valueOffsetOffset));
|
||||
return gstring (0, (void*) (pool.data() + valueOffset), false, nextValueOffset - 1 - valueOffset, ref);
|
||||
}
|
||||
/** How many elements are in the pool. */
|
||||
static uint32_t poolSize (const gstring& pool) {
|
||||
if (pool.length() < 4) return 0;
|
||||
uint32_t valuesStart = ntohl (*(uint32_t*) pool.data());
|
||||
return (valuesStart - 4) / 4;
|
||||
}
|
||||
void toBytes (gstring& newPool, uint32_t size, const gstring* oldPool) const {
|
||||
newPool.clear();
|
||||
const Impl* impl = _impl.get();
|
||||
const std::vector<bool>& changed = impl->_changed;
|
||||
const std::vector<gstring>& changes = impl->_changes;
|
||||
if (changed.empty()) return;
|
||||
uint32_t valuesStart = 4 + size * 4;
|
||||
uint32_t networkOrder = 0;
|
||||
newPool.append ((char*) &(networkOrder = htonl (valuesStart)), 4);
|
||||
for (uint32_t num = 0; num < size; ++num) newPool.append ((char*) &(networkOrder = 0), 4);
|
||||
for (uint32_t num = 0; num < size; ++num) {
|
||||
uint32_t start = newPool.length();
|
||||
if (num < changed.size() && changed[num]) newPool << changes[num];
|
||||
else newPool << original (oldPool ? *oldPool : impl->_pool, num);
|
||||
newPool << '\0';
|
||||
uint32_t valuesOffsetOffset = 4 + num * 4; assert (valuesOffsetOffset < valuesStart);
|
||||
*(uint32_t*)(newPool.data() + valuesOffsetOffset) = htonl (start);
|
||||
}
|
||||
}
|
||||
public:
|
||||
/** Field, old value, new value. Might be used to maintain indexes. */
|
||||
typedef std::function<void(uint32_t, const gstring&, const gstring&)> ChangeVisitor;
|
||||
|
||||
SerializablePoolTpl() = default;
|
||||
/** Copy the given pool bytes from the outside source (e.g. from the database). */
|
||||
SerializablePoolTpl (const gstring& poolBytes): _impl (poolBytes) {}
|
||||
/** Returns a view into the original serialized field (ignores the current changes).\n
|
||||
* Returns an empty string if the field is not in the pool (num > size).
|
||||
* @param ref Return a zero-copy view. The view becomes invalid after the value has been changed or when the pool's `Impl` is destroyed. */
|
||||
const gstring original (uint32_t num, bool ref = false) const {return original (_impl->_pool, num, ref);}
|
||||
/** Returns the original serialized field (ignores the current changes).\n
|
||||
* Returns an empty string if the field is not in the pool (num > size). */
|
||||
const char* cstringOriginal (uint32_t num) const {
|
||||
gstring gs (original (_impl->_pool, num));
|
||||
return gs.empty() ? "" : gs.data(); // All fields in the _pool are 0-terminated.
|
||||
}
|
||||
/** Returns the field.
|
||||
* @param ref Return a zero-copy view. The view becomes invalid after the value has been changed or when the pool's `Impl` is destroyed. */
|
||||
const gstring current (uint32_t num, bool ref = false) const {
|
||||
const Impl* impl = _impl.get(); if (!impl) return gstring();
|
||||
if (num < impl->_changed.size() && impl->_changed[num]) {
|
||||
const gstring& value = impl->_changes[num]; return ref ? value.ref() : value;}
|
||||
return original (impl->_pool, num);
|
||||
}
|
||||
/** Set the new value of the field. */
|
||||
void set (uint32_t num, const gstring& value) {
|
||||
Impl* impl = _impl.instance();
|
||||
if (__builtin_expect (impl->_readOnly, 0)) throw std::runtime_error ("Attempt to modify a read-only SerializablePool");
|
||||
if (num >= impl->_changed.size()) {impl->_changed.resize (num + 1); impl->_changes.resize (num + 1);}
|
||||
impl->_changed[num] = true;
|
||||
impl->_changes[num] = value;
|
||||
}
|
||||
void reserve (uint32_t fields) {
|
||||
Impl* impl = _impl.instance();
|
||||
if (__builtin_expect (impl->_readOnly, 0)) throw std::runtime_error ("Attempt to modify a read-only SerializablePool");
|
||||
impl->_changed.reserve (fields);
|
||||
impl->_changes.reserve (fields);
|
||||
}
|
||||
/** Peek into the pool.\n
|
||||
* Returned reference should not be used after the SerializablePool goes out of scope (and destroyed). */
|
||||
const gstring& originalPool() {return _impl->_pool;}
|
||||
/** Serialize the pool.
|
||||
* @param changeVisitor is called for every field that was really changed (e.g. the bytes differ). */
|
||||
void toBytes (gstring& newPool, ChangeVisitor changeVisitor = ChangeVisitor()) const {
|
||||
if (changeVisitor) {
|
||||
const Impl* impl = _impl.get();
|
||||
const std::vector<bool>& changed = impl->_changed;
|
||||
const std::vector<gstring>& changes = impl->_changes;
|
||||
for (uint32_t num = 0, size = changed.size(); num < size; ++num) if (changed[num]) {
|
||||
const gstring& from = original (impl->_pool, num); const gstring& to = changes[num];
|
||||
if (from != to) changeVisitor (num, from, to);
|
||||
}
|
||||
}
|
||||
toBytes (newPool, (uint32_t) _impl->_changed.size(), nullptr);
|
||||
}
|
||||
/**
|
||||
* Performs "delta" serialization of the pool: creates a new pool where values which has not changed are copied from the `oldPool`.\n
|
||||
* \code Use case: 1) pools X and Y are loaded from a database by users A and B;
|
||||
* 2) user A changes field 0 in pool X; 3) user B changes field 1 in pool Y;
|
||||
* 4) user A loads `oldPool` from the database, does `toBytesDelta` from pool X and saves to the database;
|
||||
* 5) user B loads `oldPool` from the database, does `toBytesDelta` from pool Y and saves to the database;
|
||||
* result: database contains both changes (field 0 from user A and field 1 from user B). \endcode
|
||||
* @param changeVisitor is called for every field that was changed between the oldPool and the current one.
|
||||
* Returns `false` and leaves `newPool` *empty* if there are no changes found against the `oldPool`.
|
||||
*/
|
||||
bool toBytesDelta (gstring& newPool, const gstring& oldPool, ChangeVisitor changeVisitor = ChangeVisitor()) const {
|
||||
newPool.clear();
|
||||
const Impl* impl = _impl.get();
|
||||
const std::vector<bool>& changed = impl->_changed;
|
||||
const std::vector<gstring>& changes = impl->_changes;
|
||||
bool verifiedChanges = false;
|
||||
for (uint32_t num = 0, size = changed.size(); num < size; ++num) if (changed[num]) {
|
||||
const gstring& from = original (oldPool, num); const gstring& to = changes[num];
|
||||
if (from != to) {
|
||||
verifiedChanges = true;
|
||||
if (changeVisitor) changeVisitor (num, from, to); else break;
|
||||
}
|
||||
}
|
||||
if (!verifiedChanges) return false;
|
||||
|
||||
toBytes (newPool, std::max ((uint32_t) changed.size(), poolSize (oldPool)), &oldPool);
|
||||
return true;
|
||||
}
|
||||
/** True if the field has been `set` in this pool instance.\n
|
||||
* NB: Does *not* check if the `set` value is equal to the `original` value or not. */
|
||||
bool changed (uint32_t num) const {const auto& changed = _impl->_changed; return num < changed.size() ? changed[num] : false;}
|
||||
/** True if a field has been `set` in this pool instance. */
|
||||
bool changed() const {return !_impl->_changed.empty();}
|
||||
|
||||
bool operator == (const SerializablePoolTpl<PI>& rhs) const {return _impl.get() == rhs._impl.get();}
|
||||
/** Useful for storing SerializablePool in a map. */
|
||||
intptr_t implId() const {return (intptr_t) _impl.get();}
|
||||
|
||||
/** If set to `true` then modifying the pool will throw an exception.\n
|
||||
* Useful for freezing the pool before sharing it with other threads. */
|
||||
void readOnly (bool ro) {if (_impl) _impl->_readOnly = true;}
|
||||
bool readOnly() const {return (_impl ? _impl->_readOnly : false);}
|
||||
|
||||
/** Number of elements in the pool. Equals to max(num)-1. */
|
||||
uint32_t size() const {
|
||||
Impl* impl = _impl.get(); if (__builtin_expect (!impl, 0)) return 0;
|
||||
return std::max (poolSize (impl->_pool), (uint32_t) impl->_changed.size());
|
||||
}
|
||||
|
||||
#ifndef _SERIALIZABLEPOOL_NOLDB
|
||||
/** Serialize the `value` with `ldbSerialize` and `set` it to `num`.
|
||||
* @param stackSize is the amount of space to preallocate on stack for the temporary buffer. */
|
||||
template<typename T> void serialize (uint32_t num, const T& value, uint32_t stackSize = 256) {
|
||||
GSTRING_ON_STACK (bytes, stackSize);
|
||||
ldbSerialize (bytes, value);
|
||||
set (num, bytes);
|
||||
}
|
||||
/** If the field is not empty then `ldbDeserialize` it into `value`. */
|
||||
template<typename T> void deserialize (uint32_t num, T& value) const {
|
||||
const gstring& bytes = current (num);
|
||||
if (bytes.length()) ldbDeserialize (current (num), value);
|
||||
}
|
||||
/** Deserialize the `num` field with `ldbDeserialize`, run `visitor` on it, then optionally serialize the field back using `ldbSerialize`.
|
||||
* Example: \code
|
||||
* typedef std::map<std::string, std::string> MyMap;
|
||||
* pool.with<MyMap> (_myMap, [](MyMap& myMap) {myMap["foo"] = "bar"; return true;});
|
||||
* \endcode
|
||||
* @param visitor must return `true` to serialize the field back to the pool.
|
||||
*/
|
||||
template<typename T> void with (uint32_t num, std::function<bool(T&)> visitor) {
|
||||
const gstring& fromBytes = current (num, true);
|
||||
T value; if (fromBytes.length()) ldbDeserialize (fromBytes, value);
|
||||
if (visitor (value)) serialize (num, value, 16 + fromBytes.length() * 2);
|
||||
}
|
||||
#endif
|
||||
};
|
||||
|
||||
using SerializablePool = SerializablePoolTpl<SerializablePoolHelpers::SharedPtr>;
|
||||
using InlineSerializablePool = SerializablePoolTpl<SerializablePoolHelpers::InlinePtr>;
|
||||
|
||||
}
|
||||
|
||||
#endif // _GLIM_SERIALIZABLEPOOL_HPP_INCLUDED
|
||||
32
external/glim/TscTimer.hpp
vendored
Normal file
32
external/glim/TscTimer.hpp
vendored
Normal file
@ -0,0 +1,32 @@
|
||||
#ifndef _TSC_TIMER_H
|
||||
#define _TSC_TIMER_H
|
||||
|
||||
namespace glim {
|
||||
|
||||
extern "C" { // http://en.wikipedia.org/wiki/Rdtsc
|
||||
#if (defined(__GNUC__) || defined(__ICC)) && defined(__i386__)
|
||||
static __inline__ unsigned long long rdTsc(void) {
|
||||
unsigned long long ret;
|
||||
__asm__ __volatile__("rdtsc": "=A" (ret));
|
||||
return ret;
|
||||
}
|
||||
#elif (defined(__GNUC__) || defined(__ICC) || defined(__SUNPRO_C)) && defined(__x86_64__)
|
||||
static __inline__ unsigned long long rdTsc(void) {
|
||||
unsigned a, d;
|
||||
asm volatile("rdtsc" : "=a" (a), "=d" (d));
|
||||
return ((unsigned long long)a) | (((unsigned long long)d) << 32);
|
||||
}
|
||||
#endif
|
||||
}
|
||||
|
||||
//! CPU cycles timer. Fast, not safe.
|
||||
//! cf. http://en.wikipedia.org/wiki/Rdtsc
|
||||
struct TscTimer {
|
||||
int64_t start;
|
||||
TscTimer (): start (rdTsc()) {}
|
||||
int64_t operator()() const {return rdTsc() - start;}
|
||||
};
|
||||
|
||||
}
|
||||
|
||||
#endif // _TSC_TIMER_H
|
||||
203
external/glim/cbcoro.hpp
vendored
Normal file
203
external/glim/cbcoro.hpp
vendored
Normal file
@ -0,0 +1,203 @@
|
||||
/** \file
|
||||
* ucontext-based coroutine library designed to emulate a normal control flow around callbacks. */
|
||||
|
||||
// http://en.wikipedia.org/wiki/Setcontext; man 3 makecontext; man 2 getcontext
|
||||
// http://www.boost.org/doc/libs/1_53_0/libs/context/doc/html/index.html
|
||||
// g++ -std=c++11 -O1 -Wall -g test_cbcoro.cc -pthread && ./a.out
|
||||
|
||||
// NB: There is now a coroutine support in Boost ASIO which can be used to make asynchronous APIs look synchronous in a similar way:
|
||||
// https://svn.boost.org/trac/boost/changeset/84311
|
||||
|
||||
#include <ucontext.h>
|
||||
#include <sys/mman.h> // mmap
|
||||
#include <string.h> // strerror
|
||||
#include <mutex>
|
||||
#include <atomic>
|
||||
#include <valgrind/valgrind.h>
|
||||
#include <glim/exception.hpp>
|
||||
#include <boost/container/flat_map.hpp>
|
||||
#include <boost/container/slist.hpp>
|
||||
|
||||
namespace glim {
|
||||
|
||||
/// Simplifies turning callback control flows into normal imperative control flows.
|
||||
class CBCoro {
|
||||
public:
|
||||
/// "Holds" the CBCoro and will delete it when it is no longer used.
|
||||
struct CBCoroPtr {
|
||||
CBCoro* _coro;
|
||||
CBCoroPtr (CBCoro* coro): _coro (coro) {
|
||||
_coro->_users++;
|
||||
}
|
||||
~CBCoroPtr() {
|
||||
if (--_coro->_users <= 0 && _coro->_delete) delete _coro;
|
||||
}
|
||||
CBCoro* operator ->() const {return _coro;}
|
||||
};
|
||||
|
||||
static constexpr size_t defaultStackSize() {return 512 * 1024;}
|
||||
static constexpr uint8_t defaultCacheSize() {return 2;}
|
||||
protected:
|
||||
typedef boost::container::flat_map<size_t, boost::container::slist<void*> > cache_t;
|
||||
/// The cached stacks; stackSize -> free list.
|
||||
static cache_t& cache() {static cache_t CACHE; return CACHE;}
|
||||
static std::mutex& cacheMutex() {static std::mutex CACHE_MUTEX; return CACHE_MUTEX;}
|
||||
|
||||
ucontext_t _context;
|
||||
ucontext_t* _returnTo;
|
||||
std::recursive_mutex _mutex; ///< This one is locked most of the time.
|
||||
std::atomic_int_fast32_t _users; ///< Counter used by `CBCoroPtr`.
|
||||
bool _delete; ///< Whether the `CBCoroPtr` should `delete` this instance when it is no longer used (default is `true`).
|
||||
bool _invokeFromYield; ///< True if `invokeFromCallback()` was called directly from `yieldForCallback()`.
|
||||
bool _yieldFromInvoke; ///< True if `yieldForCallback()` now runs from `invokeFromCallback()`.
|
||||
uint8_t const _cacheStack; ///< Tells `freeStack()` to cache the stack if the number of cached `#_stackSize` stacks is less than it.
|
||||
void* _stack;
|
||||
size_t const _stackSize; ///< Keeps the size of the stack.
|
||||
|
||||
/// Peek a stack from the cache or allocate one with `mmap` (and register with Valgrind).
|
||||
virtual void allocateStack() {
|
||||
if (_cacheStack) {
|
||||
std::lock_guard<std::mutex> lock (cacheMutex());
|
||||
auto& freeList = cache()[_stackSize];
|
||||
if (!freeList.empty()) {_stack = freeList.front(); freeList.pop_front(); return;}
|
||||
}
|
||||
_stack = mmap (nullptr, _stackSize, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS | MAP_STACK | MAP_NORESERVE, -1, 0);
|
||||
if (_stack == MAP_FAILED) GTHROW (std::string ("mmap allocation failed: ") + ::strerror (errno));
|
||||
#pragma GCC diagnostic ignored "-Wunused-value"
|
||||
VALGRIND_STACK_REGISTER (_stack, (char*) _stack + _stackSize);
|
||||
}
|
||||
/// Release a stack into the cache or free it with `munmap` (and deregister with Valgrind).
|
||||
virtual void freeStack() {
|
||||
if (_cacheStack) {
|
||||
std::lock_guard<std::mutex> lock (cacheMutex());
|
||||
auto& freeList = cache()[_stackSize];
|
||||
if (freeList.size() < _cacheStack) {freeList.push_front (_stack); _stack = nullptr; return;}
|
||||
}
|
||||
VALGRIND_STACK_DEREGISTER (_stack);
|
||||
if (munmap (_stack, _stackSize)) GTHROW (std::string ("!munmap: ") + ::strerror (errno));;
|
||||
_stack = nullptr;
|
||||
}
|
||||
|
||||
/// Prepare the coroutine (initialize context, allocate stack and register it with Valgrind).
|
||||
CBCoro (uint8_t cacheStack = defaultCacheSize(), size_t stackSize = defaultStackSize()):
|
||||
_returnTo (nullptr), _users (0), _delete (true), _invokeFromYield (false), _yieldFromInvoke (false),
|
||||
_cacheStack (cacheStack), _stack (nullptr), _stackSize (stackSize) {
|
||||
if (getcontext (&_context)) GTHROW ("!getcontext");
|
||||
allocateStack();
|
||||
_context.uc_stack.ss_sp = _stack;
|
||||
_context.uc_stack.ss_size = stackSize;
|
||||
}
|
||||
virtual ~CBCoro() {
|
||||
freeStack();
|
||||
}
|
||||
public:
|
||||
/// Starts the coroutine on the `_stack` (makecontext, swapcontext), calling the `CBCoro::run`.
|
||||
CBCoroPtr start() {
|
||||
CBCoroPtr ptr (this);
|
||||
ucontext_t back; _context.uc_link = &back;
|
||||
makecontext (&_context, (void(*)()) cbcRun, 1, (intptr_t) this);
|
||||
// Since we have to "return" from inside the `yieldForCallback`,
|
||||
// we're not actually using the `_context.uc_link` and `return`, we use `setcontext (_returnTo)` instead.
|
||||
_returnTo = &back;
|
||||
_mutex.lock();
|
||||
swapcontext (&back, &_context); // Now our stack lives and the caller stack is no longer in control.
|
||||
_mutex.unlock();
|
||||
return ptr;
|
||||
}
|
||||
protected:
|
||||
/// Logs exception thrown from `CBCoro::run`.
|
||||
virtual void log (const std::exception& ex) {
|
||||
std::cerr << "glim::CBCoro, exception: " << ex.what() << std::endl;
|
||||
}
|
||||
static void cbcRun (CBCoro* cbCoro) {
|
||||
try {
|
||||
cbCoro->run();
|
||||
} catch (const std::exception& ex) {
|
||||
cbCoro->log (ex);
|
||||
}
|
||||
cbCoro->cbcReturn(); // Return the control to the rightful owner, e.g. to a last callback who ran `invokeFromCallback`, or otherwise to `cbcStart`.
|
||||
}
|
||||
/// Relinquish the control to the original owner of the thread, restoring its stack.
|
||||
void cbcReturn() {
|
||||
ucontext_t* returnTo = _returnTo;
|
||||
if (returnTo != nullptr) {_returnTo = nullptr; setcontext (returnTo);}
|
||||
}
|
||||
/// This method is performed on the CBCoro stack, allowing it to be suspended and then reanimated from callbacks.
|
||||
virtual void run() = 0;
|
||||
public:
|
||||
/** Use this method to wrap a return-via-callback code.
|
||||
* For example, the callback code \code
|
||||
* startSomeWork ([=]() {
|
||||
* continueWhenWorkIsFinished();
|
||||
* });
|
||||
* \endcode should be turned into \code
|
||||
* yieldForCallback ([&]() {
|
||||
* startSomeWork ([&]() {
|
||||
* invokeFromCallback();
|
||||
* });
|
||||
* });
|
||||
* continueWhenWorkIsFinished();
|
||||
* \endcode
|
||||
*
|
||||
* Captures the stack, runs the `fun` and relinquish the control to `_returnTo`.\n
|
||||
* This method will never "return" by itself, in order for it to "return" the
|
||||
* `fun` MUST call `invokeFromCallback`, maybe later and from a different stack. */
|
||||
template <typename F> CBCoroPtr yieldForCallback (F fun) {
|
||||
CBCoroPtr ptr (this);
|
||||
_yieldFromInvoke = false;
|
||||
if (getcontext (&_context)) GTHROW ("!getcontext"); // Capture.
|
||||
if (_yieldFromInvoke) {
|
||||
// We're now in the future, revived by the `invokeFromCallback`.
|
||||
// All we do now is "return" to the caller whose stack we captured earlier.
|
||||
} else {
|
||||
// We're still in the present, still have some work to do.
|
||||
fun(); // The `fun` is supposed to do something resulting in the `invokeFromCallback` being called later.
|
||||
if (_invokeFromYield) {
|
||||
// The `fun` used the `invokeFromCallback` directly, not resorting to callbacks, meaning we don't have to do our magick.
|
||||
_invokeFromYield = false;
|
||||
} else {
|
||||
// So, the `fun` took measures to revive us later, it's time for us to go into torpor and return the control to whoever we've borrowed it from.
|
||||
cbcReturn();
|
||||
}
|
||||
}
|
||||
return ptr;
|
||||
}
|
||||
|
||||
/// To be called from a callback in order to lend the control to CBCoro, continuing it from where it called `yieldForCallback`.
|
||||
CBCoroPtr invokeFromCallback() {
|
||||
CBCoroPtr ptr (this);
|
||||
_mutex.lock(); // Wait for an other-thready `yieldForCallback` to finish.
|
||||
if (_returnTo != nullptr) {
|
||||
// We have not yet "returned" from the `yieldForCallback`,
|
||||
// meaning that the `invokeFromCallback` was executed immediately from inside the `yieldForCallback`.
|
||||
// In that case we must DO NOTHING, we must simply continue running on the current stack.
|
||||
_invokeFromYield = true; // Tells `yieldForCallback` to do nothing.
|
||||
} else {
|
||||
// Revive the CBCoro, letting it continue from where it was suspended in `yieldForCallback`.
|
||||
ucontext_t cbContext; _returnTo = &cbContext; _yieldFromInvoke = true;
|
||||
if (swapcontext (&cbContext, &_context)) GTHROW ("!swapcontext");
|
||||
// NB: When the CBCoro is suspended or exits, the control returns back there and then back to the callback from which we borrowed it.
|
||||
if (_returnTo == &cbContext) _returnTo = nullptr;
|
||||
}
|
||||
_mutex.unlock(); // Other-thready `yieldForCallback` has finished and `cbcReturn`ed here.
|
||||
return ptr;
|
||||
}
|
||||
};
|
||||
|
||||
/** CBCoro running a given functor.
|
||||
* The functor's first argument must be a CBCoro pointer, like this: \code (new CBCoroForFunctor ([](CBCoro* cbcoro) {}))->start(); \endcode */
|
||||
template <typename FUN> struct CBCoroForFunctor: public CBCoro {
|
||||
FUN _fun;
|
||||
template <typename CFUN> CBCoroForFunctor (CFUN&& fun, uint8_t cacheStack, size_t stackSize): CBCoro (cacheStack, stackSize), _fun (std::forward<CFUN> (fun)) {}
|
||||
virtual void run() {_fun (this);}
|
||||
virtual ~CBCoroForFunctor() {}
|
||||
};
|
||||
|
||||
/** Syntactic sugar: Runs a given functor in a CBCoro instance.
|
||||
* Example: \code glim::cbCoro ([](glim::CBCoro* cbcoro) {}); \endcode
|
||||
* Returns a `CBCoroPtr` to the CBCoro instance holding the `fun` which might be held somewhere in order to delay the deletion of `fun`. */
|
||||
template <typename FUN> inline CBCoro::CBCoroPtr cbCoro (FUN&& fun, uint8_t cacheStack = CBCoro::defaultCacheSize(), size_t stackSize = CBCoro::defaultStackSize()) {
|
||||
return (new CBCoroForFunctor<FUN> (std::forward<FUN> (fun), cacheStack, stackSize))->start();
|
||||
}
|
||||
|
||||
}
|
||||
40
external/glim/channel.hpp
vendored
Normal file
40
external/glim/channel.hpp
vendored
Normal file
@ -0,0 +1,40 @@
|
||||
#ifndef _GLIM_CHANNEL_INCLUDED
|
||||
#define _GLIM_CHANNEL_INCLUDED
|
||||
|
||||
#include <atomic>
|
||||
#include <mutex>
|
||||
#include <thread>
|
||||
|
||||
namespace glim {
|
||||
|
||||
/// Unbuffered channel.
|
||||
/// Optimized for a single value (busy-waits on a second one).
|
||||
template <typename V>
|
||||
struct Channel {
|
||||
V _v;
|
||||
std::mutex _mutex; // Locked when there is no value.
|
||||
std::atomic_int_fast8_t _state; enum State {EMPTY = 0, WRITING = 1, FULL = 2};
|
||||
Channel(): _state (EMPTY) {_mutex.lock();}
|
||||
// Waits until the Channel is empty then stores the value.
|
||||
template <typename VA> void send (VA&& v) {
|
||||
for (;;) {
|
||||
int_fast8_t expectEmpty = EMPTY; if (_state.compare_exchange_weak (expectEmpty, WRITING)) break;
|
||||
std::this_thread::sleep_for (std::chrono::milliseconds (20));
|
||||
}
|
||||
try {_v = std::forward<V> (v);} catch (...) {_state = EMPTY; throw;}
|
||||
_state = FULL;
|
||||
_mutex.unlock(); // Allows the reader to proceed.
|
||||
}
|
||||
// Waits untill there is a value to receive.
|
||||
V receive() {
|
||||
_mutex.lock(); // Wait.
|
||||
V tmp = std::move (_v);
|
||||
assert (_state == FULL);
|
||||
_state = EMPTY;
|
||||
return tmp;
|
||||
}
|
||||
};
|
||||
|
||||
} // namespace glim
|
||||
|
||||
#endif
|
||||
304
external/glim/curl.hpp
vendored
Normal file
304
external/glim/curl.hpp
vendored
Normal file
@ -0,0 +1,304 @@
|
||||
/** \file
|
||||
* Very simple header-only wrapper around libcurl.\n
|
||||
* See also: https://github.com/venam/Browser\n
|
||||
* See also: https://github.com/mologie/curl-asio\n
|
||||
* See also: http://thread.gmane.org/gmane.comp.web.curl.library/1322 (this one uses a temporary file). */
|
||||
|
||||
#ifndef _GLIM_CURL_INCLUDED
|
||||
#define _GLIM_CURL_INCLUDED
|
||||
|
||||
#include "gstring.hpp"
|
||||
#include "exception.hpp"
|
||||
#include <curl/curl.h>
|
||||
#include <algorithm>
|
||||
#include <functional>
|
||||
#include <string.h>
|
||||
#include <stdint.h>
|
||||
|
||||
namespace glim {
|
||||
|
||||
inline size_t curlWriteToString (void *buffer, size_t size, size_t nmemb, void *userp) {
|
||||
((std::string*) userp)->append ((const char*) buffer, size * nmemb);
|
||||
return size * nmemb;};
|
||||
|
||||
inline size_t curlReadFromString (void *ptr, size_t size, size_t nmemb, void *userdata);
|
||||
inline size_t curlReadFromGString (void *ptr, size_t size, size_t nmemb, void *userdata);
|
||||
inline size_t curlWriteHeader (void *ptr, size_t size, size_t nmemb, void *curlPtr);
|
||||
inline int curlDebugCB (CURL* curl, curl_infotype type, char* bytes, size_t size, void* curlPtr);
|
||||
|
||||
/**
|
||||
Simple HTTP requests using cURL.
|
||||
Example: \code
|
||||
std::string w3 = glim::Curl() .http ("http://www.w3.org/") .go().str();
|
||||
\endcode
|
||||
*/
|
||||
class Curl {
|
||||
protected:
|
||||
Curl (const Curl&): _curl (NULL), _headers (NULL), _sent (0), _needs_cleanup (true) {} // No copying.
|
||||
public:
|
||||
struct PerformError: public glim::Exception {
|
||||
PerformError (const char* message, const char* file, int32_t line):
|
||||
glim::Exception (message, file, line) {}
|
||||
};
|
||||
struct GetinfoError: public glim::Exception {
|
||||
CURLcode _code; std::string _error;
|
||||
GetinfoError (CURLcode code, const std::string& error, const char* file, int32_t line):
|
||||
glim::Exception (error, file, line),
|
||||
_code (code), _error (error) {}
|
||||
};
|
||||
public:
|
||||
CURL* _curl;
|
||||
struct curl_slist *_headers;
|
||||
std::function<void (const char* header, int len)> _headerListener;
|
||||
std::function<void (curl_infotype type, char* bytes, size_t size)> _debugListener;
|
||||
std::string _sendStr; ///< We're using `std::string` instead of `gstring` in order to support payloads larger than 16 MiB.
|
||||
glim::gstring _sendGStr; ///< `gstring::view` and `gstring::ref` allow us to zero-copy.
|
||||
uint32_t _sent;
|
||||
std::string _got;
|
||||
bool _needs_cleanup:1; ///< ~Curl will do `curl_easy_cleanup` if `true`.
|
||||
char _errorBuf[CURL_ERROR_SIZE];
|
||||
|
||||
Curl (Curl&&) = default;
|
||||
|
||||
/// @param cleanup can be turned off if the cURL is freed elsewhere.
|
||||
Curl (bool cleanup = true): _curl (curl_easy_init()), _headers (NULL), _sent (0), _needs_cleanup (cleanup) {
|
||||
curl_easy_setopt (_curl, CURLOPT_NOSIGNAL, 1L); // required per http://curl.haxx.se/libcurl/c/libcurl-tutorial.html#Multi-threading
|
||||
*_errorBuf = 0;}
|
||||
/// Wraps an existing handle (will invoke `curl_easy_cleanup` nevertheless).
|
||||
/// @param cleanup can be turned off if the cURL is freed elsewhere.
|
||||
Curl (CURL* curl, bool cleanup = true): _curl (curl), _headers (NULL), _sent (0), _needs_cleanup (cleanup) {
|
||||
curl_easy_setopt (_curl, CURLOPT_NOSIGNAL, 1L); // required per http://curl.haxx.se/libcurl/c/libcurl-tutorial.html#Multi-threading
|
||||
*_errorBuf = 0;}
|
||||
~Curl(){
|
||||
if (_headers) {curl_slist_free_all (_headers); _headers = NULL;}
|
||||
if (_curl) {if (_needs_cleanup) curl_easy_cleanup (_curl); _curl = NULL;}
|
||||
}
|
||||
|
||||
/** Stores the content to be sent into an `std::string` inside `Curl`.
|
||||
* NB: In order to have an effect this method should be used *before* the `http()` and `smtp()` methods. */
|
||||
template<typename STR> Curl& send (STR&& text) {
|
||||
_sendStr = std::forward<STR> (text);
|
||||
_sendGStr.clear();
|
||||
_sent = 0;
|
||||
return *this;}
|
||||
|
||||
/// Adds "Content-Type" header into `_headers`.
|
||||
Curl& contentType (const char* ct) {
|
||||
char ctb[64]; gstring cth (sizeof (ctb), ctb, false, 0);
|
||||
cth << "Content-Type: " << ct << "\r\n";
|
||||
_headers = curl_slist_append (_headers, cth.c_str());
|
||||
return *this;
|
||||
}
|
||||
|
||||
/// @param fullHeader is a full HTTP header and a newline, e.g. "User-Agent: Me\r\n".
|
||||
Curl& header (const char* fullHeader) {
|
||||
_headers = curl_slist_append (_headers, fullHeader);
|
||||
return *this;
|
||||
}
|
||||
|
||||
/**
|
||||
Sets the majority of options for the http request.
|
||||
NB: If `send` was used with a non-empty string then `http` will use `CURLOPT_UPLOAD`, setting http method to `PUT` (use the `method()` to override).
|
||||
\n
|
||||
Example: \code
|
||||
glim::Curl curl;
|
||||
curl.http (url.c_str()) .go();
|
||||
std::cout << curl.status() << std::endl << curl.str() << std::endl;
|
||||
\endcode
|
||||
*/
|
||||
Curl& http (const char* url, int timeoutSec = 20) {
|
||||
curl_easy_setopt (_curl, CURLOPT_NOSIGNAL, 1L); // required per http://curl.haxx.se/libcurl/c/libcurl-tutorial.html#Multi-threading
|
||||
curl_easy_setopt (_curl, CURLOPT_URL, url);
|
||||
curl_easy_setopt (_curl, CURLOPT_WRITEFUNCTION, curlWriteToString);
|
||||
curl_easy_setopt (_curl, CURLOPT_WRITEDATA, &_got);
|
||||
curl_easy_setopt (_curl, CURLOPT_TIMEOUT, timeoutSec);
|
||||
curl_easy_setopt (_curl, CURLOPT_PROTOCOLS, CURLPROTO_HTTP);
|
||||
curl_easy_setopt (_curl, CURLOPT_ERRORBUFFER, _errorBuf);
|
||||
if (_sendStr.size() || _sendGStr.size()) {
|
||||
curl_easy_setopt (_curl, CURLOPT_UPLOAD, 1L); // http://curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTUPLOAD
|
||||
if (_sendStr.size()) {
|
||||
curl_easy_setopt (_curl, CURLOPT_INFILESIZE, (long) _sendStr.size());
|
||||
curl_easy_setopt (_curl, CURLOPT_READFUNCTION, curlReadFromString);
|
||||
} else {
|
||||
curl_easy_setopt (_curl, CURLOPT_INFILESIZE, (long) _sendGStr.size());
|
||||
curl_easy_setopt (_curl, CURLOPT_READFUNCTION, curlReadFromGString);}
|
||||
curl_easy_setopt (_curl, CURLOPT_READDATA, this);}
|
||||
if (_headers)
|
||||
curl_easy_setopt (_curl, CURLOPT_HTTPHEADER, _headers); // http://curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTHTTPHEADER
|
||||
return *this;
|
||||
}
|
||||
|
||||
/**
|
||||
Set options for smtp request.
|
||||
Example: \code
|
||||
long rc = glim::Curl().send ("Subject: subject\r\n\r\n" "text\r\n") .smtp ("from", "to") .go().status();
|
||||
if (rc != 250) std::cerr << "Error sending email: " << rc << std::endl;
|
||||
\endcode */
|
||||
Curl& smtp (const char* from, const char* to) {
|
||||
curl_easy_setopt (_curl, CURLOPT_NOSIGNAL, 1L); // required per http://curl.haxx.se/libcurl/c/libcurl-tutorial.html#Multi-threading
|
||||
curl_easy_setopt (_curl, CURLOPT_URL, "smtp://127.0.0.1");
|
||||
if (from) curl_easy_setopt (_curl, CURLOPT_MAIL_FROM, from);
|
||||
bcc (to);
|
||||
if (_headers) curl_easy_setopt (_curl, CURLOPT_MAIL_RCPT, _headers);
|
||||
curl_easy_setopt (_curl, CURLOPT_WRITEFUNCTION, curlWriteToString);
|
||||
curl_easy_setopt (_curl, CURLOPT_WRITEDATA, &_got);
|
||||
if (_sendStr.size()) {
|
||||
curl_easy_setopt (_curl, CURLOPT_INFILESIZE, (long) _sendStr.size());
|
||||
curl_easy_setopt (_curl, CURLOPT_READFUNCTION, curlReadFromString);
|
||||
curl_easy_setopt (_curl, CURLOPT_READDATA, this);
|
||||
} else if (_sendGStr.size()) {
|
||||
curl_easy_setopt (_curl, CURLOPT_INFILESIZE, (long) _sendGStr.size());
|
||||
curl_easy_setopt (_curl, CURLOPT_READFUNCTION, curlReadFromGString);
|
||||
curl_easy_setopt (_curl, CURLOPT_READDATA, this);
|
||||
}
|
||||
curl_easy_setopt (_curl, CURLOPT_UPLOAD, 1L); // cURL now needs this to actually send the email, cf. "http://curl.haxx.se/mail/lib-2013-12/0152.html".
|
||||
return *this;
|
||||
}
|
||||
|
||||
/** Add SMTP recipient to the `_headers` (which are then set into `CURLOPT_MAIL_RCPT` by the `Curl::smtp`).
|
||||
* NB: Should be used *before* the `Curl::smtp`! */
|
||||
Curl& bcc (const char* to) {
|
||||
if (to) _headers = curl_slist_append (_headers, to);
|
||||
return *this;
|
||||
}
|
||||
|
||||
/**
|
||||
Uses `CURLOPT_CUSTOMREQUEST` to set the http method.
|
||||
Can be used both before and after the `http` method.\n
|
||||
Example sending a POST request to ElasticSearch: \code
|
||||
glim::Curl curl;
|
||||
curl.send (C2GSTRING (R"({"query":{"match_all":{}},"facets":{"tags":{"terms":{"field":"tags","size":1000}}}})"));
|
||||
curl.method ("POST") .http ("http://127.0.0.1:9200/froples/frople/_search", 120);
|
||||
if (curl.verbose().go().status() != 200) GTHROW ("Error fetching tags: " + std::to_string (curl.status()) + ", " + curl.str());
|
||||
cout << curl.gstr() << endl;
|
||||
\endcode */
|
||||
Curl& method (const char* method) {
|
||||
curl_easy_setopt (_curl, CURLOPT_CUSTOMREQUEST, method);
|
||||
return *this;
|
||||
}
|
||||
|
||||
/** Setup a handler to process the headers cURL gets from the response.
|
||||
* "The header callback will be called once for each header and only complete header lines are passed on to the callback".\n
|
||||
* See http://curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTHEADERFUNCTION */
|
||||
Curl& headerListener (std::function<void (const char* header, int len)> listener) {
|
||||
curl_easy_setopt (_curl, CURLOPT_HEADERFUNCTION, curlWriteHeader);
|
||||
curl_easy_setopt (_curl, CURLOPT_WRITEHEADER, this);
|
||||
_headerListener = listener;
|
||||
return *this;
|
||||
}
|
||||
|
||||
/** Setup a handler to get the debug messages generated by cURL.
|
||||
* See http://curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTDEBUGFUNCTION */
|
||||
Curl& debugListener (std::function<void (curl_infotype type, char* bytes, size_t size)> listener) {
|
||||
curl_easy_setopt (_curl, CURLOPT_DEBUGFUNCTION, curlDebugCB);
|
||||
curl_easy_setopt (_curl, CURLOPT_DEBUGDATA, this);
|
||||
_debugListener = listener;
|
||||
return verbose (true);
|
||||
}
|
||||
|
||||
/**
|
||||
Setup a handler to get some of the debug messages generated by cURL.
|
||||
Listener gets a formatted text: outbound data is prepended with "> " and inbound with "< ".\n
|
||||
Usage example: \code
|
||||
auto curlDebug = std::make_shared<std::string>();
|
||||
curl->debugListenerF ([curlDebug](const char* bytes, size_t size) {curlDebug->append (bytes, size);});
|
||||
...
|
||||
if (curl->status() != 200) std::cerr << "cURL status != 200; debug follows: " << *curlDebug << std::endl;
|
||||
\endcode
|
||||
See http://curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTDEBUGFUNCTION
|
||||
@param listener The receiver of the debug information.
|
||||
@param data Whether to pass the data (`CURLINFO_DATA_IN`, `CURLINFO_DATA_OUT`) to the `listener`.
|
||||
*/
|
||||
Curl& debugListenerF (std::function<void (const char* bytes, size_t size)> listener, bool data = false) {
|
||||
return debugListener ([listener/* = std::move (listener)*/,data] (curl_infotype type, char* bytes, size_t size) {
|
||||
GSTRING_ON_STACK (buf, 256);
|
||||
auto prepend = [&](const char* prefix) {
|
||||
buf << prefix; for (char *p = bytes, *end = bytes + size; p < end; ++p) {buf << *p; if (*p == '\n' && p + 2 < end) buf << prefix;}};
|
||||
if (type == CURLINFO_HEADER_IN || (type == CURLINFO_DATA_IN && data)) prepend ("< ");
|
||||
else if (type == CURLINFO_HEADER_OUT || (type == CURLINFO_DATA_OUT && data)) prepend ("> ");
|
||||
listener (buf.c_str(), buf.size());
|
||||
});
|
||||
}
|
||||
|
||||
/// Whether to print debug information to `CURLOPT_STDERR`.
|
||||
/// Note that when `debugListener` is used, verbose output will go to the listener and not to `CURLOPT_STDERR`.
|
||||
Curl& verbose (bool on = true) {
|
||||
curl_easy_setopt (_curl, CURLOPT_VERBOSE, on ? 1L : 0L);
|
||||
return *this;
|
||||
}
|
||||
|
||||
/// Reset the buffers and perform the cURL request.
|
||||
Curl& go() {
|
||||
_got.clear();
|
||||
*_errorBuf = 0;
|
||||
if (curl_easy_perform (_curl)) throw PerformError (_errorBuf, __FILE__, __LINE__);
|
||||
return *this;
|
||||
}
|
||||
|
||||
/// The contents of the response.
|
||||
const std::string& str() const {return _got;}
|
||||
/// CString of `str`.
|
||||
const char* c_str() const {return _got.c_str();}
|
||||
/// Returns a gstring "view" into `str`.
|
||||
gstring gstr() const {return gstring (0, (void*) _got.data(), false, _got.size());}
|
||||
|
||||
/// The status of the response (For HTTP it's 200 ok, 404 not found, 500 error, etc).
|
||||
long status() const {
|
||||
long status; CURLcode err = curl_easy_getinfo (_curl, CURLINFO_RESPONSE_CODE, &status);
|
||||
if (err) {
|
||||
GSTRING_ON_STACK (message, 128) << "CURL error " << (int) err << ": " << curl_easy_strerror (err);
|
||||
throw GetinfoError (err, message.str(), __FILE__, __LINE__);
|
||||
}
|
||||
return status;}
|
||||
};
|
||||
|
||||
/** Moves the content to be sent into a `glim::gstring` inside `Curl`.
|
||||
* NB: In order to have an effect this method should be used *before* the `http()` and `smtp()` methods. */
|
||||
template<> inline Curl& Curl::send<gstring> (gstring&& text) {
|
||||
_sendStr.clear();
|
||||
_sendGStr = std::move (text);
|
||||
_sent = 0;
|
||||
return *this;}
|
||||
|
||||
inline size_t curlReadFromString (void *ptr, size_t size, size_t nmemb, void *userdata) {
|
||||
Curl* curl = (Curl*) userdata;
|
||||
size_t len = std::min (curl->_sendStr.size() - curl->_sent, size * nmemb);
|
||||
if (len) memcpy (ptr, curl->_sendStr.data() + curl->_sent, len);
|
||||
curl->_sent += len;
|
||||
return len;}
|
||||
|
||||
inline size_t curlReadFromGString (void *ptr, size_t size, size_t nmemb, void *userdata) {
|
||||
Curl* curl = (Curl*) userdata;
|
||||
size_t len = std::min (curl->_sendGStr.size() - curl->_sent, size * nmemb);
|
||||
if (len) memcpy (ptr, curl->_sendGStr.data() + curl->_sent, len);
|
||||
curl->_sent += len;
|
||||
return len;}
|
||||
|
||||
// http://curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTHEADERFUNCTION
|
||||
inline size_t curlWriteHeader (void *ptr, size_t size, size_t nmemb, void *curlPtr) {
|
||||
Curl* curl = (Curl*) curlPtr;
|
||||
std::function<void (const char* header, int len)>& listener = curl->_headerListener;
|
||||
int len = size * nmemb;
|
||||
if (listener) listener ((const char*) ptr, len);
|
||||
return (size_t) len;
|
||||
}
|
||||
|
||||
// http://curl.haxx.se/libcurl/c/curl_easy_setopt.html#CURLOPTDEBUGFUNCTION
|
||||
inline int curlDebugCB (CURL*, curl_infotype type, char* bytes, size_t size, void* curlPtr) {
|
||||
Curl* curl = (Curl*) curlPtr;
|
||||
auto& listener = curl->_debugListener;
|
||||
if (listener) listener (type, bytes, size);
|
||||
return 0;
|
||||
}
|
||||
|
||||
/// Example: std::string w3 = glim::curl2str ("http://www.w3.org/");
|
||||
inline std::string curl2str (const char* url, int timeoutSec = 20) {
|
||||
try {
|
||||
return glim::Curl().http (url, timeoutSec) .go().str();
|
||||
} catch (const std::exception&) {}
|
||||
return std::string();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
#endif
|
||||
237
external/glim/doxyconf
vendored
Normal file
237
external/glim/doxyconf
vendored
Normal file
@ -0,0 +1,237 @@
|
||||
# Doxyfile 1.8.4; http://www.stack.nl/~dimitri/doxygen/manual/config.html
|
||||
DOXYFILE_ENCODING = UTF-8
|
||||
PROJECT_NAME = "libglim"
|
||||
PROJECT_NUMBER = 0.7
|
||||
OUTPUT_DIRECTORY = doc
|
||||
CREATE_SUBDIRS = NO
|
||||
OUTPUT_LANGUAGE = English
|
||||
BRIEF_MEMBER_DESC = YES
|
||||
REPEAT_BRIEF = YES
|
||||
ALWAYS_DETAILED_SEC = NO
|
||||
INLINE_INHERITED_MEMB = NO
|
||||
FULL_PATH_NAMES = NO
|
||||
SHORT_NAMES = NO
|
||||
JAVADOC_AUTOBRIEF = YES
|
||||
QT_AUTOBRIEF = NO
|
||||
MULTILINE_CPP_IS_BRIEF = NO
|
||||
INHERIT_DOCS = YES
|
||||
SEPARATE_MEMBER_PAGES = NO
|
||||
TAB_SIZE = 2
|
||||
OPTIMIZE_OUTPUT_FOR_C = YES
|
||||
OPTIMIZE_OUTPUT_JAVA = NO
|
||||
OPTIMIZE_FOR_FORTRAN = NO
|
||||
OPTIMIZE_OUTPUT_VHDL = NO
|
||||
# http://daringfireball.net/projects/markdown/
|
||||
MARKDOWN_SUPPORT = YES
|
||||
AUTOLINK_SUPPORT = YES
|
||||
BUILTIN_STL_SUPPORT = YES
|
||||
CPP_CLI_SUPPORT = NO
|
||||
SIP_SUPPORT = NO
|
||||
IDL_PROPERTY_SUPPORT = NO
|
||||
DISTRIBUTE_GROUP_DOC = NO
|
||||
SUBGROUPING = YES
|
||||
INLINE_GROUPED_CLASSES = NO
|
||||
INLINE_SIMPLE_STRUCTS = YES
|
||||
TYPEDEF_HIDES_STRUCT = NO
|
||||
LOOKUP_CACHE_SIZE = 0
|
||||
EXTRACT_ALL = NO
|
||||
EXTRACT_PRIVATE = NO
|
||||
EXTRACT_PACKAGE = NO
|
||||
EXTRACT_STATIC = NO
|
||||
EXTRACT_LOCAL_CLASSES = YES
|
||||
EXTRACT_LOCAL_METHODS = NO
|
||||
EXTRACT_ANON_NSPACES = NO
|
||||
HIDE_UNDOC_MEMBERS = YES
|
||||
HIDE_UNDOC_CLASSES = YES
|
||||
HIDE_FRIEND_COMPOUNDS = NO
|
||||
HIDE_IN_BODY_DOCS = NO
|
||||
INTERNAL_DOCS = NO
|
||||
CASE_SENSE_NAMES = YES
|
||||
HIDE_SCOPE_NAMES = NO
|
||||
SHOW_INCLUDE_FILES = YES
|
||||
FORCE_LOCAL_INCLUDES = NO
|
||||
INLINE_INFO = YES
|
||||
SORT_MEMBER_DOCS = YES
|
||||
SORT_BRIEF_DOCS = YES
|
||||
SORT_MEMBERS_CTORS_1ST = NO
|
||||
SORT_GROUP_NAMES = NO
|
||||
SORT_BY_SCOPE_NAME = NO
|
||||
STRICT_PROTO_MATCHING = NO
|
||||
GENERATE_TODOLIST = YES
|
||||
GENERATE_TESTLIST = YES
|
||||
GENERATE_BUGLIST = YES
|
||||
GENERATE_DEPRECATEDLIST= YES
|
||||
MAX_INITIALIZER_LINES = 30
|
||||
SHOW_USED_FILES = YES
|
||||
SHOW_FILES = YES
|
||||
SHOW_NAMESPACES = YES
|
||||
QUIET = NO
|
||||
WARNINGS = YES
|
||||
WARN_IF_UNDOCUMENTED = NO
|
||||
WARN_IF_DOC_ERROR = YES
|
||||
WARN_NO_PARAMDOC = NO
|
||||
WARN_FORMAT = "$file:$line: $text"
|
||||
|
||||
INPUT = ./
|
||||
INPUT_ENCODING = UTF-8
|
||||
FILE_PATTERNS = *.hpp
|
||||
RECURSIVE = NO
|
||||
EXCLUDE =
|
||||
EXCLUDE_SYMLINKS = NO
|
||||
EXCLUDE_PATTERNS =
|
||||
EXCLUDE_SYMBOLS =
|
||||
EXAMPLE_PATH =
|
||||
EXAMPLE_PATTERNS = test_*.cc
|
||||
EXAMPLE_RECURSIVE = NO
|
||||
IMAGE_PATH =
|
||||
FILTER_SOURCE_FILES = NO
|
||||
SOURCE_BROWSER = NO
|
||||
INLINE_SOURCES = NO
|
||||
STRIP_CODE_COMMENTS = NO
|
||||
REFERENCED_BY_RELATION = NO
|
||||
REFERENCES_RELATION = NO
|
||||
REFERENCES_LINK_SOURCE = YES
|
||||
USE_HTAGS = NO
|
||||
VERBATIM_HEADERS = YES
|
||||
CLANG_ASSISTED_PARSING = NO
|
||||
ALPHABETICAL_INDEX = YES
|
||||
COLS_IN_ALPHA_INDEX = 5
|
||||
GENERATE_HTML = YES
|
||||
HTML_OUTPUT = html
|
||||
HTML_FILE_EXTENSION = .html
|
||||
HTML_HEADER =
|
||||
HTML_FOOTER =
|
||||
HTML_STYLESHEET =
|
||||
HTML_EXTRA_STYLESHEET =
|
||||
HTML_EXTRA_FILES =
|
||||
HTML_COLORSTYLE_HUE = 220
|
||||
HTML_COLORSTYLE_SAT = 100
|
||||
HTML_COLORSTYLE_GAMMA = 80
|
||||
HTML_TIMESTAMP = YES
|
||||
HTML_DYNAMIC_SECTIONS = NO
|
||||
HTML_INDEX_NUM_ENTRIES = 100
|
||||
GENERATE_DOCSET = NO
|
||||
DOCSET_FEEDNAME = "Doxygen generated docs"
|
||||
DOCSET_BUNDLE_ID = org.doxygen.Project
|
||||
DOCSET_PUBLISHER_ID = org.doxygen.Publisher
|
||||
DOCSET_PUBLISHER_NAME = Publisher
|
||||
GENERATE_HTMLHELP = NO
|
||||
CHM_FILE =
|
||||
HHC_LOCATION =
|
||||
GENERATE_CHI = NO
|
||||
CHM_INDEX_ENCODING =
|
||||
BINARY_TOC = NO
|
||||
TOC_EXPAND = NO
|
||||
GENERATE_QHP = NO
|
||||
QCH_FILE =
|
||||
QHP_NAMESPACE = org.doxygen.Project
|
||||
QHP_VIRTUAL_FOLDER = doc
|
||||
QHP_CUST_FILTER_NAME =
|
||||
QHP_CUST_FILTER_ATTRS =
|
||||
QHP_SECT_FILTER_ATTRS =
|
||||
QHG_LOCATION =
|
||||
GENERATE_ECLIPSEHELP = NO
|
||||
ECLIPSE_DOC_ID = org.doxygen.Project
|
||||
DISABLE_INDEX = NO
|
||||
GENERATE_TREEVIEW = NO
|
||||
ENUM_VALUES_PER_LINE = 4
|
||||
TREEVIEW_WIDTH = 250
|
||||
EXT_LINKS_IN_WINDOW = NO
|
||||
FORMULA_FONTSIZE = 10
|
||||
FORMULA_TRANSPARENT = YES
|
||||
USE_MATHJAX = NO
|
||||
MATHJAX_FORMAT = HTML-CSS
|
||||
MATHJAX_RELPATH = http://cdn.mathjax.org/mathjax/latest
|
||||
MATHJAX_EXTENSIONS =
|
||||
MATHJAX_CODEFILE =
|
||||
SEARCHENGINE = YES
|
||||
SERVER_BASED_SEARCH = NO
|
||||
EXTERNAL_SEARCH = NO
|
||||
SEARCHENGINE_URL =
|
||||
SEARCHDATA_FILE = searchdata.xml
|
||||
EXTERNAL_SEARCH_ID =
|
||||
EXTRA_SEARCH_MAPPINGS =
|
||||
GENERATE_LATEX = YES
|
||||
LATEX_OUTPUT = latex
|
||||
LATEX_CMD_NAME = latex
|
||||
MAKEINDEX_CMD_NAME = makeindex
|
||||
COMPACT_LATEX = NO
|
||||
PAPER_TYPE = a4
|
||||
EXTRA_PACKAGES =
|
||||
LATEX_HEADER =
|
||||
LATEX_FOOTER =
|
||||
LATEX_EXTRA_FILES =
|
||||
PDF_HYPERLINKS = YES
|
||||
USE_PDFLATEX = NO
|
||||
LATEX_BATCHMODE = NO
|
||||
LATEX_HIDE_INDICES = NO
|
||||
LATEX_SOURCE_CODE = NO
|
||||
LATEX_BIB_STYLE = plain
|
||||
GENERATE_RTF = NO
|
||||
RTF_OUTPUT = rtf
|
||||
COMPACT_RTF = NO
|
||||
RTF_HYPERLINKS = NO
|
||||
RTF_STYLESHEET_FILE =
|
||||
RTF_EXTENSIONS_FILE =
|
||||
GENERATE_MAN = NO
|
||||
MAN_OUTPUT = man
|
||||
MAN_EXTENSION = .3
|
||||
MAN_LINKS = NO
|
||||
GENERATE_XML = NO
|
||||
XML_OUTPUT = xml
|
||||
XML_SCHEMA =
|
||||
XML_DTD =
|
||||
XML_PROGRAMLISTING = YES
|
||||
GENERATE_DOCBOOK = NO
|
||||
DOCBOOK_OUTPUT = docbook
|
||||
GENERATE_AUTOGEN_DEF = NO
|
||||
GENERATE_PERLMOD = NO
|
||||
PERLMOD_LATEX = NO
|
||||
PERLMOD_PRETTY = YES
|
||||
PERLMOD_MAKEVAR_PREFIX =
|
||||
ENABLE_PREPROCESSING = YES
|
||||
MACRO_EXPANSION = NO
|
||||
EXPAND_ONLY_PREDEF = NO
|
||||
SEARCH_INCLUDES = YES
|
||||
INCLUDE_PATH =
|
||||
INCLUDE_FILE_PATTERNS =
|
||||
PREDEFINED =
|
||||
EXPAND_AS_DEFINED =
|
||||
SKIP_FUNCTION_MACROS = YES
|
||||
TAGFILES =
|
||||
GENERATE_TAGFILE =
|
||||
ALLEXTERNALS = NO
|
||||
EXTERNAL_GROUPS = YES
|
||||
EXTERNAL_PAGES = YES
|
||||
PERL_PATH = /usr/bin/perl
|
||||
CLASS_DIAGRAMS = YES
|
||||
MSCGEN_PATH =
|
||||
HIDE_UNDOC_RELATIONS = YES
|
||||
HAVE_DOT = NO
|
||||
DOT_NUM_THREADS = 0
|
||||
DOT_FONTNAME = Helvetica
|
||||
DOT_FONTSIZE = 10
|
||||
DOT_FONTPATH =
|
||||
CLASS_GRAPH = YES
|
||||
COLLABORATION_GRAPH = YES
|
||||
GROUP_GRAPHS = YES
|
||||
UML_LOOK = NO
|
||||
UML_LIMIT_NUM_FIELDS = 10
|
||||
TEMPLATE_RELATIONS = NO
|
||||
INCLUDE_GRAPH = YES
|
||||
INCLUDED_BY_GRAPH = YES
|
||||
CALL_GRAPH = NO
|
||||
CALLER_GRAPH = NO
|
||||
GRAPHICAL_HIERARCHY = YES
|
||||
DIRECTORY_GRAPH = YES
|
||||
DOT_IMAGE_FORMAT = png
|
||||
INTERACTIVE_SVG = NO
|
||||
DOT_PATH =
|
||||
DOTFILE_DIRS =
|
||||
MSCFILE_DIRS =
|
||||
DOT_GRAPH_MAX_NODES = 50
|
||||
MAX_DOT_GRAPH_DEPTH = 0
|
||||
DOT_TRANSPARENT = NO
|
||||
DOT_MULTI_TARGETS = NO
|
||||
GENERATE_LEGEND = YES
|
||||
DOT_CLEANUP = YES
|
||||
259
external/glim/exception.hpp
vendored
Normal file
259
external/glim/exception.hpp
vendored
Normal file
@ -0,0 +1,259 @@
|
||||
#ifndef _GLIM_EXCEPTION_HPP_INCLUDED
|
||||
#define _GLIM_EXCEPTION_HPP_INCLUDED
|
||||
|
||||
/// \file
|
||||
/// Exceptions with configurable behaviour.
|
||||
/// Requires `thread_local` support introduced in [gcc-4.8](http://gcc.gnu.org/gcc-4.8/changes.html)
|
||||
/// (`__thread` is not reliable with GCC 4.7.2 across shared libraries).
|
||||
|
||||
#include <stdexcept>
|
||||
#include <string>
|
||||
#include <sstream>
|
||||
#include <stdint.h>
|
||||
#include <stdlib.h> // free
|
||||
#include <unistd.h> // write
|
||||
|
||||
/// Throws `::glim::Exception` passing the current file and line into constructor.
|
||||
#define GTHROW(message) throw ::glim::Exception (message, __FILE__, __LINE__)
|
||||
/// Throws a `::glim::Exception` derived exception `name` passing the current file and line into constructor.
|
||||
#define GNTHROW(name, message) throw name (message, __FILE__, __LINE__)
|
||||
/// Helps defining new `::glim::Exception`-based exceptions.
|
||||
/// Named exceptions might be useful in a debugger.
|
||||
#define G_DEFINE_EXCEPTION(name) \
|
||||
struct name: public ::glim::Exception { \
|
||||
name (const ::std::string& message, const char* file, int line): ::glim::Exception (message, file, line) {} \
|
||||
}
|
||||
|
||||
// Workaround to compile under GCC 4.7.
|
||||
#if defined (__GNUC__) && __GNUC__ == 4 && __GNUC_MINOR__ == 7 && !defined (thread_local)
|
||||
# define thread_local __thread
|
||||
#endif
|
||||
|
||||
namespace glim {
|
||||
|
||||
// Ideas:
|
||||
// RAII control via thread-local integer (with bits): option to capture stack trace (printed on "what()")
|
||||
// see http://stacktrace.svn.sourceforge.net/viewvc/stacktrace/stacktrace/call_stack_gcc.cpp?revision=40&view=markup
|
||||
// A handler to log exception with VALGRIND (with optional trace)
|
||||
// A handler to log thread id and *pause* the thread in exception constructor (user can attach GDB and investigate)
|
||||
// (or we might call an empty function: "I once used something similar,
|
||||
// but with an empty function debug_breakpoint. When debugging, I simply entered "bre debug_breakpoint"
|
||||
// at the gdb prompt - no asembler needed (compile debug_breakpoint in a separate compilation unit to avoid having the call optimized away)."
|
||||
// - http://stackoverflow.com/a/4720403/257568)
|
||||
// A handler to call a debugger? (see: http://stackoverflow.com/a/4732119/257568)
|
||||
|
||||
// todo: Try a helper which uses cairo's backtrace-symbols.c
|
||||
// http://code.ohloh.net/file?fid=zUOUdEl-Id-ijyPOmCkVnBJt2d8&cid=zGpizbyIjEw&s=addr2line&browser=Default#L7
|
||||
|
||||
// todo: Try a helper which uses cairo's lookup-symbol.c
|
||||
// http://code.ohloh.net/file?fid=Je2jZqsOxge_SvWVrvywn2I0TIs&cid=zGpizbyIjEw&s=addr2line&browser=Default#L0
|
||||
|
||||
// todo: A helper converting backtrace to addr2line invocation, e.g.
|
||||
// bin/test_exception() [0x4020cc];bin/test_exception(__cxa_throw+0x47) [0x402277];bin/test_exception() [0x401c06];/lib/x86_64-linux-gnu/libc.so.6(__libc_start_main+0xfd) [0x57f0ead];bin/test_exception() [0x401fd1];
|
||||
// should be converted to
|
||||
// addr2line -pifCa -e bin/test_exception 0x4020cc 0x402277 0x401c06 0x57f0ead 0x401fd1
|
||||
//
|
||||
// The helper should read the shared library addresses from /proc/.../map and generate separate addr2line invocations
|
||||
// for groups of addresses inside the same shared library.
|
||||
// => dladdr instead of /proc/../map; http://stackoverflow.com/a/2606152/257568
|
||||
//
|
||||
// Shared libraries (http://stackoverflow.com/a/7557756/257568).
|
||||
// Example, backtrace: /usr/local/lib/libfrople.so(_ZN5mongo14BSONObjBuilder8appendAsERKNS_11BSONElementERKNS_10StringDataE+0x1ca) [0x2aef5b45eb8a]
|
||||
// cat /proc/23630/maps | grep libfrople
|
||||
// -> 2aef5b363000-2aef5b53e000
|
||||
// 0x2aef5b45eb8a - 2aef5b363000 = FBB8A
|
||||
// addr2line -pifCa -e /usr/local/lib/libfrople.so 0xFBB8A
|
||||
//
|
||||
// cat /proc/`pidof FropleAndImg2`/maps | grep libfrople
|
||||
// addr2line -pifCa -e /usr/local/lib/libfrople.so `perl -e 'printf ("%x", 0x2aef5b45eb8a - 0x2aef5b363000)'`
|
||||
|
||||
inline void captureBacktrace (void* stdStringPtr);
|
||||
|
||||
typedef void (*exception_handler_fn)(void*);
|
||||
|
||||
/// Exception with file and line information and optional stack trace capture.
|
||||
/// Requires `thread_local` support ([gcc-4.8](http://gcc.gnu.org/gcc-4.8/changes.html)).
|
||||
class Exception: public std::runtime_error {
|
||||
protected:
|
||||
const char* _file; int32_t _line;
|
||||
std::string _what;
|
||||
uint32_t _options;
|
||||
|
||||
/// Append [{file}:{line}] into `buf`.
|
||||
void appendLine (std::string& buf) const {
|
||||
if (_file || _line > 0) {
|
||||
std::ostringstream oss;
|
||||
oss << '[';
|
||||
if (_file) oss << _file;
|
||||
if (_line >= 0) oss << ':' << _line;
|
||||
oss << "] ";
|
||||
buf.append (oss.str());
|
||||
}
|
||||
}
|
||||
|
||||
/// Append a stack trace to `_what`.
|
||||
void capture() {
|
||||
if (_options & RENDEZVOUS) rendezvous();
|
||||
if (_options & CAPTURE_TRACE) {
|
||||
appendLine (_what);
|
||||
_what += "[at ";
|
||||
captureBacktrace (&_what);
|
||||
_what.append ("] ");
|
||||
_what += std::runtime_error::what();
|
||||
}
|
||||
}
|
||||
|
||||
public:
|
||||
/** The reference to the thread-local options. */
|
||||
inline static uint32_t& options() {
|
||||
static thread_local uint32_t EXCEPTION_OPTIONS = 0;
|
||||
return EXCEPTION_OPTIONS;
|
||||
}
|
||||
enum Options: uint32_t {
|
||||
PLAIN_WHAT = 1, ///< Pass `what` as is, do not add any information to it.
|
||||
HANDLE_ALL = 1 << 1, ///< Run the custom handler from `__cxa_throw`.
|
||||
CAPTURE_TRACE = 1 << 2, ///< Append a stack trace into the `Exception::_what` (with the help of the `captureBacktrace`).
|
||||
RENDEZVOUS = 1 << 3 ///< Call the rendezvous function in `throw` and in `what`, so that the GDB can catch it (break glim::Exception::rendezvous).
|
||||
};
|
||||
|
||||
/** The pointer to the thread-local exception handler. */
|
||||
inline static exception_handler_fn* handler() {
|
||||
static thread_local exception_handler_fn EXCEPTION_HANDLER = nullptr;
|
||||
return &EXCEPTION_HANDLER;
|
||||
}
|
||||
/** The pointer to the thread-local argument for the exception handler. */
|
||||
inline static void** handlerArg() {
|
||||
static thread_local void* EXCEPTION_HANDLER_ARG = nullptr;
|
||||
return &EXCEPTION_HANDLER_ARG;
|
||||
}
|
||||
|
||||
/// Invoked when the `RENDEZVOUS` option is set in order to help the debugger catch the exception (break glim::Exception::rendezvous).
|
||||
static void rendezvous() __attribute__((noinline)) {
|
||||
asm (""); // Prevents the function from being optimized away.
|
||||
}
|
||||
|
||||
Exception (const std::string& message):
|
||||
std::runtime_error (message), _file (0), _line (-1), _options (options()) {
|
||||
capture();}
|
||||
Exception (const std::string& message, const char* file, int32_t line):
|
||||
std::runtime_error (message), _file (file), _line (line), _options (options()) {
|
||||
capture();}
|
||||
~Exception() throw() {}
|
||||
virtual const char* what() const throw() {
|
||||
if (_options & RENDEZVOUS) rendezvous();
|
||||
if (_options & PLAIN_WHAT) return std::runtime_error::what();
|
||||
std::string& buf = const_cast<std::string&> (_what);
|
||||
if (buf.empty()) {
|
||||
appendLine (buf);
|
||||
buf.append (std::runtime_error::what());
|
||||
}
|
||||
return buf.c_str();
|
||||
}
|
||||
};
|
||||
|
||||
/// RAII control of thrown `Exception`s.
|
||||
/// Example: \code
|
||||
/// glim::ExceptionControl trace (glim::Exception::Options::CAPTURE_TRACE);
|
||||
/// \endcode
|
||||
/// Modifies the `Exception` options via a thread-local variable and restores them back upon destruction.\n
|
||||
/// Currently uses http://gcc.gnu.org/onlinedocs/gcc-4.7.2/gcc/Thread_002dLocal.html
|
||||
/// (might use C++11 `thread_local` in the future).
|
||||
class ExceptionControl {
|
||||
protected:
|
||||
uint32_t _savedOptions;
|
||||
public:
|
||||
ExceptionControl (uint32_t newOptions) {
|
||||
uint32_t& options = Exception::options();
|
||||
_savedOptions = options;
|
||||
options = newOptions;
|
||||
}
|
||||
~ExceptionControl() {
|
||||
Exception::options() = _savedOptions;
|
||||
}
|
||||
};
|
||||
|
||||
class ExceptionHandler {
|
||||
protected:
|
||||
uint32_t _savedOptions;
|
||||
exception_handler_fn _savedHandler;
|
||||
void* _savedHandlerArg;
|
||||
public:
|
||||
ExceptionHandler (uint32_t newOptions, exception_handler_fn handler, void* handlerArg) {
|
||||
uint32_t& options = Exception::options(); _savedOptions = options; options = newOptions;
|
||||
exception_handler_fn* handler_ = Exception::handler(); _savedHandler = *handler_; *handler_ = handler;
|
||||
void** handlerArg_ = Exception::handlerArg(); _savedHandlerArg = *handlerArg_; *handlerArg_ = handlerArg;
|
||||
}
|
||||
~ExceptionHandler() {
|
||||
Exception::options() = _savedOptions;
|
||||
*Exception::handler() = _savedHandler;
|
||||
*Exception::handlerArg() = _savedHandlerArg;
|
||||
}
|
||||
};
|
||||
|
||||
} // namespace glim
|
||||
|
||||
#if defined(__GNUC__) && (defined (__linux__) || defined (_SYSTYPE_BSD))
|
||||
# include <execinfo.h> // backtrace; http://www.gnu.org/software/libc/manual/html_node/Backtraces.html
|
||||
# define _GLIM_USE_EXECINFO
|
||||
#endif
|
||||
|
||||
namespace glim {
|
||||
|
||||
/** If `stdStringPtr` is not null then backtrace is saved there (must point to an std::string instance),
|
||||
* otherwise printed to write(2). */
|
||||
void captureBacktrace (void* stdStringPtr) {
|
||||
#ifdef _GLIM_USE_EXECINFO
|
||||
const int arraySize = 10; void *array[arraySize];
|
||||
int got = ::backtrace (array, arraySize);
|
||||
if (stdStringPtr) {
|
||||
std::string* out = (std::string*) stdStringPtr;
|
||||
char **strings = ::backtrace_symbols (array, got);
|
||||
for (int tn = 0; tn < got; ++tn) {out->append (strings[tn]); out->append (1, ';');}
|
||||
::free (strings);
|
||||
} else ::backtrace_symbols_fd (array, got, 2);
|
||||
#else
|
||||
# warning captureBacktrace: I do not know how to capture backtrace there. Patches welcome.
|
||||
#endif
|
||||
}
|
||||
|
||||
} // namespace glim
|
||||
|
||||
#endif // _GLIM_EXCEPTION_HPP_INCLUDED
|
||||
|
||||
/**
|
||||
* Special handler for ALL exceptions. Usage:
|
||||
* 1) In the `main` module inject this code with:
|
||||
* #define _GLIM_ALL_EXCEPTIONS_CODE
|
||||
* #include <glim/exception.hpp>
|
||||
* 2) Link with "-ldl" (for `dlsym`).
|
||||
* 3) Use the ExceptionHandler to enable special behaviour in the current thread:
|
||||
* glim::ExceptionHandler traceExceptions (glim::Exception::Options::HANDLE_ALL, glim::captureBacktrace, nullptr);
|
||||
*
|
||||
* About handing all exceptions see:
|
||||
* http://stackoverflow.com/a/11674810/257568
|
||||
* http://blog.sjinks.pro/c-cpp/969-track-uncaught-exceptions/
|
||||
*/
|
||||
#ifdef _GLIM_ALL_EXCEPTIONS_CODE
|
||||
|
||||
#include <dlfcn.h> // dlsym
|
||||
|
||||
typedef void(*cxa_throw_type)(void*, void*, void(*)(void*)); // Tested with GCC 4.7.
|
||||
static cxa_throw_type NATIVE_CXA_THROW = 0;
|
||||
|
||||
extern "C" void __cxa_throw (void* thrown_exception, void* tinfo, void (*dest)(void*)) {
|
||||
if (!NATIVE_CXA_THROW) NATIVE_CXA_THROW = reinterpret_cast<cxa_throw_type> (::dlsym (RTLD_NEXT, "__cxa_throw"));
|
||||
if (!NATIVE_CXA_THROW) ::std::terminate();
|
||||
|
||||
using namespace glim;
|
||||
uint32_t options = Exception::options();
|
||||
if (options & Exception::RENDEZVOUS) Exception::rendezvous();
|
||||
if (options & Exception::HANDLE_ALL) {
|
||||
exception_handler_fn handler = *Exception::handler();
|
||||
if (handler) handler (*Exception::handlerArg());
|
||||
}
|
||||
|
||||
NATIVE_CXA_THROW (thrown_exception, tinfo, dest);
|
||||
}
|
||||
|
||||
#undef _GLIM_ALL_EXCEPTIONS_CODE
|
||||
#endif // _GLIM_ALL_EXCEPTIONS_CODE
|
||||
578
external/glim/gstring.hpp
vendored
Normal file
578
external/glim/gstring.hpp
vendored
Normal file
@ -0,0 +1,578 @@
|
||||
#ifndef _GSTRING_INCLUDED
|
||||
#define _GSTRING_INCLUDED
|
||||
|
||||
/**
|
||||
* A C++ char string.\n
|
||||
* Can reuse (stack-allocated) buffers.\n
|
||||
* Can create zero-copy views.
|
||||
* @code
|
||||
Copyright 2012 Kozarezov Artem Aleksandrovich
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
* @endcode
|
||||
* @file
|
||||
*/
|
||||
|
||||
#include <assert.h>
|
||||
#include <stdlib.h> // malloc, realloc, free
|
||||
#include <stdint.h>
|
||||
#include <string.h> // memcpy, memmem
|
||||
#include <stdio.h> // snprintf
|
||||
#include <stdexcept>
|
||||
#include <iostream>
|
||||
#include <iterator>
|
||||
|
||||
#include "exception.hpp"
|
||||
|
||||
/// Make a read-only gstring from a C string: `const gstring foo = C2GSTRING("foo")`.
|
||||
#define C2GSTRING(CSTR) ::glim::gstring (::glim::gstring::ReferenceConstructor(), CSTR, sizeof (CSTR) - 1, true)
|
||||
/// Usage: GSTRING_ON_STACK (buf, 64) << "foo" << "bar";
|
||||
#define GSTRING_ON_STACK(NAME, SIZE) char NAME##Buf[SIZE]; ::glim::gstring NAME (SIZE, NAME##Buf, false, 0); NAME.self()
|
||||
|
||||
namespace glim {
|
||||
|
||||
/**
|
||||
* Based on: C++ version 0.4 char* style "itoa": Written by Lukás Chmela, http://www.strudel.org.uk/itoa/ (GPLv3).
|
||||
* Returns a pointer to the end of the string.
|
||||
* NB about `inline`: http://stackoverflow.com/a/1759575/257568
|
||||
* @param base Maximum is 36 (see http://en.wikipedia.org/wiki/Base_36).
|
||||
*/
|
||||
inline char* itoa (char* ptr, int64_t value, const int base = 10) {
|
||||
// check that the base is valid
|
||||
if (base < 2 || base > 36) {*ptr = '\0'; return ptr;}
|
||||
|
||||
char *ptr1 = ptr;
|
||||
int64_t tmp_value;
|
||||
|
||||
do {
|
||||
tmp_value = value;
|
||||
value /= base;
|
||||
*ptr++ = "zyxwvutsrqponmlkjihgfedcba9876543210123456789abcdefghijklmnopqrstuvwxyz" [35 + (tmp_value - value * base)];
|
||||
} while (value);
|
||||
|
||||
// Apply negative sign
|
||||
if (tmp_value < 0) *ptr++ = '-';
|
||||
char* end = ptr;
|
||||
*ptr-- = '\0';
|
||||
char tmp_char;
|
||||
while (ptr1 < ptr) {
|
||||
tmp_char = *ptr;
|
||||
*ptr--= *ptr1;
|
||||
*ptr1++ = tmp_char;
|
||||
}
|
||||
return end;
|
||||
}
|
||||
|
||||
class gstring_stream;
|
||||
|
||||
class gstring {
|
||||
enum Flags {
|
||||
FREE_FLAG = 0x80000000, // 1st bit; `_buf` needs `free`ing
|
||||
FREE_OFFSET = 31,
|
||||
REF_FLAG = 0x40000000, // 2nd bit; `_buf` has an extended life-time (such as C string literals) and can be shared (passed by reference)
|
||||
REF_OFFSET = 30,
|
||||
CAPACITY_MASK = 0x3F000000, // 3..8 bits; `_buf` size is 2^this
|
||||
CAPACITY_OFFSET = 24,
|
||||
LENGTH_MASK = 0x00FFFFFF, // 9th bit; allocated capacity
|
||||
};
|
||||
uint32_t _meta;
|
||||
public:
|
||||
void* _buf;
|
||||
public:
|
||||
constexpr gstring() noexcept: _meta (0), _buf (nullptr) {}
|
||||
/**
|
||||
* Reuse `buf` of size `bufSize`.
|
||||
* To fully use `buf` the `bufSize` should be the power of two.
|
||||
* @param bufSize The size of the memory allocated to `buf`.
|
||||
* @param buf The memory region to be reused.
|
||||
* @param free Whether the `buf` should be `free`d on resize or gstring destruction.
|
||||
* @param length String length inside the `buf`.
|
||||
* @param ref If true then the `buf` isn't copied by gstring's copy constructors.
|
||||
* This is useful for wrapping C string literals.
|
||||
*/
|
||||
explicit gstring (uint32_t bufSize, void* buf, bool free, uint32_t length, bool ref = false) noexcept {
|
||||
uint32_t power = 0; while (((uint32_t) 1 << (power + 1)) <= bufSize) ++power;
|
||||
_meta = ((uint32_t) free << FREE_OFFSET) |
|
||||
((uint32_t) ref << REF_OFFSET) |
|
||||
(power << CAPACITY_OFFSET) |
|
||||
(length & LENGTH_MASK);
|
||||
_buf = buf;
|
||||
}
|
||||
|
||||
struct ReferenceConstructor {};
|
||||
/// Make a view to the given cstring.
|
||||
/// @param buf The memory region to be reused.
|
||||
/// @param length String length inside the `buf`.
|
||||
/// @param ref If true then the `buf` isn't copied by gstring's copy constructors.
|
||||
/// This is useful for wrapping C string literals.
|
||||
explicit constexpr gstring (ReferenceConstructor, const char* buf, uint32_t length, bool ref = false) noexcept:
|
||||
_meta (((uint32_t) ref << REF_OFFSET) | (length & LENGTH_MASK)), _buf ((void*) buf) {}
|
||||
|
||||
/// Copy the characters into `gstring`.
|
||||
gstring (const char* chars): _meta (0), _buf (nullptr) {
|
||||
if (chars && *chars) {
|
||||
size_t length = ::strlen (chars);
|
||||
_buf = ::malloc (length);
|
||||
::memcpy (_buf, chars, length);
|
||||
_meta = (uint32_t) FREE_FLAG |
|
||||
(length & LENGTH_MASK);
|
||||
}
|
||||
}
|
||||
|
||||
/// Copy the characters into `gstring`.
|
||||
gstring (const char* chars, size_t length) {
|
||||
if (length != 0) {
|
||||
_buf = ::malloc (length);
|
||||
::memcpy (_buf, chars, length);
|
||||
} else _buf = nullptr;
|
||||
_meta = (uint32_t) FREE_FLAG |
|
||||
(length & LENGTH_MASK);
|
||||
}
|
||||
|
||||
/// Copy into `gstring`.
|
||||
gstring (const std::string& str): _meta (0), _buf (nullptr) {
|
||||
if (!str.empty()) {
|
||||
_buf = ::malloc (str.length());
|
||||
::memcpy (_buf, str.data(), str.length());
|
||||
_meta = (uint32_t) FREE_FLAG |
|
||||
(str.length() & LENGTH_MASK);
|
||||
}
|
||||
}
|
||||
|
||||
/// If `gstr` is `copiedByReference` then make a shallow copy of it,
|
||||
/// otherwise copy `gstr` contents into a `malloc`ed buffer.
|
||||
gstring (const gstring& gstr) {
|
||||
uint32_t glen = gstr.length();
|
||||
if (glen != 0) {
|
||||
if (gstr.copiedByReference()) {
|
||||
_meta = gstr._meta; _buf = gstr._buf;
|
||||
} else {
|
||||
_buf = ::malloc (glen);
|
||||
if (!_buf) GTHROW ("!malloc");
|
||||
::memcpy (_buf, gstr._buf, glen);
|
||||
_meta = (uint32_t) FREE_FLAG |
|
||||
(glen & LENGTH_MASK);
|
||||
}
|
||||
} else {
|
||||
_meta = 0; _buf = nullptr;
|
||||
}
|
||||
}
|
||||
gstring (gstring&& gstr) noexcept: _meta (gstr._meta), _buf (gstr._buf) {
|
||||
gstr._meta = 0; gstr._buf = nullptr;
|
||||
}
|
||||
gstring& operator = (const gstring& gstr) {
|
||||
// cf. http://stackoverflow.com/questions/9322174/move-assignment-operator-and-if-this-rhs
|
||||
if (this != &gstr) {
|
||||
uint32_t glen = gstr.length();
|
||||
uint32_t power = 0;
|
||||
uint32_t capacity = this->capacity();
|
||||
if (glen <= capacity && capacity > 1) { // `capacity <= 1` means there is no _buf.
|
||||
// We reuse existing buffer. Keep capacity info.
|
||||
power = (_meta & CAPACITY_MASK) >> CAPACITY_OFFSET;
|
||||
} else {
|
||||
if (_buf != nullptr && needsFreeing()) ::free (_buf);
|
||||
if (gstr.copiedByReference()) {
|
||||
_meta = gstr._meta; _buf = gstr._buf;
|
||||
return *this;
|
||||
}
|
||||
_buf = ::malloc (glen);
|
||||
if (_buf == nullptr) GTHROW ("malloc failed");
|
||||
}
|
||||
::memcpy (_buf, gstr._buf, glen);
|
||||
_meta = (uint32_t) FREE_FLAG |
|
||||
(power << CAPACITY_OFFSET) |
|
||||
(glen & LENGTH_MASK);
|
||||
}
|
||||
return *this;
|
||||
}
|
||||
gstring& operator = (gstring&& gstr) noexcept {
|
||||
assert (this != &gstr);
|
||||
if (_buf != nullptr && needsFreeing()) free (_buf);
|
||||
_meta = gstr._meta; _buf = gstr._buf;
|
||||
gstr._meta = 0; gstr._buf = nullptr;
|
||||
return *this;
|
||||
}
|
||||
|
||||
/// Return a copy of the string.
|
||||
gstring clone() const {return gstring (data(), length());}
|
||||
/// If the gstring's buffer is not owned then copy the bytes into the owned one.
|
||||
/// Useful for turning a stack-allocated gstring into a heap-allocated gstring.
|
||||
gstring& owned() {if (!needsFreeing()) *this = gstring (data(), length()); return *this;}
|
||||
/** Returns a reference to the gstring: when the reference is copied the internal buffer is not copied but referenced (shallow copy).\n
|
||||
* This method should only be used if it is know that the life-time of the reference and its copies is less than the life-time of the buffer. */
|
||||
gstring ref() const noexcept {return gstring (0, _buf, false, length(), true);}
|
||||
|
||||
bool needsFreeing() const noexcept {return _meta & FREE_FLAG;}
|
||||
bool copiedByReference() const noexcept {return _meta & REF_FLAG;}
|
||||
/// Current buffer capacity (memory allocated to the string). Returns 1 if no memory allocated.
|
||||
uint32_t capacity() const noexcept {return 1 << ((_meta & CAPACITY_MASK) >> CAPACITY_OFFSET);}
|
||||
uint32_t length() const noexcept {return _meta & LENGTH_MASK;}
|
||||
size_t size() const noexcept {return _meta & LENGTH_MASK;}
|
||||
bool empty() const noexcept {return (_meta & LENGTH_MASK) == 0;}
|
||||
std::string str() const {size_t len = size(); return len ? std::string ((const char*) _buf, len) : std::string();}
|
||||
/// NB: might move the string to a new buffer.
|
||||
const char* c_str() const {
|
||||
uint32_t len = length(); if (len == 0) return "";
|
||||
uint32_t cap = capacity();
|
||||
// c_str should work even for const gstring's, otherwise it's too much of a pain.
|
||||
if (cap < len + 1) const_cast<gstring*> (this) ->reserve (len + 1);
|
||||
char* buf = (char*) _buf; buf[len] = 0; return buf;
|
||||
}
|
||||
bool equals (const char* cstr) const noexcept {
|
||||
const char* cstr_; uint32_t clen_;
|
||||
if (cstr != nullptr) {cstr_ = cstr; clen_ = strlen (cstr);} else {cstr_ = ""; clen_ = 0;}
|
||||
const uint32_t len = length();
|
||||
if (len != clen_) return false;
|
||||
const char* gstr_ = _buf != nullptr ? (const char*) _buf : "";
|
||||
return memcmp (gstr_, cstr_, len) == 0;
|
||||
}
|
||||
bool equals (const gstring& gs) const noexcept {
|
||||
uint32_t llen = length(), olen = gs.length();
|
||||
if (llen != olen) return false;
|
||||
return memcmp ((const char*) _buf, (const char*) gs._buf, llen) == 0;
|
||||
}
|
||||
|
||||
char& operator[] (unsigned index) noexcept {return ((char*)_buf)[index];}
|
||||
const char& operator[] (unsigned index) const noexcept {return ((const char*)_buf)[index];}
|
||||
|
||||
/// Access `_buf` as `char*`. `_buf` might be nullptr.
|
||||
char* data() noexcept {return (char*)_buf;}
|
||||
const char* data() const noexcept {return (const char*)_buf;}
|
||||
|
||||
char* endp() noexcept {return (char*)_buf + length();}
|
||||
const char* endp() const noexcept {return (const char*)_buf + length();}
|
||||
|
||||
gstring view (uint32_t pos, int32_t count = -1) noexcept {
|
||||
return gstring (0, data() + pos, false, count >= 0 ? count : length() - pos, copiedByReference());}
|
||||
const gstring view (uint32_t pos, int32_t count = -1) const noexcept {
|
||||
return gstring (0, (void*)(data() + pos), false, count >= 0 ? count : length() - pos, copiedByReference());}
|
||||
|
||||
// http://en.cppreference.com/w/cpp/concept/Iterator
|
||||
template<typename CT> struct iterator_t: public std::iterator<std::random_access_iterator_tag, CT, int32_t> {
|
||||
CT* _ptr;
|
||||
iterator_t () noexcept: _ptr (nullptr) {}
|
||||
iterator_t (CT* ptr) noexcept: _ptr (ptr) {}
|
||||
iterator_t (const iterator_t<CT>& it) noexcept: _ptr (it._ptr) {}
|
||||
|
||||
CT& operator*() const noexcept {return *_ptr;}
|
||||
CT* operator->() const noexcept {return _ptr;}
|
||||
CT& operator[](int32_t ofs) const noexcept {return _ptr[ofs];}
|
||||
|
||||
iterator_t<CT>& operator++() noexcept {++_ptr; return *this;}
|
||||
iterator_t<CT> operator++(int) noexcept {return iterator_t<CT> (_ptr++);};
|
||||
iterator_t<CT>& operator--() noexcept {--_ptr; return *this;}
|
||||
iterator_t<CT> operator--(int) noexcept {return iterator_t<CT> (_ptr--);};
|
||||
bool operator == (const iterator_t<CT>& i2) const noexcept {return _ptr == i2._ptr;}
|
||||
bool operator != (const iterator_t<CT>& i2) const noexcept {return _ptr != i2._ptr;}
|
||||
bool operator < (const iterator_t<CT>& i2) const noexcept {return _ptr < i2._ptr;}
|
||||
bool operator > (const iterator_t<CT>& i2) const noexcept {return _ptr > i2._ptr;}
|
||||
bool operator <= (const iterator_t<CT>& i2) const noexcept {return _ptr <= i2._ptr;}
|
||||
bool operator >= (const iterator_t<CT>& i2) const noexcept {return _ptr >= i2._ptr;}
|
||||
iterator_t<CT> operator + (int32_t ofs) const noexcept {return iterator (_ptr + ofs);}
|
||||
iterator_t<CT>& operator += (int32_t ofs) noexcept {_ptr += ofs; return *this;}
|
||||
iterator_t<CT> operator - (int32_t ofs) const noexcept {return iterator (_ptr - ofs);}
|
||||
iterator_t<CT>& operator -= (int32_t ofs) noexcept {_ptr -= ofs; return *this;}
|
||||
};
|
||||
// http://en.cppreference.com/w/cpp/concept/Container
|
||||
typedef char value_type;
|
||||
typedef char& reference;
|
||||
typedef const char& const_reference;
|
||||
typedef uint32_t size_type;
|
||||
typedef int32_t difference_type;
|
||||
typedef iterator_t<char> iterator;
|
||||
typedef iterator_t<const char> const_iterator;
|
||||
iterator begin() noexcept {return iterator ((char*) _buf);}
|
||||
const_iterator begin() const noexcept {return const_iterator ((char*) _buf);}
|
||||
iterator end() noexcept {return iterator ((char*) _buf + size());}
|
||||
const_iterator end() const noexcept {return const_iterator ((char*) _buf + size());}
|
||||
const_iterator cbegin() const noexcept {return const_iterator ((char*) _buf);}
|
||||
const_iterator cend() const noexcept {return const_iterator ((char*) _buf + size());}
|
||||
|
||||
/** Returns -1 if not found. */
|
||||
int32_t find (const char* str, int32_t pos, int32_t count) const noexcept {
|
||||
const int32_t hlen = (int32_t) length() - pos;
|
||||
if (hlen <= 0) return -1;
|
||||
char* haystack = (char*) _buf + pos;
|
||||
void* mret = memmem (haystack, hlen, str, count);
|
||||
if (mret == 0) return -1;
|
||||
return (char*) mret - (char*) _buf;
|
||||
}
|
||||
int32_t find (const char* str, int32_t pos = 0) const noexcept {return find (str, pos, strlen (str));}
|
||||
|
||||
/** Index of `ch` inside the string or -1 if not found. */
|
||||
int32_t indexOf (char ch) const noexcept {
|
||||
void* ret = memchr (_buf, ch, size());
|
||||
return ret == nullptr ? -1 : (char*) ret - (char*) _buf;
|
||||
}
|
||||
|
||||
// Helps to workaround the "statement has no effect" warning in `GSTRING_ON_STACK`.
|
||||
gstring& self() noexcept {return *this;}
|
||||
|
||||
/** Grow buffer to be at least `to` characters long. */
|
||||
void reserve (uint32_t to) {
|
||||
uint32_t power = (_meta & CAPACITY_MASK) >> CAPACITY_OFFSET;
|
||||
if (((uint32_t) 1 << power) < to) {
|
||||
++power;
|
||||
while (((uint32_t) 1 << power) < to) ++power;
|
||||
if (power > 24) {GSTRING_ON_STACK (error, 64) << "gstring too large: " << (int) to; GTHROW (error.str());}
|
||||
} else if (power) {
|
||||
// No need to grow.
|
||||
return;
|
||||
}
|
||||
_meta = (_meta & ~CAPACITY_MASK) | (power << CAPACITY_OFFSET);
|
||||
if (needsFreeing() && _buf != nullptr) {
|
||||
_buf = ::realloc (_buf, capacity());
|
||||
if (_buf == nullptr) GTHROW ("realloc failed");
|
||||
} else {
|
||||
const char* oldBuf = (const char*) _buf;
|
||||
_buf = ::malloc (capacity());
|
||||
if (_buf == nullptr) GTHROW ("malloc failed");
|
||||
if (oldBuf != nullptr) ::memcpy (_buf, oldBuf, length());
|
||||
_meta |= FREE_FLAG;
|
||||
}
|
||||
}
|
||||
|
||||
/** Length setter. Useful when you manually write into the buffer or to cut the string. */
|
||||
void length (uint32_t len) noexcept {
|
||||
_meta = (_meta & ~LENGTH_MASK) | (len & LENGTH_MASK);
|
||||
}
|
||||
|
||||
protected:
|
||||
friend class gstring_stream;
|
||||
public:
|
||||
/** Appends an integer to the string.
|
||||
* @param base Radix, from 1 to 36 (default 10).
|
||||
* @param bytes How many bytes to reserve (24 by default). */
|
||||
void append64 (int64_t iv, int base = 10, uint_fast8_t bytes = 24) {
|
||||
uint32_t pos = length();
|
||||
if (capacity() < pos + bytes) reserve (pos + bytes);
|
||||
length (itoa ((char*) _buf + pos, iv, base) - (char*) _buf);
|
||||
}
|
||||
void append (char ch) {
|
||||
uint32_t pos = length();
|
||||
const uint32_t cap = capacity();
|
||||
if (pos >= cap || cap <= 1) reserve (pos + 1);
|
||||
((char*)_buf)[pos] = ch;
|
||||
length (++pos);
|
||||
}
|
||||
void append (const char* cstr, uint32_t clen) {
|
||||
uint32_t len = length();
|
||||
uint32_t need = len + clen;
|
||||
const uint32_t cap = capacity();
|
||||
if (need > cap || cap <= 1) reserve (need);
|
||||
::memcpy ((char*) _buf + len, cstr, clen);
|
||||
length (need);
|
||||
}
|
||||
/** This one is for http://code.google.com/p/re2/; `clear` then `append`. */
|
||||
bool ParseFrom (const char* cstr, int clen) {
|
||||
if (clen < 0 || clen > (int) LENGTH_MASK) return false;
|
||||
length (0); append (cstr, (uint32_t) clen); return true;}
|
||||
gstring& operator << (const gstring& gs) {append (gs.data(), gs.length()); return *this;}
|
||||
gstring& operator << (const std::string& str) {append (str.data(), str.length()); return *this;}
|
||||
gstring& operator << (const char* cstr) {if (cstr) append (cstr, ::strlen (cstr)); return *this;}
|
||||
gstring& operator << (char ch) {append (ch); return *this;}
|
||||
gstring& operator << (int iv) {append64 (iv, 10, sizeof (int) * 3); return *this;}
|
||||
gstring& operator << (long iv) {append64 (iv, 10, sizeof (long) * 3); return *this;}
|
||||
gstring& operator << (long long iv) {append64 (iv, 10, sizeof (long long) * 3); return *this;}
|
||||
gstring& operator << (double dv) {
|
||||
uint32_t len = length();
|
||||
reserve (len + 32);
|
||||
int rc = snprintf (endp(), 31, "%f", dv);
|
||||
if (rc > 0) {length (len + std::min (rc, 31));}
|
||||
return *this;
|
||||
}
|
||||
|
||||
bool operator < (const gstring &gs) const noexcept {
|
||||
uint32_t len1 = length(); uint32_t len2 = gs.length();
|
||||
if (len1 == len2) return ::strncmp (data(), gs.data(), len1) < 0;
|
||||
int cmp = ::strncmp (data(), gs.data(), std::min (len1, len2));
|
||||
if (cmp) return cmp < 0;
|
||||
return len1 < len2;
|
||||
}
|
||||
|
||||
/// Asks `strftime` to generate a time string. Capacity is increased if necessary (up to a limit of +1024 bytes).
|
||||
gstring& appendTime (const char* format, struct tm* tmv) {
|
||||
int32_t pos = length(), cap = capacity(), left = cap - pos;
|
||||
if (left < 8) {reserve (pos + 8); return appendTime (format, tmv);}
|
||||
size_t got = strftime ((char*) _buf + pos, left, format, tmv);
|
||||
if (got == 0) {
|
||||
if (left > 1024) return *this; // Guard against perpetual growth.
|
||||
reserve (pos + left * 2); return appendTime (format, tmv);
|
||||
}
|
||||
length (pos + got);
|
||||
return *this;
|
||||
}
|
||||
|
||||
/// Append the characters to this `gstring` wrapping them in the netstring format.
|
||||
gstring& appendNetstring (const char* cstr, uint32_t clen) {
|
||||
*this << (int) clen; append (':'); append (cstr, clen); append (','); return *this;}
|
||||
/// Append the `gstr` wrapping it in the netstring format.
|
||||
gstring& appendNetstring (const gstring& gstr) {return appendNetstring (gstr.data(), gstr.length());}
|
||||
|
||||
std::ostream& writeAsNetstring (std::ostream& stream) const;
|
||||
|
||||
/// Parse netstring at `pos` and return a `gstring` *pointing* at the parsed netstring.\n
|
||||
/// No heap space allocated.\n
|
||||
/// Throws std::runtime_error if netstring parsing fails.\n
|
||||
/// If parsing was successfull, then `after` is set to point after the parsed netstring.
|
||||
gstring netstringAt (uint32_t pos, uint32_t* after = nullptr) const {
|
||||
const uint32_t len = length(); char* buf = (char*) _buf;
|
||||
if (buf == nullptr) GTHROW ("gstring: netstringAt: nullptr");
|
||||
uint32_t next = pos;
|
||||
while (next < len && buf[next] >= '0' && buf[next] <= '9') ++next;
|
||||
if (next >= len || buf[next] != ':' || next - pos > 10) GTHROW ("gstring: netstringAt: no header");
|
||||
char* endptr = 0;
|
||||
long nlen = ::strtol (buf + pos, &endptr, 10);
|
||||
if (endptr != buf + next) GTHROW ("gstring: netstringAt: unexpected header end");
|
||||
pos = next + 1; next = pos + nlen;
|
||||
if (next >= len || buf[next] != ',') GTHROW ("gstring: netstringAt: no body");
|
||||
if (after) *after = next + 1;
|
||||
return gstring (0, buf + pos, false, next - pos);
|
||||
}
|
||||
|
||||
/// Wrapper around strtol, not entirely safe (make sure the string is terminated with a non-digit, by calling c_str, for example).
|
||||
long intAt (uint32_t pos, uint32_t* after = nullptr, int base = 10) const {
|
||||
// BTW: http://www.kumobius.com/2013/08/c-string-to-int/
|
||||
const uint32_t len = length(); char* buf = (char*) _buf;
|
||||
if (pos >= len || buf == nullptr) GTHROW ("gstring: intAt: pos >= len");
|
||||
char* endptr = 0;
|
||||
long lv = ::strtol (buf + pos, &endptr, base);
|
||||
uint32_t next = endptr - buf;
|
||||
if (next > len) GTHROW ("gstring: intAt: endptr > len");
|
||||
if (after) *after = next;
|
||||
return lv;
|
||||
}
|
||||
|
||||
/// Wrapper around strtol. Copies the string into a temporary buffer in order to pass it to strtol. Empty string returns 0.
|
||||
long toInt (int base = 10) const noexcept {
|
||||
const uint32_t len = length(); if (len == 0) return 0;
|
||||
char buf[len + 1]; memcpy (buf, _buf, len); buf[len] = 0;
|
||||
return ::strtol (buf, nullptr, base);
|
||||
}
|
||||
|
||||
/// Get a single netstring from the `stream` and append it to the end of `gstring`.
|
||||
/// Throws an exception if the input is not a well-formed netstring.
|
||||
gstring& readNetstring (std::istream& stream) {
|
||||
int32_t nlen; stream >> nlen;
|
||||
if (!stream.good() || nlen < 0) GTHROW ("!netstring");
|
||||
int ch = stream.get();
|
||||
if (!stream.good() || ch != ':') GTHROW ("!netstring");
|
||||
uint32_t glen = length();
|
||||
const uint32_t cap = capacity();
|
||||
if (cap < glen + nlen || cap <= 1) reserve (glen + nlen);
|
||||
stream.read ((char*) _buf + glen, nlen);
|
||||
if (!stream.good()) GTHROW ("!netstring");
|
||||
ch = stream.get();
|
||||
if (ch != ',') GTHROW ("!netstring");
|
||||
length (glen + nlen);
|
||||
return *this;
|
||||
}
|
||||
|
||||
/// Set length to 0. `_buf` not changed.
|
||||
gstring& clear() noexcept {length (0); return *this;}
|
||||
|
||||
/// Removes `count` characters starting at `pos`.
|
||||
gstring& erase (uint32_t pos, uint32_t count = 1) noexcept {
|
||||
const char* buf = (const char*) _buf;
|
||||
const char* pt1 = buf + pos;
|
||||
const char* pt2 = pt1 + count;
|
||||
uint32_t len = length();
|
||||
const char* end = buf + len;
|
||||
if (pt2 <= end) {
|
||||
length (len - count);
|
||||
::memmove ((void*) pt1, (void*) pt2, end - pt2);
|
||||
}
|
||||
return *this;
|
||||
}
|
||||
/// Remove characters [from,till) and return `from`.\n
|
||||
/// Compatible with "boost/algorithm/string/trim.hpp".
|
||||
iterator_t<char> erase (iterator_t<char> from, iterator_t<char> till) noexcept {
|
||||
intptr_t ipos = from._ptr - (char*) _buf;
|
||||
intptr_t count = till._ptr - from._ptr;
|
||||
if (ipos >= 0 && count > 0) erase (ipos, count);
|
||||
return from;
|
||||
}
|
||||
|
||||
~gstring() noexcept {
|
||||
if (_buf != nullptr && needsFreeing()) {::free (_buf); _buf = nullptr;}
|
||||
}
|
||||
};
|
||||
|
||||
inline bool operator == (const gstring& gs1, const gstring& gs2) noexcept {return gs1.equals (gs2);}
|
||||
inline bool operator == (const char* cstr, const gstring& gstr) noexcept {return gstr.equals (cstr);}
|
||||
inline bool operator == (const gstring& gstr, const char* cstr) noexcept {return gstr.equals (cstr);}
|
||||
inline bool operator != (const gstring& gs1, const gstring& gs2) noexcept {return !gs1.equals (gs2);}
|
||||
inline bool operator != (const char* cstr, const gstring& gstr) noexcept {return !gstr.equals (cstr);}
|
||||
inline bool operator != (const gstring& gstr, const char* cstr) noexcept {return !gstr.equals (cstr);}
|
||||
|
||||
inline bool operator == (const gstring& gstr, const std::string& str) noexcept {
|
||||
return gstr.equals (gstring (gstring::ReferenceConstructor(), str.data(), str.size()));}
|
||||
inline bool operator != (const gstring& gstr, const std::string& str) noexcept {return !(gstr == str);}
|
||||
inline bool operator == (const std::string& str, const gstring& gstr) noexcept {return gstr == str;}
|
||||
inline bool operator != (const std::string& str, const gstring& gstr) noexcept {return !(gstr == str);}
|
||||
inline std::string operator += (std::string& str, const gstring& gstr) {return str.append (gstr.data(), gstr.size());}
|
||||
inline std::string operator + (const std::string& str, const gstring& gstr) {return std::string (str) .append (gstr.data(), gstr.size());}
|
||||
|
||||
inline std::ostream& operator << (std::ostream& os, const gstring& gstr) {
|
||||
if (gstr._buf != nullptr) os.write ((const char*) gstr._buf, gstr.length());
|
||||
return os;
|
||||
}
|
||||
|
||||
/// Encode this `gstring` into `stream` as a netstring.
|
||||
inline std::ostream& gstring::writeAsNetstring (std::ostream& stream) const {
|
||||
stream << length() << ':' << *this << ',';
|
||||
return stream;
|
||||
}
|
||||
|
||||
// http://www.mr-edd.co.uk/blog/beginners_guide_streambuf
|
||||
// http://www.dreamincode.net/code/snippet2499.htm
|
||||
// http://spec.winprog.org/streams/
|
||||
class gstring_stream: public std::basic_streambuf<char, std::char_traits<char> > {
|
||||
gstring& _gstr;
|
||||
public:
|
||||
gstring_stream (gstring& gstr) noexcept: _gstr (gstr) {
|
||||
char* buf = (char*) gstr._buf;
|
||||
if (buf != nullptr) setg (buf, buf, buf + gstr.length());
|
||||
}
|
||||
protected:
|
||||
virtual int_type overflow (int_type ch) {
|
||||
if (__builtin_expect (ch != traits_type::eof(), 1)) _gstr.append ((char) ch);
|
||||
return 0;
|
||||
}
|
||||
// no copying
|
||||
gstring_stream (const gstring_stream &);
|
||||
gstring_stream& operator = (const gstring_stream &);
|
||||
};
|
||||
|
||||
} // namespace glim
|
||||
|
||||
// hash specialization
|
||||
// cf. http://stackoverflow.com/questions/8157937/how-to-specialize-stdhashkeyoperator-for-user-defined-type-in-unordered
|
||||
namespace std {
|
||||
template <> struct hash<glim::gstring> {
|
||||
size_t operator()(const glim::gstring& gs) const noexcept {
|
||||
// cf. http://stackoverflow.com/questions/7666509/hash-function-for-string
|
||||
// Would be nice to use https://131002.net/siphash/ here.
|
||||
uint32_t hash = 5381;
|
||||
uint32_t len = gs.length();
|
||||
if (len) {
|
||||
const char* str = (const char*) gs._buf;
|
||||
const char* end = str + len;
|
||||
while (str < end) hash = ((hash << 5) + hash) + *str++; /* hash * 33 + c */
|
||||
}
|
||||
return hash;
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
#endif // _GSTRING_INCLUDED
|
||||
255
external/glim/hget.hpp
vendored
Normal file
255
external/glim/hget.hpp
vendored
Normal file
@ -0,0 +1,255 @@
|
||||
// Simple header-only wrapper around libevent's evhttp client.
|
||||
// See also: https://github.com/cpp-netlib/cpp-netlib/issues/160
|
||||
|
||||
#ifndef _GLIM_HGET_INCLUDED
|
||||
#define _GLIM_HGET_INCLUDED
|
||||
|
||||
#include <event2/event.h>
|
||||
#include <event2/dns.h>
|
||||
#include <evhttp.h> // http://stackoverflow.com/a/5237994; http://archives.seul.org/libevent/users/Sep-2010/msg00050.html
|
||||
|
||||
#include <memory>
|
||||
#include <functional>
|
||||
#include <stdexcept>
|
||||
#include <iostream>
|
||||
#include <vector>
|
||||
|
||||
#include <stdint.h>
|
||||
#include <string.h>
|
||||
#include <errno.h>
|
||||
|
||||
#include "exception.hpp"
|
||||
#include "gstring.hpp"
|
||||
|
||||
namespace glim {
|
||||
|
||||
/// HTTP results
|
||||
struct hgot {
|
||||
int32_t status = 0;
|
||||
/// Uses errno codes.
|
||||
int32_t error = 0;
|
||||
struct evbuffer* body = 0;
|
||||
struct evhttp_request* req = 0;
|
||||
size_t bodyLength() const {return body ? evbuffer_get_length (body) : 0;}
|
||||
/// Warning: the string is NOT zero-terminated.
|
||||
const char* bodyData() {return body ? (const char*) evbuffer_pullup (body, -1) : "";}
|
||||
/// Returns a zero-terminated string. Warning: modifies the `body` every time in order to add the terminator.
|
||||
const char* cbody() {if (!body) return ""; evbuffer_add (body, "", 1); return (const char*) evbuffer_pullup (body, -1);}
|
||||
/// A gstring *view* into the `body`.
|
||||
glim::gstring gbody() {
|
||||
if (!body) return glim::gstring();
|
||||
return glim::gstring (glim::gstring::ReferenceConstructor(), (const char*) evbuffer_pullup (body, -1), evbuffer_get_length (body));}
|
||||
};
|
||||
|
||||
/// Used internally to pass both connection and handler into callback.
|
||||
struct hgetContext {
|
||||
struct evhttp_connection* conn;
|
||||
std::function<void(hgot&)> handler;
|
||||
hgetContext (struct evhttp_connection* conn, std::function<void(hgot&)> handler): conn (conn), handler (handler) {}
|
||||
};
|
||||
|
||||
/// Invoked when evhttp finishes a request.
|
||||
inline void hgetCB (struct evhttp_request* req, void* ctx_){
|
||||
hgetContext* ctx = (hgetContext*) ctx_;
|
||||
|
||||
hgot gt;
|
||||
if (req == NULL) gt.error = ETIMEDOUT;
|
||||
else if (req->response_code == 0) gt.error = ECONNREFUSED;
|
||||
else {
|
||||
gt.status = req->response_code;
|
||||
gt.body = req->input_buffer;
|
||||
gt.req = req;
|
||||
}
|
||||
|
||||
try {
|
||||
ctx->handler (gt);
|
||||
} catch (const std::runtime_error& ex) { // Shouldn't normally happen:
|
||||
std::cerr << "glim::hget, handler exception: " << ex.what() << std::endl;
|
||||
}
|
||||
|
||||
evhttp_connection_free ((struct evhttp_connection*) ctx->conn);
|
||||
//freed by libevent//if (req != NULL) evhttp_request_free (req);
|
||||
delete ctx;
|
||||
}
|
||||
|
||||
/**
|
||||
C++ wrapper around libevent's http client.
|
||||
Example: \code
|
||||
hget (evbase, dnsbase) .setRequestBuilder ([](struct evhttp_request* req){
|
||||
evbuffer_add (req->output_buffer, "foo", 3);
|
||||
evhttp_add_header (req->output_headers, "Content-Length", "3");
|
||||
}) .go ("http://127.0.0.1:8080/test", [](hgot& got){
|
||||
if (got.error) log_warn ("127.0.0.1:8080 " << strerror (got.error));
|
||||
else if (got.status != 200) log_warn ("127.0.0.1:8080 != 200");
|
||||
else log_info ("got " << evbuffer_get_length (got.body) << " bytes from /test: " << evbuffer_pullup (got.body, -1));
|
||||
}); \endcode
|
||||
*/
|
||||
class hget {
|
||||
public:
|
||||
std::shared_ptr<struct event_base> _evbase;
|
||||
std::shared_ptr<struct evdns_base> _dnsbase;
|
||||
std::function<void(struct evhttp_request*)> _requestBuilder;
|
||||
enum evhttp_cmd_type _method;
|
||||
public:
|
||||
typedef std::shared_ptr<struct evhttp_uri> uri_t;
|
||||
/// The third parameter is the request number, starting from 1.
|
||||
typedef std::function<float(hgot&,uri_t,int32_t)> until_handler_t;
|
||||
public:
|
||||
hget (std::shared_ptr<struct event_base> evbase, std::shared_ptr<struct evdns_base> dnsbase):
|
||||
_evbase (evbase), _dnsbase (dnsbase), _method (EVHTTP_REQ_GET) {}
|
||||
|
||||
/// Modifies the request before its execution.
|
||||
hget& setRequestBuilder (std::function<void(struct evhttp_request*)> rb) {
|
||||
_requestBuilder = rb;
|
||||
return *this;
|
||||
}
|
||||
|
||||
/** Uses a simple request builder to send the `str`.
|
||||
* `str` is a `char` string class with methods `data` and `size`. */
|
||||
template<typename STR> hget& payload (STR str, const char* contentType = nullptr, enum evhttp_cmd_type method = EVHTTP_REQ_POST) {
|
||||
_method = method;
|
||||
return setRequestBuilder ([str,contentType](struct evhttp_request* req) {
|
||||
if (contentType) evhttp_add_header (req->output_headers, "Content-Type", contentType);
|
||||
char buf[64];
|
||||
*glim::itoa (buf, (int) str.size()) = 0;
|
||||
evhttp_add_header (req->output_headers, "Content-Length", buf);
|
||||
evbuffer_add (req->output_buffer, (const void*) str.data(), (size_t) str.size());
|
||||
});
|
||||
}
|
||||
|
||||
struct evhttp_request* go (uri_t uri, int32_t timeoutSec, std::function<void(hgot&)> handler) {
|
||||
int port = evhttp_uri_get_port (uri.get());
|
||||
if (port == -1) port = 80;
|
||||
struct evhttp_connection* conn = evhttp_connection_base_new (_evbase.get(), _dnsbase.get(),
|
||||
evhttp_uri_get_host (uri.get()), port);
|
||||
evhttp_connection_set_timeout (conn, timeoutSec);
|
||||
struct evhttp_request *req = evhttp_request_new (hgetCB, new hgetContext(conn, handler));
|
||||
int ret = evhttp_add_header (req->output_headers, "Host", evhttp_uri_get_host (uri.get()));
|
||||
if (ret) throw std::runtime_error ("hget: evhttp_add_header(Host) != 0");
|
||||
if (_requestBuilder) _requestBuilder (req);
|
||||
const char* get = evhttp_uri_get_path (uri.get());
|
||||
const char* qs = evhttp_uri_get_query (uri.get());
|
||||
if (qs == NULL) {
|
||||
ret = evhttp_make_request (conn, req, _method, get);
|
||||
} else {
|
||||
size_t getLen = strlen (get);
|
||||
size_t qsLen = strlen (qs);
|
||||
char buf[getLen + 1 + qsLen + 1];
|
||||
char* caret = stpcpy (buf, get);
|
||||
*caret++ = '?';
|
||||
caret = stpcpy (caret, qs);
|
||||
assert (caret - buf < sizeof (buf));
|
||||
ret = evhttp_make_request (conn, req, _method, buf);
|
||||
}
|
||||
if (ret) throw std::runtime_error ("hget: evhttp_make_request != 0");
|
||||
return req;
|
||||
}
|
||||
struct evhttp_request* go (const char* url, int32_t timeoutSec, std::function<void(hgot&)> handler) {
|
||||
return go (std::shared_ptr<struct evhttp_uri> (evhttp_uri_parse (url), evhttp_uri_free), timeoutSec, handler);
|
||||
}
|
||||
|
||||
void goUntil (std::vector<uri_t> urls, until_handler_t handler, int32_t timeoutSec = 20);
|
||||
/**
|
||||
Parse urls and call `goUntil`.
|
||||
Example (trying ten times to reach the servers): \code
|
||||
std::string path ("/path");
|
||||
hget.goUntilS (boost::assign::list_of ("http://server1" + path) ("http://server2" + path),
|
||||
[](hgot& got, hget::uri_t uri, int32_t num)->float {
|
||||
std::cout << "server: " << evhttp_uri_get_host (uri.get()) << "; request number: " << num << std::endl;
|
||||
if (got.status != 200 && num < 10) return 1.f; // Retry in a second.
|
||||
return -1.f; // No need to retry the request.
|
||||
});
|
||||
\endcode
|
||||
@param urls is a for-compatible container of strings (where string has methods `data` and `size`).
|
||||
*/
|
||||
template<typename URLS> void goUntilS (URLS&& urls, until_handler_t handler, int32_t timeoutSec = 20) {
|
||||
std::vector<uri_t> parsedUrls;
|
||||
for (auto&& url: urls) {
|
||||
// Copying to stack might be cheaper than malloc in c_str.
|
||||
int len = url.size(); char buf[len + 1]; memcpy (buf, url.data(), len); buf[len] = 0;
|
||||
struct evhttp_uri* uri = evhttp_uri_parse (buf);
|
||||
if (!uri) GTHROW (std::string ("!evhttp_uri_parse: ") + buf);
|
||||
parsedUrls.push_back (uri_t (uri, evhttp_uri_free));
|
||||
}
|
||||
goUntil (parsedUrls, handler, timeoutSec);
|
||||
}
|
||||
/**
|
||||
Parse urls and call `goUntil`.
|
||||
Example (trying ten times to reach the servers): \code
|
||||
hget.goUntilC (boost::assign::list_of ("http://server1/") ("http://server2/"),
|
||||
[](hgot& got, hget::uri_t uri, int32_t num)->float {
|
||||
std::cout << "server: " << evhttp_uri_get_host (uri.get()) << "; request number: " << num << std::endl;
|
||||
if (got.status != 200 && num < 10) return 1.f; // Retry in a second.
|
||||
return -1.f; // No need to retry the request.
|
||||
});
|
||||
\endcode
|
||||
Or with `std::array` instead of `boost::assign::list_of`: \code
|
||||
std::array<const char*, 2> urls {{"http://server1/", "http://server2/"}};
|
||||
hget.goUntilC (urls, [](hgot& got, hget::uri_t uri, int32_t num)->float {
|
||||
return got.status != 200 && num < 10 ? 0.f : -1.f;});
|
||||
\endcode
|
||||
@param urls is a for-compatible container of C strings (const char*).
|
||||
*/
|
||||
template<typename URLS> void goUntilC (URLS&& urls, until_handler_t handler, int32_t timeoutSec = 20) {
|
||||
std::vector<uri_t> parsedUrls;
|
||||
for (auto url: urls) {
|
||||
struct evhttp_uri* uri = evhttp_uri_parse (url);
|
||||
if (!uri) GTHROW (std::string ("Can't parse url: ") + url);
|
||||
parsedUrls.push_back (uri_t (uri, evhttp_uri_free));
|
||||
}
|
||||
goUntil (parsedUrls, handler, timeoutSec);
|
||||
}
|
||||
};
|
||||
|
||||
inline void hgetUntilRetryCB (evutil_socket_t, short, void* utilHandlerPtr); // event_callback_fn
|
||||
|
||||
/** `hget::goUntil` implementation.
|
||||
* This function object is passed to `hget::go` as a handler and calls `hget::go` again if necessary. */
|
||||
struct HgetUntilHandler {
|
||||
hget _hget;
|
||||
hget::until_handler_t _handler;
|
||||
std::vector<hget::uri_t> _urls;
|
||||
int32_t _timeoutSec;
|
||||
int32_t _requestNum;
|
||||
uint8_t _nextUrl; ///< A round-robin pointer to the next url in `_urls`.
|
||||
HgetUntilHandler (hget& hg, hget::until_handler_t handler, std::vector<hget::uri_t> urls, int32_t timeoutSec):
|
||||
_hget (hg), _handler (handler), _urls (urls), _timeoutSec (timeoutSec), _requestNum (0), _nextUrl (0) {}
|
||||
void operator() (hgot& got) {
|
||||
uint8_t urlNum = _nextUrl ? _nextUrl - 1 : _urls.size() - 1;
|
||||
float retryAfterSec = _handler (got, _urls[urlNum], _requestNum);
|
||||
if (retryAfterSec == 0.f) retry();
|
||||
else if (retryAfterSec > 0.f) {
|
||||
struct timeval wait;
|
||||
wait.tv_sec = (int) retryAfterSec;
|
||||
retryAfterSec -= wait.tv_sec;
|
||||
wait.tv_usec = (int) (retryAfterSec * 1000000.f);
|
||||
int rc = event_base_once (_hget._evbase.get(), -1, EV_TIMEOUT, hgetUntilRetryCB, new HgetUntilHandler (*this), &wait);
|
||||
if (rc) throw std::runtime_error ("HgetUntilHandler: event_base_once != 0");
|
||||
}
|
||||
}
|
||||
void start() {retry();}
|
||||
void retry() {
|
||||
uint8_t nextUrl = _nextUrl++;
|
||||
if (_nextUrl >= _urls.size()) _nextUrl = 0;
|
||||
++_requestNum;
|
||||
_hget.go (_urls[nextUrl], _timeoutSec, *this);
|
||||
}
|
||||
};
|
||||
|
||||
/// Used in `hget::goUntil` to wait in `evtimer_new` before repeating the request.
|
||||
inline void hgetUntilRetryCB (evutil_socket_t, short, void* utilHandlerPtr) { // event_callback_fn
|
||||
std::unique_ptr<HgetUntilHandler> untilHandler ((HgetUntilHandler*) utilHandlerPtr);
|
||||
untilHandler->retry();
|
||||
}
|
||||
|
||||
/**
|
||||
* Allows to retry the request using multiple URLs in a round-robin fashion.
|
||||
* The `handler` returns the number of seconds to wait before retrying the request or -1 if no retry is necessary.
|
||||
*/
|
||||
inline void hget::goUntil (std::vector<uri_t> urls, until_handler_t handler, int32_t timeoutSec) {
|
||||
HgetUntilHandler (*this, handler, urls, timeoutSec) .start();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
#endif // _GLIM_HGET_INCLUDED
|
||||
384
external/glim/ldb.hpp
vendored
Normal file
384
external/glim/ldb.hpp
vendored
Normal file
@ -0,0 +1,384 @@
|
||||
#ifndef _GLIM_LDB_HPP_INCLUDED
|
||||
#define _GLIM_LDB_HPP_INCLUDED
|
||||
|
||||
/**
|
||||
* Leveldb (http://code.google.com/p/leveldb/) wrapper.
|
||||
* @code
|
||||
Copyright 2012 Kozarezov Artem Aleksandrovich
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
* @endcode
|
||||
* @file
|
||||
*/
|
||||
|
||||
#include <string>
|
||||
//#include <unordered_map> // having SIGFPE as in http://stackoverflow.com/q/13580823/257568
|
||||
#include <map>
|
||||
#include <climits> // CHAR_MAX
|
||||
|
||||
#include <leveldb/db.h>
|
||||
#include <leveldb/write_batch.h>
|
||||
#include <leveldb/filter_policy.h>
|
||||
#include <boost/archive/binary_oarchive.hpp>
|
||||
#include <boost/archive/binary_iarchive.hpp>
|
||||
#include <boost/serialization/serialization.hpp>
|
||||
#include <boost/noncopyable.hpp>
|
||||
#include <boost/iterator/iterator_facade.hpp>
|
||||
#include <boost/range/iterator_range.hpp> // http://www.boost.org/doc/libs/1_52_0/libs/range/doc/html/range/reference/utilities/iterator_range.html
|
||||
|
||||
#include <arpa/inet.h> // htonl, ntohl
|
||||
#include <sys/stat.h> // mkdir
|
||||
#include <sys/types.h> // mkdir
|
||||
#include <string.h> // strerror
|
||||
#include <errno.h>
|
||||
#include <sstream>
|
||||
|
||||
#include "gstring.hpp"
|
||||
#include "exception.hpp"
|
||||
|
||||
namespace glim {
|
||||
|
||||
G_DEFINE_EXCEPTION (LdbEx);
|
||||
|
||||
template <typename T> inline void ldbSerialize (gstring& bytes, const T& data) {
|
||||
gstring_stream stream (bytes);
|
||||
boost::archive::binary_oarchive oa (stream, boost::archive::no_header);
|
||||
oa << data;
|
||||
}
|
||||
template <typename V> inline void ldbDeserialize (const gstring& bytes, V& data) {
|
||||
gstring_stream stream (const_cast<gstring&> (bytes));
|
||||
boost::archive::binary_iarchive ia (stream, boost::archive::no_header);
|
||||
ia >> data;
|
||||
}
|
||||
|
||||
/** uint32_t keys are stored big-endian (network byte order) in order to be compatible with lexicographic ordering. */
|
||||
template <> inline void ldbSerialize<uint32_t> (gstring& bytes, const uint32_t& ui) {
|
||||
uint32_t nui = htonl (ui); bytes.append ((const char*) &nui, sizeof (uint32_t));}
|
||||
/** Deserialize uint32_t from big-endian (network byte order). */
|
||||
template <> inline void ldbDeserialize<uint32_t> (const gstring& bytes, uint32_t& ui) {
|
||||
if (bytes.size() != sizeof (uint32_t)) GNTHROW (LdbEx, "Not uint32_t, wrong number of bytes");
|
||||
uint32_t nui = * (uint32_t*) bytes.data(); ui = ntohl (nui);}
|
||||
|
||||
/** If the data is `gstring` then use the data's buffer directly, no copy. */
|
||||
template <> inline void ldbSerialize<gstring> (gstring& bytes, const gstring& data) {
|
||||
bytes = gstring (0, (void*) data.data(), false, data.length());}
|
||||
/** Deserializing into `gstring` copies the bytes into it, reusing its buffer. */
|
||||
template <> inline void ldbDeserialize<gstring> (const gstring& bytes, gstring& data) {
|
||||
data.clear() << bytes;}
|
||||
|
||||
/** If the data is `std::string` then use the data's buffer directly, no copy. */
|
||||
template <> inline void ldbSerialize<std::string> (gstring& bytes, const std::string& data) {
|
||||
bytes = gstring (0, (void*) data.data(), false, data.length());}
|
||||
/** Deserializing into `std::string` copies the bytes into it, reusing its buffer. */
|
||||
template <> inline void ldbDeserialize<std::string> (const gstring& bytes, std::string& data) {
|
||||
data.clear(); data.append (bytes.data(), bytes.size());}
|
||||
|
||||
/**
|
||||
* Header-only Leveldb wrapper.\n
|
||||
* Uses Boost Serialization to pack keys and values (glim::gstring can be used for raw bytes).\n
|
||||
* Allows semi-automatic indexing with triggers.
|
||||
*/
|
||||
struct Ldb {
|
||||
std::shared_ptr<leveldb::DB> _db;
|
||||
std::shared_ptr<const leveldb::FilterPolicy> _filter;
|
||||
|
||||
struct IteratorEntry { ///< Something to be `dereference`d from the Iterator. Also a pImpl allowing to keep the `_valid` and the `_lit` in sync.
|
||||
leveldb::Iterator* _lit;
|
||||
bool _valid:1;
|
||||
|
||||
IteratorEntry (const IteratorEntry&) = delete; // Owns `leveldb::Iterator`, should not be copied.
|
||||
IteratorEntry (IteratorEntry&&) = default;
|
||||
|
||||
IteratorEntry (leveldb::Iterator* lit, bool valid = false): _lit (lit), _valid (valid) {}
|
||||
~IteratorEntry() {delete _lit;}
|
||||
|
||||
/** Zero-copy view of the current key bytes. Should *not* be used after the Iterator is changed or destroyed.
|
||||
* @param ref If true then the *copies* of the returned gstring will keep pointing to LevelDB memory which is valid only until the iterator changes. */
|
||||
const gstring keyView (bool ref = false) const {
|
||||
if (!_valid) return gstring();
|
||||
const leveldb::Slice& key = _lit->key();
|
||||
return gstring (0, (void*) key.data(), false, key.size(), ref);} // Zero copy.
|
||||
/** Zero-copy view of the current value bytes. Should *not* be used after the Iterator is changed or destroyed.
|
||||
* @param ref If true then the *copies* of the returned gstring will keep pointing to LevelDB memory which is valid only until the iterator changes. */
|
||||
const gstring valueView (bool ref = false) const {
|
||||
if (!_valid) return gstring();
|
||||
const leveldb::Slice& val = _lit->value();
|
||||
return gstring (0, (void*) val.data(), false, val.size(), ref);} // Zero copy.
|
||||
/** Deserialize into `key`. */
|
||||
template <typename T> void getKey (T& key) const {ldbDeserialize (keyView (true), key);}
|
||||
/** Deserialize the key into a temporary and return it. */
|
||||
template <typename T> T getKey() const {T key; getKey (key); return key;}
|
||||
/** Deserialize into `value`. */
|
||||
template <typename T> void getValue (T& value) const {ldbDeserialize (valueView (true), value);}
|
||||
/** Deserialize the value into a temporary and return it. */
|
||||
template <typename T> T getValue() const {T value; getValue (value); return value;}
|
||||
};
|
||||
struct NoSeekFlag {}; ///< Tells the `Iterator` constructor not to seek to the beginning of the database.
|
||||
/** Wraps Leveldb iterator.
|
||||
* Note: "In fact the iterator is a light-weight snapshot. It will see exactly the version of the DB that existed when the iterator was created
|
||||
* (i.e., any insert/delete done after the iterator is created, regardless of which thread does them) will be invisible to the iterator"
|
||||
* (https://groups.google.com/d/msg/leveldb/nX8S5KKiSn4/PI92Yf1Hf6UJ). */
|
||||
struct Iterator: public boost::iterator_facade<Iterator, IteratorEntry, boost::bidirectional_traversal_tag> {
|
||||
std::shared_ptr<IteratorEntry> _entry; ///< The Iterator might be copied around, therefore we keep the real iterator and the state in the shared_ptr.
|
||||
|
||||
Iterator (const Iterator&) = default;
|
||||
Iterator (Iterator&&) = default;
|
||||
Iterator& operator= (const Iterator&) = default;
|
||||
Iterator& operator= (Iterator&&) = default;
|
||||
/** Iterate from the beginning or the end of the database.
|
||||
* @param position can be MDB_FIRST or MDB_LAST */
|
||||
Iterator (Ldb* ldb, leveldb::ReadOptions options = leveldb::ReadOptions()):
|
||||
_entry (std::make_shared<IteratorEntry> (ldb->_db->NewIterator (options))) {
|
||||
IteratorEntry* entry = _entry.get();
|
||||
entry->_lit->SeekToFirst();
|
||||
entry->_valid = entry->_lit->Valid();
|
||||
}
|
||||
|
||||
Iterator (Ldb* ldb, NoSeekFlag, leveldb::ReadOptions options = leveldb::ReadOptions()):
|
||||
_entry (std::make_shared<IteratorEntry> (ldb->_db->NewIterator (options))) {}
|
||||
/** True if the iterator isn't pointing anywhere. */
|
||||
bool end() const {return !_entry->_valid;}
|
||||
|
||||
bool equal (const Iterator& other) const {
|
||||
bool weAreValid = _entry->_valid, theyAreValid = other._entry->_valid;
|
||||
if (!weAreValid) return !theyAreValid;
|
||||
if (!theyAreValid) return false;
|
||||
auto&& ourKey = _entry->_lit->key(), theirKey = other._entry->_lit->key();
|
||||
if (ourKey.size() != theirKey.size()) return false;
|
||||
return memcmp (ourKey.data(), theirKey.data(), ourKey.size()) == 0;
|
||||
}
|
||||
IteratorEntry& dereference() const {
|
||||
// NB: Boost iterator_facade expects the `dereference` to be a `const` method.
|
||||
// I guess Iterator is not modified, so the `dereference` is `const`, even though the Entry can be modified.
|
||||
return *_entry;
|
||||
}
|
||||
virtual void increment() {
|
||||
IteratorEntry* entry = _entry.get();
|
||||
if (entry->_valid) entry->_lit->Next();
|
||||
else entry->_lit->SeekToFirst();
|
||||
entry->_valid = entry->_lit->Valid();
|
||||
}
|
||||
virtual void decrement() {
|
||||
IteratorEntry* entry = _entry.get();
|
||||
if (entry->_valid) entry->_lit->Prev();
|
||||
else entry->_lit->SeekToLast();
|
||||
entry->_valid = entry->_lit->Valid();
|
||||
}
|
||||
|
||||
Iterator& seek (const gstring& key) {
|
||||
IteratorEntry* entry = _entry.get();
|
||||
entry->_lit->Seek (leveldb::Slice (key.data(), key.size()));
|
||||
entry->_valid = entry->_lit->Valid();
|
||||
return *this;
|
||||
}
|
||||
};
|
||||
Iterator begin() {return Iterator (this);}
|
||||
Iterator end() {return Iterator (this, NoSeekFlag());}
|
||||
|
||||
/** Range from `from` (inclusive) to `till` (exclusive). */
|
||||
template <typename K>
|
||||
boost::iterator_range<Iterator> range (const K& from, const K& till, leveldb::ReadOptions options = leveldb::ReadOptions()) {
|
||||
char kbuf[64]; // Allow up to 64 bytes to be serialized without heap allocations.
|
||||
gstring kbytes (sizeof (kbuf), kbuf, false, 0);
|
||||
ldbSerialize (kbytes, from);
|
||||
Iterator fit (this, NoSeekFlag(), options); fit.seek (kbytes);
|
||||
|
||||
ldbSerialize (kbytes.clear(), till);
|
||||
Iterator tit (this, NoSeekFlag(), options); tit.seek (kbytes);
|
||||
|
||||
return boost::iterator_range<Iterator> (fit, tit);
|
||||
}
|
||||
|
||||
struct StartsWithIterator: public Iterator {
|
||||
gstring _starts;
|
||||
StartsWithIterator (const StartsWithIterator&) = default;
|
||||
StartsWithIterator (StartsWithIterator&&) = default;
|
||||
StartsWithIterator& operator= (const StartsWithIterator&) = default;
|
||||
StartsWithIterator& operator= (StartsWithIterator&&) = default;
|
||||
|
||||
StartsWithIterator (Ldb* ldb, const char* data, uint32_t length, leveldb::ReadOptions options = leveldb::ReadOptions()):
|
||||
Iterator (ldb, NoSeekFlag(), options), _starts (data, length) {
|
||||
IteratorEntry* entry = _entry.get();
|
||||
entry->_lit->Seek (leveldb::Slice (data, length));
|
||||
entry->_valid = checkValidity();
|
||||
}
|
||||
/** End iterator, pointing nowhere. */
|
||||
StartsWithIterator (Ldb* ldb, const char* data, uint32_t length, NoSeekFlag, leveldb::ReadOptions options = leveldb::ReadOptions()):
|
||||
Iterator (ldb, NoSeekFlag(), options), _starts (data, length) {}
|
||||
|
||||
bool checkValidity() const {
|
||||
IteratorEntry* entry = _entry.get();
|
||||
return entry->_lit->Valid() && entry->_lit->key().starts_with (leveldb::Slice (_starts.data(), _starts.length()));
|
||||
}
|
||||
virtual void increment() override {
|
||||
IteratorEntry* entry = _entry.get();
|
||||
if (entry->_valid) entry->_lit->Next();
|
||||
else entry->_lit->Seek (leveldb::Slice (_starts.data(), _starts.length()));
|
||||
entry->_valid = checkValidity();
|
||||
}
|
||||
virtual void decrement() override {
|
||||
IteratorEntry* entry = _entry.get();
|
||||
if (entry->_valid) entry->_lit->Prev(); else seekLast();
|
||||
entry->_valid = checkValidity();
|
||||
}
|
||||
void seekLast() {
|
||||
leveldb::Iterator* lit = _entry->_lit;
|
||||
// Go somewhere *below* the `_starts` prefix.
|
||||
char after[_starts.length()]; if (sizeof (after)) {
|
||||
memcpy (after, _starts.data(), sizeof (after));
|
||||
uint32_t pos = sizeof (after); while (--pos >= 0) if (after[pos] < CHAR_MAX) {++after[pos]; break;}
|
||||
if (pos >= 0) {lit->Seek (leveldb::Slice (after, sizeof (after))); if (!lit->Valid()) lit->SeekToLast();} else lit->SeekToLast();
|
||||
} else lit->SeekToLast();
|
||||
// Seek back until we are in the `_starts` prefix.
|
||||
for (leveldb::Slice prefix (_starts.data(), _starts.length()); lit->Valid(); lit->Prev()) {
|
||||
leveldb::Slice key (lit->key());
|
||||
if (key.starts_with (prefix)) break; // We're "back" in the `_starts` prefix.
|
||||
if (key.compare (prefix) < 0) break; // Gone too far (no prefix entries in the db).
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
/** Range over entries starting with `key`. */
|
||||
template <typename K>
|
||||
boost::iterator_range<StartsWithIterator> startsWith (const K& key) {
|
||||
char kbuf[64]; // Allow up to 64 bytes to be serialized without heap allocations.
|
||||
gstring kbytes (sizeof (kbuf), kbuf, false, 0);
|
||||
ldbSerialize (kbytes, key);
|
||||
return boost::iterator_range<StartsWithIterator> (
|
||||
StartsWithIterator (this, kbytes.data(), kbytes.length()),
|
||||
StartsWithIterator (this, kbytes.data(), kbytes.length(), NoSeekFlag()));
|
||||
}
|
||||
|
||||
struct Trigger {
|
||||
virtual gstring triggerName() const {return C2GSTRING ("defaultTriggerName");};
|
||||
virtual void put (Ldb& ldb, void* key, gstring& kbytes, void* value, gstring& vbytes, leveldb::WriteBatch& batch) = 0;
|
||||
virtual void del (Ldb& ldb, void* key, gstring& kbytes, leveldb::WriteBatch& batch) = 0;
|
||||
};
|
||||
std::map<gstring, std::shared_ptr<Trigger>> _triggers;
|
||||
|
||||
/** Register the trigger (by its `triggerName`). */
|
||||
void putTrigger (std::shared_ptr<Trigger> trigger) {
|
||||
_triggers[trigger->triggerName()] = trigger;
|
||||
}
|
||||
|
||||
public:
|
||||
|
||||
Ldb() {}
|
||||
|
||||
/** Opens Leveldb database. */
|
||||
Ldb (const char* path, leveldb::Options* options = nullptr, mode_t mode = 0770) {
|
||||
int rc = ::mkdir (path, mode);
|
||||
if (rc && errno != EEXIST) GNTHROW (LdbEx, std::string ("Can't create ") + path + ": " + ::strerror (errno));
|
||||
leveldb::DB* db;
|
||||
leveldb::Status status;
|
||||
if (options) {
|
||||
status = leveldb::DB::Open (*options, path, &db);
|
||||
} else {
|
||||
leveldb::Options localOptions;
|
||||
localOptions.create_if_missing = true;
|
||||
_filter.reset (leveldb::NewBloomFilterPolicy (8));
|
||||
localOptions.filter_policy = _filter.get();
|
||||
status = leveldb::DB::Open (localOptions, path, &db);
|
||||
}
|
||||
if (!status.ok()) GNTHROW (LdbEx, std::string ("Ldb: Can't open ") + path + ": " + status.ToString());
|
||||
_db.reset (db);
|
||||
}
|
||||
|
||||
/** Wraps an existing Leveldb handler. */
|
||||
Ldb (std::shared_ptr<leveldb::DB> db): _db (db) {}
|
||||
|
||||
template <typename K, typename V> void put (const K& key, const V& value, leveldb::WriteBatch& batch) {
|
||||
char kbuf[64]; // Allow up to 64 bytes to be serialized without heap allocations.
|
||||
gstring kbytes (sizeof (kbuf), kbuf, false, 0);
|
||||
ldbSerialize (kbytes, key);
|
||||
|
||||
char vbuf[64]; // Allow up to 64 bytes to be serialized without heap allocations.
|
||||
gstring vbytes (sizeof (vbuf), vbuf, false, 0);
|
||||
ldbSerialize (vbytes, value);
|
||||
|
||||
for (auto& trigger: _triggers) trigger.second->put (*this, (void*) &key, kbytes, (void*) &value, vbytes, batch);
|
||||
|
||||
batch.Put (leveldb::Slice (kbytes.data(), kbytes.size()), leveldb::Slice (vbytes.data(), vbytes.size()));
|
||||
}
|
||||
template <typename K, typename V> void put (const K& key, const V& value) {
|
||||
leveldb::WriteBatch batch;
|
||||
put (key, value, batch);
|
||||
leveldb::Status status (_db->Write (leveldb::WriteOptions(), &batch));
|
||||
if (!status.ok()) GNTHROW (LdbEx, "Ldb: add: " + status.ToString());
|
||||
}
|
||||
|
||||
/** Returns `true` if the key exists. Throws on error. */
|
||||
template <typename K> bool have (const K& key, leveldb::ReadOptions options = leveldb::ReadOptions()) {
|
||||
char kbuf[64]; // Allow up to 64 bytes to be serialized without heap allocations.
|
||||
gstring kbytes (sizeof (kbuf), kbuf, false, 0);
|
||||
ldbSerialize (kbytes, key);
|
||||
leveldb::Slice keySlice (kbytes.data(), kbytes.size());
|
||||
|
||||
// NB: "BloomFilter only helps for Get() calls" - https://groups.google.com/d/msg/leveldb/oEiDztqHiHc/LMY3tHxzRGAJ
|
||||
// "Apart from the lack of Bloom filter functionality, creating an iterator is really quite slow" - qpu2jSA8mCEJ
|
||||
std::string str;
|
||||
leveldb::Status status (_db->Get (options, keySlice, &str));
|
||||
if (status.ok()) return true;
|
||||
else if (status.IsNotFound()) return false;
|
||||
else GTHROW ("Ldb.have: " + status.ToString());
|
||||
}
|
||||
|
||||
/** Returns `true` and modifies `value` if `key` is found. */
|
||||
template <typename K, typename V> bool get (const K& key, V& value, leveldb::ReadOptions options = leveldb::ReadOptions()) {
|
||||
char kbuf[64]; // Allow up to 64 bytes to be serialized without heap allocations.
|
||||
gstring kbytes (sizeof (kbuf), kbuf, false, 0);
|
||||
ldbSerialize (kbytes, key);
|
||||
leveldb::Slice keySlice (kbytes.data(), kbytes.size());
|
||||
|
||||
// NB: "BloomFilter only helps for Get() calls" - https://groups.google.com/d/msg/leveldb/oEiDztqHiHc/LMY3tHxzRGAJ
|
||||
// "Apart from the lack of Bloom filter functionality, creating an iterator is really quite slow" - qpu2jSA8mCEJ
|
||||
std::string str;
|
||||
leveldb::Status status (_db->Get (options, keySlice, &str));
|
||||
if (status.ok()) {
|
||||
ldbDeserialize (gstring (0, (void*) str.data(), false, str.size()), value);
|
||||
return true;
|
||||
} else if (status.IsNotFound()) return false;
|
||||
else GTHROW ("Ldb.get: " + status.ToString());
|
||||
}
|
||||
|
||||
template <typename K> void del (const K& key, leveldb::WriteBatch& batch) {
|
||||
char kbuf[64]; // Allow up to 64 bytes to be serialized without heap allocations.
|
||||
gstring kbytes (sizeof (kbuf), kbuf, false, 0);
|
||||
ldbSerialize (kbytes, key);
|
||||
if (kbytes.empty()) GNTHROW (LdbEx, "del: key is empty");
|
||||
|
||||
for (auto& trigger: _triggers) trigger.second->del (*this, (void*) &key, kbytes, batch);
|
||||
|
||||
batch.Delete (leveldb::Slice (kbytes.data(), kbytes.size()));
|
||||
}
|
||||
template <typename K> void del (const K& key) {
|
||||
leveldb::WriteBatch batch;
|
||||
del (key, batch);
|
||||
leveldb::Status status (_db->Write (leveldb::WriteOptions(), &batch));
|
||||
if (!status.ok()) GNTHROW (LdbEx, "Ldb: del: " + status.ToString());
|
||||
}
|
||||
|
||||
/** Writes the batch. Throws LdbEx if not successfull. */
|
||||
void write (leveldb::WriteBatch& batch, leveldb::WriteOptions options = leveldb::WriteOptions()) {
|
||||
leveldb::Status status (_db->Write (options, &batch));
|
||||
if (!status.ok()) GNTHROW (LdbEx, status.ToString());
|
||||
}
|
||||
|
||||
virtual ~Ldb() {
|
||||
_triggers.clear(); // Destroy triggers before closing the database.
|
||||
}
|
||||
};
|
||||
|
||||
} // namespace glim
|
||||
|
||||
#endif // _GLIM_LDB_HPP_INCLUDED
|
||||
93
external/glim/makefile
vendored
Normal file
93
external/glim/makefile
vendored
Normal file
@ -0,0 +1,93 @@
|
||||
|
||||
PREFIX = /usr/local
|
||||
INSTALL2 = ${PREFIX}/include/glim
|
||||
CXXFLAGS = -std=c++1y -Wall -O2 -ggdb -DBOOST_ALL_DYN_LINK
|
||||
|
||||
all: test
|
||||
|
||||
help:
|
||||
@echo "make test\nmake install\nmake uninstall\nmake clean"
|
||||
|
||||
doc: doxyconf *.hpp
|
||||
mkdir -p doc
|
||||
doxygen doxyconf
|
||||
|
||||
test: test_sqlite test_gstring test_runner test_exception test_ldb
|
||||
|
||||
test_sqlite: bin/test_sqlite
|
||||
cp bin/test_sqlite /tmp/libglim_test_sqlite && chmod +x /tmp/libglim_test_sqlite && /tmp/libglim_test_sqlite && rm -f /tmp/libglim_test_sqlite
|
||||
|
||||
bin/test_sqlite: test_sqlite.cc
|
||||
mkdir -p bin
|
||||
g++ $(CXXFLAGS) test_sqlite.cc -o bin/test_sqlite -lsqlite3
|
||||
|
||||
test_memcache: bin/test_memcache
|
||||
cp bin/test_memcache /tmp/libglim_test_memcache && chmod +x /tmp/libglim_test_memcache && /tmp/libglim_test_memcache && rm -f /tmp/libglim_test_memcache
|
||||
|
||||
bin/test_memcache: test_memcache.cc memcache.hpp
|
||||
mkdir -p bin
|
||||
g++ $(CXXFLAGS) test_memcache.cc -o bin/test_memcache -lmemcache
|
||||
|
||||
bin/test_gstring: test_gstring.cc gstring.hpp
|
||||
mkdir -p bin
|
||||
g++ $(CXXFLAGS) test_gstring.cc -o bin/test_gstring
|
||||
|
||||
test_gstring: bin/test_gstring
|
||||
cp bin/test_gstring /tmp/libglim_test_gstring
|
||||
chmod +x /tmp/libglim_test_gstring
|
||||
/tmp/libglim_test_gstring
|
||||
rm -f /tmp/libglim_test_gstring
|
||||
|
||||
bin/test_runner: test_runner.cc runner.hpp curl.hpp
|
||||
mkdir -p bin
|
||||
g++ $(CXXFLAGS) test_runner.cc -o bin/test_runner -pthread -lboost_log -levent -levent_pthreads -lcurl
|
||||
|
||||
test_runner: bin/test_runner
|
||||
valgrind -q bin/test_runner
|
||||
|
||||
bin/test_exception: test_exception.cc exception.hpp
|
||||
mkdir -p bin
|
||||
g++ $(CXXFLAGS) test_exception.cc -o bin/test_exception -ldl -rdynamic
|
||||
|
||||
test_exception: bin/test_exception
|
||||
valgrind -q bin/test_exception
|
||||
|
||||
test_ldb: test_ldb.cc ldb.hpp
|
||||
mkdir -p bin
|
||||
g++ $(CXXFLAGS) test_ldb.cc -o bin/test_ldb \
|
||||
-lleveldb -lboost_serialization -lboost_filesystem -lboost_system
|
||||
valgrind -q bin/test_ldb
|
||||
|
||||
bin/test_cbcoro: test_cbcoro.cc
|
||||
mkdir -p bin
|
||||
g++ $(CXXFLAGS) test_cbcoro.cc -o bin/test_cbcoro -pthread
|
||||
|
||||
test_cbcoro: bin/test_cbcoro
|
||||
bin/test_cbcoro
|
||||
|
||||
install:
|
||||
mkdir -p ${INSTALL2}/
|
||||
cp sqlite.hpp ${INSTALL2}/
|
||||
cp NsecTimer.hpp ${INSTALL2}/
|
||||
cp TscTimer.hpp ${INSTALL2}/
|
||||
cp memcache.hpp ${INSTALL2}/
|
||||
cp gstring.hpp ${INSTALL2}/
|
||||
cp runner.hpp ${INSTALL2}/
|
||||
cp hget.hpp ${INSTALL2}/
|
||||
cp curl.hpp ${INSTALL2}/
|
||||
cp mdb.hpp ${INSTALL2}/
|
||||
cp ldb.hpp ${INSTALL2}/
|
||||
cp exception.hpp ${INSTALL2}/
|
||||
cp SerializablePool.hpp ${INSTALL2}/
|
||||
cp cbcoro.hpp ${INSTALL2}/
|
||||
cp raii.hpp ${INSTALL2}/
|
||||
cp channel.hpp ${INSTALL2}/
|
||||
|
||||
uninstall:
|
||||
rm -rf ${INSTALL2}
|
||||
|
||||
clean:
|
||||
rm -rf bin/*
|
||||
rm -rf doc
|
||||
rm -f /tmp/libglim_test_*
|
||||
rm -f *.exe.stackdump
|
||||
499
external/glim/mdb.hpp
vendored
Normal file
499
external/glim/mdb.hpp
vendored
Normal file
@ -0,0 +1,499 @@
|
||||
#ifndef _GLIM_MDB_HPP_INCLUDED
|
||||
#define _GLIM_MDB_HPP_INCLUDED
|
||||
|
||||
/**
|
||||
* A C++ wrapper around MDB (http://www.symas.com/mdb/).
|
||||
* @code
|
||||
Copyright 2012 Kozarezov Artem Aleksandrovich
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
* @endcode
|
||||
* @file
|
||||
*/
|
||||
|
||||
#include <mdb.h>
|
||||
#include <boost/archive/binary_oarchive.hpp>
|
||||
#include <boost/archive/binary_iarchive.hpp>
|
||||
#include <boost/serialization/serialization.hpp>
|
||||
#include <boost/noncopyable.hpp>
|
||||
#include <boost/iterator/iterator_facade.hpp>
|
||||
#include <boost/range/iterator_range.hpp>
|
||||
|
||||
#include <arpa/inet.h> // htonl, ntohl
|
||||
|
||||
#include "gstring.hpp"
|
||||
|
||||
namespace glim {
|
||||
|
||||
struct MdbEx: public std::runtime_error {MdbEx (std::string message): std::runtime_error (message) {}};
|
||||
|
||||
template <typename T> inline void mdbSerialize (gstring& bytes, const T& data) {
|
||||
gstring_stream stream (bytes);
|
||||
boost::archive::binary_oarchive oa (stream, boost::archive::no_header);
|
||||
oa << data;
|
||||
}
|
||||
template <typename V> inline void mdbDeserialize (const gstring& bytes, V& data) {
|
||||
gstring_stream stream (const_cast<gstring&> (bytes));
|
||||
boost::archive::binary_iarchive ia (stream, boost::archive::no_header);
|
||||
ia >> data;
|
||||
}
|
||||
|
||||
/** uint32_t keys are stored big-endian (network byte order) in order to be compatible with lexicographic ordering. */
|
||||
template <> inline void mdbSerialize<uint32_t> (gstring& bytes, const uint32_t& ui) {
|
||||
uint32_t nui = htonl (ui); bytes.append ((const char*) &nui, sizeof (uint32_t));}
|
||||
/** Deserialize uint32_t from big-endian (network byte order). */
|
||||
template <> inline void mdbDeserialize<uint32_t> (const gstring& bytes, uint32_t& ui) {
|
||||
if (bytes.size() != sizeof (uint32_t)) throw MdbEx ("Not uint32_t, wrong number of bytes");
|
||||
uint32_t nui = * (uint32_t*) bytes.data(); ui = ntohl (nui);}
|
||||
|
||||
/** If the data is `gstring` then use the data's buffer directly, no copy. */
|
||||
template <> inline void mdbSerialize<gstring> (gstring& bytes, const gstring& data) {
|
||||
bytes = gstring (0, (void*) data.data(), false, data.length());}
|
||||
/** Deserializing into `gstring` copies the bytes into it, reusing its buffer. */
|
||||
template <> inline void mdbDeserialize<gstring> (const gstring& bytes, gstring& data) {
|
||||
data.clear() << bytes;}
|
||||
|
||||
/**
|
||||
* Header-only C++ wrapper around OpenLDAP-MDB.\n
|
||||
* Uses Boost Serialization to pack keys and values (glim::gstring can be used for raw bytes).\n
|
||||
* Allows semi-automatic indexing with triggers.\n
|
||||
* Known issues: http://www.openldap.org/its/index.cgi?findid=7448
|
||||
*/
|
||||
struct Mdb {
|
||||
std::shared_ptr<MDB_env> _env;
|
||||
MDB_dbi _dbi = 0;
|
||||
|
||||
typedef std::unique_ptr<MDB_txn, void(*)(MDB_txn*)> Transaction;
|
||||
|
||||
/** Holds the current key and value of the Iterator. */
|
||||
struct IteratorEntry {
|
||||
MDB_val _key = {0, 0}, _val = {0, 0};
|
||||
/** Zero-copy view of the current key bytes. Should *not* be used after the Iterator is changed or destroyed. */
|
||||
const gstring keyView() const {return gstring (0, _key.mv_data, false, _key.mv_size, true);} // Zero copy.
|
||||
/** Zero-copy view of the current value bytes. Should *not* be used after the Iterator is changed or destroyed. */
|
||||
const gstring valueView() const {return gstring (0, _val.mv_data, false, _val.mv_size, true);} // Zero copy.
|
||||
/** Deserialize into `key`. */
|
||||
template <typename T> void getKey (T& key) const {mdbDeserialize (keyView(), key);}
|
||||
/** Deserialize the key into a temporary and return it. */
|
||||
template <typename T> T getKey() const {T key; getKey (key); return key;}
|
||||
/** Deserialize into `value`. */
|
||||
template <typename T> void getValue (T& value) const {mdbDeserialize (valueView(), value);}
|
||||
/** Deserialize the value into a temporary and return it. */
|
||||
template <typename T> T getValue() const {T value; getValue (value); return value;}
|
||||
};
|
||||
/** Holds the Iterator's unique transaction and cursor, allowing the Iterator to be copied. */
|
||||
struct IteratorImpl: boost::noncopyable {
|
||||
Mdb* _mdb;
|
||||
Transaction _txn;
|
||||
MDB_cursor* _cur;
|
||||
IteratorImpl (Mdb* mdb, Transaction&& txn, MDB_cursor* cur): _mdb (mdb), _txn (std::move (txn)), _cur (cur) {}
|
||||
~IteratorImpl() {
|
||||
if (_cur) {::mdb_cursor_close (_cur); _cur = nullptr;}
|
||||
if (_mdb && _txn) {_mdb->commitTransaction (_txn); _mdb = nullptr;}
|
||||
}
|
||||
};
|
||||
/** Wraps MDB cursor and cursor's transaction. */
|
||||
struct Iterator: public boost::iterator_facade<Iterator, IteratorEntry, boost::bidirectional_traversal_tag> {
|
||||
std::shared_ptr<IteratorImpl> _impl; // Iterator might be copied around, thus we keep the unique things in IteratorImpl.
|
||||
IteratorEntry _entry;
|
||||
bool _stayInKey = false;
|
||||
|
||||
Iterator (const Iterator&) = default;
|
||||
Iterator (Iterator&&) = default;
|
||||
/** Iterate from the beginning or the end of the database.
|
||||
* @param position can be MDB_FIRST or MDB_LAST */
|
||||
Iterator (Mdb* mdb, int position = 0): _stayInKey (false) {
|
||||
Transaction txn (mdb->beginTransaction());
|
||||
MDB_cursor* cur = nullptr; int rc = ::mdb_cursor_open (txn.get(), mdb->_dbi, &cur);
|
||||
if (rc) throw MdbEx ("mdb_cursor_open");
|
||||
_impl = std::make_shared<IteratorImpl> (mdb, std::move (txn), cur);
|
||||
if (position == ::MDB_FIRST || position == ::MDB_LAST) {
|
||||
rc = ::mdb_cursor_get (cur, &_entry._key, &_entry._val, (MDB_cursor_op) position);
|
||||
if (rc) throw MdbEx ("mdb_cursor_get");
|
||||
}
|
||||
}
|
||||
/** Iterate over `key` values.
|
||||
* @param stayInKey if `false` then iterator can go farther than the `key`. */
|
||||
Iterator (Mdb* mdb, const gstring& key, bool stayInKey = true): _stayInKey (stayInKey) {
|
||||
Transaction txn (mdb->beginTransaction());
|
||||
MDB_cursor* cur = nullptr; int rc = ::mdb_cursor_open (txn.get(), mdb->_dbi, &cur);
|
||||
if (rc) throw MdbEx ("mdb_cursor_open");
|
||||
_impl = std::make_shared<IteratorImpl> (mdb, std::move (txn), cur);
|
||||
_entry._key = {key.size(), (void*) key.data()};
|
||||
rc = ::mdb_cursor_get (cur, &_entry._key, &_entry._val, ::MDB_SET_KEY);
|
||||
if (rc == MDB_NOTFOUND) {_entry._key = {0, 0}; _entry._val = {0, 0};}
|
||||
else if (rc) throw MdbEx ("mdb_cursor_get");
|
||||
}
|
||||
|
||||
struct EndIteratorFlag {};
|
||||
/** The "end" iterator does not open an MDB transaction (this is essential for having a pair of iterators without a deadlock). */
|
||||
Iterator (EndIteratorFlag): _stayInKey (false) {}
|
||||
/** True if the iterator isn't pointing anywhere. */
|
||||
bool end() const {return _entry._key.mv_size == 0;}
|
||||
|
||||
bool equal (const Iterator& other) const {
|
||||
IteratorImpl* impl = _impl.get();
|
||||
if (mdb_cmp (impl->_txn.get(), impl->_mdb->_dbi, &_entry._key, &other._entry._key)) return false;
|
||||
if (mdb_dcmp (impl->_txn.get(), impl->_mdb->_dbi, &_entry._val, &other._entry._val)) return false;
|
||||
return true;
|
||||
}
|
||||
IteratorEntry& dereference() const {
|
||||
// NB: Boost iterator_facade expects the `dereference` to be a `const` method.
|
||||
// I guess Iterator is not modified, so the `dereference` is `const`, even though the Entry can be modified.
|
||||
return const_cast<IteratorEntry&> (_entry);}
|
||||
void increment() {
|
||||
int rc = ::mdb_cursor_get (_impl->_cur, &_entry._key, &_entry._val, _stayInKey ? ::MDB_NEXT_DUP : ::MDB_NEXT);
|
||||
if (rc) {_entry._key = {0,0}; _entry._val = {0,0};}
|
||||
}
|
||||
void decrement() {
|
||||
int rc = ::mdb_cursor_get (_impl->_cur, &_entry._key, &_entry._val, _stayInKey ? ::MDB_PREV_DUP : ::MDB_PREV);
|
||||
if (rc) {_entry._key = {0,0}; _entry._val = {0,0};}
|
||||
}
|
||||
};
|
||||
Iterator begin() {return Iterator (this, ::MDB_FIRST);}
|
||||
const Iterator end() {return Iterator (Iterator::EndIteratorFlag());}
|
||||
/** Position the cursor at the first `key` record.\n
|
||||
* The iterator increment will use `MDB_NEXT_DUP`, staying withing the `key`.\n
|
||||
* See also the `all` method. */
|
||||
template <typename K> Iterator values (const K& key) {
|
||||
char kbuf[64]; // Allow up to 64 bytes to be serialized without heap allocations.
|
||||
gstring kbytes (sizeof (kbuf), kbuf, false, 0);
|
||||
mdbSerialize (kbytes, key);
|
||||
return Iterator (this, kbytes);
|
||||
}
|
||||
/** Range over the `key` values.\n
|
||||
* See also the `all` method. */
|
||||
template <typename K> boost::iterator_range<Iterator> valuesRange (const K& key) {return boost::iterator_range<Iterator> (values (key), end());}
|
||||
|
||||
struct Trigger {
|
||||
virtual gstring getTriggerName() const {return C2GSTRING ("defaultTriggerName");};
|
||||
virtual void add (Mdb& mdb, void* key, gstring& kbytes, void* value, gstring& vbytes, Transaction& txn) = 0;
|
||||
virtual void erase (Mdb& mdb, void* key, gstring& kbytes, Transaction& txn) = 0;
|
||||
virtual void eraseKV (Mdb& mdb, void* key, gstring& kbytes, void* value, gstring& vbytes, Transaction& txn) = 0;
|
||||
};
|
||||
std::map<gstring, std::shared_ptr<Trigger>> _triggers;
|
||||
|
||||
void setTrigger (std::shared_ptr<Trigger> trigger) {
|
||||
_triggers[trigger->getTriggerName()] = trigger;
|
||||
}
|
||||
|
||||
/** `flags` can be `MDB_RDONLY` */
|
||||
Transaction beginTransaction (unsigned flags = 0) {
|
||||
MDB_txn* txn = 0; int rc = ::mdb_txn_begin (_env.get(), nullptr, flags, &txn);
|
||||
if (rc) throw MdbEx (std::string ("mdb_txn_begin: ") + ::strerror (rc));
|
||||
return Transaction (txn, ::mdb_txn_abort);
|
||||
}
|
||||
void commitTransaction (Transaction& txn) {
|
||||
int rc = ::mdb_txn_commit (txn.get());
|
||||
txn.release(); // Must prevent `mdb_txn_abort` from happening (even if rc != 0).
|
||||
if (rc) throw MdbEx (std::string ("mdb_txn_commit: ") + ::strerror (rc));
|
||||
}
|
||||
|
||||
virtual unsigned envFlags (uint8_t sync) {
|
||||
unsigned flags = MDB_NOSUBDIR;
|
||||
if (sync < 1) flags |= MDB_NOSYNC; else if (sync < 2) flags |= MDB_NOMETASYNC;
|
||||
return flags;
|
||||
}
|
||||
/** Used before `mdb_env_open`. By default sets the number of database to 32. */
|
||||
virtual void envConf (MDB_env* env) {
|
||||
int rc = ::mdb_env_set_maxdbs (env, 32);
|
||||
if (rc) throw MdbEx (std::string ("envConf: ") + ::strerror (rc));
|
||||
}
|
||||
virtual void dbFlags (unsigned& flags) {}
|
||||
|
||||
protected:
|
||||
void open (const char* dbName, bool dup) {
|
||||
auto txn = beginTransaction();
|
||||
unsigned flags = MDB_CREATE;
|
||||
if (dup) flags |= MDB_DUPSORT;
|
||||
dbFlags (flags);
|
||||
int rc = ::mdb_open (txn.get(), dbName, flags, &_dbi);
|
||||
if (rc) throw MdbEx (std::string ("mdb_open (") + dbName + "): " + ::strerror (rc));
|
||||
commitTransaction (txn);
|
||||
}
|
||||
public:
|
||||
|
||||
/** Opens MDB environment and MDB database. */
|
||||
Mdb (const char* path, size_t maxSizeMb = 1024, const char* dbName = "main", uint8_t sync = 0, bool dup = true, mode_t mode = 0660) {
|
||||
MDB_env* env = 0; int rc = ::mdb_env_create (&env);
|
||||
if (rc) throw MdbEx (std::string ("mdb_env_create: ") + ::strerror (rc));
|
||||
_env.reset (env, ::mdb_env_close);
|
||||
rc = ::mdb_env_set_mapsize (env, maxSizeMb * 1024 * 1024);
|
||||
if (rc) throw MdbEx (std::string ("mdb_env_set_mapsize: ") + ::strerror (rc));
|
||||
envConf (env);
|
||||
rc = ::mdb_env_open (env, path, envFlags (sync), mode);
|
||||
_dbi = 0; open (dbName, dup);
|
||||
}
|
||||
|
||||
/** Opens MDB database in the provided environment. */
|
||||
Mdb (std::shared_ptr<MDB_env> env, const char* dbName, bool dup = true): _env (env), _dbi (0) {
|
||||
open (dbName, dup);
|
||||
}
|
||||
|
||||
template <typename K, typename V> void add (const K& key, const V& value, Transaction& txn) {
|
||||
char kbuf[64]; // Allow up to 64 bytes to be serialized without heap allocations.
|
||||
gstring kbytes (sizeof (kbuf), kbuf, false, 0);
|
||||
mdbSerialize (kbytes, key);
|
||||
MDB_val mkey = {kbytes.size(), (void*) kbytes.data()};
|
||||
|
||||
char vbuf[64]; // Allow up to 64 bytes to be serialized without heap allocations.
|
||||
gstring vbytes (sizeof (vbuf), vbuf, false, 0);
|
||||
mdbSerialize (vbytes, value);
|
||||
MDB_val mvalue = {vbytes.size(), (void*) vbytes.data()};
|
||||
|
||||
for (auto& trigger: _triggers) trigger.second->add (*this, (void*) &key, kbytes, (void*) &value, vbytes, txn);
|
||||
|
||||
int rc = ::mdb_put (txn.get(), _dbi, &mkey, &mvalue, 0);
|
||||
if (rc) throw MdbEx (std::string ("mdb_put: ") + ::strerror (rc));
|
||||
}
|
||||
template <typename K, typename V> void add (const K& key, const V& value) {
|
||||
Transaction txn (beginTransaction());
|
||||
add (key, value, txn);
|
||||
commitTransaction (txn);
|
||||
}
|
||||
|
||||
template <typename K, typename V> bool first (const K& key, V& value, Transaction& txn) {
|
||||
char kbuf[64]; // Allow up to 64 bytes to be serialized without heap allocations.
|
||||
gstring kbytes (sizeof (kbuf), kbuf, false, 0);
|
||||
mdbSerialize (kbytes, key);
|
||||
MDB_val mkey = {kbytes.size(), (void*) kbytes.data()};
|
||||
MDB_val mvalue;
|
||||
int rc = ::mdb_get (txn.get(), _dbi, &mkey, &mvalue);
|
||||
if (rc == MDB_NOTFOUND) return false;
|
||||
if (rc) throw MdbEx (std::string ("mdb_get: ") + ::strerror (rc));
|
||||
gstring vstr (0, mvalue.mv_data, false, mvalue.mv_size);
|
||||
mdbDeserialize (vstr, value);
|
||||
return true;
|
||||
}
|
||||
template <typename K, typename V> bool first (const K& key, V& value) {
|
||||
Transaction txn (beginTransaction (MDB_RDONLY));
|
||||
bool rb = first (key, value, txn);
|
||||
commitTransaction (txn);
|
||||
return rb;
|
||||
}
|
||||
|
||||
/** Iterate over `key` values until `visitor` returns `false`. Return the number of values visited. */
|
||||
template <typename K, typename V> int32_t all (const K& key, std::function<bool(const V&)> visitor, Transaction& txn) {
|
||||
char kbuf[64]; // Allow up to 64 bytes to be serialized without heap allocations.
|
||||
gstring kbytes (sizeof (kbuf), kbuf, false, 0);
|
||||
mdbSerialize (kbytes, key);
|
||||
MDB_val mkey = {kbytes.size(), (void*) kbytes.data()};
|
||||
|
||||
MDB_cursor* cur = 0; int rc = ::mdb_cursor_open (txn.get(), _dbi, &cur);
|
||||
if (rc) throw MdbEx (std::string ("mdb_cursor_open: ") + ::strerror (rc));
|
||||
std::unique_ptr<MDB_cursor, void(*)(MDB_cursor*)> curHolder (cur, ::mdb_cursor_close);
|
||||
MDB_val mval = {0, 0};
|
||||
rc = ::mdb_cursor_get (cur, &mkey, &mval, ::MDB_SET_KEY); if (rc == MDB_NOTFOUND) return 0;
|
||||
if (rc) throw MdbEx (std::string ("mdb_cursor_get: ") + ::strerror (rc));
|
||||
|
||||
V value;
|
||||
gstring vstr (0, mval.mv_data, false, mval.mv_size);
|
||||
mdbDeserialize (vstr, value);
|
||||
bool goOn = visitor (value);
|
||||
int32_t count = 1;
|
||||
|
||||
while (goOn) {
|
||||
rc = ::mdb_cursor_get (cur, &mkey, &mval, ::MDB_NEXT_DUP); if (rc == MDB_NOTFOUND) return count;
|
||||
if (rc) throw MdbEx (std::string ("mdb_cursor_get: ") + ::strerror (rc));
|
||||
gstring vstr (0, mval.mv_data, false, mval.mv_size);
|
||||
mdbDeserialize (vstr, value);
|
||||
goOn = visitor (value);
|
||||
++count;
|
||||
}
|
||||
return count;
|
||||
}
|
||||
/** Iterate over `key` values until `visitor` returns `false`. Return the number of values visited. */
|
||||
template <typename K, typename V> int32_t all (const K& key, std::function<bool(const V&)> visitor) {
|
||||
Transaction txn (beginTransaction (MDB_RDONLY));
|
||||
int32_t count = all (key, visitor, txn);
|
||||
commitTransaction (txn);
|
||||
return count;
|
||||
}
|
||||
|
||||
template <typename K, typename V> bool eraseKV (const K& key, const V& value, Transaction& txn) {
|
||||
char kbuf[64]; // Allow up to 64 bytes to be serialized without heap allocations.
|
||||
gstring kbytes (sizeof (kbuf), kbuf, false, 0);
|
||||
mdbSerialize (kbytes, key);
|
||||
if (kbytes.empty()) throw MdbEx ("eraseKV: key is empty");
|
||||
MDB_val mkey = {kbytes.size(), (void*) kbytes.data()};
|
||||
|
||||
char vbuf[64]; // Allow up to 64 bytes to be serialized without heap allocations.
|
||||
gstring vbytes (sizeof (vbuf), vbuf, false, 0);
|
||||
mdbSerialize (vbytes, value);
|
||||
MDB_val mvalue = {vbytes.size(), (void*) vbytes.data()};
|
||||
|
||||
for (auto& trigger: _triggers) trigger.second->eraseKV (*this, (void*) &key, kbytes, (void*) &value, vbytes, txn);
|
||||
|
||||
int rc = ::mdb_del (txn.get(), _dbi, &mkey, &mvalue);
|
||||
if (rc == MDB_NOTFOUND) return false;
|
||||
if (rc) throw MdbEx (std::string ("mdb_del: ") + ::strerror (rc));
|
||||
return true;
|
||||
}
|
||||
template <typename K, typename V> bool eraseKV (const K& key, const V& value) {
|
||||
Transaction txn (beginTransaction());
|
||||
bool rb = eraseKV (key, value, txn);
|
||||
commitTransaction (txn);
|
||||
return rb;
|
||||
}
|
||||
|
||||
/** Erase all values of the `key`. */
|
||||
template <typename K> bool erase (const K& key, Transaction& txn) {
|
||||
char kbuf[64]; // Allow up to 64 bytes to be serialized without heap allocations.
|
||||
gstring kbytes (sizeof (kbuf), kbuf, false, 0);
|
||||
mdbSerialize (kbytes, key);
|
||||
if (kbytes.empty()) throw MdbEx ("erase: key is empty");
|
||||
MDB_val mkey = {kbytes.size(), (void*) kbytes.data()};
|
||||
|
||||
for (auto& trigger: _triggers) trigger.second->erase (*this, (void*) &key, kbytes, txn);
|
||||
|
||||
int rc = ::mdb_del (txn.get(), _dbi, &mkey, nullptr);
|
||||
if (rc == MDB_NOTFOUND) return false;
|
||||
if (rc) throw MdbEx (std::string ("mdb_del: ") + ::strerror (rc));
|
||||
return true;
|
||||
}
|
||||
/** Erase all values of the `key`. */
|
||||
template <typename K> bool erase (const K& key) {
|
||||
Transaction txn (beginTransaction());
|
||||
bool rb = erase (key, txn);
|
||||
commitTransaction (txn);
|
||||
return rb;
|
||||
}
|
||||
|
||||
static void test (Mdb& mdb) {
|
||||
mdb.add (std::string ("foo"), std::string ("bar"));
|
||||
// NB: "MDB_DUPSORT doesn't allow duplicate duplicates" (Howard Chu)
|
||||
mdb.add (std::string ("foo"), std::string ("bar"));
|
||||
mdb.add ((uint32_t) 123, 1);
|
||||
mdb.add ((uint32_t) 123, 2);
|
||||
mdb.add (C2GSTRING ("foo"), 3);
|
||||
mdb.add (C2GSTRING ("foo"), 4);
|
||||
mdb.add (C2GSTRING ("gsk"), C2GSTRING ("gsv"));
|
||||
string ts; int ti; gstring tgs;
|
||||
auto fail = [](string msg) {throw std::runtime_error ("assertion failed: " + msg);};
|
||||
if (!mdb.first (std::string ("foo"), ts) || ts != "bar") fail ("!foo=bar");
|
||||
if (!mdb.first ((uint32_t) 123, ti) || ti < 1 || ti > 2) fail ("!123");
|
||||
if (!mdb.first (C2GSTRING ("foo"), ti) || ti < 3 || ti > 4) fail ("!foo=3,4");
|
||||
if (!mdb.first (C2GSTRING ("gsk"), tgs) || tgs != "gsv") fail ("!gsk=gsv");
|
||||
|
||||
// Test range-based for.
|
||||
int count = 0; bool haveGskGsv = false;
|
||||
for (auto&& entry: mdb) {
|
||||
if (!entry._key.mv_size || !entry._val.mv_size) fail ("!entry");
|
||||
if (entry.keyView() == "gsk") {
|
||||
if (entry.getKey<gstring>() != "gsk") fail ("getKey(gsk)!=gsk");
|
||||
if (entry.getValue<gstring>() != "gsv") fail ("getValue(gsk)!=gsv");
|
||||
haveGskGsv = true;
|
||||
}
|
||||
++count;}
|
||||
if (count != 6) fail ("count!=6"); // foo=bar, 123=1, 123=2, foo=3, foo=4, gsk=gsv
|
||||
if (!haveGskGsv) fail ("!haveGskGsv");
|
||||
|
||||
// Test `values`.
|
||||
count = 0; int sum = 0;
|
||||
for (auto&& entry: mdb.valuesRange ((uint32_t) 123)) {
|
||||
if (entry.getKey<uint32_t>() != (uint32_t) 123) fail("values(123).key!=123");
|
||||
++count; sum += entry.getValue<int>();
|
||||
}
|
||||
if (count != 2) fail ("count(123)!=2");
|
||||
if (sum != 3) fail ("sum(123)!=3");
|
||||
|
||||
if (!mdb.eraseKV ((uint32_t) 123, 1)) fail ("!eraseKV(123,1)");
|
||||
if (!mdb.first ((uint32_t) 123, ti) || ti != 2) fail ("!123=2");
|
||||
if (!mdb.eraseKV ((uint32_t) 123, 2)) fail ("!eraseKV(123,2)");
|
||||
if (mdb.first ((uint32_t) 123, ti)) fail ("123");
|
||||
|
||||
if (!mdb.erase (C2GSTRING ("foo"))) fail ("!erase(g(foo))");
|
||||
if (mdb.first (C2GSTRING ("foo"), ti)) fail ("foo");
|
||||
if (!mdb.erase (std::string ("foo"))) fail ("!erase(str(foo))");
|
||||
|
||||
{ // We've erased "123" and "foo", the only key left is "gsk" (gsk=gsv), let's test the iterator boundaries on this small dataset.
|
||||
auto&& it = mdb.begin();
|
||||
if (it->getKey<gstring>() != "gsk") fail ("first key !gsk " + it->keyView().str());
|
||||
if (!(++it).end()) fail ("++it != end");
|
||||
if ((--it)->getKey<gstring>() != "gsk") fail ("can't go back to gsk");
|
||||
if (!(--it).end()) fail ("--it != end");
|
||||
if ((++it)->getKey<gstring>() != "gsk") fail ("can't go forward to gsk");
|
||||
}
|
||||
|
||||
struct SimpleIndexTrigger: public Trigger {
|
||||
const char* _name; Mdb _indexDb;
|
||||
SimpleIndexTrigger (Mdb& mdb, const char* name = "index"): _name (name), _indexDb (mdb._env, name) {}
|
||||
gstring getTriggerName() {return gstring (0, (void*) _name, false, strlen (_name), true);}
|
||||
void add (Mdb& mdb, void* key, gstring& kbytes, void* value, gstring& vbytes, Transaction& txn) {
|
||||
MDB_val mkey = {vbytes.size(), (void*) vbytes.data()};
|
||||
MDB_val mvalue = {kbytes.size(), (void*) kbytes.data()};
|
||||
int rc = ::mdb_put (txn.get(), _indexDb._dbi, &mkey, &mvalue, 0);
|
||||
if (rc) throw MdbEx (std::string ("index, mdb_put: ") + ::strerror (rc));
|
||||
}
|
||||
void erase (Mdb& mdb, void* ekey, gstring& kbytes, Transaction& txn) {
|
||||
// Get all the values and remove them from the index.
|
||||
MDB_cursor* cur = 0; int rc = ::mdb_cursor_open (txn.get(), mdb._dbi, &cur);
|
||||
if (rc) throw MdbEx (std::string ("index, erase, mdb_cursor_open: ") + ::strerror (rc));
|
||||
std::unique_ptr<MDB_cursor, void(*)(MDB_cursor*)> curHolder (cur, ::mdb_cursor_close);
|
||||
MDB_val mkey = {kbytes.size(), (void*) kbytes.data()}, val = {0, 0};
|
||||
rc = ::mdb_cursor_get (cur, &mkey, &val, ::MDB_SET_KEY); if (rc == MDB_NOTFOUND) return;
|
||||
if (rc) throw MdbEx (std::string ("index, erase, mdb_cursor_get: ") + ::strerror (rc));
|
||||
rc = ::mdb_del (txn.get(), _indexDb._dbi, &val, &mkey);
|
||||
if (rc && rc != MDB_NOTFOUND) throw MdbEx (std::string ("index, erase, mdb_del: ") + ::strerror (rc));
|
||||
for (;;) {
|
||||
rc = ::mdb_cursor_get (cur, &mkey, &val, ::MDB_NEXT_DUP); if (rc == MDB_NOTFOUND) return;
|
||||
if (rc) throw MdbEx (std::string ("index, erase, mdb_cursor_get: ") + ::strerror (rc));
|
||||
rc = ::mdb_del (txn.get(), _indexDb._dbi, &val, &mkey);
|
||||
if (rc && rc != MDB_NOTFOUND) throw MdbEx (std::string ("index, erase, mdb_del: ") + ::strerror (rc));
|
||||
}
|
||||
}
|
||||
void eraseKV (Mdb& mdb, void* key, gstring& kbytes, void* value, gstring& vbytes, Transaction& txn) {
|
||||
MDB_val mkey = {vbytes.size(), (void*) vbytes.data()};
|
||||
MDB_val mvalue = {kbytes.size(), (void*) kbytes.data()};
|
||||
int rc = ::mdb_del (txn.get(), _indexDb._dbi, &mkey, &mvalue);
|
||||
if (rc && rc != MDB_NOTFOUND) throw MdbEx (std::string ("index, mdb_del: ") + ::strerror (rc));
|
||||
}
|
||||
};
|
||||
auto indexTrigger = std::make_shared<SimpleIndexTrigger> (mdb); mdb.setTrigger (indexTrigger); auto& indexDb = indexTrigger->_indexDb;
|
||||
mdb.erase (C2GSTRING ("gsk")); // NB: "gsk" wasn't indexed here. `IndexTrigger.erase` should handle this gracefully.
|
||||
|
||||
// Add indexed.
|
||||
mdb.add (C2GSTRING ("ik"), C2GSTRING ("iv1"));
|
||||
mdb.add (C2GSTRING ("ik"), string ("iv2"));
|
||||
mdb.add (C2GSTRING ("ik"), 3);
|
||||
// Check the index.
|
||||
gstring ik;
|
||||
if (!indexDb.first (C2GSTRING ("iv1"), ik) || ik != "ik") fail ("!iv1=ik");
|
||||
if (!indexDb.first (string ("iv2"), ik) || ik != "ik") fail ("!iv2=ik");
|
||||
if (!indexDb.first (3, ik) || ik != "ik") fail ("!iv3=ik");
|
||||
|
||||
// Remove indexed.
|
||||
mdb.eraseKV (C2GSTRING ("ik"), string ("iv2"));
|
||||
// Check the index.
|
||||
if (!indexDb.first (C2GSTRING ("iv1"), ik) || ik != "ik") fail ("!iv1=ik");
|
||||
if (indexDb.first (string ("iv2"), ik)) fail ("iv2=ik");
|
||||
if (!indexDb.first (3, ik) || ik != "ik") fail ("!iv3=ik");
|
||||
|
||||
// Remove indexed.
|
||||
mdb.erase (C2GSTRING ("ik"));
|
||||
// Check the index.
|
||||
if (indexDb.first (C2GSTRING ("iv1"), ik)) fail ("iv1");
|
||||
if (indexDb.first (3, ik)) fail ("iv3");
|
||||
// Check the data.
|
||||
if (mdb.first (C2GSTRING ("ik"), ik)) fail ("ik");
|
||||
}
|
||||
|
||||
virtual ~Mdb() {
|
||||
_triggers.clear(); // Destroy triggers before closing the database.
|
||||
if (_dbi) {::mdb_close (_env.get(), _dbi); _dbi = 0;}
|
||||
}
|
||||
};
|
||||
|
||||
} // namespace glim
|
||||
|
||||
#endif // _GLIM_MDB_HPP_INCLUDED
|
||||
3741
external/glim/ql2.pb.cc
vendored
Normal file
3741
external/glim/ql2.pb.cc
vendored
Normal file
File diff suppressed because it is too large
Load Diff
2532
external/glim/ql2.pb.h
vendored
Normal file
2532
external/glim/ql2.pb.h
vendored
Normal file
File diff suppressed because it is too large
Load Diff
34
external/glim/raii.hpp
vendored
Normal file
34
external/glim/raii.hpp
vendored
Normal file
@ -0,0 +1,34 @@
|
||||
#include <functional>
|
||||
namespace glim {
|
||||
|
||||
// http://stackoverflow.com/questions/2121607/any-raii-template-in-boost-or-c0x/
|
||||
|
||||
/// RAII helper. Keeps the functor and runs it in the destructor.
|
||||
/// Example: \code auto unmap = raiiFun ([&]() {munmap (fd, size);}); \endcode
|
||||
template<typename Fun> struct RAIIFun {
|
||||
Fun _fun;
|
||||
RAIIFun (RAIIFun&&) = default;
|
||||
RAIIFun (const RAIIFun&) = default;
|
||||
template<typename FunArg> RAIIFun (FunArg&& fun): _fun (std::forward<Fun> (fun)) {}
|
||||
~RAIIFun() {_fun();}
|
||||
};
|
||||
|
||||
/// The idea to name it `finally` comes from http://www.codeproject.com/Tips/476970/finally-clause-in-Cplusplus.
|
||||
/// Example: \code finally unmap ([&]() {munmap (fd, size);}); \endcode
|
||||
typedef RAIIFun<std::function<void(void)>> finally;
|
||||
|
||||
/// Runs the given functor when going out of scope.
|
||||
/// Example: \code
|
||||
/// auto closeFd = raiiFun ([&]() {close (fd);});
|
||||
/// auto unmap = raiiFun ([&]() {munmap (fd, size);});
|
||||
/// \endcode
|
||||
template<typename Fun> RAIIFun<Fun> raiiFun (const Fun& fun) {return RAIIFun<Fun> (fun);}
|
||||
|
||||
/// Runs the given functor when going out of scope.
|
||||
/// Example: \code
|
||||
/// auto closeFd = raiiFun ([&]() {close (fd);});
|
||||
/// auto unmap = raiiFun ([&]() {munmap (fd, size);});
|
||||
/// \endcode
|
||||
template<typename Fun> RAIIFun<Fun> raiiFun (Fun&& fun) {return RAIIFun<Fun> (std::move (fun));}
|
||||
|
||||
}
|
||||
402
external/glim/runner.hpp
vendored
Normal file
402
external/glim/runner.hpp
vendored
Normal file
@ -0,0 +1,402 @@
|
||||
#ifndef _GLIM_RUNNER_INCLUDED
|
||||
#define _GLIM_RUNNER_INCLUDED
|
||||
|
||||
#include <algorithm> // min
|
||||
#include <atomic>
|
||||
#include <condition_variable>
|
||||
#include <chrono>
|
||||
#include <functional>
|
||||
#include <mutex>
|
||||
#include <memory>
|
||||
#include <stdexcept>
|
||||
#include <thread>
|
||||
#include <unordered_map>
|
||||
|
||||
#include <curl/curl.h>
|
||||
#include <event2/event.h> // cf. hiperfifo.cpp at http://article.gmane.org/gmane.comp.web.curl.library/37752
|
||||
|
||||
#include <boost/intrusive_ptr.hpp>
|
||||
#include <boost/lockfree/queue.hpp> // http://www.boost.org/doc/libs/1_53_0/doc/html/boost/lockfree/queue.html
|
||||
#include <boost/log/trivial.hpp>
|
||||
|
||||
#include <time.h>
|
||||
#include <stdlib.h> // rand
|
||||
#include <sys/eventfd.h>
|
||||
|
||||
#include "gstring.hpp"
|
||||
#include "exception.hpp"
|
||||
|
||||
namespace glim {
|
||||
|
||||
/// Listens to messages returned by `curl_multi_info_read`.
|
||||
/// NB: When CURL is queued with `addToCURLM` the CURL's `CURLOPT_PRIVATE` must point to the instance of `CurlmInformationListener`.
|
||||
struct CurlmInformationListener {
|
||||
enum FreeOptions {REMOVE_CURL_FROM_CURLM = 1, CURL_CLEANUP = 2, DELETE_LISTENER = 4, REMOVE_CLEAN_DELETE = 1|2|4};
|
||||
virtual FreeOptions information (CURLMsg*, CURLM*) = 0;
|
||||
virtual ~CurlmInformationListener() {}
|
||||
};
|
||||
|
||||
/// Listener deferring to a lambda.
|
||||
struct FunCurlmLisneter: public glim::CurlmInformationListener {
|
||||
std::function <void(CURLMsg*, CURLM*)> _fun;
|
||||
FreeOptions _freeOptions;
|
||||
FunCurlmLisneter (std::function <void(CURLMsg*, CURLM*)>&& fun, FreeOptions freeOptions): _fun (std::move (fun)), _freeOptions (freeOptions) {}
|
||||
virtual FreeOptions information (CURLMsg* msg, CURLM* curlm) override {
|
||||
if (__builtin_expect ((bool) _fun, 1))
|
||||
try {_fun (msg, curlm);} catch (const std::exception& ex) {BOOST_LOG_TRIVIAL (error) << "FunCurlmLisneter] " << ex.what();}
|
||||
return _freeOptions;
|
||||
}
|
||||
};
|
||||
|
||||
/// Running cURL jobs in a single thread.
|
||||
/// NB: The RunnerV2 *must* be allocated with `boost::intrusive_ptr` (typically you'd use `RunnerV2::instance()`).
|
||||
class RunnerV2 {
|
||||
std::atomic_int_fast32_t _references {0}; // For intrusive_ptr.
|
||||
CURLM* _multi = nullptr; ///< Initialized in `run`. Should not be used outside of it.
|
||||
int _eventFd = 0; ///< Used to give the `curl_multi_wait` some work when there's no cURL descriptors and to wake it from `withCURLM`.
|
||||
boost::lockfree::queue<CURL*, boost::lockfree::capacity<64>> _queue; ///< `CURL` handles waiting to be added to `CURL_MULTI`.
|
||||
std::thread _thread;
|
||||
std::atomic_bool _running {false}; /// True if the `_thread` is running.
|
||||
|
||||
using FreeOptions = CurlmInformationListener::FreeOptions;
|
||||
|
||||
friend inline void intrusive_ptr_add_ref (RunnerV2*);
|
||||
friend inline void intrusive_ptr_release (RunnerV2*);
|
||||
|
||||
void run() noexcept {
|
||||
try {
|
||||
if (__builtin_expect (_references <= 0, 0)) GTHROW ("RunnerV2] Must be allocated with boost::intrusive_ptr!");
|
||||
_running = true; // NB: _running only becomes true if we're in the intrusive_ptr. ^^
|
||||
pthread_setname_np (pthread_self(), "Runner");
|
||||
_multi = curl_multi_init(); if (__builtin_expect (_multi == nullptr, 0)) GTHROW ("!curl_multi_init");
|
||||
_eventFd = eventfd (0, EFD_CLOEXEC | EFD_NONBLOCK); // Used to pause `curl_multi_wait` when there's no other jobs.
|
||||
if (__builtin_expect (_eventFd == -1, 0)) GTHROW (std::string ("eventfd: ") + ::strerror (errno));
|
||||
while (__builtin_expect (_references > 0, 0)) {
|
||||
// Reset the CURL_EVENT_FD value to 0, so that the `curl_multi_wait` can sleep.
|
||||
if (__builtin_expect (_eventFd > 0, 1)) {eventfd_t count = 0; eventfd_read (_eventFd, &count);}
|
||||
|
||||
// Add the queued CURL handles to our CURLM.
|
||||
CURL* easy = nullptr; while (_queue.pop (easy)) curl_multi_add_handle (_multi, easy);
|
||||
|
||||
// Run the cURL.
|
||||
int runningHandles = 0;
|
||||
CURLMcode rc = curl_multi_perform (_multi, &runningHandles); // http://curl.haxx.se/libcurl/c/curl_multi_perform.html
|
||||
if (__builtin_expect (rc != CURLM_OK, 0)) BOOST_LOG_TRIVIAL (error) << "Runner] curl_multi_perform: " << curl_multi_strerror (rc);
|
||||
|
||||
// Process the finished handles.
|
||||
for (;;) {
|
||||
int messagesLeft = 0; CURLMsg* msg = curl_multi_info_read (_multi, &messagesLeft); if (msg) try {
|
||||
CURL* curl = msg->easy_handle; CurlmInformationListener* listener = 0;
|
||||
if (__builtin_expect (curl_easy_getinfo (curl, CURLINFO_PRIVATE, &listener) == CURLE_OK, 1)) {
|
||||
using FOP = CurlmInformationListener::FreeOptions;
|
||||
FOP fop = listener->information (msg, _multi);
|
||||
if (fop & FOP::REMOVE_CURL_FROM_CURLM) curl_multi_remove_handle (_multi, curl);
|
||||
if (fop & FOP::CURL_CLEANUP) curl_easy_cleanup (curl);
|
||||
if (fop & FOP::DELETE_LISTENER) delete listener;
|
||||
} else {
|
||||
curl_multi_remove_handle (_multi, curl);
|
||||
curl_easy_cleanup (curl);
|
||||
}
|
||||
} catch (const std::exception& ex) {BOOST_LOG_TRIVIAL (error) << "Runner] " << ex.what();}
|
||||
if (messagesLeft == 0) break;
|
||||
}
|
||||
|
||||
// Wait on the cURL file descriptors.
|
||||
int descriptors = 0;
|
||||
curl_waitfd waitfd = {_eventFd, CURL_WAIT_POLLIN, 0};
|
||||
eventfd_t eValue = 0; eventfd_read (_eventFd, &eValue); // Reset the curlEventFd value to zero.
|
||||
rc = curl_multi_wait (_multi, &waitfd, 1, 100, &descriptors); // http://curl.haxx.se/libcurl/c/curl_multi_wait.html
|
||||
if (__builtin_expect (rc != CURLM_OK, 0)) BOOST_LOG_TRIVIAL (error) << "Runner] curl_multi_wait: " << curl_multi_strerror (rc);
|
||||
}
|
||||
} catch (const std::exception& ex) {BOOST_LOG_TRIVIAL (error) << "Runner] " << ex.what();}
|
||||
// Delayed destruction: when we're in intrusive_ptr (_running == true) but no longer referenced.
|
||||
if (_running && _references == 0) delete this; // http://www.parashift.com/c++-faq-lite/delete-this.html
|
||||
else _running = false;
|
||||
}
|
||||
public:
|
||||
RunnerV2() {
|
||||
// Start a thread using CURLM in a thread-safe way (that is, from this single thread only).
|
||||
// NB: Handles *can* be passed between threads: http://article.gmane.org/gmane.comp.web.curl.library/33188
|
||||
_thread = std::thread (&RunnerV2::run, this);
|
||||
}
|
||||
~RunnerV2() {
|
||||
_thread.detach();
|
||||
}
|
||||
|
||||
/// A singletone instance of the Runner used in order for different programes to reuse the same cURL thread.
|
||||
static boost::intrusive_ptr<RunnerV2>& instance() {
|
||||
static boost::intrusive_ptr<RunnerV2> INSTANCE (new RunnerV2());
|
||||
return INSTANCE;
|
||||
}
|
||||
|
||||
/// Schedule a CURL handler to be executed in the cURL thread.
|
||||
/// NB: If the handle have a `CURLOPT_PRIVATE` option then it MUST point to an instance of `CurlmInformationListener`.
|
||||
void addToCURLM (CURL* easyHandle) {
|
||||
if (__builtin_expect (!_queue.push (easyHandle), 0)) GTHROW ("Can't push CURL* into the queue.");
|
||||
if (__builtin_expect (_eventFd > 0, 1)) eventfd_write (_eventFd, 1); // Will wake the `curl_multi_wait` up, in order to run the `curl_multi_add_handle`.
|
||||
}
|
||||
|
||||
/// Schedule a CURL handler to be executed in the cURL thread.
|
||||
/// NB: `CURLOPT_PRIVATE` is overwritten with a pointer to `FunCurlmLisneter`.
|
||||
void addToCURLM (CURL* easyHandle, std::function <void(CURLMsg*, CURLM*)>&& listener,
|
||||
FreeOptions freeOptions = static_cast<FreeOptions> (FreeOptions::REMOVE_CURL_FROM_CURLM | FreeOptions::DELETE_LISTENER)) {
|
||||
FunCurlmLisneter* funListener = new FunCurlmLisneter (std::move (listener), freeOptions); // Will be deleted by the Runner.
|
||||
curl_easy_setopt (easyHandle, CURLOPT_PRIVATE, funListener); // Tells `addToCURLM` to call this listener later.
|
||||
addToCURLM (easyHandle);
|
||||
}
|
||||
};
|
||||
|
||||
inline void intrusive_ptr_add_ref (RunnerV2* runner) {++ runner->_references;}
|
||||
inline void intrusive_ptr_release (RunnerV2* runner) {if (-- runner->_references == 0 && !runner->_running) delete runner;}
|
||||
|
||||
/// Run CURLM requests and completion handlers, as well as other periodic jobs.
|
||||
class Runner {
|
||||
G_DEFINE_EXCEPTION (RunnerEx);
|
||||
/// Free CURL during stack unwinding.
|
||||
struct FreeCurl {
|
||||
Runner* runner; CURL* curl;
|
||||
FreeCurl (Runner* runner, CURL* curl): runner (runner), curl (curl) {}
|
||||
~FreeCurl() {
|
||||
runner->_handlers.erase (curl);
|
||||
curl_multi_remove_handle (runner->_curlm, curl);
|
||||
curl_easy_cleanup (curl);
|
||||
}
|
||||
};
|
||||
public:
|
||||
struct JobInfo;
|
||||
/// The job must return `true` if Runner is to continue invoking it.
|
||||
typedef std::function<bool(JobInfo& jobInfo)> job_t;
|
||||
struct JobInfo {
|
||||
job_t job;
|
||||
float pauseSec = 1.0f;
|
||||
struct timespec ran = {0, 0};
|
||||
};
|
||||
protected:
|
||||
typedef std::function<void(CURLMsg*)> handler_t;
|
||||
typedef std::function<void(const char* error)> errlog_t;
|
||||
std::shared_ptr<struct event_base> _evbase;
|
||||
errlog_t _errlog;
|
||||
std::recursive_mutex _mutex;
|
||||
typedef std::unique_ptr<struct event, void(*)(struct event*)> event_t;
|
||||
std::unordered_map<CURL*, std::pair<handler_t, event_t>> _handlers;
|
||||
/// Functions to run periodically.
|
||||
typedef std::unordered_map<gstring, JobInfo> jobs_map_t;
|
||||
jobs_map_t _jobs;
|
||||
CURLM* _curlm = nullptr;
|
||||
struct event* _timer = nullptr;
|
||||
|
||||
/// Schedule a function to be run on the event loop. Useful to run all cURL methods on the single event loop thread.
|
||||
template<typename F>
|
||||
void doInEv (F fun, struct timeval after = {0, 0}) {
|
||||
struct Dugout {F fun; struct event* timer; Dugout (F&& fun): fun (std::move (fun)), timer (nullptr) {}} *dugout = new Dugout (std::move (fun));
|
||||
event_callback_fn cb = [](evutil_socket_t, short, void* dugout_)->void {
|
||||
Dugout* dugout = static_cast<Dugout*> (dugout_);
|
||||
event_free (dugout->timer); dugout->timer = nullptr;
|
||||
F fun = std::move (dugout->fun); delete dugout;
|
||||
fun();
|
||||
};
|
||||
dugout->timer = evtimer_new (_evbase.get(), cb, dugout);
|
||||
evtimer_add (dugout->timer, &after);
|
||||
}
|
||||
|
||||
bool shouldRun (jobs_map_t::value_type& entry, const struct timespec& ct) {
|
||||
JobInfo& jobInfo = entry.second;
|
||||
if (jobInfo.pauseSec <= 0.f) return true; // Run always.
|
||||
if (jobInfo.ran.tv_sec == 0) {jobInfo.ran = ct; return true;}
|
||||
float delta = (float)(ct.tv_sec - jobInfo.ran.tv_sec);
|
||||
delta += (float)(ct.tv_nsec - jobInfo.ran.tv_nsec) / 1000000000.0f;
|
||||
if (delta >= jobInfo.pauseSec) {jobInfo.ran = ct; return true;}
|
||||
return false;
|
||||
}
|
||||
|
||||
/// Used for debugging.
|
||||
static uint64_t ms() {
|
||||
return std::chrono::duration_cast<std::chrono::milliseconds> (std::chrono::system_clock::now().time_since_epoch()) .count();
|
||||
}
|
||||
/// Tells CURL to check its sockets.
|
||||
void callCurlWithTimeout() {
|
||||
//std::cout << __LINE__ << ',' << ms() << ": callCurlWithTimeout" << std::endl;
|
||||
int running_handles = 0;
|
||||
CURLMcode rc = curl_multi_socket_action (_curlm, CURL_SOCKET_TIMEOUT, 0, &running_handles);
|
||||
if (rc != CURLM_OK) {GSTRING_ON_STACK (err, 256) << "glim::Runner: curl_multi_socket_action: " << curl_multi_strerror (rc); _errlog (err.c_str());}
|
||||
}
|
||||
|
||||
/// Should only be run when the _mutex is locked.
|
||||
void checkForFinishedCurlJobs() {
|
||||
//std::cout << __LINE__ << ',' << ms() << ": checkForFinishedCurlJobs" << std::endl;
|
||||
nextMessage:
|
||||
int msgs_in_queue = 0;
|
||||
CURLMsg* msg = curl_multi_info_read (_curlm, &msgs_in_queue);
|
||||
if (msg) try {
|
||||
auto curl = msg->easy_handle;
|
||||
FreeCurl freeCurl (this, curl);
|
||||
auto it = _handlers.find (curl);
|
||||
if (it != _handlers.end()) it->second.first (msg);
|
||||
if (msgs_in_queue > 0) goto nextMessage;
|
||||
} catch (const std::exception& ex) {
|
||||
char eBuf[512]; gstring err (sizeof(eBuf), eBuf, false, 0);
|
||||
err << "glim::Runner: handler: " << ex.what();
|
||||
_errlog (err.c_str());
|
||||
}
|
||||
}
|
||||
/// Will reset the timer unless there is a shorter timer already set.
|
||||
void restartTimer (uint32_t nextInMicro = 100000) { // 100ms = 100000µs
|
||||
struct timeval tv;
|
||||
if (event_pending (_timer, EV_TIMEOUT, &tv) && !tv.tv_sec && tv.tv_usec < nextInMicro) return; // Already have a shorter timeout.
|
||||
tv = {0, nextInMicro};
|
||||
evtimer_add (_timer, &tv);
|
||||
}
|
||||
static void evTimerCB (evutil_socket_t, short, void* runner_) {
|
||||
//std::cout << __LINE__ << ',' << ms() << ": evTimerCB" << std::endl;
|
||||
Runner* runner = (Runner*) runner_;
|
||||
runner->callCurlWithTimeout();
|
||||
runner->run();
|
||||
}
|
||||
/// event_callback_fn: There is an activity on a socket we are monitoring for CURL.
|
||||
static void evSocketCB (evutil_socket_t sock, short events, void* runner_) {
|
||||
//std::cout << __LINE__ << ',' << ms() << ": evSocketCB; sock: " << sock << "; events: " << events << std::endl;
|
||||
Runner* runner = (Runner*) runner_;
|
||||
int ev_bitmask = (events & EV_READ ? CURL_CSELECT_IN : 0) | (events & EV_WRITE ? CURL_CSELECT_OUT : 0);
|
||||
int running_handles = 0;
|
||||
CURLMcode rc = curl_multi_socket_action (runner->_curlm, sock, ev_bitmask, &running_handles);
|
||||
if (rc != CURLM_OK) {GSTRING_ON_STACK (err, 256) << "glim::Runner: curl_multi_socket_action: " << curl_multi_strerror (rc); runner->_errlog (err.c_str());}
|
||||
}
|
||||
static void deleteEvent (struct event* ev) {
|
||||
//std::cout << __LINE__ << ',' << ms() << ": deleteEvent: " << ev << std::endl;
|
||||
event_del (ev); event_free (ev);
|
||||
};
|
||||
/// curl_socket_callback: CURL asks us to monitor the socket.
|
||||
static int curlSocketCB (CURL* easy, curl_socket_t sock, int what, void* runner_, void* socketp) {
|
||||
//std::cout << __LINE__ << ',' << ms() << ": curlSocketCB; sock: " << sock << "; what: " << what;
|
||||
//std::cout << " (" << (what == 0 ? "none" : what == 1 ? "in" : what == 2 ? "out" : what == 3 ? "inout" : what == 4 ? "remove" : "?") << ")" << std::endl;
|
||||
Runner* runner = (Runner*) runner_;
|
||||
std::lock_guard<std::recursive_mutex> lock (runner->_mutex);
|
||||
if (what & CURL_POLL_REMOVE) {
|
||||
auto it = runner->_handlers.find (easy); if (it != runner->_handlers.end()) it->second.second.reset();
|
||||
// We can't run `checkForFinishedCurlJobs` from there or bad things would happen
|
||||
// (`curl_multi_remove_handle` will be called while we are still in the `curl_multi_socket_action`),
|
||||
// but we can schedule the check via the libevent timer.
|
||||
runner->restartTimer (0);
|
||||
} else {
|
||||
auto it = runner->_handlers.find (easy); if (it != runner->_handlers.end() && !it->second.second) {
|
||||
event_callback_fn cb = evSocketCB;
|
||||
struct event* ev = event_new (runner->_evbase.get(), sock, EV_READ | EV_WRITE | EV_ET | EV_PERSIST, cb, runner);
|
||||
event_add (ev, nullptr);
|
||||
//std::cout << __LINE__ << ',' << ms() << ": new event: " << ev << std::endl;
|
||||
it->second.second = event_t (ev, deleteEvent);
|
||||
}
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
/// curl_multi_timer_callback: Schedule a CURL timer event or if `timeout_ms` is 0 then run immediately.
|
||||
static int curlTimerCB (CURLM* multi, long timeout_ms, void* runner_) {
|
||||
//std::cout << __LINE__ << ',' << ms() << ": curlTimerCB; timeout_ms: " << timeout_ms << std::endl;
|
||||
if (timeout_ms == -1) return 0; // CURL tells us it doesn't need no timer.
|
||||
Runner* runner = (Runner*) runner_;
|
||||
if (timeout_ms == 0) { // CURL tells us it wants to run NOW.
|
||||
runner->callCurlWithTimeout();
|
||||
return 0;
|
||||
}
|
||||
// CURL asks us to run it `timeout_ms` from now.
|
||||
runner->restartTimer (std::min ((uint32_t) timeout_ms, (uint32_t) 100) * 1000); // We wait no more than 100ms.
|
||||
return 0;
|
||||
}
|
||||
public:
|
||||
Runner (std::shared_ptr<struct event_base> evbase, errlog_t errlog): _evbase (evbase), _errlog (errlog) {
|
||||
doInEv ([this]() {
|
||||
std::lock_guard<std::recursive_mutex> lock (_mutex);
|
||||
_curlm = curl_multi_init(); if (!_curlm) GNTHROW (RunnerEx, "!curl_multi_init");
|
||||
auto check = [this](CURLMcode rc) {if (rc != CURLM_OK) {curl_multi_cleanup (_curlm); GNTHROW (RunnerEx, "curl_multi_setopt: " + std::to_string (rc));}};
|
||||
check (curl_multi_setopt (_curlm, CURLMOPT_SOCKETDATA, this));
|
||||
curl_socket_callback socketCB = curlSocketCB; check (curl_multi_setopt (_curlm, CURLMOPT_SOCKETFUNCTION, socketCB));
|
||||
check (curl_multi_setopt (_curlm, CURLMOPT_TIMERDATA, this));
|
||||
curl_multi_timer_callback curlTimerCB_ = curlTimerCB; check (curl_multi_setopt (_curlm, CURLMOPT_TIMERFUNCTION, curlTimerCB_));
|
||||
event_callback_fn evTimerCB_ = evTimerCB; _timer = evtimer_new (_evbase.get(), evTimerCB_, this);
|
||||
restartTimer();
|
||||
});
|
||||
}
|
||||
~Runner() {
|
||||
//std::cout << __LINE__ << ',' << ms() << ": ~Runner" << std::endl;
|
||||
std::lock_guard<std::recursive_mutex> lock (_mutex);
|
||||
if (_timer) {evtimer_del (_timer); event_free (_timer); _timer = nullptr;}
|
||||
doInEv ([curlm = _curlm, handlers = std::move (_handlers)]() {
|
||||
for (auto it = handlers.begin(), end = handlers.end(); it != end; ++it) {
|
||||
curl_multi_remove_handle (curlm, it->first);
|
||||
curl_easy_cleanup (it->first);
|
||||
}
|
||||
if (curlm) {curl_multi_cleanup (curlm);}
|
||||
});
|
||||
_curlm = nullptr;
|
||||
}
|
||||
|
||||
/** Turns HTTP Pipelining on (or off).
|
||||
* See http://curl.haxx.se/libcurl/c/curl_multi_setopt.html#CURLMOPTPIPELINING */
|
||||
Runner& pipeline (long enabled = 1) {
|
||||
CURLMcode rc = curl_multi_setopt (_curlm, CURLMOPT_PIPELINING, enabled);
|
||||
if (rc != CURLM_OK) GNTHROW (RunnerEx, "curl_multi_setopt: " + std::to_string (rc));
|
||||
return *this;
|
||||
}
|
||||
|
||||
/// Wait for the operation to complete, then call the `handler`, then free the `curl`.
|
||||
void multi (CURL* curl, handler_t handler) {
|
||||
{ std::lock_guard<std::recursive_mutex> lock (_mutex);
|
||||
_handlers.insert (std::make_pair (curl, std::make_pair (std::move (handler), event_t (nullptr, nullptr)))); }
|
||||
doInEv ([this,curl]() {
|
||||
curl_multi_add_handle (_curlm, curl);
|
||||
});
|
||||
}
|
||||
/// Register a new job to be run on the thread loop.
|
||||
JobInfo& job (const gstring& name) {
|
||||
std::lock_guard<std::recursive_mutex> lock (_mutex);
|
||||
return _jobs[name];
|
||||
}
|
||||
/// Register a new job to be run on the thread loop.
|
||||
void schedule (const gstring& name, float pauseSec, job_t job) {
|
||||
struct timespec ct; if (pauseSec > 0.f) clock_gettime (CLOCK_MONOTONIC, &ct);
|
||||
std::lock_guard<std::recursive_mutex> lock (_mutex);
|
||||
JobInfo& jobInfo = _jobs[name];
|
||||
jobInfo.job = job;
|
||||
jobInfo.pauseSec = pauseSec;
|
||||
if (pauseSec > 0.f) jobInfo.ran = ct; // If we need a pause then we also need to know when the job was scheduled.
|
||||
}
|
||||
/// Register a new job to be run on the thread loop.
|
||||
void schedule (float pauseSec, job_t job) {
|
||||
// Find a unique job name.
|
||||
anotherName:
|
||||
GSTRING_ON_STACK (name, 64) << "job" << rand();
|
||||
if (_jobs.find (name) != _jobs.end()) goto anotherName;
|
||||
schedule (name, pauseSec, std::move (job));
|
||||
}
|
||||
void removeJob (const gstring& name) {
|
||||
std::lock_guard<std::recursive_mutex> lock (_mutex);
|
||||
_jobs.erase (name);
|
||||
}
|
||||
/// Invoked automatically from a libevent timer; can also be invoked manually.
|
||||
void run() {
|
||||
_mutex.lock();
|
||||
checkForFinishedCurlJobs();
|
||||
// Run non-CURL jobs. Copy jobs into a local array in order not to run them with the `_mutex` locked.
|
||||
struct timespec ct; clock_gettime (CLOCK_MONOTONIC, &ct);
|
||||
JobInfo jobs[_jobs.size()]; gstring jobNames[_jobs.size()]; int jn = -1; {
|
||||
for (auto it = _jobs.begin(), end = _jobs.end(); it != end; ++it) if (shouldRun (*it, ct)) {
|
||||
++jn; jobNames[jn] = it->first; jobs[jn] = it->second;
|
||||
} }
|
||||
_mutex.unlock();
|
||||
|
||||
for (; jn >= 0; --jn) try {
|
||||
if (!jobs[jn].job (jobs[jn])) removeJob (jobNames[jn]);
|
||||
} catch (const std::exception& ex) {
|
||||
char eBuf[512]; gstring err (sizeof(eBuf), eBuf, false, 0);
|
||||
err << "glim::Runner: error in job " << jobNames[jn] << ": " << ex.what();
|
||||
_errlog (err.c_str());
|
||||
}
|
||||
restartTimer();
|
||||
}
|
||||
|
||||
/// Expose CURLM. Useful for curl_multi_setopt (http://curl.haxx.se/libcurl/c/curl_multi_setopt.html).
|
||||
CURLM* curlm() const {return _curlm;}
|
||||
};
|
||||
|
||||
} // namespace glim
|
||||
|
||||
#endif // _GLIM_RUNNER_INCLUDED
|
||||
538
external/glim/sqlite.hpp
vendored
Normal file
538
external/glim/sqlite.hpp
vendored
Normal file
@ -0,0 +1,538 @@
|
||||
#ifndef GLIM_SQLITE_HPP_
|
||||
#define GLIM_SQLITE_HPP_
|
||||
|
||||
/**
|
||||
* A threaded interface to <a href="http://sqlite.org/">SQLite</a>.
|
||||
* This file is a header-only library,
|
||||
* whose sole dependencies should be standard STL and posix threading libraries.
|
||||
* You can extract this file out of the "glim" library to include it separately in your project.
|
||||
* @code
|
||||
Copyright 2006-2012 Kozarezov Artem Aleksandrovich
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
* @endcode
|
||||
* @file
|
||||
*/
|
||||
|
||||
#include <stdexcept>
|
||||
#include <string>
|
||||
#include <sqlite3.h>
|
||||
#include <pthread.h>
|
||||
#include <string.h> // strerror
|
||||
#include <sys/types.h> // stat
|
||||
#include <sys/stat.h> // stat
|
||||
#include <unistd.h> // stat
|
||||
#include <errno.h> // stat
|
||||
#include <stdio.h> // snprintf
|
||||
#include <stdint.h>
|
||||
|
||||
namespace glim {
|
||||
|
||||
class SqliteSession;
|
||||
class SqliteQuery;
|
||||
|
||||
struct SqliteEx: public std::runtime_error {
|
||||
SqliteEx (const std::string& what): std::runtime_error (what) {}
|
||||
};
|
||||
|
||||
/**
|
||||
* The database.
|
||||
* According to sqlite3_open <a href="http://sqlite.org/capi3ref.html#sqlite3_open">documentation</a>,
|
||||
* only the thread that opened the database can safely access it. This restriction was
|
||||
* relaxed, as described in the <a href="http://www.sqlite.org/faq.html#q8">FAQ</a>
|
||||
* (the "Is SQLite threadsafe?" question), so we can use the library from multiple
|
||||
* threads, but only if no more than one thread at a time accesses the database.
|
||||
* This restriction is, in fact, beneficial if the database is used from a single application:
|
||||
* by restricting access to a sigle thread at a time, we effectively avoid all deadlock issues.\n
|
||||
* This library goals are:\n
|
||||
* \li to ensure that SQLite is used in a thread-safe way,
|
||||
* \li to provide additional threaded quirks, such as delayed updates (not implemented).
|
||||
*
|
||||
* The library is targeted at SQLite setup which is \b not \c -DTHREADSAFE,
|
||||
* since this is the default setup on UNIX architectures.\n
|
||||
* \n
|
||||
* This file is a header-only library,
|
||||
* whose sole dependencies should be standard STL and posix threading libraries.
|
||||
* You can extract this file out of the "glim" library to include it separately in your project.\n
|
||||
* \n
|
||||
* This library is targeted at UTF-8 API. There is no plans to support the UTF-16 API.\n
|
||||
* \n
|
||||
* See also:\n
|
||||
* \li http://www.sqlite.org/cvstrac/fileview?f=sqlite/src/server.c\n
|
||||
* for another way of handling multithreading with SQLite.
|
||||
*/
|
||||
class Sqlite {
|
||||
/// No copying allowed.
|
||||
Sqlite& operator = (const Sqlite& other) {return *this;}
|
||||
/// No copying allowed.
|
||||
Sqlite (const Sqlite& other) = delete;
|
||||
friend class SqliteSession;
|
||||
protected:
|
||||
/// Filename the database was opened with; we need it to reopen the database on fork()s.
|
||||
/// std::string is used to avoid memory allocation issues.
|
||||
std::string filename;
|
||||
::sqlite3* handler;
|
||||
::pthread_mutex_t mutex;
|
||||
public:
|
||||
/// Flags for the Sqlite constructor.
|
||||
enum Flags {
|
||||
/**
|
||||
* The file will be checked for existence.
|
||||
* SqliteEx is thrown if the file is not accessible;
|
||||
* format of the error description is "$filename: $strerror".\n
|
||||
* Usage example: \code Sqlite db ("filename", Sqlite::existing); \endcode
|
||||
*/
|
||||
existing = 1
|
||||
};
|
||||
/**
|
||||
* Opens the database.
|
||||
* @param filename Database filename (UTF-8).
|
||||
* @param flags Optional. Currently there is the #existing flag.
|
||||
* @throws SqliteEx Thrown if we can't open the database.
|
||||
*/
|
||||
Sqlite (std::string filename, int flags = 0) {
|
||||
if (flags & existing) {
|
||||
// Check if the file exists already.
|
||||
struct stat st; if (stat (filename.c_str(), &st))
|
||||
throw SqliteEx (filename + ": " + ::strerror(errno));
|
||||
}
|
||||
::pthread_mutex_init (&mutex, NULL);
|
||||
this->filename = filename;
|
||||
if (::sqlite3_open(filename.c_str(), &handler) != SQLITE_OK)
|
||||
throw SqliteEx (std::string("sqlite3_open(") + filename + "): " + ::sqlite3_errmsg(handler));
|
||||
}
|
||||
/**
|
||||
* Closes the database.
|
||||
* @throws SqliteEx Thrown if we can't close the database.
|
||||
*/
|
||||
~Sqlite () {
|
||||
::pthread_mutex_destroy (&mutex);
|
||||
if (::sqlite3_close(handler) != SQLITE_OK)
|
||||
throw SqliteEx (std::string ("sqlite3_close(): ") + ::sqlite3_errmsg(handler));
|
||||
}
|
||||
|
||||
Sqlite& exec (const char* query);
|
||||
/**
|
||||
* Invokes `exec` on `query.c_str()`.
|
||||
* Example:\code
|
||||
* glim::Sqlite sqlite (":memory:");
|
||||
* for (std::string pv: {"page_size = 4096", "secure_delete = 1"}) sqlite->exec2 ("PRAGMA " + pv); \endcode
|
||||
*/
|
||||
template <typename StringLike> Sqlite& exec2 (StringLike query) {return exec (query.c_str());}
|
||||
};
|
||||
|
||||
/**
|
||||
* A single thread session with Sqlite.
|
||||
* Only a sigle thread at a time can have an SqliteSession,
|
||||
* all other threads will wait, in the SqliteSession constructor,
|
||||
* till the active session is either closed or destructed.
|
||||
*/
|
||||
class SqliteSession {
|
||||
/// No copying allowed.
|
||||
SqliteSession& operator = (const SqliteSession& other) {return *this;}
|
||||
/// No copying allowed.
|
||||
SqliteSession(SqliteSession& other): db (NULL) {}
|
||||
protected:
|
||||
Sqlite* db;
|
||||
public:
|
||||
/**
|
||||
* Locks the database.
|
||||
* @throws SqliteEx if a mutex error occurs.
|
||||
*/
|
||||
SqliteSession (Sqlite* sqlite): db (sqlite) {
|
||||
int err = ::pthread_mutex_lock (&(db->mutex));
|
||||
if (err != 0) throw SqliteEx (std::string ("error locking the mutex: ") + ::strerror(err));
|
||||
}
|
||||
/**
|
||||
* A shorter way to construct query from the session.
|
||||
* Usage example: \code ses.query(S("create table test (i integer)")).step() \endcode
|
||||
* @see SqliteQuery#qstep
|
||||
*/
|
||||
template <typename T>
|
||||
SqliteQuery query (T t);
|
||||
/// Automatically unlocks the database.
|
||||
/// @see close
|
||||
~SqliteSession () {close();}
|
||||
/**
|
||||
* Unlock the database.
|
||||
* It is safe to call this method multiple times.\n
|
||||
* You must not use the session after it was closed.\n
|
||||
* All resources allocated within this session must be released before the session is closed.
|
||||
* @throws SqliteEx if a mutex error occurs.
|
||||
*/
|
||||
void close () {
|
||||
if (db == NULL) return;
|
||||
int err = ::pthread_mutex_unlock (&(db->mutex));
|
||||
db = NULL;
|
||||
if (err != 0) throw SqliteEx (std::string ("error unlocking the mutex: ") + ::strerror(err));
|
||||
}
|
||||
/// True if the \c close method has been already called on this SqliteSession.
|
||||
bool isClosed () const {
|
||||
return db == NULL;
|
||||
}
|
||||
/**
|
||||
* This class can be used in place of the SQLite handler.
|
||||
* Make sure you've released any resources thus manually acquired before this SqliteSession is closed.
|
||||
* Usage example:
|
||||
* @code
|
||||
* glim::Sqlite db (":memory:");
|
||||
* glim::SqliteSession ses (&db);
|
||||
* sqlite3_exec (ses, "PRAGMA page_size = 4096;", NULL, NULL, NULL);
|
||||
* @endcode
|
||||
*/
|
||||
operator ::sqlite3* () const {return db->handler;}
|
||||
};
|
||||
|
||||
/**
|
||||
* Execute the given query, throwing SqliteEx on failure.\n
|
||||
* Example:\code
|
||||
* glim::Sqlite sqlite (":memory:");
|
||||
* sqlite.exec ("PRAGMA page_size = 4096") .exec ("PRAGMA secure_delete = 1"); \endcode
|
||||
*/
|
||||
inline Sqlite& Sqlite::exec (const char* query) {
|
||||
SqliteSession ses (this); // Maintains the locks.
|
||||
char* errmsg = NULL; ::sqlite3_exec (handler, query, NULL, NULL, &errmsg);
|
||||
if (errmsg) throw SqliteEx (std::string ("Sqlite::exec, error in query (") + query + "): " + errmsg);
|
||||
return *this;
|
||||
}
|
||||
|
||||
/**
|
||||
* Wraps the sqlite3_stmt; will prepare it, bind values, query and finalize.
|
||||
*/
|
||||
class SqliteQuery {
|
||||
protected:
|
||||
::sqlite3_stmt* statement;
|
||||
SqliteSession* session;
|
||||
int bindCounter;
|
||||
/// -1 if statement isn't DONE.
|
||||
int mChanges;
|
||||
void prepare (SqliteSession* session, char const* query, int queryLength) {
|
||||
::sqlite3* handler = *session;
|
||||
if (::sqlite3_prepare_v2 (handler, query, queryLength, &statement, NULL) != SQLITE_OK)
|
||||
throw SqliteEx (std::string(query, queryLength) + ": " + ::sqlite3_errmsg(handler));
|
||||
}
|
||||
/** Shan't copy. */
|
||||
SqliteQuery (const SqliteQuery& other) = delete;
|
||||
public:
|
||||
SqliteQuery (SqliteQuery&& rvalue) {
|
||||
statement = rvalue.statement;
|
||||
session = rvalue.session;
|
||||
bindCounter = rvalue.bindCounter;
|
||||
mChanges = rvalue.mChanges;
|
||||
rvalue.statement = nullptr;
|
||||
}
|
||||
/**
|
||||
* Prepares the query.
|
||||
* @throws SqliteEx if sqlite3_prepare fails; format of the error message is "$query: $errmsg".
|
||||
*/
|
||||
SqliteQuery (SqliteSession* session, char const* query, int queryLength)
|
||||
: statement (NULL), session (session), bindCounter (0), mChanges (-1) {
|
||||
prepare (session, query, queryLength);
|
||||
}
|
||||
/**
|
||||
* Prepares the query.
|
||||
* @throws SqliteEx if sqlite3_prepare fails; format of the error message is "$query: $errmsg".
|
||||
*/
|
||||
SqliteQuery (SqliteSession* session, std::pair<char const*, int> query)
|
||||
: statement (NULL), session (session), bindCounter (0), mChanges (-1) {
|
||||
prepare (session, query.first, query.second);
|
||||
}
|
||||
/**
|
||||
* Prepares the query.
|
||||
* @throws SqliteEx if sqlite3_prepare fails; format of the error message is "$query: $errmsg".
|
||||
*/
|
||||
SqliteQuery (SqliteSession* session, std::string query)
|
||||
: statement (NULL), session (session), bindCounter (0), mChanges (-1) {
|
||||
prepare (session, query.c_str(), query.length());
|
||||
}
|
||||
/**
|
||||
* Release resources.
|
||||
* @see http://sqlite.org/capi3ref.html#sqlite3_finalize
|
||||
*/
|
||||
~SqliteQuery () {
|
||||
if (statement) ::sqlite3_finalize (statement);
|
||||
}
|
||||
|
||||
/// Call this (followed by the #step) if you need the query to be re-executed.
|
||||
/// @see http://sqlite.org/capi3ref.html#sqlite3_reset
|
||||
SqliteQuery& reset () {
|
||||
bindCounter = 0;
|
||||
mChanges = -1;
|
||||
::sqlite3_reset (statement);
|
||||
return *this;
|
||||
}
|
||||
|
||||
/// Synonym for #step.
|
||||
bool next () {return step();}
|
||||
/**
|
||||
* Invokes sqlite3_step.
|
||||
* @return \c true if there was a row fetched successfully, \c false if there is no more rows.
|
||||
* @see http://sqlite.org/capi3ref.html#sqlite3_step
|
||||
*/
|
||||
bool step () {
|
||||
if (mChanges >= 0) {mChanges = 0; return false;}
|
||||
int ret = ::sqlite3_step (statement);
|
||||
if (ret == SQLITE_ROW) return true;
|
||||
if (ret == SQLITE_DONE) {
|
||||
mChanges = ::sqlite3_changes (*session);
|
||||
return false;
|
||||
}
|
||||
throw SqliteEx (std::string(::sqlite3_errmsg(*session)));
|
||||
}
|
||||
/**
|
||||
* Perform #step and throw an exception if #step has returned \c false.
|
||||
* Usage example:
|
||||
* \code (ses.query(S("select count(*) from test where idx = ?")) << 12345).qstep().intAt(1) \endcode
|
||||
*/
|
||||
SqliteQuery& qstep () {
|
||||
if (!step())
|
||||
throw SqliteEx (std::string("qstep: no rows returned / affected"));
|
||||
return *this;
|
||||
}
|
||||
/**
|
||||
* Invokes a DML query and returns the number of rows affected.
|
||||
* Example: \code
|
||||
* int affected = (ses.query(S("update test set count = count + ? where id = ?")) << 1 << 9).ustep();
|
||||
* \endcode
|
||||
* @see http://sqlite.org/capi3ref.html#sqlite3_step
|
||||
*/
|
||||
int ustep () {
|
||||
int ret = ::sqlite3_step (statement);
|
||||
if (ret == SQLITE_DONE) {
|
||||
mChanges = ::sqlite3_changes (*session);
|
||||
return mChanges;
|
||||
}
|
||||
if (ret == SQLITE_ROW) return 0;
|
||||
throw SqliteEx (std::string(::sqlite3_errmsg(*session)));
|
||||
}
|
||||
|
||||
/**
|
||||
* The number of rows changed by the query.
|
||||
* Providing the query was a DML (Data Modification Language),
|
||||
* returns the number of rows updated.\n
|
||||
* If the query wasn't a DML, returned value is undefined.\n
|
||||
* -1 is returned if the query wasn't executed, or after #reset.\n
|
||||
* Example: \code
|
||||
* SqliteQuery query (&ses, S("update test set count = count + ? where id = ?"));
|
||||
* query.bind (1, 1);
|
||||
* query.bind (2, 9);
|
||||
* query.step ();
|
||||
* int affected = query.changes ();
|
||||
* \endcode
|
||||
* @see #ustep
|
||||
*/
|
||||
int changes () {return mChanges;}
|
||||
|
||||
/**
|
||||
* The integer value of the given column.
|
||||
* @param column 1-based.
|
||||
* @see http://sqlite.org/capi3ref.html#sqlite3_column_text
|
||||
*/
|
||||
int intAt (int column) {
|
||||
return ::sqlite3_column_int (statement, --column);
|
||||
}
|
||||
|
||||
/**
|
||||
* The integer value of the given column.
|
||||
* @param column 1-based.
|
||||
* @see http://sqlite.org/capi3ref.html#sqlite3_column_text
|
||||
*/
|
||||
sqlite3_int64 int64at (int column) {
|
||||
return ::sqlite3_column_int64 (statement, --column);
|
||||
}
|
||||
|
||||
/**
|
||||
* The floating point number from the given column.
|
||||
* @param column 1-based.
|
||||
* @see http://sqlite.org/capi3ref.html#sqlite3_column_text
|
||||
*/
|
||||
double doubleAt (int column) {
|
||||
return ::sqlite3_column_double (statement, --column);
|
||||
}
|
||||
|
||||
/**
|
||||
* Return the column as UTF-8 characters, which can be used until the next #step.
|
||||
* @param column 1-based.
|
||||
* @see http://sqlite.org/capi3ref.html#sqlite3_column_text
|
||||
*/
|
||||
std::pair<char const*, int> charsAt (int column) {
|
||||
return std::pair<char const*, int> ((char const*) ::sqlite3_column_text (statement, column-1),
|
||||
::sqlite3_column_bytes (statement, column-1));
|
||||
}
|
||||
|
||||
/**
|
||||
* Return the column as C++ string (UTF-8).
|
||||
* @param column 1-based.
|
||||
*/
|
||||
std::string stringAt (int column) {
|
||||
return std::string ((char const*) ::sqlite3_column_text (statement, column-1),
|
||||
::sqlite3_column_bytes (statement, column-1));
|
||||
}
|
||||
|
||||
/**
|
||||
* The type of the column.
|
||||
* SQLITE_INTEGER, SQLITE_FLOAT, SQLITE_TEXT, SQLITE_BLOB or SQLITE_NULL.
|
||||
* @param column 1-based.
|
||||
* @see http://sqlite.org/capi3ref.html#sqlite3_column_text
|
||||
*/
|
||||
int typeAt (int column) {
|
||||
return ::sqlite3_column_type (statement, --column);
|
||||
}
|
||||
|
||||
/**
|
||||
* Binds a value using one of the bind methods.
|
||||
*/
|
||||
template<typename T>
|
||||
SqliteQuery& operator << (T value) {
|
||||
return bind (++bindCounter, value);
|
||||
}
|
||||
/**
|
||||
* Binds a value using the named parameter and one of the bind methods.
|
||||
* @throws SqliteEx if the name could not be found.
|
||||
* @see http://sqlite.org/capi3ref.html#sqlite3_bind_parameter_index
|
||||
*/
|
||||
template<typename T>
|
||||
SqliteQuery& bind (char const* name, T value) {
|
||||
int index = ::sqlite3_bind_parameter_index (statement, name);
|
||||
if (index == 0)
|
||||
throw SqliteEx (std::string ("No such parameter in the query: ") + name);
|
||||
return bind (index, value);
|
||||
}
|
||||
|
||||
/**
|
||||
* Bind a string to the query.
|
||||
* @param transient must be true, if lifetime of the string might be shorter than that of the query.
|
||||
*/
|
||||
SqliteQuery& bind (int index, const char* text, int length, bool transient = false) {
|
||||
if (::sqlite3_bind_text (statement, index, text, length,
|
||||
transient ? SQLITE_TRANSIENT : SQLITE_STATIC) != SQLITE_OK)
|
||||
throw SqliteEx (std::string (::sqlite3_errmsg (*session)));
|
||||
return *this;
|
||||
}
|
||||
/**
|
||||
* Bind a string to the query.
|
||||
* @param transient must be true, if lifetime of the string might be shorter than that of the query.
|
||||
*/
|
||||
SqliteQuery& bind (int index, std::pair<const char*, int> text, bool transient = false) {
|
||||
if (::sqlite3_bind_text (statement, index, text.first, text.second,
|
||||
transient ? SQLITE_TRANSIENT : SQLITE_STATIC) != SQLITE_OK)
|
||||
throw SqliteEx (std::string (::sqlite3_errmsg (*session)));
|
||||
return *this;
|
||||
}
|
||||
/**
|
||||
* Bind a string to the query.
|
||||
* @param transient must be true, if lifetime of the string might be shorter than that of the query.
|
||||
*/
|
||||
SqliteQuery& bind (int index, const std::string& text, bool transient = true) {
|
||||
if (::sqlite3_bind_text (statement, index, text.data(), text.length(),
|
||||
transient ? SQLITE_TRANSIENT : SQLITE_STATIC) != SQLITE_OK)
|
||||
throw SqliteEx (std::string (::sqlite3_errmsg (*session)));
|
||||
return *this;
|
||||
}
|
||||
/**
|
||||
* Bind an integer to the query.
|
||||
*/
|
||||
SqliteQuery& bind (int index, int value) {
|
||||
if (::sqlite3_bind_int (statement, index, value) != SQLITE_OK)
|
||||
throw SqliteEx (std::string (::sqlite3_errmsg (*session)));
|
||||
return *this;
|
||||
}
|
||||
/**
|
||||
* Bind an 64-bit integer to the query.
|
||||
*/
|
||||
SqliteQuery& bind (int index, sqlite3_int64 value) {
|
||||
if (::sqlite3_bind_int64 (statement, index, value) != SQLITE_OK)
|
||||
throw SqliteEx (std::string (::sqlite3_errmsg (*session)));
|
||||
return *this;
|
||||
}
|
||||
};
|
||||
|
||||
/**
|
||||
* Version of SqliteQuery suitable for using SQLite in parallel with other processes.
|
||||
* Will automatically handle the SQLITE_SCHEMA error
|
||||
* and will automatically repeat attempts after SQLITE_BUSY,
|
||||
* but it requires that the query string supplied
|
||||
* is constant and available during the SqliteParQuery lifetime.
|
||||
* Error messages, contained in exceptions, may differ from SqliteQuery by containing the query
|
||||
* (for example, the #step method will throw "$query: $errmsg" instead of just "$errmsg").
|
||||
*/
|
||||
class SqliteParQuery: public SqliteQuery {
|
||||
protected:
|
||||
char const* query;
|
||||
int queryLength;
|
||||
int repeat;
|
||||
int wait;
|
||||
public:
|
||||
/**
|
||||
* Prepares the query.
|
||||
* @param repeat the number of times we try to repeat the query when SQLITE_BUSY is returned.
|
||||
* @param wait how long, in milliseconds (1/1000 of a second) we are to wait before repeating.
|
||||
* @throws SqliteEx if sqlite3_prepare fails; format of the error message is "$query: $errmsg".
|
||||
*/
|
||||
SqliteParQuery (SqliteSession* session, char const* query, int queryLength, int repeat = 90, int wait = 20)
|
||||
: SqliteQuery (session, query, queryLength) {
|
||||
this->query = query;
|
||||
this->queryLength = queryLength;
|
||||
this->repeat = repeat;
|
||||
this->wait = wait;
|
||||
}
|
||||
/**
|
||||
* Prepares the query.
|
||||
* @param query the SQL query together with its length.
|
||||
* @param repeat the number of times we try to repeat the query when SQLITE_BUSY is returned.
|
||||
* @param wait how long, in milliseconds (1/1000 of a second) we are to wait before repeating.
|
||||
* @throws SqliteEx if sqlite3_prepare fails; format of the error message is "$query: $errmsg".
|
||||
*/
|
||||
SqliteParQuery (SqliteSession* session, std::pair<char const*, int> query, int repeat = 90, int wait = 20)
|
||||
: SqliteQuery (session, query) {
|
||||
this->query = query.first;
|
||||
this->queryLength = query.second;
|
||||
this->repeat = repeat;
|
||||
this->wait = wait;
|
||||
}
|
||||
|
||||
bool next () {return step();}
|
||||
bool step () {
|
||||
if (mChanges >= 0) {mChanges = 0; return false;}
|
||||
repeat:
|
||||
int ret = ::sqlite3_step (statement);
|
||||
if (ret == SQLITE_ROW) return true;
|
||||
if (ret == SQLITE_DONE) {
|
||||
mChanges = ::sqlite3_changes (*session);
|
||||
return false;
|
||||
}
|
||||
if (ret == SQLITE_SCHEMA) {
|
||||
::sqlite3_stmt* old = statement;
|
||||
prepare (session, query, queryLength);
|
||||
::sqlite3_transfer_bindings(old, statement);
|
||||
::sqlite3_finalize (old);
|
||||
goto repeat;
|
||||
}
|
||||
if (ret == SQLITE_BUSY) for (int repeat = this->repeat; ret == SQLITE_BUSY && repeat >= 0; --repeat) {
|
||||
//struct timespec ts; ts.tv_sec = 0; ts.tv_nsec = wait * 1000000; // nan is 10^-9 of sec.
|
||||
//while (::nanosleep (&ts, &ts) == EINTR);
|
||||
::sqlite3_sleep (wait);
|
||||
ret = ::sqlite3_step (statement);
|
||||
}
|
||||
throw SqliteEx (std::string(query, queryLength) + ::sqlite3_errmsg(*session));
|
||||
}
|
||||
};
|
||||
|
||||
template <typename T>
|
||||
SqliteQuery SqliteSession::query (T t) {
|
||||
return SqliteQuery (this, t);
|
||||
}
|
||||
|
||||
}; // namespace glim
|
||||
|
||||
#endif // GLIM_SQLITE_HPP_
|
||||
89
external/glim/test_cbcoro.cc
vendored
Normal file
89
external/glim/test_cbcoro.cc
vendored
Normal file
@ -0,0 +1,89 @@
|
||||
// http://en.wikipedia.org/wiki/Setcontext; man 3 makecontext; man 2 getcontext
|
||||
// http://www.boost.org/doc/libs/1_53_0/libs/context/doc/html/index.html
|
||||
// g++ -std=c++11 -O1 -Wall -g test_cbcoro.cc -pthread && ./a.out
|
||||
|
||||
#include <glim/exception.hpp>
|
||||
#include <glim/NsecTimer.hpp>
|
||||
|
||||
#include "cbcoro.hpp"
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <unistd.h> // sleep
|
||||
#include <string.h> // strerror
|
||||
#include <errno.h>
|
||||
#include <functional>
|
||||
using std::function;
|
||||
#include <thread>
|
||||
#include <memory>
|
||||
using std::shared_ptr; using std::make_shared;
|
||||
#include <string>
|
||||
using std::string; using std::to_string;
|
||||
#include <iostream>
|
||||
using std::cout; using std::endl;
|
||||
|
||||
/** A typical remote service with callback. */
|
||||
void esDelete (int frople, std::function<void(int)> cb) {
|
||||
std::thread th ([cb,frople]() {
|
||||
cout << "esDelete: sleeping for a second" << endl;
|
||||
std::this_thread::sleep_for (std::chrono::seconds (1));
|
||||
cb (frople);
|
||||
}); th.detach();
|
||||
}
|
||||
|
||||
struct RemoveFroples: public glim::CBCoro {
|
||||
const char* _argument;
|
||||
RemoveFroples (const char* argument): _argument (argument) {
|
||||
cout << "RF: constructor" << endl;
|
||||
}
|
||||
virtual ~RemoveFroples() {puts ("~RemoveFroples");}
|
||||
virtual void run() override {
|
||||
for (int i = 1; i <= 4; ++i) {
|
||||
cout << "RF: Removing frople " << i << "..." << endl;
|
||||
int returnedFrople = 0;
|
||||
yieldForCallback ([this,i,&returnedFrople]() {
|
||||
if (i != 2) {
|
||||
// Sometimes we use a callback.
|
||||
esDelete (i, [this,&returnedFrople](int frople) {
|
||||
cout << "RF,CB: frople " << frople << "." << endl;
|
||||
returnedFrople = frople;
|
||||
invokeFromCallback();
|
||||
});
|
||||
} else {
|
||||
// Sometimes we don't use a callback.
|
||||
returnedFrople = 0;
|
||||
invokeFromCallback();
|
||||
}
|
||||
});
|
||||
cout << "RF: Returned from callback; _returnTo is: " << (intptr_t) _returnTo << "; frople " << returnedFrople << endl;
|
||||
}
|
||||
cout << "RF: finish! _returnTo is: " << (intptr_t) _returnTo << endl;
|
||||
};
|
||||
};
|
||||
|
||||
int main() {
|
||||
glim::cbCoro ([](glim::CBCoro* cbcoro) {
|
||||
cout << "main: run1, thread " << std::this_thread::get_id() << endl; // Runs on the `main` thread.
|
||||
cbcoro->yieldForCallback ([&]() {
|
||||
std::thread callbackThread ([&]() {
|
||||
std::this_thread::sleep_for (std::chrono::seconds (4));
|
||||
cbcoro->invokeFromCallback();
|
||||
}); callbackThread.detach();
|
||||
});
|
||||
cout << "main: run2, thread " << std::this_thread::get_id() << endl; // Runs on the `callbackThread`.
|
||||
});
|
||||
|
||||
(new RemoveFroples ("argument"))->start();
|
||||
cout << "main: returned from RemoveFroples" << endl;
|
||||
|
||||
glim::NsecTimer timer; const int ops = RUNNING_ON_VALGRIND ? 999 : 9999;
|
||||
for (int i = 0; i < ops; ++i) glim::cbCoro ([](glim::CBCoro* cbcoro) {});
|
||||
double speedEmpty = ops / timer.sec();
|
||||
timer.restart();
|
||||
for (int i = 0; i < ops; ++i) glim::cbCoro ([](glim::CBCoro* cbcoro) {cbcoro->yieldForCallback ([&]() {cbcoro->invokeFromCallback();});});
|
||||
double speedImmediate = ops / timer.sec();
|
||||
|
||||
sleep (5);
|
||||
cout << "speed: empty: " << speedEmpty << " o/s" << endl;
|
||||
cout << "speed: immediate: " << speedImmediate << " o/s" << endl;
|
||||
return 0;
|
||||
}
|
||||
73
external/glim/test_exception.cc
vendored
Normal file
73
external/glim/test_exception.cc
vendored
Normal file
@ -0,0 +1,73 @@
|
||||
#define _GLIM_ALL_EXCEPTIONS_CODE
|
||||
#include "exception.hpp"
|
||||
#include <iostream>
|
||||
#include <typeinfo>
|
||||
#include <assert.h>
|
||||
|
||||
// NB: Controlling exceptions across shared object (.so) boundaries is tested separately in frople/src/test.cpp/testExceptionControl.
|
||||
|
||||
static void testThrowLine() {
|
||||
int line = 0; std::string message; try {
|
||||
line = __LINE__; GTHROW ("message");
|
||||
} catch (const std::exception& ex) {
|
||||
message = ex.what();
|
||||
}
|
||||
//std::cout << message << ' ' << std::flush;
|
||||
assert (message.size());
|
||||
assert (std::string (message) .find (":" + std::to_string (line)) != std::string::npos);
|
||||
|
||||
line = 0; message.clear(); std::string name; try {
|
||||
line = __LINE__; G_DEFINE_EXCEPTION (FooEx); GNTHROW (FooEx, "foo");
|
||||
} catch (const std::exception& ex) {
|
||||
message = ex.what(); name = typeid (ex) .name();
|
||||
}
|
||||
//std::cout << "testThrowLine: " << message << ' ' << name << ' ' << std::flush;
|
||||
assert (message.size());
|
||||
assert (std::string (message) .find (":" + std::to_string (line)) != std::string::npos);
|
||||
assert (name.find ("FooEx") != std::string::npos);
|
||||
|
||||
message.clear(); try {
|
||||
glim::ExceptionControl plainWhat (glim::Exception::PLAIN_WHAT);
|
||||
GTHROW ("bar");
|
||||
} catch (const std::exception& ex) {
|
||||
message = ex.what();
|
||||
}
|
||||
assert (message == "bar");
|
||||
assert (glim::Exception::options() == 0);
|
||||
}
|
||||
|
||||
static void testBacktrace() {
|
||||
assert (glim::Exception::options() == 0);
|
||||
glim::ExceptionControl captureTrace (glim::Exception::CAPTURE_TRACE);
|
||||
assert (glim::Exception::options() != 0);
|
||||
std::string message;
|
||||
try {
|
||||
GTHROW ("message");
|
||||
} catch (const std::exception& ex) {
|
||||
message = ex.what();
|
||||
}
|
||||
//std::cout << "testBacktrace: " << message << std::endl;
|
||||
if (message.find ("[at bin/test_exception") == std::string::npos && message.find ("[test_exception") == std::string::npos)
|
||||
GTHROW ("No expected string in " + message);
|
||||
}
|
||||
|
||||
static void testAllExceptionsHack() {
|
||||
assert (glim::Exception::options() == 0);
|
||||
std::string traceBuf;
|
||||
glim::ExceptionHandler traceExceptions (glim::Exception::HANDLE_ALL | glim::Exception::RENDEZVOUS, glim::captureBacktrace, &traceBuf);
|
||||
assert (glim::Exception::options() != 0);
|
||||
try {
|
||||
throw "catch me"; // Catched by `_GLIM_ALL_EXCEPTIONS_CODE` and handled with `glim::ExceptionControl::backtrace`.
|
||||
} catch (const char* skip) {}
|
||||
//std::cout << "testAllExceptionsHack: " << std::endl << traceBuf << std::endl;
|
||||
assert (traceBuf.size());
|
||||
}
|
||||
|
||||
int main () {
|
||||
std::cout << "Testing exception.hpp ... " << std::flush;
|
||||
testThrowLine();
|
||||
testBacktrace();
|
||||
testAllExceptionsHack();
|
||||
std::cout << "pass." << std::endl;
|
||||
return 0;
|
||||
}
|
||||
131
external/glim/test_gstring.cc
vendored
Normal file
131
external/glim/test_gstring.cc
vendored
Normal file
@ -0,0 +1,131 @@
|
||||
#include "gstring.hpp"
|
||||
using glim::gstring;
|
||||
#include <assert.h>
|
||||
#include <stdlib.h>
|
||||
#include <iostream>
|
||||
#include <string>
|
||||
#include <stdexcept>
|
||||
#include <sstream>
|
||||
#include <unordered_map>
|
||||
|
||||
#include <boost/algorithm/string/predicate.hpp>
|
||||
#include <boost/algorithm/string/trim.hpp>
|
||||
#include <boost/algorithm/string/case_conv.hpp>
|
||||
|
||||
static void testIterators();
|
||||
static void testBoost();
|
||||
static void testStrftime();
|
||||
|
||||
int main () {
|
||||
std::cout << "Testing gstring.hpp ... " << std::flush;
|
||||
|
||||
gstring gs;
|
||||
if (gs.needsFreeing()) throw std::runtime_error ("Default gstring needsFreeing");
|
||||
if (gs.capacity() != 1) throw std::runtime_error ("Default gstring capacity is not 1");
|
||||
char buf16[16];
|
||||
gstring gs16 (sizeof (buf16), buf16, false, 0);
|
||||
if (gs16.capacity() != 16) throw std::runtime_error ("gs16 capacity != 16");
|
||||
if (gs16.size() != 0) throw std::runtime_error ("gs16 size != 0");
|
||||
gstring gsFree (17, NULL, true, 0);
|
||||
if (!gsFree.needsFreeing()) throw std::runtime_error ("!needsFreeing");
|
||||
if (gsFree.capacity() != 16) throw std::runtime_error ("gsFree capacity != 16");
|
||||
if (gsFree.size() != 0) throw std::runtime_error ("gsFree size != 0");
|
||||
gstring gsRO (0, NULL, false, 0);
|
||||
if (gsRO.needsFreeing()) throw std::runtime_error ("needsFreeing");
|
||||
if (gsRO.capacity() != 1) throw std::runtime_error ("gsRO capacity != 1");
|
||||
if (gsRO.size() != 0) throw std::runtime_error ("gsRO size != 0");
|
||||
char buf32[32];
|
||||
gstring gs32 (sizeof (buf32), buf32, false, 0);
|
||||
if (gs32.capacity() != 32) throw std::runtime_error ("capacity != 32");
|
||||
if (gs32.size() != 0) throw std::runtime_error ("gs32 size != 0");
|
||||
const gstring foo = C2GSTRING ("foo");
|
||||
if (foo.needsFreeing()) throw std::runtime_error ("foo needsFreeing");
|
||||
if (foo != "foo") throw std::runtime_error ("foo != foo");
|
||||
if (foo.size() != 3) throw std::runtime_error ("foo not 3");
|
||||
std::ostringstream oss; oss << gs16 << gsFree << gsRO << gs32 << foo;
|
||||
if (oss.str() != "foo") throw std::runtime_error ("oss foo != foo");
|
||||
glim::gstring_stream gss (gs16); std::ostream gsos (&gss);
|
||||
gsos << "bar" << std::flush;
|
||||
if (gs16 != "bar") throw std::runtime_error ("gs16 != bar");
|
||||
gsos << "beer" << std::flush;
|
||||
if (gs16 != "barbeer") throw std::runtime_error ("gs16 != barbeer");
|
||||
gsos << "123456789" << std::flush;
|
||||
if (gs16 != "barbeer123456789") throw std::runtime_error ("gs16 != barbeer123456789");
|
||||
if (gs16.capacity() != 16) throw std::runtime_error ("gs16 != 16");
|
||||
gsos << '0' << std::flush;
|
||||
if (gs16 != "barbeer1234567890") throw std::runtime_error ("gs16 != barbeer1234567890");
|
||||
if (gs16.capacity() != 32) throw std::runtime_error ("gs16 != 32");
|
||||
|
||||
gstring gsb; std::string str ("abc");
|
||||
gsb << 'a' << 1 << 2LL << str;
|
||||
std::string ns ("1:3,"); std::istringstream nsi (ns);
|
||||
gsb.readNetstring (nsi);
|
||||
if (gsb != "a12abc3") throw std::runtime_error ("gsb != a12abc3");
|
||||
if (strcmp (gsb.c_str(), "a12abc3") != 0) throw std::runtime_error ("strcmp ! 0");
|
||||
|
||||
gsb.clear().appendNetstring ("foo") .appendNetstring ("bar");
|
||||
if (gsb != "3:foo,3:bar,") throw std::runtime_error ("gsb != 3:foo,3:bar,");
|
||||
uint32_t pos = 0;
|
||||
if (gsb.netstringAt (pos, &pos) != "foo" || gsb.netstringAt (pos, &pos) != "bar" || pos != gsb.length())
|
||||
throw std::runtime_error ("gsb !netstringAt");
|
||||
|
||||
gs32.clear() << 12345 << ',';
|
||||
if (gs32.intAt (0, &pos) != 12345 || pos != 5) throw std::runtime_error ("gsb !12345");
|
||||
if (gs32.intAt (1, &pos) != 2345 || pos != 5) throw std::runtime_error ("gsb !2345");
|
||||
if (gs32.intAt (5, &pos) != 0 || pos != 5) throw std::runtime_error ("gsb !0");
|
||||
|
||||
if ((gs32.clear() << 123).erase (0) != "23") throw std::runtime_error ("!23");
|
||||
if ((gs32.clear() << 123).erase (1) != "13") throw std::runtime_error ("!13");
|
||||
if ((gs32.clear() << 123).erase (2) != "12") throw std::runtime_error ("!12");
|
||||
|
||||
std::unordered_map<glim::gstring, int> map;
|
||||
map[glim::gstring ("foo")] = 1;
|
||||
glim::gstring bar ("bar");
|
||||
map[bar] = 1;
|
||||
map[glim::gstring ("sum")] = map[glim::gstring ("foo")] + map[glim::gstring ("bar")];
|
||||
if (map[glim::gstring ("sum")] != 2) throw std::runtime_error ("sum != 2");
|
||||
map.clear();
|
||||
|
||||
gstring gs1 ("foo"); gstring gs2 ("bar");
|
||||
gs1 = gstring (gs2 << "_"); // Copying in order to malloc length() bytes.
|
||||
if (gs1 != "bar_") throw std::runtime_error ("!bar_");
|
||||
if (gs1.capacity() != 1) throw std::runtime_error ("bar_ != 4");
|
||||
|
||||
testIterators();
|
||||
testBoost();
|
||||
testStrftime();
|
||||
|
||||
std::cout << "pass." << std::endl;
|
||||
return 0;
|
||||
}
|
||||
|
||||
static void testIterators() {
|
||||
gstring foo (C2GSTRING ("foo"));
|
||||
gstring buf; for (auto it = foo.begin(), end = foo.end(); it != end; ++it) buf << *it;
|
||||
assert (buf == "foo");
|
||||
assert (boost::starts_with (foo, "f") && boost::ends_with (foo, "oo"));
|
||||
}
|
||||
|
||||
static void testBoost() {
|
||||
gstring str (" foo\t\r\n");
|
||||
boost::trim (str);
|
||||
assert (str == "foo");
|
||||
|
||||
gstring up ("FOO"); boost::to_lower (up);
|
||||
assert (up == "foo");
|
||||
}
|
||||
|
||||
static void testStrftime() {
|
||||
time_t tim = time(0); struct tm ltime; memset (<ime, 0, sizeof ltime); if (!localtime_r (&tim, <ime)) GTHROW ("!localtime_r");
|
||||
GSTRING_ON_STACK (t1, 8); assert (t1.capacity() == 8);
|
||||
t1.appendTime ("", <ime); assert (t1 == "");
|
||||
t1.appendTime ("foo %a, %d %b %Y %T %z bar", <ime);
|
||||
assert (t1.capacity() > 8); // Capacity increased to account for the large string.
|
||||
assert (boost::starts_with (t1, "foo "));
|
||||
assert (boost::ends_with (t1, " bar"));
|
||||
|
||||
GSTRING_ON_STACK (t2, 8); assert (t2.capacity() == 8);
|
||||
t2.appendTime ("%H", <ime);
|
||||
assert (t2.capacity() == 8); // 8 is big enought, isn't it?
|
||||
assert (t2.intAt (0) == ltime.tm_hour); // NB: intAt is safe here because strftime adds an uncounted null-terminator.
|
||||
}
|
||||
165
external/glim/test_ldb.cc
vendored
Normal file
165
external/glim/test_ldb.cc
vendored
Normal file
@ -0,0 +1,165 @@
|
||||
#include "ldb.hpp"
|
||||
using glim::Ldb;
|
||||
using glim::gstring;
|
||||
#include <iostream>
|
||||
using std::cout; using std::flush; using std::endl;
|
||||
#include <assert.h>
|
||||
#include <boost/filesystem.hpp>
|
||||
|
||||
void test1 (Ldb& ldb) {
|
||||
ldb.put (std::string ("foo_"), std::string ("bar"));
|
||||
ldb.put ((uint32_t) 123, 1);
|
||||
ldb.put ((uint32_t) 123, 2);
|
||||
ldb.put (C2GSTRING ("foo"), 3);
|
||||
ldb.put (C2GSTRING ("foo"), 4);
|
||||
ldb.put (C2GSTRING ("gsk"), C2GSTRING ("gsv"));
|
||||
std::string ts; int ti; gstring tgs;
|
||||
auto fail = [](std::string msg) {throw std::runtime_error ("assertion failed: " + msg);};
|
||||
if (!ldb.get (std::string ("foo_"), ts) || ts != "bar") fail ("!foo_=bar");
|
||||
if (!ldb.get ((uint32_t) 123, ti) || ti != 2) fail ("!123=2");
|
||||
if (!ldb.get (C2GSTRING ("foo"), ti) || ti != 4) fail ("!foo=4");
|
||||
if (!ldb.get (C2GSTRING ("gsk"), tgs) || tgs != "gsv") fail ("!gsk=gsv");
|
||||
|
||||
// Test range-based for.
|
||||
int count = 0; bool haveGskGsv = false;
|
||||
for (auto&& entry: ldb) {
|
||||
if (!entry._lit->Valid()) fail ("!entry");
|
||||
if (entry.keyView() == "gsk") {
|
||||
if (entry.getKey<gstring>() != "gsk") fail ("getKey(gsk)!=gsk");
|
||||
if (entry.getValue<gstring>() != "gsv") fail ("getValue(gsk)!=gsv");
|
||||
haveGskGsv = true;
|
||||
}
|
||||
++count;}
|
||||
if (count != 4) fail ("count!=4"); // foo_=bar, 123=2, foo=4, gsk=gsv
|
||||
if (!haveGskGsv) fail ("!haveGskGsv");
|
||||
|
||||
ldb.del ((uint32_t) 123); if (ldb.get ((uint32_t) 123, ti)) fail ("123");
|
||||
ldb.del (C2GSTRING ("foo")); if (ldb.get (C2GSTRING ("foo"), ti)) fail ("foo");
|
||||
ldb.del (std::string ("foo_"));
|
||||
|
||||
{ // We've erased "123" and "foo", the only key left is "gsk" (gsk=gsv), let's test the iterator boundaries on this small dataset.
|
||||
auto&& it = ldb.begin();
|
||||
if (it->getKey<gstring>() != "gsk") fail ("first key !gsk " + it->keyView().str());
|
||||
if (!(++it).end()) fail ("++it != end");
|
||||
if ((--it)->getKey<gstring>() != "gsk") fail ("can't go back to gsk");
|
||||
if (!(--it).end()) fail ("--it != end");
|
||||
if ((++it)->getKey<gstring>() != "gsk") fail ("can't go forward to gsk");
|
||||
}
|
||||
|
||||
// todo: index trigger example
|
||||
// struct SimpleIndexTrigger: public Trigger { // Uses key space partitioning (cf. http://stackoverflow.com/a/12503799/257568)
|
||||
// const char* _name; Ldb _indexDb;
|
||||
// SimpleIndexTrigger (Ldb& ldb, const char* name = "index"): _name (name), _indexDb (ldb._env, name) {}
|
||||
// gstring triggerName() {return gstring (0, (void*) _name, false, strlen (_name), true);}
|
||||
// void add (Ldb& ldb, void* key, gstring& kbytes, void* value, gstring& vbytes, Transaction& txn) {
|
||||
// MDB_val mkey = {vbytes.size(), (void*) vbytes.data()};
|
||||
// MDB_val mvalue = {kbytes.size(), (void*) kbytes.data()};
|
||||
// int rc = ::ldb_put (txn.get(), _indexDb._dbi, &mkey, &mvalue, 0);
|
||||
// if (rc) GNTHROW (LdbEx, std::string ("index, ldb_put: ") + ::strerror (rc));
|
||||
// }
|
||||
// void erase (Ldb& ldb, void* ekey, gstring& kbytes, Transaction& txn) {
|
||||
// // Get all the values and remove them from the index.
|
||||
// MDB_cursor* cur = 0; int rc = ::ldb_cursor_open (txn.get(), ldb._dbi, &cur);
|
||||
// if (rc) GNTHROW (LdbEx, std::string ("index, erase, ldb_cursor_open: ") + ::strerror (rc));
|
||||
// std::unique_ptr<MDB_cursor, void(*)(MDB_cursor*)> curHolder (cur, ::ldb_cursor_close);
|
||||
// MDB_val mkey = {kbytes.size(), (void*) kbytes.data()}, val = {0, 0};
|
||||
// rc = ::ldb_cursor_get (cur, &mkey, &val, ::MDB_SET_KEY); if (rc == MDB_NOTFOUND) return;
|
||||
// if (rc) GNTHROW (LdbEx, std::string ("index, erase, ldb_cursor_get: ") + ::strerror (rc));
|
||||
// rc = ::ldb_del (txn.get(), _indexDb._dbi, &val, &mkey);
|
||||
// if (rc && rc != MDB_NOTFOUND) GNTHROW (LdbEx, std::string ("index, erase, ldb_del: ") + ::strerror (rc));
|
||||
// for (;;) {
|
||||
// rc = ::ldb_cursor_get (cur, &mkey, &val, ::MDB_NEXT_DUP); if (rc == MDB_NOTFOUND) return;
|
||||
// if (rc) GNTHROW (LdbEx, std::string ("index, erase, ldb_cursor_get: ") + ::strerror (rc));
|
||||
// rc = ::ldb_del (txn.get(), _indexDb._dbi, &val, &mkey);
|
||||
// if (rc && rc != MDB_NOTFOUND) GNTHROW (LdbEx, std::string ("index, erase, ldb_del: ") + ::strerror (rc));
|
||||
// }
|
||||
// }
|
||||
// void eraseKV (Ldb& ldb, void* key, gstring& kbytes, void* value, gstring& vbytes, Transaction& txn) {
|
||||
// MDB_val mkey = {vbytes.size(), (void*) vbytes.data()};
|
||||
// MDB_val mvalue = {kbytes.size(), (void*) kbytes.data()};
|
||||
// int rc = ::ldb_del (txn.get(), _indexDb._dbi, &mkey, &mvalue);
|
||||
// if (rc && rc != MDB_NOTFOUND) GNTHROW (LdbEx, std::string ("index, ldb_del: ") + ::strerror (rc));
|
||||
// }
|
||||
// };
|
||||
// auto indexTrigger = std::make_shared<SimpleIndexTrigger> (ldb); ldb.setTrigger (indexTrigger); auto& indexDb = indexTrigger->_indexDb;
|
||||
// ldb.erase (C2GSTRING ("gsk")); // NB: "gsk" wasn't indexed here. `IndexTrigger.erase` should handle this gracefully.
|
||||
|
||||
// Add indexed.
|
||||
// ldb.put (C2GSTRING ("ik"), C2GSTRING ("iv1"));
|
||||
// ldb.put (C2GSTRING ("ik"), string ("iv2"));
|
||||
// ldb.put (C2GSTRING ("ik"), 3);
|
||||
// Check the index.
|
||||
// gstring ik;
|
||||
// if (!indexDb.first (C2GSTRING ("iv1"), ik) || ik != "ik") fail ("!iv1=ik");
|
||||
// if (!indexDb.first (string ("iv2"), ik) || ik != "ik") fail ("!iv2=ik");
|
||||
// if (!indexDb.first (3, ik) || ik != "ik") fail ("!iv3=ik");
|
||||
|
||||
// Remove indexed.
|
||||
// ldb.eraseKV (C2GSTRING ("ik"), string ("iv2"));
|
||||
// Check the index.
|
||||
// if (!indexDb.first (C2GSTRING ("iv1"), ik) || ik != "ik") fail ("!iv1=ik");
|
||||
// if (indexDb.first (string ("iv2"), ik)) fail ("iv2=ik");
|
||||
// if (!indexDb.first (3, ik) || ik != "ik") fail ("!iv3=ik");
|
||||
|
||||
// Remove indexed.
|
||||
// ldb.erase (C2GSTRING ("ik"));
|
||||
// Check the index.
|
||||
// if (indexDb.first (C2GSTRING ("iv1"), ik)) fail ("iv1");
|
||||
// if (indexDb.first (3, ik)) fail ("iv3");
|
||||
// Check the data.
|
||||
// if (ldb.first (C2GSTRING ("ik"), ik)) fail ("ik");
|
||||
}
|
||||
|
||||
void testStartsWith (Ldb& ldb) {
|
||||
// Using `gstring`s because the Boost Serialization encoding for CStrings is not prefix-friendly.
|
||||
ldb.put (C2GSTRING ("01"), ""); ldb.put (C2GSTRING ("02"), "");
|
||||
ldb.put (C2GSTRING ("11"), "");
|
||||
ldb.put (C2GSTRING ("21"), ""); ldb.put (C2GSTRING ("222"), ""); ldb.put (C2GSTRING ("2"), "");
|
||||
|
||||
auto range = ldb.startsWith (C2GSTRING ("0")); auto it = range.begin();
|
||||
assert (it->keyView() == "01"); assert (it != range.end());
|
||||
assert ((++it)->keyView() == "02"); assert (it != range.end());
|
||||
assert ((++it)->keyView().empty()); assert (it == range.end());
|
||||
assert ((--it)->keyView() == "02"); assert (it != range.end());
|
||||
assert ((--it)->keyView() == "01"); assert (it != range.end());
|
||||
assert ((--it)->keyView().empty()); assert (it == range.end());
|
||||
// `it` and `range.begin` point to the same `leveldb::Iterator`.
|
||||
assert (range.begin()._entry->_lit == it._entry->_lit);
|
||||
assert (!range.begin()._entry->_valid); assert (range.begin()->keyView().empty());
|
||||
|
||||
range = ldb.startsWith (C2GSTRING ("0")); it = range.end();
|
||||
assert (it.end() && it->keyView().empty()); assert (it != range.begin());
|
||||
assert ((--it)->keyView() == "02"); assert (it != range.begin());
|
||||
assert ((--it)->keyView() == "01"); assert (it == range.begin());
|
||||
assert ((--it)->keyView().empty()); assert (it != range.begin());
|
||||
|
||||
int8_t count = 0; for (auto& en: ldb.startsWith (C2GSTRING ("1"))) {en.keyView(); ++count;} assert (count == 1);
|
||||
count = 0; for (auto& en: ldb.startsWith (C2GSTRING ("2"))) {en.keyView(); ++count;} assert (count == 3);
|
||||
count = 0; for (auto& en: ldb.startsWith (C2GSTRING ("-"))) {en.keyView(); ++count;} assert (count == 0);
|
||||
count = 0; for (auto& en: ldb.startsWith (C2GSTRING (""))) {en.keyView(); ++count;} assert (count == 6);
|
||||
|
||||
assert (ldb.startsWith (C2GSTRING ("-")) .empty());
|
||||
|
||||
count = 0; for (auto& en: boost::make_iterator_range (ldb.end().seek ("1"), ldb.end().seek ("2"))) {en.keyView(); ++count;} assert (count == 1);
|
||||
count = 0; for (auto& en: boost::make_iterator_range (ldb.end().seek ("2"), ldb.end().seek ("3"))) {en.keyView(); ++count;} assert (count == 3);
|
||||
count = 0; for (auto& en: ldb.range (C2GSTRING ("1"), C2GSTRING ("2"))) {en.keyView(); ++count;} assert (count == 1);
|
||||
|
||||
{ auto range = ldb.range (C2GSTRING ("0"), C2GSTRING ("1")); // 01 and 02, but not 11.
|
||||
count = 0; for (auto& en: range) {en.keyView(); ++count;} assert (count == 2); }
|
||||
}
|
||||
|
||||
int main() {
|
||||
cout << "Testing ldb.hpp ... " << flush;
|
||||
boost::filesystem::remove_all ("/dev/shm/ldbTest");
|
||||
|
||||
Ldb ldb ("/dev/shm/ldbTest");
|
||||
test1 (ldb);
|
||||
|
||||
for (auto& en: ldb) ldb.del (en.keyView());
|
||||
testStartsWith (ldb);
|
||||
|
||||
ldb._db.reset(); // Close.
|
||||
boost::filesystem::remove_all ("/dev/shm/ldbTest");
|
||||
cout << "pass." << endl;
|
||||
return 0;
|
||||
}
|
||||
74
external/glim/test_runner.cc
vendored
Normal file
74
external/glim/test_runner.cc
vendored
Normal file
@ -0,0 +1,74 @@
|
||||
#include "runner.hpp"
|
||||
#include "curl.hpp"
|
||||
#include <future>
|
||||
#include <thread>
|
||||
#include <iostream>
|
||||
#include <assert.h>
|
||||
|
||||
static void testRunner() {
|
||||
using glim::RunnerV2;
|
||||
auto runner = RunnerV2::instance();
|
||||
|
||||
struct Dugout: public glim::CurlmInformationListener {
|
||||
glim::Curl _curl;
|
||||
std::promise<std::string> _promise;
|
||||
void schedule (RunnerV2* runner) {
|
||||
_curl.http ("http://glim.ru/", 2);
|
||||
curl_easy_setopt (_curl._curl, CURLOPT_PRIVATE, this); // Tells `addToCURLM` to call this listener later.
|
||||
runner->addToCURLM (_curl._curl);
|
||||
}
|
||||
virtual FreeOptions information (CURLMsg* msg, CURLM* curlm) override {
|
||||
_promise.set_value (_curl.str());
|
||||
return static_cast<FreeOptions> (REMOVE_CURL_FROM_CURLM | DELETE_LISTENER);
|
||||
}
|
||||
virtual ~Dugout() {}
|
||||
};
|
||||
{ Dugout* dugout = new Dugout();
|
||||
auto future = dugout->_promise.get_future();
|
||||
dugout->schedule (runner.get());
|
||||
if (future.get().find ("<html") == std::string::npos) GTHROW ("!html"); }
|
||||
|
||||
auto curl = std::make_shared<glim::Curl>(); curl->http ("http://glim.ru/", 2);
|
||||
auto promise = std::make_shared<std::promise<std::string>>(); auto future = promise->get_future();
|
||||
runner->addToCURLM (curl->_curl, [curl,promise](CURLMsg*, CURLM*) {promise->set_value (curl->str());});
|
||||
if (future.get().find ("<html") == std::string::npos) GTHROW ("!html");
|
||||
}
|
||||
|
||||
static void oldTest() {
|
||||
std::shared_ptr<struct event_base> evbase (event_base_new(), event_base_free);
|
||||
glim::Runner runner (evbase, [](const char* error) {std::cerr << error << std::endl;});
|
||||
|
||||
auto scheduledJobFired = std::make_shared<bool> (false);
|
||||
runner.schedule (0.f, [=](glim::Runner::JobInfo&)->bool {*scheduledJobFired = true; return false;});
|
||||
|
||||
auto curl = std::make_shared<glim::Curl> (false); curl->http ("http://glim.ru/env.cgi?pause=50", 5);
|
||||
auto curlDebug = std::make_shared<std::string>(); curl->debugListenerF ([curlDebug](const char* bytes, size_t size) {curlDebug->append (bytes, size);});
|
||||
volatile bool ran = false;
|
||||
runner.multi (curl->_curl, [curl,&ran,evbase,curlDebug](CURLMsg* msg) {
|
||||
std::cout << " status: " << curl->status();
|
||||
if (curl->status() == 200) std::cout << " ip: " << curl->gstr().view (0, std::max (curl->gstr().find ("\n"), 0));
|
||||
if (curlDebug->find ("GET /env.cgi") == std::string::npos) std::cerr << " No headers in debug? " << *curlDebug << std::endl;
|
||||
ran = true;
|
||||
event_base_loopbreak (evbase.get());
|
||||
});
|
||||
//struct timeval tv {1, 0}; event_base_loopexit (evbase.get(), &tv); // Exit the loop in a sec.
|
||||
event_base_dispatch (evbase.get());
|
||||
if (!ran) GTHROW ("!ran");
|
||||
if (!*scheduledJobFired) GTHROW ("!scheduledJobFired");
|
||||
|
||||
std::cout << " pass." << std::endl;
|
||||
//waiting: "was introduced in Libevent 2.1.1-alpha"//libevent_global_shutdown();
|
||||
}
|
||||
|
||||
int main () {
|
||||
std::cout << "Testing runner.hpp ..." << std::flush; try {
|
||||
|
||||
testRunner();
|
||||
oldTest();
|
||||
|
||||
} catch (const std::exception& ex) {
|
||||
std::cerr << " exception: " << ex.what() << std::endl;
|
||||
return 1;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
18
external/glim/test_sqlite.cc
vendored
Normal file
18
external/glim/test_sqlite.cc
vendored
Normal file
@ -0,0 +1,18 @@
|
||||
|
||||
#include "sqlite.hpp"
|
||||
#define S(cstr) (std::pair<char const*, int> (cstr, sizeof (cstr) - 1))
|
||||
#include <assert.h>
|
||||
#include <iostream>
|
||||
using namespace glim;
|
||||
|
||||
int main () {
|
||||
std::cout << "Testing sqlite.hpp ... " << std::flush;
|
||||
Sqlite sqlite (":memory:");
|
||||
SqliteSession sqs (&sqlite);
|
||||
assert (sqs.query ("CREATE TABLE test (t TEXT, i INTEGER)") .ustep() == 0);
|
||||
assert (sqs.query ("INSERT INTO test VALUES (?, ?)") .bind (1, S("foo")) .bind (2, 27) .ustep() == 1);
|
||||
assert (sqs.query ("SELECT t FROM test") .qstep() .stringAt (1) == "foo");
|
||||
assert (sqs.query ("SELECT i FROM test") .qstep() .intAt (1) == 27);
|
||||
std::cout << "pass." << std::endl;
|
||||
return 0;
|
||||
}
|
||||
Loading…
Reference in New Issue
Block a user