Git-Mirrors/RetroShare
1
0
Fork 0
mirror of https://github.com/RetroShare/RetroShare.git synced 2024-10-01 02:35:48 -04:00
Code Issues Actions Packages Projects Releases Wiki Activity

FeedReader: Don't add items as new item when pubDate less than storage time

Browse Source
This commit is contained in:
thunder2 2024-04-16 19:25:04 +02:00
parent 83b433a2e2
commit a60f3f44f7
1 changed files with 30 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

37
plugins/FeedReader/services/p3FeedReaderThread.cc
View File

@ -848,6 +848,22 @@ RsFeedReaderErrorState p3FeedReaderThread::process(const RsFeedReaderFeed &feed,
RsFeedReaderErrorState result = RS_FEED_ERRORSTATE_OK; RsFeedReaderErrorState result = RS_FEED_ERRORSTATE_OK;
time_t minimumPubDate = 0;
if (feed.lastUpdate == 0) {
// Get all items on first scan
} else {
// Get storage time
uint32_t storageTime = 0;
if (feed.flag & RS_FEED_FLAG_STANDARD_STORAGE_TIME) {
storageTime = mFeedReader->getStandardStorageTime();
} else {
storageTime = feed.storageTime;
}
if (storageTime > 0) {
minimumPubDate = time(NULL) - storageTime;
}
}
XMLWrapper xml; XMLWrapper xml;
if (xml.readXML(feed.content.c_str())) { if (xml.readXML(feed.content.c_str())) {
xmlNodePtr root = xml.getRootElement(); xmlNodePtr root = xml.getRootElement();
@ -1006,6 +1022,13 @@ RsFeedReaderErrorState p3FeedReaderThread::process(const RsFeedReaderFeed &feed,
} }
} }
if (minimumPubDate) {
if (item->pubDate < minimumPubDate) {
// pubDate is less than storage time, don't add as new item
continue;
}
}
entries.push_back(item); entries.push_back(item);
} }
} else { } else {
@ -1097,7 +1120,7 @@ RsFeedReaderErrorState p3FeedReaderThread::processMsg(const RsFeedReaderFeed &fe
std::string url; std::string url;
if (feed.flag & RS_FEED_FLAG_SAVE_COMPLETE_PAGE) { if (feed.flag & RS_FEED_FLAG_SAVE_COMPLETE_PAGE) {
#ifdef FEEDREADER_DEBUG #ifdef FEEDREADER_DEBUG
std::cerr << "p3FeedReaderThread::processHTML - feed " << feed.feedId << " (" << feed.name << ") download page " << msg->link << std::endl; std::cerr << "p3FeedReaderThread::processMsg - feed " << feed.feedId << " (" << feed.name << ") download page " << msg->link << std::endl;
#endif #endif
std::string content; std::string content;
CURLWrapper CURL(proxy); CURLWrapper CURL(proxy);
@ -1134,7 +1157,7 @@ RsFeedReaderErrorState p3FeedReaderThread::processMsg(const RsFeedReaderFeed &fe
if (result != RS_FEED_ERRORSTATE_OK) { if (result != RS_FEED_ERRORSTATE_OK) {
#ifdef FEEDREADER_DEBUG #ifdef FEEDREADER_DEBUG
std::cerr << "p3FeedReaderThread::processHTML - feed " << feed.feedId << " (" << feed.name << ") cannot download page, CURLCode = " << code << ", error = " << errorString << std::endl; std::cerr << "p3FeedReaderThread::processMsg - feed " << feed.feedId << " (" << feed.name << ") cannot download page, CURLCode = " << code << ", error = " << errorString << std::endl;
#endif #endif
return result; return result;
} }
@ -1280,7 +1303,7 @@ RsFeedReaderErrorState p3FeedReaderThread::processMsg(const RsFeedReaderFeed &fe
if (!src.empty()) { if (!src.empty()) {
/* download image */ /* download image */
#ifdef FEEDREADER_DEBUG #ifdef FEEDREADER_DEBUG
std::cerr << "p3FeedReaderThread::processHTML - feed " << feed.feedId << " (" << feed.name << ") download image " << src << std::endl; std::cerr << "p3FeedReaderThread::processMsg - feed " << feed.feedId << " (" << feed.name << ") download image " << src << std::endl;
#endif #endif
std::vector<unsigned char> data; std::vector<unsigned char> data;
CURLWrapper CURL(proxy); CURLWrapper CURL(proxy);
@ -1348,7 +1371,7 @@ RsFeedReaderErrorState p3FeedReaderThread::processMsg(const RsFeedReaderFeed &fe
if (!html.saveHTML(msg->postedDescriptionWithoutFirstImage)) { if (!html.saveHTML(msg->postedDescriptionWithoutFirstImage)) {
errorString = html.lastError(); errorString = html.lastError();
#ifdef FEEDREADER_DEBUG #ifdef FEEDREADER_DEBUG
std::cerr << "p3FeedReaderThread::processHTML - feed " << feed.feedId << " (" << feed.name << ") cannot dump html" << std::endl; std::cerr << "p3FeedReaderThread::processMsg - feed " << feed.feedId << " (" << feed.name << ") cannot dump html" << std::endl;
std::cerr << " Error: " << errorString << std::endl; std::cerr << " Error: " << errorString << std::endl;
#endif #endif
result = RS_FEED_ERRORSTATE_PROCESS_INTERNAL_ERROR; result = RS_FEED_ERRORSTATE_PROCESS_INTERNAL_ERROR;
@ -1357,7 +1380,7 @@ RsFeedReaderErrorState p3FeedReaderThread::processMsg(const RsFeedReaderFeed &fe
} else { } else {
errorString = html.lastError(); errorString = html.lastError();
#ifdef FEEDREADER_DEBUG #ifdef FEEDREADER_DEBUG
std::cerr << "p3FeedReaderThread::processHTML - feed " << feed.feedId << " (" << feed.name << ") cannot dump html" << std::endl; std::cerr << "p3FeedReaderThread::processMsg - feed " << feed.feedId << " (" << feed.name << ") cannot dump html" << std::endl;
std::cerr << " Error: " << errorString << std::endl; std::cerr << " Error: " << errorString << std::endl;
#endif #endif
result = RS_FEED_ERRORSTATE_PROCESS_INTERNAL_ERROR; result = RS_FEED_ERRORSTATE_PROCESS_INTERNAL_ERROR;
@ -1366,14 +1389,14 @@ RsFeedReaderErrorState p3FeedReaderThread::processMsg(const RsFeedReaderFeed &fe
} }
} else { } else {
#ifdef FEEDREADER_DEBUG #ifdef FEEDREADER_DEBUG
std::cerr << "p3FeedReaderThread::processHTML - feed " << feed.feedId << " (" << feed.name << ") no root element" << std::endl; std::cerr << "p3FeedReaderThread::processMsg - feed " << feed.feedId << " (" << feed.name << ") no root element" << std::endl;
#endif #endif
result = RS_FEED_ERRORSTATE_PROCESS_HTML_ERROR; result = RS_FEED_ERRORSTATE_PROCESS_HTML_ERROR;
} }
} else { } else {
errorString = html.lastError(); errorString = html.lastError();
#ifdef FEEDREADER_DEBUG #ifdef FEEDREADER_DEBUG
std::cerr << "p3FeedReaderThread::processHTML - feed " << feed.feedId << " (" << feed.name << ") cannot read html" << std::endl; std::cerr << "p3FeedReaderThread::processMsg - feed " << feed.feedId << " (" << feed.name << ") cannot read html" << std::endl;
std::cerr << " Error: " << errorString << std::endl; std::cerr << " Error: " << errorString << std::endl;
#endif #endif
result = RS_FEED_ERRORSTATE_PROCESS_HTML_ERROR; result = RS_FEED_ERRORSTATE_PROCESS_HTML_ERROR;