diff --git a/.gitignore b/.gitignore index 70977c8..670a70d 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,8 @@ +*.o +moc_* Makefile *.so *.ts *.qm -.moc -.obj +*.moc +*.obj diff --git a/src/common/socialnetworksyncadaptor.cpp b/src/common/socialnetworksyncadaptor.cpp index 7d9ece0..4574304 100644 --- a/src/common/socialnetworksyncadaptor.cpp +++ b/src/common/socialnetworksyncadaptor.cpp @@ -63,11 +63,12 @@ namespace { SocialNetworkSyncAdaptor::SocialNetworkSyncAdaptor(const QString &serviceName, SocialNetworkSyncAdaptor::DataType dataType, + QNetworkAccessManager *qnam, QObject *parent) : QObject(parent) , m_dataType(dataType) , m_accountManager(new Accounts::Manager(this)) - , m_networkAccessManager(new SocialdNetworkAccessManager(this)) + , m_networkAccessManager(qnam != 0 ? qnam : new SocialdNetworkAccessManager) , m_accountSyncProfile(NULL) , m_syncDb(new SocialNetworkSyncDatabase()) , m_status(SocialNetworkSyncAdaptor::Invalid) @@ -79,6 +80,7 @@ SocialNetworkSyncAdaptor::SocialNetworkSyncAdaptor(const QString &serviceName, SocialNetworkSyncAdaptor::~SocialNetworkSyncAdaptor() { + delete m_networkAccessManager; delete m_accountSyncProfile; delete m_syncDb; } diff --git a/src/common/socialnetworksyncadaptor.h b/src/common/socialnetworksyncadaptor.h index 308b21c..1e930ee 100644 --- a/src/common/socialnetworksyncadaptor.h +++ b/src/common/socialnetworksyncadaptor.h @@ -80,7 +80,7 @@ class SocialNetworkSyncAdaptor : public QObject static QString dataTypeName(DataType t); public: - SocialNetworkSyncAdaptor(const QString &serviceName, SocialNetworkSyncAdaptor::DataType dataType, QObject *parent); + SocialNetworkSyncAdaptor(const QString &serviceName, SocialNetworkSyncAdaptor::DataType dataType, QNetworkAccessManager *qnam, QObject *parent); virtual ~SocialNetworkSyncAdaptor(); virtual QString syncServiceName() const = 0; diff --git a/src/dropbox/dropboxdatatypesyncadaptor.cpp b/src/dropbox/dropboxdatatypesyncadaptor.cpp index 67df2e1..609b4fa 100644 --- a/src/dropbox/dropboxdatatypesyncadaptor.cpp +++ b/src/dropbox/dropboxdatatypesyncadaptor.cpp @@ -43,7 +43,7 @@ #include DropboxDataTypeSyncAdaptor::DropboxDataTypeSyncAdaptor(SocialNetworkSyncAdaptor::DataType dataType, QObject *parent) - : SocialNetworkSyncAdaptor("dropbox", dataType, parent), m_triedLoading(false) + : SocialNetworkSyncAdaptor("dropbox", dataType, 0, parent), m_triedLoading(false) { } diff --git a/src/facebook/facebookdatatypesyncadaptor.cpp b/src/facebook/facebookdatatypesyncadaptor.cpp index 4a1343f..dfefdda 100644 --- a/src/facebook/facebookdatatypesyncadaptor.cpp +++ b/src/facebook/facebookdatatypesyncadaptor.cpp @@ -43,7 +43,7 @@ #include FacebookDataTypeSyncAdaptor::FacebookDataTypeSyncAdaptor(SocialNetworkSyncAdaptor::DataType dataType, QObject *parent) - : SocialNetworkSyncAdaptor("facebook", dataType, parent), m_triedLoading(false) + : SocialNetworkSyncAdaptor("facebook", dataType, 0, parent), m_triedLoading(false) { } diff --git a/src/google/googledatatypesyncadaptor.cpp b/src/google/googledatatypesyncadaptor.cpp index bdcccca..e4dd822 100644 --- a/src/google/googledatatypesyncadaptor.cpp +++ b/src/google/googledatatypesyncadaptor.cpp @@ -43,7 +43,7 @@ #include GoogleDataTypeSyncAdaptor::GoogleDataTypeSyncAdaptor(SocialNetworkSyncAdaptor::DataType dataType, QObject *parent) - : SocialNetworkSyncAdaptor("google", dataType, parent), m_triedLoading(false) + : SocialNetworkSyncAdaptor("google", dataType, 0, parent), m_triedLoading(false) { } diff --git a/src/onedrive/onedrivedatatypesyncadaptor.cpp b/src/onedrive/onedrivedatatypesyncadaptor.cpp index ebb216c..26bd016 100644 --- a/src/onedrive/onedrivedatatypesyncadaptor.cpp +++ b/src/onedrive/onedrivedatatypesyncadaptor.cpp @@ -43,7 +43,7 @@ #include OneDriveDataTypeSyncAdaptor::OneDriveDataTypeSyncAdaptor(SocialNetworkSyncAdaptor::DataType dataType, QObject *parent) - : SocialNetworkSyncAdaptor("onedrive", dataType, parent), m_triedLoading(false) + : SocialNetworkSyncAdaptor("onedrive", dataType, 0, parent), m_triedLoading(false) { } diff --git a/src/twitter/twitterdatatypesyncadaptor.cpp b/src/twitter/twitterdatatypesyncadaptor.cpp index a938f68..02e95ee 100644 --- a/src/twitter/twitterdatatypesyncadaptor.cpp +++ b/src/twitter/twitterdatatypesyncadaptor.cpp @@ -50,7 +50,7 @@ #include TwitterDataTypeSyncAdaptor::TwitterDataTypeSyncAdaptor(SocialNetworkSyncAdaptor::DataType dataType, QObject *parent) - : SocialNetworkSyncAdaptor("twitter", dataType, parent), m_triedLoading(false) + : SocialNetworkSyncAdaptor("twitter", dataType, 0, parent), m_triedLoading(false) { } diff --git a/src/vk/vk-calendars/vkcalendarsyncadaptor.cpp b/src/vk/vk-calendars/vkcalendarsyncadaptor.cpp index 323f3a4..53a8853 100644 --- a/src/vk/vk-calendars/vkcalendarsyncadaptor.cpp +++ b/src/vk/vk-calendars/vkcalendarsyncadaptor.cpp @@ -223,6 +223,19 @@ void VKCalendarSyncAdaptor::beginSync(int accountId, const QString &accessToken) requestEvents(accountId, accessToken); } +void VKCalendarSyncAdaptor::retryThrottledRequest(const QString &request, const QVariantList &args, bool retryLimitReached) +{ + int accountId = args[0].toInt(); + if (retryLimitReached) { + SOCIALD_LOG_ERROR("hit request retry limit! unable to request data from VK account with id" << accountId); + setStatus(SocialNetworkSyncAdaptor::Error); + } else { + SOCIALD_LOG_DEBUG("retrying Calendars" << request << "request for VK account:" << accountId); + requestEvents(accountId, args[1].toString(), args[2].toInt()); + } + decrementSemaphore(accountId); // finished waiting for the request. +} + void VKCalendarSyncAdaptor::requestEvents(int accountId, const QString &accessToken, int offset) { QUrlQuery urlQuery; @@ -252,15 +265,21 @@ void VKCalendarSyncAdaptor::requestEvents(int accountId, const QString &accessTo incrementSemaphore(accountId); setupReplyTimeout(accountId, reply); } else { - SOCIALD_LOG_ERROR("unable to request events from VK account with id:" << accountId); + // request was throttled by VKNetworkAccessManager + QVariantList args; + args << accountId << accessToken << offset; + enqueueThrottledRequest(QStringLiteral("requestEvents"), args); + + // we are waiting to request data. Increment the semaphore so that we know we're still busy. + incrementSemaphore(accountId); // decremented in retryThrottledRequest(). } } void VKCalendarSyncAdaptor::finishedHandler() { QNetworkReply *reply = qobject_cast(sender()); - QString accessToken = reply->property("accessToken").toString(); int accountId = reply->property("accountId").toInt(); + QString accessToken = reply->property("accessToken").toString(); int offset = reply->property("offset").toInt(); QByteArray replyData = reply->readAll(); bool isError = reply->property("isError").toBool(); @@ -316,6 +335,15 @@ void VKCalendarSyncAdaptor::finishedHandler() SOCIALD_LOG_DEBUG("done fetching calendar results"); } } else { + QVariantList args; + args << accountId << accessToken << offset; + if (enqueueServerThrottledRequestIfRequired(parsed, QStringLiteral("requestEvents"), args)) { + // we hit the throttle limit, let throttle timer repeat the request + // don't decrement semaphore yet as we're still waiting for it. + // it will be decremented in retryThrottledRequest(). + return; + } + // error occurred during request. SOCIALD_LOG_ERROR("unable to parse calendar data from request with account" << accountId << "; got:" << QString::fromUtf8(replyData)); diff --git a/src/vk/vk-calendars/vkcalendarsyncadaptor.h b/src/vk/vk-calendars/vkcalendarsyncadaptor.h index 9b8abff..d135182 100644 --- a/src/vk/vk-calendars/vkcalendarsyncadaptor.h +++ b/src/vk/vk-calendars/vkcalendarsyncadaptor.h @@ -32,6 +32,7 @@ class VKCalendarSyncAdaptor : public VKDataTypeSyncAdaptor void beginSync(int accountId, const QString &accessToken); void finalize(int accountId); void finalCleanup(); + void retryThrottledRequest(const QString &request, const QVariantList &args, bool retryLimitReached); private: void requestEvents(int accountId, const QString &accessToken, int offset = 0); diff --git a/src/vk/vk-common.pri b/src/vk/vk-common.pri index 39bbaf1..ec545bc 100644 --- a/src/vk/vk-common.pri +++ b/src/vk/vk-common.pri @@ -1,3 +1,3 @@ INCLUDEPATH += $$PWD -SOURCES += $$PWD/vkdatatypesyncadaptor.cpp -HEADERS += $$PWD/vkdatatypesyncadaptor.h +SOURCES += $$PWD/vkdatatypesyncadaptor.cpp $$PWD/vknetworkaccessmanager.cpp +HEADERS += $$PWD/vkdatatypesyncadaptor.h $$PWD/vknetworkaccessmanager_p.h diff --git a/src/vk/vk-contacts/vkcontactsyncadaptor.cpp b/src/vk/vk-contacts/vkcontactsyncadaptor.cpp index 198eec6..1666bff 100644 --- a/src/vk/vk-contacts/vkcontactsyncadaptor.cpp +++ b/src/vk/vk-contacts/vkcontactsyncadaptor.cpp @@ -277,6 +277,20 @@ void VKContactSyncAdaptor::purgeDataForOldAccount(int oldId, SocialNetworkSyncAd } } +void VKContactSyncAdaptor::retryThrottledRequest(const QString &request, const QVariantList &args, bool retryLimitReached) +{ + int accountId = args[0].toInt(); + if (retryLimitReached) { + SOCIALD_LOG_ERROR("hit request retry limit! unable to request data from VK account with id" << accountId); + purgeSyncStateData(QString::number(accountId)); + setStatus(SocialNetworkSyncAdaptor::Error); + } else { + SOCIALD_LOG_DEBUG("retrying Contacts" << request << "request for VK account:" << accountId); + requestData(accountId, args[1].toString(), args[2].toInt(), args[3].toDateTime()); + } + decrementSemaphore(accountId); // finished waiting for the request. +} + void VKContactSyncAdaptor::beginSync(int accountId, const QString &accessToken) { // clear our cache lists if necessary. @@ -326,18 +340,21 @@ void VKContactSyncAdaptor::requestData(int accountId, const QString &accessToken if (reply) { reply->setProperty("accountId", accountId); reply->setProperty("accessToken", accessToken); - reply->setProperty("lastSyncTimestamp", syncTimestamp); reply->setProperty("startIndex", startIndex); + reply->setProperty("lastSyncTimestamp", syncTimestamp); connect(reply, SIGNAL(finished()), this, SLOT(contactsFinishedHandler())); connect(reply, SIGNAL(error(QNetworkReply::NetworkError)), this, SLOT(errorHandler(QNetworkReply::NetworkError))); connect(reply, SIGNAL(sslErrors(QList)), this, SLOT(sslErrorsHandler(QList))); m_apiRequestsRemaining[accountId] = m_apiRequestsRemaining[accountId] - 1; setupReplyTimeout(accountId, reply); } else { - SOCIALD_LOG_ERROR("unable to request data from VK account with id:" << accountId); - purgeSyncStateData(QString::number(accountId)); - setStatus(SocialNetworkSyncAdaptor::Error); - decrementSemaphore(accountId); + // request was throttled by VKNetworkAccessManager + QVariantList args; + args << accountId << accessToken << startIndex << syncTimestamp; + enqueueThrottledRequest(QStringLiteral("requestData"), args); + + // we are waiting to request data. Increment the semaphore so that we know we're still busy. + incrementSemaphore(accountId); // decremented in retryThrottledRequest(). } } @@ -345,9 +362,9 @@ void VKContactSyncAdaptor::contactsFinishedHandler() { QNetworkReply *reply = qobject_cast(sender()); QByteArray data = reply->readAll(); - int startIndex = reply->property("startIndex").toInt(); int accountId = reply->property("accountId").toInt(); QString accessToken = reply->property("accessToken").toString(); + int startIndex = reply->property("startIndex").toInt(); QDateTime lastSyncTimestamp = reply->property("lastSyncTimestamp").toDateTime(); bool isError = reply->property("isError").toBool(); reply->deleteLater(); @@ -359,6 +376,16 @@ void VKContactSyncAdaptor::contactsFinishedHandler() } if (isError) { + QVariantList args; + args << accountId << accessToken << startIndex << lastSyncTimestamp; + bool ok = true; + QJsonObject parsed = parseJsonObjectReplyData(data, &ok); + if (enqueueServerThrottledRequestIfRequired(parsed, QStringLiteral("requestData"), args)) { + // we hit the throttle limit, let throttle timer repeat the request + // don't decrement semaphore yet as we're still waiting for it. + // it will be decremented in retryThrottledRequest(). + return; + } SOCIALD_LOG_ERROR("error occurred when performing contacts request for VK account:" << accountId); purgeSyncStateData(QString::number(accountId)); setStatus(SocialNetworkSyncAdaptor::Error); diff --git a/src/vk/vk-contacts/vkcontactsyncadaptor.h b/src/vk/vk-contacts/vkcontactsyncadaptor.h index 19dac4e..f997fa3 100644 --- a/src/vk/vk-contacts/vkcontactsyncadaptor.h +++ b/src/vk/vk-contacts/vkcontactsyncadaptor.h @@ -48,6 +48,7 @@ class VKContactSyncAdaptor : public VKDataTypeSyncAdaptor, public QtContactsSqli void beginSync(int accountId, const QString &accessToken); void finalize(int accountId); void finalCleanup(); + void retryThrottledRequest(const QString &request, const QVariantList &args, bool retryLimitReached); // implementing TWCSA interface bool testAccountProvenance(const QContact &contact, const QString &accountId); bool readSyncStateData(QDateTime *remoteSince, const QString &accountId, TwoWayContactSyncAdapter::ReadStateMode readMode = TwoWayContactSyncAdapter::ReadAllState); diff --git a/src/vk/vk-images/vkimagesyncadaptor.cpp b/src/vk/vk-images/vkimagesyncadaptor.cpp index 4d96198..96db121 100644 --- a/src/vk/vk-images/vkimagesyncadaptor.cpp +++ b/src/vk/vk-images/vkimagesyncadaptor.cpp @@ -30,8 +30,6 @@ #include #define VK_IMAGES_MAX_COUNT 1000 /* maximum images returned per request */ -#define VK_THROTTLE_TIMER_TIMEOUT 5000 -#define VK_THROTTLE_ERROR 6 // Currently, we integrate with the device image gallery via saving thumbnails to the // ~/.local/share/system/privileged/Images directory, and filling the @@ -44,7 +42,6 @@ VKImageSyncAdaptor::VKImageSyncAdaptor(QObject *parent) , m_currentAlbumIndex(0) { setInitialActive(m_db.isValid()); - connect(&m_throttleTimer, SIGNAL(timeout()), this, SLOT(throttleTimerTimeout())); } VKImageSyncAdaptor::~VKImageSyncAdaptor() @@ -80,6 +77,7 @@ void VKImageSyncAdaptor::finalize(int accountId) SOCIALD_LOG_INFO("sync aborted, won't commit database changes"); } else if (m_syncError) { SOCIALD_LOG_INFO("sync error, won't commit database changes"); + setStatus(SocialNetworkSyncAdaptor::Error); } else { // Determine album delta. QHash > deletedAlbumIds; // user to deleted album ids @@ -198,12 +196,34 @@ void VKImageSyncAdaptor::finalize(int accountId) } } +void VKImageSyncAdaptor::retryThrottledRequest(const QString &request, const QVariantList &args, bool retryLimitReached) +{ + int accountId = args[0].toInt(); + if (retryLimitReached) { + SOCIALD_LOG_ERROR("hit request retry limit! unable to request data from VK account with id" << accountId); + m_syncError = true; + } else { + SOCIALD_LOG_DEBUG("retrying Images" << request << "request for VK account:" << accountId); + if (request == QStringLiteral("requestData")) { + requestData(accountId, + args[1].toString(), + args[2].toString(), + args[3].toString(), + args[4].toString()); + } else { + possiblyAddNewUser(accountId, + args[1].toString(), + args[2].toString()); + } + } + decrementSemaphore(accountId); // finished waiting for the request. +} + void VKImageSyncAdaptor::requestData(int accountId, const QString &accessToken, const QString &continuationUrl, const QString &vkUserId, - const QString &vkAlbumId, - bool restarted) + const QString &vkAlbumId) { if (syncAborted()) { SOCIALD_LOG_DEBUG("skipping data request due to sync abort"); @@ -262,15 +282,17 @@ void VKImageSyncAdaptor::requestData(int accountId, connect(reply, SIGNAL(finished()), this, SLOT(imagesFinishedHandler())); } - // we're requesting data. Increment the semaphore so that we know we're still busy - // unless this was restarted call. - if (!restarted) { - incrementSemaphore(accountId); - } + // we're requesting data. Increment the semaphore so that we know we're still busy. + incrementSemaphore(accountId); setupReplyTimeout(accountId, reply); } else { - SOCIALD_LOG_ERROR("unable to request data from VK account with id" << accountId); - m_syncError = true; + // request was throttled by VKNetworkAccessManager + QVariantList args; + args << accountId << accessToken << continuationUrl << vkUserId << vkAlbumId; + enqueueThrottledRequest(QStringLiteral("requestData"), args); + + // we are waiting to request data. Increment the semaphore so that we know we're still busy. + incrementSemaphore(accountId); // decremented in retryThrottledRequest(). } } @@ -289,10 +311,11 @@ void VKImageSyncAdaptor::albumsFinishedHandler() bool ok = false; QJsonObject parsed = parseJsonObjectReplyData(replyData, &ok); if (isError || !ok || !parsed.contains(QLatin1String("response"))) { - if (startThrottleTimerIfRequired(parsed, accountId, accessToken, QString(), QString(), QString())) { + QVariantList args; args << accountId << accessToken << QString() << QString() << QString(); + if (enqueueServerThrottledRequestIfRequired(parsed, QStringLiteral("requestData"), args)) { // we hit the throttle limit, let throttle timer repeat the request. - // don't decrement semaphore as this call will continue - // after the break. + // don't decrement semaphore yet as we're still waiting for it. + // it will be decremented in retryThrottledRequest(). return; } @@ -346,7 +369,7 @@ void VKImageSyncAdaptor::albumsFinishedHandler() } // request the information for user who owns this album if necessary - possiblyAddNewUser(ownerId, accountId, accessToken); + possiblyAddNewUser(accountId, accessToken, ownerId); } // start downloading album content @@ -375,10 +398,11 @@ void VKImageSyncAdaptor::imagesFinishedHandler() bool ok = false; QJsonObject parsed = parseJsonObjectReplyData(replyData, &ok); if (isError || !ok || !parsed.contains(QLatin1String("response"))) { - if (startThrottleTimerIfRequired(parsed, accountId, accessToken, vkUserId, vkAlbumId, continuationUrl)) { + QVariantList args; args << accountId << accessToken << vkUserId << vkAlbumId << continuationUrl; + if (enqueueServerThrottledRequestIfRequired(parsed, QStringLiteral("requestData"), args)) { // we hit the throttle limit, let throttle timer repeat the request - // don't decrement semaphore as this call will continue - // after the break. + // don't decrement semaphore yet as we're still waiting for it. + // it will be decremented in retryThrottledRequest(). return; } @@ -464,7 +488,7 @@ void VKImageSyncAdaptor::imagesFinishedHandler() decrementSemaphore(accountId); } -void VKImageSyncAdaptor::possiblyAddNewUser(const QString &vkUserId, int accountId, const QString &accessToken) +void VKImageSyncAdaptor::possiblyAddNewUser(int accountId, const QString &accessToken, const QString &vkUserId) { QString dbUserId; VKUser::ConstPtr dbUser = m_db.user(accountId); @@ -490,6 +514,7 @@ void VKImageSyncAdaptor::possiblyAddNewUser(const QString &vkUserId, int account if (reply) { reply->setProperty("accountId", accountId); reply->setProperty("accessToken", accessToken); + reply->setProperty("vkUserId", vkUserId); connect(reply, SIGNAL(error(QNetworkReply::NetworkError)), this, SLOT(errorHandler(QNetworkReply::NetworkError))); connect(reply, SIGNAL(sslErrors(QList)), @@ -498,6 +523,14 @@ void VKImageSyncAdaptor::possiblyAddNewUser(const QString &vkUserId, int account incrementSemaphore(accountId); setupReplyTimeout(accountId, reply); + } else { + // request was throttled by VKNetworkAccessManager + QVariantList args; + args << accountId << accessToken << vkUserId; + enqueueThrottledRequest(QStringLiteral("possiblyAddNewUser"), args); + + // we are waiting to request data. Increment the semaphore so that we know we're still busy. + incrementSemaphore(accountId); // decremented in retryThrottledRequest(). } } @@ -506,12 +539,22 @@ void VKImageSyncAdaptor::userFinishedHandler() QNetworkReply *reply = qobject_cast(sender()); QByteArray replyData = reply->readAll(); int accountId = reply->property("accountId").toInt(); + QString accessToken = reply->property("accessToken").toString(); + QString vkUserId = reply->property("vkUserId").toString(); disconnect(reply); reply->deleteLater(); bool ok = false; QJsonObject parsed = parseJsonObjectReplyData(replyData, &ok); if (!ok || !parsed.contains(QLatin1String("response")) || !parsed.value(QLatin1String("response")).toArray().size()) { + QVariantList args; + args << accountId << accessToken << vkUserId; + if (enqueueServerThrottledRequestIfRequired(parsed, QStringLiteral("possiblyAddNewUser"), args)) { + // we hit the throttle limit, let throttle timer repeat the request + // don't decrement semaphore yet as we're still waiting for it. + // it will be decremented in retryThrottledRequest(). + return; + } SOCIALD_LOG_ERROR("unable to read users.get response for VK account with id" << accountId); return; } @@ -526,21 +569,6 @@ void VKImageSyncAdaptor::userFinishedHandler() decrementSemaphore(accountId); } -void VKImageSyncAdaptor::throttleTimerTimeout() -{ - SOCIALD_LOG_DEBUG("VK throttle timer expired"); - - m_throttleTimer.stop(); - int accountId = m_throttleTimer.property("accountId").toInt(); - QString accessToken = m_throttleTimer.property("accessToken").toString(); - QString vkUserId = m_throttleTimer.property("vkUserId").toString(); - QString vkAlbumId = m_throttleTimer.property("vkAlbumId").toString(); - QString continuationUrl = m_throttleTimer.property("continuationUrl").toString(); - - requestData(accountId, accessToken, continuationUrl, - vkUserId, vkAlbumId, true); -} - void VKImageSyncAdaptor::requestQueuedAlbum(const QString &accessToken) { // take next album from the queue and load it @@ -554,27 +582,3 @@ void VKImageSyncAdaptor::requestQueuedAlbum(const QString &accessToken) requestData(accountId, accessToken, QString(), ownerId, id); } } - -bool VKImageSyncAdaptor::startThrottleTimerIfRequired(QJsonObject &parsed, int accountId, const QString &accessToken, - const QString &vkUserId, const QString &vkAlbumId, - const QString &continuationUrl) -{ - if (parsed.contains(QLatin1String("error"))) { - QJsonObject error = parsed.value(QLatin1String("error")).toObject(); - int errorCode = error.value(QLatin1String("error_code")).toInt(); - if (errorCode == VK_THROTTLE_ERROR) { - // we have hit the server rate limit. - // wait a few of seconds and try again. - SOCIALD_LOG_DEBUG("VK server rate limit exceeded, start throttle timer"); - m_throttleTimer.setProperty("accountId", accountId); - m_throttleTimer.setProperty("accessToken", accessToken); - m_throttleTimer.setProperty("vkUserId", vkUserId); - m_throttleTimer.setProperty("vkAlbumId", vkAlbumId); - m_throttleTimer.setProperty("continuationUrl", continuationUrl); - m_throttleTimer.start(VK_THROTTLE_TIMER_TIMEOUT); - return true; - } - } - - return false; -} diff --git a/src/vk/vk-images/vkimagesyncadaptor.h b/src/vk/vk-images/vkimagesyncadaptor.h index a70e04e..4d41687 100644 --- a/src/vk/vk-images/vkimagesyncadaptor.h +++ b/src/vk/vk-images/vkimagesyncadaptor.h @@ -51,21 +51,18 @@ class VKImageSyncAdaptor : public VKDataTypeSyncAdaptor void purgeDataForOldAccount(int oldId, SocialNetworkSyncAdaptor::PurgeMode mode); void beginSync(int accountId, const QString &accessToken); void finalize(int accountId); + void retryThrottledRequest(const QString &request, const QVariantList &args, bool retryLimitReached); private: void requestData(int accountId, const QString &accessToken, const QString &continuationUrl, - const QString &vkUserId, const QString &vkAlbumId, bool restarted = false); - void possiblyAddNewUser(const QString &vkUserId, int accountId, const QString &accessToken); + const QString &vkUserId, const QString &vkAlbumId); + void possiblyAddNewUser(int accountId, const QString &accessToken, const QString &vkUserId); void requestQueuedAlbum(const QString &accessToken); - bool startThrottleTimerIfRequired(QJsonObject &parsed, int accountId, const QString &accessToken, - const QString &vkUserId, const QString &vkAlbumId, - const QString &continuationUrl); private Q_SLOTS: void albumsFinishedHandler(); void imagesFinishedHandler(); void userFinishedHandler(); - void throttleTimerTimeout(); private: QList m_receivedAlbums; @@ -77,7 +74,6 @@ private Q_SLOTS: VKImagesDatabase m_db; bool m_syncError; int m_currentAlbumIndex; - QTimer m_throttleTimer; }; #endif // VKIMAGESYNCADAPTOR_H diff --git a/src/vk/vk-notifications/vknotificationsyncadaptor.cpp b/src/vk/vk-notifications/vknotificationsyncadaptor.cpp index 7efa41b..410ddb1 100644 --- a/src/vk/vk-notifications/vknotificationsyncadaptor.cpp +++ b/src/vk/vk-notifications/vknotificationsyncadaptor.cpp @@ -63,6 +63,19 @@ void VKNotificationSyncAdaptor::finalize(int accountId) } } +void VKNotificationSyncAdaptor::retryThrottledRequest(const QString &request, const QVariantList &args, bool retryLimitReached) +{ + int accountId = args[0].toInt(); + if (retryLimitReached) { + SOCIALD_LOG_ERROR("hit request retry limit! unable to request data from VK account with id" << accountId); + setStatus(SocialNetworkSyncAdaptor::Error); + } else { + SOCIALD_LOG_DEBUG("retrying Notifications" << request << "request for VK account:" << accountId); + requestNotifications(accountId, args[1].toString(), args[2].toString(), args[3].toString()); + } + decrementSemaphore(accountId); // finished waiting for the request. +} + void VKNotificationSyncAdaptor::requestNotifications(int accountId, const QString &accessToken, const QString &until, const QString &pagingToken) { // TODO: result paging @@ -90,7 +103,13 @@ void VKNotificationSyncAdaptor::requestNotifications(int accountId, const QStrin incrementSemaphore(accountId); setupReplyTimeout(accountId, reply); } else { - SOCIALD_LOG_ERROR("error: unable to request home posts from VK account with id:" << accountId); + // request was throttled by VKNetworkAccessManager + QVariantList args; + args << accountId << accessToken << until << pagingToken; + enqueueThrottledRequest(QStringLiteral("requestNotifications"), args); + + // we are waiting to request data. Increment the semaphore so that we know we're still busy. + incrementSemaphore(accountId); // decremented in retryThrottledRequest(). } } diff --git a/src/vk/vk-notifications/vknotificationsyncadaptor.h b/src/vk/vk-notifications/vknotificationsyncadaptor.h index 897c664..0dd6339 100644 --- a/src/vk/vk-notifications/vknotificationsyncadaptor.h +++ b/src/vk/vk-notifications/vknotificationsyncadaptor.h @@ -30,6 +30,7 @@ class VKNotificationSyncAdaptor : public VKDataTypeSyncAdaptor void purgeDataForOldAccount(int oldId, SocialNetworkSyncAdaptor::PurgeMode mode); void beginSync(int accountId, const QString &accessToken); void finalize(int accountId); + void retryThrottledRequest(const QString &request, const QVariantList &args, bool retryLimitReached); private: void requestNotifications(int accountId, const QString &accessToken, diff --git a/src/vk/vk-posts/vkpostsyncadaptor.cpp b/src/vk/vk-posts/vkpostsyncadaptor.cpp index 85c0a48..1da2381 100644 --- a/src/vk/vk-posts/vkpostsyncadaptor.cpp +++ b/src/vk/vk-posts/vkpostsyncadaptor.cpp @@ -85,6 +85,19 @@ void VKPostSyncAdaptor::finalize(int accountId) } } +void VKPostSyncAdaptor::retryThrottledRequest(const QString &request, const QVariantList &args, bool retryLimitReached) +{ + int accountId = args[0].toInt(); + if (retryLimitReached) { + SOCIALD_LOG_ERROR("hit request retry limit! unable to request data from VK account with id" << accountId); + setStatus(SocialNetworkSyncAdaptor::Error); + } else { + SOCIALD_LOG_DEBUG("retrying Posts" << request << "request for VK account:" << accountId); + requestPosts(accountId, args[1].toString()); + } + decrementSemaphore(accountId); // finished waiting for the request. +} + void VKPostSyncAdaptor::requestPosts(int accountId, const QString &accessToken) { QDateTime since = lastSuccessfulSyncTime(accountId); @@ -119,7 +132,13 @@ void VKPostSyncAdaptor::requestPosts(int accountId, const QString &accessToken) incrementSemaphore(accountId); setupReplyTimeout(accountId, reply); } else { - SOCIALD_LOG_ERROR("error: unable to request home posts from VK account with id:" << accountId); + // request was throttled by VKNetworkAccessManager + QVariantList args; + args << accountId << accessToken; + enqueueThrottledRequest(QStringLiteral("requestPosts"), args); + + // we are waiting to request data. Increment the semaphore so that we know we're still busy. + incrementSemaphore(accountId); // decremented in retryThrottledRequest(). } } @@ -128,6 +147,7 @@ void VKPostSyncAdaptor::finishedPostsHandler() QNetworkReply *reply = qobject_cast(sender()); bool isError = reply->property("isError").toBool(); int accountId = reply->property("accountId").toInt(); + QString accessToken = reply->property("accessToken").toString(); QByteArray replyData = reply->readAll(); disconnect(reply); @@ -181,6 +201,14 @@ void VKPostSyncAdaptor::finishedPostsHandler() } } } else { + QVariantList args; + args << accountId << accessToken; + if (enqueueServerThrottledRequestIfRequired(parsed, QStringLiteral("requestPosts"), args)) { + // we hit the throttle limit, let throttle timer repeat the request. + // don't decrement semaphore yet as we're still waiting for it. + // it will be decremented in retryThrottledRequest(). + return; + } // error occurred during request. SOCIALD_LOG_ERROR("error: unable to parse event feed data from request with account" << accountId << "got:" << QString::fromUtf8(replyData)); diff --git a/src/vk/vk-posts/vkpostsyncadaptor.h b/src/vk/vk-posts/vkpostsyncadaptor.h index ce39aef..2b7cfa9 100644 --- a/src/vk/vk-posts/vkpostsyncadaptor.h +++ b/src/vk/vk-posts/vkpostsyncadaptor.h @@ -36,6 +36,7 @@ class VKPostSyncAdaptor : public VKDataTypeSyncAdaptor void purgeDataForOldAccount(int oldId, SocialNetworkSyncAdaptor::PurgeMode mode); void beginSync(int accountId, const QString &accessToken); void finalize(int accountId); + void retryThrottledRequest(const QString &request, const QVariantList &args, bool retryLimitReached); private: void requestPosts(int accountId, const QString &accessToken); diff --git a/src/vk/vkdatatypesyncadaptor.cpp b/src/vk/vkdatatypesyncadaptor.cpp index d7b2030..0ea043a 100644 --- a/src/vk/vkdatatypesyncadaptor.cpp +++ b/src/vk/vkdatatypesyncadaptor.cpp @@ -6,6 +6,7 @@ ****************************************************************************/ #include "vkdatatypesyncadaptor.h" +#include "vknetworkaccessmanager_p.h" #include "trace.h" #include @@ -154,14 +155,70 @@ QDateTime VKDataTypeSyncAdaptor::parseVKDateTime(const QJsonValue &v) VKDataTypeSyncAdaptor::VKDataTypeSyncAdaptor(SocialNetworkSyncAdaptor::DataType dataType, QObject *parent) - : SocialNetworkSyncAdaptor("vk", dataType, parent), m_triedLoading(false) + : SocialNetworkSyncAdaptor("vk", dataType, new VKNetworkAccessManager, parent), m_triedLoading(false) { + m_throttleTimer.setSingleShot(true); + connect(&m_throttleTimer, &QTimer::timeout, this, &VKDataTypeSyncAdaptor::throttleTimerTimeout); } VKDataTypeSyncAdaptor::~VKDataTypeSyncAdaptor() { } +void VKDataTypeSyncAdaptor::enqueueThrottledRequest(const QString &request, const QVariantList &args, int interval) +{ + m_throttledRequestQueue.append(qMakePair(request, args)); + if (!m_throttleTimer.isActive() || m_throttleTimer.interval() < interval) { + // start the timer if it is inactive, or if we are requested to + // enqueue a request with a larger interval (e.g., if the server + // throttled us, hence we are using VK_THROTTLE_EXTRA_INTERVAL). + m_throttleTimer.setInterval(interval ? interval : VK_THROTTLE_INTERVAL); + m_throttleTimer.start(); + } +} + +bool VKDataTypeSyncAdaptor::enqueueServerThrottledRequestIfRequired(const QJsonObject &parsed, + const QString &request, + const QVariantList &args) +{ + if (parsed.contains(QLatin1String("error"))) { + QJsonObject error = parsed.value(QLatin1String("error")).toObject(); + int errorCode = error.value(QLatin1String("error_code")).toInt(); + if (errorCode == VK_THROTTLE_ERROR_CODE) { + // we have hit the server rate limit. + // wait a few of seconds and try again. + SOCIALD_LOG_DEBUG("VK server rate limit exceeded, start throttle timer"); + enqueueThrottledRequest(request, args, VK_THROTTLE_EXTRA_INTERVAL); + return true; + } + } + + return false; +} + +void VKDataTypeSyncAdaptor::throttleTimerTimeout() +{ + if (m_throttledRequestQueue.isEmpty()) { + return; + } + + QPair request = m_throttledRequestQueue.takeFirst(); + static int totalRetryCount; + totalRetryCount += 1; + bool retryLimitReached = totalRetryCount > VK_THROTTLE_RETRY_LIMIT; + + // even if the retry limit has been reached, we still call the derived-type function. + // this is because they may have special handling (e.g., cleanup / error conditions). + retryThrottledRequest(request.first, request.second, retryLimitReached); + + // we still handle every queued request even if the limit has been reached, as each + // request will have a semaphore associated with it which will need to be decremented. + if (!m_throttledRequestQueue.isEmpty()) { + m_throttleTimer.setInterval(retryLimitReached ? 0 : VK_THROTTLE_INTERVAL); + m_throttleTimer.start(); + } +} + void VKDataTypeSyncAdaptor::sync(const QString &dataTypeString, int accountId) { if (dataTypeString != SocialNetworkSyncAdaptor::dataTypeName(m_dataType)) { diff --git a/src/vk/vkdatatypesyncadaptor.h b/src/vk/vkdatatypesyncadaptor.h index 5bdfafd..d42bf77 100644 --- a/src/vk/vkdatatypesyncadaptor.h +++ b/src/vk/vkdatatypesyncadaptor.h @@ -12,6 +12,7 @@ #include #include +#include #include #include #include @@ -85,6 +86,12 @@ class VKDataTypeSyncAdaptor : public SocialNetworkSyncAdaptor static UserProfile findUserProfile(const QList &profiles, int uid); static GroupProfile findGroupProfile(const QList &profiles, int uid); + void enqueueThrottledRequest(const QString &request, const QVariantList &args, int interval = 0); + bool enqueueServerThrottledRequestIfRequired(const QJsonObject &parsed, + const QString &request, + const QVariantList &args); + virtual void retryThrottledRequest(const QString &request, const QVariantList &args, bool retryLimitReached) = 0; + protected Q_SLOTS: virtual void errorHandler(QNetworkReply::NetworkError err); virtual void sslErrorsHandler(const QList &errs); @@ -92,6 +99,7 @@ protected Q_SLOTS: private Q_SLOTS: void signOnError(const SignOn::Error &error); void signOnResponse(const SignOn::SessionData &responseData); + void throttleTimerTimeout(); private: void loadClientId(); @@ -99,6 +107,8 @@ private Q_SLOTS: void signIn(Accounts::Account *account); bool m_triedLoading; // Is true if we tried to load (even if we failed) QString m_clientId; + QTimer m_throttleTimer; + QList > m_throttledRequestQueue; }; #endif // VKDATATYPESYNCADAPTOR_H diff --git a/src/vk/vknetworkaccessmanager.cpp b/src/vk/vknetworkaccessmanager.cpp new file mode 100644 index 0000000..29484ae --- /dev/null +++ b/src/vk/vknetworkaccessmanager.cpp @@ -0,0 +1,103 @@ +/**************************************************************************** + ** + ** Copyright (C) 2015 Jolla Ltd. + ** Contact: Chris Adams + ** + ** This program/library is free software; you can redistribute it and/or + ** modify it under the terms of the GNU Lesser General Public License + ** version 2.1 as published by the Free Software Foundation. + ** + ** This program/library is distributed in the hope that it will be useful, + ** but WITHOUT ANY WARRANTY; without even the implied warranty of + ** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + ** Lesser General Public License for more details. + ** + ** You should have received a copy of the GNU Lesser General Public + ** License along with this program/library; if not, write to the Free + ** Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA + ** 02110-1301 USA + ** + ****************************************************************************/ + +#include "vknetworkaccessmanager_p.h" +#include "trace.h" + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +namespace { + bool touchTimestampFile() + { + static const QString timestampFileName = QString::fromLatin1("%1/%2/vktimestamp") + .arg(QString::fromLatin1(PRIVILEGED_DATA_DIR)) + .arg(QString::fromLatin1(SYNC_DATABASE_DIR)); + QByteArray tsfnba = timestampFileName.toUtf8(); + + int fd = open(tsfnba.constData(), O_WRONLY|O_CREAT|O_NOCTTY|O_NONBLOCK, 0666); + if (fd < 0) { + return false; + } + + int rv = utimensat(AT_FDCWD, tsfnba.constData(), 0, 0); + close(fd); + if (rv != 0) { + return false; + } + + return true; + } + + qint64 readTimestampFile() + { + static const QString timestampFileName = QString::fromLatin1("%1/%2/vktimestamp") + .arg(QString::fromLatin1(PRIVILEGED_DATA_DIR)) + .arg(QString::fromLatin1(SYNC_DATABASE_DIR)); + QByteArray tsfnba = timestampFileName.toUtf8(); + + struct stat buf; + if (stat(tsfnba.constData(), &buf) < 0) { + return 0; + } + + time_t tvsec = buf.st_mtim.tv_sec; + long nanosec = buf.st_mtim.tv_nsec; + qint64 msecs = (tvsec*1000) + (nanosec/1000000); + return msecs; + } +} + +VKNetworkAccessManager::VKNetworkAccessManager(QObject *parent) + : SocialdNetworkAccessManager(parent) +{ +} + +QNetworkReply *VKNetworkAccessManager::createRequest( + QNetworkAccessManager::Operation op, + const QNetworkRequest &req, + QIODevice *outgoingData) +{ + // VK throttles requests. We want to wait at least 550 ms between each request. + // To do this properly, we need to protect the file access with a semaphore + // or link-lock to prevent concurrent process access. For now, we use the + // naive approach. + + qint64 currTime = QDateTime::currentDateTimeUtc().toMSecsSinceEpoch(); + qint64 lastRequestTime = readTimestampFile(); + qint64 delta = currTime - lastRequestTime; + if (lastRequestTime == 0 || delta > VK_THROTTLE_INTERVAL) { + touchTimestampFile(); + return SocialdNetworkAccessManager::createRequest(op, req, outgoingData); + } + + SOCIALD_LOG_DEBUG("Throttling request! lastRequestTime:" << lastRequestTime << ", currTime:" << currTime << ", so delta:" << delta); + return 0; // tell the client to resubmit their request, it was throttled. +} diff --git a/src/vk/vknetworkaccessmanager_p.h b/src/vk/vknetworkaccessmanager_p.h new file mode 100644 index 0000000..f79408d --- /dev/null +++ b/src/vk/vknetworkaccessmanager_p.h @@ -0,0 +1,45 @@ +/**************************************************************************** + ** + ** Copyright (C) 2015 Jolla Ltd. + ** Contact: Chris Adams + ** + ** This program/library is free software; you can redistribute it and/or + ** modify it under the terms of the GNU Lesser General Public License + ** version 2.1 as published by the Free Software Foundation. + ** + ** This program/library is distributed in the hope that it will be useful, + ** but WITHOUT ANY WARRANTY; without even the implied warranty of + ** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + ** Lesser General Public License for more details. + ** + ** You should have received a copy of the GNU Lesser General Public + ** License along with this program/library; if not, write to the Free + ** Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA + ** 02110-1301 USA + ** + ****************************************************************************/ + +#ifndef SOCIALD_VK_QNAMFACTORY_P_H +#define SOCIALD_VK_QNAMFACTORY_P_H + +#include "socialdnetworkaccessmanager_p.h" + +#define VK_THROTTLE_INTERVAL 550 /* msec */ +#define VK_THROTTLE_EXTRA_INTERVAL 3000 /* msec */ +#define VK_THROTTLE_ERROR_CODE 6 +#define VK_THROTTLE_RETRY_LIMIT 30 + +class VKNetworkAccessManager : public SocialdNetworkAccessManager +{ + Q_OBJECT + +public: + VKNetworkAccessManager(QObject *parent = 0); + +protected: + QNetworkReply *createRequest(QNetworkAccessManager::Operation op, + const QNetworkRequest &req, + QIODevice *outgoingData = 0); +}; + +#endif // SOCIALD_VK_QNAMFACTORY_P_H