Skip to content

Commit

Permalink
Browse files Browse the repository at this point in the history
[buteo-sync-plugins-social] Implement generic VK request throttling. …
…Contributes to JB#33334

This commit ensures that we meter-out requests to the VK service
to avoid hitting the server-enforced throttle limits.

Contributes to JB#33334
  • Loading branch information
Chris Adams committed Nov 13, 2015
1 parent 77ade53 commit 699a055
Show file tree
Hide file tree
Showing 23 changed files with 413 additions and 88 deletions.
6 changes: 4 additions & 2 deletions .gitignore
@@ -1,6 +1,8 @@
*.o
moc_*
Makefile
*.so
*.ts
*.qm
.moc
.obj
*.moc
*.obj
4 changes: 3 additions & 1 deletion src/common/socialnetworksyncadaptor.cpp
Expand Up @@ -63,11 +63,12 @@ namespace {

SocialNetworkSyncAdaptor::SocialNetworkSyncAdaptor(const QString &serviceName,
SocialNetworkSyncAdaptor::DataType dataType,
QNetworkAccessManager *qnam,
QObject *parent)
: QObject(parent)
, m_dataType(dataType)
, m_accountManager(new Accounts::Manager(this))
, m_networkAccessManager(new SocialdNetworkAccessManager(this))
, m_networkAccessManager(qnam != 0 ? qnam : new SocialdNetworkAccessManager)
, m_accountSyncProfile(NULL)
, m_syncDb(new SocialNetworkSyncDatabase())
, m_status(SocialNetworkSyncAdaptor::Invalid)
Expand All @@ -79,6 +80,7 @@ SocialNetworkSyncAdaptor::SocialNetworkSyncAdaptor(const QString &serviceName,

SocialNetworkSyncAdaptor::~SocialNetworkSyncAdaptor()
{
delete m_networkAccessManager;
delete m_accountSyncProfile;
delete m_syncDb;
}
Expand Down
2 changes: 1 addition & 1 deletion src/common/socialnetworksyncadaptor.h
Expand Up @@ -80,7 +80,7 @@ class SocialNetworkSyncAdaptor : public QObject
static QString dataTypeName(DataType t);

public:
SocialNetworkSyncAdaptor(const QString &serviceName, SocialNetworkSyncAdaptor::DataType dataType, QObject *parent);
SocialNetworkSyncAdaptor(const QString &serviceName, SocialNetworkSyncAdaptor::DataType dataType, QNetworkAccessManager *qnam, QObject *parent);
virtual ~SocialNetworkSyncAdaptor();

virtual QString syncServiceName() const = 0;
Expand Down
2 changes: 1 addition & 1 deletion src/dropbox/dropboxdatatypesyncadaptor.cpp
Expand Up @@ -43,7 +43,7 @@
#include <SignOn/SessionData>

DropboxDataTypeSyncAdaptor::DropboxDataTypeSyncAdaptor(SocialNetworkSyncAdaptor::DataType dataType, QObject *parent)
: SocialNetworkSyncAdaptor("dropbox", dataType, parent), m_triedLoading(false)
: SocialNetworkSyncAdaptor("dropbox", dataType, 0, parent), m_triedLoading(false)
{
}

Expand Down
2 changes: 1 addition & 1 deletion src/facebook/facebookdatatypesyncadaptor.cpp
Expand Up @@ -43,7 +43,7 @@
#include <SignOn/SessionData>

FacebookDataTypeSyncAdaptor::FacebookDataTypeSyncAdaptor(SocialNetworkSyncAdaptor::DataType dataType, QObject *parent)
: SocialNetworkSyncAdaptor("facebook", dataType, parent), m_triedLoading(false)
: SocialNetworkSyncAdaptor("facebook", dataType, 0, parent), m_triedLoading(false)
{
}

Expand Down
2 changes: 1 addition & 1 deletion src/google/googledatatypesyncadaptor.cpp
Expand Up @@ -43,7 +43,7 @@
#include <SignOn/SessionData>

GoogleDataTypeSyncAdaptor::GoogleDataTypeSyncAdaptor(SocialNetworkSyncAdaptor::DataType dataType, QObject *parent)
: SocialNetworkSyncAdaptor("google", dataType, parent), m_triedLoading(false)
: SocialNetworkSyncAdaptor("google", dataType, 0, parent), m_triedLoading(false)
{
}

Expand Down
2 changes: 1 addition & 1 deletion src/onedrive/onedrivedatatypesyncadaptor.cpp
Expand Up @@ -43,7 +43,7 @@
#include <SignOn/SessionData>

OneDriveDataTypeSyncAdaptor::OneDriveDataTypeSyncAdaptor(SocialNetworkSyncAdaptor::DataType dataType, QObject *parent)
: SocialNetworkSyncAdaptor("onedrive", dataType, parent), m_triedLoading(false)
: SocialNetworkSyncAdaptor("onedrive", dataType, 0, parent), m_triedLoading(false)
{
}

Expand Down
2 changes: 1 addition & 1 deletion src/twitter/twitterdatatypesyncadaptor.cpp
Expand Up @@ -50,7 +50,7 @@
#include <SignOn/SessionData>

TwitterDataTypeSyncAdaptor::TwitterDataTypeSyncAdaptor(SocialNetworkSyncAdaptor::DataType dataType, QObject *parent)
: SocialNetworkSyncAdaptor("twitter", dataType, parent), m_triedLoading(false)
: SocialNetworkSyncAdaptor("twitter", dataType, 0, parent), m_triedLoading(false)
{
}

Expand Down
32 changes: 30 additions & 2 deletions src/vk/vk-calendars/vkcalendarsyncadaptor.cpp
Expand Up @@ -223,6 +223,19 @@ void VKCalendarSyncAdaptor::beginSync(int accountId, const QString &accessToken)
requestEvents(accountId, accessToken);
}

void VKCalendarSyncAdaptor::retryThrottledRequest(const QString &request, const QVariantList &args, bool retryLimitReached)
{
int accountId = args[0].toInt();
if (retryLimitReached) {
SOCIALD_LOG_ERROR("hit request retry limit! unable to request data from VK account with id" << accountId);
setStatus(SocialNetworkSyncAdaptor::Error);
} else {
SOCIALD_LOG_DEBUG("retrying Calendars" << request << "request for VK account:" << accountId);
requestEvents(accountId, args[1].toString(), args[2].toInt());
}
decrementSemaphore(accountId); // finished waiting for the request.
}

void VKCalendarSyncAdaptor::requestEvents(int accountId, const QString &accessToken, int offset)
{
QUrlQuery urlQuery;
Expand Down Expand Up @@ -252,15 +265,21 @@ void VKCalendarSyncAdaptor::requestEvents(int accountId, const QString &accessTo
incrementSemaphore(accountId);
setupReplyTimeout(accountId, reply);
} else {
SOCIALD_LOG_ERROR("unable to request events from VK account with id:" << accountId);
// request was throttled by VKNetworkAccessManager
QVariantList args;
args << accountId << accessToken << offset;
enqueueThrottledRequest(QStringLiteral("requestEvents"), args);

// we are waiting to request data. Increment the semaphore so that we know we're still busy.
incrementSemaphore(accountId); // decremented in retryThrottledRequest().
}
}

void VKCalendarSyncAdaptor::finishedHandler()
{
QNetworkReply *reply = qobject_cast<QNetworkReply*>(sender());
QString accessToken = reply->property("accessToken").toString();
int accountId = reply->property("accountId").toInt();
QString accessToken = reply->property("accessToken").toString();
int offset = reply->property("offset").toInt();
QByteArray replyData = reply->readAll();
bool isError = reply->property("isError").toBool();
Expand Down Expand Up @@ -316,6 +335,15 @@ void VKCalendarSyncAdaptor::finishedHandler()
SOCIALD_LOG_DEBUG("done fetching calendar results");
}
} else {
QVariantList args;
args << accountId << accessToken << offset;
if (enqueueServerThrottledRequestIfRequired(parsed, QStringLiteral("requestEvents"), args)) {
// we hit the throttle limit, let throttle timer repeat the request
// don't decrement semaphore yet as we're still waiting for it.
// it will be decremented in retryThrottledRequest().
return;
}

// error occurred during request.
SOCIALD_LOG_ERROR("unable to parse calendar data from request with account" << accountId <<
"; got:" << QString::fromUtf8(replyData));
Expand Down
1 change: 1 addition & 0 deletions src/vk/vk-calendars/vkcalendarsyncadaptor.h
Expand Up @@ -32,6 +32,7 @@ class VKCalendarSyncAdaptor : public VKDataTypeSyncAdaptor
void beginSync(int accountId, const QString &accessToken);
void finalize(int accountId);
void finalCleanup();
void retryThrottledRequest(const QString &request, const QVariantList &args, bool retryLimitReached);

private:
void requestEvents(int accountId, const QString &accessToken, int offset = 0);
Expand Down
4 changes: 2 additions & 2 deletions src/vk/vk-common.pri
@@ -1,3 +1,3 @@
INCLUDEPATH += $$PWD
SOURCES += $$PWD/vkdatatypesyncadaptor.cpp
HEADERS += $$PWD/vkdatatypesyncadaptor.h
SOURCES += $$PWD/vkdatatypesyncadaptor.cpp $$PWD/vknetworkaccessmanager.cpp
HEADERS += $$PWD/vkdatatypesyncadaptor.h $$PWD/vknetworkaccessmanager_p.h
39 changes: 33 additions & 6 deletions src/vk/vk-contacts/vkcontactsyncadaptor.cpp
Expand Up @@ -277,6 +277,20 @@ void VKContactSyncAdaptor::purgeDataForOldAccount(int oldId, SocialNetworkSyncAd
}
}

void VKContactSyncAdaptor::retryThrottledRequest(const QString &request, const QVariantList &args, bool retryLimitReached)
{
int accountId = args[0].toInt();
if (retryLimitReached) {
SOCIALD_LOG_ERROR("hit request retry limit! unable to request data from VK account with id" << accountId);
purgeSyncStateData(QString::number(accountId));
setStatus(SocialNetworkSyncAdaptor::Error);
} else {
SOCIALD_LOG_DEBUG("retrying Contacts" << request << "request for VK account:" << accountId);
requestData(accountId, args[1].toString(), args[2].toInt(), args[3].toDateTime());
}
decrementSemaphore(accountId); // finished waiting for the request.
}

void VKContactSyncAdaptor::beginSync(int accountId, const QString &accessToken)
{
// clear our cache lists if necessary.
Expand Down Expand Up @@ -326,28 +340,31 @@ void VKContactSyncAdaptor::requestData(int accountId, const QString &accessToken
if (reply) {
reply->setProperty("accountId", accountId);
reply->setProperty("accessToken", accessToken);
reply->setProperty("lastSyncTimestamp", syncTimestamp);
reply->setProperty("startIndex", startIndex);
reply->setProperty("lastSyncTimestamp", syncTimestamp);
connect(reply, SIGNAL(finished()), this, SLOT(contactsFinishedHandler()));
connect(reply, SIGNAL(error(QNetworkReply::NetworkError)), this, SLOT(errorHandler(QNetworkReply::NetworkError)));
connect(reply, SIGNAL(sslErrors(QList<QSslError>)), this, SLOT(sslErrorsHandler(QList<QSslError>)));
m_apiRequestsRemaining[accountId] = m_apiRequestsRemaining[accountId] - 1;
setupReplyTimeout(accountId, reply);
} else {
SOCIALD_LOG_ERROR("unable to request data from VK account with id:" << accountId);
purgeSyncStateData(QString::number(accountId));
setStatus(SocialNetworkSyncAdaptor::Error);
decrementSemaphore(accountId);
// request was throttled by VKNetworkAccessManager
QVariantList args;
args << accountId << accessToken << startIndex << syncTimestamp;
enqueueThrottledRequest(QStringLiteral("requestData"), args);

// we are waiting to request data. Increment the semaphore so that we know we're still busy.
incrementSemaphore(accountId); // decremented in retryThrottledRequest().
}
}

void VKContactSyncAdaptor::contactsFinishedHandler()
{
QNetworkReply *reply = qobject_cast<QNetworkReply*>(sender());
QByteArray data = reply->readAll();
int startIndex = reply->property("startIndex").toInt();
int accountId = reply->property("accountId").toInt();
QString accessToken = reply->property("accessToken").toString();
int startIndex = reply->property("startIndex").toInt();
QDateTime lastSyncTimestamp = reply->property("lastSyncTimestamp").toDateTime();
bool isError = reply->property("isError").toBool();
reply->deleteLater();
Expand All @@ -359,6 +376,16 @@ void VKContactSyncAdaptor::contactsFinishedHandler()
}

if (isError) {
QVariantList args;
args << accountId << accessToken << startIndex << lastSyncTimestamp;
bool ok = true;
QJsonObject parsed = parseJsonObjectReplyData(data, &ok);
if (enqueueServerThrottledRequestIfRequired(parsed, QStringLiteral("requestData"), args)) {
// we hit the throttle limit, let throttle timer repeat the request
// don't decrement semaphore yet as we're still waiting for it.
// it will be decremented in retryThrottledRequest().
return;
}
SOCIALD_LOG_ERROR("error occurred when performing contacts request for VK account:" << accountId);
purgeSyncStateData(QString::number(accountId));
setStatus(SocialNetworkSyncAdaptor::Error);
Expand Down
1 change: 1 addition & 0 deletions src/vk/vk-contacts/vkcontactsyncadaptor.h
Expand Up @@ -48,6 +48,7 @@ class VKContactSyncAdaptor : public VKDataTypeSyncAdaptor, public QtContactsSqli
void beginSync(int accountId, const QString &accessToken);
void finalize(int accountId);
void finalCleanup();
void retryThrottledRequest(const QString &request, const QVariantList &args, bool retryLimitReached);
// implementing TWCSA interface
bool testAccountProvenance(const QContact &contact, const QString &accountId);
bool readSyncStateData(QDateTime *remoteSince, const QString &accountId, TwoWayContactSyncAdapter::ReadStateMode readMode = TwoWayContactSyncAdapter::ReadAllState);
Expand Down

0 comments on commit 699a055

Please sign in to comment.