Skip to content

Commit

Permalink
Merge branch 'jb53795-upload-session' into 'master'
Browse files Browse the repository at this point in the history 
[buteo-sync-plugins-social] OneDrive backup uploads should use upload sessions. Fixes JB#53795

See merge request mer-core/buteo-sync-plugins-social!93
  • Loading branch information
blam committed Apr 6, 2021
2 parents 6244a0b + 02fe26c commit aef9916
Show file tree
Hide file tree
Showing 3 changed files with 156 additions and 36 deletions.
179 changes: 145 additions & 34 deletions src/onedrive/onedrivebackupoperationsyncadaptor.cpp
View file
Expand Up @@ -35,6 +35,9 @@

namespace {

// OneDrive upload fragments must be multiple of 320kb
static const qint64 UploadChunkSize = 327680;

void debugDumpResponse(const QByteArray &data)
{
QString alldata = QString::fromUtf8(data);
Expand Down Expand Up @@ -738,14 +741,14 @@ void OneDriveBackupOperationSyncAdaptor::remoteFileFinishedHandler()
// note: no access token is required to access the content redirect url.
bool ok = false;
QJsonObject parsed = parseJsonObjectReplyData(data, &ok);
if (!ok || !parsed.contains("@content.downloadUrl")) {
redirectUrl = parsed.value("@microsoft.graph.downloadUrl").toString();
if (!ok || redirectUrl.isEmpty()) {
SOCIALD_LOG_ERROR("no content redirect url exists in file metadata for file:" << remoteFile);
debugDumpJsonResponse(data);
setStatus(SocialNetworkSyncAdaptor::Error);
decrementSemaphore(accountId);
return;
}
redirectUrl = parsed.value("@content.downloadUrl").toString();
SOCIALD_LOG_DEBUG("redirected from:" << remoteFileName << "to:" << redirectUrl);
requestData(accountId, QString(), localPath, remotePath, remoteFile, redirectUrl);
} else {
Expand Down Expand Up @@ -813,24 +816,58 @@ void OneDriveBackupOperationSyncAdaptor::uploadData(int accountId, const QString
SOCIALD_LOG_DEBUG("with data:" << createFolderJson);

reply = m_networkAccessManager->post(request, data);

} else if (m_uploadSessionUrl.isEmpty()) {
// Create an upload session
const QString createUploadSessionJson = QStringLiteral(
"{"
"\"name\": \"%1\""
"}").arg(m_localFileInfo.fileName());
const QByteArray data = createUploadSessionJson.toUtf8();

const QUrl url = QUrl(QStringLiteral("%1/%2:/%3/%4:/createUploadSession").arg(api(), m_remoteAppDir, remotePath, localFile));
QNetworkRequest request(url);
request.setHeader(QNetworkRequest::ContentLengthHeader, data.size());
request.setHeader(QNetworkRequest::ContentTypeHeader,
QVariant::fromValue<QString>(QString::fromLatin1("application/json")));
request.setRawHeader(QString(QLatin1String("Authorization")).toUtf8(),
QString(QLatin1String("Bearer ")).toUtf8() + accessToken.toUtf8());
SOCIALD_LOG_DEBUG("Creating upload session for remote file:"
<< QStringLiteral("%1/%2").arg(remotePath).arg(localFile)
<< "via request:" << url.toString());
reply = m_networkAccessManager->post(request, data);

} else {
// attempt to create a remote file.
QUrl url = QUrl(QStringLiteral("%1/%2:/%3/%4:/content").arg(api(), m_remoteAppDir, remotePath, localFile));
QString localFileName = QStringLiteral("%1/%2").arg(localPath, localFile);
QFile f(localFileName, this);
if(!f.open(QIODevice::ReadOnly)){
SOCIALD_LOG_ERROR("unable to open local file:" << localFileName << "for upload to OneDrive Backup with account:" << accountId);
} else {
QByteArray data(f.readAll());
f.close();
QNetworkRequest req(url);
req.setHeader(QNetworkRequest::ContentLengthHeader, data.size());
req.setHeader(QNetworkRequest::ContentTypeHeader, "application/octet-stream");
req.setRawHeader(QString(QLatin1String("Authorization")).toUtf8(),
QString(QLatin1String("Bearer ")).toUtf8() + accessToken.toUtf8());
SOCIALD_LOG_DEBUG("Attempting to create the remote file:" << QStringLiteral("%1/%2").arg(remotePath).arg(localFile) << "via request:" << url.toString());
reply = m_networkAccessManager->put(req, data);
QUrl url(m_uploadSessionUrl);

if (!m_uploadFile) {
m_uploadFile = new QFile(m_localFileInfo.filePath());
m_nextFileUploadPos = 0;
if (!m_uploadFile->open(QIODevice::ReadOnly)){
SOCIALD_LOG_ERROR("unable to open local file:" << m_localFileInfo.filePath()
<< "for upload to OneDrive Backup with account:" << accountId);
return;
}
}

const qint64 readSize = qMin(UploadChunkSize, m_uploadFile->size() - m_nextFileUploadPos);
m_uploadFile->seek(m_nextFileUploadPos);
const QByteArray data(m_uploadFile->read(readSize));
const QString contentRange = QStringLiteral("bytes %1-%2/%3") // e.g. "bytes 0-25/128"
.arg(m_nextFileUploadPos)
.arg(m_nextFileUploadPos + data.size() - 1) // -1 because range is inclusive
.arg(m_uploadFile->size());

QNetworkRequest req(url);
req.setHeader(QNetworkRequest::ContentLengthHeader, data.size());
req.setRawHeader(QByteArrayLiteral("Content-Range"), contentRange.toLatin1());
req.setRawHeader(QByteArrayLiteral("Authorization"),
QByteArrayLiteral("Bearer ") + accessToken.toUtf8());
SOCIALD_LOG_DEBUG("Attempting to upload" << contentRange << "of file:"
<< m_localFileInfo.filePath()
<< "via request:" << url.toString());
reply = m_networkAccessManager->put(req, data);
}

if (reply) {
Expand All @@ -843,10 +880,12 @@ void OneDriveBackupOperationSyncAdaptor::uploadData(int accountId, const QString
connect(reply, SIGNAL(error(QNetworkReply::NetworkError)), this, SLOT(errorHandler(QNetworkReply::NetworkError)));
connect(reply, SIGNAL(sslErrors(QList<QSslError>)), this, SLOT(sslErrorsHandler(QList<QSslError>)));
if (localFile.isEmpty()) {
connect(reply, SIGNAL(finished()), this, SLOT(createRemotePathFinishedHandler()));
connect(reply, &QNetworkReply::finished, this, &OneDriveBackupOperationSyncAdaptor::createRemotePathFinishedHandler);
} else if (m_uploadSessionUrl.isEmpty()) {
connect(reply, &QNetworkReply::finished, this, &OneDriveBackupOperationSyncAdaptor::createUploadSessionFinishedHandler);
} else {
connect(reply, SIGNAL(uploadProgress(qint64,qint64)), this, SLOT(uploadProgressHandler(qint64,qint64)));
connect(reply, SIGNAL(finished()), this, SLOT(createRemoteFileFinishedHandler()));
connect(reply, &QNetworkReply::uploadProgress, this, &OneDriveBackupOperationSyncAdaptor::uploadProgressHandler);
connect(reply, &QNetworkReply::finished, this, &OneDriveBackupOperationSyncAdaptor::filePartUploadFinishedHandler);
}

// we're requesting data. Increment the semaphore so that we know we're still busy.
Expand Down Expand Up @@ -918,29 +957,101 @@ void OneDriveBackupOperationSyncAdaptor::createRemotePathFinishedHandler()
decrementSemaphore(accountId);
}

void OneDriveBackupOperationSyncAdaptor::createRemoteFileFinishedHandler()
void OneDriveBackupOperationSyncAdaptor::createUploadSessionFinishedHandler()
{
QNetworkReply *reply = qobject_cast<QNetworkReply*>(sender());
QByteArray data = reply->readAll();
int accountId = reply->property("accountId").toInt();
QString localPath = reply->property("localPath").toString();
QString remotePath = reply->property("remotePath").toString();
QString localFile = reply->property("localFile").toString();
bool isError = reply->property("isError").toBool();
int httpCode = reply->attribute(QNetworkRequest::HttpStatusCodeAttribute).toInt();
const QByteArray data = reply->readAll();
const int accountId = reply->property("accountId").toInt();
const QString localPath = reply->property("localPath").toString();
const QString remotePath = reply->property("remotePath").toString();
const QString localFile = reply->property("localFile").toString();
const QString accessToken = reply->property("accessToken").toString();
const bool isError = reply->property("isError").toBool();
const int httpCode = reply->attribute(QNetworkRequest::HttpStatusCodeAttribute).toInt();
reply->deleteLater();
removeReplyTimeout(accountId, reply);
if (isError) {

bool ok = false;
const QJsonObject parsed = parseJsonObjectReplyData(data, &ok);
const QString uploadSessionUrl = parsed.value("uploadUrl").toString();
if (isError || !ok || uploadSessionUrl.isEmpty()) {
SOCIALD_LOG_ERROR("failed to read uploadUrl from createUploadSessionRequest for path" << remotePath
<< "to upload file" << localPath << localFile
<< "for OneDrive account:" << accountId << ", code:" << httpCode
<< "response:" << data);
debugDumpJsonResponse(data);
setStatus(SocialNetworkSyncAdaptor::Error);
} else {
m_uploadSessionUrl = uploadSessionUrl;

SOCIALD_LOG_DEBUG("successfully created upload session to upload to:" << m_uploadSessionUrl
<< localPath << localFile << "to:" << remotePath
<< "for OneDrive account:" << accountId);
uploadData(accountId, accessToken, localPath, remotePath, localFile);
}

decrementSemaphore(accountId);
}

void OneDriveBackupOperationSyncAdaptor::filePartUploadFinishedHandler()
{
QNetworkReply *reply = qobject_cast<QNetworkReply*>(sender());
const QByteArray data = reply->readAll();
const int accountId = reply->property("accountId").toInt();
const QString localPath = reply->property("localPath").toString();
const QString remotePath = reply->property("remotePath").toString();
const QString localFile = reply->property("localFile").toString();
const QString accessToken = reply->property("accessToken").toString();

const int httpCode = reply->attribute(QNetworkRequest::HttpStatusCodeAttribute).toInt();
reply->deleteLater();
removeReplyTimeout(accountId, reply);

int nextRangeStart = 0;

if (httpCode == 200 || httpCode == 201) { // OK or Created
SOCIALD_LOG_DEBUG("successfully uploaded backup of file:" << localPath << localFile << "to:" << remotePath <<
"for OneDrive account:" << accountId);
} else if (httpCode == 202) { // Accepted
bool ok = false;
const QJsonObject parsed = parseJsonObjectReplyData(data, &ok);
const QJsonArray nextExpectedRanges = parsed.value("nextExpectedRanges").toArray();

for (const QJsonValue &value : nextExpectedRanges) {
const QString range = value.toString();
const int sepIndex = range.indexOf('-');
if (sepIndex > 0) {
nextRangeStart = range.mid(0, sepIndex).toInt();
break;
}
}

if (nextRangeStart == 0) {
SOCIALD_LOG_ERROR("Cannot find nextExpectedRanges data to upload next part of"
<< m_localFileInfo.filePath() << "to" << remotePath
<< "for OneDrive account:" << accountId << ", code:" << httpCode
<< "response:" << data);
debugDumpJsonResponse(data);
setStatus(SocialNetworkSyncAdaptor::Error);
}

} else {
SOCIALD_LOG_ERROR("failed to backup file:" << localPath << localFile << "to:" << remotePath <<
"for OneDrive account:" << accountId << ", code:" << httpCode);
"for OneDrive account:" << accountId << ", code:" << httpCode
<< "response:" << data);
debugDumpJsonResponse(data);
setStatus(SocialNetworkSyncAdaptor::Error);
decrementSemaphore(accountId);
return;
}

SOCIALD_LOG_DEBUG("successfully uploaded backup of file:" << localPath << localFile << "to:" << remotePath <<
"for OneDrive account:" << accountId);
if (nextRangeStart > 0) {
m_nextFileUploadPos = nextRangeStart;
uploadData(accountId, accessToken, localPath, remotePath, localFile);
} else {
m_uploadFile->close();
delete m_uploadFile;
m_uploadFile = nullptr;
}

decrementSemaphore(accountId);
}

Expand Down
8 changes: 7 additions & 1 deletion src/onedrive/onedrivebackupoperationsyncadaptor.h
View file
Expand Up @@ -80,7 +80,8 @@ private Q_SLOTS:
void remotePathFinishedHandler();
void remoteFileFinishedHandler();
void createRemotePathFinishedHandler();
void createRemoteFileFinishedHandler();
void filePartUploadFinishedHandler();
void createUploadSessionFinishedHandler();
void downloadProgressHandler(qint64 bytesReceived, qint64 bytesTotal);
void uploadProgressHandler(qint64 bytesSent, qint64 bytesTotal);

Expand All @@ -102,7 +103,12 @@ private Q_SLOTS:
QList<RemoteDirectory> m_remoteDirectories;
QString m_accessToken;
QString m_remoteDirPath;
QString m_uploadSessionUrl;
QFileInfo m_localFileInfo;

QFile *m_uploadFile = nullptr;
qint64 m_nextFileUploadPos = 0;

int m_accountId = 0;
};

Expand Down
5 changes: 4 additions & 1 deletion src/onedrive/onedrivedatatypesyncadaptor.cpp
View file
Expand Up @@ -93,6 +93,8 @@ void OneDriveDataTypeSyncAdaptor::errorHandler(QNetworkReply::NetworkError err)
{
// OneDrive sends error code 204 (HTTP code 401) for Unauthorized Error
QNetworkReply *reply = qobject_cast<QNetworkReply*>(sender());
const int httpCode = reply->attribute(QNetworkRequest::HttpStatusCodeAttribute).toInt();

if (err == QNetworkReply::AuthenticationRequiredError) {
int httpCode = reply->attribute(QNetworkRequest::HttpStatusCodeAttribute).toInt();
SOCIALD_LOG_INFO("sociald:OneDrive: received:" << httpCode <<
Expand All @@ -103,7 +105,8 @@ void OneDriveDataTypeSyncAdaptor::errorHandler(QNetworkReply::NetworkError err)

SOCIALD_LOG_ERROR(SocialNetworkSyncAdaptor::dataTypeName(m_dataType) <<
"request with account" << sender()->property("accountId").toInt() <<
"experienced error:" << err);
"experienced error:" << err << "HTTP code:" << httpCode
<< "data:" << reply->readAll());
// set "isError" on the reply so that adapters know to ignore the result in the finished() handler
reply->setProperty("isError", QVariant::fromValue<bool>(true));
// Note: not all errors are "unrecoverable" errors, so we don't change the status here.
Expand Down

0 comments on commit aef9916

Please sign in to comment.