Commit 5c0e80da authored by Paweł Wegner's avatar Paweł Wegner

Optimized upload

Uploaded file isn't loaded to memory anymore.
parent 738d4f12
......@@ -213,7 +213,7 @@ HttpRequest::Pointer CloudProvider::listDirectoryRequest(const IItem&,
HttpRequest::Pointer CloudProvider::uploadFileRequest(const IItem&,
const std::string&,
std::istream&,
std::ostream&,
std::ostream&) const {
return nullptr;
}
......
......@@ -85,8 +85,8 @@ class CloudProvider : public ICloudProvider,
const IItem&, const std::string& page_token,
std::ostream& input_stream) const;
virtual HttpRequest::Pointer uploadFileRequest(
const IItem& directory, const std::string& filename, std::istream& stream,
std::ostream& input_stream) const;
const IItem& directory, const std::string& filename,
std::ostream& prefix_stream, std::ostream& suffix_stream) const;
virtual HttpRequest::Pointer downloadFileRequest(
const IItem&, std::ostream& input_stream) const;
virtual HttpRequest::Pointer getThumbnailRequest(
......
......@@ -62,8 +62,8 @@ HttpRequest::Pointer GoogleDrive::listDirectoryRequest(
}
HttpRequest::Pointer GoogleDrive::uploadFileRequest(
const IItem& f, const std::string& filename, std::istream& stream,
std::ostream& input_stream) const {
const IItem& f, const std::string& filename, std::ostream& prefix_stream,
std::ostream& suffix_stream) const {
const std::string separator = "fWoDm9QNn3v3Bq3bScUX";
const Item& item = static_cast<const Item&>(f);
HttpRequest::Pointer request = make_unique<HttpRequest>(
......@@ -76,13 +76,12 @@ HttpRequest::Pointer GoogleDrive::uploadFileRequest(
request_data["parents"].append(item.id());
std::string json_data = Json::FastWriter().write(request_data);
json_data.pop_back();
input_stream << "--" << separator << "\r\n"
<< "Content-Type: application/json; charset=UTF-8\r\n\r\n"
<< json_data << "\r\n"
<< "--" << separator << "\r\n"
<< "Content-Type: \r\n\r\n"
<< stream.rdbuf() << "\r\n"
<< "--" << separator << "--";
prefix_stream << "--" << separator << "\r\n"
<< "Content-Type: application/json; charset=UTF-8\r\n\r\n"
<< json_data << "\r\n"
<< "--" << separator << "\r\n"
<< "Content-Type: \r\n\r\n";
suffix_stream << "\r\n--" << separator << "--\r\n";
return request;
}
......
......@@ -43,8 +43,8 @@ class GoogleDrive : public CloudProvider {
std::ostream& input_stream) const;
HttpRequest::Pointer uploadFileRequest(const IItem& directory,
const std::string& filename,
std::istream& stream,
std::ostream& input_stream) const;
std::ostream& prefix_stream,
std::ostream& suffix_stream) const;
HttpRequest::Pointer downloadFileRequest(const IItem&,
std::ostream& input_stream) const;
HttpRequest::Pointer deleteItemRequest(const IItem&,
......
......@@ -150,14 +150,14 @@ ICloudProvider::UploadFileRequest::Pointer YandexDisk::uploadFileAsync(
output >> response;
std::string url = response["href"].asString();
UploadStreamWrapper wrapper(
std::bind(&IUploadFileCallback::putData, callback.get(), _1, _2));
std::istream stream(&wrapper);
std::bind(&IUploadFileCallback::putData, callback.get(), _1, _2),
callback->size());
code = r->sendRequest(
[url, callback, &stream](std::ostream& input) {
[url, callback, &wrapper](std::ostream& input) {
auto request =
make_unique<HttpRequest>(url, HttpRequest::Type::PUT);
callback->reset();
input << stream.rdbuf();
input.rdbuf(&wrapper);
return request;
},
output, nullptr,
......
......@@ -74,6 +74,7 @@ class IUploadFileCallback {
virtual void reset() = 0;
virtual uint32_t putData(char* data, uint32_t maxlength) = 0;
virtual uint64_t size() = 0;
virtual void done() = 0;
virtual void error(const std::string& description) = 0;
virtual void progress(uint32_t total, uint32_t now) = 0;
......
......@@ -36,7 +36,8 @@ UploadFileRequest::UploadFileRequest(
: Request(p),
directory_(std::move(directory)),
filename_(filename),
stream_wrapper_(std::bind(&ICallback::putData, callback.get(), _1, _2)),
stream_wrapper_(std::bind(&ICallback::putData, callback.get(), _1, _2),
callback->size()),
callback_(callback) {
if (!stream_wrapper_.callback_)
throw std::logic_error("Callback can't be null.");
......@@ -49,9 +50,13 @@ UploadFileRequest::UploadFileRequest(
int code = sendRequest(
[this](std::ostream& input) {
callback_->reset();
std::istream stream(&stream_wrapper_);
return provider()->uploadFileRequest(*directory_, filename_, stream,
input);
stream_wrapper_.prefix_ = std::stringstream();
stream_wrapper_.suffix_ = std::stringstream();
stream_wrapper_.read_ = 0;
input.rdbuf(&stream_wrapper_);
return provider()->uploadFileRequest(*directory_, filename_,
stream_wrapper_.prefix_,
stream_wrapper_.suffix_);
},
response_stream, nullptr,
std::bind(&ICallback::progress, callback_.get(), _1, _2));
......@@ -66,12 +71,36 @@ void UploadFileRequest::error(int code, const std::string& description) {
}
UploadStreamWrapper::UploadStreamWrapper(
std::function<uint32_t(char*, uint32_t)> callback)
: callback_(std::move(callback)) {}
std::function<uint32_t(char*, uint32_t)> callback, uint64_t size)
: callback_(std::move(callback)), size_(size), read_(), position_() {}
UploadStreamWrapper::pos_type UploadStreamWrapper::seekoff(
off_type off, std::ios_base::seekdir way, std::ios_base::openmode) {
if (off != 0) return pos_type(off_type(-1));
if (way == std::ios_base::beg)
return position_ = 0;
else if (way == std::ios_base::end)
return position_ = prefix_.str().size() + size_ + suffix_.str().size();
else
return position_;
}
std::streambuf::int_type UploadStreamWrapper::underflow() {
uint32_t size = callback_(buffer_, BUFFER_SIZE);
if (gptr() == egptr()) setg(buffer_, buffer_, buffer_ + size);
pos_type read_data = 0;
if (prefix_) {
prefix_.read(buffer_ + read_data, BUFFER_SIZE - read_data);
read_data += prefix_.gcount();
}
if (read_ < size_ && !prefix_) {
uint32_t size = callback_(buffer_ + read_data, BUFFER_SIZE - read_data);
read_data += size;
read_ += size;
}
if (read_ == size_ && !prefix_) {
suffix_.read(buffer_ + read_data, BUFFER_SIZE - read_data);
read_data += suffix_.gcount();
}
if (gptr() == egptr()) setg(buffer_, buffer_, buffer_ + read_data);
return gptr() == egptr() ? std::char_traits<char>::eof()
: std::char_traits<char>::to_int_type(*gptr());
}
......
......@@ -33,12 +33,19 @@ class UploadStreamWrapper : public std::streambuf {
public:
static constexpr uint32_t BUFFER_SIZE = 1024;
UploadStreamWrapper(std::function<uint32_t(char*, uint32_t)> callback);
UploadStreamWrapper(std::function<uint32_t(char*, uint32_t)> callback,
uint64_t size);
pos_type seekoff(off_type, std::ios_base::seekdir, std::ios_base::openmode);
int_type underflow();
char buffer_[BUFFER_SIZE];
std::function<uint32_t(char*, uint32_t)> callback_;
std::stringstream prefix_;
std::stringstream suffix_;
uint64_t size_;
uint64_t read_;
pos_type position_;
};
class UploadFileRequest : public Request<void> {
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment