whytofly
/
qt6-bb10
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Make use of QDecompressHelper for HTTP downloads 

Changes are not too big for now. Just replaces use of the previous
calls to the zlib decompression function. And initialize
QDecompressHelper when we know the content-encoding.

Task-number: QTBUG-83269
Change-Id: I41358feaef2e7ac5f48f14e3f95ec094e0c110b7
Reviewed-by: Timur Pocheptsov <timur.pocheptsov@qt.io>
bb10
Mårten Nordheim 2020-05-07 17:34:22 +02:00
parent 7ec0786d78
commit 7b76379a89
7 changed files with 70 additions and 142 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
src/network/access/qdecompresshelper.cpp
View File

@ -79,6 +79,19 @@ bool QDecompressHelper::isSupportedEncoding(const QByteArray &encoding)
return encodingFromByteArray(encoding) != QDecompressHelper::None;
}
QByteArrayList QDecompressHelper::acceptedEncoding()
{
static QByteArrayList accepted = []() {
QByteArrayList list;
list.reserve(sizeof(contentEncodingMapping) / sizeof(contentEncodingMapping[0]));
for (const auto &mapping : contentEncodingMapping) {
list << QByteArray(mapping.name);
}
return list;
}();
return accepted;
}
QDecompressHelper::~QDecompressHelper()
{
clear();

1
src/network/access/qdecompresshelper_p.h
View File

@ -90,6 +90,7 @@ public:
void clear();
static bool isSupportedEncoding(const QByteArray &encoding);
static QByteArrayList acceptedEncoding();
private:
bool countInternal();

22
src/network/access/qhttp2protocolhandler.cpp
View File

@ -1163,8 +1163,11 @@ void QHttp2ProtocolHandler::updateStream(Stream &stream, const HPack::HttpHeader
if (QHttpNetworkReply::isHttpRedirect(statusCode) && redirectUrl.isValid())
httpReply->setRedirectUrl(redirectUrl);
if (httpReplyPrivate->isCompressed() && httpRequest.d->autoDecompress)
if (httpReplyPrivate->isCompressed() && httpRequest.d->autoDecompress) {
httpReplyPrivate->removeAutoDecompressHeader();
httpReplyPrivate->decompressHelper.setEncoding(
httpReplyPrivate->headerField("content-encoding"));
}
if (QHttpNetworkReply::isHttpRedirect(statusCode)
|| statusCode == 401 || statusCode == 407) {
@ -1207,12 +1210,17 @@ void QHttp2ProtocolHandler::updateStream(Stream &stream, const Frame &frame,
const QByteArray wrapped(data, length);
if (httpRequest.d->autoDecompress && replyPrivate->isCompressed()) {
QByteDataBuffer inDataBuffer;
inDataBuffer.append(wrapped);
replyPrivate->uncompressBodyData(&inDataBuffer, &replyPrivate->responseData);
// Now, make sure replyPrivate's destructor will properly clean up
// buffers allocated (if any) by zlib.
replyPrivate->autoDecompress = true;
Q_ASSERT(replyPrivate->decompressHelper.isValid());
replyPrivate->decompressHelper.feed(wrapped);
while (replyPrivate->decompressHelper.hasData()) {
QByteArray output(4 * 1024, Qt::Uninitialized);
qint64 read = replyPrivate->decompressHelper.read(output.data(), output.size());
if (read > 0) {
output.resize(read);
replyPrivate->responseData.append(std::move(output));
}
}
} else {
replyPrivate->responseData.append(wrapped);
}

3
src/network/access/qhttpnetworkconnection.cpp
View File

@ -297,7 +297,8 @@ void QHttpNetworkConnectionPrivate::prepareRequest(HttpMessagePair &messagePair)
value = request.headerField("accept-encoding");
if (value.isEmpty()) {
#ifndef QT_NO_COMPRESS
request.setHeaderField("Accept-Encoding", "gzip, deflate");
const QByteArrayList &acceptedEncoding = QDecompressHelper::acceptedEncoding();
request.setHeaderField("Accept-Encoding", acceptedEncoding.join(", "));
request.d->autoDecompress = true;
#else
// if zlib is not available set this to false always

149
src/network/access/qhttpnetworkreply.cpp
View File

@ -46,10 +46,6 @@
# include <QtNetwork/qsslconfiguration.h>
#endif
#ifndef QT_NO_COMPRESS
#include <zlib.h>
#endif
QT_BEGIN_NAMESPACE
QHttpNetworkReply::QHttpNetworkReply(const QUrl &url, QObject *parent)
@ -63,11 +59,6 @@ QHttpNetworkReply::~QHttpNetworkReply()
if (d->connection) {
d->connection->d_func()->removeReply(this);
}
#ifndef QT_NO_COMPRESS
if (d->autoDecompress && d->isCompressed() && d->inflateStrm)
inflateEnd(d->inflateStrm);
#endif
}
QUrl QHttpNetworkReply::url() const
@ -335,9 +326,6 @@ QHttpNetworkReplyPrivate::QHttpNetworkReplyPrivate(const QUrl &newUrl)
autoDecompress(false), responseData(), requestIsPrepared(false)
,pipeliningUsed(false), h2Used(false), downstreamLimited(false)
,userProvidedDownloadBuffer(nullptr)
#ifndef QT_NO_COMPRESS
,inflateStrm(nullptr)
#endif
{
QString scheme = newUrl.scheme();
@ -347,13 +335,7 @@ QHttpNetworkReplyPrivate::QHttpNetworkReplyPrivate(const QUrl &newUrl)
connectionCloseEnabled = false;
}
QHttpNetworkReplyPrivate::~QHttpNetworkReplyPrivate()
{
#ifndef QT_NO_COMPRESS
if (inflateStrm)
delete inflateStrm;
#endif
}
QHttpNetworkReplyPrivate::~QHttpNetworkReplyPrivate() = default;
void QHttpNetworkReplyPrivate::clearHttpLayerInformation()
{
@ -366,10 +348,7 @@ void QHttpNetworkReplyPrivate::clearHttpLayerInformation()
currentChunkRead = 0;
lastChunkRead = false;
connectionCloseEnabled = true;
#ifndef QT_NO_COMPRESS
if (autoDecompress && inflateStrm)
inflateEnd(inflateStrm);
#endif
decompressHelper.clear();
fields.clear();
}
@ -388,11 +367,15 @@ qint64 QHttpNetworkReplyPrivate::bytesAvailable() const
return (state != ReadingDataState ? 0 : fragment.size());
}
bool QHttpNetworkReplyPrivate::isCompressed()
bool QHttpNetworkReplyPrivate::isCompressed() const
{
QByteArray encoding = headerField("content-encoding");
return encoding.compare("gzip", Qt::CaseInsensitive) == 0 ||
encoding.compare("deflate", Qt::CaseInsensitive) == 0;
return QDecompressHelper::isSupportedEncoding(headerField("content-encoding"));
}
bool QHttpNetworkReply::isCompressed() const
{
Q_D(const QHttpNetworkReply);
return d->isCompressed();
}
void QHttpNetworkReplyPrivate::removeAutoDecompressHeader()
@ -596,18 +579,10 @@ qint64 QHttpNetworkReplyPrivate::readHeader(QAbstractSocket *socket)
headerField("proxy-connection").toLower().contains("close")) ||
(majorVersion == 1 && minorVersion == 0 &&
(connectionHeaderField.isEmpty() && !headerField("proxy-connection").toLower().contains("keep-alive")));
#ifndef QT_NO_COMPRESS
if (autoDecompress && isCompressed()) {
// allocate inflate state
if (!inflateStrm)
inflateStrm = new z_stream;
int ret = initializeInflateStream();
if (ret != Z_OK)
return -1;
if (!decompressHelper.setEncoding(headerField("content-encoding")))
return -1; // Either the encoding was unsupported or the decoder could not be set up
}
#endif
}
return bytes;
}
@ -709,12 +684,8 @@ qint64 QHttpNetworkReplyPrivate::readBody(QAbstractSocket *socket, QByteDataBuff
{
qint64 bytes = 0;
#ifndef QT_NO_COMPRESS
// for gzip we'll allocate a temporary one that we then decompress
// for compressed data we'll allocate a temporary one that we then decompress
QByteDataBuffer *tempOutDataBuffer = (autoDecompress ? new QByteDataBuffer : out);
#else
QByteDataBuffer *tempOutDataBuffer = out;
#endif
if (isChunked()) {
@ -730,95 +701,29 @@ qint64 QHttpNetworkReplyPrivate::readBody(QAbstractSocket *socket, QByteDataBuff
bytes += readReplyBodyRaw(socket, tempOutDataBuffer, socket->bytesAvailable());
}
#ifndef QT_NO_COMPRESS
// This is true if there is compressed encoding and we're supposed to use it.
if (autoDecompress) {
qint64 uncompressRet = uncompressBodyData(tempOutDataBuffer, out);
delete tempOutDataBuffer;
if (uncompressRet < 0)
QScopedPointer holder(tempOutDataBuffer);
if (!decompressHelper.isValid())
return -1;
decompressHelper.feed(std::move(*tempOutDataBuffer));
while (decompressHelper.hasData()) {
QByteArray output(4 * 1024, Qt::Uninitialized);
qint64 read = decompressHelper.read(output.data(), output.size());
if (read < 0) {
return -1;
} else if (read > 0) {
output.resize(read);
out->append(std::move(output));
}
}
}
#endif
contentRead += bytes;
return bytes;
}
#ifndef QT_NO_COMPRESS
int QHttpNetworkReplyPrivate::initializeInflateStream()
{
Q_ASSERT(inflateStrm);
inflateStrm->zalloc = Z_NULL;
inflateStrm->zfree = Z_NULL;
inflateStrm->opaque = Z_NULL;
inflateStrm->avail_in = 0;
inflateStrm->next_in = Z_NULL;
// "windowBits can also be greater than 15 for optional gzip decoding.
// Add 32 to windowBits to enable zlib and gzip decoding with automatic header detection"
// http://www.zlib.net/manual.html
int ret = inflateInit2(inflateStrm, MAX_WBITS+32);
Q_ASSERT(ret == Z_OK);
return ret;
}
qint64 QHttpNetworkReplyPrivate::uncompressBodyData(QByteDataBuffer *in, QByteDataBuffer *out)
{
if (!inflateStrm) { // happens when called from the SPDY protocol handler
inflateStrm = new z_stream;
initializeInflateStream();
}
if (!inflateStrm)
return -1;
bool triedRawDeflate = false;
for (int i = 0; i < in->bufferCount(); i++) {
QByteArray &bIn = (*in)[i];
inflateStrm->avail_in = bIn.size();
inflateStrm->next_in = reinterpret_cast<Bytef*>(bIn.data());
do {
QByteArray bOut;
// make a wild guess about the uncompressed size.
bOut.reserve(inflateStrm->avail_in * 3 + 512);
inflateStrm->avail_out = bOut.capacity();
inflateStrm->next_out = reinterpret_cast<Bytef*>(bOut.data());
int ret = inflate(inflateStrm, Z_NO_FLUSH);
//All negative return codes are errors, in the context of HTTP compression, Z_NEED_DICT is also an error.
// in the case where we get Z_DATA_ERROR this could be because we received raw deflate compressed data.
if (ret == Z_DATA_ERROR && !triedRawDeflate) {
inflateEnd(inflateStrm);
triedRawDeflate = true;
inflateStrm->zalloc = Z_NULL;
inflateStrm->zfree = Z_NULL;
inflateStrm->opaque = Z_NULL;
inflateStrm->avail_in = 0;
inflateStrm->next_in = Z_NULL;
int ret = inflateInit2(inflateStrm, -MAX_WBITS);
if (ret != Z_OK) {
return -1;
} else {
inflateStrm->avail_in = bIn.size();
inflateStrm->next_in = reinterpret_cast<Bytef*>(bIn.data());
continue;
}
} else if (ret < 0 || ret == Z_NEED_DICT) {
return -1;
}
bOut.resize(bOut.capacity() - inflateStrm->avail_out);
out->append(bOut);
if (ret == Z_STREAM_END)
return out->byteAmount();
} while (inflateStrm->avail_in > 0);
}
return out->byteAmount();
}
#endif
qint64 QHttpNetworkReplyPrivate::readReplyBodyRaw(QAbstractSocket *socket, QByteDataBuffer *out, qint64 size)
{
// FIXME get rid of this function and just use readBodyFast and give it socket->bytesAvailable()

16
src/network/access/qhttpnetworkreply_p.h
View File

@ -55,10 +55,6 @@
#include <qplatformdefs.h>
#ifndef QT_NO_COMPRESS
struct z_stream_s;
#endif
#include <QtNetwork/qtcpsocket.h>
// it's safe to include these even if SSL support is not enabled
#include <QtNetwork/qsslsocket.h>
@ -80,6 +76,8 @@ Q_MOC_INCLUDE(<QtNetwork/QNetworkProxy>)
#endif
Q_MOC_INCLUDE(<QtNetwork/QAuthenticator>)
#include <private/qdecompresshelper_p.h>
QT_REQUIRE_CONFIG(http);
QT_BEGIN_NAMESPACE
@ -157,6 +155,8 @@ public:
static bool isHttpRedirect(int statusCode);
bool isCompressed() const;
#ifndef QT_NO_SSL
QSslConfiguration sslConfiguration() const;
void setSslConfiguration(const QSslConfiguration &config);
@ -224,7 +224,7 @@ public:
bool isChunked();
bool isConnectionCloseEnabled();
bool isCompressed();
bool isCompressed() const;
void removeAutoDecompressHeader();
enum ReplyState {
@ -276,11 +276,7 @@ public:
char* userProvidedDownloadBuffer;
QUrl redirectUrl;
#ifndef QT_NO_COMPRESS
z_stream_s *inflateStrm;
int initializeInflateStream();
qint64 uncompressBodyData(QByteDataBuffer *in, QByteDataBuffer *out);
#endif
QDecompressHelper decompressHelper;
};

8
tests/auto/network/access/qdecompresshelper/tst_qdecompresshelper.cpp
View File

@ -82,11 +82,15 @@ void tst_QDecompressHelper::cleanupTestCase()
void tst_QDecompressHelper::encodingSupported()
{
QVERIFY(!QDecompressHelper::isSupportedEncoding("identity"));
QVERIFY(!QDecompressHelper::isSupportedEncoding("fake"));
const QByteArrayList &accepted = QDecompressHelper::acceptedEncoding();
QVERIFY(QDecompressHelper::isSupportedEncoding("deflate"));
QVERIFY(accepted.contains("deflate"));
QVERIFY(QDecompressHelper::isSupportedEncoding("gzip"));
QVERIFY(accepted.contains("gzip"));
int expected = 2;
QCOMPARE(expected, accepted.size());
}
void tst_QDecompressHelper::sharedDecompress_data()