From 7ce6920aacfcba485cd8017e01c6aeb324292e75 Mon Sep 17 00:00:00 2001 From: Giuseppe D'Angelo Date: Mon, 19 Feb 2024 13:05:54 +0100 Subject: [PATCH] Containers: add max_size() One more method for STL compatibility. This one is particularly subtle as it's required by the `reservable-container` concept: https://eel.is/c++draft/ranges#range.utility.conv.general-3 Without this concept, ranges::to won't reserve() before copying the elements (out of a sized range which isn't a common_range). Implementation notes: there were already a couple of constants denoting the maximum QByteArray and QString size. Centralize that implementation in QTypedArrayData, so that QList can use it too. The maximum allocation size (private constant) needs a even more central place so that even QVLA can use it. Lacking anything better, I've put it in qcontainerfwd.h. Since our containers aren't allocator-aware, I can make max_size() a static member, and replace the existing constants throughout the rest of qtbase. (I can't kill them yet as they're used by other submodules.) [ChangeLog][QtCore][QList] Added max_size(). [ChangeLog][QtCore][QString] Added max_size(). [ChangeLog][QtCore][QByteArray] Added max_size(). [ChangeLog][QtCore][QVarLengthArray] Added max_size(). Change-Id: I176142e31b998f4f787c96333894b8f6653eb70d Reviewed-by: Qt CI Bot Reviewed-by: Thiago Macieira --- src/corelib/io/qiodevice.cpp | 13 ++++++------- src/corelib/serialization/qcborstreamreader.cpp | 5 ++--- src/corelib/serialization/qcborvalue.cpp | 9 ++++----- src/corelib/text/qbytearray.cpp | 12 ++++++++++-- src/corelib/text/qbytearray.h | 5 +++++ src/corelib/text/qbytearray_p.h | 5 ++--- src/corelib/text/qstring.cpp | 10 ++++++++++ src/corelib/text/qstring.h | 5 +++++ src/corelib/tools/qarraydata.h | 7 +++++++ src/corelib/tools/qcontainerfwd.h | 7 +++++++ src/corelib/tools/qlist.h | 4 ++++ src/corelib/tools/qlist.qdoc | 9 +++++++++ src/corelib/tools/qringbuffer.cpp | 9 ++++----- src/corelib/tools/qtools_p.h | 3 --- src/corelib/tools/qvarlengtharray.h | 7 +++++++ src/corelib/tools/qvarlengtharray.qdoc | 9 +++++++++ src/network/access/qdecompresshelper.cpp | 1 - src/network/socket/qsctpsocket.cpp | 3 +-- .../qcborstreamreader/tst_qcborstreamreader.cpp | 6 ++---- .../serialization/qcborvalue/tst_qcborvalue.cpp | 6 ++---- 20 files changed, 96 insertions(+), 39 deletions(-) diff --git a/src/corelib/io/qiodevice.cpp b/src/corelib/io/qiodevice.cpp index 723cc3e323..b0029e2af7 100644 --- a/src/corelib/io/qiodevice.cpp +++ b/src/corelib/io/qiodevice.cpp @@ -9,7 +9,6 @@ #include "qfile.h" #include "qstringlist.h" #include "qdir.h" -#include "private/qbytearray_p.h" #include "private/qtools_p.h" #include @@ -89,9 +88,9 @@ static void checkWarnMessage(const QIODevice *device, const char *function, cons #define CHECK_MAXBYTEARRAYSIZE(function) \ do { \ - if (maxSize >= MaxByteArraySize) { \ + if (maxSize >= QByteArray::max_size()) { \ checkWarnMessage(this, #function, "maxSize argument exceeds QByteArray size limit"); \ - maxSize = MaxByteArraySize - 1; \ + maxSize = QByteArray::max_size() - 1; \ } \ } while (0) @@ -1243,7 +1242,7 @@ QByteArray QIODevice::readAll() : d->buffer.size()); qint64 readResult; do { - if (readBytes + readChunkSize >= MaxByteArraySize) { + if (readBytes + readChunkSize >= QByteArray::max_size()) { // If resize would fail, don't read more, return what we have. break; } @@ -1257,8 +1256,8 @@ QByteArray QIODevice::readAll() } else { // Read it all in one go. readBytes -= d->pos; - if (readBytes >= MaxByteArraySize) - readBytes = MaxByteArraySize; + if (readBytes >= QByteArray::max_size()) + readBytes = QByteArray::max_size(); result.resize(readBytes); readBytes = d->read(result.data(), readBytes); } @@ -1449,7 +1448,7 @@ QByteArray QIODevice::readLine(qint64 maxSize) qint64 readBytes = 0; if (maxSize == 0) { // Size is unknown, read incrementally. - maxSize = MaxByteArraySize - 1; + maxSize = QByteArray::max_size() - 1; // The first iteration needs to leave an extra byte for the terminating null result.resize(1); diff --git a/src/corelib/serialization/qcborstreamreader.cpp b/src/corelib/serialization/qcborstreamreader.cpp index 1b7847de4e..ccddfa534e 100644 --- a/src/corelib/serialization/qcborstreamreader.cpp +++ b/src/corelib/serialization/qcborstreamreader.cpp @@ -6,7 +6,6 @@ #define CBOR_NO_ENCODER_API #include -#include #include #include #include @@ -1752,7 +1751,7 @@ QCborStreamReaderPrivate::readStringChunk_byte(ReadStringChunk params, qsizetype // the distinction between DataTooLarge and OOM is mostly for // compatibility with Qt 5; in Qt 6, we could consider everything // to be OOM. - handleError(newSize > MaxByteArraySize ? CborErrorDataTooLarge: CborErrorOutOfMemory); + handleError(newSize > QByteArray::max_size() ? CborErrorDataTooLarge: CborErrorOutOfMemory); return -1; } @@ -1791,7 +1790,7 @@ QCborStreamReaderPrivate::readStringChunk_unicode(ReadStringChunk params, qsizet // conversion uses the same number of words or less. qsizetype currentSize = params.string->size(); size_t newSize = size_t(utf8len) + size_t(currentSize); // can't overflow - if (utf8len > MaxStringSize || qsizetype(newSize) < 0) { + if (utf8len > QString::max_size() || qsizetype(newSize) < 0) { handleError(CborErrorDataTooLarge); return -1; } diff --git a/src/corelib/serialization/qcborvalue.cpp b/src/corelib/serialization/qcborvalue.cpp index 4fb7618017..57fdae322e 100644 --- a/src/corelib/serialization/qcborvalue.cpp +++ b/src/corelib/serialization/qcborvalue.cpp @@ -19,7 +19,6 @@ #include #include #include -#include #include #include @@ -1616,7 +1615,7 @@ void QCborContainerPrivate::decodeStringFromCbor(QCborStreamReader &reader) // add space for aligned ByteData (this can't overflow) offset += sizeof(QtCbor::ByteData) + alignof(QtCbor::ByteData); offset &= ~(alignof(QtCbor::ByteData) - 1); - if (offset > size_t(MaxByteArraySize)) { + if (offset > size_t(QByteArray::max_size())) { // overflow setErrorInReader(reader, { QCborError::DataTooLarge }); return; @@ -1629,9 +1628,9 @@ void QCborContainerPrivate::decodeStringFromCbor(QCborStreamReader &reader) // so capa how much we allocate newCapacity = offset + MaxMemoryIncrement - EstimatedOverhead; } - if (newCapacity > size_t(MaxByteArraySize)) { + if (newCapacity > size_t(QByteArray::max_size())) { // this may cause an allocation failure - newCapacity = MaxByteArraySize; + newCapacity = QByteArray::max_size(); } if (newCapacity > size_t(data.capacity())) data.reserve(newCapacity); @@ -1682,7 +1681,7 @@ void QCborContainerPrivate::decodeStringFromCbor(QCborStreamReader &reader) // check that this UTF-8 text string can be loaded onto a QString if (e.type == QCborValue::String) { - if (Q_UNLIKELY(b->len > MaxStringSize)) { + if (Q_UNLIKELY(b->len > QString::max_size())) { setErrorInReader(reader, { QCborError::DataTooLarge }); status = QCborStreamReader::Error; } diff --git a/src/corelib/text/qbytearray.cpp b/src/corelib/text/qbytearray.cpp index 3df77f179c..8e34ab0742 100644 --- a/src/corelib/text/qbytearray.cpp +++ b/src/corelib/text/qbytearray.cpp @@ -14,7 +14,6 @@ #include "private/qsimd_p.h" #include "qstringalgorithms_p.h" #include "qscopedpointer.h" -#include "qbytearray_p.h" #include "qstringconverter_p.h" #include #include @@ -780,7 +779,7 @@ QByteArray qUncompress(const uchar* data, qsizetype nbytes) return QByteArray(); } - constexpr auto MaxDecompressedSize = size_t(MaxByteArraySize); + constexpr auto MaxDecompressedSize = size_t(QByteArray::max_size()); if constexpr (MaxDecompressedSize < std::numeric_limits::max()) { if (expectedSize > MaxDecompressedSize) return tooMuchData(ZLibOp::Decompression); @@ -1389,6 +1388,15 @@ QByteArray &QByteArray::operator=(const char *str) \sa isEmpty(), resize() */ +/*! \fn qsizetype QByteArray::max_size() + \since 6.8 + + This function is provided for STL compatibility. + It returns the maximum number of elements that the byte array can + theoretically hold. In practice, the number can be much smaller, + limited by the amount of memory available to the system. +*/ + /*! \fn bool QByteArray::isEmpty() const Returns \c true if the byte array has size 0; otherwise returns \c false. diff --git a/src/corelib/text/qbytearray.h b/src/corelib/text/qbytearray.h index a673383438..248bcc0b7b 100644 --- a/src/corelib/text/qbytearray.h +++ b/src/corelib/text/qbytearray.h @@ -519,6 +519,11 @@ public: void shrink_to_fit() { squeeze(); } iterator erase(const_iterator first, const_iterator last); inline iterator erase(const_iterator it) { return erase(it, it + 1); } + static constexpr qsizetype max_size() noexcept + { + // -1 to deal with the NUL terminator + return Data::max_size() - 1; + } static QByteArray fromStdString(const std::string &s); std::string toStdString() const; diff --git a/src/corelib/text/qbytearray_p.h b/src/corelib/text/qbytearray_p.h index 8fc3d7e357..7103a0428a 100644 --- a/src/corelib/text/qbytearray_p.h +++ b/src/corelib/text/qbytearray_p.h @@ -16,13 +16,12 @@ // #include -#include "private/qtools_p.h" QT_BEGIN_NAMESPACE // -1 because of the terminating NUL -constexpr qsizetype MaxByteArraySize = MaxAllocSize - sizeof(std::remove_pointer::type) - 1; -constexpr qsizetype MaxStringSize = (MaxAllocSize - sizeof(std::remove_pointer::type)) / 2 - 1; +constexpr qsizetype MaxByteArraySize = QtPrivate::MaxAllocSize - sizeof(std::remove_pointer::type) - 1; +constexpr qsizetype MaxStringSize = (QtPrivate::MaxAllocSize - sizeof(std::remove_pointer::type)) / 2 - 1; QT_END_NAMESPACE diff --git a/src/corelib/text/qstring.cpp b/src/corelib/text/qstring.cpp index d3bb1d92d1..f8fab76d73 100644 --- a/src/corelib/text/qstring.cpp +++ b/src/corelib/text/qstring.cpp @@ -6261,6 +6261,16 @@ QString& QString::fill(QChar ch, qsizetype size) \sa isEmpty(), resize() */ +/*! + \fn qsizetype QString::max_size() + \since 6.8 + + This function is provided for STL compatibility. + It returns the maximum number of elements that the string can + theoretically hold. In practice, the number can be much smaller, + limited by the amount of memory available to the system. +*/ + /*! \fn bool QString::isNull() const Returns \c true if this string is null; otherwise returns \c false. diff --git a/src/corelib/text/qstring.h b/src/corelib/text/qstring.h index 780b3090c1..5936f76ed4 100644 --- a/src/corelib/text/qstring.h +++ b/src/corelib/text/qstring.h @@ -941,6 +941,11 @@ public: void shrink_to_fit() { squeeze(); } iterator erase(const_iterator first, const_iterator last); inline iterator erase(const_iterator it) { return erase(it, it + 1); } + static constexpr qsizetype max_size() noexcept + { + // -1 to deal with the NUL terminator + return Data::max_size() - 1; + } static inline QString fromStdString(const std::string &s); inline std::string toStdString() const; diff --git a/src/corelib/tools/qarraydata.h b/src/corelib/tools/qarraydata.h index 2900b0178e..da83fc1a21 100644 --- a/src/corelib/tools/qarraydata.h +++ b/src/corelib/tools/qarraydata.h @@ -8,6 +8,7 @@ #include #include #include +#include #include QT_BEGIN_NAMESPACE @@ -169,6 +170,12 @@ struct QTypedArrayData (quintptr(data) + sizeof(QArrayData) + alignment - 1) & ~(alignment - 1)); return static_cast(start); } + + constexpr static qsizetype max_size() noexcept + { + // -1 to deal with the pointer one-past-the-end + return (QtPrivate::MaxAllocSize - sizeof(QtPrivate::AlignedQArrayData) - 1) / sizeof(T); + } }; namespace QtPrivate { diff --git a/src/corelib/tools/qcontainerfwd.h b/src/corelib/tools/qcontainerfwd.h index fa0fc87728..d5590553fa 100644 --- a/src/corelib/tools/qcontainerfwd.h +++ b/src/corelib/tools/qcontainerfwd.h @@ -14,6 +14,7 @@ // std headers can unfortunately not be forward declared #include // std::size_t #include +#include QT_BEGIN_NAMESPACE @@ -52,6 +53,12 @@ using QVariantMap = QMap; using QVariantHash = QHash; using QVariantPair = std::pair; +namespace QtPrivate +{ +[[maybe_unused]] +constexpr qsizetype MaxAllocSize = (std::numeric_limits::max)(); +} + QT_END_NAMESPACE #endif // QCONTAINERFWD_H diff --git a/src/corelib/tools/qlist.h b/src/corelib/tools/qlist.h index 684922cf08..89e0e3f380 100644 --- a/src/corelib/tools/qlist.h +++ b/src/corelib/tools/qlist.h @@ -689,6 +689,10 @@ public: inline reference back() { return last(); } inline const_reference back() const noexcept { return last(); } void shrink_to_fit() { squeeze(); } + static qsizetype max_size() noexcept + { + return Data::max_size(); + } // comfort QList &operator+=(const QList &l) { append(l); return *this; } diff --git a/src/corelib/tools/qlist.qdoc b/src/corelib/tools/qlist.qdoc index bf0f52eb6b..e07b74cd53 100644 --- a/src/corelib/tools/qlist.qdoc +++ b/src/corelib/tools/qlist.qdoc @@ -1329,6 +1329,15 @@ returns \c false. */ +/*! \fn template qsizetype QList::max_size() + \since 6.8 + + This function is provided for STL compatibility. + It returns the maximum number of elements that the list can + theoretically hold. In practice, the number can be much smaller, + limited by the amount of memory available to the system. +*/ + /*! \fn template QList &QList::operator+=(const QList &other) Appends the items of the \a other list to this list and diff --git a/src/corelib/tools/qringbuffer.cpp b/src/corelib/tools/qringbuffer.cpp index ea4224eefe..0645759118 100644 --- a/src/corelib/tools/qringbuffer.cpp +++ b/src/corelib/tools/qringbuffer.cpp @@ -3,7 +3,6 @@ // SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #include "private/qringbuffer_p.h" -#include "private/qbytearray_p.h" #include @@ -91,7 +90,7 @@ void QRingBuffer::free(qint64 bytes) clear(); // try to minify/squeeze us } } else { - Q_ASSERT(bytes < MaxByteArraySize); + Q_ASSERT(bytes < QByteArray::max_size()); chunk.advance(bytes); bufferSize -= bytes; } @@ -106,7 +105,7 @@ void QRingBuffer::free(qint64 bytes) char *QRingBuffer::reserve(qint64 bytes) { - Q_ASSERT(bytes > 0 && bytes < MaxByteArraySize); + Q_ASSERT(bytes > 0 && bytes < QByteArray::max_size()); const qsizetype chunkSize = qMax(qint64(basicBlockSize), bytes); qsizetype tail = 0; @@ -136,7 +135,7 @@ char *QRingBuffer::reserve(qint64 bytes) */ char *QRingBuffer::reserveFront(qint64 bytes) { - Q_ASSERT(bytes > 0 && bytes < MaxByteArraySize); + Q_ASSERT(bytes > 0 && bytes < QByteArray::max_size()); const qsizetype chunkSize = qMax(qint64(basicBlockSize), bytes); if (bufferSize == 0) { @@ -182,7 +181,7 @@ void QRingBuffer::chop(qint64 bytes) clear(); // try to minify/squeeze us } } else { - Q_ASSERT(bytes < MaxByteArraySize); + Q_ASSERT(bytes < QByteArray::max_size()); chunk.grow(-bytes); bufferSize -= bytes; } diff --git a/src/corelib/tools/qtools_p.h b/src/corelib/tools/qtools_p.h index 93432a9524..105aa40c02 100644 --- a/src/corelib/tools/qtools_p.h +++ b/src/corelib/tools/qtools_p.h @@ -115,9 +115,6 @@ constexpr inline int qt_lencmp(qsizetype lhs, qsizetype rhs) noexcept } // namespace QtMiscUtils -// We typically need an extra bit for qNextPowerOfTwo when determining the next allocation size. -constexpr qsizetype MaxAllocSize = (std::numeric_limits::max)(); - struct CalculateGrowingBlockSizeResult { qsizetype size; diff --git a/src/corelib/tools/qvarlengtharray.h b/src/corelib/tools/qvarlengtharray.h index 8fa6da9a9a..afc345d3be 100644 --- a/src/corelib/tools/qvarlengtharray.h +++ b/src/corelib/tools/qvarlengtharray.h @@ -183,6 +183,12 @@ public: iterator erase(const_iterator begin, const_iterator end); iterator erase(const_iterator pos) { return erase(pos, pos + 1); } + static constexpr qsizetype max_size() noexcept + { + // -1 to deal with the pointer one-past-the-end + return (QtPrivate::MaxAllocSize / sizeof(T)) - 1; + } + size_t hash(size_t seed) const noexcept(QtPrivate::QNothrowHashable_v) { return qHashRange(begin(), end(), seed); @@ -397,6 +403,7 @@ public: } #ifdef Q_QDOC inline qsizetype size() const { return this->s; } + static constexpr qsizetype max_size() noexcept { return QVLABase::max_size(); } #endif using Base::size; inline qsizetype count() const { return size(); } diff --git a/src/corelib/tools/qvarlengtharray.qdoc b/src/corelib/tools/qvarlengtharray.qdoc index 4ebbcc43dc..4467e0c65a 100644 --- a/src/corelib/tools/qvarlengtharray.qdoc +++ b/src/corelib/tools/qvarlengtharray.qdoc @@ -140,6 +140,15 @@ \sa isEmpty(), resize() */ +/*! \fn template qsizetype QVarLengthArray::max_size() + \since 6.8 + + This function is provided for STL compatibility. + It returns the maximum number of elements that the array can + theoretically hold. In practice, the number can be much smaller, + limited by the amount of memory available to the system. +*/ + /*! \fn template T& QVarLengthArray::first() Returns a reference to the first item in the array. The array must diff --git a/src/network/access/qdecompresshelper.cpp b/src/network/access/qdecompresshelper.cpp index 677c1a6bb2..52a0d9fc06 100644 --- a/src/network/access/qdecompresshelper.cpp +++ b/src/network/access/qdecompresshelper.cpp @@ -3,7 +3,6 @@ #include "qdecompresshelper_p.h" -#include #include #include diff --git a/src/network/socket/qsctpsocket.cpp b/src/network/socket/qsctpsocket.cpp index 27c6fc930c..868c9e3785 100644 --- a/src/network/socket/qsctpsocket.cpp +++ b/src/network/socket/qsctpsocket.cpp @@ -83,7 +83,6 @@ #include "qsctpsocket_p.h" #include "qabstractsocketengine_p.h" -#include "private/qbytearray_p.h" #ifdef QSCTPSOCKET_DEBUG #include @@ -133,7 +132,7 @@ bool QSctpSocketPrivate::canReadNotification() bytesToRead = 4096; } - Q_ASSERT((datagramSize + qsizetype(bytesToRead)) < MaxByteArraySize); + Q_ASSERT((datagramSize + qsizetype(bytesToRead)) < QByteArray::max_size()); incomingDatagram.resize(datagramSize + int(bytesToRead)); #if defined (QSCTPSOCKET_DEBUG) diff --git a/tests/auto/corelib/serialization/qcborstreamreader/tst_qcborstreamreader.cpp b/tests/auto/corelib/serialization/qcborstreamreader/tst_qcborstreamreader.cpp index edd3821b22..a45b95e64e 100644 --- a/tests/auto/corelib/serialization/qcborstreamreader/tst_qcborstreamreader.cpp +++ b/tests/auto/corelib/serialization/qcborstreamreader/tst_qcborstreamreader.cpp @@ -5,8 +5,6 @@ #include #include -#include - class tst_QCborStreamReader : public QObject { Q_OBJECT @@ -920,7 +918,7 @@ void tst_QCborStreamReader::validation_data() // Add QCborStreamReader-specific limitations due to use of QByteArray and // QString, which are allocated by QArrayData::allocate(). const qsizetype MaxInvalid = std::numeric_limits::max(); - const qsizetype MinInvalid = MaxByteArraySize + 1; + const qsizetype MinInvalid = QByteArray::max_size() + 1; addValidationColumns(); addValidationData(MinInvalid); @@ -997,7 +995,7 @@ void tst_QCborStreamReader::validation() void tst_QCborStreamReader::hugeDeviceValidation_data() { - addValidationHugeDevice(MaxByteArraySize + 1, MaxStringSize + 1); + addValidationHugeDevice(QByteArray::max_size() + 1, QString::max_size() + 1); } void tst_QCborStreamReader::hugeDeviceValidation() diff --git a/tests/auto/corelib/serialization/qcborvalue/tst_qcborvalue.cpp b/tests/auto/corelib/serialization/qcborvalue/tst_qcborvalue.cpp index ec6a6afec3..a686ca0f2e 100644 --- a/tests/auto/corelib/serialization/qcborvalue/tst_qcborvalue.cpp +++ b/tests/auto/corelib/serialization/qcborvalue/tst_qcborvalue.cpp @@ -11,8 +11,6 @@ #include #include -#include - Q_DECLARE_METATYPE(QCborKnownTags) Q_DECLARE_METATYPE(QCborValue) Q_DECLARE_METATYPE(QCborValue::EncodingOptions) @@ -2221,7 +2219,7 @@ void tst_QCborValue::validation_data() // Add QCborStreamReader-specific limitations due to use of QByteArray and // QString, which are allocated by QArrayData::allocate(). const qsizetype MaxInvalid = std::numeric_limits::max(); - const qsizetype MinInvalid = MaxByteArraySize + 1 - sizeof(QByteArray::size_type); + const qsizetype MinInvalid = QByteArray::max_size() + 1 - sizeof(QByteArray::size_type); addValidationColumns(); addValidationData(MinInvalid); addValidationLargeData(MinInvalid, MaxInvalid); @@ -2394,7 +2392,7 @@ void tst_QCborValue::hugeDeviceValidation_data() { // because QCborValue will attempt to retain the original string in UTF-8, // the size which it can't store is actually the byte array size - addValidationHugeDevice(MaxByteArraySize + 1, MaxByteArraySize + 1); + addValidationHugeDevice(QByteArray::max_size() + 1, QByteArray::max_size() + 1); } void tst_QCborValue::hugeDeviceValidation()