diff --git a/src/widgets/itemviews/qtableview.cpp b/src/widgets/itemviews/qtableview.cpp
index 62bc9d1ff1..d1b82c2511 100644
--- a/src/widgets/itemviews/qtableview.cpp
+++ b/src/widgets/itemviews/qtableview.cpp
@@ -3467,7 +3467,7 @@ void QTableViewPrivate::selectColumn(int column, bool anchor)
         if (q->selectionMode() != QTableView::SingleSelection
             && command.testFlag(QItemSelectionModel::Toggle)) {
             if (anchor)
-                ctrlDragSelectionFlag = horizontalHeader->selectionModel()->selectedColumns().contains(index)
+                ctrlDragSelectionFlag = horizontalHeader->selectionModel()->selectedColumns(row).contains(index)
                                     ? QItemSelectionModel::Deselect : QItemSelectionModel::Select;
             command &= ~QItemSelectionModel::Toggle;
             command |= ctrlDragSelectionFlag;
diff --git a/tests/auto/widgets/itemviews/qtableview/tst_qtableview.cpp b/tests/auto/widgets/itemviews/qtableview/tst_qtableview.cpp
index 662226de52..6ed3b4ecdd 100644
--- a/tests/auto/widgets/itemviews/qtableview/tst_qtableview.cpp
+++ b/tests/auto/widgets/itemviews/qtableview/tst_qtableview.cpp
@@ -439,6 +439,8 @@ private slots:
     void deselectRow();
     void selectRowsAndCells();
     void selectColumnsAndCells();
+    void selectWithHeader_data();
+    void selectWithHeader();
 
 #if QT_CONFIG(wheelevent)
     void mouseWheel_data();
@@ -4855,6 +4857,60 @@ void tst_QTableView::selectColumnsAndCells()
     checkColumns(tw.selectionModel()->selectedColumns());
 }
 
+void tst_QTableView::selectWithHeader_data()
+{
+    QTest::addColumn<Qt::Orientation>("orientation");
+
+    QTest::addRow("horizontal") << Qt::Horizontal;
+    QTest::addRow("vertical") << Qt::Vertical;
+}
+
+void tst_QTableView::selectWithHeader()
+{
+    QFETCH(Qt::Orientation, orientation);
+
+    QTableWidget view(10, 10);
+    view.resize(200, 100);
+    view.show();
+
+    QVERIFY(QTest::qWaitForWindowExposed(&view));
+
+    QHeaderView *header;
+    QPoint clickPos;
+    QModelIndex lastIndex;
+
+    switch (orientation) {
+    case Qt::Horizontal:
+        header = view.horizontalHeader();
+        clickPos.rx() = header->sectionPosition(0) + header->sectionSize(0) / 2;
+        clickPos.ry() = header->height() / 2;
+        lastIndex = view.model()->index(9, 0);
+        break;
+    case Qt::Vertical:
+        header = view.verticalHeader();
+        clickPos.rx() = header->width() / 2;
+        clickPos.ry() = header->sectionPosition(0) + header->sectionSize(0) / 2;
+        lastIndex = view.model()->index(0, 9);
+        break;
+    }
+
+    const auto isSelected = [&]{
+        return orientation == Qt::Horizontal
+             ? view.selectionModel()->isColumnSelected(0)
+             : view.selectionModel()->isRowSelected(0);
+    };
+
+    QTest::mouseClick(header->viewport(), Qt::LeftButton, {}, clickPos);
+    QVERIFY(isSelected());
+    QTest::mouseClick(header->viewport(), Qt::LeftButton, Qt::ControlModifier, clickPos);
+    QVERIFY(!isSelected());
+    QTest::mouseClick(header->viewport(), Qt::LeftButton, {}, clickPos);
+    QVERIFY(isSelected());
+    view.scrollTo(lastIndex);
+    QTest::mouseClick(header->viewport(), Qt::LeftButton, Qt::ControlModifier, clickPos);
+    QVERIFY(!isSelected());
+}
+
 // This has nothing to do with QTableView, but it's convenient to reuse the QtTestTableModel
 #if QT_CONFIG(textmarkdownwriter)