diff --git a/src/corelib/CMakeLists.txt b/src/corelib/CMakeLists.txt index 789982d57a..61e960dcf2 100644 --- a/src/corelib/CMakeLists.txt +++ b/src/corelib/CMakeLists.txt @@ -1236,167 +1236,50 @@ qt_internal_extend_target(Core CONDITION QT_FEATURE_permissions AND WASM kernel/qpermissions_wasm.cpp ) - -#### Keys ignored in scope 171:.:mimetypes:mimetypes/mimetypes.pri:QT_FEATURE_mimetype: -# MIME_DATABASE = "mimetypes/mime/packages/freedesktop.org.xml" -# OTHER_FILES = "$$MIME_DATABASE" - -# special case begin -# qt_internal_extend_target(Core CONDITION QT_FEATURE_mimetype AND QT_FEATURE_mimetype_database -# INCLUDE_DIRECTORIES -# .rcc -#) -# special case end - -#### Keys ignored in scope 172:.:mimetypes:mimetypes/mimetypes.pri:QT_FEATURE_mimetype_database: -# QMAKE_EXTRA_COMPILERS = "mimedb" -# mimedb.commands = "${QMAKE_FILE_IN}" ">" "${QMAKE_FILE_OUT}" -# mimedb.depends = "$$PWD/mime/generate.pl" -# mimedb.input = "MIME_DATABASE" -# mimedb.output = "$$outpath/qmimeprovider_database.cpp" -# mimedb.variable_out = "INCLUDED_SOURCES" - -#### Keys ignored in scope 173:.:mimetypes:mimetypes/mimetypes.pri:ANDROID: -# outpath = "$$outpath/$${QT_ARCH}" - -#### Keys ignored in scope 175:.:mimetypes:mimetypes/mimetypes.pri:(CMAKE_BUILD_TYPE STREQUAL Debug): -# outpath = "$$outpath/debug" - -#### Keys ignored in scope 176:.:mimetypes:mimetypes/mimetypes.pri:else: -# outpath = "$$outpath/release" - -#### Keys ignored in scope 177:.:mimetypes:mimetypes/mimetypes.pri:MAKEFILE_GENERATOR___equals___MSVC.NET OR MAKEFILE_GENERATOR___equals___MSBUILD OR QMAKE_SH_ISEMPTY: -# mimedb.commands = "cmd" "/c" "$$shell_path($$PWD/mime/generate.bat)" -# mimedb.depends = "$$PWD/mime/generate.bat" "$$PWD/mime/hexdump.ps1" - -#### Keys ignored in scope 178:.:mimetypes:mimetypes/mimetypes.pri:else: -# mimedb.commands = "perl" "$${mimedb.depends}" - -#### Keys ignored in scope 179:.:mimetypes:mimetypes/mimetypes.pri:QT_FEATURE_zstd: -# mimedb.commands = "--zstd" -# Resources: -# special case begin -# We can't specify the resources directly as we have unit tests that -# depend on these as well and we no longer support the use of hand -# edited qrc files. if(QT_FEATURE_mimetype AND QT_FEATURE_mimetype_database) include(${CMAKE_CURRENT_SOURCE_DIR}/mimetypes/mimetypes_resources.cmake) -# Generate qmimeprovider_database.cpp - set(qmimeprovider_db_output_dir "${CMAKE_CURRENT_BINARY_DIR}/.rcc") - set(qmimeprovider_db_output "${qmimeprovider_db_output_dir}/qmimeprovider_database.cpp") if(CMAKE_VERSION VERSION_LESS 3.18 OR QT_AVOID_CMAKE_ARCHIVING_API) - set(command_args "") - set(mime_dir "${CMAKE_CURRENT_SOURCE_DIR}/mimetypes/mime") - set(command_depends "${mime_dir}/generate.pl" "${corelib_mimetypes_resource_file}") - if (MSVC) - list(APPEND command_args "${mime_dir}/generate.bat") - list(APPEND command_depends "${mime_dir}/generate.bat" "${mime_dir}/hexdump.ps1" ) - else() - file(MAKE_DIRECTORY ${qmimeprovider_db_output_dir}) - list(APPEND command_args perl "${mime_dir}/generate.pl" ) - endif() - - if (QT_FEATURE_zstd) - list(APPEND command_args "--zstd") - endif() - - list(APPEND command_args "${corelib_mimetypes_resource_file}" ">" "${qmimeprovider_db_output}") - - add_custom_command(OUTPUT "${qmimeprovider_db_output}" - DEPENDS ${command_depends} - COMMAND ${command_args} - COMMENT "Generating ${qmimeprovider_db_output}" - VERBATIM - ) + set(archiving_api "External") else() - if(QT_FEATURE_zstd) - if(NOT QT_CMAKE_ZSTD_SUPPORT) - message(FATAL_ERROR - "CMake was not built with zstd support. " - "Rebuild CMake or set QT_AVOID_CMAKE_ARCHIVING_API=ON.") - endif() - set(qmime_db_compression Zstd) - string(APPEND qmime_db_content "#define MIME_DATABASE_IS_ZSTD\n") - else() - set(qmime_db_compression GZip) - string(APPEND qmime_db_content "#define MIME_DATABASE_IS_GZIP\n") - endif() - - file(MAKE_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/.rcc") - get_filename_component(mime_types_resource_file "${corelib_mimetypes_resource_file}" NAME) - - set(mimetypes_resource_file_minified "${CMAKE_CURRENT_BINARY_DIR}/.rcc/${mime_types_resource_file}") - - set(mimetypes_resfile_timestamp_file "${CMAKE_CURRENT_BINARY_DIR}/.rcc/${mime_types_resource_file}.timestamp") - file(TIMESTAMP "${corelib_mimetypes_resource_file}" mimetypes_resfile_timestamp) - - set(compute_db_archive ON) - if (EXISTS "${mimetypes_resfile_timestamp_file}") - file(READ "${mimetypes_resfile_timestamp_file}" old_mimetypes_resfile_timestamp) - if ("${mimetypes_resfile_timestamp}" STREQUAL "${old_mimetypes_resfile_timestamp}") - set(compute_db_archive OFF) - endif() - endif() - - if (compute_db_archive) - find_program(xmlstarlet NAMES xmlstarlet xml) - if (xmlstarlet) - execute_process( - COMMAND "${xmlstarlet}" sel -D -B -t -c / "${corelib_mimetypes_resource_file}" - OUTPUT_FILE "${mimetypes_resource_file_minified}" - RESULT_VARIABLE failed_to_minify - ) - if (NOT failed_to_minify) - set(corelib_mimetypes_resource_file "${mimetypes_resource_file_minified}") - endif() - else() - message(STATUS "xmlstarlet command was not found. ${mime_types_resource_file} will not be minified.") - endif() - - if (CMAKE_VERSION GREATER_EQUAL 3.19) - set(additional_file_archive_create_parameters COMPRESSION_LEVEL 9) - endif() - - file(ARCHIVE_CREATE OUTPUT "${qmimeprovider_db_output}.archive" - PATHS "${corelib_mimetypes_resource_file}" - FORMAT raw - COMPRESSION ${qmime_db_compression} - ${additional_file_archive_create_parameters} - ) - file(READ "${qmimeprovider_db_output}.archive" qmime_db_archive HEX) - file(SIZE "${qmimeprovider_db_output}.archive" qmime_db_archive_size) - file(SIZE "${corelib_mimetypes_resource_file}" qmime_db_resource_size) - file(REMOVE ${qmimeprovider_db_output}.archive) - - string(APPEND qmime_db_content "static const unsigned char mimetype_database[] = { ") - - string(REGEX MATCHALL "([a-f0-9][a-f0-9])" qmime_db_hex "${qmime_db_archive}") - - list(TRANSFORM qmime_db_hex PREPEND "0x") - math(EXPR qmime_db_archive_size "${qmime_db_archive_size} - 1") - foreach(index RANGE 0 ${qmime_db_archive_size} 12) - list(APPEND index_list ${index}) - endforeach() - list(TRANSFORM qmime_db_hex PREPEND "\n " AT ${index_list}) - list(JOIN qmime_db_hex ", " qmime_db_hex_joined) - - string(APPEND qmime_db_content "${qmime_db_hex_joined}") - string(APPEND qmime_db_content "\n};\n") - string(APPEND qmime_db_content "static constexpr size_t MimeTypeDatabaseOriginalSize = ${qmime_db_resource_size};\n") - - file(WRITE "${qmimeprovider_db_output}" "${qmime_db_content}") - - file(WRITE "${mimetypes_resfile_timestamp_file}" "${mimetypes_resfile_timestamp}") + set(archiving_api "CMake") + if(QT_FEATURE_zstd AND NOT QT_CMAKE_ZSTD_SUPPORT) + message(FATAL_ERROR + "CMake was not built with zstd support. " + "Rebuild CMake or set QT_AVOID_CMAKE_ARCHIVING_API=ON.") endif() endif() - target_sources(Core PRIVATE ${qmimeprovider_db_output}) - set_source_files_properties(${qmimeprovider_db_output} PROPERTIES HEADER_FILE_ONLY TRUE) - target_include_directories(Core PRIVATE "${CMAKE_CURRENT_BINARY_DIR}/.rcc") -endif() + if(QT_FEATURE_zstd) + set(compression_type "zstd") + else() + set(compression_type "gzip") + endif() -# special case end + # Generate qmimeprovider_database.cpp + set(qmimeprovider_db_output "${CMAKE_CURRENT_BINARY_DIR}/.rcc/qmimeprovider_database.cpp") + add_custom_command(OUTPUT "${qmimeprovider_db_output}" + COMMAND ${CMAKE_COMMAND} + -DINPUT_FILE=${corelib_mimetypes_resource_file} + -DOUTPUT_FILE=${qmimeprovider_db_output} + -DARCHIVING_API=${archiving_api} + -DCOMPRESSION_TYPE=${compression_type} + -P "${CMAKE_CURRENT_SOURCE_DIR}/QtCompressMimeDatabase.cmake" + DEPENDS + "${CMAKE_CURRENT_SOURCE_DIR}/QtCompressMimeDatabase.cmake" + "${corelib_mimetypes_resource_file}" + VERBATIM + ) + + qt_internal_extend_target(Core + SOURCES ${qmimeprovider_db_output} + INCLUDE_DIRECTORIES "${CMAKE_CURRENT_BINARY_DIR}/.rcc" + ) + set_source_files_properties(${qmimeprovider_db_output} PROPERTIES + GENERATED TRUE + HEADER_FILE_ONLY TRUE + ) +endif() qt_internal_extend_target(Core CONDITION WASM SOURCES diff --git a/src/corelib/QtCompressMimeDatabase.cmake b/src/corelib/QtCompressMimeDatabase.cmake new file mode 100644 index 0000000000..718d330f28 --- /dev/null +++ b/src/corelib/QtCompressMimeDatabase.cmake @@ -0,0 +1,156 @@ +# Copyright (C) 2022 The Qt Company Ltd. +# SPDX-License-Identifier: BSD-3-Clause +cmake_minimum_required(VERSION 3.16) + +get_filename_component(output_directory "${OUTPUT_FILE}" DIRECTORY) +file(MAKE_DIRECTORY "${output_directory}") + +find_program(xmlstarlet NAMES xmlstarlet xml) +if(xmlstarlet) + set(XML_STARLET_COMMAND COMMAND "${xmlstarlet}" sel -D -B -t -c / "${INPUT_FILE}") +else() + get_filename_component(mime_types_resource_filename "${INPUT_FILE}" NAME) + message(STATUS "xmlstarlet command was not found. ${mime_types_resource_filename}" + " will not be minified.") +endif() + +# The function tries to minify 'input_file' using the globally defined 'XML_STARLET_COMMAND'. +# Sets the full path of the output file to the 'out_var' argument. If 'XML_STARLET_COMMAND' fails +# to minify the 'input_file', 'out_var' is set to 'input_file'. +function(try_minify_xml out_var input_file output_directory) + get_filename_component(output_filename "${input_file}" NAME) + set(minified_output "${output_directory}/${output_filename}") + + # Assume XML_STARLET_COMMAND is globally defined. + if(XML_STARLET_COMMAND) + execute_process(${XML_STARLET_COMMAND} + OUTPUT_FILE "${minified_output}" + RESULT_VARIABLE failed_to_minify + ) + if(NOT failed_to_minify) + set(${out_var} "${minified_output}" PARENT_SCOPE) + return() + endif() + endif() + set(${out_var} "${input_file}" PARENT_SCOPE) +endfunction() + +unset(archive_command) + +if(COMPRESSION_TYPE STREQUAL "none") + try_minify_xml(qmimeprovider_db_data_file "${INPUT_FILE}" "${output_directory}") +else() + # CMake versions less than 3.26 don't support the value of the zstd compression level higher + # than 9. We want zstd to compress with level 19 if it's possible, so if zstd executable is + # found on a file system we prefer to use the 'External' archiving API instead of the CMake API. + # See for details: https://gitlab.kitware.com/cmake/cmake/-/issues/24160 + if(COMPRESSION_TYPE STREQUAL "zstd" AND CMAKE_VERSION VERSION_LESS 3.26) + find_program(archive_command NAMES zstd zstd.exe) + if(archive_command) + set(ARCHIVING_API "External") + endif() + endif() + + set(qmimeprovider_db_data_file "${OUTPUT_FILE}.archive") + if(ARCHIVING_API STREQUAL "External") + message(STATUS "Using external archive command to compress the mime type database") + if(COMPRESSION_TYPE STREQUAL "zstd") + find_program(archive_command NAMES zstd zstd.exe) + if(archive_command) + set(archive_command_args -cq19 -T1) + else() + message(WARNING + "Unable to use the preffered ${COMPRESSION_TYPE} compression for the mime type database." + " ${COMPRESSION_TYPE} binary is not found. Trying gzip.") + endif() + endif() + if(NOT archive_command) + # Trying to use gzip if zstd is binary is not found. + set(COMPRESSION_TYPE "gzip") + find_program(archive_command NAMES gzip gzip.exe) + set(archive_command_args -nc9) + endif() + if(archive_command) + if(NOT XML_STARLET_COMMAND) + set(intput_file_arg INPUT_FILE "${INPUT_FILE}") + endif() + execute_process(${XML_STARLET_COMMAND} + COMMAND ${archive_command} + ${archive_command_args} + ${intput_file_arg} + OUTPUT_FILE "${qmimeprovider_db_data_file}" + RESULTS_VARIABLE results + ERROR_VARIABLE error_string + ) + + foreach(result IN LISTS results) + if(NOT result EQUAL 0) + message(WARNING "Unable to compress mime type database: ${error_string}") + endif() + endforeach() + else() + message(WARNING "Unable to find ${COMPRESSION_TYPE} binary." + " Please make sure that the ${COMPRESSION_TYPE} binary is in PATH." + " Adding the uncompressed mime type database.") + set(COMPRESSION_TYPE "none") + set(qmimeprovider_db_data_file "${INPUT_FILE}") + endif() + else() + message(STATUS "Using CMake archive command to compress the mime type database") + + try_minify_xml(mimetypes_resource_file_minified "${INPUT_FILE}" "${output_directory}") + + if(CMAKE_VERSION VERSION_GREATER_EQUAL 3.26 AND COMPRESSION_TYPE STREQUAL "zstd") + set(additional_file_archive_create_parameters COMPRESSION_LEVEL 19) + elseif(CMAKE_VERSION VERSION_GREATER_EQUAL 3.19) + set(additional_file_archive_create_parameters COMPRESSION_LEVEL 9) + endif() + + if(COMPRESSION_TYPE STREQUAL "gzip") + set(cmake_compression_type "GZip") + elseif(COMPRESSION_TYPE STREQUAL "zstd") + set(cmake_compression_type "Zstd") + endif() + + file(ARCHIVE_CREATE OUTPUT "${qmimeprovider_db_data_file}" + PATHS "${mimetypes_resource_file_minified}" + FORMAT raw + COMPRESSION ${cmake_compression_type} + ${additional_file_archive_create_parameters} + ) + if(NOT mimetypes_resource_file_minified STREQUAL INPUT_FILE) + file(REMOVE "${mimetypes_resource_file_minified}") + endif() + endif() +endif() +file(READ "${qmimeprovider_db_data_file}" qmime_db_data HEX) +file(SIZE "${qmimeprovider_db_data_file}" qmime_db_data_size) +if(NOT qmimeprovider_db_data_file STREQUAL INPUT_FILE) + file(SIZE "${INPUT_FILE}" qmime_db_resource_size) + file(REMOVE "${qmimeprovider_db_data_file}") +else() + set(qmime_db_resource_size ${qmime_db_data_size}) +endif() + +string(REGEX MATCHALL "([a-f0-9][a-f0-9])" qmime_db_hex "${qmime_db_data}") + +list(TRANSFORM qmime_db_hex PREPEND "0x") +math(EXPR qmime_db_data_size "${qmime_db_data_size} - 1") +foreach(index RANGE 0 ${qmime_db_data_size} 12) + list(APPEND index_list ${index}) +endforeach() +list(TRANSFORM qmime_db_hex PREPEND "\n " AT ${index_list}) +list(JOIN qmime_db_hex ", " qmime_db_hex_joined) + +if(NOT COMPRESSION_TYPE STREQUAL "none") + string(TOUPPER "${COMPRESSION_TYPE}" compression_type_upper) + set(qmime_db_content "#define MIME_DATABASE_IS_${compression_type_upper}\n") +endif() +string(APPEND qmime_db_content + "static const unsigned char mimetype_database[] = {" + "${qmime_db_hex_joined}" + "\n};\n" + "static constexpr size_t MimeTypeDatabaseOriginalSize = ${qmime_db_resource_size};\n" +) + +file(WRITE "${OUTPUT_FILE}" "${qmime_db_content}") diff --git a/src/corelib/mimetypes/mime/generate.bat b/src/corelib/mimetypes/mime/generate.bat deleted file mode 100644 index 5ea9e5ea55..0000000000 --- a/src/corelib/mimetypes/mime/generate.bat +++ /dev/null @@ -1,48 +0,0 @@ -:: Copyright (C) 2019 Intel Corporation. -:: SPDX-License-Identifier: LicenseRef-Qt-Commercial OR GPL-3.0-only WITH Qt-GPL-exception-1.0 - -@echo off -setlocal ENABLEDELAYEDEXPANSION ENABLEEXTENSIONS -set me=%~dp0 - -:: Check if certain tools are in PATH -for %%C in (gzip.exe zstd.exe perl.exe) do set %%C=%%~$PATH:C - -:: If perl is in PATH, just let it do everything -if not "%perl.exe%" == "" goto PuntToPerl - -set COMPRESS= -set MACRO=MIME_DATABASE_IS_UNCOMPRESSED -if not "%gzip.exe%" == "" ( - set COMPRESS=gzip -9 - set MACRO=MIME_DATABASE_IS_GZIP -) - -:: Check if zstd support was enabled -if /i "%~1" == "--zstd" ( - shift - if not "%zstd.exe%" == "" ( - set COMPRESS=zstd -19 - set MACRO=MIME_DATABASE_IS_ZSTD - ) -) - -if not "%COMPRESS%" == "" goto CompressedCommon - -:: No Compression and no Perl -:: Just hex-dump with Powershell -powershell -ExecutionPolicy Bypass %me%hexdump.ps1 %1 %1 -exit /b %errorlevel% - -:CompressedCommon -:: Compress to a temporary file, then hex-dump using Powershell -echo #define %MACRO% -set tempfile=generate%RANDOM%.tmp -%COMPRESS% < %1 > %tempfile% -powershell -ExecutionPolicy Bypass %me%hexdump.ps1 %tempfile% %1 -del %tempfile% -exit /b %errorlevel% - -:PuntToPerl -perl %me%generate.pl %* -exit /b %errorlevel% diff --git a/src/corelib/mimetypes/mime/generate.pl b/src/corelib/mimetypes/mime/generate.pl deleted file mode 100644 index a354377079..0000000000 --- a/src/corelib/mimetypes/mime/generate.pl +++ /dev/null @@ -1,93 +0,0 @@ -#!/usr/bin/perl -# Copyright (C) 2019 Intel Corporation. -# SPDX-License-Identifier: LicenseRef-Qt-Commercial OR GPL-3.0-only WITH Qt-GPL-exception-1.0 -use strict; -use warnings; -use Config; -local $/; # Enable "slurp" mode - -sub checkCommand($) { - use File::Spec::Functions; - my $cmd = $_[0] . $Config{_exe}; - for my $path (path()) { - return 1 if -x catfile($path, $cmd); - } - return 0; -} - -my $data; -my $compress; -my $macro; -my $zlib = eval 'use Compress::Zlib; use IO::Compress::Gzip; return 1;'; -my $fname = shift @ARGV; - -if ($zlib) { - # Prefer internal zlib support (useful on Windows where gzip.exe isn't - # always presnet) - $macro = "MIME_DATABASE_IS_GZIP"; -} elsif (checkCommand("gzip")) { - # No builtin support for compression (old Perl?) - $compress = "gzip -c9"; - $macro = "MIME_DATABASE_IS_GZIP"; -} - -# Check if Qt is being built with zstd support -if ($fname eq "--zstd") { - $fname = shift @ARGV; - if (checkCommand("zstd")) { - $compress = "zstd -cq19 -T1"; - $macro = "MIME_DATABASE_IS_ZSTD"; - } -} - -# Check if xml (from xmlstarlet) is in $PATH -my $cmd; -if (checkCommand("xmlstarlet")) { - # Minify the data before compressing - $cmd = "xmlstarlet sel -D -B -t -c / $fname"; - $cmd .= "| $compress" if $compress; -} elsif (checkCommand("xml")) { - # Minify the data before compressing - $cmd = "xml sel -D -B -t -c / $fname"; - $cmd .= "| $compress" if $compress; -} elsif ($compress) { - $cmd = "$compress < $fname" -} -if ($cmd) { - # Run the command and read everything - open CMD, "$cmd |"; - binmode CMD; - $data = ; - close CMD; - die("Failed to run $cmd") if ($? >> 8); -} else { - # No command, just read the file - open F, "<$fname"; - $data = ; - close F; -} - -# Do we need to compress with zlib? -if (!$compress && $zlib) { - $data = eval q{ - use Compress::Zlib; - use IO::Compress::Gzip qw(gzip); - my $compressed; - gzip \$data => \$compressed, - Minimal => 1, - Level => Z_BEST_COMPRESSION; - return $compressed; - }; -} - -# Now print as hex -printf "#define %s\n", $macro if $macro; -printf "static const unsigned char mimetype_database[] = {"; -my $i = 0; -map { - printf "\n " if $i++ % 12 == 0; - printf "0x%02x, ", ord $_ -} split //, $data; -printf "\n};\n"; -printf "static constexpr size_t MimeTypeDatabaseOriginalSize = %d;\n", - (stat $fname)[7];