Android: Do not extract QML assets data

Instead to extract the assets QML file, we create a .rcc bundle file
which is register by android qpa plugin before the it invokes the main
function. Thsi way we avoid extracting the QML files from assets as they
can be accessed directly from resources.

[ChangeLog][Android] Instead of bundling QML resources in assets
and extracting them on first start, Qt now creates an .rcc file
and register it before invoking the main function.

Change-Id: Icb2fda79d82c5af102cc9a0276ff26bb0d1599e8
Reviewed-by: Eskil Abrahamsen Blomfeldt <eskil.abrahamsen-blomfeldt@qt.io>
bb10
BogDan Vatra 2019-08-15 08:25:30 +03:00 committed by Eskil Abrahamsen Blomfeldt
parent fb09a8bfcf
commit c8b07f7da3
6 changed files with 87 additions and 398 deletions

View File

@ -44,8 +44,6 @@ import android.content.DialogInterface;
import android.content.Intent;
import android.content.ServiceConnection;
import android.content.pm.ComponentInfo;
import android.content.pm.PackageInfo;
import android.content.res.AssetManager;
import android.os.Build;
import android.os.Bundle;
import android.os.IBinder;
@ -55,15 +53,8 @@ import android.util.Log;
import org.kde.necessitas.ministro.IMinistro;
import org.kde.necessitas.ministro.IMinistroCallback;
import java.io.DataInputStream;
import java.io.DataOutputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.lang.reflect.Array;
import java.lang.reflect.Method;
import java.util.ArrayList;
import java.util.Arrays;
@ -88,8 +79,6 @@ public abstract class QtLoader {
public static final String ENVIRONMENT_VARIABLES_KEY = "environment.variables";
public static final String APPLICATION_PARAMETERS_KEY = "application.parameters";
public static final String BUNDLED_LIBRARIES_KEY = "bundled.libraries";
public static final String BUNDLED_IN_LIB_RESOURCE_ID_KEY = "android.app.bundled_in_lib_resource_id";
public static final String BUNDLED_IN_ASSETS_RESOURCE_ID_KEY = "android.app.bundled_in_assets_resource_id";
public static final String MAIN_LIBRARY_KEY = "main.library";
public static final String STATIC_INIT_CLASSES_KEY = "static.init.classes";
public static final String NECESSITAS_API_LEVEL_KEY = "necessitas.api.level";
@ -141,7 +130,6 @@ public abstract class QtLoader {
public String QT_ANDROID_DEFAULT_THEME = null; // sets the default theme.
public static final int INCOMPATIBLE_MINISTRO_VERSION = 1; // Incompatible Ministro version. Ministro needs to be upgraded.
public static final int BUFFER_SIZE = 1024;
public String[] m_sources = {"https://download.qt-project.org/ministro/android/qt5/qt-5.7"}; // Make sure you are using ONLY secure locations
public String m_repository = "default"; // Overwrites the default Ministro repository
@ -368,263 +356,6 @@ public abstract class QtLoader {
errorDialog.show();
}
static private void copyFile(InputStream inputStream, OutputStream outputStream)
throws IOException
{
byte[] buffer = new byte[BUFFER_SIZE];
int count;
while ((count = inputStream.read(buffer)) > 0)
outputStream.write(buffer, 0, count);
}
private void copyAsset(String source, String destination)
throws IOException
{
// Already exists, we don't have to do anything
File destinationFile = new File(destination);
if (destinationFile.exists())
return;
File parentDirectory = destinationFile.getParentFile();
if (!parentDirectory.exists())
parentDirectory.mkdirs();
destinationFile.createNewFile();
AssetManager assetsManager = m_context.getAssets();
InputStream inputStream = null;
FileOutputStream outputStream = null;
try {
inputStream = assetsManager.open(source);
outputStream = new FileOutputStream(destinationFile);
copyFile(inputStream, outputStream);
} catch (Exception e) {
e.printStackTrace();
} finally {
if (inputStream != null)
inputStream.close();
if (outputStream != null)
// Ensure that the buffered data is flushed to the OS for writing.
outputStream.flush();
}
// Mark the output stream as still needing to be written to physical disk.
// The output stream will be closed after this sync completes.
m_fileOutputStreams.add(outputStream);
}
private static void createBundledBinary(String source, String destination)
throws IOException
{
// Already exists, we don't have to do anything
File destinationFile = new File(destination);
if (destinationFile.exists())
return;
File parentDirectory = destinationFile.getParentFile();
if (!parentDirectory.exists())
parentDirectory.mkdirs();
destinationFile.createNewFile();
InputStream inputStream = null;
FileOutputStream outputStream = null;
try {
inputStream = new FileInputStream(source);
outputStream = new FileOutputStream(destinationFile);
copyFile(inputStream, outputStream);
} catch (Exception e) {
e.printStackTrace();
} finally {
if (inputStream != null)
inputStream.close();
if (outputStream != null)
// Ensure that the buffered data is flushed to the OS for writing.
outputStream.flush();
}
// Mark the output stream as still needing to be written to physical disk.
// The output stream will be closed after this sync completes.
m_fileOutputStreams.add(outputStream);
}
private boolean cleanCacheIfNecessary(String pluginsPrefix, long packageVersion)
{
File versionFile = new File(pluginsPrefix + "cache.version");
long cacheVersion = 0;
if (versionFile.exists() && versionFile.canRead()) {
DataInputStream inputStream = null;
try {
inputStream = new DataInputStream(new FileInputStream(versionFile));
cacheVersion = inputStream.readLong();
} catch (Exception e) {
e.printStackTrace();
} finally {
if (inputStream != null) {
try {
inputStream.close();
} catch (Exception e) {
e.printStackTrace();
}
}
}
}
if (cacheVersion != packageVersion) {
deleteRecursively(new File(pluginsPrefix));
return true;
} else {
return false;
}
}
private void extractBundledPluginsAndImports(String pluginsPrefix, String libsDir)
throws IOException
{
long packageVersion = -1;
try {
PackageInfo packageInfo = m_context.getPackageManager().getPackageInfo(m_context.getPackageName(), 0);
packageVersion = packageInfo.lastUpdateTime;
} catch (Exception e) {
e.printStackTrace();
}
if (!cleanCacheIfNecessary(pluginsPrefix, packageVersion))
return;
{
// why can't we load the plugins directly from libs ?!?!
String key = BUNDLED_IN_LIB_RESOURCE_ID_KEY;
if (m_contextInfo.metaData.containsKey(key)) {
int resourceId = m_contextInfo.metaData.getInt(key);
ArrayList<String> list = prefferedAbiLibs(m_context.getResources().getStringArray(resourceId));
for (String bundledImportBinary : list) {
String[] split = bundledImportBinary.split(":");
String sourceFileName = libsDir + split[0];
String destinationFileName = pluginsPrefix + split[1];
createBundledBinary(sourceFileName, destinationFileName);
}
}
}
{
String key = BUNDLED_IN_ASSETS_RESOURCE_ID_KEY;
if (m_contextInfo.metaData.containsKey(key)) {
String[] list = m_context.getResources().getStringArray(m_contextInfo.metaData.getInt(key));
for (String fileName : list) {
String[] split = fileName.split(":");
String sourceFileName = split[0];
String destinationFileName = pluginsPrefix + split[1];
copyAsset(sourceFileName, destinationFileName);
}
}
}
// The Java compiler must be assured that variables belonging to this parent thread will not
// go out of scope during the runtime of the spawned thread (since in general spawned
// threads can outlive their parent threads). Copy variables and declare as 'final' before
// passing into the spawned thread.
final String pluginsPrefixFinal = pluginsPrefix;
final long packageVersionFinal = packageVersion;
// Spawn a worker thread to write all installed files to physical disk and indicate
// successful installation by creating the 'cache.version' file.
new Thread(new Runnable() {
@Override
public void run() {
try {
finalizeInstallation(pluginsPrefixFinal, packageVersionFinal);
} catch (Exception e) {
Log.e(QtApplication.QtTAG, e.getMessage());
e.printStackTrace();
return;
}
}
}).start();
}
private void finalizeInstallation(String pluginsPrefix, long packageVersion)
throws IOException
{
{
// Write all installed files to physical disk and close each output stream
for (FileOutputStream fileOutputStream : m_fileOutputStreams) {
fileOutputStream.getFD().sync();
fileOutputStream.close();
}
m_fileOutputStreams.clear();
}
{
// Create 'cache.version' file
File versionFile = new File(pluginsPrefix + "cache.version");
File parentDirectory = versionFile.getParentFile();
if (!parentDirectory.exists())
parentDirectory.mkdirs();
versionFile.createNewFile();
DataOutputStream outputStream = null;
try {
outputStream = new DataOutputStream(new FileOutputStream(versionFile));
outputStream.writeLong(packageVersion);
} catch (Exception e) {
e.printStackTrace();
} finally {
if (outputStream != null)
outputStream.close();
}
}
}
private void deleteRecursively(File directory)
{
File[] files = directory.listFiles();
if (files != null) {
for (File file : files) {
if (file.isDirectory())
deleteRecursively(file);
else
file.delete();
}
directory.delete();
}
}
private void cleanOldCacheIfNecessary(String oldLocalPrefix, String localPrefix)
{
File newCache = new File(localPrefix);
if (!newCache.exists()) {
{
File oldPluginsCache = new File(oldLocalPrefix + "plugins/");
if (oldPluginsCache.exists() && oldPluginsCache.isDirectory())
deleteRecursively(oldPluginsCache);
}
{
File oldImportsCache = new File(oldLocalPrefix + "imports/");
if (oldImportsCache.exists() && oldImportsCache.isDirectory())
deleteRecursively(oldImportsCache);
}
{
File oldQmlCache = new File(oldLocalPrefix + "qml/");
if (oldQmlCache.exists() && oldQmlCache.isDirectory())
deleteRecursively(oldQmlCache);
}
}
}
public void startApp(final boolean firstStart)
{
try {
@ -688,29 +419,13 @@ public abstract class QtLoader {
if (m_contextInfo.metaData.containsKey("android.app.bundle_local_qt_libs")
&& m_contextInfo.metaData.getInt("android.app.bundle_local_qt_libs") == 1) {
File dataDir = new File(m_context.getApplicationInfo().dataDir);
String dataPath = dataDir.getCanonicalPath() + "/";
String pluginsPrefix = dataPath + "qt-reserved-files/";
if (libsDir == null)
throw new Exception("Invalid libsDir");
cleanOldCacheIfNecessary(dataPath, pluginsPrefix);
extractBundledPluginsAndImports(pluginsPrefix, libsDir);
if (m_contextInfo.metaData.containsKey(BUNDLED_IN_LIB_RESOURCE_ID_KEY)) {
int resourceId = m_contextInfo.metaData.getInt("android.app.load_local_libs_resource_id");
for (String libs : prefferedAbiLibs(m_context.getResources().getStringArray(resourceId))) {
for (String lib : libs.split(":")) {
if (!lib.isEmpty())
libraryList.add(libsDir + lib);
}
int resourceId = m_contextInfo.metaData.getInt("android.app.load_local_libs_resource_id");
for (String libs : prefferedAbiLibs(m_context.getResources().getStringArray(resourceId))) {
for (String lib : libs.split(":")) {
if (!lib.isEmpty())
libraryList.add(libsDir + lib);
}
}
ENVIRONMENT_VARIABLES += "\tQML2_IMPORT_PATH=" + pluginsPrefix + "/qml"
+ "\tQML_IMPORT_PATH=" + pluginsPrefix + "/imports"
+ "\tQT_PLUGIN_PATH=" + pluginsPrefix + "/plugins";
if (bundledLibsDir != null)
ENVIRONMENT_VARIABLES += "\tQT_BUNDLED_LIBS_PATH=" + bundledLibsDir;
}

View File

@ -34,8 +34,6 @@
<meta-data android:name="android.app.bundled_libs_resource_id" android:resource="@array/bundled_libs"/>
<!-- Deploy Qt libs as part of package -->
<meta-data android:name="android.app.bundle_local_qt_libs" android:value="-- %%BUNDLE_LOCAL_QT_LIBS%% --"/>
<meta-data android:name="android.app.bundled_in_lib_resource_id" android:resource="@array/bundled_in_lib"/>
<meta-data android:name="android.app.bundled_in_assets_resource_id" android:resource="@array/bundled_in_assets"/>
<!-- Run with local libs -->
<meta-data android:name="android.app.use_local_qt_libs" android:value="-- %%USE_LOCAL_QT_LIBS%% --"/>

View File

@ -54,4 +54,9 @@ android {
lintOptions {
abortOnError false
}
// Do not compress Qt binary resources file
aaptOptions {
noCompress 'rcc'
}
}

View File

@ -11,20 +11,12 @@
<!-- %%INSERT_EXTRA_LIBS%% -->
</array>
<array name="qt_libs">
<array name="qt_libs">
<!-- %%INSERT_QT_LIBS%% -->
</array>
<array name="bundled_in_lib">
<!-- %%INSERT_BUNDLED_IN_LIB%% -->
</array>
<array name="load_local_libs">
<!-- %%INSERT_LOCAL_LIBS%% -->
</array>
<array name="bundled_in_assets">
<!-- %%INSERT_BUNDLED_IN_ASSETS%% -->
</array>
</resources>

View File

@ -60,6 +60,7 @@
#include "qandroideventdispatcher.h"
#include <android/api-level.h>
#include <QtCore/qresource.h>
#include <QtCore/qthread.h>
#include <QtCore/private/qjnihelpers_p.h>
#include <QtCore/private/qjni_p.h>
@ -525,6 +526,10 @@ static jboolean startQtApplication(JNIEnv */*env*/, jclass /*clazz*/)
vm->AttachCurrentThread(&env, &args);
}
// Register resources if they are available
if (QFile{QStringLiteral("assets:/android_rcc_bundle.rcc")}.exists())
QResource::registerResource(QStringLiteral("assets:/android_rcc_bundle.rcc"));
QVarLengthArray<const char *> params(m_applicationParams.size());
for (int i = 0; i < m_applicationParams.size(); i++)
params[i] = static_cast<const char *>(m_applicationParams[i].constData());

View File

@ -116,7 +116,6 @@ struct Options
: helpRequested(false)
, verbose(false)
, timing(false)
, generateAssetsFileList(true)
, build(true)
, auxMode(false)
, deploymentMechanism(Bundled)
@ -146,7 +145,6 @@ struct Options
bool helpRequested;
bool verbose;
bool timing;
bool generateAssetsFileList;
bool build;
bool auxMode;
ActionTimer timer;
@ -521,8 +519,6 @@ Options parseOptions()
options.protectedAuthenticationPath = true;
} else if (argument.compare(QLatin1String("--jarsigner"), Qt::CaseInsensitive) == 0) {
options.jarSigner = true;
} else if (argument.compare(QLatin1String("--no-generated-assets-cache"), Qt::CaseInsensitive) == 0) {
options.generateAssetsFileList = false;
} else if (argument.compare(QLatin1String("--aux-mode"), Qt::CaseInsensitive) == 0) {
options.auxMode = true;
}
@ -1242,8 +1238,6 @@ bool updateLibsXml(Options *options)
}
QString qtLibs;
QString bundledInLibs;
QString bundledInAssets;
QString allLocalLibs;
QString extraLibs;
@ -1256,33 +1250,6 @@ bool updateLibsXml(Options *options)
QString s = bundledFile.second.mid(sizeof("lib/lib") - 1);
s.chop(sizeof(".so") - 1);
qtLibs += QLatin1String(" <item>%1;%2</item>\n").arg(it.key(), s);
} else if (bundledFile.first.startsWith(libsPath)) {
QString s = bundledFile.first.mid(libsPath.length());
bundledInLibs += QString::fromLatin1(" <item>%1;%2:%3</item>\n")
.arg(it.key(), s, bundledFile.second);
} else if (bundledFile.first.startsWith(QLatin1String("assets/"))) {
QString s = bundledFile.first.mid(sizeof("assets/") - 1);
bundledInAssets += QString::fromLatin1(" <item>%1:%2</item>\n")
.arg(s, bundledFile.second);
}
}
if (!options->archExtraPlugins[it.key()].isEmpty()) {
for (const QString &extraRes : options->archExtraPlugins[it.key()]) {
QDir resourceDir(extraRes);
const QStringList files = allFilesInside(resourceDir, resourceDir);
for (const QString &file : files) {
QString destinationPath = resourceDir.dirName() + QLatin1Char('/') + file;
if (!file.endsWith(QLatin1String(".so"))) {
bundledInAssets += QLatin1String(" <item>%1:%1</item>\n")
.arg(destinationPath);
} else {
bundledInLibs += QLatin1String(" <item>%1;lib%2:%3</item>\n")
.arg(it.key(),
QString(destinationPath).replace(QLatin1Char('/'), QLatin1Char('_')),
destinationPath);
}
}
}
}
@ -1342,11 +1309,6 @@ bool updateLibsXml(Options *options)
replacements[QStringLiteral("<!-- %%INSERT_LOCAL_LIBS%% -->")] = allLocalLibs.trimmed();
replacements[QStringLiteral("<!-- %%INSERT_EXTRA_LIBS%% -->")] = extraLibs.trimmed();
if (options->deploymentMechanism == Options::Bundled) {
replacements[QStringLiteral("<!-- %%INSERT_BUNDLED_IN_LIB%% -->")] += bundledInLibs.trimmed();
replacements[QStringLiteral("<!-- %%INSERT_BUNDLED_IN_ASSETS%% -->")] += bundledInAssets.trimmed();
}
if (!updateFile(fileName, replacements))
return false;
@ -1871,6 +1833,70 @@ bool scanImports(Options *options, QSet<QString> *usedDependencies)
return true;
}
bool runCommand(const Options &options, const QString &command)
{
if (options.verbose)
fprintf(stdout, "Running command '%s'\n", qPrintable(command));
FILE *runCommand = openProcess(command);
if (runCommand == nullptr) {
fprintf(stderr, "Cannot run command '%s'\n", qPrintable(command));
return false;
}
char buffer[4096];
while (fgets(buffer, sizeof(buffer), runCommand) != nullptr) {
if (options.verbose)
fprintf(stdout, "%s", buffer);
}
pclose(runCommand);
fflush(stdout);
fflush(stderr);
return true;
}
bool createRcc(const Options &options)
{
auto assetsDir = QLatin1String("%1/assets").arg(options.outputDirectory);
if (!QDir{QLatin1String("%1/android_rcc_bundle").arg(assetsDir)}.exists()) {
fprintf(stdout, "Skipping createRCC\n");
return true;
}
if (options.verbose)
fprintf(stdout, "Create rcc bundle.\n");
QString rcc = options.qtInstallDirectory + QLatin1String("/bin/rcc");
#if defined(Q_OS_WIN32)
rcc += QLatin1String(".exe");
#endif
if (!QFile::exists(rcc)) {
fprintf(stderr, "rcc not found: %s\n", qPrintable(rcc));
return false;
}
auto currentDir = QDir::currentPath();
if (!QDir::setCurrent(QLatin1String("%1/android_rcc_bundle").arg(assetsDir))) {
fprintf(stderr, "Cannot set current dir to: %s\n", qPrintable(QLatin1String("%1/android_rcc_bundle").arg(assetsDir)));
return false;
}
bool res = runCommand(options, QLatin1String("%1 --project -o %2").arg(rcc, shellQuote(QLatin1String("%1/android_rcc_bundle.qrc").arg(assetsDir))));
if (!res)
return false;
QFile::rename(QLatin1String("%1/android_rcc_bundle.qrc").arg(assetsDir), QLatin1String("%1/android_rcc_bundle/android_rcc_bundle.qrc").arg(assetsDir));
res = runCommand(options, QLatin1String("%1 %2 --binary -o %3 android_rcc_bundle.qrc").arg(rcc, shellQuote(QLatin1String("--root=/android_rcc_bundle/")),
shellQuote(QLatin1String("%1/android_rcc_bundle.rcc").arg(assetsDir))));
if (!QDir::setCurrent(currentDir)) {
fprintf(stderr, "Cannot set current dir to: %s\n", qPrintable(currentDir));
return false;
}
QFile::remove(QLatin1String("%1/android_rcc_bundle.qrc").arg(assetsDir));
QDir{QLatin1String("%1/android_rcc_bundle").arg(assetsDir)}.removeRecursively();
return res;
}
bool readDependencies(Options *options)
{
if (options->verbose)
@ -2025,7 +2051,7 @@ bool copyQtFiles(Options *options)
QString libsDirectory = QLatin1String("libs/");
// Copy other Qt dependencies
auto assetsDestinationDirectory = QLatin1String("assets/--Added-by-androiddeployqt--/");
auto assetsDestinationDirectory = QLatin1String("assets/android_rcc_bundle/");
for (const QtDependency &qtDependency : qAsConst(options->qtDependencies[options->currentArchitecture])) {
QString sourceFileName = qtDependency.absolutePath;
QString destinationFileName;
@ -2681,57 +2707,6 @@ bool signPackage(const Options &options)
return apkSignerRunner() && QFile::remove(packagePath(options, UnsignedAPK));
}
bool generateAssetsFileList(const Options &options)
{
if (options.verbose)
fprintf(stdout, "Pregenerating entry list for assets file engine.\n");
QString assetsPath = options.outputDirectory + QLatin1String("/assets/");
QString addedByAndroidDeployQtPath = assetsPath + QLatin1String("--Added-by-androiddeployqt--/");
if (!QDir().mkpath(addedByAndroidDeployQtPath)) {
fprintf(stderr, "Failed to create directory '%s'", qPrintable(addedByAndroidDeployQtPath));
return false;
}
QFile file(addedByAndroidDeployQtPath + QLatin1String("/qt_cache_pregenerated_file_list"));
if (file.open(QIODevice::WriteOnly)) {
QDirIterator dirIterator(assetsPath,
QDir::Dirs | QDir::Files | QDir::NoDotAndDotDot,
QDirIterator::Subdirectories);
QHash<QString, QStringList> directoryContents;
while (dirIterator.hasNext()) {
const QString name = dirIterator.next().mid(assetsPath.length());
int slashIndex = name.lastIndexOf(QLatin1Char('/'));
QString pathName = slashIndex >= 0 ? name.left(slashIndex) : QStringLiteral("/");
QString fileName = slashIndex >= 0 ? name.mid(pathName.length() + 1) : name;
if (!fileName.isEmpty() && dirIterator.fileInfo().isDir() && !fileName.endsWith(QLatin1Char('/')))
fileName += QLatin1Char('/');
if (fileName.isEmpty() && !directoryContents.contains(pathName))
directoryContents[pathName] = QStringList();
else if (!fileName.isEmpty())
directoryContents[pathName].append(fileName);
}
QDataStream stream(&file);
stream.setVersion(QDataStream::Qt_5_3);
for (auto it = directoryContents.cbegin(), end = directoryContents.cend(); it != end; ++it) {
const QStringList &entryList = it.value();
stream << it.key() << entryList.size();
for (const QString &entry : entryList)
stream << entry;
}
} else {
fprintf(stderr, "Pregenerating entry list for assets file engine failed!\n");
return false;
}
return true;
}
enum ErrorCode
{
Success,
@ -2749,9 +2724,9 @@ enum ErrorCode
CannotBuildAndroidProject = 14,
CannotSignPackage = 15,
CannotInstallApk = 16,
CannotGenerateAssetsFileList = 18,
CannotCopyAndroidExtraResources = 19,
CannotCopyApk = 20
CannotCopyApk = 20,
CannotCreateRcc = 21
};
int main(int argc, char *argv[])
@ -2848,14 +2823,16 @@ int main(int argc, char *argv[])
}
}
if (!createRcc(options))
return CannotCreateRcc;
if (options.auxMode) {
if (!updateAndroidFiles(options))
return CannotUpdateAndroidFiles;
if (options.generateAssetsFileList && !generateAssetsFileList(options))
return CannotGenerateAssetsFileList;
return 0;
}
if (options.build) {
if (!copyAndroidSources(options))
return CannotCopyAndroidSources;
@ -2866,9 +2843,6 @@ int main(int argc, char *argv[])
if (!updateAndroidFiles(options))
return CannotUpdateAndroidFiles;
if (options.generateAssetsFileList && !generateAssetsFileList(options))
return CannotGenerateAssetsFileList;
if (Q_UNLIKELY(options.timing))
fprintf(stdout, "[TIMING] %d ms: Updated files\n", options.timer.elapsed());