diff --git a/sources/TerminalStrip/terminalstrip.cpp b/sources/TerminalStrip/terminalstrip.cpp index 337f56766..cbb537cdb 100644 --- a/sources/TerminalStrip/terminalstrip.cpp +++ b/sources/TerminalStrip/terminalstrip.cpp @@ -1020,8 +1020,9 @@ bool TerminalStrip::fromXml(QDomElement &xml_element) terminals.insert(QUuid(xml_reals.at(i).toElement().attribute(QStringLiteral("element_uuid")))); m_project->buildFoliosHoldingSymbols(terminals); - //Get all free elements terminal of the project - const ElementProvider ep(m_project); + //Get all free elements terminal of the project: its own are on + //the folios just built + const ElementProvider ep(m_project->builtFolios()); auto free_terminals = ep.freeTerminal(); //Read each physical terminal diff --git a/sources/cli_export.cpp b/sources/cli_export.cpp index 80e7f1aa4..cdb00096f 100644 --- a/sources/cli_export.cpp +++ b/sources/cli_export.cpp @@ -870,9 +870,8 @@ int exportLinks(QETProject &project, const QString &output) int resaveProject(QETProject &project, const QString &output) { if (QETProject::isContainerPath(output)) { - const QDomDocument doc = project.toXml(); QString error; - if (!QETProject::writeContainer(doc, output, &error)) { + if (!project.writeContainerFile(output, &error)) { err << "Cannot write '" << output << "': " << error << "\n"; return 1; } @@ -968,9 +967,8 @@ int setTitleBlock(QETProject &project, const QString &output, } if (QETProject::isContainerPath(output)) { - const QDomDocument doc = project.toXml(); QString error; - if (!QETProject::writeContainer(doc, output, &error)) { + if (!project.writeContainerFile(output, &error)) { err << "Cannot write '" << output << "': " << error << "\n"; return 1; } diff --git a/sources/container/qetcontainer.cpp b/sources/container/qetcontainer.cpp index 7f27789c4..3cb20fd84 100644 --- a/sources/container/qetcontainer.cpp +++ b/sources/container/qetcontainer.cpp @@ -258,6 +258,43 @@ QList QetContainer::split(const QDomDocument &project, const QStr return entries; } +/** + @brief QetContainer::readFolio + The folio part @p file of the .qetz whose entries are @p files, as + join() puts it in the project: its pictures back as base64 and its + engineering data back from project.sqlite. +*/ +bool QetContainer::readFolio(const QHash &files, const QString &file, + QDomDocument *folio, QString *error) +{ + if (!files.contains(file)) { + setError(error, QStringLiteral("part \"%1\" is missing").arg(file)); + return false; + } + QString part_error; + if (!parse(files.value(file), folio, &part_error)) { + setError(error, QStringLiteral("%1: %2").arg(file, part_error)); + return false; + } + const QDomNodeList images = folio->elementsByTagName(QStringLiteral("image")); + for (int i = 0 ; i < images.size() ; ++i) + { + QDomElement image = images.at(i).toElement(); + if (!image.hasAttribute(ImageFileAttribute)) continue; + const QString picture = image.attribute(ImageFileAttribute); + if (!files.contains(picture)) { + setError(error, QStringLiteral("picture \"%1\" is missing").arg(picture)); + return false; + } + image.removeAttribute(ImageFileAttribute); + image.insertBefore(folio->createTextNode(QString::fromLatin1(files.value(picture).toBase64())), + image.firstChild()); + } + return !files.contains(QStringLiteral("project.sqlite")) + || QetContainerDb::restoreFolio(folio->documentElement(), + files.value(QStringLiteral("project.sqlite")), error); +} + /** @brief QetContainer::join Rebuild in @p project the document split() made @p entries from. @@ -265,7 +302,8 @@ QList QetContainer::split(const QDomDocument &project, const QStr (the manifest's min-reader), or a part missing or unreadable. Entries nothing names are ignored, so a later format may add some. */ -bool QetContainer::join(const QList &entries, QDomDocument *project, QString *error) +bool QetContainer::join(const QList &entries, QDomDocument *project, QString *error, + QStringList *folio_files) { QHash files; for (const QetZip::Entry &entry : entries) @@ -297,6 +335,16 @@ bool QetContainer::join(const QList &entries, QDomDocument *proje for (QDomElement part : placeholders) { const QString file = part.attribute(FileAttribute); + //With @p folio_files, the folios stay parts, read when built + if (folio_files && part.parentNode() == project->documentElement() + && file.startsWith(QLatin1String("folios/"))) { + if (!files.contains(file)) { + setError(error, QStringLiteral("part \"%1\" is missing").arg(file)); + return false; + } + folio_files->append(file); + continue; + } if (!files.contains(file)) { setError(error, QStringLiteral("part \"%1\" is missing").arg(file)); return false; diff --git a/sources/container/qetcontainer.h b/sources/container/qetcontainer.h index d92df084c..ad050efb4 100644 --- a/sources/container/qetcontainer.h +++ b/sources/container/qetcontainer.h @@ -53,7 +53,11 @@ namespace QetContainer const QString &writer = QString(), QString *error = nullptr); bool join(const QList &entries, QDomDocument *project, - QString *error = nullptr); + QString *error = nullptr, QStringList *folio_files = nullptr); + /// A folio part of @p files, which join() left out, with its + /// pictures and its engineering data back (stage D2) + bool readFolio(const QHash &files, const QString &file, + QDomDocument *folio, QString *error = nullptr); /// Parse @p xml as a .qet is read (QETProject::openFile) bool parse(const QByteArray &xml, QDomDocument *document, QString *error = nullptr); diff --git a/sources/container/qetcontainerdb.cpp b/sources/container/qetcontainerdb.cpp index 321fddd49..e5c48c121 100644 --- a/sources/container/qetcontainerdb.cpp +++ b/sources/container/qetcontainerdb.cpp @@ -434,6 +434,143 @@ QStringList keys(const QList &items, const QString &prefix) } } +namespace { + //@p database opened read only in @p db, its tables checked to be plain + //tables: a file is untrusted +bool openDatabase(Database &db, const QByteArray &database, QString *error) +{ + if (sqlite3_open(":memory:", &db.m_db) != SQLITE_OK) { + setError(error, QStringLiteral("cannot open the project database")); + return false; + } + unsigned char *copy = static_cast(sqlite3_malloc64(sqlite3_uint64(database.size()))); + if (!copy) { setError(error, QStringLiteral("out of memory")); return false; } + memcpy(copy, database.constData(), size_t(database.size())); + if (sqlite3_deserialize(db.m_db, "main", copy, database.size(), database.size(), + SQLITE_DESERIALIZE_FREEONCLOSE | SQLITE_DESERIALIZE_READONLY) != SQLITE_OK) { + setError(error, QStringLiteral("the project database cannot be read")); + return false; + } + //A file is untrusted: nothing in its schema may run, and every name + //read must be a plain table. + db.exec(QStringLiteral("PRAGMA trusted_schema = OFF")); + db.exec(QStringLiteral("PRAGMA query_only = ON")); + for (const char *table : {"meta", "project_property", "folio", "folio_property", + "element", "element_info", "link", "conductor", "terminal_strip", + "strip_terminal", "strip_bridge", "strip_bridge_terminal"}) { + Statement kind(db.m_db, QStringLiteral("SELECT type FROM sqlite_master WHERE name = ?")); + kind.bind({QString::fromLatin1(table)}); + if (!kind.next() || kind.value(0).toString() != QLatin1String("table")) { + setError(error, QStringLiteral("the project database has no table %1").arg(QLatin1String(table))); + return false; + } + } + return true; +} + +QDomElement entryElement(QDomDocument &document, const QVariant &name_attr, const QString &name, + const QVariant &value, const QVariant &attrs, const QString &default_tag) +{ + const QJsonObject o = QJsonDocument::fromJson(attrs.toString().toUtf8()).object(); + QDomElement e = document.createElement(o.value(QStringLiteral("tag")).toString(default_tag)); + if (!name_attr.isNull()) e.setAttribute(name, name_attr.toString()); + setAttributesJson(e, attrs); + if (!value.isNull()) e.appendChild(document.createTextNode(value.toString())); + return e; +} + + //The engineering data of the folio @p d (its qetz-folio id) back from @p db +bool restoreDiagram(Database &db, QDomElement d, QString *error) +{ + QDomDocument document = d.ownerDocument(); + auto entry = [&document](const QVariant &name_attr, const QString &name, const QVariant &value, + const QVariant &attrs, const QString &default_tag) { + return entryElement(document, name_attr, name, value, attrs, default_tag); + }; + const QString id = d.attribute(FolioAttribute); + if (id.isEmpty()) return true; + d.removeAttribute(FolioAttribute); + Statement f(db.m_db, QStringLiteral("SELECT %1, props_at, props_attrs FROM folio WHERE id = ?") + .arg(QStringLiteral("\"") + FolioData.join(QStringLiteral("\", \"")) + QStringLiteral("\""))); + f.bind({id}); + if (!f.next()) { setError(error, QStringLiteral("folio %1 is not in the database").arg(id)); return false; } + QVariantList values; + for (int i = 0 ; i < FolioData.size() + 2 ; ++i) values << f.value(i); + put(d, FolioData, values, 0); + if (!values.at(FolioData.size()).isNull()) { + QDomElement props = document.createElement(QStringLiteral("properties")); + setAttributesJson(props, values.at(FolioData.size() + 1)); + Statement p(db.m_db, QStringLiteral("SELECT name, value, attrs FROM folio_property WHERE folio_id = ? ORDER BY ord")); + p.bind({id}); + while (p.next()) + props.appendChild(entry(p.value(0), QStringLiteral("name"), p.value(1), p.value(2), + QStringLiteral("property"))); + restoreWhitespace(props, values.at(FolioData.size() + 1)); + if (!replaceMarker(d, props)) { + setError(error, QStringLiteral("folio %1's properties have no place").arg(id)); + return false; + } + } + + for (const QDomElement &list : children(d, QStringLiteral("elements"))) { + for (QDomElement e : children(list, QStringLiteral("element"))) { + if (!e.hasAttribute(KeyAttribute)) continue; + const QString key = e.attribute(KeyAttribute); + e.removeAttribute(KeyAttribute); + Statement s(db.m_db, QStringLiteral("SELECT %1, info_at, info_attrs, links_at, links_attrs FROM element WHERE key = ?") + .arg(QStringLiteral("\"") + ElementData.join(QStringLiteral("\", \"")) + QStringLiteral("\""))); + s.bind({key}); + if (!s.next()) { setError(error, QStringLiteral("symbol %1 is not in the database").arg(key)); return false; } + QVariantList v; + for (int i = 0 ; i < ElementData.size() + 4 ; ++i) v << s.value(i); + put(e, ElementData, v, 0); + const int n = ElementData.size(); + QDomElement info, links; + if (!v.at(n).isNull()) { + info = document.createElement(QStringLiteral("elementInformations")); + setAttributesJson(info, v.at(n + 1)); + Statement i(db.m_db, QStringLiteral("SELECT name, value, attrs FROM element_info WHERE element_key = ? ORDER BY ord")); + i.bind({key}); + while (i.next()) + info.appendChild(entry(i.value(0), QStringLiteral("name"), i.value(1), i.value(2), + QStringLiteral("elementInformation"))); + restoreWhitespace(info, v.at(n + 1)); + } + if (!v.at(n + 2).isNull()) { + links = document.createElement(QStringLiteral("links_uuids")); + setAttributesJson(links, v.at(n + 3)); + Statement l(db.m_db, QStringLiteral("SELECT linked_uuid, attrs FROM link WHERE element_key = ? ORDER BY ord")); + l.bind({key}); + while (l.next()) + links.appendChild(entry(l.value(0), QStringLiteral("uuid"), QVariant(), l.value(1), + QStringLiteral("link_uuid"))); + restoreWhitespace(links, v.at(n + 3)); + } + if ((!info.isNull() && !replaceMarker(e, info)) + || (!links.isNull() && !replaceMarker(e, links))) { + setError(error, QStringLiteral("symbol %1's information has no place").arg(key)); + return false; + } + } + } + for (const QDomElement &list : children(d, QStringLiteral("conductors"))) { + for (QDomElement c : children(list, QStringLiteral("conductor"))) { + if (!c.hasAttribute(KeyAttribute)) continue; + const QString key = c.attribute(KeyAttribute); + c.removeAttribute(KeyAttribute); + Statement s(db.m_db, QStringLiteral("SELECT %1 FROM conductor WHERE key = ?") + .arg(QStringLiteral("\"") + ConductorData.join(QStringLiteral("\", \"")) + QStringLiteral("\""))); + s.bind({key}); + if (!s.next()) { setError(error, QStringLiteral("wire %1 is not in the database").arg(key)); return false; } + QVariantList v; + for (int i = 0 ; i < ConductorData.size() ; ++i) v << s.value(i); + put(c, ConductorData, v, 0); + } + } + return true; +} +} + /** @brief QetContainerDb::extract Move the engineering data of @p project (the element, its @@ -609,32 +746,8 @@ bool QetContainerDb::restore(QDomElement project, const QByteArray &database, QS { QDomDocument document = project.ownerDocument(); Database db; - if (sqlite3_open(":memory:", &db.m_db) != SQLITE_OK) { - setError(error, QStringLiteral("cannot open the project database")); + if (!openDatabase(db, database, error)) return false; - } - unsigned char *copy = static_cast(sqlite3_malloc64(sqlite3_uint64(database.size()))); - if (!copy) { setError(error, QStringLiteral("out of memory")); return false; } - memcpy(copy, database.constData(), size_t(database.size())); - if (sqlite3_deserialize(db.m_db, "main", copy, database.size(), database.size(), - SQLITE_DESERIALIZE_FREEONCLOSE | SQLITE_DESERIALIZE_READONLY) != SQLITE_OK) { - setError(error, QStringLiteral("the project database cannot be read")); - return false; - } - //A file is untrusted: nothing in its schema may run, and every name - //read must be a plain table. - db.exec(QStringLiteral("PRAGMA trusted_schema = OFF")); - db.exec(QStringLiteral("PRAGMA query_only = ON")); - for (const char *table : {"meta", "project_property", "folio", "folio_property", - "element", "element_info", "link", "conductor", "terminal_strip", - "strip_terminal", "strip_bridge", "strip_bridge_terminal"}) { - Statement kind(db.m_db, QStringLiteral("SELECT type FROM sqlite_master WHERE name = ?")); - kind.bind({QString::fromLatin1(table)}); - if (!kind.next() || kind.value(0).toString() != QLatin1String("table")) { - setError(error, QStringLiteral("the project database has no table %1").arg(QLatin1String(table))); - return false; - } - } auto entry = [&document](const QVariant &name_attr, const QString &name, const QVariant &value, const QVariant &attrs, const QString &default_tag) { @@ -687,87 +800,18 @@ bool QetContainerDb::restore(QDomElement project, const QByteArray &database, QS } for (QDomElement d : children(project, QStringLiteral("diagram"))) - { - const QString id = d.attribute(FolioAttribute); - if (id.isEmpty()) continue; - d.removeAttribute(FolioAttribute); - Statement f(db.m_db, QStringLiteral("SELECT %1, props_at, props_attrs FROM folio WHERE id = ?") - .arg(QStringLiteral("\"") + FolioData.join(QStringLiteral("\", \"")) + QStringLiteral("\""))); - f.bind({id}); - if (!f.next()) { setError(error, QStringLiteral("folio %1 is not in the database").arg(id)); return false; } - QVariantList values; - for (int i = 0 ; i < FolioData.size() + 2 ; ++i) values << f.value(i); - put(d, FolioData, values, 0); - if (!values.at(FolioData.size()).isNull()) { - QDomElement props = document.createElement(QStringLiteral("properties")); - setAttributesJson(props, values.at(FolioData.size() + 1)); - Statement p(db.m_db, QStringLiteral("SELECT name, value, attrs FROM folio_property WHERE folio_id = ? ORDER BY ord")); - p.bind({id}); - while (p.next()) - props.appendChild(entry(p.value(0), QStringLiteral("name"), p.value(1), p.value(2), - QStringLiteral("property"))); - restoreWhitespace(props, values.at(FolioData.size() + 1)); - if (!replaceMarker(d, props)) { - setError(error, QStringLiteral("folio %1's properties have no place").arg(id)); - return false; - } - } - - for (const QDomElement &list : children(d, QStringLiteral("elements"))) { - for (QDomElement e : children(list, QStringLiteral("element"))) { - if (!e.hasAttribute(KeyAttribute)) continue; - const QString key = e.attribute(KeyAttribute); - e.removeAttribute(KeyAttribute); - Statement s(db.m_db, QStringLiteral("SELECT %1, info_at, info_attrs, links_at, links_attrs FROM element WHERE key = ?") - .arg(QStringLiteral("\"") + ElementData.join(QStringLiteral("\", \"")) + QStringLiteral("\""))); - s.bind({key}); - if (!s.next()) { setError(error, QStringLiteral("symbol %1 is not in the database").arg(key)); return false; } - QVariantList v; - for (int i = 0 ; i < ElementData.size() + 4 ; ++i) v << s.value(i); - put(e, ElementData, v, 0); - const int n = ElementData.size(); - QDomElement info, links; - if (!v.at(n).isNull()) { - info = document.createElement(QStringLiteral("elementInformations")); - setAttributesJson(info, v.at(n + 1)); - Statement i(db.m_db, QStringLiteral("SELECT name, value, attrs FROM element_info WHERE element_key = ? ORDER BY ord")); - i.bind({key}); - while (i.next()) - info.appendChild(entry(i.value(0), QStringLiteral("name"), i.value(1), i.value(2), - QStringLiteral("elementInformation"))); - restoreWhitespace(info, v.at(n + 1)); - } - if (!v.at(n + 2).isNull()) { - links = document.createElement(QStringLiteral("links_uuids")); - setAttributesJson(links, v.at(n + 3)); - Statement l(db.m_db, QStringLiteral("SELECT linked_uuid, attrs FROM link WHERE element_key = ? ORDER BY ord")); - l.bind({key}); - while (l.next()) - links.appendChild(entry(l.value(0), QStringLiteral("uuid"), QVariant(), l.value(1), - QStringLiteral("link_uuid"))); - restoreWhitespace(links, v.at(n + 3)); - } - if ((!info.isNull() && !replaceMarker(e, info)) - || (!links.isNull() && !replaceMarker(e, links))) { - setError(error, QStringLiteral("symbol %1's information has no place").arg(key)); - return false; - } - } - } - for (const QDomElement &list : children(d, QStringLiteral("conductors"))) { - for (QDomElement c : children(list, QStringLiteral("conductor"))) { - if (!c.hasAttribute(KeyAttribute)) continue; - const QString key = c.attribute(KeyAttribute); - c.removeAttribute(KeyAttribute); - Statement s(db.m_db, QStringLiteral("SELECT %1 FROM conductor WHERE key = ?") - .arg(QStringLiteral("\"") + ConductorData.join(QStringLiteral("\", \"")) + QStringLiteral("\""))); - s.bind({key}); - if (!s.next()) { setError(error, QStringLiteral("wire %1 is not in the database").arg(key)); return false; } - QVariantList v; - for (int i = 0 ; i < ConductorData.size() ; ++i) v << s.value(i); - put(c, ConductorData, v, 0); - } - } - } + if (!restoreDiagram(db, d, error)) + return false; return true; } + +/** + @brief QetContainerDb::restoreFolio + restore() for the one folio @p diagram, read from its own part (a folio + built after its project was opened from a .qetz, stage D2) +*/ +bool QetContainerDb::restoreFolio(QDomElement diagram, const QByteArray &database, QString *error) +{ + Database db; + return openDatabase(db, database, error) && restoreDiagram(db, diagram, error); +} diff --git a/sources/container/qetcontainerdb.h b/sources/container/qetcontainerdb.h index a70dfb1fe..e43c87946 100644 --- a/sources/container/qetcontainerdb.h +++ b/sources/container/qetcontainerdb.h @@ -39,6 +39,7 @@ namespace QetContainerDb { QByteArray extract(QDomElement project, QString *error = nullptr); bool restore(QDomElement project, const QByteArray &database, QString *error = nullptr); + bool restoreFolio(QDomElement diagram, const QByteArray &database, QString *error = nullptr); } #endif // QETCONTAINERDB_H diff --git a/sources/dataBase/projectdatabase.cpp b/sources/dataBase/projectdatabase.cpp index 68a98486e..3ebe41382 100644 --- a/sources/dataBase/projectdatabase.cpp +++ b/sources/dataBase/projectdatabase.cpp @@ -49,6 +49,8 @@ #include #include #include +#include +#include #include #include #include @@ -865,6 +867,282 @@ bool projectDataBase::populateFromDocument(const QDomDocument &document, QString return true; } +/** + @brief projectDataBase::schemaStamp + @return a stamp of the tables of this database as created: an index + file (indexFile()) is read back only into a database with the same +*/ +QString projectDataBase::schemaStamp() const +{ + QSqlQuery query(m_data_base); + query.exec(QStringLiteral("SELECT sql FROM sqlite_master WHERE type IN ('table', 'index') " + "AND name NOT LIKE 'sqlite_%' ORDER BY name")); + QCryptographicHash hash(QCryptographicHash::Sha256); + while (query.next()) + hash.addData(query.value(0).toString().toUtf8() + ';'); + return QString::fromLatin1(hash.result().toHex()); +} + +namespace { + //The tables an index file carries, all of this database's but the + //drawing items, which exist for built folios only +QStringList indexTables(const QSqlDatabase &db) +{ + QStringList tables; + QSqlQuery query(db); + query.exec(QStringLiteral("SELECT name FROM sqlite_master WHERE type = 'table' " + "AND name NOT LIKE 'sqlite_%' ORDER BY name")); + while (query.next()) { + const QString name = query.value(0).toString(); + if (name != QLatin1String("shape") && name != QLatin1String("independent_text") + && name != QLatin1String("image")) + tables << name; + } + return tables; +} +} + +/** + @brief projectDataBase::indexFile + This database written to an SQLite file, for a .qetz to be opened from + (stage D2): every table but the drawing items, brought + up to date first, with the folios' own data @p folio_headers (in folio + order, each as Diagram::headerXml() writes it) and the stamp of the + schema it was written with. + @return the file, or nothing if it could not be written +*/ +QByteArray projectDataBase::indexFile(const QList> &folio_headers) +{ + if (m_content_changed) { + updateDB(); + } + flushLinks(); + flushElementPositions(); + flushConductorProperties(); + + QTemporaryDir dir; + if (!dir.isValid()) { + return QByteArray(); + } + const QString path = dir.filePath(QStringLiteral("index.sqlite")); + QString escaped = path; + escaped.replace(QLatin1Char('\''), QStringLiteral("''")); + QSqlQuery vacuum(m_data_base); + if (!vacuum.exec(QStringLiteral("VACUUM INTO '%1'").arg(escaped))) { + qWarning() << "projectDataBase::indexFile:" << vacuum.lastError().text(); + return QByteArray(); + } + const QString stamp = schemaStamp(); + const QString connection = QStringLiteral("qet_index_") + QUuid::createUuid().toString(); + bool ok = false; + { + QSqlDatabase file = QSqlDatabase::addDatabase(QStringLiteral("QSQLITE"), connection); + file.setDatabaseName(path); + if (file.open()) + { + QSqlQuery query(file); + //Its views read the drawing items too: an index holds tables + QStringList views; + query.exec(QStringLiteral("SELECT name FROM sqlite_master WHERE type = 'view'")); + while (query.next()) views << query.value(0).toString(); + for (const QString &view : std::as_const(views)) + query.exec(QStringLiteral("DROP VIEW \"%1\"").arg(view)); + ok = query.exec(QStringLiteral("DROP TABLE shape")) + && query.exec(QStringLiteral("DROP TABLE independent_text")) + && query.exec(QStringLiteral("DROP TABLE image")) + && query.exec(QStringLiteral("CREATE TABLE index_meta (key TEXT PRIMARY KEY, value TEXT)")) + && query.exec(QStringLiteral("CREATE TABLE folio_header (pos INTEGER PRIMARY KEY, " + "uuid TEXT NOT NULL, xml TEXT NOT NULL)")); + file.transaction(); + query.prepare(QStringLiteral("INSERT INTO index_meta (key, value) VALUES (?, ?)")); + for (const auto &meta : {std::pair{QStringLiteral("schema"), stamp}, + std::pair{QStringLiteral("writer"), + QStringLiteral("QElectroTech ") + QetVersion::currentVersion().toString()}}) { + query.addBindValue(meta.first); + query.addBindValue(meta.second); + ok = ok && query.exec(); + } + query.prepare(QStringLiteral("INSERT INTO folio_header (pos, uuid, xml) VALUES (?, ?, ?)")); + int pos = 0; + for (const auto &header : folio_headers) { + query.addBindValue(pos++); + query.addBindValue(header.first.toString()); + query.addBindValue(header.second); + ok = ok && query.exec(); + } + ok = file.commit() && ok; + ok = query.exec(QStringLiteral("VACUUM")) && ok; + file.close(); + } + } + QSqlDatabase::removeDatabase(connection); + if (!ok) { + return QByteArray(); + } + QFile file(path); + if (!file.open(QIODevice::ReadOnly)) { + return QByteArray(); + } + return file.readAll(); +} + +/** + @brief projectDataBase::discardFill + Undo fillFromIndex(): the project is read from its document instead +*/ +void projectDataBase::discardFill() +{ + QSqlQuery query(m_data_base); + for (const QString &table : indexTables(m_data_base)) + query.exec(QStringLiteral("DELETE FROM \"%1\"").arg(table)); + m_element_information.clear(); + m_folio_titleblocks.clear(); + m_conductor_properties.clear(); + m_conductor_attributes.clear(); + m_prefilled_elements.clear(); + m_prefilled_folios.clear(); + m_prefilled_conductors.clear(); + m_rows_from_document = false; + m_content_changed = true; +} + +/** + @brief projectDataBase::fillFromIndex + Fill this database from @p index, a file indexFile() wrote, instead of + from the project's document: its tables, and the stores of symbol + information and wire properties (the folios' title blocks are filled + from @p folio_headers by the folios). The file is untrusted: read only + when written with this database's schema, its tables plain tables, and + nothing in its schema run. + @param folio_headers : the folios' own data, in folio order + @return false, with @p why set and nothing changed in the tables, if + the file cannot be read +*/ +bool projectDataBase::fillFromIndex(const QByteArray &index, + QList> *folio_headers, + QString *why) +{ + auto refuse = [why](const QString &reason) { + if (why) *why = reason; + return false; + }; + QTemporaryDir dir; + if (!dir.isValid()) { + return refuse(QStringLiteral("no temporary folder")); + } + const QString path = dir.filePath(QStringLiteral("index.sqlite")); + { + QFile file(path); + if (!file.open(QIODevice::WriteOnly) || file.write(index) != index.size()) { + return refuse(QStringLiteral("cannot write the index to a temporary file")); + } + } + QString escaped = path; + escaped.replace(QLatin1Char('\''), QStringLiteral("''")); + QSqlQuery query(m_data_base); + query.exec(QStringLiteral("PRAGMA trusted_schema = OFF")); + if (!query.exec(QStringLiteral("ATTACH DATABASE '%1' AS idx").arg(escaped))) { + return refuse(QStringLiteral("the index cannot be read")); + } + auto detach = [this]() { + QSqlQuery q(m_data_base); + q.exec(QStringLiteral("DETACH DATABASE idx")); + }; + + const QStringList tables = indexTables(m_data_base); + auto isTable = [this](const QString &name) { + QSqlQuery kind(m_data_base); + kind.prepare(QStringLiteral("SELECT type FROM idx.sqlite_master WHERE name = ?")); + kind.addBindValue(name); + return kind.exec() && kind.next() && kind.value(0).toString() == QLatin1String("table"); + }; + for (const QString &table : tables + QStringList{QStringLiteral("index_meta"), + QStringLiteral("folio_header")}) { + if (!isTable(table)) { + detach(); + return refuse(QStringLiteral("the index has no table %1").arg(table)); + } + } + query.exec(QStringLiteral("SELECT value FROM idx.index_meta WHERE key = 'schema'")); + if (!query.next() || query.value(0).toString() != schemaStamp()) { + detach(); + return refuse(QStringLiteral("the index was written with another database schema")); + } + + m_data_base.transaction(); + bool ok = true; + for (const QString &table : tables) { + QSqlQuery columns(m_data_base); + columns.exec(QStringLiteral("PRAGMA main.table_info(\"%1\")").arg(table)); + QStringList names; + while (columns.next()) + names << QStringLiteral("\"%1\"").arg(columns.value(1).toString()); + const QString list = names.join(QLatin1Char(',')); + ok = ok && query.exec(QStringLiteral("DELETE FROM main.\"%1\"").arg(table)) + && query.exec(QStringLiteral("INSERT INTO main.\"%1\" (%2) SELECT %2 FROM idx.\"%1\"") + .arg(table, list)); + } + if (!ok) { + m_data_base.rollback(); + detach(); + return refuse(QStringLiteral("the index's rows cannot be read")); + } + m_data_base.commit(); + + folio_headers->clear(); + query.exec(QStringLiteral("SELECT uuid, xml FROM idx.folio_header ORDER BY pos")); + while (query.next()) + folio_headers->append({QUuid(query.value(0).toString()), query.value(1).toString()}); + detach(); + + //The stores, from their tables, as the document would fill them + m_prefilled_elements.clear(); + m_prefilled_folios.clear(); + m_prefilled_conductors.clear(); + for (PrefillTally &tally : m_prefill_tally) tally = PrefillTally(); + query.exec(QStringLiteral("SELECT element_uuid, name, value, show FROM element_information " + "ORDER BY element_uuid, ord")); + while (query.next()) { + const QUuid uuid(query.value(0).toString()); + m_prefilled_elements[uuid].addValue(query.value(1).toString(), query.value(2).toString(), + query.value(3).toInt() != 0); + } + QHash> attributes; + query.exec(QStringLiteral("SELECT conductor_uuid, name, value FROM conductor_properties")); + while (query.next()) + attributes[QUuid(query.value(0).toString())].insert(query.value(1).toString(), + query.value(2).toString()); + QDomDocument scratch; + for (auto it = attributes.cbegin() ; it != attributes.cend() ; ++it) { + QDomElement e = scratch.createElement(QStringLiteral("conductor")); + for (auto a = it.value().cbegin() ; a != it.value().cend() ; ++a) + e.setAttribute(a.key(), a.value()); + ConductorProperties properties; + properties.fromXml(e); + m_prefilled_conductors.insert(it.key(), properties); + } + for (auto it = m_prefilled_elements.cbegin() ; it != m_prefilled_elements.cend() ; ++it) + m_element_information.insert(it.key(), it.value()); + for (auto it = m_prefilled_conductors.cbegin() ; it != m_prefilled_conductors.cend() ; ++it) + m_conductor_properties.insert(it.key(), it.value()); + m_conductor_attributes = attributes; + for (const auto &header : std::as_const(*folio_headers)) { + QDomDocument document; + if (!document.setContent(header.second)) continue; + TitleBlockProperties properties; + properties.fromXml(document.documentElement()); + properties.version = QetVersion::displayedVersion(); + properties.collection = QET::QetCollection::Embedded; + m_prefilled_folios.insert(header.first, properties); + m_folio_titleblocks.insert(header.first, properties); + } + + m_rows_from_document = true; + m_content_changed = false; + emit dataBaseUpdated(); + return true; +} + /** @brief projectDataBase::foliosWithUnjoinedWires @return the uuids of the folios whose document has a wire that building diff --git a/sources/dataBase/projectdatabase.h b/sources/dataBase/projectdatabase.h index be3322c7d..15119a61d 100644 --- a/sources/dataBase/projectdatabase.h +++ b/sources/dataBase/projectdatabase.h @@ -163,6 +163,13 @@ class projectDataBase : public QObject void setBuildingFolio(bool building); bool isBuildingFolio() const {return m_building_folios > 0;} bool rowsFromDocument() const {return m_rows_from_document;} + QString schemaStamp() const; + QByteArray indexFile(const QList> &folio_headers); + void discardFill(); + /// The tables are right as they are (filled from an index) + void markCurrent() {m_content_changed = false;} + bool fillFromIndex(const QByteArray &index, QList> *folio_headers, + QString *why = nullptr); QString prefillReport() const; QSet foliosWithUnjoinedWires() const; QSet folioUuidsOfElements(const QSet &elements) const; diff --git a/sources/diagram.cpp b/sources/diagram.cpp index 9586b9555..5ce6078ec 100644 --- a/sources/diagram.cpp +++ b/sources/diagram.cpp @@ -778,7 +778,7 @@ void Diagram::keyReleaseEvent(QKeyEvent *e) */ void Diagram::ensureLoaded() { - if (m_deferred.isNull()) { + if (isLoaded()) { return; } if (m_project) { @@ -788,6 +788,23 @@ void Diagram::ensureLoaded() } } +/** + @brief Diagram::deferredXml + @return the XML of this folio's items while they are not built: kept + from the project's document, or read now from the folio's part of the + .qetz the project was opened from +*/ +QDomElement Diagram::deferredXml() +{ + if (m_deferred.isNull() && !m_deferred_part.isEmpty() && m_project) { + m_deferred = m_project->readFolioPart(m_deferred_part); + if (!m_deferred.isNull()) { + m_deferred_part.clear(); + } + } + return m_deferred; +} + /** @brief Diagram::buildDeferred Build this folio's items from the XML kept for later, and nothing else. @@ -797,11 +814,15 @@ void Diagram::ensureLoaded() */ bool Diagram::buildDeferred() { - if (m_deferred.isNull()) { + if (isLoaded()) { return false; } - QDomElement xml = m_deferred; + QDomElement xml = deferredXml(); m_deferred = QDomElement(); + m_deferred_part.clear(); + if (xml.isNull()) { + return true; //its part could not be read: QETProject::readFolioPart() said why + } m_kept_text.clear(); projectDataBase *db = m_project ? m_project->dataBase() : nullptr; if (db) db->setBuildingFolio(true); diff --git a/sources/diagram.h b/sources/diagram.h index 44748c181..ca5e13951 100644 --- a/sources/diagram.h +++ b/sources/diagram.h @@ -142,6 +142,7 @@ class Diagram : public QGraphicsScene QUuid m_uuid = QUuid::createUuid(); /// The XML of this folio's items, while they are not built yet QDomElement m_deferred; + QString m_deferred_part; QString m_opened_header; QByteArray m_kept_text; bool m_loading_deferred = false; @@ -212,8 +213,12 @@ class Diagram : public QGraphicsScene DiagramContent * = nullptr); /// false while this folio's items wait in its XML to be /// built (QET_LAZY_FOLIOS) - bool isLoaded() const {return m_deferred.isNull();} - QDomElement deferredXml() const {return m_deferred;} + bool isLoaded() const {return m_deferred.isNull() && m_deferred_part.isEmpty();} + QDomElement deferredXml(); + /// The folio's items are in the part @p file of the .qetz the + /// project was opened from, read when they are first needed + /// (stage D2) + void setDeferredPart(const QString &file) {m_deferred = QDomElement(); m_deferred_part = file;} QString headerXml(); /// The folio's text in the file it was read from, while it is /// not built (QETProject::toXml()) diff --git a/sources/qetproject.cpp b/sources/qetproject.cpp index 92aeefc6c..f6c02dfc9 100644 --- a/sources/qetproject.cpp +++ b/sources/qetproject.cpp @@ -30,6 +30,7 @@ #include "autoNum/elementautonumschemecommand.h" #include "diagram.h" #include "qetgraphicsitem/element.h" +#include "qetgraphicsitem/conductor.h" #include "qetgraphicsitem/ViewItem/qetgraphicstableitem.h" #include "qetgraphicsitem/ViewItem/projectdbmodel.h" #include "qetapp.h" @@ -462,7 +463,40 @@ QETProject::ProjectState QETProject::openFile(QFile *file) // QetContainer::parse() does exactly that, for a .qet and for each part // of a .qetz alike. QString read_error; - if (!QetContainer::readDocument(content, &xml_project, &read_error)) + //A .qetz holding the index of its database opens from it, the + //folios' parts read when they are built (stage D2); + //any other, or one whose index cannot be used, is read whole + m_opened_from_index = false; + if (QetContainer::isZip(content) && qEnvironmentVariableIntValue("QET_LAZY_FOLIOS") == 1) + { + QList entries; + QString why; + if (QetZip::fromBytes(content, &entries, &why)) + { + QHash files; + for (const QetZip::Entry &entry : std::as_const(entries)) + files.insert(entry.name, entry.data); + QStringList folio_files; + QDomDocument document; + if (files.contains(QStringLiteral("index.sqlite")) + && QetContainer::join(entries, &document, &why, &folio_files)) + { + if (m_data_base.fillFromIndex(files.value(QStringLiteral("index.sqlite")), + &m_folio_headers, &why) + && m_folio_headers.size() == folio_files.size()) { + xml_project = document; + m_container_files = files; + m_folio_files = folio_files; + m_opened_from_index = true; + } else { + m_data_base.discardFill(); + m_folio_headers.clear(); + qInfo().noquote() << "Not opened from the index of" << fi.fileName() << ":" << why; + } + } + } + } + if (!m_opened_from_index && !QetContainer::readDocument(content, &xml_project, &read_error)) { qWarning().noquote() << "Unable to read" << fi.fileName() << ":" << read_error; if(opened_here) { @@ -798,6 +832,16 @@ void QETProject::buildFoliosHoldingSymbols(const QSet &symbols) return; } QList to_build; + //Opened from an index, the database knows where every symbol is + //without reading the folios' parts + if (m_opened_from_index) { + const QSet folios = m_data_base.folioUuidsOfElements(symbols); + for (Diagram *d : std::as_const(m_diagrams_list)) + if (!d->isLoaded() && folios.contains(d->uuid())) + to_build << d; + buildFolios(to_build); + return; + } for (Diagram *d : std::as_const(m_diagrams_list)) { if (d->isLoaded()) { @@ -2471,15 +2515,59 @@ bool QETProject::isContainerPath(const QString &path) document, in parts, zipped (#1440). The file is replaced only once the whole archive is written. */ -bool QETProject::writeContainer(const QDomDocument &project, const QString &path, QString *error) +bool QETProject::writeContainer(const QDomDocument &project, const QString &path, QString *error, + const QByteArray &index) { - const QList entries = QetContainer::split( + QList entries = QetContainer::split( project, QStringLiteral("QElectroTech %1").arg(QetVersion::currentVersion().toString()), error); + //The project database, for opening without reading the folios + //(stage D2); a reader that does not know it skips it + if (!entries.isEmpty() && !index.isEmpty()) + entries.append(QetZip::Entry{QStringLiteral("index.sqlite"), index, true}); return !entries.isEmpty() && QetZip::write(path, entries, error); } +/** + @brief QETProject::writeContainerFile + Write this project to @p path as a .qetz, with the index of its + database a .qetz is opened from (stage D2) when every + folio, symbol and wire has a uuid of its own -- what opening from the + index relies on, as opening without building folios does. +*/ +bool QETProject::writeContainerFile(const QString &path, QString *error) +{ + const QDomDocument document = toXml(); //every folio built + return writeContainer(document, path, error, containerIndex()); +} + +/** + @brief QETProject::containerIndex + @return the index of this project's database for a .qetz, empty when a + folio, symbol or wire has no uuid of its own +*/ +QByteArray QETProject::containerIndex() +{ + loadFolios(); + QSet seen; + QList> headers; + auto unique = [&seen](const QUuid &uuid) { + if (uuid.isNull() || seen.contains(uuid)) return false; + seen.insert(uuid); + return true; + }; + for (Diagram *diagram : std::as_const(m_diagrams_list)) { + if (!unique(diagram->uuid())) return QByteArray(); + for (Element *element : diagram->elements()) + if (!unique(element->uuid())) return QByteArray(); + for (Conductor *conductor : diagram->conductors()) + if (!unique(conductor->uuid())) return QByteArray(); + headers.append({diagram->uuid(), diagram->headerXml()}); + } + return m_data_base.indexFile(headers); +} + /** @brief QETProject::besideContainerPath @return the .qetz a project read from a .qet is saved to: the same @@ -2539,9 +2627,9 @@ QETResult QETProject::write() QString error_message; if (isContainerPath(m_file_path)) { - // A container must contain complete folios, not lazy placeholders. - QDomDocument xml_project(toXml()); - if (!writeContainer(xml_project, m_file_path, &error_message)) + //A container is written from the whole document: every folio + //built (stage D4 writes it from the database) + if (!writeContainerFile(m_file_path, &error_message)) return(error_message); } else @@ -3059,7 +3147,9 @@ void QETProject::readProjectXml(QDomDocument &xml_project) //The stores of symbol information, folio title blocks and wire //properties are filled from the document before the folios are //built - m_data_base.prefillFromDocument(xml_project); + if (!m_opened_from_index) { + m_data_base.prefillFromDocument(xml_project); + } //QET_LAZY_FOLIOS leaves the folios to be built when needed only //when the database can be filled from the document: otherwise @@ -3075,7 +3165,11 @@ void QETProject::readProjectXml(QDomDocument &xml_project) } //Load the diagrams - readDiagramsXml(xml_project); + if (m_opened_from_index) { + readFoliosFromIndex(); + } else { + readDiagramsXml(xml_project); + } const qint64 diagrams_ms = phase_timer.restart(); //Tie the elements to the numbering schemes they follow @@ -3104,7 +3198,14 @@ void QETProject::readProjectXml(QDomDocument &xml_project) m_data_base.blockSignals(false); m_data_base.setUpdateBlocked(false); - m_data_base.updateDB(xml_project); + if (m_opened_from_index) { + //Filled from the index already: making the folios changed + //nothing their rows say + m_data_base.markCurrent(); + m_data_base.updateDB(); + } else { + m_data_base.updateDB(xml_project); + } m_data_base.endPrefill(unloadedFolioCount() > 0); //Saving may write the folios not built as they were read, when //building them would write the same: a file written by this @@ -3145,6 +3246,51 @@ void QETProject::readProjectXml(QDomDocument &xml_project) m_state = Ok; } +/** + @brief QETProject::readFoliosFromIndex + readDiagramsXml() for a .qetz opened from its index: each folio made + from its own data as the index has it, its items left in its part + (stage D2) +*/ +void QETProject::readFoliosFromIndex() +{ + for (int i = 0 ; i < m_folio_headers.size() ; ++i) + { + QDomDocument header; + header.setContent(m_folio_headers.at(i).second); + auto diagram = new Diagram(this); + m_diagrams_list << diagram; + connect(&diagram->border_and_titleblock, &BorderTitleBlock::needFolioData, + this, &QETProject::updateDiagramsFolioData); + connect(diagram, &Diagram::usedTitleBlockTemplateChanged, + this, &QETProject::usedTitleBlockTemplateChanged); + m_deferring_folios = true; + QDomElement header_element = header.documentElement(); + diagram->initFromXml(header_element); + m_deferring_folios = false; + diagram->setDeferredPart(m_folio_files.at(i)); + m_data_base.placeFolio(diagram); + } + updateDiagramsFolioData(); +} + +/** + @brief QETProject::readFolioPart + @return the folio part @p file of the .qetz this project was opened + from, read now, its engineering data back from project.sqlite; null + if it cannot be read (said in the log) +*/ +QDomElement QETProject::readFolioPart(const QString &file) +{ + QDomDocument folio; + QString error; + if (!QetContainer::readFolio(m_container_files, file, &folio, &error)) { + qWarning().noquote() << "Unable to read the folio" << file << ":" << error; + return QDomElement(); + } + return folio.documentElement(); +} + /** @brief QETProject::readDiagramsXml Load the diagrams from the xml description of the project. diff --git a/sources/qetproject.h b/sources/qetproject.h index 07d4abd51..a9f79f919 100644 --- a/sources/qetproject.h +++ b/sources/qetproject.h @@ -443,7 +443,11 @@ class QETProject : public QObject QString besideContainerPath() const; QETResult writeCopyForOlderVersions(const QString &path); static bool writeContainer(const QDomDocument &project, const QString &path, - QString *error = nullptr); + QString *error = nullptr, const QByteArray &index = QByteArray()); + bool writeContainerFile(const QString &path, QString *error = nullptr); + QByteArray containerIndex(); + QDomElement readFolioPart(const QString &file); + bool openedFromIndex() const {return m_opened_from_index;} private: static QUuid derivedUuid(const QByteArray &content); void refresh(); @@ -528,6 +532,13 @@ class QETProject : public QObject bool m_kept_folios_trusted = false; /// Read from a .qet whose path nobody chose since: saved beside it bool m_opened_from_qet = false; + /// Opened from a .qetz's index (stage D2): the + /// database filled from it, each folio's items in its part + bool m_opened_from_index = false; + QHash m_container_files; + QStringList m_folio_files; + QList> m_folio_headers; + void readFoliosFromIndex(); bool m_kept_folios_stale = false; /// Something was edited since the project was opened: a table /// filled from the database may have changed diff --git a/sources/scripting/liveserver.cpp b/sources/scripting/liveserver.cpp index 132b244f8..167f50930 100644 --- a/sources/scripting/liveserver.cpp +++ b/sources/scripting/liveserver.cpp @@ -919,11 +919,11 @@ QJsonObject LiveServer::snapshot(const QString &raw_path) if (raw_path.trimmed().isEmpty() || info.isRelative()) return failure(QStringLiteral("path must be absolute")); if (info.exists()) return failure(QStringLiteral("%1 already exists").arg(raw_path)); - QDomDocument xml(project->toXml()); QString error; if (QETProject::isContainerPath(info.absoluteFilePath()) - ? !QETProject::writeContainer(xml, info.absoluteFilePath(), &error) - : !QET::writeXmlFile(xml, info.absoluteFilePath(), &error)) + ? !project->writeContainerFile(info.absoluteFilePath(), &error) + : ![&]() { QDomDocument xml(project->toXml()); + return QET::writeXmlFile(xml, info.absoluteFilePath(), &error); }()) return failure(QStringLiteral("snapshot not written: %1").arg(error)); return {{QStringLiteral("ok"), true}, {QStringLiteral("path"), info.absoluteFilePath()}, {QStringLiteral("folio"), e->currentDiagramView() diff --git a/sources/scripting/qetscriptapi.cpp b/sources/scripting/qetscriptapi.cpp index 936088c81..be97fc618 100644 --- a/sources/scripting/qetscriptapi.cpp +++ b/sources/scripting/qetscriptapi.cpp @@ -434,11 +434,11 @@ bool QetScriptApi::save(const QString &output) return true; } - QDomDocument xml_doc(m_project->toXml()); QString error_message; if (QETProject::isContainerPath(output) - ? !QETProject::writeContainer(xml_doc, output, &error_message) - : !QET::writeXmlFile(xml_doc, output, &error_message)) { + ? !m_project->writeContainerFile(output, &error_message) + : ![&]() { QDomDocument xml_doc(m_project->toXml()); + return QET::writeXmlFile(xml_doc, output, &error_message); }()) { log(QStringLiteral("qet.save: %1").arg(error_message)); return false; } diff --git a/tests/qttest/lazy_folios_probe.cpp b/tests/qttest/lazy_folios_probe.cpp index 2ed2d085e..c9f3c5c1b 100644 --- a/tests/qttest/lazy_folios_probe.cpp +++ b/tests/qttest/lazy_folios_probe.cpp @@ -607,6 +607,25 @@ int main(int argc, char **argv) check(opened.state() == QETProject::Ok, "a project with a shared uuid opens"); check(opened.unloadedFolioCount() == 0, "a project with a shared uuid opens with every folio built"); check(built_late == 0, "a project with a shared uuid builds its folios as it reads them, not later"); + } + //A .qetz opened from its index (stage D2): no + //folio read, each read from its part when shown, drawn and + //linked as when built, saved as the .qet is + { + const QString zipped = output + "/lazy_folios.qetz"; + QString error; + check(eager.writeContainerFile(zipped, &error), "saved as a .qetz"); + QETProject indexed(zipped); + check(indexed.state() == QETProject::Ok, "the .qetz opens"); + check(indexed.openedFromIndex(), "the .qetz opens from its index"); + std::printf(".qetz opened from its index: %d of %d folios not built (strip folios %d)\n", + indexed.unloadedFolioCount(), int(folios.size()), int(strip_folios.size())); + check(indexed.unloadedFolioCount() == folios.size() - strip_folios.size(), + "the .qetz opens building only the folios of the terminal strip's terminals"); + check(numbering(indexed) == numbering(eager), "symbols' numbering from the index as when built"); + for (int i = 0; i < folios.size(); ++i) + same(indexed, i, "shown from the .qetz"); + check(indexed.toXml().toString() == eager.toXml().toString(), "the .qetz saves as when built"); } std::printf("PASS: %d folios\n", int(folios.size())); } catch (const std::exception &error) {