diff --git a/sources/container/qetcontainer.cpp b/sources/container/qetcontainer.cpp index 5749113ee..1bfa5f8e6 100644 --- a/sources/container/qetcontainer.cpp +++ b/sources/container/qetcontainer.cpp @@ -22,6 +22,7 @@ #include #include #include +#include const char QetContainer::MimeType[] = "application/x-qelectrotech-project+zip"; @@ -228,7 +229,8 @@ void QetContainer::dropComputed(QDomElement diagram, bool terminal_lists) set if @p project is not a project */ QList QetContainer::split(const QDomDocument &project, const QString &writer, - QString *error, const QByteArray &index) + QString *error, const QByteArray &index, + const QHash &kept_from) { QDomDocument document = project.cloneNode(true).toDocument(); QDomElement root = document.documentElement(); @@ -238,7 +240,11 @@ QList QetContainer::split(const QDomDocument &project, const QStr } //The engineering data out of the document, into project.sqlite - const QByteArray database = QetContainerDb::extract(root, error, index); + //A folio written as it was read (, + //stage D4) keeps its part and, through + //@p kept_from's project.sqlite, its rows + const QByteArray database = QetContainerDb::extract( + root, error, index, kept_from.value(QStringLiteral("project.sqlite"))); if (database.isEmpty()) return {}; @@ -247,11 +253,35 @@ QList QetContainer::split(const QDomDocument &project, const QStr //Folios, each with its pictures taken out int folio = 0; - QMap pictures; //name -> png, once each //Nothing QElectroTech computes is kept (format 3) for (QDomElement diagram : childElements(root, QStringLiteral("diagram"))) dropComputed(diagram, !index.isEmpty()); + //The parts of the folios kept as they were, and their pictures + QMap pictures; //name -> png, once each + for (QDomElement diagram : childElements(root, QStringLiteral("diagram"))) + { + const QString kept = diagram.attribute(QStringLiteral("qetz-kept-part")); + if (kept.isEmpty()) continue; + if (!kept_from.contains(kept)) { + setError(error, QStringLiteral("the part \"%1\" of a folio kept as it was is missing").arg(kept)); + return {}; + } + taken.insert(kept); + const QByteArray part = kept_from.value(kept); + static const QRegularExpression image_file(QStringLiteral("%1=\"(images/[^\"]+)\"").arg(ImageFileAttribute)); + for (const auto &match : image_file.globalMatch(QString::fromUtf8(part))) { + const QString name = match.captured(1); + if (!kept_from.contains(name)) { + setError(error, QStringLiteral("the picture \"%1\" of a folio kept as it was is missing").arg(name)); + return {}; + } + pictures.insert(name, kept_from.value(name)); + } + parts.append(QetZip::Entry{kept, part, true}); + root.replaceChild(placeholder(document, kept), diagram); + } + for (QDomElement diagram : childElements(root, QStringLiteral("diagram"))) { ++folio; diff --git a/sources/container/qetcontainer.h b/sources/container/qetcontainer.h index 1d654f49d..16da23a29 100644 --- a/sources/container/qetcontainer.h +++ b/sources/container/qetcontainer.h @@ -21,6 +21,7 @@ #include "qetzip.h" #include +#include /** @brief The QetContainer namespace @@ -52,7 +53,8 @@ namespace QetContainer QList split(const QDomDocument &project, const QString &writer = QString(), QString *error = nullptr, - const QByteArray &index = QByteArray()); + const QByteArray &index = QByteArray(), + const QHash &kept_from = QHash()); bool join(const QList &entries, QDomDocument *project, QString *error = nullptr, QStringList *folio_files = nullptr); /// A folio part of @p files, which join() left out, with its diff --git a/sources/container/qetcontainerdb.cpp b/sources/container/qetcontainerdb.cpp index 49beba7b9..7ec4c6e0a 100644 --- a/sources/container/qetcontainerdb.cpp +++ b/sources/container/qetcontainerdb.cpp @@ -24,12 +24,15 @@ #include #include #include +#include #include #include namespace { const QString KeyAttribute = QStringLiteral("qetz-key"); + //A folio written as it was read (stage D4): its part +const QString KeptPartAttribute = QStringLiteral("qetz-kept-part"); const QString FolioAttribute = QStringLiteral("qetz-folio"); //What a moved block leaves where it was. Removing it instead would leave //the whitespace before and after it side by side, read back as one text @@ -604,7 +607,8 @@ bool restoreDiagram(Database &db, QDomElement d, QString *error) folios still in it) into a new database. @return the database file, or an empty array with @p error set */ -QByteArray QetContainerDb::extract(QDomElement project, QString *error, const QByteArray &index) +QByteArray QetContainerDb::extract(QDomElement project, QString *error, const QByteArray &index, + const QByteArray &kept_from) { Database db; db.m_prefix = QStringLiteral("doc_"); @@ -627,6 +631,28 @@ QByteArray QetContainerDb::extract(QDomElement project, QString *error, const QB if (!db.exec(schema())) { setError(error, QStringLiteral("cannot create the project database")); return {}; + } + //The database of the .qetz this project was read from, for the folios + //kept as they were (stage D4) + if (!kept_from.isEmpty()) { + unsigned char *copy = static_cast(sqlite3_malloc64(sqlite3_uint64(kept_from.size()))); + if (!copy) { setError(error, QStringLiteral("out of memory")); return {}; } + memcpy(copy, kept_from.constData(), size_t(kept_from.size())); + if (!db.exec(QStringLiteral("ATTACH DATABASE ':memory:' AS old")) + || sqlite3_deserialize(db.m_db, "old", copy, kept_from.size(), kept_from.size(), + SQLITE_DESERIALIZE_FREEONCLOSE | SQLITE_DESERIALIZE_READONLY) != SQLITE_OK) { + setError(error, QStringLiteral("cannot read the database the kept folios come from")); + return {}; + } + db.exec(QStringLiteral("PRAGMA trusted_schema = OFF")); + for (const QString &table : DocTables) { + Statement kind(db, QStringLiteral("SELECT type FROM old.sqlite_master WHERE name = ?")); + kind.bind({db.table(table)}); + if (!kind.next() || kind.value(0).toString() != QLatin1String("table")) { + setError(error, QStringLiteral("the kept folios' database has no table %1").arg(table)); + return {}; + } + } } db.exec(QStringLiteral("BEGIN")); @@ -683,10 +709,53 @@ QByteArray QetContainerDb::extract(QDomElement project, QString *error, const QB const QStringList conductor_keys = keys(all_conductors, QStringLiteral("c")); int e_index = 0, c_index = 0; + //A folio's id is its uuid when every folio has one of its own, so + //that a folio kept as it was keeps its rows (format 3, D4) + bool uuid_ids = true; + { + QSet seen; + for (const QDomElement &d : diagrams) { + const QString uuid = d.attribute(QStringLiteral("uuid")); + if (QUuid(uuid).isNull() || seen.contains(uuid)) { uuid_ids = false; break; } + seen.insert(uuid); + } + } + if (uuid_ids) { + ok = ok && meta.run({QStringLiteral("folio_ids"), QStringLiteral("uuid")}); + } + int pos = 0; for (QDomElement d : diagrams) { - const QString id = QStringLiteral("f%1").arg(++pos); + ++pos; + const QString id = uuid_ids ? d.attribute(QStringLiteral("uuid")) : QStringLiteral("f%1").arg(pos); + //Kept as it was: its rows from the database it was read from + if (d.hasAttribute(KeptPartAttribute)) { + if (kept_from.isEmpty() || !uuid_ids) { + setError(error, QStringLiteral("a folio kept as it was, without its database")); + return {}; + } + Statement copy(db, QStringLiteral("INSERT INTO folio SELECT id, ?, %1, props_at, props_attrs " + "FROM old.folio WHERE id = ?") + .arg(QStringLiteral("\"") + FolioData.join(QStringLiteral("\", \"")) + QStringLiteral("\""))); + ok = ok && copy.run({pos, id}) && sqlite3_changes(db.m_db) == 1; + for (const QString &sql : { + QStringLiteral("INSERT INTO folio_property SELECT * FROM old.folio_property WHERE folio_id = ? ORDER BY rowid"), + QStringLiteral("INSERT INTO element SELECT * FROM old.element WHERE folio_id = ? ORDER BY rowid"), + QStringLiteral("INSERT INTO element_info SELECT * FROM old.element_info WHERE element_key IN " + "(SELECT key FROM old.element WHERE folio_id = ?) ORDER BY rowid"), + QStringLiteral("INSERT INTO link SELECT * FROM old.link WHERE element_key IN " + "(SELECT key FROM old.element WHERE folio_id = ?) ORDER BY rowid"), + QStringLiteral("INSERT INTO conductor SELECT * FROM old.conductor WHERE folio_id = ? ORDER BY rowid")}) { + Statement rows(db, sql); + ok = ok && rows.run({id}); + } + if (!ok) { + setError(error, QStringLiteral("folio %1 kept as it was is not in its database").arg(id)); + return {}; + } + continue; + } QVariantList row{id, pos}; row += take(d, FolioData); QDomElement props = d.firstChildElement(QStringLiteral("properties")); @@ -859,3 +928,18 @@ bool QetContainerDb::restoreFolio(QDomElement diagram, const QByteArray &databas Database db; return openDatabase(db, database, error) && restoreDiagram(db, diagram, error); } + +/** + @brief QetContainerDb::foliosByUuid + @return true if @p database, a format 3 project.sqlite, names its + folios by their uuid: a folio written as it was read keeps its rows + under it (stage D4) +*/ +bool QetContainerDb::foliosByUuid(const QByteArray &database) +{ + Database db; + if (!openDatabase(db, database, nullptr) || db.m_prefix.isEmpty()) + return false; + Statement s(db, QStringLiteral("SELECT value FROM meta WHERE key = 'folio_ids'")); + return s.next() && s.value(0).toString() == QLatin1String("uuid"); +} diff --git a/sources/container/qetcontainerdb.h b/sources/container/qetcontainerdb.h index 5e1ec63ca..8e49eca63 100644 --- a/sources/container/qetcontainerdb.h +++ b/sources/container/qetcontainerdb.h @@ -38,9 +38,12 @@ namespace QetContainerDb { QByteArray extract(QDomElement project, QString *error = nullptr, - const QByteArray &index = QByteArray()); + const QByteArray &index = QByteArray(), + const QByteArray &kept_from = QByteArray()); bool restore(QDomElement project, const QByteArray &database, QString *error = nullptr); bool restoreFolio(QDomElement diagram, const QByteArray &database, QString *error = nullptr); + /// true if @p database names its folios by their uuid (format 3) + bool foliosByUuid(const QByteArray &database); } #endif // QETCONTAINERDB_H diff --git a/sources/diagram.cpp b/sources/diagram.cpp index 5ce6078ec..fe7d34346 100644 --- a/sources/diagram.cpp +++ b/sources/diagram.cpp @@ -798,9 +798,6 @@ QDomElement Diagram::deferredXml() { if (m_deferred.isNull() && !m_deferred_part.isEmpty() && m_project) { m_deferred = m_project->readFolioPart(m_deferred_part); - if (!m_deferred.isNull()) { - m_deferred_part.clear(); - } } return m_deferred; } diff --git a/sources/diagram.h b/sources/diagram.h index ca5e13951..a73f9938c 100644 --- a/sources/diagram.h +++ b/sources/diagram.h @@ -219,6 +219,7 @@ class Diagram : public QGraphicsScene /// project was opened from, read when they are first needed /// (stage D2) void setDeferredPart(const QString &file) {m_deferred = QDomElement(); m_deferred_part = file;} + QString deferredPart() const {return m_deferred_part;} QString headerXml(); /// The folio's text in the file it was read from, while it is /// not built (QETProject::toXml()) diff --git a/sources/qetproject.cpp b/sources/qetproject.cpp index 11b8c3f22..0e1a2b726 100644 --- a/sources/qetproject.cpp +++ b/sources/qetproject.cpp @@ -18,6 +18,7 @@ #include "qetproject.h" #include "container/qetcontainer.h" +#include "container/qetcontainerdb.h" #include "ElementsCollection/xmlelementcollection.h" #include "autoNum/assignvariables.h" @@ -2293,8 +2294,12 @@ void QETProject::autoFolioNumberingSelectedFolios(int from, @brief QETProject::toXml @return un document XML representant le projet */ -QDomDocument QETProject::toXml(QHash *kept_folios) +QDomDocument QETProject::toXml(QHash *kept_folios, QStringList *kept_parts) { + //With @p kept_parts, the same for a .qetz opened from its index: a + //folio not built is written as , its + //part and its rows copied by QetContainer::split() + //(stage D4) //With @p kept_folios, a folio not built (QET_LAZY_FOLIOS) is //written as a placeholder and its text //as read from the file goes in kept_folios, for @@ -2308,8 +2313,8 @@ QDomDocument QETProject::toXml(QHash *kept_folios) //is built with the folios its links reach, as when it is shown: //their saved texts can show its data (a report, its partner's //folio label). - const bool keep = kept_folios && unloadedFolioCount() - && m_kept_folios_trusted && !m_kept_folios_stale; + const bool keep = unloadedFolioCount() && !m_kept_folios_stale + && ((kept_folios && m_kept_folios_trusted) || (kept_parts && m_kept_parts_trusted)); if (keep) { for (Diagram *diagram : std::as_const(m_diagrams_list)) if (diagram->headerChanged()) @@ -2423,6 +2428,15 @@ QDomDocument QETProject::toXml(QHash *kept_folios) << "[" << diagram << "]"; + if (keep && !diagram->isLoaded() && kept_parts) { + auto placeholder = xml_doc.createElement(QStringLiteral("diagram")); + placeholder.setAttribute(QStringLiteral("qetz-kept-part"), diagram->deferredPart()); + placeholder.setAttribute(QStringLiteral("uuid"), diagram->uuid().toString()); + kept_parts->append(diagram->deferredPart()); + project_root.appendChild(placeholder); + ++order_num; + continue; + } if (keep && !diagram->isLoaded()) { auto placeholder = xml_doc.createElement(QStringLiteral("diagram")); placeholder.setAttribute(QStringLiteral("qet_kept_folio"), order_num); @@ -2520,7 +2534,7 @@ bool QETProject::isContainerPath(const QString &path) the whole archive is written. */ bool QETProject::writeContainer(const QDomDocument &project, const QString &path, QString *error, - const QByteArray &index) + const QByteArray &index, const QHash &kept_from) { //The project database, for opening without reading the folios //(stage D2), is in project.sqlite with the @@ -2528,7 +2542,7 @@ bool QETProject::writeContainer(const QDomDocument &project, const QString &path QList entries = QetContainer::split( project, QStringLiteral("QElectroTech %1").arg(QetVersion::currentVersion().toString()), - error, index); + error, index, kept_from); return !entries.isEmpty() && QetZip::write(path, entries, error); } @@ -2541,8 +2555,12 @@ bool QETProject::writeContainer(const QDomDocument &project, const QString &path */ bool QETProject::writeContainerFile(const QString &path, QString *error) { - const QDomDocument document = toXml(); //every folio built - return writeContainer(document, path, error, containerIndex()); + //The folios not built, written as they were read, when that is + //what building them would write (toXml()); every other folio built + QStringList kept; + const QDomDocument document = toXml(nullptr, &kept); + return writeContainer(document, path, error, containerIndex(), + kept.isEmpty() ? QHash() : m_container_files); } /** @@ -2552,7 +2570,11 @@ bool QETProject::writeContainerFile(const QString &path, QString *error) */ QByteArray QETProject::containerIndex() { - loadFolios(); + //Opened from an index, the folios not built have their rows in the + //database, whose keys are unique; the built ones are looked at + if (!m_kept_parts_trusted) { + loadFolios(); + } QSet seen; QList> headers; auto unique = [&seen](const QUuid &uuid) { @@ -2562,6 +2584,10 @@ QByteArray QETProject::containerIndex() }; for (Diagram *diagram : std::as_const(m_diagrams_list)) { if (!unique(diagram->uuid())) return QByteArray(); + if (!diagram->isLoaded()) { + headers.append({diagram->uuid(), diagram->headerXml()}); + continue; + } for (Element *element : diagram->elements()) if (!unique(element->uuid())) return QByteArray(); for (Conductor *conductor : diagram->conductors()) @@ -3224,9 +3250,13 @@ void QETProject::readProjectXml(QDomDocument &xml_project) for (Diagram *diagram : std::as_const(m_diagrams_list)) if (!diagram->isLoaded() && diagram->keptText().isEmpty()) m_kept_folios_trusted = false; + m_kept_parts_trusted = m_opened_from_index && unloadedFolioCount() > 0 + && m_project_qet_version == QetVersion::currentVersion() + && m_data_base.rowsFromDocument() + && QetContainerDb::foliosByUuid(m_container_files.value(QStringLiteral("project.sqlite"))); m_kept_folios_stale = false; m_edited_since_open = false; - if (m_kept_folios_trusted) + if (m_kept_folios_trusted || m_kept_parts_trusted) for (Diagram *diagram : std::as_const(m_diagrams_list)) diagram->rememberHeader(); if (qEnvironmentVariableIntValue("QET_LAZY_FOLIOS_TRACE") == 1) diff --git a/sources/qetproject.h b/sources/qetproject.h index a9f79f919..18e061505 100644 --- a/sources/qetproject.h +++ b/sources/qetproject.h @@ -313,7 +313,8 @@ class QETProject : public QObject void autoFolioNumberingNewFolios (); void autoFolioNumberingSelectedFolios(int, int, const QString&); - QDomDocument toXml(QHash *kept_folios = nullptr); + QDomDocument toXml(QHash *kept_folios = nullptr, + QStringList *kept_parts = nullptr); static QString spliceKeptFolios(QString text, const QHash &kept_folios); bool close(); QETResult write(); @@ -443,7 +444,8 @@ class QETProject : public QObject QString besideContainerPath() const; QETResult writeCopyForOlderVersions(const QString &path); static bool writeContainer(const QDomDocument &project, const QString &path, - QString *error = nullptr, const QByteArray &index = QByteArray()); + QString *error = nullptr, const QByteArray &index = QByteArray(), + const QHash &kept_from = QHash()); bool writeContainerFile(const QString &path, QString *error = nullptr); QByteArray containerIndex(); QDomElement readFolioPart(const QString &file); @@ -535,6 +537,9 @@ class QETProject : public QObject /// Opened from a .qetz's index (stage D2): the /// database filled from it, each folio's items in its part bool m_opened_from_index = false; + /// Saving a .qetz may copy the part of a folio not built + /// (toXml(), stage D4) + bool m_kept_parts_trusted = false; QHash m_container_files; QStringList m_folio_files; QList> m_folio_headers; diff --git a/tests/qttest/lazy_folios_probe.cpp b/tests/qttest/lazy_folios_probe.cpp index c9f3c5c1b..79a5815b9 100644 --- a/tests/qttest/lazy_folios_probe.cpp +++ b/tests/qttest/lazy_folios_probe.cpp @@ -176,6 +176,23 @@ static QString saved(QETProject &project) return QETProject::spliceKeptFolios(document.toString(4), kept); } + //@p xml's lines in order, any uuid not in the file @p path "new": a + //pasted copy's and a new wire's are random, and a folio's symbols are + //saved in uuid order +static QStringList sortedLines(QString xml, const QString &path) +{ + QFile file(path); file.open(QIODevice::ReadOnly); + const QString before = QString::fromUtf8(file.readAll()); + static const QRegularExpression uuid_re(QStringLiteral("\\{[0-9a-f-]{36}\\}")); + for (auto it = uuid_re.globalMatch(QString(xml)) ; it.hasNext() ; ) { + const QString uuid = it.next().captured(0); + if (!before.contains(uuid)) xml.replace(uuid, QStringLiteral("new")); + } + QStringList list = xml.split(QLatin1Char('\n')); + list.sort(); + return list; +} + //The rows of the project database's tables, in their order, read as //every caller reads them (newQuery(): brought up to date first) static QStringList tables(QETProject &project) @@ -626,6 +643,43 @@ int main(int argc, char **argv) for (int i = 0; i < folios.size(); ++i) same(indexed, i, "shown from the .qetz"); check(indexed.toXml().toString() == eager.toXml().toString(), "the .qetz saves as when built"); + + //Saved as a .qetz without building the folios not touched + //(D4): their parts and rows copied as they were + QETProject untouched(zipped); + const int unbuilt = untouched.unloadedFolioCount(); + const QString kept_path = output + "/kept.qetz"; + check(untouched.writeContainerFile(kept_path, &error), "saved as a .qetz without building"); + std::printf(".qetz saved untouched: %d of %d folios not built before, %d after\n", + unbuilt, int(folios.size()), untouched.unloadedFolioCount()); + check(untouched.unloadedFolioCount() > 0, "saving an untouched .qetz does not build every folio"); + qunsetenv("QET_LAZY_FOLIOS"); + QETProject kept_back(kept_path); + qputenv("QET_LAZY_FOLIOS", "1"); + check(kept_back.toXml().toString() == eager.toXml().toString(), + "a .qetz saved without building holds the project"); + + //After everyday edits on a folio + QETProject edited_z(zipped); + edit(edited_z, busiest); + const QString edited_path = output + "/edited.qetz"; + check(edited_z.writeContainerFile(edited_path, &error), "edited .qetz saved"); + std::printf(".qetz saved after the edits: %d folios not built\n", edited_z.unloadedFolioCount()); + check(edited_z.unloadedFolioCount() > 0, "saving an edited .qetz does not build every folio"); + //Against the same edits made with every folio built, saved and + //read back too: what a save and a reading do to a project is + //not what is checked here + const QString usual_path = output + "/edited-usual.qetz"; + qunsetenv("QET_LAZY_FOLIOS"); + { + QETProject edited_usual(path); + edit(edited_usual, busiest); + check(edited_usual.writeContainerFile(usual_path, &error), "edits saved with every folio built"); + } + QETProject edited_back(edited_path), usual_back(usual_path); + qputenv("QET_LAZY_FOLIOS", "1"); + check(sortedLines(edited_back.toXml().toString(), path) == sortedLines(usual_back.toXml().toString(), path), + "an edited .qetz saved without building holds what the edits made"); } std::printf("PASS: %d folios\n", int(folios.size())); } catch (const std::exception &error) {