From c259cae4295b5efebf2a7f18cf2ad44c442b0c1b Mon Sep 17 00:00:00 2001 From: ispyisail Date: Wed, 14 Oct 2026 07:35:34 +1300 Subject: [PATCH] Keep each fact once in a .qetz: drop the stored project database A .qetz held the engineering data twice: the document's tables, and beside them QElectroTech's own project database (stage D2/D3), which QElectroTech also builds from the document. Two copies of one fact can disagree, and the copy cost a third of the file. Now project.sqlite holds the document's tables only; QElectroTech builds its own project database from them when the project opens, as it does for a .qet. Measured (this build, lazy folios): industrial (50 folios): 620 -> 374 KB, open 0.37 -> 0.65 s f2099/nuri (136 folios): 3.27 -> 2.20 MB, open 2.2 -> 3.5 s The same projects as .qet open in 3.3 s and 23 s. Saving still copies the file of every folio not built, with its rows (stage D4): the project keeps the part each folio was read from. nuri opened and saved again: 6.3 s, no folio built, the same bytes. A symbol's terminal list is left out of the folio file when every folio, symbol and wire has a uuid of its own, now checked on the document (QetContainer::ownUuids()) rather than by the stored database being written. Checked: 342 corpus projects open, .qetz -> .qet equals the plain resave, a .qetz saved twice gives the same bytes; tst_lazyfolios, tst_qetcontainer, tst_resaveunchanged (which now checks project.sqlite holds only the document's tables). Co-Authored-By: Claude Opus 5.5 --- sources/container/qetcontainer.cpp | 91 +++---- sources/container/qetcontainer.h | 12 +- sources/container/qetcontainerdb.cpp | 115 +-------- sources/container/qetcontainerdb.h | 15 -- sources/dataBase/projectdatabase.cpp | 371 --------------------------- sources/dataBase/projectdatabase.h | 11 - sources/diagram.cpp | 21 +- sources/diagram.h | 16 +- sources/qetproject.cpp | 260 ++++--------------- sources/qetproject.h | 21 +- tests/qttest/CMakeLists.txt | 7 +- tests/qttest/lazy_folios_probe.cpp | 27 +- tests/qttest/tst_qetcontainer.cpp | 7 +- tests/qttest/tst_resaveunchanged.cpp | 33 +++ 14 files changed, 165 insertions(+), 842 deletions(-) diff --git a/sources/container/qetcontainer.cpp b/sources/container/qetcontainer.cpp index 4b8f18a14..15ffe995d 100644 --- a/sources/container/qetcontainer.cpp +++ b/sources/container/qetcontainer.cpp @@ -23,6 +23,7 @@ #include #include #include +#include const char QetContainer::MimeType[] = "application/x-qelectrotech-project+zip"; @@ -107,6 +108,7 @@ void removeElement(QDomElement e) } } + //The collection's symbol definitions, each to elements// void splitCollection(QDomDocument &document, const QDomElement &category, const QString &path, QList *parts, QSet *taken) @@ -184,6 +186,33 @@ QString QetContainer::pictureFile(const QDomElement &image) return image.attribute(ImageFileAttribute); } +/** + @brief QetContainer::ownUuids + @return true if every folio, symbol and wire of @p project has a uuid + no other has: a wire then finds its ends without the symbols' terminal + lists, which split() leaves out +*/ +bool QetContainer::ownUuids(const QDomElement &project) +{ + QSet seen; + auto own = [&seen](const QDomElement &item) { + const QString uuid = item.attribute(QStringLiteral("uuid")); + if (QUuid(uuid).isNull() || seen.contains(uuid)) return false; + seen.insert(uuid); + return true; + }; + for (const QDomElement &diagram : childElements(project, QStringLiteral("diagram"))) { + if (!own(diagram)) return false; + for (const QDomElement &list : childElements(diagram, QStringLiteral("elements"))) + for (const QDomElement &element : childElements(list, QStringLiteral("element"))) + if (!own(element)) return false; + for (const QDomElement &list : childElements(diagram, QStringLiteral("conductors"))) + for (const QDomElement &wire : childElements(list, QStringLiteral("conductor"))) + if (!own(wire)) return false; + } + return true; +} + /** @brief QetContainer::dropComputed Take out of the folio @p diagram what QElectroTech computes when it @@ -235,8 +264,7 @@ void QetContainer::dropComputed(QDomElement diagram, bool terminal_lists) set if @p project is not a project */ QList QetContainer::split(const QDomDocument &project, const QString &writer, - QString *error, const QByteArray &index, - const QHash &kept_from) + QString *error, const QHash &kept_from) { QDomDocument document = project.cloneNode(true).toDocument(); QDomElement root = document.documentElement(); @@ -250,7 +278,7 @@ QList QetContainer::split(const QDomDocument &project, const QStr //stage D4) keeps its part and, through //@p kept_from's project.sqlite, its rows const QByteArray database = QetContainerDb::extract( - root, error, index, kept_from.value(QStringLiteral("project.sqlite"))); + root, error, kept_from.value(QStringLiteral("project.sqlite"))); if (database.isEmpty()) return {}; @@ -259,9 +287,10 @@ QList QetContainer::split(const QDomDocument &project, const QStr //Folios, each with its pictures taken out int folio = 0; - //Nothing QElectroTech computes is kept (format 3) + //Nothing QElectroTech computes is kept (format 4) + const bool own_uuids = ownUuids(root); for (QDomElement diagram : childElements(root, QStringLiteral("diagram"))) - dropComputed(diagram, !index.isEmpty()); + dropComputed(diagram, own_uuids); //The parts of the folios kept as they were, and their pictures QMap pictures; //name -> png, once each @@ -354,46 +383,6 @@ QList QetContainer::split(const QDomDocument &project, const QStr return entries; } -/** - @brief QetContainer::readFolio - The folio part @p file of the .qetz whose entries are @p files, as - join() puts it in the project: its pictures back as base64 and its - engineering data back from project.sqlite. -*/ -bool QetContainer::readFolio(const QHash &files, const QString &file, - QDomDocument *folio, QString *error, - QetContainerDb::FolioReader *reader) -{ - if (!files.contains(file)) { - setError(error, QStringLiteral("part \"%1\" is missing").arg(file)); - return false; - } - QString part_error; - if (!parse(files.value(file), folio, &part_error)) { - setError(error, QStringLiteral("%1: %2").arg(file, part_error)); - return false; - } - const QDomNodeList images = folio->elementsByTagName(QStringLiteral("image")); - for (int i = 0 ; i < images.size() ; ++i) - { - QDomElement image = images.at(i).toElement(); - if (!image.hasAttribute(ImageFileAttribute)) continue; - const QString picture = image.attribute(ImageFileAttribute); - if (!files.contains(picture)) { - setError(error, QStringLiteral("picture \"%1\" is missing").arg(picture)); - return false; - } - image.removeAttribute(ImageFileAttribute); - image.insertBefore(folio->createTextNode(QString::fromLatin1(files.value(picture).toBase64())), - image.firstChild()); - } - if (!files.contains(QStringLiteral("project.sqlite"))) - return true; - return reader ? reader->restore(folio->documentElement(), error) - : QetContainerDb::restoreFolio(folio->documentElement(), - files.value(QStringLiteral("project.sqlite")), error); -} - /** @brief QetContainer::join Rebuild in @p project the document split() made @p entries from. @@ -401,7 +390,8 @@ bool QetContainer::readFolio(const QHash &files, const QStr (the manifest's min-reader), or a part missing or unreadable. Entries nothing names are ignored, so a later format may add some. With @p pictures, the pictures stay out of the document: see the - namespace. + namespace. With @p folio_files, the part each folio was read from, in + order. */ bool QetContainer::join(const QList &entries, QDomDocument *project, QString *error, QStringList *folio_files, Pictures *pictures) @@ -436,16 +426,9 @@ bool QetContainer::join(const QList &entries, QDomDocument *proje for (QDomElement part : placeholders) { const QString file = part.attribute(FileAttribute); - //With @p folio_files, the folios stay parts, read when built if (folio_files && part.parentNode() == project->documentElement() - && file.startsWith(QLatin1String("folios/"))) { - if (!files.contains(file)) { - setError(error, QStringLiteral("part \"%1\" is missing").arg(file)); - return false; - } + && file.startsWith(QLatin1String("folios/"))) folio_files->append(file); - continue; - } if (!files.contains(file)) { setError(error, QStringLiteral("part \"%1\" is missing").arg(file)); return false; diff --git a/sources/container/qetcontainer.h b/sources/container/qetcontainer.h index 33740306f..54d75a693 100644 --- a/sources/container/qetcontainer.h +++ b/sources/container/qetcontainer.h @@ -18,7 +18,6 @@ #ifndef QETCONTAINER_H #define QETCONTAINER_H -#include "qetcontainerdb.h" #include "qetzip.h" #include @@ -61,27 +60,24 @@ namespace QetContainer /// 1/0, its PRAGMA user_version the format. /// 4: what QElectroTech computes left out of the folios /// (dropComputed()), the tables written without keys or indexes. + /// project.sqlite holds the engineering data once: QElectroTech's + /// own project database is built from it when the project opens. /// Formats 2 and 3 are still read. QList split(const QDomDocument &project, const QString &writer = QString(), QString *error = nullptr, - const QByteArray &index = QByteArray(), const QHash &kept_from = QHash()); bool join(const QList &entries, QDomDocument *project, QString *error = nullptr, QStringList *folio_files = nullptr, Pictures *pictures = nullptr); - /// A folio part of @p files, which join() left out, with its - /// pictures and its engineering data back (stage D2) - /// @p reader, if any, reads the engineering data of @p files - bool readFolio(const QHash &files, const QString &file, - QDomDocument *folio, QString *error = nullptr, - QetContainerDb::FolioReader *reader = nullptr); /// The file join() left the picture of @p image in, or an empty string QString pictureFile(const QDomElement &image); /// Take what QElectroTech computes out of a folio (format 4) void dropComputed(QDomElement diagram, bool terminal_lists); + /// true if every folio, symbol and wire has a uuid of its own + bool ownUuids(const QDomElement &project); /// Parse @p xml as a .qet is read (QETProject::openFile) bool parse(const QByteArray &xml, QDomDocument *document, QString *error = nullptr); /// true if @p content is a zip, so a .qetz rather than a .qet diff --git a/sources/container/qetcontainerdb.cpp b/sources/container/qetcontainerdb.cpp index aea1b4768..059fe41cf 100644 --- a/sources/container/qetcontainerdb.cpp +++ b/sources/container/qetcontainerdb.cpp @@ -19,7 +19,6 @@ #include "qetcontainer.h" -#include #include #include #include @@ -134,9 +133,7 @@ QString schema() .arg(columns(FolioData), columns(ElementData), columns(ConductorData)); } - //The tables of a document's engineering data. While the project - //database's own tables were in the same file they were named - //doc_ (stage D3) + //The tables of a document's engineering data const QStringList DocTables = { QStringLiteral("strip_bridge_terminal"), QStringLiteral("strip_bridge"), QStringLiteral("strip_terminal"), QStringLiteral("terminal_strip"), @@ -144,26 +141,16 @@ const QStringList DocTables = { QStringLiteral("element_info"), QStringLiteral("element"), QStringLiteral("link"), QStringLiteral("conductor"), QStringLiteral("meta")}; - //Owns an sqlite3 handle; its SQL names the document's tables by their - //plain names, given their prefix here, and its columns as the format of - //the database read names them + //Owns an sqlite3 handle; names its columns as the format of the + //database read names them class Database { public: ~Database() { if (m_db) sqlite3_close(m_db); } sqlite3 *m_db = nullptr; - QString m_prefix; int m_format = QetContainer::Format; - QString sql(const QString &text) const { - if (m_prefix.isEmpty()) return text; - static const QRegularExpression names(QStringLiteral("(? - { - Statement doc(db, QStringLiteral("SELECT count(*) FROM sqlite_master WHERE name = 'doc_meta'")); - if (doc.next() && doc.value(0).toInt() > 0) db.m_prefix = QStringLiteral("doc_"); - } - for (const char *table : {"meta", "project_property", "folio", "folio_property", - "element", "element_info", "link", "conductor", "terminal_strip", - "strip_terminal", "strip_bridge", "strip_bridge_terminal"}) { + for (const QString &table : DocTables) { Statement kind(db, QStringLiteral("SELECT type FROM sqlite_master WHERE name = ?")); - kind.bind({db.table(QString::fromLatin1(table))}); + kind.bind({table}); if (!kind.next() || kind.value(0).toString() != QLatin1String("table")) { - setError(error, QStringLiteral("the project database has no table %1").arg(QLatin1String(table))); + setError(error, QStringLiteral("the project database has no table %1").arg(table)); return false; } } @@ -755,28 +734,10 @@ bool restoreDiagram(Database &db, QDomElement d, QString *error) folios still in it) into a new database. @return the database file, or an empty array with @p error set */ -QByteArray QetContainerDb::extract(QDomElement project, QString *error, const QByteArray &index, - const QByteArray &kept_from) +QByteArray QetContainerDb::extract(QDomElement project, QString *error, const QByteArray &kept_from) { Database db; - if (sqlite3_open(":memory:", &db.m_db) != SQLITE_OK) { - setError(error, QStringLiteral("cannot create the project database")); - return {}; - } - //The project database's own tables first, when given: one file - //holds both (stage D3), the document's named doc_ - if (!index.isEmpty()) { - db.m_prefix = QStringLiteral("doc_"); - unsigned char *copy = static_cast(sqlite3_malloc64(sqlite3_uint64(index.size()))); - if (!copy) { setError(error, QStringLiteral("out of memory")); return {}; } - memcpy(copy, index.constData(), size_t(index.size())); - if (sqlite3_deserialize(db.m_db, "main", copy, index.size(), index.size(), - SQLITE_DESERIALIZE_FREEONCLOSE | SQLITE_DESERIALIZE_RESIZEABLE) != SQLITE_OK) { - setError(error, QStringLiteral("cannot read the project database's index")); - return {}; - } - } - if (!db.exec(schema())) { + if (sqlite3_open(":memory:", &db.m_db) != SQLITE_OK || !db.exec(schema())) { setError(error, QStringLiteral("cannot create the project database")); return {}; } @@ -795,7 +756,7 @@ QByteArray QetContainerDb::extract(QDomElement project, QString *error, const QB db.exec(QStringLiteral("PRAGMA trusted_schema = OFF")); for (const QString &table : DocTables) { Statement kind(db, QStringLiteral("SELECT type FROM old.sqlite_master WHERE name = ?")); - kind.bind({db.table(table)}); + kind.bind({table}); if (!kind.next() || kind.value(0).toString() != QLatin1String("table")) { setError(error, QStringLiteral("the kept folios' database has no table %1").arg(table)); return {}; @@ -1105,58 +1066,6 @@ bool QetContainerDb::restore(QDomElement project, const QByteArray &database, QS return true; } -/** - @brief QetContainerDb::restoreFolio - restore() for the one folio @p diagram, read from its own part (a folio - built after its project was opened from a .qetz, stage D2) -*/ -bool QetContainerDb::restoreFolio(QDomElement diagram, const QByteArray &database, QString *error) -{ - Database db; - return openDatabase(db, database, error) && restoreDiagram(db, diagram, error); -} - -struct QetContainerDb::FolioReader::Private -{ - QByteArray database; - Database db; - bool opened = false; - QString open_error; -}; - -/** - @brief QetContainerDb::FolioReader::FolioReader - Reads folios' engineering data from @p database, opened the first time - one is read and kept open: a folio built after the project was opened - from a .qetz no longer copies the whole database (stage D5) -*/ -QetContainerDb::FolioReader::FolioReader(const QByteArray &database) : - d(std::make_unique()) -{ - d->database = database; -} - -QetContainerDb::FolioReader::~FolioReader() = default; - -/** - @brief QetContainerDb::FolioReader::restore - restoreFolio() for @p diagram -*/ -bool QetContainerDb::FolioReader::restore(QDomElement diagram, QString *error) -{ - if (!d->opened) { - d->opened = true; - if (!openDatabase(d->db, d->database, &d->open_error) && d->open_error.isEmpty()) - d->open_error = QStringLiteral("the project database cannot be read"); - d->database = QByteArray(); - } - if (!d->open_error.isEmpty()) { - setError(error, d->open_error); - return false; - } - return restoreDiagram(d->db, diagram, error); -} - /** @brief QetContainerDb::currentFormat @return true if @p database, a project.sqlite, is of the format this diff --git a/sources/container/qetcontainerdb.h b/sources/container/qetcontainerdb.h index 5e34f307d..4f4b31408 100644 --- a/sources/container/qetcontainerdb.h +++ b/sources/container/qetcontainerdb.h @@ -21,8 +21,6 @@ #include #include -#include - /** @brief The QetContainerDb namespace The engineering data of a project, moved between its document and the @@ -42,21 +40,8 @@ namespace QetContainerDb { QByteArray extract(QDomElement project, QString *error = nullptr, - const QByteArray &index = QByteArray(), const QByteArray &kept_from = QByteArray()); bool restore(QDomElement project, const QByteArray &database, QString *error = nullptr); - bool restoreFolio(QDomElement diagram, const QByteArray &database, QString *error = nullptr); - /// restoreFolio() for the folios of one database, opened once - class FolioReader - { - public: - explicit FolioReader(const QByteArray &database); - ~FolioReader(); - bool restore(QDomElement diagram, QString *error = nullptr); - private: - struct Private; - std::unique_ptr d; - }; /// true if @p database is of the format this writes bool currentFormat(const QByteArray &database); } diff --git a/sources/dataBase/projectdatabase.cpp b/sources/dataBase/projectdatabase.cpp index 6dd82ea82..b688865b9 100644 --- a/sources/dataBase/projectdatabase.cpp +++ b/sources/dataBase/projectdatabase.cpp @@ -49,10 +49,8 @@ #include #include #include -#include #include #include -#include #include #include #include @@ -896,375 +894,6 @@ bool projectDataBase::populateFromDocument(const QDomDocument &document, QString return true; } -/** - @brief projectDataBase::schemaStamp - @return a stamp of the tables of this database as created: an index - file (indexFile()) is read back only into a database with the same -*/ -QString projectDataBase::schemaStamp() const -{ - QSqlQuery query(m_data_base); - query.exec(QStringLiteral("SELECT sql FROM sqlite_master WHERE type IN ('table', 'index') " - "AND name NOT LIKE 'sqlite_%' ORDER BY name")); - QCryptographicHash hash(QCryptographicHash::Sha256); - while (query.next()) - hash.addData(query.value(0).toString().toUtf8() + ';'); - return QString::fromLatin1(hash.result().toHex()); -} - -namespace { - //The tables an index file carries, all of this database's but the - //drawing items, which exist for built folios only -QStringList indexTables(const QSqlDatabase &db) -{ - QStringList tables; - QSqlQuery query(db); - query.exec(QStringLiteral("SELECT name FROM sqlite_master WHERE type = 'table' " - "AND name NOT LIKE 'sqlite_%' ORDER BY name")); - while (query.next()) { - const QString name = query.value(0).toString(); - if (name != QLatin1String("shape") && name != QLatin1String("independent_text") - && name != QLatin1String("image")) - tables << name; - } - return tables; -} -} - -namespace { - //A wire's properties read from @p attributes, as a saved wire's are -ConductorProperties propertiesFrom(const QHash &attributes) -{ - QDomDocument scratch; - QDomElement e = scratch.createElement(QStringLiteral("conductor")); - for (auto it = attributes.cbegin() ; it != attributes.cend() ; ++it) - e.setAttribute(it.key(), it.value()); - ConductorProperties properties; - properties.fromXml(e); - return properties; -} - - //@p attributes without those reading them would give anyway: each - //left out in turn when the properties read without it are the same - //(stage D3b) -QHash fewestAttributes(QHash attributes) -{ - const ConductorProperties full = propertiesFrom(attributes); - QStringList names = attributes.keys(); - names.sort(); - for (const QString &name : std::as_const(names)) { - QHash without = attributes; - without.remove(name); - const ConductorProperties read = propertiesFrom(without); - if (read == full && read.attributes() == full.attributes()) - attributes = without; - } - return attributes; -} -} - -/** - @brief projectDataBase::indexFile - This database written to an SQLite file, for a .qetz to be opened from - (stage D2): every table but the drawing items, brought - up to date first, with the folios' own data @p folio_headers (in folio - order, each as Diagram::headerXml() writes it), the drawn tables - @p tables and how they continue one another (which folios a folio's - tables must be built with), and the stamp of the schema it was written - with. - @return the file, or nothing if it could not be written -*/ -QByteArray projectDataBase::indexFile(const QList> &folio_headers, - const QList &tables) -{ - if (m_content_changed) { - updateDB(); - } - flushLinks(); - flushElementPositions(); - flushConductorProperties(); - - QTemporaryDir dir; - if (!dir.isValid()) { - return QByteArray(); - } - const QString path = dir.filePath(QStringLiteral("index.sqlite")); - QString escaped = path; - escaped.replace(QLatin1Char('\''), QStringLiteral("''")); - QSqlQuery vacuum(m_data_base); - if (!vacuum.exec(QStringLiteral("VACUUM INTO '%1'").arg(escaped))) { - qWarning() << "projectDataBase::indexFile:" << vacuum.lastError().text(); - return QByteArray(); - } - const QString stamp = schemaStamp(); - const QString connection = QStringLiteral("qet_index_") + QUuid::createUuid().toString(); - bool ok = false; - { - QSqlDatabase file = QSqlDatabase::addDatabase(QStringLiteral("QSQLITE"), connection); - file.setDatabaseName(path); - if (file.open()) - { - QSqlQuery query(file); - //Its views read the drawing items too: an index holds tables - QStringList views; - query.exec(QStringLiteral("SELECT name FROM sqlite_master WHERE type = 'view'")); - while (query.next()) views << query.value(0).toString(); - for (const QString &view : std::as_const(views)) - query.exec(QStringLiteral("DROP VIEW \"%1\"").arg(view)); - ok = query.exec(QStringLiteral("DROP TABLE shape")) - && query.exec(QStringLiteral("DROP TABLE independent_text")) - && query.exec(QStringLiteral("DROP TABLE image")) - && query.exec(QStringLiteral("CREATE TABLE index_meta (key TEXT PRIMARY KEY, value TEXT)")) - && query.exec(QStringLiteral("CREATE TABLE folio_header (pos INTEGER PRIMARY KEY, " - "uuid TEXT NOT NULL, xml TEXT NOT NULL)")) - && query.exec(QStringLiteral("CREATE TABLE folio_table (uuid TEXT NOT NULL, " - "folio_uuid TEXT NOT NULL, previous_uuid TEXT)")); - file.transaction(); - query.prepare(QStringLiteral("INSERT INTO index_meta (key, value) VALUES (?, ?)")); - for (const auto &meta : {std::pair{QStringLiteral("schema"), stamp}, - std::pair{QStringLiteral("writer"), - QStringLiteral("QElectroTech ") + QetVersion::currentVersion().toString()}}) { - query.addBindValue(meta.first); - query.addBindValue(meta.second); - ok = ok && query.exec(); - } - query.prepare(QStringLiteral("INSERT INTO folio_header (pos, uuid, xml) VALUES (?, ?, ?)")); - int pos = 0; - for (const auto &header : folio_headers) { - query.addBindValue(pos++); - query.addBindValue(header.first.toString()); - query.addBindValue(header.second); - ok = ok && query.exec(); - } - query.prepare(QStringLiteral("INSERT INTO folio_table (uuid, folio_uuid, previous_uuid) " - "VALUES (?, ?, ?)")); - for (const FolioTable &table : tables) { - query.addBindValue(table.table.toString()); - query.addBindValue(table.folio.toString()); - query.addBindValue(table.previous.isNull() ? QVariant() : QVariant(table.previous.toString())); - ok = ok && query.exec(); - } - //A wire's properties: only the attributes reading them needs, - //by wire and name (fillFromIndex() writes the others back) - { - QMap> wires; - QSqlQuery rows(file); - rows.exec(QStringLiteral("SELECT conductor_uuid, name, value FROM conductor_properties")); - while (rows.next()) - wires[rows.value(0).toString()].insert(rows.value(1).toString(), rows.value(2).toString()); - ok = ok && query.exec(QStringLiteral("DELETE FROM conductor_properties")); - query.prepare(QStringLiteral("INSERT INTO conductor_properties (conductor_uuid, name, value) " - "VALUES (?, ?, ?)")); - for (auto wire = wires.cbegin() ; wire != wires.cend() ; ++wire) { - const QHash fewest = fewestAttributes(wire.value()); - QStringList names = fewest.keys(); - names.sort(); - for (const QString &name : std::as_const(names)) { - query.addBindValue(wire.key()); - query.addBindValue(name); - query.addBindValue(fewest.value(name)); - ok = ok && query.exec(); - } - } - } - ok = file.commit() && ok; - ok = query.exec(QStringLiteral("VACUUM")) && ok; - file.close(); - } - } - QSqlDatabase::removeDatabase(connection); - if (!ok) { - return QByteArray(); - } - QFile file(path); - if (!file.open(QIODevice::ReadOnly)) { - return QByteArray(); - } - return file.readAll(); -} - -/** - @brief projectDataBase::discardFill - Undo fillFromIndex(): the project is read from its document instead -*/ -void projectDataBase::discardFill() -{ - QSqlQuery query(m_data_base); - for (const QString &table : indexTables(m_data_base)) - query.exec(QStringLiteral("DELETE FROM \"%1\"").arg(table)); - m_element_information.clear(); - m_folio_titleblocks.clear(); - m_conductor_properties.clear(); - m_conductor_attributes.clear(); - m_prefilled_elements.clear(); - m_prefilled_folios.clear(); - m_prefilled_conductors.clear(); - m_rows_from_document = false; - m_content_changed = true; -} - -/** - @brief projectDataBase::fillFromIndex - Fill this database from @p index, a file indexFile() wrote, instead of - from the project's document: its tables, and the stores of symbol - information and wire properties (the folios' title blocks are filled - from @p folio_headers by the folios). The file is untrusted: read only - when written with this database's schema, its tables plain tables, and - nothing in its schema run. - @param folio_headers : the folios' own data, in folio order - @param folio_tables : if not null, the drawn tables indexFile() wrote, or - nothing if the index predates them - @return false, with @p why set and nothing changed in the tables, if - the file cannot be read -*/ -bool projectDataBase::fillFromIndex(const QByteArray &index, - QList> *folio_headers, - QString *why, - std::optional> *folio_tables) -{ - auto refuse = [why](const QString &reason) { - if (why) *why = reason; - return false; - }; - QTemporaryDir dir; - if (!dir.isValid()) { - return refuse(QStringLiteral("no temporary folder")); - } - const QString path = dir.filePath(QStringLiteral("index.sqlite")); - { - QFile file(path); - if (!file.open(QIODevice::WriteOnly) || file.write(index) != index.size()) { - return refuse(QStringLiteral("cannot write the index to a temporary file")); - } - } - QString escaped = path; - escaped.replace(QLatin1Char('\''), QStringLiteral("''")); - QSqlQuery query(m_data_base); - query.exec(QStringLiteral("PRAGMA trusted_schema = OFF")); - if (!query.exec(QStringLiteral("ATTACH DATABASE '%1' AS idx").arg(escaped))) { - return refuse(QStringLiteral("the index cannot be read")); - } - auto detach = [this]() { - QSqlQuery q(m_data_base); - q.exec(QStringLiteral("DETACH DATABASE idx")); - }; - - const QStringList tables = indexTables(m_data_base); - auto isTable = [this](const QString &name) { - QSqlQuery kind(m_data_base); - kind.prepare(QStringLiteral("SELECT type FROM idx.sqlite_master WHERE name = ?")); - kind.addBindValue(name); - return kind.exec() && kind.next() && kind.value(0).toString() == QLatin1String("table"); - }; - for (const QString &table : tables + QStringList{QStringLiteral("index_meta"), - QStringLiteral("folio_header")}) { - if (!isTable(table)) { - detach(); - return refuse(QStringLiteral("the index has no table %1").arg(table)); - } - } - query.exec(QStringLiteral("SELECT value FROM idx.index_meta WHERE key = 'schema'")); - if (!query.next() || query.value(0).toString() != schemaStamp()) { - detach(); - return refuse(QStringLiteral("the index was written with another database schema")); - } - - m_data_base.transaction(); - bool ok = true; - for (const QString &table : tables) { - QSqlQuery columns(m_data_base); - columns.exec(QStringLiteral("PRAGMA main.table_info(\"%1\")").arg(table)); - QStringList names; - while (columns.next()) - names << QStringLiteral("\"%1\"").arg(columns.value(1).toString()); - const QString list = names.join(QLatin1Char(',')); - ok = ok && query.exec(QStringLiteral("DELETE FROM main.\"%1\"").arg(table)) - && query.exec(QStringLiteral("INSERT INTO main.\"%1\" (%2) SELECT %2 FROM idx.\"%1\"") - .arg(table, list)); - } - if (!ok) { - m_data_base.rollback(); - detach(); - return refuse(QStringLiteral("the index's rows cannot be read")); - } - m_data_base.commit(); - - folio_headers->clear(); - query.exec(QStringLiteral("SELECT uuid, xml FROM idx.folio_header ORDER BY pos")); - while (query.next()) - folio_headers->append({QUuid(query.value(0).toString()), query.value(1).toString()}); - if (folio_tables) { - folio_tables->reset(); - if (isTable(QStringLiteral("folio_table"))) { - QList read; - query.exec(QStringLiteral("SELECT uuid, folio_uuid, previous_uuid FROM idx.folio_table")); - while (query.next()) - read.append({QUuid(query.value(0).toString()), QUuid(query.value(1).toString()), - QUuid(query.value(2).toString())}); - *folio_tables = read; - } - } - detach(); - - //The stores, from their tables, as the document would fill them - m_prefilled_elements.clear(); - m_prefilled_folios.clear(); - m_prefilled_conductors.clear(); - for (PrefillTally &tally : m_prefill_tally) tally = PrefillTally(); - query.exec(QStringLiteral("SELECT element_uuid, name, value, show FROM element_information " - "ORDER BY element_uuid, ord")); - while (query.next()) { - const QUuid uuid(query.value(0).toString()); - m_prefilled_elements[uuid].addValue(query.value(1).toString(), query.value(2).toString(), - query.value(3).toInt() != 0); - } - QHash> attributes; - QList wire_order; - query.exec(QStringLiteral("SELECT conductor_uuid, name, value FROM conductor_properties ORDER BY rowid")); - while (query.next()) { - const QUuid wire(query.value(0).toString()); - if (!attributes.contains(wire)) wire_order << wire; - attributes[wire].insert(query.value(1).toString(), query.value(2).toString()); - } - //A file may hold only the attributes reading needs (indexFile()): - //the table gets every attribute a saved wire carries, as before - query.exec(QStringLiteral("DELETE FROM conductor_properties")); - query.prepare(QStringLiteral("INSERT INTO conductor_properties (conductor_uuid, name, value) VALUES (?, ?, ?)")); - for (const QUuid &wire : std::as_const(wire_order)) { - const ConductorProperties properties = propertiesFrom(attributes.value(wire)); - m_prefilled_conductors.insert(wire, properties); - QHash &full = attributes[wire]; - full.clear(); - for (const auto &attribute : properties.attributes()) { - full.insert(attribute.first, attribute.second); - query.addBindValue(wire.toString()); - query.addBindValue(attribute.first); - query.addBindValue(attribute.second); - query.exec(); - } - } - for (auto it = m_prefilled_elements.cbegin() ; it != m_prefilled_elements.cend() ; ++it) - m_element_information.insert(it.key(), it.value()); - for (auto it = m_prefilled_conductors.cbegin() ; it != m_prefilled_conductors.cend() ; ++it) - m_conductor_properties.insert(it.key(), it.value()); - m_conductor_attributes = attributes; - for (const auto &header : std::as_const(*folio_headers)) { - QDomDocument document; - if (!document.setContent(header.second)) continue; - TitleBlockProperties properties; - properties.fromXml(document.documentElement()); - properties.version = QetVersion::displayedVersion(); - properties.collection = QET::QetCollection::Embedded; - m_prefilled_folios.insert(header.first, properties); - m_folio_titleblocks.insert(header.first, properties); - } - - m_rows_from_document = true; - m_content_changed = false; - emit dataBaseUpdated(); - return true; -} - /** @brief projectDataBase::foliosWithUnjoinedWires @return the uuids of the folios whose document has a wire that building diff --git a/sources/dataBase/projectdatabase.h b/sources/dataBase/projectdatabase.h index ceaeec512..6b5fcc38c 100644 --- a/sources/dataBase/projectdatabase.h +++ b/sources/dataBase/projectdatabase.h @@ -166,17 +166,6 @@ class projectDataBase : public QObject void setBuildingFolio(bool building); bool isBuildingFolio() const {return m_building_folios > 0;} bool rowsFromDocument() const {return m_rows_from_document;} - QString schemaStamp() const; - /// A drawn table of a folio and the table it continues - struct FolioTable { QUuid table; QUuid folio; QUuid previous; }; - QByteArray indexFile(const QList> &folio_headers, - const QList &tables = QList()); - void discardFill(); - /// The tables are right as they are (filled from an index) - void markCurrent() {m_content_changed = false;} - bool fillFromIndex(const QByteArray &index, QList> *folio_headers, - QString *why = nullptr, - std::optional> *folio_tables = nullptr); QString prefillReport() const; QSet foliosWithUnjoinedWires() const; QSet folioUuidsOfElements(const QSet &elements) const; diff --git a/sources/diagram.cpp b/sources/diagram.cpp index 483f2dd62..1eb549fd8 100644 --- a/sources/diagram.cpp +++ b/sources/diagram.cpp @@ -882,20 +882,6 @@ QDomElement Diagram::deferredXmlWithHeader(QDomDocument &document) return folio; } -/** - @brief Diagram::deferredXml - @return the XML of this folio's items while they are not built: kept - from the project's document, or read now from the folio's part of the - .qetz the project was opened from -*/ -QDomElement Diagram::deferredXml() -{ - if (m_deferred.isNull() && !m_deferred_part.isEmpty() && m_project) { - m_deferred = m_project->readFolioPart(m_deferred_part); - } - return m_deferred; -} - /** @brief Diagram::buildDeferred Build this folio's items from the XML kept for later, and nothing else. @@ -908,12 +894,9 @@ bool Diagram::buildDeferred() if (isLoaded()) { return false; } - QDomElement xml = deferredXml(); + QDomElement xml = m_deferred; m_deferred = QDomElement(); - m_deferred_part.clear(); - if (xml.isNull()) { - return true; //its part could not be read: QETProject::readFolioPart() said why - } + m_kept_part.clear(); m_kept_text.clear(); projectDataBase *db = m_project ? m_project->dataBase() : nullptr; if (db) db->setBuildingFolio(true); diff --git a/sources/diagram.h b/sources/diagram.h index 5785b7efd..233ba48d9 100644 --- a/sources/diagram.h +++ b/sources/diagram.h @@ -142,7 +142,7 @@ class Diagram : public QGraphicsScene QUuid m_uuid = QUuid::createUuid(); /// The XML of this folio's items, while they are not built yet QDomElement m_deferred; - QString m_deferred_part; + QString m_kept_part; QHash m_partial_elements; QHash m_partial_table; bool m_partial_changed = false; @@ -218,13 +218,13 @@ class Diagram : public QGraphicsScene DiagramContent * = nullptr); /// false while this folio's items wait in its XML to be /// built (lazy folios) - bool isLoaded() const {return m_deferred.isNull() && m_deferred_part.isEmpty();} - QDomElement deferredXml(); - /// The folio's items are in the part @p file of the .qetz the - /// project was opened from, read when they are first needed - /// (stage D2) - void setDeferredPart(const QString &file) {m_deferred = QDomElement(); m_deferred_part = file;} - QString deferredPart() const {return m_deferred_part;} + bool isLoaded() const {return m_deferred.isNull();} + QDomElement deferredXml() const {return m_deferred;} + /// The part @p file of the .qetz the project was opened from + /// holds this folio as it is while not built: saving may copy it + /// (QETProject::toXml(), stage D4) + void setKeptPart(const QString &file) {m_kept_part = file;} + QString keptPart() const {return m_kept_part;} QList buildSymbols(const QSet &uuids); /// Symbols built as partners (buildSymbols()) changed since: /// the folio must be built to be saved diff --git a/sources/qetproject.cpp b/sources/qetproject.cpp index e5600445c..afd94ab65 100644 --- a/sources/qetproject.cpp +++ b/sources/qetproject.cpp @@ -479,48 +479,27 @@ QETProject::ProjectState QETProject::openFile(QFile *file) // QetContainer::parse() does exactly that, for a .qet and for each part // of a .qetz alike. QString read_error; - //A .qetz holding the index of its database opens from it, the - //folios' parts read when they are built (stage D2); - //any other, or one whose index cannot be used, is read whole - m_opened_from_index = false; - if (QetContainer::isZip(content) && lazyFolios()) - { + //A .qetz: its entries are kept, for a save to copy the parts of the + //folios not built (stage D4) + m_container_files.clear(); + m_folio_parts.clear(); + bool read = false; + if (QetContainer::isZip(content)) { QList entries; - QString why; - if (QetZip::fromBytes(content, &entries, &why)) - { - QHash files; + read = QetZip::fromBytes(content, &entries, &read_error) + && QetContainer::join(entries, &xml_project, &read_error, &m_folio_parts, + &m_container_pictures); + if (read) for (const QetZip::Entry &entry : std::as_const(entries)) - files.insert(entry.name, entry.data); - QStringList folio_files; - QDomDocument document; - //The index: in project.sqlite from format 3, its own entry - //before - const QByteArray index = files.contains(QStringLiteral("index.sqlite")) - ? files.value(QStringLiteral("index.sqlite")) - : files.value(QStringLiteral("project.sqlite")); - if (!index.isEmpty() && index.contains("folio_header") - && QetContainer::join(entries, &document, &why, &folio_files)) - { - if (m_data_base.fillFromIndex(index, &m_folio_headers, &why, &m_index_tables) - && m_folio_headers.size() == folio_files.size()) { - xml_project = document; - m_container_files = files; - m_folio_files = folio_files; - m_opened_from_index = true; - } else { - m_data_base.discardFill(); - m_folio_headers.clear(); - m_index_tables.reset(); - qInfo().noquote() << "Not opened from the index of" << fi.fileName() << ":" << why; - } - } - } + m_container_files.insert(entry.name, entry.data); + } else { + read = QetContainer::readDocument(content, &xml_project, &read_error); } - if (!m_opened_from_index && !QetContainer::readDocument(content, &xml_project, &read_error, &m_container_pictures)) + if (!read) { qWarning().noquote() << "Unable to read" << fi.fileName() << ":" << read_error; m_container_pictures.clear(); + m_folio_parts.clear(); if(opened_here) { file->close(); } @@ -627,9 +606,8 @@ QList QETProject::folios() const folio when the project has cables (a core is followed through folio reports to any folio); without cables, the folios built and those not built yet holding a wire that still refers to a cable -- the only - wires the pass changes then, taking the reference away. Read from the - project database for a .qetz opened from it, from the folios' XML - otherwise. + wires the pass changes then, taking the reference away, read from the + folios' XML. */ QList QETProject::cableLabelFolios() { @@ -637,25 +615,14 @@ QList QETProject::cableLabelFolios() return diagrams(); } QList referring; - if (m_opened_from_index) { - QSet folios; - QSqlQuery query = m_data_base.newQuery(QStringLiteral( - "SELECT DISTINCT c.diagram_uuid FROM conductor c JOIN conductor_properties p " - "ON p.conductor_uuid = c.uuid WHERE p.name = 'cable_uuid' AND p.value != ''")); - if (query.exec()) - while (query.next()) folios.insert(QUuid(query.value(0).toString())); - for (Diagram *d : std::as_const(m_diagrams_list)) - if (!d->isLoaded() && folios.contains(d->uuid())) referring << d; - } else { - for (Diagram *d : std::as_const(m_diagrams_list)) { - if (d->isLoaded()) continue; - for (const QDomElement &wire : QET::findInDomElement(d->deferredXml(), QStringLiteral("conductors"), - QStringLiteral("conductor"))) - if (!QUuid(wire.attribute(QStringLiteral("cable_uuid"))).isNull()) { - referring << d; - break; - } - } + for (Diagram *d : std::as_const(m_diagrams_list)) { + if (d->isLoaded()) continue; + for (const QDomElement &wire : QET::findInDomElement(d->deferredXml(), QStringLiteral("conductors"), + QStringLiteral("conductor"))) + if (!QUuid(wire.attribute(QStringLiteral("cable_uuid"))).isNull()) { + referring << d; + break; + } } buildFolios(referring); return builtFolios(); @@ -875,9 +842,8 @@ void QETProject::buildFoliosHoldingSymbols(const QSet &symbols) return; } QList to_build; - //Opened from an index, the database knows where every symbol is - //without reading the folios' parts - if (m_opened_from_index || m_data_base.rowsFromDocument()) { + //Filled from the document, the database knows where every symbol is + if (m_data_base.rowsFromDocument()) { const QSet folios = m_data_base.folioUuidsOfElements(symbols); for (Diagram *d : std::as_const(m_diagrams_list)) if (!d->isLoaded() && folios.contains(d->uuid())) @@ -951,16 +917,6 @@ QSet QETProject::tableChainFolios(Diagram *folio) const } } continue; - } - //Opened from an index: its tables, not the folio's part - if (m_index_tables) { - for (const projectDataBase::FolioTable &table : std::as_const(*m_index_tables)) { - if (table.folio == d->uuid()) { - folio_of.insert(table.table, d); - previous_of.insert(table.table, table.previous); - } - } - continue; } const QDomNodeList tables = d->deferredXml().elementsByTagName( QetGraphicsTableItem::xmlTagName()); @@ -2365,7 +2321,7 @@ void QETProject::autoFolioNumberingSelectedFolios(int from, */ QDomDocument QETProject::toXml(QHash *kept_folios, QStringList *kept_parts) { - //With @p kept_parts, the same for a .qetz opened from its index: a + //With @p kept_parts, for a .qetz: a //folio not built is written as , its //part and its rows copied by QetContainer::split() //(stage D4) @@ -2556,7 +2512,7 @@ QDomDocument QETProject::toXml(QHash *kept_folios, QStringList } if (keep && !diagram->isLoaded() && kept_parts) { auto placeholder = xml_doc.createElement(QStringLiteral("diagram")); - placeholder.setAttribute(QStringLiteral("qetz-kept-part"), diagram->deferredPart()); + placeholder.setAttribute(QStringLiteral("qetz-kept-part"), diagram->keptPart()); placeholder.setAttribute(QStringLiteral("uuid"), diagram->uuid().toString()); //Symbols whose information changed while the folio was //not built: the rows QetContainer::split() writes for them @@ -2579,7 +2535,7 @@ QDomDocument QETProject::toXml(QHash *kept_folios, QStringList wire.setAttribute(attribute.first, attribute.second); placeholder.appendChild(wire); } - kept_parts->append(diagram->deferredPart()); + kept_parts->append(diagram->keptPart()); project_root.appendChild(placeholder); ++order_num; continue; @@ -2682,24 +2638,18 @@ bool QETProject::isContainerPath(const QString &path) the whole archive is written. */ bool QETProject::writeContainer(const QDomDocument &project, const QString &path, QString *error, - const QByteArray &index, const QHash &kept_from) + const QHash &kept_from) { - //The project database, for opening without reading the folios - //(stage D2), is in project.sqlite with the - //document's engineering data (format 3, D3) QList entries = QetContainer::split( project, QStringLiteral("QElectroTech %1").arg(QetVersion::currentVersion().toString()), - error, index, kept_from); + error, kept_from); return !entries.isEmpty() && QetZip::write(path, entries, error); } /** @brief QETProject::writeContainerFile - Write this project to @p path as a .qetz, with the index of its - database a .qetz is opened from (stage D2) when every - folio, symbol and wire has a uuid of its own -- what opening from the - index relies on, as opening without building folios does. + Write this project to @p path as a .qetz */ bool QETProject::writeContainerFile(const QString &path, QString *error) { @@ -2707,67 +2657,10 @@ bool QETProject::writeContainerFile(const QString &path, QString *error) //what building them would write (toXml()); every other folio built QStringList kept; const QDomDocument document = toXml(nullptr, &kept); - return writeContainer(document, path, error, containerIndex(), + return writeContainer(document, path, error, kept.isEmpty() ? QHash() : m_container_files); } -/** - @brief QETProject::containerIndex - @return the index of this project's database for a .qetz, empty when a - folio, symbol or wire has no uuid of its own -*/ -QByteArray QETProject::containerIndex() -{ - //Opened from an index, the folios not built have their rows in the - //database, whose keys are unique; the built ones are looked at - if (!m_kept_parts_trusted) { - loadFolios(); - } - QSet seen; - QList> headers; - auto unique = [&seen](const QUuid &uuid) { - if (uuid.isNull() || seen.contains(uuid)) return false; - seen.insert(uuid); - return true; - }; - //The drawn tables and the tables they continue: which folios a - //folio's tables are built with (tableChainFolios()) - QList tables; - for (Diagram *diagram : std::as_const(m_diagrams_list)) { - if (!unique(diagram->uuid())) return QByteArray(); - if (!diagram->isLoaded()) { - headers.append({diagram->uuid(), diagram->headerXml()}); - if (m_index_tables) { - for (const projectDataBase::FolioTable &table : std::as_const(*m_index_tables)) - if (table.folio == diagram->uuid()) - tables.append(table); - continue; - } - const QDomNodeList items = diagram->deferredXml().elementsByTagName( - QetGraphicsTableItem::xmlTagName()); - for (int i = 0 ; i < items.size() ; ++i) { - const QDomElement table = items.at(i).toElement(); - tables.append({QUuid(table.attribute(QStringLiteral("uuid"))), diagram->uuid(), - QUuid(table.firstChildElement(QStringLiteral("previous_table")) - .attribute(QStringLiteral("uuid")))}); - } - continue; - } - for (Element *element : diagram->elements()) - if (!unique(element->uuid())) return QByteArray(); - for (Conductor *conductor : diagram->conductors()) - if (!unique(conductor->uuid())) return QByteArray(); - headers.append({diagram->uuid(), diagram->headerXml()}); - for (QGraphicsItem *item : diagram->items()) { - if (item->type() == QetGraphicsTableItem::Type) { - auto table = static_cast(item); - tables.append({table->uuid(), diagram->uuid(), table->previousTableUuid()}); - } - } - } - return m_data_base.indexFile(headers, tables); -} - /** @brief QETProject::besideContainerPath @return the .qetz a project read from a .qet is saved to: the same @@ -3093,8 +2986,7 @@ QDomDocument QETProject::folioXml(Diagram *folio) /** @brief QETProject::buildFoliosHoldingTables Build the folios not built yet (lazy folios) that hold a drawn - table, found from the index the project was opened from when it has - one: what a list of the project's tables needs, and no more + table: what a list of the project's tables needs, and no more */ void QETProject::buildFoliosHoldingTables() { @@ -3114,15 +3006,10 @@ void QETProject::buildFoliosHoldingTables() /** @brief QETProject::unbuiltFolioHasTables @return true if @p folio, not built yet (lazy folios), has a drawn - table: from the index it was opened from, else from its XML + table, read from its XML */ bool QETProject::unbuiltFolioHasTables(Diagram *folio) const { - if (m_index_tables) { - for (const projectDataBase::FolioTable &table : std::as_const(*m_index_tables)) - if (table.folio == folio->uuid()) return true; - return false; - } return !folio->deferredXml().elementsByTagName(QetGraphicsTableItem::xmlTagName()).isEmpty(); } @@ -3408,9 +3295,7 @@ void QETProject::readProjectXml(QDomDocument &xml_project) //The stores of symbol information, folio title blocks and wire //properties are filled from the document before the folios are //built - if (!m_opened_from_index) { - m_data_base.prefillFromDocument(xml_project); - } + m_data_base.prefillFromDocument(xml_project); //Folios are left to be built when needed (lazyFolios()) only //when the database can be filled from the document: otherwise @@ -3426,11 +3311,7 @@ void QETProject::readProjectXml(QDomDocument &xml_project) } //Load the diagrams - if (m_opened_from_index) { - readFoliosFromIndex(); - } else { - readDiagramsXml(xml_project); - } + readDiagramsXml(xml_project); const qint64 diagrams_ms = phase_timer.restart(); //Tie the elements to the numbering schemes they follow @@ -3459,14 +3340,7 @@ void QETProject::readProjectXml(QDomDocument &xml_project) m_data_base.blockSignals(false); m_data_base.setUpdateBlocked(false); - if (m_opened_from_index) { - //Filled from the index already: making the folios changed - //nothing their rows say - m_data_base.markCurrent(); - m_data_base.updateDB(); - } else { - m_data_base.updateDB(xml_project); - } + m_data_base.updateDB(xml_project); m_data_base.endPrefill(unloadedFolioCount() > 0); //Saving may write the folios not built as they were read, when //building them would write the same: a file written by this @@ -3482,10 +3356,13 @@ void QETProject::readProjectXml(QDomDocument &xml_project) for (Diagram *diagram : std::as_const(m_diagrams_list)) if (!diagram->isLoaded() && diagram->keptText().isEmpty()) m_kept_folios_trusted = false; - m_kept_parts_trusted = m_opened_from_index && unloadedFolioCount() > 0 + m_kept_parts_trusted = !m_folio_parts.isEmpty() && unloadedFolioCount() > 0 && m_project_qet_version == QetVersion::currentVersion() && m_data_base.rowsFromDocument() && QetContainerDb::currentFormat(m_container_files.value(QStringLiteral("project.sqlite"))); + for (Diagram *diagram : std::as_const(m_diagrams_list)) + if (!diagram->isLoaded() && diagram->keptPart().isEmpty()) + m_kept_parts_trusted = false; m_kept_folios_stale = false; m_edited_since_open = false; if (m_kept_folios_trusted || m_kept_parts_trusted) @@ -3511,34 +3388,6 @@ void QETProject::readProjectXml(QDomDocument &xml_project) m_state = Ok; } -/** - @brief QETProject::readFoliosFromIndex - readDiagramsXml() for a .qetz opened from its index: each folio made - from its own data as the index has it, its items left in its part - (stage D2) -*/ -void QETProject::readFoliosFromIndex() -{ - for (int i = 0 ; i < m_folio_headers.size() ; ++i) - { - QDomDocument header; - header.setContent(m_folio_headers.at(i).second); - auto diagram = new Diagram(this); - m_diagrams_list << diagram; - connect(&diagram->border_and_titleblock, &BorderTitleBlock::needFolioData, - this, &QETProject::updateDiagramsFolioData); - connect(diagram, &Diagram::usedTitleBlockTemplateChanged, - this, &QETProject::usedTitleBlockTemplateChanged); - m_deferring_folios = true; - QDomElement header_element = header.documentElement(); - diagram->initFromXml(header_element); - m_deferring_folios = false; - diagram->setDeferredPart(m_folio_files.at(i)); - m_data_base.placeFolio(diagram); - } - updateDiagramsFolioData(); -} - /** @brief QETProject::lazyFolios @see the declaration @@ -3548,27 +3397,6 @@ bool QETProject::lazyFolios() return qEnvironmentVariableIntValue("QET_EAGER_FOLIOS") != 1; } -/** - @brief QETProject::readFolioPart - @return the folio part @p file of the .qetz this project was opened - from, read now, its engineering data back from project.sqlite; null - if it cannot be read (said in the log) -*/ -QDomElement QETProject::readFolioPart(const QString &file) -{ - QDomDocument folio; - QString error; - if (!m_folio_reader) - m_folio_reader = std::make_unique( - m_container_files.value(QStringLiteral("project.sqlite"))); - ++m_folio_parts_read; - if (!QetContainer::readFolio(m_container_files, file, &folio, &error, m_folio_reader.get())) { - qWarning().noquote() << "Unable to read the folio" << file << ":" << error; - return QDomElement(); - } - return folio.documentElement(); -} - /** @brief QETProject::readDiagramsXml Load the diagrams from the xml description of the project. @@ -3665,6 +3493,8 @@ void QETProject::readDiagramsXml(QDomDocument &xml_project) m_deferring_folios = false; if (!diagram->isLoaded() && !line_starts.isEmpty()) diagram->setKeptText(folio_text(diagram_xml_element)); + if (!diagram->isLoaded() && m_folio_parts.size() == diagram_nodes.length()) + diagram->setKeptPart(m_folio_parts.at(i)); m_data_base.placeFolio(diagram); if(dlgWaiting) dlgWaiting->setDetail(diagram->title()); diff --git a/sources/qetproject.h b/sources/qetproject.h index a76a5d60f..b98e9cf0a 100644 --- a/sources/qetproject.h +++ b/sources/qetproject.h @@ -460,14 +460,9 @@ class QETProject : public QObject QString besideContainerPath() const; QETResult writeCopyForOlderVersions(const QString &path); static bool writeContainer(const QDomDocument &project, const QString &path, - QString *error = nullptr, const QByteArray &index = QByteArray(), + QString *error = nullptr, const QHash &kept_from = QHash()); bool writeContainerFile(const QString &path, QString *error = nullptr); - QByteArray containerIndex(); - QDomElement readFolioPart(const QString &file); - /// How many folio parts of the .qetz were read so far - int folioPartsRead() const {return m_folio_parts_read;} - bool openedFromIndex() const {return m_opened_from_index;} private: static QUuid derivedUuid(const QByteArray &content); void refresh(); @@ -555,22 +550,14 @@ class QETProject : public QObject bool m_kept_folios_trusted = false; /// Read from a .qet whose path nobody chose since: saved beside it bool m_opened_from_qet = false; - /// Opened from a .qetz's index (stage D2): the - /// database filled from it, each folio's items in its part - bool m_opened_from_index = false; /// Saving a .qetz may copy the part of a folio not built /// (toXml(), stage D4) bool m_kept_parts_trusted = false; QSet m_waiting_link_targets; + /// The entries of the .qetz opened, and the part each folio + /// was read from, in order QHash m_container_files; - ///Reads the folio parts' engineering data, opened once - std::unique_ptr m_folio_reader; - int m_folio_parts_read = 0; - QStringList m_folio_files; - QList> m_folio_headers; - ///The drawn tables of the index opened from, if it has them - std::optional> m_index_tables; - void readFoliosFromIndex(); + QStringList m_folio_parts; bool m_kept_folios_stale = false; /// Something was edited since the project was opened: a table /// filled from the database may have changed diff --git a/tests/qttest/CMakeLists.txt b/tests/qttest/CMakeLists.txt index 18f285472..a71e43618 100644 --- a/tests/qttest/CMakeLists.txt +++ b/tests/qttest/CMakeLists.txt @@ -1060,10 +1060,13 @@ endif() add_executable( tst_resaveunchanged - tst_resaveunchanged.cpp) + tst_resaveunchanged.cpp + ${QET_DIR}/sources/container/qetzip.cpp + ${QET_DIR}/sources/container/miniz/miniz.c) add_test(NAME tst_resaveunchanged COMMAND tst_resaveunchanged) add_dependencies(tst_resaveunchanged qelectrotech) -target_link_libraries(tst_resaveunchanged PRIVATE Qt::Test) +target_include_directories(tst_resaveunchanged PRIVATE ${QET_DIR}/sources) +target_link_libraries(tst_resaveunchanged PRIVATE Qt::Test SQLite3::SQLite3) target_compile_definitions(tst_resaveunchanged PRIVATE "QET_TEST_BINARY_PATH=\"$\"" "QET_EXAMPLES_DIR=\"${QET_DIR}/examples\"") diff --git a/tests/qttest/lazy_folios_probe.cpp b/tests/qttest/lazy_folios_probe.cpp index 9e0d0e1cb..4e888e142 100644 --- a/tests/qttest/lazy_folios_probe.cpp +++ b/tests/qttest/lazy_folios_probe.cpp @@ -288,7 +288,7 @@ static QStringList savedLinks(const QString &qetz, const QString &path) db.setDatabaseName(db_path); check(db.open(), "a .qetz's database opened"); QSqlQuery query(db); - check(query.exec(QStringLiteral("SELECT element_uuid, linked_uuid FROM doc_link ORDER BY element_uuid, ord")), + check(query.exec(QStringLiteral("SELECT element_uuid, linked_uuid FROM link ORDER BY element_uuid, ord")), "saved links read"); while (query.next()) links[query.value(0).toString()] << query.value(1).toString(); @@ -750,8 +750,8 @@ int main(int argc, char **argv) check(opened.unloadedFolioCount() == 0, "a project with a shared uuid opens with every folio built"); check(built_late == 0, "a project with a shared uuid builds its folios as it reads them, not later"); } - //A .qetz opened from its index (stage D2): no - //folio read, each read from its part when shown, drawn and + //A .qetz opened lazily: no folio built but those + //the terminal strip needs, each built when shown, drawn and //linked as when built, saved as the .qet is { const QString zipped = output + "/lazy_folios.qetz"; @@ -759,12 +759,11 @@ int main(int argc, char **argv) check(eager.writeContainerFile(zipped, &error), "saved as a .qetz"); QETProject indexed(zipped); check(indexed.state() == QETProject::Ok, "the .qetz opens"); - check(indexed.openedFromIndex(), "the .qetz opens from its index"); - std::printf(".qetz opened from its index: %d of %d folios not built (strip folios %d)\n", + std::printf(".qetz opened: %d of %d folios not built (strip folios %d)\n", indexed.unloadedFolioCount(), int(folios.size()), int(strip_folios.size())); check(indexed.unloadedFolioCount() == folios.size() - strip_folios.size(), "the .qetz opens building only the folios of the terminal strip's terminals"); - check(numbering(indexed) == numbering(eager), "symbols' numbering from the index as when built"); + check(numbering(indexed) == numbering(eager), "symbols' numbering of a .qetz as when built"); //The drawn items, through the view made of their tables: the //folios built for it, as for the tables (D6: none were) { @@ -809,7 +808,7 @@ int main(int argc, char **argv) if (row.startsWith("diagram")) rows << row; return rows; }; - check(folioRows(indexed) == folioRows(eager), "the folios' rows from the index as when built"); + check(folioRows(indexed) == folioRows(eager), "the folios' rows of a .qetz as when built"); //Folio numbers that count the folios ("%id/%total") const QString totals = output + "/totals.qetz"; qputenv("QET_EAGER_FOLIOS", "1"); @@ -823,20 +822,19 @@ int main(int argc, char **argv) QETProject usual_totals(totals); qunsetenv("QET_EAGER_FOLIOS"); QETProject lazy_totals(totals); - check(lazy_totals.openedFromIndex(), "opened from its index"); + check(lazy_totals.unloadedFolioCount() > 0, "opened lazily"); const QStringList lazy_rows = folioRows(lazy_totals), usual_rows = folioRows(usual_totals); if (lazy_rows != usual_rows) for (int i = 0 ; i < lazy_rows.size() && i < 3 ; ++i) std::printf(" lazily %s\n built %s\n", qPrintable(lazy_rows.at(i).left(140)), qPrintable(usual_rows.value(i).left(140))); - check(lazy_rows == usual_rows, "folio numbers with totals from the index as when built"); + check(lazy_rows == usual_rows, "folio numbers with totals of a .qetz as when built"); } for (int i = 0; i < folios.size(); ++i) same(indexed, i, "shown from the .qetz"); check(indexed.toXml().toString() == eager.toXml().toString(), "the .qetz saves as when built"); //A folio whose table continues one on another folio, shown - //first: the index, not the folios' parts, says which folios - //its table is built with (D5) + //first: the folios its table is built with (D5) int continued = 0; for (int i = 0; i < folios.size(); ++i) { bool continues = false; @@ -852,8 +850,7 @@ int main(int argc, char **argv) check(continued > 0, "a folio whose table continues another"); //Numbering, the searches for symbols and the unused symbols - //are answered from the database: no folio file is read - //but those of the folios built (D5) + //are answered from the database (D5) { QETProject asked(zipped); check(numbering(asked) == numbering(eager), "numbering of a .qetz as when built"); @@ -874,10 +871,8 @@ int main(int argc, char **argv) check(found(search.freeElement(kinds)) == found(usual_search.freeElement(kinds)), "free symbols of a .qetz found as when built"); const int built = int(folios.size()) - asked.unloadedFolioCount(); - std::printf(".qetz asked numbering, searches, unused symbols: %d folio files read, %d folios built\n", - asked.folioPartsRead(), built); + std::printf(".qetz asked numbering, searches, unused symbols: %d folios built\n", built); check(asked.unloadedFolioCount() > 0, "asking a .qetz does not build every folio"); - check(asked.folioPartsRead() == built, "asking a .qetz reads no file of a folio not built"); } //A folio added, moved and removed: no other folio built, the diff --git a/tests/qttest/tst_qetcontainer.cpp b/tests/qttest/tst_qetcontainer.cpp index dd818596a..60986444e 100644 --- a/tests/qttest/tst_qetcontainer.cpp +++ b/tests/qttest/tst_qetcontainer.cpp @@ -70,14 +70,15 @@ class tst_qetcontainer : public QObject } //@p document without what QElectroTech computes, which a .qetz - //does not keep (format 3); tst_resaveunchanged checks that + //does not keep (format 4); tst_resaveunchanged checks that //QElectroTech gives it back static QDomDocument withoutComputed(const QDomDocument &document) { QDomDocument copy = document.cloneNode(true).toDocument(); + const bool own_uuids = QetContainer::ownUuids(copy.documentElement()); for (QDomElement diagram = copy.documentElement().firstChildElement(QStringLiteral("diagram")); !diagram.isNull() ; diagram = diagram.nextSiblingElement(QStringLiteral("diagram"))) - QetContainer::dropComputed(diagram, false); + QetContainer::dropComputed(diagram, own_uuids); return copy; } @@ -243,7 +244,7 @@ private slots: QCOMPARE(count("element_info"), int(original.elementsByTagName(QStringLiteral("elementInformation")).size())); QCOMPARE(count("link"), int(original.elementsByTagName(QStringLiteral("link_uuid")).size())); QCOMPARE(count("conductor"), int(original.elementsByTagName(QStringLiteral("conductor")).size())); - QVERIFY(count("folio_property") + count("doc_project_property") > 0); + QVERIFY(count("folio_property") + count("project_property") > 0); sqlite3_close(db); } diff --git a/tests/qttest/tst_resaveunchanged.cpp b/tests/qttest/tst_resaveunchanged.cpp index 9cceb2b50..72a8186a7 100644 --- a/tests/qttest/tst_resaveunchanged.cpp +++ b/tests/qttest/tst_resaveunchanged.cpp @@ -8,6 +8,10 @@ #include #include +#include "container/qetzip.h" + +#include + // Saving a project that was just saved must change nothing. Two things // made the second save differ from the first, both cleanup done on save // but not on load: @@ -306,6 +310,10 @@ private slots: QVERIFY2(!zipped.isEmpty(), "--resave as .qetz failed"); QVERIFY2(read(zipped).startsWith("PK\x03\x04"), "the .qetz is not a zip"); QVERIFY2(read(zipped).size() < read(straight).size() / 2, "the .qetz is not smaller"); + QCOMPARE(databaseTables(zipped), QStringList({ + "conductor", "element", "element_info", "folio", "folio_property", "link", "meta", + "project_property", "strip_bridge", "strip_bridge_terminal", "strip_terminal", + "terminal_strip"})); const QString back = resave(zipped); QVERIFY2(!back.isEmpty(), "opening the .qetz failed"); @@ -317,6 +325,31 @@ private slots: QCOMPARE(without_save_stamps(read(back)), without_save_stamps(read(straight))); } + // The tables of the database in the .qetz @p path: the document's + // engineering data only, each fact once -- not QElectroTech's own + // project database, built from it when the project opens + static QStringList databaseTables(const QString &path) + { + QList entries; + if (!QetZip::fromBytes(read(path), &entries)) return {QStringLiteral("not a zip")}; + QByteArray database; + for (const QetZip::Entry &entry : std::as_const(entries)) + if (entry.name == QLatin1String("project.sqlite")) database = entry.data; + sqlite3 *db = nullptr; + sqlite3_open(":memory:", &db); + unsigned char *copy = static_cast(sqlite3_malloc64(sqlite3_uint64(database.size()))); + memcpy(copy, database.constData(), size_t(database.size())); + sqlite3_deserialize(db, "main", copy, database.size(), database.size(), SQLITE_DESERIALIZE_FREEONCLOSE); + sqlite3_stmt *list = nullptr; + QStringList tables; + sqlite3_prepare_v2(db, "SELECT name FROM sqlite_master WHERE type = 'table' ORDER BY name", -1, &list, nullptr); + while (list && sqlite3_step(list) == SQLITE_ROW) + tables << QString::fromUtf8(reinterpret_cast(sqlite3_column_text(list, 0))); + sqlite3_finalize(list); + sqlite3_close(db); + return tables; + } + // QElectroTech run in a sandbox of its own with @p args, the store // check on (QET_CHECK_ELEMENT_INFO_STORE); its output QString runChecked(const QStringList &args, bool eager = false)