From c240e4e6f7027316269bd6b3f826bf88fb0c5015 Mon Sep 17 00:00:00 2001 From: ispyisail Date: Tue, 13 Oct 2026 21:49:34 +1300 Subject: [PATCH 1/4] Fix the dates of the files inside a .qetz project miniz is built with MINIZ_NO_TIME so that saving the same project twice gives the same bytes. Without a clock it writes 0 as each entry's DOS date: day 0 of month 0, 1980, which is not a date. Unzip tools show it as 30 Nov 1979, others fail the conversion and show 13 Dec 1901 (reported on PR #1462). Write 1 Jan 1980 00:00, the first valid DOS date, into the local and central header of every entry once the archive is built. The bytes stay reproducible, and miniz stays unmodified. Using the save time instead was rejected: two saves would no longer give the same file, and miniz converts through the local time zone, so even a fixed time_t would differ between machines. Co-Authored-By: Claude Opus 5.5 --- sources/container/qetzip.cpp | 43 +++++++++++++++++++++++++++++++++++- sources/container/qetzip.h | 4 ++-- tests/qttest/tst_qetzip.cpp | 28 +++++++++++++++++++++++ 3 files changed, 72 insertions(+), 3 deletions(-) diff --git a/sources/container/qetzip.cpp b/sources/container/qetzip.cpp index 7a0be3c26..dc9e2c9f7 100644 --- a/sources/container/qetzip.cpp +++ b/sources/container/qetzip.cpp @@ -22,6 +22,7 @@ #include #include #include +#include namespace { void setError(QString *error, const QString &text) @@ -44,6 +45,42 @@ bool safeName(const QString &name) } return true; } + + //Built with MINIZ_NO_TIME, miniz leaves each entry's DOS date at 0, + //day 0 of month 0, which is no date: unzip shows 30 Nov 1979, other + //tools 13 Dec 1901. Give every entry 1 Jan 1980 00:00 instead, the + //first valid DOS date, so the bytes still do not depend on the time. +constexpr quint16 EntryDosDate = (1 << 5) | 1; + +bool setEntryDates(QByteArray *bytes) +{ + char *data = bytes->data(); + mz_zip_archive zip; + mz_zip_zero_struct(&zip); + if (!mz_zip_reader_init_mem(&zip, data, size_t(bytes->size()), 0)) return false; + + bool ok = true; + const mz_uint count = mz_zip_reader_get_num_files(&zip); + for (mz_uint i = 0 ; i < count && ok ; ++i) + { + mz_zip_archive_file_stat stat; + if (!mz_zip_reader_file_stat(&zip, i, &stat)) { + ok = false; + break; + } + //date at offset 12 of the local header, 14 of the central one + const quint64 local = stat.m_local_header_ofs + 12; + const quint64 central = zip.m_central_directory_file_ofs + stat.m_central_dir_ofs + 14; + if (local + 2 > quint64(bytes->size()) || central + 2 > quint64(bytes->size())) { + ok = false; + break; + } + qToLittleEndian(EntryDosDate, data + local); + qToLittleEndian(EntryDosDate, data + central); + } + mz_zip_reader_end(&zip); + return ok; +} } /** @@ -87,8 +124,12 @@ QByteArray QetZip::toBytes(const QList &entries, QString *error) mz_zip_writer_end(&zip); return {}; } - const QByteArray bytes(static_cast(buffer), qsizetype(size)); + QByteArray bytes(static_cast(buffer), qsizetype(size)); mz_zip_writer_end(&zip); //frees buffer + if (!setEntryDates(&bytes)) { + setError(error, QStringLiteral("cannot date the zip entries")); + return {}; + } return bytes; } diff --git a/sources/container/qetzip.h b/sources/container/qetzip.h index 9238f6ebf..808a861ce 100644 --- a/sources/container/qetzip.h +++ b/sources/container/qetzip.h @@ -30,8 +30,8 @@ megabytes, not gigabytes -- and written through QSaveFile, so a failed save leaves the previous file as it was. - Entries carry no timestamp: saving the same project twice gives the - same bytes. + Every entry is dated 1 Jan 1980 00:00, not the time of saving: saving + the same project twice gives the same bytes. */ namespace QetZip { diff --git a/tests/qttest/tst_qetzip.cpp b/tests/qttest/tst_qetzip.cpp index 4c69afef3..156cb8fb6 100644 --- a/tests/qttest/tst_qetzip.cpp +++ b/tests/qttest/tst_qetzip.cpp @@ -2,6 +2,7 @@ #include #include +#include #include "container/qetzip.h" #include "container/miniz/miniz.h" @@ -72,6 +73,33 @@ private slots: QCOMPARE(QetZip::toBytes(sample()), QetZip::toBytes(sample())); } + // every entry carries a valid date in both of its headers, not the + // 0 miniz writes without a clock (shown as 1979 or 1901 by unzip tools) + void entriesHaveValidDate() + { + const QByteArray zip = QetZip::toBytes(sample()); + QVERIFY(!zip.isEmpty()); + mz_zip_archive archive; + mz_zip_zero_struct(&archive); + QVERIFY(mz_zip_reader_init_mem(&archive, zip.constData(), size_t(zip.size()), 0)); + const mz_uint count = mz_zip_reader_get_num_files(&archive); + QCOMPARE(int(count), sample().size()); + for (mz_uint i = 0 ; i < count ; ++i) { + mz_zip_archive_file_stat stat; + QVERIFY(mz_zip_reader_file_stat(&archive, i, &stat)); + const char *local = zip.constData() + stat.m_local_header_ofs; + const char *central = zip.constData() + archive.m_central_directory_file_ofs + stat.m_central_dir_ofs; + QCOMPARE(qFromLittleEndian(local), quint16(0x50 | 0x4b << 8)); // "PK" + QCOMPARE(qFromLittleEndian(central), quint16(0x50 | 0x4b << 8)); + // 1 Jan 1980: day 1, month 1, year 0 since 1980; time 00:00:00 + QCOMPARE(qFromLittleEndian(local + 12), quint16(0x21)); + QCOMPARE(qFromLittleEndian(central + 14), quint16(0x21)); + QCOMPARE(qFromLittleEndian(local + 10), quint16(0)); + QCOMPARE(qFromLittleEndian(central + 12), quint16(0)); + } + mz_zip_reader_end(&archive); + } + void refusesUnsafeNamesOnWrite_data() { QTest::addColumn("name"); From 1ad3c8e2b219683697eb6e6c3d3409390b4b4391 Mon Sep 17 00:00:00 2001 From: ispyisail Date: Tue, 13 Oct 2026 23:02:47 +1300 Subject: [PATCH 2/4] Speed up opening .qetz projects with large pictures A .qetz keeps each folio picture as a PNG file. On opening, join() put every picture back into the document as base64 text, and the folio then decoded that text again, twice. On a project with 20 pictures of 7 MB (140 MB of PNG, the shape of the project on PR #1462) that cost about 8 of the 12 seconds the open took: QDomDocument::createTextNode() on the base64 3.9 s DiagramImageItem::fromXml(), base64 decoded 3.9 s (2 x 1.9 s) the PNG decoding itself 0.3 s join() now takes an optional map. Given one, each picture stays a file: the keeps its qetz-file attribute and the bytes go in the map. QETProject keeps the map for as long as the project is open, and Diagram::fromXml() hands a picture's bytes to the new DiagramImageItem::fromXml(element, picture). Kept, not cleared after the load, so a folio built later finds its pictures too; the bytes are shared with the PNG cache of the pictures built from them, so this costs almost no memory. readDocument() without a map, as the automatic numbering import uses it, still gives the whole document. fromXml() of a .qet picture also decodes its base64 once instead of twice: the first text node, which the PNG cache already used. e.text() also collects 's text, but loadFromData() stops at the end of the first picture, so the picture shown is the same. Measured, same file, 3 runs each, offscreen: .qetz 11.8 s -> 3.8 s, peak memory 1034 MB -> 655 MB .qet 15.7 s -> 13.6 s Resaving either file with either binary gives the same bytes, a cropped and colour-keyed picture included, and every folio exports to the same PNG pixels. Co-Authored-By: Claude Opus 5.5 --- sources/container/qetcontainer.cpp | 22 +++++++++-- sources/container/qetcontainer.h | 15 +++++++- sources/diagram.cpp | 7 +++- sources/qetgraphicsitem/diagramimageitem.cpp | 40 ++++++++++++-------- sources/qetgraphicsitem/diagramimageitem.h | 1 + sources/qetproject.cpp | 17 ++++++++- sources/qetproject.h | 4 ++ tests/qttest/tst_qetcontainer.cpp | 37 ++++++++++++++++++ 8 files changed, 120 insertions(+), 23 deletions(-) diff --git a/sources/container/qetcontainer.cpp b/sources/container/qetcontainer.cpp index 7f27789c4..40b8c0db4 100644 --- a/sources/container/qetcontainer.cpp +++ b/sources/container/qetcontainer.cpp @@ -155,13 +155,19 @@ bool QetContainer::isZip(const QByteArray &content) The project document held in @p content: unzipped and joined for a .qetz, parsed for a .qet. */ -bool QetContainer::readDocument(const QByteArray &content, QDomDocument *document, QString *error) +bool QetContainer::readDocument(const QByteArray &content, QDomDocument *document, + QString *error, Pictures *pictures) { if (!isZip(content)) return parse(content, document, error); QList entries; return QetZip::fromBytes(content, &entries, error) - && join(entries, document, error); + && join(entries, document, error, pictures); +} + +QString QetContainer::pictureFile(const QDomElement &image) +{ + return image.attribute(ImageFileAttribute); } /** @@ -264,8 +270,11 @@ QList QetContainer::split(const QDomDocument &project, const QStr Refused, with @p error set: not a .qetz, made for a later QElectroTech (the manifest's min-reader), or a part missing or unreadable. Entries nothing names are ignored, so a later format may add some. + With @p pictures, the pictures stay out of the document: see the + namespace. */ -bool QetContainer::join(const QList &entries, QDomDocument *project, QString *error) +bool QetContainer::join(const QList &entries, QDomDocument *project, QString *error, + Pictures *pictures) { QHash files; for (const QetZip::Entry &entry : entries) @@ -310,7 +319,8 @@ bool QetContainer::join(const QList &entries, QDomDocument *proje part.parentNode().replaceChild(project->importNode(document.documentElement(), true), part); } - //Pictures back to base64, as the image's first child + //Pictures back to base64, as the image's first child, or into + //pictures const QDomNodeList images = project->elementsByTagName(QStringLiteral("image")); for (int i = 0 ; i < images.size() ; ++i) { @@ -321,6 +331,10 @@ bool QetContainer::join(const QList &entries, QDomDocument *proje setError(error, QStringLiteral("picture \"%1\" is missing").arg(file)); return false; } + if (pictures) { + pictures->insert(file, files.value(file)); + continue; + } image.removeAttribute(ImageFileAttribute); image.insertBefore(project->createTextNode(QString::fromLatin1(files.value(file).toBase64())), image.firstChild()); diff --git a/sources/container/qetcontainer.h b/sources/container/qetcontainer.h index d92df084c..7d0b4152a 100644 --- a/sources/container/qetcontainer.h +++ b/sources/container/qetcontainer.h @@ -21,6 +21,7 @@ #include "qetzip.h" #include +#include /** @brief The QetContainer namespace @@ -41,10 +42,17 @@ join(split(d)) is d. Nothing here knows what the parts mean: the project is read and written by the same code as a .qet file. + + Given a Pictures map, join() leaves the pictures out of the document: + each keeps the name of its file, pictureFile(), and the bytes + go in the map. Turning them back into base64 text costs seconds on a + project with large pictures, and the folio would only decode it again. */ namespace QetContainer { extern const char MimeType[]; + /// Picture bytes by file name, see join() + using Pictures = QHash; /// The format written; files of a later format are refused by join() /// unless their min-reader allows this one. constexpr int Format = 2; @@ -53,14 +61,17 @@ namespace QetContainer const QString &writer = QString(), QString *error = nullptr); bool join(const QList &entries, QDomDocument *project, - QString *error = nullptr); + QString *error = nullptr, Pictures *pictures = nullptr); + /// The file join() left the picture of @p image in, or an empty string + QString pictureFile(const QDomElement &image); /// Parse @p xml as a .qet is read (QETProject::openFile) bool parse(const QByteArray &xml, QDomDocument *document, QString *error = nullptr); /// true if @p content is a zip, so a .qetz rather than a .qet bool isZip(const QByteArray &content); /// The project document in @p content, a .qet or a .qetz - bool readDocument(const QByteArray &content, QDomDocument *document, QString *error = nullptr); + bool readDocument(const QByteArray &content, QDomDocument *document, + QString *error = nullptr, Pictures *pictures = nullptr); } #endif // QETCONTAINER_H diff --git a/sources/diagram.cpp b/sources/diagram.cpp index a04d4b433..e98aa7a83 100644 --- a/sources/diagram.cpp +++ b/sources/diagram.cpp @@ -48,6 +48,7 @@ #include "diagramsortkeys.h" #include "itemgroups.h" #include "textgrid.h" +#include "container/qetcontainer.h" #include "foliogrid.h" #include #include @@ -1915,7 +1916,11 @@ bool Diagram::fromXml(QDomElement &document, QStringLiteral("images"), QStringLiteral("image"))) { DiagramImageItem *dii = new DiagramImageItem (); - dii -> fromXml(image_xml); + const QString picture_file = QetContainer::pictureFile(image_xml); + if (!picture_file.isEmpty() && project()) + dii -> fromXml(image_xml, project() -> containerPicture(picture_file)); + else + dii -> fromXml(image_xml); settle_uuid(dii, image_xml, QStringLiteral("image"), added_images.size()); ItemGroups::setGroup(dii, ItemGroups::read(image_xml)); addItem(dii); diff --git a/sources/qetgraphicsitem/diagramimageitem.cpp b/sources/qetgraphicsitem/diagramimageitem.cpp index a952c4851..4c5f05145 100644 --- a/sources/qetgraphicsitem/diagramimageitem.cpp +++ b/sources/qetgraphicsitem/diagramimageitem.cpp @@ -1787,6 +1787,17 @@ const QByteArray &DiagramImageItem::encodedPng(const QPixmap &pixmap, QByteArray @return true if successfully loaded. */ bool DiagramImageItem::fromXml(const QDomElement &e) +{ + return fromXml(e, QByteArray()); +} + +/** + @brief DiagramImageItem::fromXml + Load this image from xml element e, its picture being @p picture, not + the base64 text of @p e, unless @p picture is empty. A .qetz keeps its + pictures as files (QetContainer::join()). +*/ +bool DiagramImageItem::fromXml(const QDomElement &e, const QByteArray &picture) { if (e.tagName() != "image") { return (false); @@ -1796,28 +1807,27 @@ bool DiagramImageItem::fromXml(const QDomElement &e) const QUuid uuid(e.attribute(QStringLiteral("uuid"))); if (!uuid.isNull() && uuid != m_uuid) setUuid(uuid); - QDomNode image_node = e.firstChild(); - if (!image_node.isText()) { - return (false); + // From the first text node only, not e.text(): that also collects the + // text of every child element, so on a cropped or colour-keyed picture + // it carries 's bytes too. Decoded once: on a project with + // large pictures each decoding costs seconds. + QByteArray array = picture; + if (array.isEmpty()) + { + QDomNode image_node = e.firstChild(); + if (!image_node.isText()) { + return (false); + } + array = QByteArray::fromBase64(image_node.toText().data().toLatin1()); } - //load xml image to QByteArray - QByteArray array; - array = QByteArray::fromBase64(e.text().toLatin1()); - //Set QPixmap from the array QPixmap pixmap; pixmap.loadFromData(array); setPixmap(pixmap); - // From the first text node only, not e.text(): that also collects the - // text of every child element, so on a cropped or colour-keyed picture - // it carries 's bytes too. loadFromData() stops at the end - // of the first PNG and never notices; a cache written back verbatim - // would. - const QByteArray ownArray = QByteArray::fromBase64(image_node.toText().data().toLatin1()); - if (isPng(ownArray) && !pixmap_.isNull()) + if (isPng(array) && !pixmap_.isNull()) { - m_png_cache = ownArray; + m_png_cache = array; m_png_cache_key = pixmap_.cacheKey(); } diff --git a/sources/qetgraphicsitem/diagramimageitem.h b/sources/qetgraphicsitem/diagramimageitem.h index 6e0f308f8..df8f77b1a 100644 --- a/sources/qetgraphicsitem/diagramimageitem.h +++ b/sources/qetgraphicsitem/diagramimageitem.h @@ -120,6 +120,7 @@ class DiagramImageItem : public QetGraphicsItem { int type() const override { return Type; } virtual bool fromXml(const QDomElement &); + bool fromXml(const QDomElement &, const QByteArray &picture); virtual QDomElement toXml(QDomDocument &) const; QUuid uuid() const {return m_uuid;} void setUuid(const QUuid &uuid) {m_uuid = uuid; emit uuidChanged();} diff --git a/sources/qetproject.cpp b/sources/qetproject.cpp index 70416ef25..500b5b1f9 100644 --- a/sources/qetproject.cpp +++ b/sources/qetproject.cpp @@ -276,6 +276,20 @@ QUuid QETProject::uuid() const return m_uuid; } +/** + @brief QETProject::containerPicture + The folio pictures of a .qetz are kept as bytes beside the document, + not as base64 text in it (QetContainer::join()), for as long as the + project is open: a folio built after opening still finds its pictures. + They cost almost no memory, sharing their bytes with the pictures + built from them. + @return the picture in @p file, or an empty array when there is none +*/ +QByteArray QETProject::containerPicture(const QString &file) const +{ + return m_container_pictures.value(file); +} + /** @brief QETProject::derivedItemUuid A uuid for an item of this project that was saved without one, the same @@ -438,9 +452,10 @@ QETProject::ProjectState QETProject::openFile(QFile *file) // QetContainer::parse() does exactly that, for a .qet and for each part // of a .qetz alike. QString read_error; - if (!QetContainer::readDocument(content, &xml_project, &read_error)) + if (!QetContainer::readDocument(content, &xml_project, &read_error, &m_container_pictures)) { qWarning().noquote() << "Unable to read" << fi.fileName() << ":" << read_error; + m_container_pictures.clear(); if(opened_here) { file->close(); } diff --git a/sources/qetproject.h b/sources/qetproject.h index 58dd124c6..b71bb7424 100644 --- a/sources/qetproject.h +++ b/sources/qetproject.h @@ -117,6 +117,7 @@ class QETProject : public QObject projectDataBase *dataBase(); QUuid uuid() const; QUuid derivedItemUuid(const QString &kind, const QString &key); + QByteArray containerPicture(const QString &file) const; ProjectState state() const; QList diagrams() const; int folioIndex(const Diagram *) const; @@ -416,6 +417,9 @@ class QETProject : public QObject static bool m_backup_enabled; /// Something changed since the last backup, see writeBackup() bool m_backup_needed = true; + /// The pictures of the .qetz this project was opened from, see + /// containerPicture() + QHash m_container_pictures; /// File path this project is saved to QString m_file_path; /// Current state of the project diff --git a/tests/qttest/tst_qetcontainer.cpp b/tests/qttest/tst_qetcontainer.cpp index ea2a9525c..42e7b0c8f 100644 --- a/tests/qttest/tst_qetcontainer.cpp +++ b/tests/qttest/tst_qetcontainer.cpp @@ -242,6 +242,43 @@ private slots: QVERIFY(pictures < 121); // the 121 placed pictures repeat a few files } + // Given a map, join() hands the pictures over as bytes instead of + // putting them back in the document as base64 + void picturesKeptAside() + { + QDomDocument original; + QList entries = splitFile( + QStringLiteral(QET_EXAMPLES_DIR "/weneedpolonez-Polonez_MR89_wiring_diagram.qet"), &original); + QVERIFY(!entries.isEmpty()); + QHash files; + for (const QetZip::Entry &e : entries) files.insert(e.name, e.data); + + QDomDocument back; + QetContainer::Pictures pictures; + QString error; + QVERIFY2(QetContainer::join(entries, &back, &error, &pictures), qPrintable(error)); + const QDomNodeList images = back.elementsByTagName(QStringLiteral("image")); + const QDomNodeList original_images = original.elementsByTagName(QStringLiteral("image")); + QCOMPARE(images.size(), 121); + QCOMPARE(images.size(), original_images.size()); + for (int i = 0 ; i < images.size() ; ++i) { + const QDomElement image = images.at(i).toElement(); + const QString file = QetContainer::pictureFile(image); + QVERIFY(!file.isEmpty()); + QVERIFY(!image.firstChild().isText()); + QCOMPARE(pictures.value(file), files.value(file)); + QCOMPARE(pictures.value(file), + QByteArray::fromBase64(original_images.at(i).firstChild().toText().data().toLatin1())); + } + + // still refused when a picture is missing + for (int i = 0 ; i < entries.size() ; ++i) + if (entries.at(i).name.startsWith(QStringLiteral("images/"))) { entries.removeAt(i); break; } + QDomDocument broken; + QVERIFY(!QetContainer::join(entries, &broken, &error, &pictures)); + QVERIFY2(error.contains(QStringLiteral("missing")), qPrintable(error)); + } + // A file whose manifest needs a later reader is refused, not half read void laterFormatRefused() { From 1c4618c8dd415cab65cda0c7fd0d6e595093fdfd Mon Sep 17 00:00:00 2001 From: ispyisail Date: Tue, 13 Oct 2026 22:00:52 +1300 Subject: [PATCH 3/4] Open with every folio built when the database must come from them With QET_LAZY_FOLIOS=1, a project whose database cannot be filled from the file (folios or symbols with no saved uuid, two symbols sharing one, a folio number using %autonum, wires naming their ends the older way) had its folios left unbuilt, then built all at once while the database was filled from them. That is slower than building them as they are read: a 17 MB project reported on the pull request took 24.8 s to open instead of 19.2 s, and most files saved by an older version have no folio uuids. The file is now checked for those reasons before the folios are read; when one applies the project opens as without QET_LAZY_FOLIOS. The rarer reasons, which need the definitions or every folio's links, still fall back as before. tst_lazyfolios opens a copy of industrial.qet with one symbol's uuid given to another, and checks no folio is built after reading. Co-Authored-By: Claude Opus 5.5 --- sources/dataBase/projectdatabase.cpp | 52 ++++++++++++++++++++++++++++ sources/dataBase/projectdatabase.h | 8 +++++ sources/qetproject.cpp | 17 +++++++-- sources/qetproject.h | 3 ++ tests/qttest/lazy_folios_probe.cpp | 40 +++++++++++++++++++++ 5 files changed, 118 insertions(+), 2 deletions(-) diff --git a/sources/dataBase/projectdatabase.cpp b/sources/dataBase/projectdatabase.cpp index d71bedfe4..095945401 100644 --- a/sources/dataBase/projectdatabase.cpp +++ b/sources/dataBase/projectdatabase.cpp @@ -253,6 +253,58 @@ void projectDataBase::updateDB(const QDomDocument &document) updateDB(); } +/** + @brief projectDataBase::documentRefusal + @return why populateFromDocument() will refuse @p document, for the + reasons it gives that the document alone can tell -- a folio, symbol + or wire with no saved uuid, two symbols sharing one, a folio number + using %autonum -- or an empty string. The other reasons need the + project's definitions or every folio's links; a document refused for + one of those is still filled from the folios, only later. +*/ +QString projectDataBase::documentRefusal(const QDomDocument &document) +{ + if (qEnvironmentVariableIntValue("QET_DATABASE_FROM_FOLIOS") == 1) { + return QStringLiteral("QET_DATABASE_FROM_FOLIOS is set"); + } + QSet element_uuids; + const QDomNodeList diagram_nodes = document.elementsByTagName(QStringLiteral("diagram")); + for (int i = 0 ; i < diagram_nodes.size() ; ++i) + { + QDomElement diagram_xml = diagram_nodes.at(i).toElement(); + if (QUuid(diagram_xml.attribute(QStringLiteral("uuid"))).isNull()) { + return QStringLiteral("a folio has no saved uuid"); + } + if (diagram_xml.attribute(QStringLiteral("folio")).contains(QStringLiteral("%autonum"))) { + return QStringLiteral("a folio number uses %autonum"); + } + for (QDomElement element_xml : QET::findInDomElement( + diagram_xml, QStringLiteral("elements"), QStringLiteral("element"))) + { + //Skipped by Diagram::fromXml() as well + if (!Element::valideXml(element_xml)) { + continue; + } + const QUuid uuid(element_xml.attribute(QStringLiteral("uuid"))); + if (uuid.isNull() || element_uuids.contains(uuid)) { + return QStringLiteral("an element has no saved uuid, or shares one"); + } + element_uuids.insert(uuid); + } + for (QDomElement conductor_xml : QET::findInDomElement( + diagram_xml, QStringLiteral("conductors"), QStringLiteral("conductor"))) + { + if (Conductor::valideXml(conductor_xml) + && (QUuid(conductor_xml.attribute(QStringLiteral("uuid"))).isNull() + || !conductor_xml.hasAttribute(QStringLiteral("element1")) + || !conductor_xml.hasAttribute(QStringLiteral("element2")))) { + return QStringLiteral("a conductor has no saved uuid, or names its ends the older way"); + } + } + } + return QString(); +} + namespace { struct DocumentTerminal { diff --git a/sources/dataBase/projectdatabase.h b/sources/dataBase/projectdatabase.h index 0dd92aaa1..faf92ba6f 100644 --- a/sources/dataBase/projectdatabase.h +++ b/sources/dataBase/projectdatabase.h @@ -67,6 +67,14 @@ class projectDataBase : public QObject populateFromDocument()), from the built folios otherwise. */ void updateDB(const QDomDocument &document); + /** + Why updateDB(document) will fill the tables from the folios + rather than from @p document, for the reasons the document + alone can tell; empty when it knows of none. Read before the + folios are: a project whose tables will come from the folios + builds every folio as it opens, even with QET_LAZY_FOLIOS. + */ + static QString documentRefusal(const QDomDocument &document); /** Suppress the full rebuild performed by updateDB(). diff --git a/sources/qetproject.cpp b/sources/qetproject.cpp index 913bc9b77..676f20efb 100644 --- a/sources/qetproject.cpp +++ b/sources/qetproject.cpp @@ -3018,6 +3018,19 @@ void QETProject::readProjectXml(QDomDocument &xml_project) //built (DB-ACCESSORS-PLAN.md stage 4.6) m_data_base.prefillFromDocument(xml_project); + //QET_LAZY_FOLIOS leaves the folios to be built when needed only + //when the database can be filled from the document: otherwise + //filling it from the folios builds every one of them anyway, late + //and more slowly than reading them now + m_lazy_open = qEnvironmentVariableIntValue("QET_LAZY_FOLIOS") == 1; + if (m_lazy_open) { + const QString why = projectDataBase::documentRefusal(xml_project); + if (!why.isEmpty()) { + qInfo().noquote() << "Project opened with every folio built:" << why; + m_lazy_open = false; + } + } + //Load the diagrams readDiagramsXml(xml_project); const qint64 diagrams_ms = phase_timer.restart(); @@ -3134,7 +3147,7 @@ void QETProject::readDiagramsXml(QDomDocument &xml_project) //Where each line of the file starts, for a folio's text: lines //[its start, the next element's start) (QETProject::toXml()) QList line_starts; - if (qEnvironmentVariableIntValue("QET_LAZY_FOLIOS") == 1 && !m_opened_content.isEmpty()) { + if (m_lazy_open && !m_opened_content.isEmpty()) { line_starts << 0; for (qsizetype i = 0 ; i < m_opened_content.size() ; ++i) if (m_opened_content.at(i) == '\n') line_starts << i + 1; @@ -3181,7 +3194,7 @@ void QETProject::readDiagramsXml(QDomDocument &xml_project) connect(diagram, &Diagram::usedTitleBlockTemplateChanged, this, &QETProject::usedTitleBlockTemplateChanged); - m_deferring_folios = qEnvironmentVariableIntValue("QET_LAZY_FOLIOS") == 1; + m_deferring_folios = m_lazy_open; diagram->initFromXml(diagram_xml_element); m_deferring_folios = false; if (!diagram->isLoaded() && !line_starts.isEmpty()) diff --git a/sources/qetproject.h b/sources/qetproject.h index 30da9d1a4..9bdc240c7 100644 --- a/sources/qetproject.h +++ b/sources/qetproject.h @@ -512,6 +512,9 @@ class QETProject : public QObject const DiagramContext &information, const QHash &titles_by_formula) const; bool m_deferring_folios = false; + /// Opening with folios built only when needed: QET_LAZY_FOLIOS + /// is set and the database can be filled from the document + bool m_lazy_open = false; /// Saving may write a folio not built as it was read /// (writesKeptFolios()): the file is this version's, its /// database rows came from it, and nothing project-wide that diff --git a/tests/qttest/lazy_folios_probe.cpp b/tests/qttest/lazy_folios_probe.cpp index daebbba50..94f238a35 100644 --- a/tests/qttest/lazy_folios_probe.cpp +++ b/tests/qttest/lazy_folios_probe.cpp @@ -37,6 +37,8 @@ #include "../../sources/undocommand/deleteqgraphicsitemcommand.h" #include "../../sources/factory/elementfactory.h" #include "../../sources/undocommand/addgraphicsobjectcommand.h" +#include "../../sources/dataBase/projectdatabase.h" +#include static void check(bool value, const char *message) { if (!value) throw std::runtime_error(message); } @@ -567,6 +569,44 @@ int main(int argc, char **argv) check(potential(wire) == potential(usual_wire), "a potential followed as when built"); std::printf("potential through reports: %d wires, %d folios not built\n", int(potential(wire).size()), followed.unloadedFolioCount()); + } + //Two symbols sharing a uuid: the database is filled from the + //folios, which builds every one of them, so the project opens + //with every folio built rather than building them all later + { + QFile file(path); + check(file.open(QIODevice::ReadOnly), "fixture read"); + QDomDocument document; + check(bool(document.setContent(&file)), "fixture parsed"); + file.close(); + check(projectDataBase::documentRefusal(document).isEmpty(), "the fixture fills the database itself"); + QDomNodeList elements = document.elementsByTagName(QStringLiteral("element")); + QList placed; + for (int i = 0; i < elements.size(); ++i) { + QDomElement e = elements.at(i).toElement(); + if (Element::valideXml(e)) placed << e; + } + check(placed.size() > 1, "two placed symbols"); + placed.last().setAttribute(QStringLiteral("uuid"), placed.first().attribute(QStringLiteral("uuid"))); + check(!projectDataBase::documentRefusal(document).isEmpty(), "a shared uuid is seen before the folios are read"); + const QString shared = output + "/lazy_folios_shared_uuid.qet"; + QFile out(shared); + check(out.open(QIODevice::WriteOnly), "copy written"); + out.write(document.toByteArray(4)); + out.close(); + qputenv("QET_LAZY_FOLIOS", "1"); + qputenv("QET_LAZY_FOLIOS_TRACE", "1"); + static int built_late; built_late = 0; + const QtMessageHandler previous = qInstallMessageHandler( + [](QtMsgType, const QMessageLogContext &, const QString &message) { + if (message.contains(QStringLiteral("built for a caller of diagrams()"))) ++built_late; + }); + QETProject opened(shared); + qInstallMessageHandler(previous); + qunsetenv("QET_LAZY_FOLIOS_TRACE"); + check(opened.state() == QETProject::Ok, "a project with a shared uuid opens"); + check(opened.unloadedFolioCount() == 0, "a project with a shared uuid opens with every folio built"); + check(built_late == 0, "a project with a shared uuid builds its folios as it reads them, not later"); } std::printf("PASS: %d folios\n", int(folios.size())); } catch (const std::exception &error) { From 00e200b50436b77400460dfeaf58ff26dc248118 Mon Sep 17 00:00:00 2001 From: ispyisail Date: Tue, 13 Oct 2026 23:03:14 +1300 Subject: [PATCH 4/4] Remove references to notes that are not in this repository Comments added with the lazy folios cited planning notes kept outside the QElectroTech repository, which a reader of the source cannot open. The sentences now stand on their own. Co-Authored-By: Claude Opus 5.5 --- sources/bordertitleblock.cpp | 4 ++-- sources/bordertitleblock.h | 4 ++-- sources/dataBase/projectdatabase.cpp | 2 +- sources/dataBase/projectdatabase.h | 8 ++++---- sources/diagram.h | 2 +- sources/qetgraphicsitem/conductor.cpp | 2 +- sources/qetgraphicsitem/element.cpp | 4 ++-- sources/qetproject.cpp | 8 +++----- sources/qetproject.h | 2 +- tests/qttest/lazy_folios_probe.cpp | 2 +- tests/qttest/tst_resaveunchanged.cpp | 6 +++--- 11 files changed, 21 insertions(+), 23 deletions(-) diff --git a/sources/bordertitleblock.cpp b/sources/bordertitleblock.cpp index f0e6f65c1..8747aa674 100644 --- a/sources/bordertitleblock.cpp +++ b/sources/bordertitleblock.cpp @@ -321,8 +321,8 @@ TitleBlockProperties BorderTitleBlock::localTitleBlock() const /** @brief BorderTitleBlock::current @return the properties of the titleblock: for a folio placed in a - project the project's store holds them (DB-ACCESSORS-PLAN.md stage - 4.4), for any other this object's own copy. + project the project's store holds them, for any other this object's + own copy. */ TitleBlockProperties BorderTitleBlock::current() const { diff --git a/sources/bordertitleblock.h b/sources/bordertitleblock.h index adc422752..009089c75 100644 --- a/sources/bordertitleblock.h +++ b/sources/bordertitleblock.h @@ -110,8 +110,8 @@ class BorderTitleBlock : public QObject QRectF insideBorderRect() const; // methods to get title block basic data. A folio placed in a - // project reads them from the project's store - // (DB-ACCESSORS-PLAN.md stage 4.4), any other from its own copy. + // project reads them from the project's store, any other from its + // own copy. /// @return the value of the title block "Author" field QString author() const { const TitleBlockProperties *s = stored(); return(s ? s->author : btb_author_); } /// @return the value of the title block "Date" field diff --git a/sources/dataBase/projectdatabase.cpp b/sources/dataBase/projectdatabase.cpp index 095945401..68a98486e 100644 --- a/sources/dataBase/projectdatabase.cpp +++ b/sources/dataBase/projectdatabase.cpp @@ -2497,7 +2497,7 @@ void projectDataBase::populateElementTable() /** @brief projectDataBase::populateElementInfoTable Populate the element info table from the symbol information store - (DB-ACCESSORS-PLAN.md stage 4.3), in the order of the element table. + in the order of the element table. The label column is the label a symbol shows, which its folio's numbering decides: the symbol holding the uuid gives it. diff --git a/sources/dataBase/projectdatabase.h b/sources/dataBase/projectdatabase.h index faf92ba6f..be3322c7d 100644 --- a/sources/dataBase/projectdatabase.h +++ b/sources/dataBase/projectdatabase.h @@ -114,7 +114,7 @@ class projectDataBase : public QObject //The information of every symbol placed in the project, by //symbol uuid: the store each symbol's information is written - //through (DB-ACCESSORS-PLAN.md stage 4.1). A symbol not placed + //through. A symbol not placed //(a preview, the symbol editor) keeps its own. DiagramContext elementInformation(const QUuid &element) const; bool hasElementInformation(const QUuid &element) const; @@ -132,7 +132,7 @@ class projectDataBase : public QObject //The title block properties of every folio of the project, by //folio uuid: the store each folio's title block is written - //through (DB-ACCESSORS-PLAN.md stage 4.4). A folio not in a + //through. A folio not in a //project keeps its own. TitleBlockProperties folioTitleBlock(const QUuid &folio) const; const TitleBlockProperties *storedFolioTitleBlock(const QUuid &folio) const; @@ -144,7 +144,7 @@ class projectDataBase : public QObject //The properties of every wire placed in the project, by wire //uuid: the store each wire's properties are written through - //(DB-ACCESSORS-PLAN.md stage 4.5). A wire not placed keeps its own. + //A wire not placed keeps its own. ConductorProperties conductorProperties(const QUuid &conductor) const; bool hasConductorProperties(const QUuid &conductor) const; int placedConductorCount(const QUuid &conductor) const; @@ -153,7 +153,7 @@ class projectDataBase : public QObject QStringList conductorPropertiesMismatches() const; //Fill the three stores from a project's document before its - //folios are built (DB-ACCESSORS-PLAN.md stage 4.6), then, once + //folios are built, then, once //they are, say how many of those values building agreed with. void prefillFromDocument(const QDomDocument &document); void endPrefill(bool keep_unplaced = false); diff --git a/sources/diagram.h b/sources/diagram.h index 6b2e6c604..44748c181 100644 --- a/sources/diagram.h +++ b/sources/diagram.h @@ -211,7 +211,7 @@ class Diagram : public QGraphicsScene bool = true, DiagramContent * = nullptr); /// false while this folio's items wait in its XML to be - /// built (QET_LAZY_FOLIOS, LAZY-FOLIO-PLAN.md) + /// built (QET_LAZY_FOLIOS) bool isLoaded() const {return m_deferred.isNull();} QDomElement deferredXml() const {return m_deferred;} QString headerXml(); diff --git a/sources/qetgraphicsitem/conductor.cpp b/sources/qetgraphicsitem/conductor.cpp index 720db396c..339c4a243 100644 --- a/sources/qetgraphicsitem/conductor.cpp +++ b/sources/qetgraphicsitem/conductor.cpp @@ -1857,7 +1857,7 @@ ConductorProperties Conductor::properties() const /** @brief Conductor::placedStore @return the project database holding this wire's properties - (DB-ACCESSORS-PLAN.md stage 4.5), or nullptr if it is not placed in a + or nullptr if it is not placed in a project, or shares its uuid with another wire for the moment a copy takes to get its own */ diff --git a/sources/qetgraphicsitem/element.cpp b/sources/qetgraphicsitem/element.cpp index 779c6f7c9..6a59bbcbc 100644 --- a/sources/qetgraphicsitem/element.cpp +++ b/sources/qetgraphicsitem/element.cpp @@ -1859,7 +1859,7 @@ void Element::setUuid(const QUuid &uuid) /** @brief Element::storeInformation Write this symbol's information to its project's store, once it is - placed (DB-ACCESSORS-PLAN.md stage 4.1): every change of a placed + placed: every change of a placed symbol's information comes through here. */ void Element::storeInformation() @@ -1925,7 +1925,7 @@ ElementData Element::elementData() const /** @brief Element::elementInformations @return this symbol's information: for a placed symbol the project's - store holds it (DB-ACCESSORS-PLAN.md stage 4.2), for one not placed + store holds it, for one not placed (a preview, the symbol editor) its own copy. */ DiagramContext Element::elementInformations() const diff --git a/sources/qetproject.cpp b/sources/qetproject.cpp index 676f20efb..7cdc79970 100644 --- a/sources/qetproject.cpp +++ b/sources/qetproject.cpp @@ -544,7 +544,7 @@ QETProject::ProjectState QETProject::state() const QList QETProject::diagrams() const { //Every caller of diagrams() expects every folio built: those that - //do not are moved to folios() one by one (LAZY-FOLIO-PLAN.md) + //do not are moved to folios() one by one loadFolios(); return(m_diagrams_list); } @@ -2291,8 +2291,7 @@ QDomDocument QETProject::toXml(QHash *kept_folios) loadFolios(); } //QET_CHECK_ELEMENT_INFO_STORE=1: say whether the stores of symbol - //information, folio title blocks and wire properties - //(DB-ACCESSORS-PLAN.md) agree with every placed symbol, folio and + //information, folio title blocks and wire properties agree with every placed symbol, folio and //wire, at each save. A test and debugging aid; it changes nothing. if (qEnvironmentVariableIntValue("QET_CHECK_ELEMENT_INFO_STORE") == 1) { const QStringList mismatches = m_data_base.elementInformationMismatches(); @@ -3015,7 +3014,7 @@ void QETProject::readProjectXml(QDomDocument &xml_project) //The stores of symbol information, folio title blocks and wire //properties are filled from the document before the folios are - //built (DB-ACCESSORS-PLAN.md stage 4.6) + //built m_data_base.prefillFromDocument(xml_project); //QET_LAZY_FOLIOS leaves the folios to be built when needed only @@ -3070,7 +3069,6 @@ void QETProject::readProjectXml(QDomDocument &xml_project) //number can hold what a save rewrites (tst_freetextnonfinite, //tst_foliofreezeflags): only with QET_LAZY_SAVE_KEPT=1, until a //.qetz, whose manifest names its writer, does it - //(QETZ-DB-TRUTH-PLAN.md D4) m_kept_folios_trusted = qEnvironmentVariableIntValue("QET_LAZY_SAVE_KEPT") == 1 && unloadedFolioCount() > 0 && m_project_qet_version == QetVersion::currentVersion() diff --git a/sources/qetproject.h b/sources/qetproject.h index 9bdc240c7..97987d1a9 100644 --- a/sources/qetproject.h +++ b/sources/qetproject.h @@ -122,7 +122,7 @@ class QETProject : public QObject ProjectState state() const; QList diagrams() const; //The folios whether their items are built or not - //(QET_LAZY_FOLIOS, LAZY-FOLIO-PLAN.md); diagrams() builds them + //(QET_LAZY_FOLIOS); diagrams() builds them QList folios() const; bool deferringFolios() const {return m_deferring_folios;} int unloadedFolioCount() const; diff --git a/tests/qttest/lazy_folios_probe.cpp b/tests/qttest/lazy_folios_probe.cpp index 94f238a35..2ed2d085e 100644 --- a/tests/qttest/lazy_folios_probe.cpp +++ b/tests/qttest/lazy_folios_probe.cpp @@ -1,5 +1,5 @@ // SPDX-License-Identifier: GPL-2.0-or-later -// Lazy folios (QET_LAZY_FOLIOS, LAZY-FOLIO-PLAN.md stages 5.2 and 5.3), +// Lazy folios (QET_LAZY_FOLIOS), // linked against the application's objects: a folio shown in a project // opened without building its folios draws and links as in the project // opened as usual, building only it and the folios its links and tables diff --git a/tests/qttest/tst_resaveunchanged.cpp b/tests/qttest/tst_resaveunchanged.cpp index 1babd2c55..c36ac8127 100644 --- a/tests/qttest/tst_resaveunchanged.cpp +++ b/tests/qttest/tst_resaveunchanged.cpp @@ -340,8 +340,8 @@ private slots: return QString::fromUtf8(proc.readAllStandardOutput() + proc.readAllStandardError()); } - // The stores of symbol information and folio title blocks - // (DB-ACCESSORS-PLAN.md stages 4.1, 4.4) agree with every placed symbol + // The stores of symbol information and folio title blocks agree with + // every placed symbol // and folio once a project is open: every example. void elementInfoStoreAgrees_data() { @@ -361,7 +361,7 @@ private slots: } // The stores filled from the document before the folios are built - // (DB-ACCESSORS-PLAN.md stage 4.6) hold what building them gives, for a + // hold what building them gives, for a // project saved by this version (every symbol, folio and wire carries // a uuid): nothing differs, nothing is left unplaced. void storesFilledBeforeBuilding()