From ecc7a39131162031a9e614c7cf0517604085a9b7 Mon Sep 17 00:00:00 2001 From: ispyisail Date: Sun, 11 Oct 2026 17:09:31 +1300 Subject: [PATCH 01/26] Save as .qetz by default; a .qet opened is saved beside it (stage D1) The zipped project (.qetz) is the format QElectroTech saves. - Save As and the first Save of a new project offer .qetz first ("untitled.qetz"); .qet can still be chosen. - A project read from a .qet is saved as a .qetz beside it, the .qet left as it is (QETProject::write(), so Save, autosave, a script's save and the assistant's live save all do it). The project then works on the .qetz. A path chosen afterwards (Save As, a script naming one) is saved to as it is. The editor says so before the first such save, and asks before replacing a .qetz already there. - File > Save a copy for an older QElectroTech writes a .qet copy, for a version that cannot open a .qetz; the project keeps its own file. - The operating systems know the file: a freedesktop MIME type application/x-qelectrotech-project+zip (*.qetz, and by its mimetype entry when the name says nothing), the desktop file, macOS's document types and the Windows installer's association. Crash backups stay XML, written beside the project as before. tst_resaveunchanged::qetSavedAsQetzBeside: a .qet saved where it is leaves the .qet byte for byte and writes a .qetz beside it holding what a plain save of the .qet holds; it fails when the switch is off. In a real window: Ctrl+S on perceuse.qet shows the notice, writes perceuse.qetz and leaves perceuse.qet's md5 unchanged; the copy for an older version (reached by command search) equals a plain save but for time_spent; a new project's Save offers untitled.qetz. xdg-mime finds the type by name and by content. ctest 96/96 with and without QET_LAZY_FOLIOS=1; srclint R-C clean. Co-Authored-By: Claude Opus 5.5 --- build-aux/windows/QET64.nsi | 4 ++ misc/Info.plist | 1 + misc/org.qelectrotech.qelectrotech.desktop | 2 +- misc/qelectrotech.xml | 8 +++ sources/projectview.cpp | 57 +++++++++++++++++++++- sources/projectview.h | 1 + sources/qetdiagrameditor.cpp | 16 ++++++ sources/qetdiagrameditor.h | 1 + sources/qetproject.cpp | 48 +++++++++++++++++- sources/qetproject.h | 8 +++ tests/qttest/tst_resaveunchanged.cpp | 34 +++++++++++++ 11 files changed, 175 insertions(+), 5 deletions(-) diff --git a/build-aux/windows/QET64.nsi b/build-aux/windows/QET64.nsi index fc79b780a..529efce82 100644 --- a/build-aux/windows/QET64.nsi +++ b/build-aux/windows/QET64.nsi @@ -338,6 +338,9 @@ Section "" WriteRegStr HKEY_CLASSES_ROOT "qet_diagram_file\DefaultIcon" "" "$final_project_ico" WriteRegStr HKEY_CLASSES_ROOT "qet_diagram_file\shell\open\command" "" '"$final_qet_exe" "%1"' + ; File associations - .qetz, the zipped project: the same file type + WriteRegStr HKEY_CLASSES_ROOT ".qetz" "" "qet_diagram_file" + ; File associations - .elmt WriteRegStr HKEY_CLASSES_ROOT ".elmt" "" "qet_element_file" WriteRegStr HKEY_CLASSES_ROOT "qet_element_file" "" "Element QET" @@ -486,6 +489,7 @@ Section "Uninstall" ; Remove file associations registry keys DeleteRegKey HKEY_CLASSES_ROOT "Applications\qelectrotech.exe" DeleteRegKey HKEY_CLASSES_ROOT ".qet" + DeleteRegKey HKEY_CLASSES_ROOT ".qetz" DeleteRegKey HKEY_CLASSES_ROOT "qet_diagram_file" DeleteRegKey HKEY_CLASSES_ROOT ".elmt" DeleteRegKey HKEY_CLASSES_ROOT "qet_element_file" diff --git a/misc/Info.plist b/misc/Info.plist index 3d138a781..c41a2f03f 100644 --- a/misc/Info.plist +++ b/misc/Info.plist @@ -32,6 +32,7 @@ CFBundleTypeExtensions qet + qetz CFBundleTypeRole Editor diff --git a/misc/org.qelectrotech.qelectrotech.desktop b/misc/org.qelectrotech.qelectrotech.desktop index d76e23964..86876992c 100644 --- a/misc/org.qelectrotech.qelectrotech.desktop +++ b/misc/org.qelectrotech.qelectrotech.desktop @@ -6,7 +6,7 @@ Exec=qelectrotech %F Icon=qelectrotech Terminal=false Type=Application -MimeType=application/x-qet-project;application/x-qet-element;application/x-qet-titleblock; +MimeType=application/x-qet-project;application/x-qelectrotech-project+zip;application/x-qet-element;application/x-qet-titleblock; Categories=Graphics; Keywords=Graphics;Science;Electricity;Engineering; Comment=Edit electrical diagrams. diff --git a/misc/qelectrotech.xml b/misc/qelectrotech.xml index 9c59b116b..cefe25eb5 100644 --- a/misc/qelectrotech.xml +++ b/misc/qelectrotech.xml @@ -34,6 +34,14 @@ QElectroTech 项目文件 + + + QElectroTech zipped project file + + + + + QElectroTech element file diff --git a/sources/projectview.cpp b/sources/projectview.cpp index 47bdc0719..ab62a3591 100644 --- a/sources/projectview.cpp +++ b/sources/projectview.cpp @@ -341,13 +341,15 @@ QString ProjectView::askUserForFilePath(bool assign) { // ask the user for a filepath in order to save the project const QString qet_filter = tr("Project QElectroTech (*.qet)", "filetypes allowed when saving a project file"); const QString qetz_filter = tr("Zipped project QElectroTech (*.qetz)", "filetypes allowed when saving a project file"); - QString selected_filter = QETProject::isContainerPath(m_project->filePath()) ? qetz_filter : qet_filter; + //A project is saved as a .qetz unless a .qet is chosen + //(stage D1) + QString selected_filter = qetz_filter; QString filepath = QFileDialog::getSaveFileName( this, tr("Save as", "dialog title"), m_project -> currentDir() + "/" + tr("untitled") + (selected_filter == qetz_filter ? ".qetz" : ".qet"), - qet_filter + ";;" + qetz_filter, + qetz_filter + ";;" + qet_filter, &selected_filter ); @@ -725,6 +727,32 @@ QETResult ProjectView::saveAs() return(doSave()); } +/** + @brief ProjectView::saveCopyForOlderVersions + Ask for a .qet path and write the project there, for a version of + QElectroTech that cannot open a .qetz. The project keeps its file. + @return a QETResult object reflecting the situation; a valid one if + the operation was cancelled +*/ +QETResult ProjectView::saveCopyForOlderVersions() +{ + if (!m_project) return(noProjectResult()); + + QString base = m_project->filePath().isEmpty() + ? m_project->currentDir() + "/" + tr("untitled") + : m_project->filePath(); + for (const QString &suffix : {QStringLiteral(".qetz"), QStringLiteral(".qet")}) + if (base.endsWith(suffix, Qt::CaseInsensitive)) base.chop(suffix.size()); + QString path = QFileDialog::getSaveFileName( + this, + tr("Save a copy for an older QElectroTech", "dialog title"), + base + ".qet", + tr("Project QElectroTech (*.qet)", "filetypes allowed when saving a project file")); + if (path.isEmpty()) return(QETResult()); + if (!path.endsWith(".qet", Qt::CaseInsensitive)) path += ".qet"; + return(m_project->writeCopyForOlderVersions(path)); +} + /** Save project content, then write the project file. May call saveAs if no filepath was provided before. @@ -741,6 +769,31 @@ QETResult ProjectView::doSave() return(saveAs()); } + //A project read from a .qet is saved as a .qetz beside it: say so, + //and ask before replacing a .qetz already there + if (m_project->savesBesideOpenedFile()) { + const QString beside = m_project->besideContainerPath(); + const QString message = tr("This project was opened from %1. It is saved as %2, " + "beside it, in the zipped project format; %1 stays as " + "it is.\n\nTo write a file for a version of " + "QElectroTech that cannot open %2, use File > Save a " + "copy for an older QElectroTech.") + .arg(QFileInfo(m_project->filePath()).fileName(), QFileInfo(beside).fileName()); + if (QFileInfo::exists(beside)) { + if (QET::QetMessageBox::question( + this, tr("Save as a zipped project", "message box title"), + message + QStringLiteral("\n\n") + + tr("%1 already exists. Replace it?").arg(QFileInfo(beside).fileName()), + QMessageBox::Yes | QMessageBox::Cancel, QMessageBox::Cancel) + != QMessageBox::Yes) { + return(QETResult()); + } + } else { + QET::QetMessageBox::information( + this, tr("Save as a zipped project", "message box title"), message); + } + } + // write to file QETResult result = m_project -> write(); updateWindowTitle(); diff --git a/sources/projectview.h b/sources/projectview.h index dfbd03987..eec92dcc7 100644 --- a/sources/projectview.h +++ b/sources/projectview.h @@ -131,6 +131,7 @@ class ProjectView : public QWidget QETResult save(); QETResult saveAs(); QETResult doSave(); + QETResult saveCopyForOlderVersions(); int cleanProject(); void updateWindowTitle(); void updateTabTitle(DiagramView *); diff --git a/sources/qetdiagrameditor.cpp b/sources/qetdiagrameditor.cpp index 8c9924606..ab25c619a 100644 --- a/sources/qetdiagrameditor.cpp +++ b/sources/qetdiagrameditor.cpp @@ -978,6 +978,8 @@ void QETDiagramEditor::setUpActions() QAction *open_file = m_file_actions_group.addAction(QET::Icons::DocumentOpen, tr("&Open")); m_save_file = m_file_actions_group.addAction(QET::Icons::DocumentSave, tr("&Save")); m_save_file_as = m_file_actions_group.addAction(QET::Icons::DocumentSaveAs, tr("Save as")); + m_save_copy_for_older = m_file_actions_group.addAction(QET::Icons::DocumentSaveAs, + tr("Save a copy for an older QElectroTech")); m_close_file = m_file_actions_group.addAction(QET::Icons::ProjectClose, tr("&Close")); ShortcutManager::instance().registerAction(new_file, "diagrameditor.new_file", tr("Diagram editors"), QKeySequence::New); @@ -991,6 +993,19 @@ void QETDiagramEditor::setUpActions() m_close_file ->setStatusTip( tr("Closes the current project", "status bar tip") ); m_save_file ->setStatusTip( tr("Saves the current project and all its sheets", "status bar tip") ); m_save_file_as ->setStatusTip( tr("Saves the current project with a different file name", "status bar tip") ); + m_save_copy_for_older->setStatusTip(tr("Writes a copy of the current project as a .qet file, which " + "versions of QElectroTech that cannot open a .qetz open", + "status bar tip")); + ShortcutManager::instance().registerAction(m_save_copy_for_older, "diagrameditor.save_copy_for_older", + tr("Diagram editors"), QKeySequence()); + connect(m_save_copy_for_older, &QAction::triggered, this, [this]() { + if (ProjectView *project_view = currentProjectView()) { + const QETResult result = project_view->saveCopyForOlderVersions(); + if (!result.isOk()) { + showError(result); + } + } + }); connect(m_save_file_as, &QAction::triggered, this, &QETDiagramEditor::saveAs); connect(m_save_file, &QAction::triggered, this, &QETDiagramEditor::save); @@ -2750,6 +2765,7 @@ void QETDiagramEditor::slot_updateActions() m_close_file-> setEnabled(opened_project); m_save_file-> setEnabled(opened_project); m_save_file_as-> setEnabled(opened_project); + m_save_copy_for_older-> setEnabled(opened_project); m_rotate_texts-> setEnabled(editable_project); m_export_to_images-> setEnabled(opened_diagram); m_print-> setEnabled(opened_diagram); diff --git a/sources/qetdiagrameditor.h b/sources/qetdiagrameditor.h index 2ff471b3f..f9287cf07 100644 --- a/sources/qetdiagrameditor.h +++ b/sources/qetdiagrameditor.h @@ -301,6 +301,7 @@ class QETDiagramEditor : public QETMainWindow *m_close_file, ///< Close current project file *m_save_file, ///< Save current project *m_save_file_as, ///< Save current project as a specific file + *m_save_copy_for_older, ///< Write a .qet copy for an older QElectroTech *m_find = nullptr, *m_jump_to_element = nullptr; ///< Open the "jump to element" quick-open popup diff --git a/sources/qetproject.cpp b/sources/qetproject.cpp index 7cdc79970..92aeefc6c 100644 --- a/sources/qetproject.cpp +++ b/sources/qetproject.cpp @@ -192,6 +192,8 @@ QETProject::QETProject(KAutoSaveFile *backup, QObject *parent) : } //Set the real path, instead of the path of the backup. setFilePath(backup->managedFile().path()); + //A project read from a .qet is saved as a .qetz beside it + m_opened_from_qet = !isContainerPath(m_file_path); delete backup; //Set the project to read only mode if the file it. @@ -437,8 +439,12 @@ QETProject::ProjectState QETProject::openFile(QFile *file) //Extract the content of the xml. The raw bytes are kept : a project //file without a persisted uuid derives its uuid from them. const QByteArray content = file->readAll(); - //Each folio's text is kept while it is not built (readDiagramsXml()) - m_opened_content = content; + //Each folio's text is kept while it is not built (readDiagramsXml()): + //from a .qet only, a .qetz is a zip of parts + if (!content.startsWith("PK\x03\x04")) + m_opened_content = content; + //A project read from a .qet is saved as a .qetz beside it + m_opened_from_qet = !content.startsWith("PK\x03\x04"); QDomDocument xml_project; // PreserveSpacingOnlyNodes: without it, a text node that is entirely // whitespace -- e.g. a title-block custom variable deliberately set to @@ -1007,6 +1013,8 @@ void QETProject::setFilePath(const QString &filepath) } m_next_backup_slot = 0; m_file_path = filepath; + //A path chosen is saved to as it is + m_opened_from_qet = false; QFileInfo fi(m_file_path); if (fi.isWritable()) { @@ -2472,12 +2480,48 @@ bool QETProject::writeContainer(const QDomDocument &project, const QString &path return !entries.isEmpty() && QetZip::write(path, entries, error); } +/** + @brief QETProject::besideContainerPath + @return the .qetz a project read from a .qet is saved to: the same + name, beside it +*/ +QString QETProject::besideContainerPath() const +{ + QString path = m_file_path; + if (path.endsWith(QStringLiteral(".qet"), Qt::CaseInsensitive)) { + path.chop(4); + } + return path + QStringLiteral(".qetz"); +} + +/** + @brief QETProject::writeCopyForOlderVersions + Write this project as a .qet at @p path, which a QElectroTech that + reads no .qetz opens. The project keeps its own file and state. +*/ +QETResult QETProject::writeCopyForOlderVersions(const QString &path) +{ + QHash kept_folios; + QDomDocument xml_project(toXml(&kept_folios)); + QString error_message; + if (!QET::writeXmlText(spliceKeptFolios(xml_project.toString(4), kept_folios), + path, &error_message)) + return(error_message); + return(QETResult()); +} + QETResult QETProject::write() { // this operation requires a filepath if (m_file_path.isEmpty()) return(QString("unable to save project to file: no filepath was specified")); + //A project read from a .qet is saved as a .qetz beside it; the + //.qet stays as it was (stage D1) + if (m_opened_from_qet) { + setFilePath(besideContainerPath()); + } + // If the project was opened read-only, only refuse when the target // really can't be written: an existing file that is not writable, or a // new file (e.g. "Save As" to another location) whose directory is not diff --git a/sources/qetproject.h b/sources/qetproject.h index 97987d1a9..07d4abd51 100644 --- a/sources/qetproject.h +++ b/sources/qetproject.h @@ -436,6 +436,12 @@ class QETProject : public QObject ProjectState openFile(QFile *file); public: static bool isContainerPath(const QString &path); + /// Saving writes a .qetz: one read from a .qet is saved as + /// a .qetz beside it, the .qet left as it is + /// (stage D1) + bool savesBesideOpenedFile() const {return m_opened_from_qet;} + QString besideContainerPath() const; + QETResult writeCopyForOlderVersions(const QString &path); static bool writeContainer(const QDomDocument &project, const QString &path, QString *error = nullptr); private: @@ -520,6 +526,8 @@ class QETProject : public QObject /// database rows came from it, and nothing project-wide that /// a folio's saved items depend on has changed since bool m_kept_folios_trusted = false; + /// Read from a .qet whose path nobody chose since: saved beside it + bool m_opened_from_qet = false; bool m_kept_folios_stale = false; /// Something was edited since the project was opened: a table /// filled from the database may have changed diff --git a/tests/qttest/tst_resaveunchanged.cpp b/tests/qttest/tst_resaveunchanged.cpp index c36ac8127..c7d7bc409 100644 --- a/tests/qttest/tst_resaveunchanged.cpp +++ b/tests/qttest/tst_resaveunchanged.cpp @@ -479,6 +479,40 @@ private slots: QCOMPARE(rowsOf(edited), rowsOf(reopened)); } + // A project read from a .qet and saved where it is, is saved as a .qetz + // beside it, and the .qet is left as it was (stage D1). + // What the .qetz holds is what a plain save of the .qet holds. + void qetSavedAsQetzBeside() + { +#ifndef QET_HAS_SCRIPTING + QSKIP("needs --run: this QElectroTech is built without Qt Qml"); +#endif + const QString qet = m_dir.filePath(QStringLiteral("beside.qet")); + QFile::remove(qet); + QVERIFY(QFile::copy(QStringLiteral(QET_EXAMPLES_DIR "/industrial.qet"), qet)); + const QByteArray before = read(qet); + const QString script = m_dir.filePath(QStringLiteral("beside.js")); + QFile js(script); + QVERIFY(js.open(QIODevice::WriteOnly)); + js.write("qet.log('SAVED ' + qet.save(''));\n"); + js.close(); + const QString log = runChecked({QStringLiteral("--run"), script, qet}); + QVERIFY2(log.contains(QStringLiteral("SAVED true")), qPrintable(log.right(400))); + + QCOMPARE(read(qet), before); + const QString qetz = m_dir.filePath(QStringLiteral("beside.qetz")); + QVERIFY2(read(qetz).startsWith("PK\x03\x04"), "no .qetz beside the .qet"); + + auto without_save_stamps = [](QByteArray xml) { + static const QRegularExpression stamp(QStringLiteral( + "\\s*]*name=\"saved(filename|filepath|filenamedir|date|date-us|date-eu|time)\"[^>]*>[^<]*")); + return QString::fromUtf8(xml).remove(stamp); + }; + const QString straight = resave(qet); + const QString back = resave(qetz); + QVERIFY(!straight.isEmpty() && !back.isEmpty()); + QCOMPARE(without_save_stamps(read(back)), without_save_stamps(read(straight))); + } // A title-block value that is a single space is kept through two saves // (#973), and a value with accents comes back as it went in. From e89ccf2a4089715efce29d916e4b67af07506902 Mon Sep 17 00:00:00 2001 From: ispyisail Date: Sun, 11 Oct 2026 18:58:35 +1300 Subject: [PATCH 02/26] Open a .qetz from its database, each folio read when built (stage D2) A .qetz now carries index.sqlite: the project database as QElectroTech keeps it -- every table but the drawing items, which exist for built folios only -- with each folio's own data (Diagram::headerXml()) and a stamp of the schema it was written with (projectDataBase::indexFile(), QETProject::writeContainerFile(): Save, --resave, --set-titleblock, a script's and live mode's save). Written when every folio, symbol and wire has a uuid of its own; a reader that does not know the entry skips it. With QET_LAZY_FOLIOS=1, a .qetz with an index opens from it: the project document is joined without its folios (QetContainer::join() with folio_files), the database and the stores of symbol information and wire properties are filled from the index (fillFromIndex()), each folio is made from its own data and keeps the name of its part, read when the folio is built (Diagram::deferredXml(), QETProject::readFolioPart(), QetContainer::readFolio() -- pictures and engineering data back from project.sqlite, QetContainerDb::restoreFolio()). A terminal strip finds its terminals' folios in the database. Any other .qetz, or one whose index cannot be used, is read whole as before. The index is untrusted: refused unless written with this database's schema, every table read a plain table (not a view), nothing in its schema run (trusted_schema off); refused, the project is read whole. Checked: an index of garbage, a wrong stamp, a table made a view -- each refused and the project saved as a plain save. The 136-folio f2099 as .qetz opens in 1.8 s (3.65 s read whole, 29 s building every folio); what is left is mostly its 7 MB of symbol definitions (0.8 s). The file grows from 2.6 to 5.2 MB, the index overlapping project.sqlite until D3. tst_lazyfolios: a .qetz opened from its index builds only the terminal strip's folios, numbering read as when built, every folio shown in turn draws and links as when built, the save as when built; fails when a folio part's engineering data is not put back. Corpus, 342 projects: 340 .qetz carry an index and open from it (2 have symbols sharing a uuid); saved back as .qet = eager = master on all 342. Probe on industrial and f2099 .qetz: every folio as eager but the 3 render-noise folios of f2099. In a real window the .qetz opens and pages building only shown folios and partners. ctest 96/96 with and without QET_LAZY_FOLIOS=1. Co-Authored-By: Claude Opus 5.5 --- sources/TerminalStrip/terminalstrip.cpp | 5 +- sources/cli_export.cpp | 6 +- sources/container/qetcontainer.cpp | 50 ++++- sources/container/qetcontainer.h | 6 +- sources/container/qetcontainerdb.cpp | 258 +++++++++++++--------- sources/container/qetcontainerdb.h | 1 + sources/dataBase/projectdatabase.cpp | 278 ++++++++++++++++++++++++ sources/dataBase/projectdatabase.h | 7 + sources/diagram.cpp | 27 ++- sources/diagram.h | 9 +- sources/qetproject.cpp | 164 +++++++++++++- sources/qetproject.h | 13 +- sources/scripting/liveserver.cpp | 6 +- sources/scripting/qetscriptapi.cpp | 6 +- tests/qttest/lazy_folios_probe.cpp | 19 ++ 15 files changed, 719 insertions(+), 136 deletions(-) diff --git a/sources/TerminalStrip/terminalstrip.cpp b/sources/TerminalStrip/terminalstrip.cpp index 337f56766..cbb537cdb 100644 --- a/sources/TerminalStrip/terminalstrip.cpp +++ b/sources/TerminalStrip/terminalstrip.cpp @@ -1020,8 +1020,9 @@ bool TerminalStrip::fromXml(QDomElement &xml_element) terminals.insert(QUuid(xml_reals.at(i).toElement().attribute(QStringLiteral("element_uuid")))); m_project->buildFoliosHoldingSymbols(terminals); - //Get all free elements terminal of the project - const ElementProvider ep(m_project); + //Get all free elements terminal of the project: its own are on + //the folios just built + const ElementProvider ep(m_project->builtFolios()); auto free_terminals = ep.freeTerminal(); //Read each physical terminal diff --git a/sources/cli_export.cpp b/sources/cli_export.cpp index 80e7f1aa4..cdb00096f 100644 --- a/sources/cli_export.cpp +++ b/sources/cli_export.cpp @@ -870,9 +870,8 @@ int exportLinks(QETProject &project, const QString &output) int resaveProject(QETProject &project, const QString &output) { if (QETProject::isContainerPath(output)) { - const QDomDocument doc = project.toXml(); QString error; - if (!QETProject::writeContainer(doc, output, &error)) { + if (!project.writeContainerFile(output, &error)) { err << "Cannot write '" << output << "': " << error << "\n"; return 1; } @@ -968,9 +967,8 @@ int setTitleBlock(QETProject &project, const QString &output, } if (QETProject::isContainerPath(output)) { - const QDomDocument doc = project.toXml(); QString error; - if (!QETProject::writeContainer(doc, output, &error)) { + if (!project.writeContainerFile(output, &error)) { err << "Cannot write '" << output << "': " << error << "\n"; return 1; } diff --git a/sources/container/qetcontainer.cpp b/sources/container/qetcontainer.cpp index 7f27789c4..3cb20fd84 100644 --- a/sources/container/qetcontainer.cpp +++ b/sources/container/qetcontainer.cpp @@ -258,6 +258,43 @@ QList QetContainer::split(const QDomDocument &project, const QStr return entries; } +/** + @brief QetContainer::readFolio + The folio part @p file of the .qetz whose entries are @p files, as + join() puts it in the project: its pictures back as base64 and its + engineering data back from project.sqlite. +*/ +bool QetContainer::readFolio(const QHash &files, const QString &file, + QDomDocument *folio, QString *error) +{ + if (!files.contains(file)) { + setError(error, QStringLiteral("part \"%1\" is missing").arg(file)); + return false; + } + QString part_error; + if (!parse(files.value(file), folio, &part_error)) { + setError(error, QStringLiteral("%1: %2").arg(file, part_error)); + return false; + } + const QDomNodeList images = folio->elementsByTagName(QStringLiteral("image")); + for (int i = 0 ; i < images.size() ; ++i) + { + QDomElement image = images.at(i).toElement(); + if (!image.hasAttribute(ImageFileAttribute)) continue; + const QString picture = image.attribute(ImageFileAttribute); + if (!files.contains(picture)) { + setError(error, QStringLiteral("picture \"%1\" is missing").arg(picture)); + return false; + } + image.removeAttribute(ImageFileAttribute); + image.insertBefore(folio->createTextNode(QString::fromLatin1(files.value(picture).toBase64())), + image.firstChild()); + } + return !files.contains(QStringLiteral("project.sqlite")) + || QetContainerDb::restoreFolio(folio->documentElement(), + files.value(QStringLiteral("project.sqlite")), error); +} + /** @brief QetContainer::join Rebuild in @p project the document split() made @p entries from. @@ -265,7 +302,8 @@ QList QetContainer::split(const QDomDocument &project, const QStr (the manifest's min-reader), or a part missing or unreadable. Entries nothing names are ignored, so a later format may add some. */ -bool QetContainer::join(const QList &entries, QDomDocument *project, QString *error) +bool QetContainer::join(const QList &entries, QDomDocument *project, QString *error, + QStringList *folio_files) { QHash files; for (const QetZip::Entry &entry : entries) @@ -297,6 +335,16 @@ bool QetContainer::join(const QList &entries, QDomDocument *proje for (QDomElement part : placeholders) { const QString file = part.attribute(FileAttribute); + //With @p folio_files, the folios stay parts, read when built + if (folio_files && part.parentNode() == project->documentElement() + && file.startsWith(QLatin1String("folios/"))) { + if (!files.contains(file)) { + setError(error, QStringLiteral("part \"%1\" is missing").arg(file)); + return false; + } + folio_files->append(file); + continue; + } if (!files.contains(file)) { setError(error, QStringLiteral("part \"%1\" is missing").arg(file)); return false; diff --git a/sources/container/qetcontainer.h b/sources/container/qetcontainer.h index d92df084c..ad050efb4 100644 --- a/sources/container/qetcontainer.h +++ b/sources/container/qetcontainer.h @@ -53,7 +53,11 @@ namespace QetContainer const QString &writer = QString(), QString *error = nullptr); bool join(const QList &entries, QDomDocument *project, - QString *error = nullptr); + QString *error = nullptr, QStringList *folio_files = nullptr); + /// A folio part of @p files, which join() left out, with its + /// pictures and its engineering data back (stage D2) + bool readFolio(const QHash &files, const QString &file, + QDomDocument *folio, QString *error = nullptr); /// Parse @p xml as a .qet is read (QETProject::openFile) bool parse(const QByteArray &xml, QDomDocument *document, QString *error = nullptr); diff --git a/sources/container/qetcontainerdb.cpp b/sources/container/qetcontainerdb.cpp index 321fddd49..e5c48c121 100644 --- a/sources/container/qetcontainerdb.cpp +++ b/sources/container/qetcontainerdb.cpp @@ -434,6 +434,143 @@ QStringList keys(const QList &items, const QString &prefix) } } +namespace { + //@p database opened read only in @p db, its tables checked to be plain + //tables: a file is untrusted +bool openDatabase(Database &db, const QByteArray &database, QString *error) +{ + if (sqlite3_open(":memory:", &db.m_db) != SQLITE_OK) { + setError(error, QStringLiteral("cannot open the project database")); + return false; + } + unsigned char *copy = static_cast(sqlite3_malloc64(sqlite3_uint64(database.size()))); + if (!copy) { setError(error, QStringLiteral("out of memory")); return false; } + memcpy(copy, database.constData(), size_t(database.size())); + if (sqlite3_deserialize(db.m_db, "main", copy, database.size(), database.size(), + SQLITE_DESERIALIZE_FREEONCLOSE | SQLITE_DESERIALIZE_READONLY) != SQLITE_OK) { + setError(error, QStringLiteral("the project database cannot be read")); + return false; + } + //A file is untrusted: nothing in its schema may run, and every name + //read must be a plain table. + db.exec(QStringLiteral("PRAGMA trusted_schema = OFF")); + db.exec(QStringLiteral("PRAGMA query_only = ON")); + for (const char *table : {"meta", "project_property", "folio", "folio_property", + "element", "element_info", "link", "conductor", "terminal_strip", + "strip_terminal", "strip_bridge", "strip_bridge_terminal"}) { + Statement kind(db.m_db, QStringLiteral("SELECT type FROM sqlite_master WHERE name = ?")); + kind.bind({QString::fromLatin1(table)}); + if (!kind.next() || kind.value(0).toString() != QLatin1String("table")) { + setError(error, QStringLiteral("the project database has no table %1").arg(QLatin1String(table))); + return false; + } + } + return true; +} + +QDomElement entryElement(QDomDocument &document, const QVariant &name_attr, const QString &name, + const QVariant &value, const QVariant &attrs, const QString &default_tag) +{ + const QJsonObject o = QJsonDocument::fromJson(attrs.toString().toUtf8()).object(); + QDomElement e = document.createElement(o.value(QStringLiteral("tag")).toString(default_tag)); + if (!name_attr.isNull()) e.setAttribute(name, name_attr.toString()); + setAttributesJson(e, attrs); + if (!value.isNull()) e.appendChild(document.createTextNode(value.toString())); + return e; +} + + //The engineering data of the folio @p d (its qetz-folio id) back from @p db +bool restoreDiagram(Database &db, QDomElement d, QString *error) +{ + QDomDocument document = d.ownerDocument(); + auto entry = [&document](const QVariant &name_attr, const QString &name, const QVariant &value, + const QVariant &attrs, const QString &default_tag) { + return entryElement(document, name_attr, name, value, attrs, default_tag); + }; + const QString id = d.attribute(FolioAttribute); + if (id.isEmpty()) return true; + d.removeAttribute(FolioAttribute); + Statement f(db.m_db, QStringLiteral("SELECT %1, props_at, props_attrs FROM folio WHERE id = ?") + .arg(QStringLiteral("\"") + FolioData.join(QStringLiteral("\", \"")) + QStringLiteral("\""))); + f.bind({id}); + if (!f.next()) { setError(error, QStringLiteral("folio %1 is not in the database").arg(id)); return false; } + QVariantList values; + for (int i = 0 ; i < FolioData.size() + 2 ; ++i) values << f.value(i); + put(d, FolioData, values, 0); + if (!values.at(FolioData.size()).isNull()) { + QDomElement props = document.createElement(QStringLiteral("properties")); + setAttributesJson(props, values.at(FolioData.size() + 1)); + Statement p(db.m_db, QStringLiteral("SELECT name, value, attrs FROM folio_property WHERE folio_id = ? ORDER BY ord")); + p.bind({id}); + while (p.next()) + props.appendChild(entry(p.value(0), QStringLiteral("name"), p.value(1), p.value(2), + QStringLiteral("property"))); + restoreWhitespace(props, values.at(FolioData.size() + 1)); + if (!replaceMarker(d, props)) { + setError(error, QStringLiteral("folio %1's properties have no place").arg(id)); + return false; + } + } + + for (const QDomElement &list : children(d, QStringLiteral("elements"))) { + for (QDomElement e : children(list, QStringLiteral("element"))) { + if (!e.hasAttribute(KeyAttribute)) continue; + const QString key = e.attribute(KeyAttribute); + e.removeAttribute(KeyAttribute); + Statement s(db.m_db, QStringLiteral("SELECT %1, info_at, info_attrs, links_at, links_attrs FROM element WHERE key = ?") + .arg(QStringLiteral("\"") + ElementData.join(QStringLiteral("\", \"")) + QStringLiteral("\""))); + s.bind({key}); + if (!s.next()) { setError(error, QStringLiteral("symbol %1 is not in the database").arg(key)); return false; } + QVariantList v; + for (int i = 0 ; i < ElementData.size() + 4 ; ++i) v << s.value(i); + put(e, ElementData, v, 0); + const int n = ElementData.size(); + QDomElement info, links; + if (!v.at(n).isNull()) { + info = document.createElement(QStringLiteral("elementInformations")); + setAttributesJson(info, v.at(n + 1)); + Statement i(db.m_db, QStringLiteral("SELECT name, value, attrs FROM element_info WHERE element_key = ? ORDER BY ord")); + i.bind({key}); + while (i.next()) + info.appendChild(entry(i.value(0), QStringLiteral("name"), i.value(1), i.value(2), + QStringLiteral("elementInformation"))); + restoreWhitespace(info, v.at(n + 1)); + } + if (!v.at(n + 2).isNull()) { + links = document.createElement(QStringLiteral("links_uuids")); + setAttributesJson(links, v.at(n + 3)); + Statement l(db.m_db, QStringLiteral("SELECT linked_uuid, attrs FROM link WHERE element_key = ? ORDER BY ord")); + l.bind({key}); + while (l.next()) + links.appendChild(entry(l.value(0), QStringLiteral("uuid"), QVariant(), l.value(1), + QStringLiteral("link_uuid"))); + restoreWhitespace(links, v.at(n + 3)); + } + if ((!info.isNull() && !replaceMarker(e, info)) + || (!links.isNull() && !replaceMarker(e, links))) { + setError(error, QStringLiteral("symbol %1's information has no place").arg(key)); + return false; + } + } + } + for (const QDomElement &list : children(d, QStringLiteral("conductors"))) { + for (QDomElement c : children(list, QStringLiteral("conductor"))) { + if (!c.hasAttribute(KeyAttribute)) continue; + const QString key = c.attribute(KeyAttribute); + c.removeAttribute(KeyAttribute); + Statement s(db.m_db, QStringLiteral("SELECT %1 FROM conductor WHERE key = ?") + .arg(QStringLiteral("\"") + ConductorData.join(QStringLiteral("\", \"")) + QStringLiteral("\""))); + s.bind({key}); + if (!s.next()) { setError(error, QStringLiteral("wire %1 is not in the database").arg(key)); return false; } + QVariantList v; + for (int i = 0 ; i < ConductorData.size() ; ++i) v << s.value(i); + put(c, ConductorData, v, 0); + } + } + return true; +} +} + /** @brief QetContainerDb::extract Move the engineering data of @p project (the element, its @@ -609,32 +746,8 @@ bool QetContainerDb::restore(QDomElement project, const QByteArray &database, QS { QDomDocument document = project.ownerDocument(); Database db; - if (sqlite3_open(":memory:", &db.m_db) != SQLITE_OK) { - setError(error, QStringLiteral("cannot open the project database")); + if (!openDatabase(db, database, error)) return false; - } - unsigned char *copy = static_cast(sqlite3_malloc64(sqlite3_uint64(database.size()))); - if (!copy) { setError(error, QStringLiteral("out of memory")); return false; } - memcpy(copy, database.constData(), size_t(database.size())); - if (sqlite3_deserialize(db.m_db, "main", copy, database.size(), database.size(), - SQLITE_DESERIALIZE_FREEONCLOSE | SQLITE_DESERIALIZE_READONLY) != SQLITE_OK) { - setError(error, QStringLiteral("the project database cannot be read")); - return false; - } - //A file is untrusted: nothing in its schema may run, and every name - //read must be a plain table. - db.exec(QStringLiteral("PRAGMA trusted_schema = OFF")); - db.exec(QStringLiteral("PRAGMA query_only = ON")); - for (const char *table : {"meta", "project_property", "folio", "folio_property", - "element", "element_info", "link", "conductor", "terminal_strip", - "strip_terminal", "strip_bridge", "strip_bridge_terminal"}) { - Statement kind(db.m_db, QStringLiteral("SELECT type FROM sqlite_master WHERE name = ?")); - kind.bind({QString::fromLatin1(table)}); - if (!kind.next() || kind.value(0).toString() != QLatin1String("table")) { - setError(error, QStringLiteral("the project database has no table %1").arg(QLatin1String(table))); - return false; - } - } auto entry = [&document](const QVariant &name_attr, const QString &name, const QVariant &value, const QVariant &attrs, const QString &default_tag) { @@ -687,87 +800,18 @@ bool QetContainerDb::restore(QDomElement project, const QByteArray &database, QS } for (QDomElement d : children(project, QStringLiteral("diagram"))) - { - const QString id = d.attribute(FolioAttribute); - if (id.isEmpty()) continue; - d.removeAttribute(FolioAttribute); - Statement f(db.m_db, QStringLiteral("SELECT %1, props_at, props_attrs FROM folio WHERE id = ?") - .arg(QStringLiteral("\"") + FolioData.join(QStringLiteral("\", \"")) + QStringLiteral("\""))); - f.bind({id}); - if (!f.next()) { setError(error, QStringLiteral("folio %1 is not in the database").arg(id)); return false; } - QVariantList values; - for (int i = 0 ; i < FolioData.size() + 2 ; ++i) values << f.value(i); - put(d, FolioData, values, 0); - if (!values.at(FolioData.size()).isNull()) { - QDomElement props = document.createElement(QStringLiteral("properties")); - setAttributesJson(props, values.at(FolioData.size() + 1)); - Statement p(db.m_db, QStringLiteral("SELECT name, value, attrs FROM folio_property WHERE folio_id = ? ORDER BY ord")); - p.bind({id}); - while (p.next()) - props.appendChild(entry(p.value(0), QStringLiteral("name"), p.value(1), p.value(2), - QStringLiteral("property"))); - restoreWhitespace(props, values.at(FolioData.size() + 1)); - if (!replaceMarker(d, props)) { - setError(error, QStringLiteral("folio %1's properties have no place").arg(id)); - return false; - } - } - - for (const QDomElement &list : children(d, QStringLiteral("elements"))) { - for (QDomElement e : children(list, QStringLiteral("element"))) { - if (!e.hasAttribute(KeyAttribute)) continue; - const QString key = e.attribute(KeyAttribute); - e.removeAttribute(KeyAttribute); - Statement s(db.m_db, QStringLiteral("SELECT %1, info_at, info_attrs, links_at, links_attrs FROM element WHERE key = ?") - .arg(QStringLiteral("\"") + ElementData.join(QStringLiteral("\", \"")) + QStringLiteral("\""))); - s.bind({key}); - if (!s.next()) { setError(error, QStringLiteral("symbol %1 is not in the database").arg(key)); return false; } - QVariantList v; - for (int i = 0 ; i < ElementData.size() + 4 ; ++i) v << s.value(i); - put(e, ElementData, v, 0); - const int n = ElementData.size(); - QDomElement info, links; - if (!v.at(n).isNull()) { - info = document.createElement(QStringLiteral("elementInformations")); - setAttributesJson(info, v.at(n + 1)); - Statement i(db.m_db, QStringLiteral("SELECT name, value, attrs FROM element_info WHERE element_key = ? ORDER BY ord")); - i.bind({key}); - while (i.next()) - info.appendChild(entry(i.value(0), QStringLiteral("name"), i.value(1), i.value(2), - QStringLiteral("elementInformation"))); - restoreWhitespace(info, v.at(n + 1)); - } - if (!v.at(n + 2).isNull()) { - links = document.createElement(QStringLiteral("links_uuids")); - setAttributesJson(links, v.at(n + 3)); - Statement l(db.m_db, QStringLiteral("SELECT linked_uuid, attrs FROM link WHERE element_key = ? ORDER BY ord")); - l.bind({key}); - while (l.next()) - links.appendChild(entry(l.value(0), QStringLiteral("uuid"), QVariant(), l.value(1), - QStringLiteral("link_uuid"))); - restoreWhitespace(links, v.at(n + 3)); - } - if ((!info.isNull() && !replaceMarker(e, info)) - || (!links.isNull() && !replaceMarker(e, links))) { - setError(error, QStringLiteral("symbol %1's information has no place").arg(key)); - return false; - } - } - } - for (const QDomElement &list : children(d, QStringLiteral("conductors"))) { - for (QDomElement c : children(list, QStringLiteral("conductor"))) { - if (!c.hasAttribute(KeyAttribute)) continue; - const QString key = c.attribute(KeyAttribute); - c.removeAttribute(KeyAttribute); - Statement s(db.m_db, QStringLiteral("SELECT %1 FROM conductor WHERE key = ?") - .arg(QStringLiteral("\"") + ConductorData.join(QStringLiteral("\", \"")) + QStringLiteral("\""))); - s.bind({key}); - if (!s.next()) { setError(error, QStringLiteral("wire %1 is not in the database").arg(key)); return false; } - QVariantList v; - for (int i = 0 ; i < ConductorData.size() ; ++i) v << s.value(i); - put(c, ConductorData, v, 0); - } - } - } + if (!restoreDiagram(db, d, error)) + return false; return true; } + +/** + @brief QetContainerDb::restoreFolio + restore() for the one folio @p diagram, read from its own part (a folio + built after its project was opened from a .qetz, stage D2) +*/ +bool QetContainerDb::restoreFolio(QDomElement diagram, const QByteArray &database, QString *error) +{ + Database db; + return openDatabase(db, database, error) && restoreDiagram(db, diagram, error); +} diff --git a/sources/container/qetcontainerdb.h b/sources/container/qetcontainerdb.h index a70dfb1fe..e43c87946 100644 --- a/sources/container/qetcontainerdb.h +++ b/sources/container/qetcontainerdb.h @@ -39,6 +39,7 @@ namespace QetContainerDb { QByteArray extract(QDomElement project, QString *error = nullptr); bool restore(QDomElement project, const QByteArray &database, QString *error = nullptr); + bool restoreFolio(QDomElement diagram, const QByteArray &database, QString *error = nullptr); } #endif // QETCONTAINERDB_H diff --git a/sources/dataBase/projectdatabase.cpp b/sources/dataBase/projectdatabase.cpp index 68a98486e..3ebe41382 100644 --- a/sources/dataBase/projectdatabase.cpp +++ b/sources/dataBase/projectdatabase.cpp @@ -49,6 +49,8 @@ #include #include #include +#include +#include #include #include #include @@ -865,6 +867,282 @@ bool projectDataBase::populateFromDocument(const QDomDocument &document, QString return true; } +/** + @brief projectDataBase::schemaStamp + @return a stamp of the tables of this database as created: an index + file (indexFile()) is read back only into a database with the same +*/ +QString projectDataBase::schemaStamp() const +{ + QSqlQuery query(m_data_base); + query.exec(QStringLiteral("SELECT sql FROM sqlite_master WHERE type IN ('table', 'index') " + "AND name NOT LIKE 'sqlite_%' ORDER BY name")); + QCryptographicHash hash(QCryptographicHash::Sha256); + while (query.next()) + hash.addData(query.value(0).toString().toUtf8() + ';'); + return QString::fromLatin1(hash.result().toHex()); +} + +namespace { + //The tables an index file carries, all of this database's but the + //drawing items, which exist for built folios only +QStringList indexTables(const QSqlDatabase &db) +{ + QStringList tables; + QSqlQuery query(db); + query.exec(QStringLiteral("SELECT name FROM sqlite_master WHERE type = 'table' " + "AND name NOT LIKE 'sqlite_%' ORDER BY name")); + while (query.next()) { + const QString name = query.value(0).toString(); + if (name != QLatin1String("shape") && name != QLatin1String("independent_text") + && name != QLatin1String("image")) + tables << name; + } + return tables; +} +} + +/** + @brief projectDataBase::indexFile + This database written to an SQLite file, for a .qetz to be opened from + (stage D2): every table but the drawing items, brought + up to date first, with the folios' own data @p folio_headers (in folio + order, each as Diagram::headerXml() writes it) and the stamp of the + schema it was written with. + @return the file, or nothing if it could not be written +*/ +QByteArray projectDataBase::indexFile(const QList> &folio_headers) +{ + if (m_content_changed) { + updateDB(); + } + flushLinks(); + flushElementPositions(); + flushConductorProperties(); + + QTemporaryDir dir; + if (!dir.isValid()) { + return QByteArray(); + } + const QString path = dir.filePath(QStringLiteral("index.sqlite")); + QString escaped = path; + escaped.replace(QLatin1Char('\''), QStringLiteral("''")); + QSqlQuery vacuum(m_data_base); + if (!vacuum.exec(QStringLiteral("VACUUM INTO '%1'").arg(escaped))) { + qWarning() << "projectDataBase::indexFile:" << vacuum.lastError().text(); + return QByteArray(); + } + const QString stamp = schemaStamp(); + const QString connection = QStringLiteral("qet_index_") + QUuid::createUuid().toString(); + bool ok = false; + { + QSqlDatabase file = QSqlDatabase::addDatabase(QStringLiteral("QSQLITE"), connection); + file.setDatabaseName(path); + if (file.open()) + { + QSqlQuery query(file); + //Its views read the drawing items too: an index holds tables + QStringList views; + query.exec(QStringLiteral("SELECT name FROM sqlite_master WHERE type = 'view'")); + while (query.next()) views << query.value(0).toString(); + for (const QString &view : std::as_const(views)) + query.exec(QStringLiteral("DROP VIEW \"%1\"").arg(view)); + ok = query.exec(QStringLiteral("DROP TABLE shape")) + && query.exec(QStringLiteral("DROP TABLE independent_text")) + && query.exec(QStringLiteral("DROP TABLE image")) + && query.exec(QStringLiteral("CREATE TABLE index_meta (key TEXT PRIMARY KEY, value TEXT)")) + && query.exec(QStringLiteral("CREATE TABLE folio_header (pos INTEGER PRIMARY KEY, " + "uuid TEXT NOT NULL, xml TEXT NOT NULL)")); + file.transaction(); + query.prepare(QStringLiteral("INSERT INTO index_meta (key, value) VALUES (?, ?)")); + for (const auto &meta : {std::pair{QStringLiteral("schema"), stamp}, + std::pair{QStringLiteral("writer"), + QStringLiteral("QElectroTech ") + QetVersion::currentVersion().toString()}}) { + query.addBindValue(meta.first); + query.addBindValue(meta.second); + ok = ok && query.exec(); + } + query.prepare(QStringLiteral("INSERT INTO folio_header (pos, uuid, xml) VALUES (?, ?, ?)")); + int pos = 0; + for (const auto &header : folio_headers) { + query.addBindValue(pos++); + query.addBindValue(header.first.toString()); + query.addBindValue(header.second); + ok = ok && query.exec(); + } + ok = file.commit() && ok; + ok = query.exec(QStringLiteral("VACUUM")) && ok; + file.close(); + } + } + QSqlDatabase::removeDatabase(connection); + if (!ok) { + return QByteArray(); + } + QFile file(path); + if (!file.open(QIODevice::ReadOnly)) { + return QByteArray(); + } + return file.readAll(); +} + +/** + @brief projectDataBase::discardFill + Undo fillFromIndex(): the project is read from its document instead +*/ +void projectDataBase::discardFill() +{ + QSqlQuery query(m_data_base); + for (const QString &table : indexTables(m_data_base)) + query.exec(QStringLiteral("DELETE FROM \"%1\"").arg(table)); + m_element_information.clear(); + m_folio_titleblocks.clear(); + m_conductor_properties.clear(); + m_conductor_attributes.clear(); + m_prefilled_elements.clear(); + m_prefilled_folios.clear(); + m_prefilled_conductors.clear(); + m_rows_from_document = false; + m_content_changed = true; +} + +/** + @brief projectDataBase::fillFromIndex + Fill this database from @p index, a file indexFile() wrote, instead of + from the project's document: its tables, and the stores of symbol + information and wire properties (the folios' title blocks are filled + from @p folio_headers by the folios). The file is untrusted: read only + when written with this database's schema, its tables plain tables, and + nothing in its schema run. + @param folio_headers : the folios' own data, in folio order + @return false, with @p why set and nothing changed in the tables, if + the file cannot be read +*/ +bool projectDataBase::fillFromIndex(const QByteArray &index, + QList> *folio_headers, + QString *why) +{ + auto refuse = [why](const QString &reason) { + if (why) *why = reason; + return false; + }; + QTemporaryDir dir; + if (!dir.isValid()) { + return refuse(QStringLiteral("no temporary folder")); + } + const QString path = dir.filePath(QStringLiteral("index.sqlite")); + { + QFile file(path); + if (!file.open(QIODevice::WriteOnly) || file.write(index) != index.size()) { + return refuse(QStringLiteral("cannot write the index to a temporary file")); + } + } + QString escaped = path; + escaped.replace(QLatin1Char('\''), QStringLiteral("''")); + QSqlQuery query(m_data_base); + query.exec(QStringLiteral("PRAGMA trusted_schema = OFF")); + if (!query.exec(QStringLiteral("ATTACH DATABASE '%1' AS idx").arg(escaped))) { + return refuse(QStringLiteral("the index cannot be read")); + } + auto detach = [this]() { + QSqlQuery q(m_data_base); + q.exec(QStringLiteral("DETACH DATABASE idx")); + }; + + const QStringList tables = indexTables(m_data_base); + auto isTable = [this](const QString &name) { + QSqlQuery kind(m_data_base); + kind.prepare(QStringLiteral("SELECT type FROM idx.sqlite_master WHERE name = ?")); + kind.addBindValue(name); + return kind.exec() && kind.next() && kind.value(0).toString() == QLatin1String("table"); + }; + for (const QString &table : tables + QStringList{QStringLiteral("index_meta"), + QStringLiteral("folio_header")}) { + if (!isTable(table)) { + detach(); + return refuse(QStringLiteral("the index has no table %1").arg(table)); + } + } + query.exec(QStringLiteral("SELECT value FROM idx.index_meta WHERE key = 'schema'")); + if (!query.next() || query.value(0).toString() != schemaStamp()) { + detach(); + return refuse(QStringLiteral("the index was written with another database schema")); + } + + m_data_base.transaction(); + bool ok = true; + for (const QString &table : tables) { + QSqlQuery columns(m_data_base); + columns.exec(QStringLiteral("PRAGMA main.table_info(\"%1\")").arg(table)); + QStringList names; + while (columns.next()) + names << QStringLiteral("\"%1\"").arg(columns.value(1).toString()); + const QString list = names.join(QLatin1Char(',')); + ok = ok && query.exec(QStringLiteral("DELETE FROM main.\"%1\"").arg(table)) + && query.exec(QStringLiteral("INSERT INTO main.\"%1\" (%2) SELECT %2 FROM idx.\"%1\"") + .arg(table, list)); + } + if (!ok) { + m_data_base.rollback(); + detach(); + return refuse(QStringLiteral("the index's rows cannot be read")); + } + m_data_base.commit(); + + folio_headers->clear(); + query.exec(QStringLiteral("SELECT uuid, xml FROM idx.folio_header ORDER BY pos")); + while (query.next()) + folio_headers->append({QUuid(query.value(0).toString()), query.value(1).toString()}); + detach(); + + //The stores, from their tables, as the document would fill them + m_prefilled_elements.clear(); + m_prefilled_folios.clear(); + m_prefilled_conductors.clear(); + for (PrefillTally &tally : m_prefill_tally) tally = PrefillTally(); + query.exec(QStringLiteral("SELECT element_uuid, name, value, show FROM element_information " + "ORDER BY element_uuid, ord")); + while (query.next()) { + const QUuid uuid(query.value(0).toString()); + m_prefilled_elements[uuid].addValue(query.value(1).toString(), query.value(2).toString(), + query.value(3).toInt() != 0); + } + QHash> attributes; + query.exec(QStringLiteral("SELECT conductor_uuid, name, value FROM conductor_properties")); + while (query.next()) + attributes[QUuid(query.value(0).toString())].insert(query.value(1).toString(), + query.value(2).toString()); + QDomDocument scratch; + for (auto it = attributes.cbegin() ; it != attributes.cend() ; ++it) { + QDomElement e = scratch.createElement(QStringLiteral("conductor")); + for (auto a = it.value().cbegin() ; a != it.value().cend() ; ++a) + e.setAttribute(a.key(), a.value()); + ConductorProperties properties; + properties.fromXml(e); + m_prefilled_conductors.insert(it.key(), properties); + } + for (auto it = m_prefilled_elements.cbegin() ; it != m_prefilled_elements.cend() ; ++it) + m_element_information.insert(it.key(), it.value()); + for (auto it = m_prefilled_conductors.cbegin() ; it != m_prefilled_conductors.cend() ; ++it) + m_conductor_properties.insert(it.key(), it.value()); + m_conductor_attributes = attributes; + for (const auto &header : std::as_const(*folio_headers)) { + QDomDocument document; + if (!document.setContent(header.second)) continue; + TitleBlockProperties properties; + properties.fromXml(document.documentElement()); + properties.version = QetVersion::displayedVersion(); + properties.collection = QET::QetCollection::Embedded; + m_prefilled_folios.insert(header.first, properties); + m_folio_titleblocks.insert(header.first, properties); + } + + m_rows_from_document = true; + m_content_changed = false; + emit dataBaseUpdated(); + return true; +} + /** @brief projectDataBase::foliosWithUnjoinedWires @return the uuids of the folios whose document has a wire that building diff --git a/sources/dataBase/projectdatabase.h b/sources/dataBase/projectdatabase.h index be3322c7d..15119a61d 100644 --- a/sources/dataBase/projectdatabase.h +++ b/sources/dataBase/projectdatabase.h @@ -163,6 +163,13 @@ class projectDataBase : public QObject void setBuildingFolio(bool building); bool isBuildingFolio() const {return m_building_folios > 0;} bool rowsFromDocument() const {return m_rows_from_document;} + QString schemaStamp() const; + QByteArray indexFile(const QList> &folio_headers); + void discardFill(); + /// The tables are right as they are (filled from an index) + void markCurrent() {m_content_changed = false;} + bool fillFromIndex(const QByteArray &index, QList> *folio_headers, + QString *why = nullptr); QString prefillReport() const; QSet foliosWithUnjoinedWires() const; QSet folioUuidsOfElements(const QSet &elements) const; diff --git a/sources/diagram.cpp b/sources/diagram.cpp index 9586b9555..5ce6078ec 100644 --- a/sources/diagram.cpp +++ b/sources/diagram.cpp @@ -778,7 +778,7 @@ void Diagram::keyReleaseEvent(QKeyEvent *e) */ void Diagram::ensureLoaded() { - if (m_deferred.isNull()) { + if (isLoaded()) { return; } if (m_project) { @@ -788,6 +788,23 @@ void Diagram::ensureLoaded() } } +/** + @brief Diagram::deferredXml + @return the XML of this folio's items while they are not built: kept + from the project's document, or read now from the folio's part of the + .qetz the project was opened from +*/ +QDomElement Diagram::deferredXml() +{ + if (m_deferred.isNull() && !m_deferred_part.isEmpty() && m_project) { + m_deferred = m_project->readFolioPart(m_deferred_part); + if (!m_deferred.isNull()) { + m_deferred_part.clear(); + } + } + return m_deferred; +} + /** @brief Diagram::buildDeferred Build this folio's items from the XML kept for later, and nothing else. @@ -797,11 +814,15 @@ void Diagram::ensureLoaded() */ bool Diagram::buildDeferred() { - if (m_deferred.isNull()) { + if (isLoaded()) { return false; } - QDomElement xml = m_deferred; + QDomElement xml = deferredXml(); m_deferred = QDomElement(); + m_deferred_part.clear(); + if (xml.isNull()) { + return true; //its part could not be read: QETProject::readFolioPart() said why + } m_kept_text.clear(); projectDataBase *db = m_project ? m_project->dataBase() : nullptr; if (db) db->setBuildingFolio(true); diff --git a/sources/diagram.h b/sources/diagram.h index 44748c181..ca5e13951 100644 --- a/sources/diagram.h +++ b/sources/diagram.h @@ -142,6 +142,7 @@ class Diagram : public QGraphicsScene QUuid m_uuid = QUuid::createUuid(); /// The XML of this folio's items, while they are not built yet QDomElement m_deferred; + QString m_deferred_part; QString m_opened_header; QByteArray m_kept_text; bool m_loading_deferred = false; @@ -212,8 +213,12 @@ class Diagram : public QGraphicsScene DiagramContent * = nullptr); /// false while this folio's items wait in its XML to be /// built (QET_LAZY_FOLIOS) - bool isLoaded() const {return m_deferred.isNull();} - QDomElement deferredXml() const {return m_deferred;} + bool isLoaded() const {return m_deferred.isNull() && m_deferred_part.isEmpty();} + QDomElement deferredXml(); + /// The folio's items are in the part @p file of the .qetz the + /// project was opened from, read when they are first needed + /// (stage D2) + void setDeferredPart(const QString &file) {m_deferred = QDomElement(); m_deferred_part = file;} QString headerXml(); /// The folio's text in the file it was read from, while it is /// not built (QETProject::toXml()) diff --git a/sources/qetproject.cpp b/sources/qetproject.cpp index 92aeefc6c..f6c02dfc9 100644 --- a/sources/qetproject.cpp +++ b/sources/qetproject.cpp @@ -30,6 +30,7 @@ #include "autoNum/elementautonumschemecommand.h" #include "diagram.h" #include "qetgraphicsitem/element.h" +#include "qetgraphicsitem/conductor.h" #include "qetgraphicsitem/ViewItem/qetgraphicstableitem.h" #include "qetgraphicsitem/ViewItem/projectdbmodel.h" #include "qetapp.h" @@ -462,7 +463,40 @@ QETProject::ProjectState QETProject::openFile(QFile *file) // QetContainer::parse() does exactly that, for a .qet and for each part // of a .qetz alike. QString read_error; - if (!QetContainer::readDocument(content, &xml_project, &read_error)) + //A .qetz holding the index of its database opens from it, the + //folios' parts read when they are built (stage D2); + //any other, or one whose index cannot be used, is read whole + m_opened_from_index = false; + if (QetContainer::isZip(content) && qEnvironmentVariableIntValue("QET_LAZY_FOLIOS") == 1) + { + QList entries; + QString why; + if (QetZip::fromBytes(content, &entries, &why)) + { + QHash files; + for (const QetZip::Entry &entry : std::as_const(entries)) + files.insert(entry.name, entry.data); + QStringList folio_files; + QDomDocument document; + if (files.contains(QStringLiteral("index.sqlite")) + && QetContainer::join(entries, &document, &why, &folio_files)) + { + if (m_data_base.fillFromIndex(files.value(QStringLiteral("index.sqlite")), + &m_folio_headers, &why) + && m_folio_headers.size() == folio_files.size()) { + xml_project = document; + m_container_files = files; + m_folio_files = folio_files; + m_opened_from_index = true; + } else { + m_data_base.discardFill(); + m_folio_headers.clear(); + qInfo().noquote() << "Not opened from the index of" << fi.fileName() << ":" << why; + } + } + } + } + if (!m_opened_from_index && !QetContainer::readDocument(content, &xml_project, &read_error)) { qWarning().noquote() << "Unable to read" << fi.fileName() << ":" << read_error; if(opened_here) { @@ -798,6 +832,16 @@ void QETProject::buildFoliosHoldingSymbols(const QSet &symbols) return; } QList to_build; + //Opened from an index, the database knows where every symbol is + //without reading the folios' parts + if (m_opened_from_index) { + const QSet folios = m_data_base.folioUuidsOfElements(symbols); + for (Diagram *d : std::as_const(m_diagrams_list)) + if (!d->isLoaded() && folios.contains(d->uuid())) + to_build << d; + buildFolios(to_build); + return; + } for (Diagram *d : std::as_const(m_diagrams_list)) { if (d->isLoaded()) { @@ -2471,15 +2515,59 @@ bool QETProject::isContainerPath(const QString &path) document, in parts, zipped (#1440). The file is replaced only once the whole archive is written. */ -bool QETProject::writeContainer(const QDomDocument &project, const QString &path, QString *error) +bool QETProject::writeContainer(const QDomDocument &project, const QString &path, QString *error, + const QByteArray &index) { - const QList entries = QetContainer::split( + QList entries = QetContainer::split( project, QStringLiteral("QElectroTech %1").arg(QetVersion::currentVersion().toString()), error); + //The project database, for opening without reading the folios + //(stage D2); a reader that does not know it skips it + if (!entries.isEmpty() && !index.isEmpty()) + entries.append(QetZip::Entry{QStringLiteral("index.sqlite"), index, true}); return !entries.isEmpty() && QetZip::write(path, entries, error); } +/** + @brief QETProject::writeContainerFile + Write this project to @p path as a .qetz, with the index of its + database a .qetz is opened from (stage D2) when every + folio, symbol and wire has a uuid of its own -- what opening from the + index relies on, as opening without building folios does. +*/ +bool QETProject::writeContainerFile(const QString &path, QString *error) +{ + const QDomDocument document = toXml(); //every folio built + return writeContainer(document, path, error, containerIndex()); +} + +/** + @brief QETProject::containerIndex + @return the index of this project's database for a .qetz, empty when a + folio, symbol or wire has no uuid of its own +*/ +QByteArray QETProject::containerIndex() +{ + loadFolios(); + QSet seen; + QList> headers; + auto unique = [&seen](const QUuid &uuid) { + if (uuid.isNull() || seen.contains(uuid)) return false; + seen.insert(uuid); + return true; + }; + for (Diagram *diagram : std::as_const(m_diagrams_list)) { + if (!unique(diagram->uuid())) return QByteArray(); + for (Element *element : diagram->elements()) + if (!unique(element->uuid())) return QByteArray(); + for (Conductor *conductor : diagram->conductors()) + if (!unique(conductor->uuid())) return QByteArray(); + headers.append({diagram->uuid(), diagram->headerXml()}); + } + return m_data_base.indexFile(headers); +} + /** @brief QETProject::besideContainerPath @return the .qetz a project read from a .qet is saved to: the same @@ -2539,9 +2627,9 @@ QETResult QETProject::write() QString error_message; if (isContainerPath(m_file_path)) { - // A container must contain complete folios, not lazy placeholders. - QDomDocument xml_project(toXml()); - if (!writeContainer(xml_project, m_file_path, &error_message)) + //A container is written from the whole document: every folio + //built (stage D4 writes it from the database) + if (!writeContainerFile(m_file_path, &error_message)) return(error_message); } else @@ -3059,7 +3147,9 @@ void QETProject::readProjectXml(QDomDocument &xml_project) //The stores of symbol information, folio title blocks and wire //properties are filled from the document before the folios are //built - m_data_base.prefillFromDocument(xml_project); + if (!m_opened_from_index) { + m_data_base.prefillFromDocument(xml_project); + } //QET_LAZY_FOLIOS leaves the folios to be built when needed only //when the database can be filled from the document: otherwise @@ -3075,7 +3165,11 @@ void QETProject::readProjectXml(QDomDocument &xml_project) } //Load the diagrams - readDiagramsXml(xml_project); + if (m_opened_from_index) { + readFoliosFromIndex(); + } else { + readDiagramsXml(xml_project); + } const qint64 diagrams_ms = phase_timer.restart(); //Tie the elements to the numbering schemes they follow @@ -3104,7 +3198,14 @@ void QETProject::readProjectXml(QDomDocument &xml_project) m_data_base.blockSignals(false); m_data_base.setUpdateBlocked(false); - m_data_base.updateDB(xml_project); + if (m_opened_from_index) { + //Filled from the index already: making the folios changed + //nothing their rows say + m_data_base.markCurrent(); + m_data_base.updateDB(); + } else { + m_data_base.updateDB(xml_project); + } m_data_base.endPrefill(unloadedFolioCount() > 0); //Saving may write the folios not built as they were read, when //building them would write the same: a file written by this @@ -3145,6 +3246,51 @@ void QETProject::readProjectXml(QDomDocument &xml_project) m_state = Ok; } +/** + @brief QETProject::readFoliosFromIndex + readDiagramsXml() for a .qetz opened from its index: each folio made + from its own data as the index has it, its items left in its part + (stage D2) +*/ +void QETProject::readFoliosFromIndex() +{ + for (int i = 0 ; i < m_folio_headers.size() ; ++i) + { + QDomDocument header; + header.setContent(m_folio_headers.at(i).second); + auto diagram = new Diagram(this); + m_diagrams_list << diagram; + connect(&diagram->border_and_titleblock, &BorderTitleBlock::needFolioData, + this, &QETProject::updateDiagramsFolioData); + connect(diagram, &Diagram::usedTitleBlockTemplateChanged, + this, &QETProject::usedTitleBlockTemplateChanged); + m_deferring_folios = true; + QDomElement header_element = header.documentElement(); + diagram->initFromXml(header_element); + m_deferring_folios = false; + diagram->setDeferredPart(m_folio_files.at(i)); + m_data_base.placeFolio(diagram); + } + updateDiagramsFolioData(); +} + +/** + @brief QETProject::readFolioPart + @return the folio part @p file of the .qetz this project was opened + from, read now, its engineering data back from project.sqlite; null + if it cannot be read (said in the log) +*/ +QDomElement QETProject::readFolioPart(const QString &file) +{ + QDomDocument folio; + QString error; + if (!QetContainer::readFolio(m_container_files, file, &folio, &error)) { + qWarning().noquote() << "Unable to read the folio" << file << ":" << error; + return QDomElement(); + } + return folio.documentElement(); +} + /** @brief QETProject::readDiagramsXml Load the diagrams from the xml description of the project. diff --git a/sources/qetproject.h b/sources/qetproject.h index 07d4abd51..a9f79f919 100644 --- a/sources/qetproject.h +++ b/sources/qetproject.h @@ -443,7 +443,11 @@ class QETProject : public QObject QString besideContainerPath() const; QETResult writeCopyForOlderVersions(const QString &path); static bool writeContainer(const QDomDocument &project, const QString &path, - QString *error = nullptr); + QString *error = nullptr, const QByteArray &index = QByteArray()); + bool writeContainerFile(const QString &path, QString *error = nullptr); + QByteArray containerIndex(); + QDomElement readFolioPart(const QString &file); + bool openedFromIndex() const {return m_opened_from_index;} private: static QUuid derivedUuid(const QByteArray &content); void refresh(); @@ -528,6 +532,13 @@ class QETProject : public QObject bool m_kept_folios_trusted = false; /// Read from a .qet whose path nobody chose since: saved beside it bool m_opened_from_qet = false; + /// Opened from a .qetz's index (stage D2): the + /// database filled from it, each folio's items in its part + bool m_opened_from_index = false; + QHash m_container_files; + QStringList m_folio_files; + QList> m_folio_headers; + void readFoliosFromIndex(); bool m_kept_folios_stale = false; /// Something was edited since the project was opened: a table /// filled from the database may have changed diff --git a/sources/scripting/liveserver.cpp b/sources/scripting/liveserver.cpp index 132b244f8..167f50930 100644 --- a/sources/scripting/liveserver.cpp +++ b/sources/scripting/liveserver.cpp @@ -919,11 +919,11 @@ QJsonObject LiveServer::snapshot(const QString &raw_path) if (raw_path.trimmed().isEmpty() || info.isRelative()) return failure(QStringLiteral("path must be absolute")); if (info.exists()) return failure(QStringLiteral("%1 already exists").arg(raw_path)); - QDomDocument xml(project->toXml()); QString error; if (QETProject::isContainerPath(info.absoluteFilePath()) - ? !QETProject::writeContainer(xml, info.absoluteFilePath(), &error) - : !QET::writeXmlFile(xml, info.absoluteFilePath(), &error)) + ? !project->writeContainerFile(info.absoluteFilePath(), &error) + : ![&]() { QDomDocument xml(project->toXml()); + return QET::writeXmlFile(xml, info.absoluteFilePath(), &error); }()) return failure(QStringLiteral("snapshot not written: %1").arg(error)); return {{QStringLiteral("ok"), true}, {QStringLiteral("path"), info.absoluteFilePath()}, {QStringLiteral("folio"), e->currentDiagramView() diff --git a/sources/scripting/qetscriptapi.cpp b/sources/scripting/qetscriptapi.cpp index 936088c81..be97fc618 100644 --- a/sources/scripting/qetscriptapi.cpp +++ b/sources/scripting/qetscriptapi.cpp @@ -434,11 +434,11 @@ bool QetScriptApi::save(const QString &output) return true; } - QDomDocument xml_doc(m_project->toXml()); QString error_message; if (QETProject::isContainerPath(output) - ? !QETProject::writeContainer(xml_doc, output, &error_message) - : !QET::writeXmlFile(xml_doc, output, &error_message)) { + ? !m_project->writeContainerFile(output, &error_message) + : ![&]() { QDomDocument xml_doc(m_project->toXml()); + return QET::writeXmlFile(xml_doc, output, &error_message); }()) { log(QStringLiteral("qet.save: %1").arg(error_message)); return false; } diff --git a/tests/qttest/lazy_folios_probe.cpp b/tests/qttest/lazy_folios_probe.cpp index 2ed2d085e..c9f3c5c1b 100644 --- a/tests/qttest/lazy_folios_probe.cpp +++ b/tests/qttest/lazy_folios_probe.cpp @@ -607,6 +607,25 @@ int main(int argc, char **argv) check(opened.state() == QETProject::Ok, "a project with a shared uuid opens"); check(opened.unloadedFolioCount() == 0, "a project with a shared uuid opens with every folio built"); check(built_late == 0, "a project with a shared uuid builds its folios as it reads them, not later"); + } + //A .qetz opened from its index (stage D2): no + //folio read, each read from its part when shown, drawn and + //linked as when built, saved as the .qet is + { + const QString zipped = output + "/lazy_folios.qetz"; + QString error; + check(eager.writeContainerFile(zipped, &error), "saved as a .qetz"); + QETProject indexed(zipped); + check(indexed.state() == QETProject::Ok, "the .qetz opens"); + check(indexed.openedFromIndex(), "the .qetz opens from its index"); + std::printf(".qetz opened from its index: %d of %d folios not built (strip folios %d)\n", + indexed.unloadedFolioCount(), int(folios.size()), int(strip_folios.size())); + check(indexed.unloadedFolioCount() == folios.size() - strip_folios.size(), + "the .qetz opens building only the folios of the terminal strip's terminals"); + check(numbering(indexed) == numbering(eager), "symbols' numbering from the index as when built"); + for (int i = 0; i < folios.size(); ++i) + same(indexed, i, "shown from the .qetz"); + check(indexed.toXml().toString() == eager.toXml().toString(), "the .qetz saves as when built"); } std::printf("PASS: %d folios\n", int(folios.size())); } catch (const std::exception &error) { From 2f98872d7a992aa444feeb85eb70938e145e3825 Mon Sep 17 00:00:00 2001 From: ispyisail Date: Sun, 11 Oct 2026 19:32:27 +1300 Subject: [PATCH 03/26] .qetz format 3: one project database (stage D3a) project.sqlite now holds both what QElectroTech keeps in its project database (the index D2 wrote as index.sqlite) and the document's engineering data, the latter's tables named doc_ beside the former's -- element, element_info, link and conductor exist in both (QetContainerDb's SQL names them as before; Database::sql() puts the prefix on). The manifest says format 3, min-reader 3: a format 2 reader would find no tables of its names. Format 2 files, with or without an index.sqlite entry, are read as before: the prefix is found from the file (doc_meta). A .qetz with an index opens from project.sqlite. Measured: the 136-folio f2099 1.8 s, 5.2 MB (the two table sets still overlap: D3b). Checked: tst_qetcontainer with the format 3 names, 34/34; a format 2 file of the last commit opens from its index.sqlite and saves the same; corpus 342: 340 .qetz carry the index in project.sqlite and open from it, saved back = eager = master on all 342. ctest 96/96 with and without QET_LAZY_FOLIOS=1. Co-Authored-By: Claude Opus 5.5 --- sources/container/qetcontainer.cpp | 4 +- sources/container/qetcontainer.h | 5 +- sources/container/qetcontainerdb.cpp | 106 +++++++++++++++++++-------- sources/container/qetcontainerdb.h | 3 +- sources/qetproject.cpp | 19 +++-- tests/qttest/tst_qetcontainer.cpp | 16 ++-- 6 files changed, 101 insertions(+), 52 deletions(-) diff --git a/sources/container/qetcontainer.cpp b/sources/container/qetcontainer.cpp index 3cb20fd84..84d0a293c 100644 --- a/sources/container/qetcontainer.cpp +++ b/sources/container/qetcontainer.cpp @@ -172,7 +172,7 @@ bool QetContainer::readDocument(const QByteArray &content, QDomDocument *documen set if @p project is not a project */ QList QetContainer::split(const QDomDocument &project, const QString &writer, - QString *error) + QString *error, const QByteArray &index) { QDomDocument document = project.cloneNode(true).toDocument(); QDomElement root = document.documentElement(); @@ -182,7 +182,7 @@ QList QetContainer::split(const QDomDocument &project, const QStr } //The engineering data out of the document, into project.sqlite - const QByteArray database = QetContainerDb::extract(root, error); + const QByteArray database = QetContainerDb::extract(root, error, index); if (database.isEmpty()) return {}; diff --git a/sources/container/qetcontainer.h b/sources/container/qetcontainer.h index ad050efb4..9a7d1692b 100644 --- a/sources/container/qetcontainer.h +++ b/sources/container/qetcontainer.h @@ -47,11 +47,12 @@ namespace QetContainer extern const char MimeType[]; /// The format written; files of a later format are refused by join() /// unless their min-reader allows this one. - constexpr int Format = 2; + constexpr int Format = 3; QList split(const QDomDocument &project, const QString &writer = QString(), - QString *error = nullptr); + QString *error = nullptr, + const QByteArray &index = QByteArray()); bool join(const QList &entries, QDomDocument *project, QString *error = nullptr, QStringList *folio_files = nullptr); /// A folio part of @p files, which join() left out, with its diff --git a/sources/container/qetcontainerdb.cpp b/sources/container/qetcontainerdb.cpp index e5c48c121..49beba7b9 100644 --- a/sources/container/qetcontainerdb.cpp +++ b/sources/container/qetcontainerdb.cpp @@ -17,6 +17,8 @@ */ #include "qetcontainerdb.h" +#include + #include #include #include @@ -103,14 +105,34 @@ QString schema() .arg(columns(FolioData), columns(ElementData), columns(ConductorData)); } - //Owns an sqlite3 handle + //The tables of a document's engineering data. From format 3 on they are + //named doc_, beside the project database's own tables in the same + //file (stage D3) +const QStringList DocTables = { + QStringLiteral("strip_bridge_terminal"), QStringLiteral("strip_bridge"), + QStringLiteral("strip_terminal"), QStringLiteral("terminal_strip"), + QStringLiteral("project_property"), QStringLiteral("folio_property"), QStringLiteral("folio"), + QStringLiteral("element_info"), QStringLiteral("element"), QStringLiteral("link"), + QStringLiteral("conductor"), QStringLiteral("meta")}; + + //Owns an sqlite3 handle; its SQL names the document's tables by their + //format 2 names, given their prefix here class Database { public: ~Database() { if (m_db) sqlite3_close(m_db); } sqlite3 *m_db = nullptr; - bool exec(const QString &sql) { - return sqlite3_exec(m_db, sql.toUtf8().constData(), nullptr, nullptr, nullptr) == SQLITE_OK; + QString m_prefix; + QString sql(const QString &text) const { + if (m_prefix.isEmpty()) return text; + static const QRegularExpression names(QStringLiteral("(?, format 2 not + { + Statement doc(db, QStringLiteral("SELECT count(*) FROM sqlite_master WHERE name = 'doc_meta'")); + if (doc.next() && doc.value(0).toInt() > 0) db.m_prefix = QStringLiteral("doc_"); + } for (const char *table : {"meta", "project_property", "folio", "folio_property", "element", "element_info", "link", "conductor", "terminal_strip", "strip_terminal", "strip_bridge", "strip_bridge_terminal"}) { - Statement kind(db.m_db, QStringLiteral("SELECT type FROM sqlite_master WHERE name = ?")); - kind.bind({QString::fromLatin1(table)}); + Statement kind(db, QStringLiteral("SELECT type FROM sqlite_master WHERE name = ?")); + kind.bind({db.table(QString::fromLatin1(table))}); if (!kind.next() || kind.value(0).toString() != QLatin1String("table")) { setError(error, QStringLiteral("the project database has no table %1").arg(QLatin1String(table))); return false; @@ -490,7 +517,7 @@ bool restoreDiagram(Database &db, QDomElement d, QString *error) const QString id = d.attribute(FolioAttribute); if (id.isEmpty()) return true; d.removeAttribute(FolioAttribute); - Statement f(db.m_db, QStringLiteral("SELECT %1, props_at, props_attrs FROM folio WHERE id = ?") + Statement f(db, QStringLiteral("SELECT %1, props_at, props_attrs FROM folio WHERE id = ?") .arg(QStringLiteral("\"") + FolioData.join(QStringLiteral("\", \"")) + QStringLiteral("\""))); f.bind({id}); if (!f.next()) { setError(error, QStringLiteral("folio %1 is not in the database").arg(id)); return false; } @@ -500,7 +527,7 @@ bool restoreDiagram(Database &db, QDomElement d, QString *error) if (!values.at(FolioData.size()).isNull()) { QDomElement props = document.createElement(QStringLiteral("properties")); setAttributesJson(props, values.at(FolioData.size() + 1)); - Statement p(db.m_db, QStringLiteral("SELECT name, value, attrs FROM folio_property WHERE folio_id = ? ORDER BY ord")); + Statement p(db, QStringLiteral("SELECT name, value, attrs FROM folio_property WHERE folio_id = ? ORDER BY ord")); p.bind({id}); while (p.next()) props.appendChild(entry(p.value(0), QStringLiteral("name"), p.value(1), p.value(2), @@ -517,7 +544,7 @@ bool restoreDiagram(Database &db, QDomElement d, QString *error) if (!e.hasAttribute(KeyAttribute)) continue; const QString key = e.attribute(KeyAttribute); e.removeAttribute(KeyAttribute); - Statement s(db.m_db, QStringLiteral("SELECT %1, info_at, info_attrs, links_at, links_attrs FROM element WHERE key = ?") + Statement s(db, QStringLiteral("SELECT %1, info_at, info_attrs, links_at, links_attrs FROM element WHERE key = ?") .arg(QStringLiteral("\"") + ElementData.join(QStringLiteral("\", \"")) + QStringLiteral("\""))); s.bind({key}); if (!s.next()) { setError(error, QStringLiteral("symbol %1 is not in the database").arg(key)); return false; } @@ -529,7 +556,7 @@ bool restoreDiagram(Database &db, QDomElement d, QString *error) if (!v.at(n).isNull()) { info = document.createElement(QStringLiteral("elementInformations")); setAttributesJson(info, v.at(n + 1)); - Statement i(db.m_db, QStringLiteral("SELECT name, value, attrs FROM element_info WHERE element_key = ? ORDER BY ord")); + Statement i(db, QStringLiteral("SELECT name, value, attrs FROM element_info WHERE element_key = ? ORDER BY ord")); i.bind({key}); while (i.next()) info.appendChild(entry(i.value(0), QStringLiteral("name"), i.value(1), i.value(2), @@ -539,7 +566,7 @@ bool restoreDiagram(Database &db, QDomElement d, QString *error) if (!v.at(n + 2).isNull()) { links = document.createElement(QStringLiteral("links_uuids")); setAttributesJson(links, v.at(n + 3)); - Statement l(db.m_db, QStringLiteral("SELECT linked_uuid, attrs FROM link WHERE element_key = ? ORDER BY ord")); + Statement l(db, QStringLiteral("SELECT linked_uuid, attrs FROM link WHERE element_key = ? ORDER BY ord")); l.bind({key}); while (l.next()) links.appendChild(entry(l.value(0), QStringLiteral("uuid"), QVariant(), l.value(1), @@ -558,7 +585,7 @@ bool restoreDiagram(Database &db, QDomElement d, QString *error) if (!c.hasAttribute(KeyAttribute)) continue; const QString key = c.attribute(KeyAttribute); c.removeAttribute(KeyAttribute); - Statement s(db.m_db, QStringLiteral("SELECT %1 FROM conductor WHERE key = ?") + Statement s(db, QStringLiteral("SELECT %1 FROM conductor WHERE key = ?") .arg(QStringLiteral("\"") + ConductorData.join(QStringLiteral("\", \"")) + QStringLiteral("\""))); s.bind({key}); if (!s.next()) { setError(error, QStringLiteral("wire %1 is not in the database").arg(key)); return false; } @@ -577,27 +604,44 @@ bool restoreDiagram(Database &db, QDomElement d, QString *error) folios still in it) into a new database. @return the database file, or an empty array with @p error set */ -QByteArray QetContainerDb::extract(QDomElement project, QString *error) +QByteArray QetContainerDb::extract(QDomElement project, QString *error, const QByteArray &index) { Database db; - if (sqlite3_open(":memory:", &db.m_db) != SQLITE_OK || !db.exec(schema())) { + db.m_prefix = QStringLiteral("doc_"); + if (sqlite3_open(":memory:", &db.m_db) != SQLITE_OK) { + setError(error, QStringLiteral("cannot create the project database")); + return {}; + } + //The project database's own tables first, when given: one file + //holds both (stage D3) + if (!index.isEmpty()) { + unsigned char *copy = static_cast(sqlite3_malloc64(sqlite3_uint64(index.size()))); + if (!copy) { setError(error, QStringLiteral("out of memory")); return {}; } + memcpy(copy, index.constData(), size_t(index.size())); + if (sqlite3_deserialize(db.m_db, "main", copy, index.size(), index.size(), + SQLITE_DESERIALIZE_FREEONCLOSE | SQLITE_DESERIALIZE_RESIZEABLE) != SQLITE_OK) { + setError(error, QStringLiteral("cannot read the project database's index")); + return {}; + } + } + if (!db.exec(schema())) { setError(error, QStringLiteral("cannot create the project database")); return {}; } db.exec(QStringLiteral("BEGIN")); - Statement meta(db.m_db, QStringLiteral("INSERT INTO meta VALUES (?,?)")); - Statement project_property(db.m_db, QStringLiteral("INSERT INTO project_property VALUES (?,?,?,?)")); - Statement folio(db.m_db, QStringLiteral("INSERT INTO folio VALUES (?,?,%1,?,?)").arg(placeholders(FolioData.size()))); - Statement folio_property(db.m_db, QStringLiteral("INSERT INTO folio_property VALUES (?,?,?,?,?)")); - Statement element(db.m_db, QStringLiteral("INSERT INTO element VALUES (?,?,%1,?,?,?,?)").arg(placeholders(ElementData.size()))); - Statement element_info(db.m_db, QStringLiteral("INSERT INTO element_info VALUES (?,?,?,?,?)")); - Statement link(db.m_db, QStringLiteral("INSERT INTO link VALUES (?,?,?,?)")); - Statement conductor(db.m_db, QStringLiteral("INSERT INTO conductor VALUES (?,?,%1)").arg(placeholders(ConductorData.size()))); + Statement meta(db, QStringLiteral("INSERT INTO meta VALUES (?,?)")); + Statement project_property(db, QStringLiteral("INSERT INTO project_property VALUES (?,?,?,?)")); + Statement folio(db, QStringLiteral("INSERT INTO folio VALUES (?,?,%1,?,?)").arg(placeholders(FolioData.size()))); + Statement folio_property(db, QStringLiteral("INSERT INTO folio_property VALUES (?,?,?,?,?)")); + Statement element(db, QStringLiteral("INSERT INTO element VALUES (?,?,%1,?,?,?,?)").arg(placeholders(ElementData.size()))); + Statement element_info(db, QStringLiteral("INSERT INTO element_info VALUES (?,?,?,?,?)")); + Statement link(db, QStringLiteral("INSERT INTO link VALUES (?,?,?,?)")); + Statement conductor(db, QStringLiteral("INSERT INTO conductor VALUES (?,?,%1)").arg(placeholders(ConductorData.size()))); bool ok = meta.ok() && project_property.ok() && folio.ok() && folio_property.ok() && element.ok() && element_info.ok() && link.ok() && conductor.ok(); - ok = ok && meta.run({QStringLiteral("format"), QStringLiteral("2")}); + ok = ok && meta.run({QStringLiteral("format"), QStringLiteral("3")}); // of a block, as rows of @p rows (with @p prefix values) auto moveProperties = [&](QDomElement block, Statement &rows, const QVariantList &prefix) { @@ -709,10 +753,10 @@ QByteArray QetContainerDb::extract(QDomElement project, QString *error) const QDomElement strips = project.firstChildElement(QStringLiteral("terminal_strips")); StripRows strip_rows; if (!strips.isNull() && stripRows(strips, &strip_rows)) { - Statement st(db.m_db, QStringLiteral("INSERT INTO terminal_strip VALUES (?,?,?,?,?,?,?)")); - Statement tt(db.m_db, QStringLiteral("INSERT INTO strip_terminal VALUES (?,?,?,?)")); - Statement bt(db.m_db, QStringLiteral("INSERT INTO strip_bridge VALUES (?,?,?,?)")); - Statement btt(db.m_db, QStringLiteral("INSERT INTO strip_bridge_terminal VALUES (?,?,?)")); + Statement st(db, QStringLiteral("INSERT INTO terminal_strip VALUES (?,?,?,?,?,?,?)")); + Statement tt(db, QStringLiteral("INSERT INTO strip_terminal VALUES (?,?,?,?)")); + Statement bt(db, QStringLiteral("INSERT INTO strip_bridge VALUES (?,?,?,?)")); + Statement btt(db, QStringLiteral("INSERT INTO strip_bridge_terminal VALUES (?,?,?)")); for (const QVariantList &r : strip_rows.strips) ok = ok && st.run(r); for (const QVariantList &r : strip_rows.terminals) ok = ok && tt.run(r); for (const QVariantList &r : strip_rows.bridges) ok = ok && bt.run(r); @@ -762,13 +806,13 @@ bool QetContainerDb::restore(QDomElement project, const QByteArray &database, QS //Project properties QHash meta; { - Statement s(db.m_db, QStringLiteral("SELECT key, value FROM meta")); + Statement s(db, QStringLiteral("SELECT key, value FROM meta")); while (s.next()) meta.insert(s.value(0).toString(), s.value(1)); } if (meta.contains(QStringLiteral("props_at"))) { QDomElement props = document.createElement(QStringLiteral("properties")); setAttributesJson(props, meta.value(QStringLiteral("props_attrs"))); - Statement s(db.m_db, QStringLiteral("SELECT name, value, attrs FROM project_property ORDER BY ord")); + Statement s(db, QStringLiteral("SELECT name, value, attrs FROM project_property ORDER BY ord")); while (s.next()) props.appendChild(entry(s.value(0), QStringLiteral("name"), s.value(1), s.value(2), QStringLiteral("property"))); @@ -782,7 +826,7 @@ bool QetContainerDb::restore(QDomElement project, const QByteArray &database, QS if (meta.contains(QStringLiteral("terminal_strips"))) { StripRows rows; auto read = [&db](const QString &sql, int columns, QList *out) { - Statement s(db.m_db, sql); + Statement s(db, sql); while (s.next()) { QVariantList r; for (int i = 0 ; i < columns ; ++i) r << s.value(i); diff --git a/sources/container/qetcontainerdb.h b/sources/container/qetcontainerdb.h index e43c87946..5e1ec63ca 100644 --- a/sources/container/qetcontainerdb.h +++ b/sources/container/qetcontainerdb.h @@ -37,7 +37,8 @@ */ namespace QetContainerDb { - QByteArray extract(QDomElement project, QString *error = nullptr); + QByteArray extract(QDomElement project, QString *error = nullptr, + const QByteArray &index = QByteArray()); bool restore(QDomElement project, const QByteArray &database, QString *error = nullptr); bool restoreFolio(QDomElement diagram, const QByteArray &database, QString *error = nullptr); } diff --git a/sources/qetproject.cpp b/sources/qetproject.cpp index f6c02dfc9..11b8c3f22 100644 --- a/sources/qetproject.cpp +++ b/sources/qetproject.cpp @@ -478,11 +478,15 @@ QETProject::ProjectState QETProject::openFile(QFile *file) files.insert(entry.name, entry.data); QStringList folio_files; QDomDocument document; - if (files.contains(QStringLiteral("index.sqlite")) + //The index: in project.sqlite from format 3, its own entry + //before + const QByteArray index = files.contains(QStringLiteral("index.sqlite")) + ? files.value(QStringLiteral("index.sqlite")) + : files.value(QStringLiteral("project.sqlite")); + if (!index.isEmpty() && index.contains("folio_header") && QetContainer::join(entries, &document, &why, &folio_files)) { - if (m_data_base.fillFromIndex(files.value(QStringLiteral("index.sqlite")), - &m_folio_headers, &why) + if (m_data_base.fillFromIndex(index, &m_folio_headers, &why) && m_folio_headers.size() == folio_files.size()) { xml_project = document; m_container_files = files; @@ -2518,14 +2522,13 @@ bool QETProject::isContainerPath(const QString &path) bool QETProject::writeContainer(const QDomDocument &project, const QString &path, QString *error, const QByteArray &index) { + //The project database, for opening without reading the folios + //(stage D2), is in project.sqlite with the + //document's engineering data (format 3, D3) QList entries = QetContainer::split( project, QStringLiteral("QElectroTech %1").arg(QetVersion::currentVersion().toString()), - error); - //The project database, for opening without reading the folios - //(stage D2); a reader that does not know it skips it - if (!entries.isEmpty() && !index.isEmpty()) - entries.append(QetZip::Entry{QStringLiteral("index.sqlite"), index, true}); + error, index); return !entries.isEmpty() && QetZip::write(path, entries, error); } diff --git a/tests/qttest/tst_qetcontainer.cpp b/tests/qttest/tst_qetcontainer.cpp index ea2a9525c..be09a3593 100644 --- a/tests/qttest/tst_qetcontainer.cpp +++ b/tests/qttest/tst_qetcontainer.cpp @@ -170,14 +170,14 @@ private slots: return n; }; const QDomNodeList diagrams = original.elementsByTagName(QStringLiteral("diagram")); - QCOMPARE(count("folio"), int(diagrams.size())); - QCOMPARE(count("element"), int(original.elementsByTagName(QStringLiteral("element")).size()) + QCOMPARE(count("doc_folio"), int(diagrams.size())); + QCOMPARE(count("doc_element"), int(original.elementsByTagName(QStringLiteral("element")).size()) - int(original.elementsByTagName(QStringLiteral("collection")).at(0).toElement() .elementsByTagName(QStringLiteral("element")).size())); - QCOMPARE(count("element_info"), int(original.elementsByTagName(QStringLiteral("elementInformation")).size())); - QCOMPARE(count("link"), int(original.elementsByTagName(QStringLiteral("link_uuid")).size())); - QCOMPARE(count("conductor"), int(original.elementsByTagName(QStringLiteral("conductor")).size())); - QVERIFY(count("folio_property") + count("project_property") > 0); + QCOMPARE(count("doc_element_info"), int(original.elementsByTagName(QStringLiteral("elementInformation")).size())); + QCOMPARE(count("doc_link"), int(original.elementsByTagName(QStringLiteral("link_uuid")).size())); + QCOMPARE(count("doc_conductor"), int(original.elementsByTagName(QStringLiteral("conductor")).size())); + QVERIFY(count("doc_folio_property") + count("doc_project_property") > 0); sqlite3_close(db); } @@ -285,8 +285,8 @@ private slots: memcpy(copy, e.data.constData(), size_t(e.data.size())); sqlite3_deserialize(db, "main", copy, e.data.size(), e.data.size(), SQLITE_DESERIALIZE_FREEONCLOSE | SQLITE_DESERIALIZE_RESIZEABLE); - QCOMPARE(sqlite3_exec(db, "ALTER TABLE element RENAME TO element_data;" - "CREATE VIEW element AS SELECT * FROM element_data;", + QCOMPARE(sqlite3_exec(db, "ALTER TABLE doc_element RENAME TO element_data;" + "CREATE VIEW doc_element AS SELECT * FROM element_data;", nullptr, nullptr, nullptr), SQLITE_OK); sqlite3_int64 size = 0; unsigned char *bytes = sqlite3_serialize(db, "main", &size, 0); From 8ea051acfc6e314db5c31b39fe6fb5c3df665409 Mon Sep 17 00:00:00 2001 From: ispyisail Date: Sun, 11 Oct 2026 19:57:43 +1300 Subject: [PATCH 04/26] A .qetz keeps nothing QElectroTech computes in its folio files (stage D3c) Format 3 folio files lose what building a folio computes (QetContainer::dropComputed()): a wire's description of its ends (element1_label, element1_name, element1_linked, terminalname1 and the same for end 2), the content of a symbol text showing the symbol's information or a composition of it, and -- when the project database's index is written, every symbol with a uuid of its own -- a symbol's list of its terminals. What may go was found by test, not by reading: on 342 projects saved by this version, each kind taken out of the file, the project opened and saved: the wire ends came back in all 244 that had them, the symbol texts in all 201, the terminal lists in 305 of 306 -- not in the file whose symbols share a uuid (#1408), where wires find their ends by them; such a project gets no index, so it keeps them. Stays, for now: labels made from a formula and wire numbers made from one, in the database (opening from the index and numbering read them), and a table's size (the user's as much as computed). tst_qetcontainer compares a joined document with the original less what is computed, and computedNotKept checks a file holding both kinds (one filled in by the test) loses them -- it fails when nothing is dropped. Through QElectroTech, 342 projects: a .qetz opened from its index and saved as .qet = eager = master; every folio of industrial and f2099 drawn and linked as eager (f2099 but its 3 render-noise folios). ctest 96/96 with and without QET_LAZY_FOLIOS=1. Co-Authored-By: Claude Opus 5.5 --- sources/container/qetcontainer.cpp | 60 ++++++++++++++++++++++++++++++ sources/container/qetcontainer.h | 2 + tests/qttest/tst_qetcontainer.cpp | 53 ++++++++++++++++++++++++-- 3 files changed, 112 insertions(+), 3 deletions(-) diff --git a/sources/container/qetcontainer.cpp b/sources/container/qetcontainer.cpp index 84d0a293c..5749113ee 100644 --- a/sources/container/qetcontainer.cpp +++ b/sources/container/qetcontainer.cpp @@ -93,6 +93,19 @@ QList childElements(const QDomElement &parent, const QString &tag) return list; } + //An element and the text nodes on either side of it out of its parent, + //one text node left where they were, so nothing is read twice +void removeElement(QDomElement e) +{ + QDomNode parent = e.parentNode(); + QDomNode before = e.previousSibling(), after = e.nextSibling(); + parent.removeChild(e); + if (before.isText() && after.isText()) { + before.setNodeValue(before.nodeValue() + after.nodeValue()); + parent.removeChild(after); + } +} + //The collection's symbol definitions, each to elements// void splitCollection(QDomDocument &document, const QDomElement &category, const QString &path, QList *parts, QSet *taken) @@ -164,6 +177,49 @@ bool QetContainer::readDocument(const QByteArray &content, QDomDocument *documen && join(entries, document, error); } +/** + @brief QetContainer::dropComputed + Take out of the folio @p diagram what QElectroTech computes when it + builds the folio (stage D3c): a wire's description of + its ends (element1_label, element1_name, element1_linked, terminalname1, + and the same for end 2), a symbol text's content where it shows the + symbol's information or a composition of it, and -- with + @p terminal_lists, when every symbol has a uuid of its own -- a symbol's + list of its terminals. Checked on 342 projects: opened and saved, each + comes back as it was, but the terminal lists of a file whose symbols + share a uuid (F100), where wires find their ends by them. +*/ +void QetContainer::dropComputed(QDomElement diagram, bool terminal_lists) +{ + for (QDomElement list : childElements(diagram, QStringLiteral("conductors"))) + for (QDomElement wire : childElements(list, QStringLiteral("conductor"))) + for (const char *end : {"1", "2"}) + for (const char *name : {"element%1_label", "element%1_name", "element%1_linked", + "terminalname%1"}) + wire.removeAttribute(QString::fromLatin1(name).arg(QLatin1String(end))); + for (QDomElement list : childElements(diagram, QStringLiteral("elements"))) + { + for (QDomElement element : childElements(list, QStringLiteral("element"))) + { + if (terminal_lists) + for (QDomElement terminals : childElements(element, QStringLiteral("terminals"))) + removeElement(terminals); + for (QDomElement texts : childElements(element, QStringLiteral("dynamic_texts"))) + { + for (QDomElement text : childElements(texts, QStringLiteral("dynamic_elmt_text"))) + { + const QString from = text.attribute(QStringLiteral("text_from")); + if (from != QLatin1String("ElementInfo") && from != QLatin1String("CompositeText")) + continue; + for (QDomElement content : childElements(text, QStringLiteral("text"))) + while (content.hasChildNodes()) + content.removeChild(content.firstChild()); + } + } + } + } +} + /** @brief QetContainer::split @param project : the project document, as QETProject::toXml() makes it @@ -192,6 +248,10 @@ QList QetContainer::split(const QDomDocument &project, const QStr //Folios, each with its pictures taken out int folio = 0; QMap pictures; //name -> png, once each + //Nothing QElectroTech computes is kept (format 3) + for (QDomElement diagram : childElements(root, QStringLiteral("diagram"))) + dropComputed(diagram, !index.isEmpty()); + for (QDomElement diagram : childElements(root, QStringLiteral("diagram"))) { ++folio; diff --git a/sources/container/qetcontainer.h b/sources/container/qetcontainer.h index 9a7d1692b..1d654f49d 100644 --- a/sources/container/qetcontainer.h +++ b/sources/container/qetcontainer.h @@ -60,6 +60,8 @@ namespace QetContainer bool readFolio(const QHash &files, const QString &file, QDomDocument *folio, QString *error = nullptr); + /// Take what QElectroTech computes out of a folio (format 3) + void dropComputed(QDomElement diagram, bool terminal_lists); /// Parse @p xml as a .qet is read (QETProject::openFile) bool parse(const QByteArray &xml, QDomDocument *document, QString *error = nullptr); /// true if @p content is a zip, so a .qetz rather than a .qet diff --git a/tests/qttest/tst_qetcontainer.cpp b/tests/qttest/tst_qetcontainer.cpp index be09a3593..1bccb0ce0 100644 --- a/tests/qttest/tst_qetcontainer.cpp +++ b/tests/qttest/tst_qetcontainer.cpp @@ -69,6 +69,18 @@ class tst_qetcontainer : public QObject return QetContainer::split(*original, QStringLiteral("tst_qetcontainer"), &error); } + //@p document without what QElectroTech computes, which a .qetz + //does not keep (format 3); tst_resaveunchanged checks that + //QElectroTech gives it back + static QDomDocument withoutComputed(const QDomDocument &document) + { + QDomDocument copy = document.cloneNode(true).toDocument(); + for (QDomElement diagram = copy.documentElement().firstChildElement(QStringLiteral("diagram")); + !diagram.isNull() ; diagram = diagram.nextSiblingElement(QStringLiteral("diagram"))) + QetContainer::dropComputed(diagram, false); + return copy; + } + private slots: void sameDocumentBack_data() { @@ -100,7 +112,8 @@ private slots: QVERIFY2(QetZip::fromBytes(QetZip::toBytes(entries, &error), &unzipped, &error), qPrintable(error)); QDomDocument back; QVERIFY2(QetContainer::join(unzipped, &back, &error), qPrintable(error)); - const QString d = difference(original.documentElement(), back.documentElement(), QString()); + const QString d = difference(withoutComputed(original).documentElement(), + back.documentElement(), QString()); QVERIFY2(d.isEmpty(), qPrintable(d)); } @@ -136,7 +149,9 @@ private slots: // The engineering data is in project.sqlite and nowhere else: the folio // files hold no symbol information, links, wire numbers or title-block - // fields, and the database holds them all (industrial.qet: 50 folios). + // fields, nor what QElectroTech computes from them (a wire's + // description of its ends, a symbol text showing its information), and + // the database holds them all (industrial.qet: 50 folios). void databaseHoldsTheData() { QDomDocument original; @@ -181,6 +196,37 @@ private slots: sqlite3_close(db); } + // Nor does a folio file hold what QElectroTech computes (format 3): a + // wire's description of its ends, a symbol text showing its information. + // A file holding both, so that the check can fail. + void computedNotKept() + { + QDomDocument original; + QString error; + QVERIFY(QetContainer::parse(read(QStringLiteral(QET_EXAMPLES_DIR "/schema_unifilaire_voltaique2.qet")), + &original, &error)); + const QDomNodeList texts = original.elementsByTagName(QStringLiteral("dynamic_elmt_text")); + bool filled = false; + for (int i = 0 ; i < texts.size() && !filled ; ++i) { + QDomElement text = texts.at(i).toElement(); + if (text.attribute(QStringLiteral("text_from")) != QLatin1String("ElementInfo")) continue; + QDomElement content = text.firstChildElement(QStringLiteral("text")); + while (content.hasChildNodes()) content.removeChild(content.firstChild()); + content.appendChild(original.createTextNode(QStringLiteral("COMPUTED-LABEL"))); + filled = true; + } + QVERIFY(filled); + QVERIFY(original.toString().contains(QStringLiteral("element1_label="))); + const QList entries = QetContainer::split(original, QString(), &error); + QVERIFY2(!entries.isEmpty(), qPrintable(error)); + for (const QetZip::Entry &e : entries) { + if (!e.name.startsWith(QStringLiteral("folios/"))) continue; + QVERIFY2(!e.data.contains("element1_label=") && !e.data.contains("terminalname2="), + qPrintable(e.name)); + QVERIFY2(!e.data.contains("COMPUTED-LABEL"), qPrintable(e.name)); + } + } + // Terminal strips move into the database, and come back exactly: // industrial.qet with two strips on its own terminal symbols, parsed as // a save leaves it (no whitespace text). @@ -217,7 +263,8 @@ private slots: QVERIFY2(!e.data.contains(""), "the strips stayed in project.xml"); QDomDocument back; QVERIFY2(QetContainer::join(entries, &back, &error), qPrintable(error)); - const QString d = difference(original.documentElement(), back.documentElement(), QString()); + const QString d = difference(withoutComputed(original).documentElement(), + back.documentElement(), QString()); QVERIFY2(d.isEmpty(), qPrintable(d)); } From 8ad33f628140e990c022b19eb9520de586957577 Mon Sep 17 00:00:00 2001 From: ispyisail Date: Sun, 11 Oct 2026 21:21:42 +1300 Subject: [PATCH 05/26] Save a .qetz without building the folios not touched (stage D4) With QET_LAZY_FOLIOS=1, a project opened from a .qetz's index saves a .qetz without building the folios it never built, when building them would write the same: their parts are copied as they were and their rows of the document's engineering data from the project.sqlite they were read from (QetContainerDb::extract() with kept_from, QetContainer::split() with the old entries, pictures included). Whether a folio may be kept follows the rules of stage 5.5 (QETProject::toXml() with kept_parts): nothing project-wide changed since opening; a folio whose own data changed is built with its partners; after an edit, folios holding a table filled from the database are built; the partners still waiting of every built folio are built. Trusted only for a project opened from a format 3 index written by this version whose folios are named by their uuid. Format 3 names a folio by its uuid in the database (doc_meta folio_ids = uuid) when every folio has one of its own, so that a kept folio keeps its rows whatever its place; copies keep the rows' order (ORDER BY rowid). A folio not built keeps the name of its part until it is built (Diagram::deferredPart()). Measured, the 136-folio f2099 opened from its .qetz and saved as a .qetz: 4.75 s with no folio built, 33 s with every folio built; its 136 folio parts copied byte for byte; a kept save of a kept save is the same bytes. industrial.qetz untouched: 28 of its 47 folios not built are built to save (link chains, as in 5.5); after everyday edits on its busiest folio, 7 stay not built. tst_lazyfolios: an untouched .qetz saved without building, read back, holds the project; edits saved without building hold, read back, what the same edits saved with every folio built hold (read back too: a wire numbered from its potential is saved with no sequential numbers whichever the way -- reading it back renumbers it, a separate issue); fails when a kept folio's symbol information is not copied. Corpus, 342 projects: a .qetz saved without building, read and saved as .qet = eager = master on all 342. ctest 96/96 with and without QET_LAZY_FOLIOS=1. Co-Authored-By: Claude Opus 5.5 --- sources/container/qetcontainer.cpp | 36 +++++++++++- sources/container/qetcontainer.h | 4 +- sources/container/qetcontainerdb.cpp | 88 +++++++++++++++++++++++++++- sources/container/qetcontainerdb.h | 5 +- sources/diagram.cpp | 3 - sources/diagram.h | 1 + sources/qetproject.cpp | 48 ++++++++++++--- sources/qetproject.h | 9 ++- tests/qttest/lazy_folios_probe.cpp | 54 +++++++++++++++++ 9 files changed, 227 insertions(+), 21 deletions(-) diff --git a/sources/container/qetcontainer.cpp b/sources/container/qetcontainer.cpp index 5749113ee..1bfa5f8e6 100644 --- a/sources/container/qetcontainer.cpp +++ b/sources/container/qetcontainer.cpp @@ -22,6 +22,7 @@ #include #include #include +#include const char QetContainer::MimeType[] = "application/x-qelectrotech-project+zip"; @@ -228,7 +229,8 @@ void QetContainer::dropComputed(QDomElement diagram, bool terminal_lists) set if @p project is not a project */ QList QetContainer::split(const QDomDocument &project, const QString &writer, - QString *error, const QByteArray &index) + QString *error, const QByteArray &index, + const QHash &kept_from) { QDomDocument document = project.cloneNode(true).toDocument(); QDomElement root = document.documentElement(); @@ -238,7 +240,11 @@ QList QetContainer::split(const QDomDocument &project, const QStr } //The engineering data out of the document, into project.sqlite - const QByteArray database = QetContainerDb::extract(root, error, index); + //A folio written as it was read (, + //stage D4) keeps its part and, through + //@p kept_from's project.sqlite, its rows + const QByteArray database = QetContainerDb::extract( + root, error, index, kept_from.value(QStringLiteral("project.sqlite"))); if (database.isEmpty()) return {}; @@ -247,11 +253,35 @@ QList QetContainer::split(const QDomDocument &project, const QStr //Folios, each with its pictures taken out int folio = 0; - QMap pictures; //name -> png, once each //Nothing QElectroTech computes is kept (format 3) for (QDomElement diagram : childElements(root, QStringLiteral("diagram"))) dropComputed(diagram, !index.isEmpty()); + //The parts of the folios kept as they were, and their pictures + QMap pictures; //name -> png, once each + for (QDomElement diagram : childElements(root, QStringLiteral("diagram"))) + { + const QString kept = diagram.attribute(QStringLiteral("qetz-kept-part")); + if (kept.isEmpty()) continue; + if (!kept_from.contains(kept)) { + setError(error, QStringLiteral("the part \"%1\" of a folio kept as it was is missing").arg(kept)); + return {}; + } + taken.insert(kept); + const QByteArray part = kept_from.value(kept); + static const QRegularExpression image_file(QStringLiteral("%1=\"(images/[^\"]+)\"").arg(ImageFileAttribute)); + for (const auto &match : image_file.globalMatch(QString::fromUtf8(part))) { + const QString name = match.captured(1); + if (!kept_from.contains(name)) { + setError(error, QStringLiteral("the picture \"%1\" of a folio kept as it was is missing").arg(name)); + return {}; + } + pictures.insert(name, kept_from.value(name)); + } + parts.append(QetZip::Entry{kept, part, true}); + root.replaceChild(placeholder(document, kept), diagram); + } + for (QDomElement diagram : childElements(root, QStringLiteral("diagram"))) { ++folio; diff --git a/sources/container/qetcontainer.h b/sources/container/qetcontainer.h index 1d654f49d..16da23a29 100644 --- a/sources/container/qetcontainer.h +++ b/sources/container/qetcontainer.h @@ -21,6 +21,7 @@ #include "qetzip.h" #include +#include /** @brief The QetContainer namespace @@ -52,7 +53,8 @@ namespace QetContainer QList split(const QDomDocument &project, const QString &writer = QString(), QString *error = nullptr, - const QByteArray &index = QByteArray()); + const QByteArray &index = QByteArray(), + const QHash &kept_from = QHash()); bool join(const QList &entries, QDomDocument *project, QString *error = nullptr, QStringList *folio_files = nullptr); /// A folio part of @p files, which join() left out, with its diff --git a/sources/container/qetcontainerdb.cpp b/sources/container/qetcontainerdb.cpp index 49beba7b9..7ec4c6e0a 100644 --- a/sources/container/qetcontainerdb.cpp +++ b/sources/container/qetcontainerdb.cpp @@ -24,12 +24,15 @@ #include #include #include +#include #include #include namespace { const QString KeyAttribute = QStringLiteral("qetz-key"); + //A folio written as it was read (stage D4): its part +const QString KeptPartAttribute = QStringLiteral("qetz-kept-part"); const QString FolioAttribute = QStringLiteral("qetz-folio"); //What a moved block leaves where it was. Removing it instead would leave //the whitespace before and after it side by side, read back as one text @@ -604,7 +607,8 @@ bool restoreDiagram(Database &db, QDomElement d, QString *error) folios still in it) into a new database. @return the database file, or an empty array with @p error set */ -QByteArray QetContainerDb::extract(QDomElement project, QString *error, const QByteArray &index) +QByteArray QetContainerDb::extract(QDomElement project, QString *error, const QByteArray &index, + const QByteArray &kept_from) { Database db; db.m_prefix = QStringLiteral("doc_"); @@ -627,6 +631,28 @@ QByteArray QetContainerDb::extract(QDomElement project, QString *error, const QB if (!db.exec(schema())) { setError(error, QStringLiteral("cannot create the project database")); return {}; + } + //The database of the .qetz this project was read from, for the folios + //kept as they were (stage D4) + if (!kept_from.isEmpty()) { + unsigned char *copy = static_cast(sqlite3_malloc64(sqlite3_uint64(kept_from.size()))); + if (!copy) { setError(error, QStringLiteral("out of memory")); return {}; } + memcpy(copy, kept_from.constData(), size_t(kept_from.size())); + if (!db.exec(QStringLiteral("ATTACH DATABASE ':memory:' AS old")) + || sqlite3_deserialize(db.m_db, "old", copy, kept_from.size(), kept_from.size(), + SQLITE_DESERIALIZE_FREEONCLOSE | SQLITE_DESERIALIZE_READONLY) != SQLITE_OK) { + setError(error, QStringLiteral("cannot read the database the kept folios come from")); + return {}; + } + db.exec(QStringLiteral("PRAGMA trusted_schema = OFF")); + for (const QString &table : DocTables) { + Statement kind(db, QStringLiteral("SELECT type FROM old.sqlite_master WHERE name = ?")); + kind.bind({db.table(table)}); + if (!kind.next() || kind.value(0).toString() != QLatin1String("table")) { + setError(error, QStringLiteral("the kept folios' database has no table %1").arg(table)); + return {}; + } + } } db.exec(QStringLiteral("BEGIN")); @@ -683,10 +709,53 @@ QByteArray QetContainerDb::extract(QDomElement project, QString *error, const QB const QStringList conductor_keys = keys(all_conductors, QStringLiteral("c")); int e_index = 0, c_index = 0; + //A folio's id is its uuid when every folio has one of its own, so + //that a folio kept as it was keeps its rows (format 3, D4) + bool uuid_ids = true; + { + QSet seen; + for (const QDomElement &d : diagrams) { + const QString uuid = d.attribute(QStringLiteral("uuid")); + if (QUuid(uuid).isNull() || seen.contains(uuid)) { uuid_ids = false; break; } + seen.insert(uuid); + } + } + if (uuid_ids) { + ok = ok && meta.run({QStringLiteral("folio_ids"), QStringLiteral("uuid")}); + } + int pos = 0; for (QDomElement d : diagrams) { - const QString id = QStringLiteral("f%1").arg(++pos); + ++pos; + const QString id = uuid_ids ? d.attribute(QStringLiteral("uuid")) : QStringLiteral("f%1").arg(pos); + //Kept as it was: its rows from the database it was read from + if (d.hasAttribute(KeptPartAttribute)) { + if (kept_from.isEmpty() || !uuid_ids) { + setError(error, QStringLiteral("a folio kept as it was, without its database")); + return {}; + } + Statement copy(db, QStringLiteral("INSERT INTO folio SELECT id, ?, %1, props_at, props_attrs " + "FROM old.folio WHERE id = ?") + .arg(QStringLiteral("\"") + FolioData.join(QStringLiteral("\", \"")) + QStringLiteral("\""))); + ok = ok && copy.run({pos, id}) && sqlite3_changes(db.m_db) == 1; + for (const QString &sql : { + QStringLiteral("INSERT INTO folio_property SELECT * FROM old.folio_property WHERE folio_id = ? ORDER BY rowid"), + QStringLiteral("INSERT INTO element SELECT * FROM old.element WHERE folio_id = ? ORDER BY rowid"), + QStringLiteral("INSERT INTO element_info SELECT * FROM old.element_info WHERE element_key IN " + "(SELECT key FROM old.element WHERE folio_id = ?) ORDER BY rowid"), + QStringLiteral("INSERT INTO link SELECT * FROM old.link WHERE element_key IN " + "(SELECT key FROM old.element WHERE folio_id = ?) ORDER BY rowid"), + QStringLiteral("INSERT INTO conductor SELECT * FROM old.conductor WHERE folio_id = ? ORDER BY rowid")}) { + Statement rows(db, sql); + ok = ok && rows.run({id}); + } + if (!ok) { + setError(error, QStringLiteral("folio %1 kept as it was is not in its database").arg(id)); + return {}; + } + continue; + } QVariantList row{id, pos}; row += take(d, FolioData); QDomElement props = d.firstChildElement(QStringLiteral("properties")); @@ -859,3 +928,18 @@ bool QetContainerDb::restoreFolio(QDomElement diagram, const QByteArray &databas Database db; return openDatabase(db, database, error) && restoreDiagram(db, diagram, error); } + +/** + @brief QetContainerDb::foliosByUuid + @return true if @p database, a format 3 project.sqlite, names its + folios by their uuid: a folio written as it was read keeps its rows + under it (stage D4) +*/ +bool QetContainerDb::foliosByUuid(const QByteArray &database) +{ + Database db; + if (!openDatabase(db, database, nullptr) || db.m_prefix.isEmpty()) + return false; + Statement s(db, QStringLiteral("SELECT value FROM meta WHERE key = 'folio_ids'")); + return s.next() && s.value(0).toString() == QLatin1String("uuid"); +} diff --git a/sources/container/qetcontainerdb.h b/sources/container/qetcontainerdb.h index 5e1ec63ca..8e49eca63 100644 --- a/sources/container/qetcontainerdb.h +++ b/sources/container/qetcontainerdb.h @@ -38,9 +38,12 @@ namespace QetContainerDb { QByteArray extract(QDomElement project, QString *error = nullptr, - const QByteArray &index = QByteArray()); + const QByteArray &index = QByteArray(), + const QByteArray &kept_from = QByteArray()); bool restore(QDomElement project, const QByteArray &database, QString *error = nullptr); bool restoreFolio(QDomElement diagram, const QByteArray &database, QString *error = nullptr); + /// true if @p database names its folios by their uuid (format 3) + bool foliosByUuid(const QByteArray &database); } #endif // QETCONTAINERDB_H diff --git a/sources/diagram.cpp b/sources/diagram.cpp index 5ce6078ec..fe7d34346 100644 --- a/sources/diagram.cpp +++ b/sources/diagram.cpp @@ -798,9 +798,6 @@ QDomElement Diagram::deferredXml() { if (m_deferred.isNull() && !m_deferred_part.isEmpty() && m_project) { m_deferred = m_project->readFolioPart(m_deferred_part); - if (!m_deferred.isNull()) { - m_deferred_part.clear(); - } } return m_deferred; } diff --git a/sources/diagram.h b/sources/diagram.h index ca5e13951..a73f9938c 100644 --- a/sources/diagram.h +++ b/sources/diagram.h @@ -219,6 +219,7 @@ class Diagram : public QGraphicsScene /// project was opened from, read when they are first needed /// (stage D2) void setDeferredPart(const QString &file) {m_deferred = QDomElement(); m_deferred_part = file;} + QString deferredPart() const {return m_deferred_part;} QString headerXml(); /// The folio's text in the file it was read from, while it is /// not built (QETProject::toXml()) diff --git a/sources/qetproject.cpp b/sources/qetproject.cpp index 11b8c3f22..0e1a2b726 100644 --- a/sources/qetproject.cpp +++ b/sources/qetproject.cpp @@ -18,6 +18,7 @@ #include "qetproject.h" #include "container/qetcontainer.h" +#include "container/qetcontainerdb.h" #include "ElementsCollection/xmlelementcollection.h" #include "autoNum/assignvariables.h" @@ -2293,8 +2294,12 @@ void QETProject::autoFolioNumberingSelectedFolios(int from, @brief QETProject::toXml @return un document XML representant le projet */ -QDomDocument QETProject::toXml(QHash *kept_folios) +QDomDocument QETProject::toXml(QHash *kept_folios, QStringList *kept_parts) { + //With @p kept_parts, the same for a .qetz opened from its index: a + //folio not built is written as , its + //part and its rows copied by QetContainer::split() + //(stage D4) //With @p kept_folios, a folio not built (QET_LAZY_FOLIOS) is //written as a placeholder and its text //as read from the file goes in kept_folios, for @@ -2308,8 +2313,8 @@ QDomDocument QETProject::toXml(QHash *kept_folios) //is built with the folios its links reach, as when it is shown: //their saved texts can show its data (a report, its partner's //folio label). - const bool keep = kept_folios && unloadedFolioCount() - && m_kept_folios_trusted && !m_kept_folios_stale; + const bool keep = unloadedFolioCount() && !m_kept_folios_stale + && ((kept_folios && m_kept_folios_trusted) || (kept_parts && m_kept_parts_trusted)); if (keep) { for (Diagram *diagram : std::as_const(m_diagrams_list)) if (diagram->headerChanged()) @@ -2423,6 +2428,15 @@ QDomDocument QETProject::toXml(QHash *kept_folios) << "[" << diagram << "]"; + if (keep && !diagram->isLoaded() && kept_parts) { + auto placeholder = xml_doc.createElement(QStringLiteral("diagram")); + placeholder.setAttribute(QStringLiteral("qetz-kept-part"), diagram->deferredPart()); + placeholder.setAttribute(QStringLiteral("uuid"), diagram->uuid().toString()); + kept_parts->append(diagram->deferredPart()); + project_root.appendChild(placeholder); + ++order_num; + continue; + } if (keep && !diagram->isLoaded()) { auto placeholder = xml_doc.createElement(QStringLiteral("diagram")); placeholder.setAttribute(QStringLiteral("qet_kept_folio"), order_num); @@ -2520,7 +2534,7 @@ bool QETProject::isContainerPath(const QString &path) the whole archive is written. */ bool QETProject::writeContainer(const QDomDocument &project, const QString &path, QString *error, - const QByteArray &index) + const QByteArray &index, const QHash &kept_from) { //The project database, for opening without reading the folios //(stage D2), is in project.sqlite with the @@ -2528,7 +2542,7 @@ bool QETProject::writeContainer(const QDomDocument &project, const QString &path QList entries = QetContainer::split( project, QStringLiteral("QElectroTech %1").arg(QetVersion::currentVersion().toString()), - error, index); + error, index, kept_from); return !entries.isEmpty() && QetZip::write(path, entries, error); } @@ -2541,8 +2555,12 @@ bool QETProject::writeContainer(const QDomDocument &project, const QString &path */ bool QETProject::writeContainerFile(const QString &path, QString *error) { - const QDomDocument document = toXml(); //every folio built - return writeContainer(document, path, error, containerIndex()); + //The folios not built, written as they were read, when that is + //what building them would write (toXml()); every other folio built + QStringList kept; + const QDomDocument document = toXml(nullptr, &kept); + return writeContainer(document, path, error, containerIndex(), + kept.isEmpty() ? QHash() : m_container_files); } /** @@ -2552,7 +2570,11 @@ bool QETProject::writeContainerFile(const QString &path, QString *error) */ QByteArray QETProject::containerIndex() { - loadFolios(); + //Opened from an index, the folios not built have their rows in the + //database, whose keys are unique; the built ones are looked at + if (!m_kept_parts_trusted) { + loadFolios(); + } QSet seen; QList> headers; auto unique = [&seen](const QUuid &uuid) { @@ -2562,6 +2584,10 @@ QByteArray QETProject::containerIndex() }; for (Diagram *diagram : std::as_const(m_diagrams_list)) { if (!unique(diagram->uuid())) return QByteArray(); + if (!diagram->isLoaded()) { + headers.append({diagram->uuid(), diagram->headerXml()}); + continue; + } for (Element *element : diagram->elements()) if (!unique(element->uuid())) return QByteArray(); for (Conductor *conductor : diagram->conductors()) @@ -3224,9 +3250,13 @@ void QETProject::readProjectXml(QDomDocument &xml_project) for (Diagram *diagram : std::as_const(m_diagrams_list)) if (!diagram->isLoaded() && diagram->keptText().isEmpty()) m_kept_folios_trusted = false; + m_kept_parts_trusted = m_opened_from_index && unloadedFolioCount() > 0 + && m_project_qet_version == QetVersion::currentVersion() + && m_data_base.rowsFromDocument() + && QetContainerDb::foliosByUuid(m_container_files.value(QStringLiteral("project.sqlite"))); m_kept_folios_stale = false; m_edited_since_open = false; - if (m_kept_folios_trusted) + if (m_kept_folios_trusted || m_kept_parts_trusted) for (Diagram *diagram : std::as_const(m_diagrams_list)) diagram->rememberHeader(); if (qEnvironmentVariableIntValue("QET_LAZY_FOLIOS_TRACE") == 1) diff --git a/sources/qetproject.h b/sources/qetproject.h index a9f79f919..18e061505 100644 --- a/sources/qetproject.h +++ b/sources/qetproject.h @@ -313,7 +313,8 @@ class QETProject : public QObject void autoFolioNumberingNewFolios (); void autoFolioNumberingSelectedFolios(int, int, const QString&); - QDomDocument toXml(QHash *kept_folios = nullptr); + QDomDocument toXml(QHash *kept_folios = nullptr, + QStringList *kept_parts = nullptr); static QString spliceKeptFolios(QString text, const QHash &kept_folios); bool close(); QETResult write(); @@ -443,7 +444,8 @@ class QETProject : public QObject QString besideContainerPath() const; QETResult writeCopyForOlderVersions(const QString &path); static bool writeContainer(const QDomDocument &project, const QString &path, - QString *error = nullptr, const QByteArray &index = QByteArray()); + QString *error = nullptr, const QByteArray &index = QByteArray(), + const QHash &kept_from = QHash()); bool writeContainerFile(const QString &path, QString *error = nullptr); QByteArray containerIndex(); QDomElement readFolioPart(const QString &file); @@ -535,6 +537,9 @@ class QETProject : public QObject /// Opened from a .qetz's index (stage D2): the /// database filled from it, each folio's items in its part bool m_opened_from_index = false; + /// Saving a .qetz may copy the part of a folio not built + /// (toXml(), stage D4) + bool m_kept_parts_trusted = false; QHash m_container_files; QStringList m_folio_files; QList> m_folio_headers; diff --git a/tests/qttest/lazy_folios_probe.cpp b/tests/qttest/lazy_folios_probe.cpp index c9f3c5c1b..79a5815b9 100644 --- a/tests/qttest/lazy_folios_probe.cpp +++ b/tests/qttest/lazy_folios_probe.cpp @@ -176,6 +176,23 @@ static QString saved(QETProject &project) return QETProject::spliceKeptFolios(document.toString(4), kept); } + //@p xml's lines in order, any uuid not in the file @p path "new": a + //pasted copy's and a new wire's are random, and a folio's symbols are + //saved in uuid order +static QStringList sortedLines(QString xml, const QString &path) +{ + QFile file(path); file.open(QIODevice::ReadOnly); + const QString before = QString::fromUtf8(file.readAll()); + static const QRegularExpression uuid_re(QStringLiteral("\\{[0-9a-f-]{36}\\}")); + for (auto it = uuid_re.globalMatch(QString(xml)) ; it.hasNext() ; ) { + const QString uuid = it.next().captured(0); + if (!before.contains(uuid)) xml.replace(uuid, QStringLiteral("new")); + } + QStringList list = xml.split(QLatin1Char('\n')); + list.sort(); + return list; +} + //The rows of the project database's tables, in their order, read as //every caller reads them (newQuery(): brought up to date first) static QStringList tables(QETProject &project) @@ -626,6 +643,43 @@ int main(int argc, char **argv) for (int i = 0; i < folios.size(); ++i) same(indexed, i, "shown from the .qetz"); check(indexed.toXml().toString() == eager.toXml().toString(), "the .qetz saves as when built"); + + //Saved as a .qetz without building the folios not touched + //(D4): their parts and rows copied as they were + QETProject untouched(zipped); + const int unbuilt = untouched.unloadedFolioCount(); + const QString kept_path = output + "/kept.qetz"; + check(untouched.writeContainerFile(kept_path, &error), "saved as a .qetz without building"); + std::printf(".qetz saved untouched: %d of %d folios not built before, %d after\n", + unbuilt, int(folios.size()), untouched.unloadedFolioCount()); + check(untouched.unloadedFolioCount() > 0, "saving an untouched .qetz does not build every folio"); + qunsetenv("QET_LAZY_FOLIOS"); + QETProject kept_back(kept_path); + qputenv("QET_LAZY_FOLIOS", "1"); + check(kept_back.toXml().toString() == eager.toXml().toString(), + "a .qetz saved without building holds the project"); + + //After everyday edits on a folio + QETProject edited_z(zipped); + edit(edited_z, busiest); + const QString edited_path = output + "/edited.qetz"; + check(edited_z.writeContainerFile(edited_path, &error), "edited .qetz saved"); + std::printf(".qetz saved after the edits: %d folios not built\n", edited_z.unloadedFolioCount()); + check(edited_z.unloadedFolioCount() > 0, "saving an edited .qetz does not build every folio"); + //Against the same edits made with every folio built, saved and + //read back too: what a save and a reading do to a project is + //not what is checked here + const QString usual_path = output + "/edited-usual.qetz"; + qunsetenv("QET_LAZY_FOLIOS"); + { + QETProject edited_usual(path); + edit(edited_usual, busiest); + check(edited_usual.writeContainerFile(usual_path, &error), "edits saved with every folio built"); + } + QETProject edited_back(edited_path), usual_back(usual_path); + qputenv("QET_LAZY_FOLIOS", "1"); + check(sortedLines(edited_back.toXml().toString(), path) == sortedLines(usual_back.toXml().toString(), path), + "an edited .qetz saved without building holds what the edits made"); } std::printf("PASS: %d folios\n", int(folios.size())); } catch (const std::exception &error) { From 2f94724df361e958917b5a3f0f2672793577a1a8 Mon Sep 17 00:00:00 2001 From: ispyisail Date: Sun, 11 Oct 2026 22:15:20 +1300 Subject: [PATCH 06/26] Save a .qetz without building partners: symbols save the links they wait for (stage D5a) A built symbol whose partner is on a folio not built yet (QET_LAZY_FOLIOS) now saves that link too, in the order it was saved with (Element::toXml(); QETProject::waitingLinkTargets() keeps it to partners the project has, so a link to a symbol gone is dropped as it always was). With that, and with no partner's text in a .qetz's folio files (D3c), saving a .qetz no longer builds the folios of the partners built folios wait for; table chains still are (a table saves the table before it), and a .qet saved as read (QET_LAZY_SAVE_KEPT) still builds them, its folio files holding partners' texts. industrial.qetz: an untouched save builds no folio (47 of 47 stay not built; 19 before), after everyday edits on its busiest folio 29 stay not built (7 before). tst_lazyfolios compares the links an edited .qetz saved without building holds, symbol by symbol in their order, with those the same edits saved with every folio built -- it fails when waiting links are not saved (the drawing alone cannot tell: links are kept on both sides, and a symbol reread gets its partner's back, in another order). Corpus, 342 projects: a .qetz saved without building, read and saved as .qet = eager = master. ctest 96/96 with and without QET_LAZY_FOLIOS=1. Co-Authored-By: Claude Opus 5.5 --- sources/qetgraphicsitem/element.cpp | 36 +++++++++++++++++++++----- sources/qetproject.cpp | 16 ++++++++---- sources/qetproject.h | 4 +++ tests/qttest/lazy_folios_probe.cpp | 39 +++++++++++++++++++++++++++++ 4 files changed, 84 insertions(+), 11 deletions(-) diff --git a/sources/qetgraphicsitem/element.cpp b/sources/qetgraphicsitem/element.cpp index 6a59bbcbc..43511ae37 100644 --- a/sources/qetgraphicsitem/element.cpp +++ b/sources/qetgraphicsitem/element.cpp @@ -1250,17 +1250,41 @@ QDomElement Element::toXml( //if this element is linked to other elements, //save the uuid of each other elements - if (! isFree()) { + //A partner on a folio not built yet (QET_LAZY_FOLIOS), which this + //symbol will be linked to, is saved too, in the order the symbol was + //saved with: saving needs no partner built (stage D5a) + QList> links; + if (keepsSavedLinks() && diagram() && diagram()->project()) { + const QSet waiting = diagram()->project()->waitingLinkTargets(); + QSet written; + for (const auto &link_info : std::as_const(tmp_uuids_link)) { + Element *linked = nullptr; + for (Element *elmt : std::as_const(connected_elements)) + if (elmt->uuid() == link_info.uuid && !written.contains(elmt)) { linked = elmt; break; } + if (linked) { + written.insert(linked); + links.append({linked->uuid(), m_group_index_map.value(linked, -1)}); + } else if (waiting.contains(link_info.uuid)) { + links.append({link_info.uuid, link_info.group_index}); + } + } + for (Element *elmt : std::as_const(connected_elements)) + if (!written.contains(elmt)) + links.append({elmt->uuid(), m_group_index_map.value(elmt, -1)}); + } else { + for (Element *elmt : std::as_const(connected_elements)) + links.append({elmt->uuid(), m_group_index_map.value(elmt, -1)}); + } + if (!links.isEmpty()) { QDomElement links_uuids = document.createElement(QStringLiteral("links_uuids")); - foreach (Element *elmt, connected_elements) { + for (const auto &link : std::as_const(links)) { QDomElement link_uuid = document.createElement(QStringLiteral("link_uuid")); - link_uuid.setAttribute(QStringLiteral("uuid"), elmt->uuid().toString()); + link_uuid.setAttribute(QStringLiteral("uuid"), link.first.toString()); // Save group index if assigned (for slave->master links) - int gi = m_group_index_map.value(elmt, -1); - if (gi >= 0) { - link_uuid.setAttribute(QStringLiteral("group_index"), gi); + if (link.second >= 0) { + link_uuid.setAttribute(QStringLiteral("group_index"), link.second); } links_uuids.appendChild(link_uuid); diff --git a/sources/qetproject.cpp b/sources/qetproject.cpp index 0e1a2b726..bfc52f86a 100644 --- a/sources/qetproject.cpp +++ b/sources/qetproject.cpp @@ -2327,17 +2327,21 @@ QDomDocument QETProject::toXml(QHash *kept_folios, QStringList QetGraphicsTableItem::xmlTagName()).isEmpty()) buildFolioToShow(diagram); //A built folio is written from its items, which need their - //partners: a symbol saves its links, a report the position of - //its partner. Build the folios of every partner still waiting, - //and of every table chain, until none is + //partners: a table saves the table before it, and in a .qet a + //report the position of its partner. Build the folios of every + //table chain, and for a .qet of every partner still waiting, + //until none is. A symbol saves the partners it waits for itself + //(Element::toXml()), and a .qetz keeps no partner's text (D3c): + //saving a .qetz builds no partner (stage D5a) for (int built = -1 ; built != unloadedFolioCount() ; ) { built = unloadedFolioCount(); QSet waiting; QSet chained; for (Diagram *diagram : builtFolios()) { - for (Element *element : diagram->elements()) - waiting.unite(element->pendingLinks()); + if (kept_folios) + for (Element *element : diagram->elements()) + waiting.unite(element->pendingLinks()); for (Diagram *other : tableChainFolios(diagram)) if (!other->isLoaded()) chained.insert(other); } @@ -2351,6 +2355,7 @@ QDomDocument QETProject::toXml(QHash *kept_folios, QStringList } else { loadFolios(); } + m_waiting_link_targets = unloadedFolioCount() ? symbolsOnUnbuiltFolios() : QSet(); //QET_CHECK_ELEMENT_INFO_STORE=1: say whether the stores of symbol //information, folio title blocks and wire properties agree with every placed symbol, folio and //wire, at each save. A test and debugging aid; it changes nothing. @@ -2478,6 +2483,7 @@ QDomDocument QETProject::toXml(QHash *kept_folios, QStringList // Write the elements collection. project_root.appendChild(m_elements_collection->root().cloneNode(true)); + m_waiting_link_targets.clear(); return(xml_doc); } diff --git a/sources/qetproject.h b/sources/qetproject.h index 18e061505..3a51c0d1f 100644 --- a/sources/qetproject.h +++ b/sources/qetproject.h @@ -133,6 +133,9 @@ class QETProject : public QObject void buildFoliosHolding(ElementData::Types kinds, bool free_only); void buildFoliosHoldingSymbols(const QSet &symbols); QSet symbolsOnUnbuiltFolios() const; + /// While saving: the symbols on folios not built a built symbol + /// may be saved linked to (Element::toXml()) + QSet waitingLinkTargets() const {return m_waiting_link_targets;} QSet tableChainFolios(Diagram *folio) const; void loadFolios() const; void buildFolios(const QList &folios) const; @@ -540,6 +543,7 @@ class QETProject : public QObject /// Saving a .qetz may copy the part of a folio not built /// (toXml(), stage D4) bool m_kept_parts_trusted = false; + QSet m_waiting_link_targets; QHash m_container_files; QStringList m_folio_files; QList> m_folio_headers; diff --git a/tests/qttest/lazy_folios_probe.cpp b/tests/qttest/lazy_folios_probe.cpp index 79a5815b9..1704a0a9d 100644 --- a/tests/qttest/lazy_folios_probe.cpp +++ b/tests/qttest/lazy_folios_probe.cpp @@ -23,6 +23,9 @@ #include "../../sources/qetgraphicsitem/element.h" #include "../../sources/autoNum/elementautonumschemecommand.h" #include "../../sources/elementprovider.h" +#include "../../sources/container/qetzip.h" +#include +#include #include "../../sources/TerminalStrip/terminalstrip.h" #include "../../sources/ui/elementpropertieswidget.h" #include "../../sources/conductorautonumerotation.h" @@ -193,6 +196,38 @@ static QStringList sortedLines(QString xml, const QString &path) return list; } + //The saved links of a .qetz, symbol by symbol in their order: what a + //symbol saves of its partners, as its file holds it +static QStringList savedLinks(const QString &qetz, const QString &path) +{ + QFile file(qetz); file.open(QIODevice::ReadOnly); + QList entries; + QString error; + check(QetZip::fromBytes(file.readAll(), &entries, &error), "a .qetz read"); + QTemporaryDir dir; + const QString db_path = dir.filePath(QStringLiteral("project.sqlite")); + for (const QetZip::Entry &entry : entries) + if (entry.name == QLatin1String("project.sqlite")) { + QFile db(db_path); db.open(QIODevice::WriteOnly); db.write(entry.data); + } + QMap links; //symbol -> its partners in order + { + QSqlDatabase db = QSqlDatabase::addDatabase(QStringLiteral("QSQLITE"), QStringLiteral("saved_links")); + db.setDatabaseName(db_path); + check(db.open(), "a .qetz's database opened"); + QSqlQuery query(db); + check(query.exec(QStringLiteral("SELECT element_key, linked_uuid FROM doc_link ORDER BY element_key, ord")), + "saved links read"); + while (query.next()) + links[query.value(0).toString()] << query.value(1).toString(); + } + QSqlDatabase::removeDatabase(QStringLiteral("saved_links")); + QStringList rows; + for (auto it = links.cbegin() ; it != links.cend() ; ++it) + rows << it.key() + QStringLiteral(": ") + it.value().join(QLatin1Char(',')); + return sortedLines(rows.join(QLatin1Char('\n')), path); +} + //The rows of the project database's tables, in their order, read as //every caller reads them (newQuery(): brought up to date first) static QStringList tables(QETProject &project) @@ -680,6 +715,10 @@ int main(int argc, char **argv) qputenv("QET_LAZY_FOLIOS", "1"); check(sortedLines(edited_back.toXml().toString(), path) == sortedLines(usual_back.toXml().toString(), path), "an edited .qetz saved without building holds what the edits made"); + //Each symbol's links as saved, partners on folios not built + //included, in their order (D5a) + check(savedLinks(edited_path, path) == savedLinks(usual_path, path), + "an edited .qetz saved without building keeps every symbol's links"); } std::printf("PASS: %d folios\n", int(folios.size())); } catch (const std::exception &error) { From b891b6354a06a54104144d20bb1336c0bab5067c Mon Sep 17 00:00:00 2001 From: ispyisail Date: Mon, 12 Oct 2026 02:22:25 +1300 Subject: [PATCH 07/26] Show a folio without building its partners' folios (stage D5b) Showing a folio built it and every folio holding a partner of its symbols. Now a partner is built alone, as a symbol, into its folio's scene (Diagram::buildSymbols()), and reused when that folio is built. - A linked folio report built alone had no wires when it was linked, so its texts showing a wire's properties stayed empty: after any build, linked reports on the built folios and their partners look at their wires again, as opening does. Unlinked reports keep their texts, as opening leaves them. - A report whose partner was built alone builds the partner's folio when its potential is followed. - tableChainFolios() parsed every unbuilt folio's file. The index now lists the drawn tables and the table each continues (folio_table), and a .qetz opened from it uses that list. An index without it falls back to the folio files. - A .qetz's project.sqlite is opened once per project (QetContainerDb::FolioReader), not once per folio built. nuri (136 folios), opened fresh, one folio shown: median 1104 -> 184 ms, each show builds one folio; industrial 234 -> 87 ms. tst_lazyfolios: a folio whose table continues another is shown first from a .qetz (fails when the index's chains are ignored). ctest 96/96 eager and lazy; 342-project corpus: master, lazy, .qetz lazy and .qetz kept all the same. Co-Authored-By: Claude Opus 5.5 --- sources/container/qetcontainer.cpp | 11 +- sources/container/qetcontainer.h | 5 +- sources/container/qetcontainerdb.cpp | 41 +++++++ sources/container/qetcontainerdb.h | 13 ++ sources/dataBase/projectdatabase.cpp | 37 +++++- sources/dataBase/projectdatabase.h | 10 +- sources/diagram.cpp | 176 +++++++++++++++++++-------- sources/diagram.h | 9 ++ sources/qetgraphicsitem/element.cpp | 9 +- sources/qetgraphicsitem/terminal.cpp | 8 ++ sources/qetproject.cpp | 104 ++++++++++++++-- sources/qetproject.h | 7 ++ tests/qttest/lazy_folios_probe.cpp | 17 +++ 13 files changed, 371 insertions(+), 76 deletions(-) diff --git a/sources/container/qetcontainer.cpp b/sources/container/qetcontainer.cpp index 1bfa5f8e6..df69c9e54 100644 --- a/sources/container/qetcontainer.cpp +++ b/sources/container/qetcontainer.cpp @@ -355,7 +355,8 @@ QList QetContainer::split(const QDomDocument &project, const QStr engineering data back from project.sqlite. */ bool QetContainer::readFolio(const QHash &files, const QString &file, - QDomDocument *folio, QString *error) + QDomDocument *folio, QString *error, + QetContainerDb::FolioReader *reader) { if (!files.contains(file)) { setError(error, QStringLiteral("part \"%1\" is missing").arg(file)); @@ -380,9 +381,11 @@ bool QetContainer::readFolio(const QHash &files, const QStr image.insertBefore(folio->createTextNode(QString::fromLatin1(files.value(picture).toBase64())), image.firstChild()); } - return !files.contains(QStringLiteral("project.sqlite")) - || QetContainerDb::restoreFolio(folio->documentElement(), - files.value(QStringLiteral("project.sqlite")), error); + if (!files.contains(QStringLiteral("project.sqlite"))) + return true; + return reader ? reader->restore(folio->documentElement(), error) + : QetContainerDb::restoreFolio(folio->documentElement(), + files.value(QStringLiteral("project.sqlite")), error); } /** diff --git a/sources/container/qetcontainer.h b/sources/container/qetcontainer.h index 16da23a29..814ca2595 100644 --- a/sources/container/qetcontainer.h +++ b/sources/container/qetcontainer.h @@ -18,6 +18,7 @@ #ifndef QETCONTAINER_H #define QETCONTAINER_H +#include "qetcontainerdb.h" #include "qetzip.h" #include @@ -59,8 +60,10 @@ namespace QetContainer QString *error = nullptr, QStringList *folio_files = nullptr); /// A folio part of @p files, which join() left out, with its /// pictures and its engineering data back (stage D2) + /// @p reader, if any, reads the engineering data of @p files bool readFolio(const QHash &files, const QString &file, - QDomDocument *folio, QString *error = nullptr); + QDomDocument *folio, QString *error = nullptr, + QetContainerDb::FolioReader *reader = nullptr); /// Take what QElectroTech computes out of a folio (format 3) void dropComputed(QDomElement diagram, bool terminal_lists); diff --git a/sources/container/qetcontainerdb.cpp b/sources/container/qetcontainerdb.cpp index 7ec4c6e0a..541b1d00f 100644 --- a/sources/container/qetcontainerdb.cpp +++ b/sources/container/qetcontainerdb.cpp @@ -929,6 +929,47 @@ bool QetContainerDb::restoreFolio(QDomElement diagram, const QByteArray &databas return openDatabase(db, database, error) && restoreDiagram(db, diagram, error); } +struct QetContainerDb::FolioReader::Private +{ + QByteArray database; + Database db; + bool opened = false; + QString open_error; +}; + +/** + @brief QetContainerDb::FolioReader::FolioReader + Reads folios' engineering data from @p database, opened the first time + one is read and kept open: a folio built after the project was opened + from a .qetz no longer copies the whole database (stage D5) +*/ +QetContainerDb::FolioReader::FolioReader(const QByteArray &database) : + d(std::make_unique()) +{ + d->database = database; +} + +QetContainerDb::FolioReader::~FolioReader() = default; + +/** + @brief QetContainerDb::FolioReader::restore + restoreFolio() for @p diagram +*/ +bool QetContainerDb::FolioReader::restore(QDomElement diagram, QString *error) +{ + if (!d->opened) { + d->opened = true; + if (!openDatabase(d->db, d->database, &d->open_error) && d->open_error.isEmpty()) + d->open_error = QStringLiteral("the project database cannot be read"); + d->database = QByteArray(); + } + if (!d->open_error.isEmpty()) { + setError(error, d->open_error); + return false; + } + return restoreDiagram(d->db, diagram, error); +} + /** @brief QetContainerDb::foliosByUuid @return true if @p database, a format 3 project.sqlite, names its diff --git a/sources/container/qetcontainerdb.h b/sources/container/qetcontainerdb.h index 8e49eca63..1b616acf2 100644 --- a/sources/container/qetcontainerdb.h +++ b/sources/container/qetcontainerdb.h @@ -21,6 +21,8 @@ #include #include +#include + /** @brief The QetContainerDb namespace The engineering data of a project, moved between its document and the @@ -42,6 +44,17 @@ namespace QetContainerDb const QByteArray &kept_from = QByteArray()); bool restore(QDomElement project, const QByteArray &database, QString *error = nullptr); bool restoreFolio(QDomElement diagram, const QByteArray &database, QString *error = nullptr); + /// restoreFolio() for the folios of one database, opened once + class FolioReader + { + public: + explicit FolioReader(const QByteArray &database); + ~FolioReader(); + bool restore(QDomElement diagram, QString *error = nullptr); + private: + struct Private; + std::unique_ptr d; + }; /// true if @p database names its folios by their uuid (format 3) bool foliosByUuid(const QByteArray &database); } diff --git a/sources/dataBase/projectdatabase.cpp b/sources/dataBase/projectdatabase.cpp index 3ebe41382..f43af8179 100644 --- a/sources/dataBase/projectdatabase.cpp +++ b/sources/dataBase/projectdatabase.cpp @@ -907,11 +907,14 @@ QStringList indexTables(const QSqlDatabase &db) This database written to an SQLite file, for a .qetz to be opened from (stage D2): every table but the drawing items, brought up to date first, with the folios' own data @p folio_headers (in folio - order, each as Diagram::headerXml() writes it) and the stamp of the - schema it was written with. + order, each as Diagram::headerXml() writes it), the drawn tables + @p tables and how they continue one another (which folios a folio's + tables must be built with), and the stamp of the schema it was written + with. @return the file, or nothing if it could not be written */ -QByteArray projectDataBase::indexFile(const QList> &folio_headers) +QByteArray projectDataBase::indexFile(const QList> &folio_headers, + const QList &tables) { if (m_content_changed) { updateDB(); @@ -952,7 +955,9 @@ QByteArray projectDataBase::indexFile(const QList> &fo && query.exec(QStringLiteral("DROP TABLE image")) && query.exec(QStringLiteral("CREATE TABLE index_meta (key TEXT PRIMARY KEY, value TEXT)")) && query.exec(QStringLiteral("CREATE TABLE folio_header (pos INTEGER PRIMARY KEY, " - "uuid TEXT NOT NULL, xml TEXT NOT NULL)")); + "uuid TEXT NOT NULL, xml TEXT NOT NULL)")) + && query.exec(QStringLiteral("CREATE TABLE folio_table (uuid TEXT NOT NULL, " + "folio_uuid TEXT NOT NULL, previous_uuid TEXT)")); file.transaction(); query.prepare(QStringLiteral("INSERT INTO index_meta (key, value) VALUES (?, ?)")); for (const auto &meta : {std::pair{QStringLiteral("schema"), stamp}, @@ -970,6 +975,14 @@ QByteArray projectDataBase::indexFile(const QList> &fo query.addBindValue(header.second); ok = ok && query.exec(); } + query.prepare(QStringLiteral("INSERT INTO folio_table (uuid, folio_uuid, previous_uuid) " + "VALUES (?, ?, ?)")); + for (const FolioTable &table : tables) { + query.addBindValue(table.table.toString()); + query.addBindValue(table.folio.toString()); + query.addBindValue(table.previous.isNull() ? QVariant() : QVariant(table.previous.toString())); + ok = ok && query.exec(); + } ok = file.commit() && ok; ok = query.exec(QStringLiteral("VACUUM")) && ok; file.close(); @@ -1015,12 +1028,15 @@ void projectDataBase::discardFill() when written with this database's schema, its tables plain tables, and nothing in its schema run. @param folio_headers : the folios' own data, in folio order + @param folio_tables : if not null, the drawn tables indexFile() wrote, or + nothing if the index predates them @return false, with @p why set and nothing changed in the tables, if the file cannot be read */ bool projectDataBase::fillFromIndex(const QByteArray &index, QList> *folio_headers, - QString *why) + QString *why, + std::optional> *folio_tables) { auto refuse = [why](const QString &reason) { if (why) *why = reason; @@ -1093,6 +1109,17 @@ bool projectDataBase::fillFromIndex(const QByteArray &index, query.exec(QStringLiteral("SELECT uuid, xml FROM idx.folio_header ORDER BY pos")); while (query.next()) folio_headers->append({QUuid(query.value(0).toString()), query.value(1).toString()}); + if (folio_tables) { + folio_tables->reset(); + if (isTable(QStringLiteral("folio_table"))) { + QList read; + query.exec(QStringLiteral("SELECT uuid, folio_uuid, previous_uuid FROM idx.folio_table")); + while (query.next()) + read.append({QUuid(query.value(0).toString()), QUuid(query.value(1).toString()), + QUuid(query.value(2).toString())}); + *folio_tables = read; + } + } detach(); //The stores, from their tables, as the document would fill them diff --git a/sources/dataBase/projectdatabase.h b/sources/dataBase/projectdatabase.h index 15119a61d..ec0ac2e88 100644 --- a/sources/dataBase/projectdatabase.h +++ b/sources/dataBase/projectdatabase.h @@ -29,6 +29,8 @@ #include #include +#include + #include "../diagramcontext.h" #include "../titleblockproperties.h" #include "../conductorproperties.h" @@ -164,12 +166,16 @@ class projectDataBase : public QObject bool isBuildingFolio() const {return m_building_folios > 0;} bool rowsFromDocument() const {return m_rows_from_document;} QString schemaStamp() const; - QByteArray indexFile(const QList> &folio_headers); + /// A drawn table of a folio and the table it continues + struct FolioTable { QUuid table; QUuid folio; QUuid previous; }; + QByteArray indexFile(const QList> &folio_headers, + const QList &tables = QList()); void discardFill(); /// The tables are right as they are (filled from an index) void markCurrent() {m_content_changed = false;} bool fillFromIndex(const QByteArray &index, QList> *folio_headers, - QString *why = nullptr); + QString *why = nullptr, + std::optional> *folio_tables = nullptr); QString prefillReport() const; QSet foliosWithUnjoinedWires() const; QSet folioUuidsOfElements(const QSet &elements) const; diff --git a/sources/diagram.cpp b/sources/diagram.cpp index fe7d34346..7dcbf2427 100644 --- a/sources/diagram.cpp +++ b/sources/diagram.cpp @@ -788,6 +788,48 @@ void Diagram::ensureLoaded() } } +/** + @brief Diagram::buildSymbols + Build, of this folio not built yet (QET_LAZY_FOLIOS), the symbols + @p uuids only -- partners a shown folio draws from -- and keep them for + when the folio is built (stage D5b). They belong to the + folio's scene; the folio stays not built. + @return the symbols built now +*/ +QList Diagram::buildSymbols(const QSet &uuids) +{ + QList built; + if (isLoaded() || uuids.isEmpty()) { + return built; + } + const QDomElement xml = deferredXml(); + if (xml.isNull()) { + return built; + } + projectDataBase *db = m_project ? m_project->dataBase() : nullptr; + if (db) db->setBuildingFolio(true); + for (QDomElement element_xml : + QET::findInDomElement(xml, QStringLiteral("elements"), QStringLiteral("element"))) + { + const QUuid uuid(element_xml.attribute(QStringLiteral("uuid"))); + if (uuid.isNull() || !uuids.contains(uuid) || m_partial_elements.contains(uuid) + || !Element::valideXml(element_xml)) { + continue; + } + if (Element *element = buildElement(element_xml, m_partial_table, true)) { + m_partial_elements.insert(uuid, element); + //A partner unlinked or linked again: the folio is not + //saved as it was read any more + connect(element, &Element::linkedElementChanged, this, [this]() { + if (!isLoaded()) m_partial_changed = true; + }); + built << element; + } + } + if (db) db->setBuildingFolio(false); + return built; +} + /** @brief Diagram::deferredXml @return the XML of this folio's items while they are not built: kept @@ -826,6 +868,9 @@ bool Diagram::buildDeferred() m_loading_deferred = true; fromXml(xml, QPointF(), true, nullptr); m_loading_deferred = false; + m_partial_elements.clear(); + m_partial_table.clear(); + m_partial_changed = false; if (db) db->setBuildingFolio(false); return true; } @@ -1504,6 +1549,73 @@ void Diagram::folioSequentialsToXml(QHash &table_adr_id, + bool consider_informations) +{ + // cree un element dont le type correspond a l'id type + QString type_id = element_xml.attribute(QStringLiteral("type")); + ElementsLocation element_location; + if (type_id.startsWith(QStringLiteral("embed://"))) { + element_location = ElementsLocation(type_id, m_project); + } + else { + element_location = ElementsLocation(type_id); + } + + int state = 0; + Element *nvel_elmt = + ElementFactory::Instance() -> createElement( + element_location, nullptr, &state); + if (state) + { + QString debug_message = + QString("Diagram::fromXml() : Le chargement de " + "la description de l'element %1 a " + "echoue avec le code d'erreur %2").arg( + element_location.path()).arg(state); + qDebug() << qPrintable(debug_message); + delete nvel_elmt; + return nullptr; + } + + addItem(nvel_elmt); + //Loading fail, remove item from the diagram + if (!nvel_elmt->fromXml(element_xml, table_adr_id)) + { + removeItem(nvel_elmt); + delete nvel_elmt; + qDebug() << QStringLiteral("Diagram::fromXml() : Le chargement des parametres d'un element a echoue"); + return nullptr; + } else { + //A symbol saved without a uuid got a random one from + //Element::fromXml(): a different identity on every load, + //written out on the next save. Derive it instead from what + //the symbol is and where it sits on its folio -- never from + //the folio's index, so inserting or moving a folio does not + //change it. Only for a folio being loaded: a paste renews + //uuids anyway. + if (consider_informations && m_project + && QUuid(element_xml.attribute(QStringLiteral("uuid"))).isNull()) { + nvel_elmt->setUuid(m_project->derivedItemUuid( + QStringLiteral("element"), + QStringList{type_id, + element_xml.attribute(QStringLiteral("x")), + element_xml.attribute(QStringLiteral("y")), + element_xml.attribute(QStringLiteral("orientation"))} + .join(QLatin1Char('\n')))); + } + ItemGroups::setGroup(nvel_elmt, ItemGroups::read(element_xml)); + return nvel_elmt; + } +return nullptr; +} + /** @brief Diagram::fromXml Imports the described schema into an XML document. @@ -1925,65 +2037,23 @@ bool Diagram::fromXml(QDomElement &document, //Load all elements from the XML QList added_elements; QHash table_adr_id; + if (m_loading_deferred) { + table_adr_id = m_partial_table; //the terminals of symbols built already + } for (auto element_xml : QET::findInDomElement(root, QStringLiteral("elements"), QStringLiteral("element"))) { if (!Element::valideXml(element_xml)) continue; - - // cree un element dont le type correspond a l'id type - QString type_id = element_xml.attribute(QStringLiteral("type")); - ElementsLocation element_location; - if (type_id.startsWith(QStringLiteral("embed://"))) { - element_location = ElementsLocation(type_id, m_project); - } - else { - element_location = ElementsLocation(type_id); - } - - int state = 0; - Element *nvel_elmt = - ElementFactory::Instance() -> createElement( - element_location, nullptr, &state); - if (state) - { - QString debug_message = - QString("Diagram::fromXml() : Le chargement de " - "la description de l'element %1 a " - "echoue avec le code d'erreur %2").arg( - element_location.path()).arg(state); - qDebug() << qPrintable(debug_message); - delete nvel_elmt; - continue; - } - - addItem(nvel_elmt); - //Loading fail, remove item from the diagram - if (!nvel_elmt->fromXml(element_xml, table_adr_id)) - { - removeItem(nvel_elmt); - delete nvel_elmt; - qDebug() << QStringLiteral("Diagram::fromXml() : Le chargement des parametres d'un element a echoue"); - } else { - //A symbol saved without a uuid got a random one from - //Element::fromXml(): a different identity on every load, - //written out on the next save. Derive it instead from what - //the symbol is and where it sits on its folio -- never from - //the folio's index, so inserting or moving a folio does not - //change it. Only for a folio being loaded: a paste renews - //uuids anyway. - if (consider_informations && m_project - && QUuid(element_xml.attribute(QStringLiteral("uuid"))).isNull()) { - nvel_elmt->setUuid(m_project->derivedItemUuid( - QStringLiteral("element"), - QStringList{type_id, - element_xml.attribute(QStringLiteral("x")), - element_xml.attribute(QStringLiteral("y")), - element_xml.attribute(QStringLiteral("orientation"))} - .join(QLatin1Char('\n')))); + //Built already as another folio's partner (buildSymbols()) + if (m_loading_deferred) { + Element *built = m_partial_elements.value(QUuid(element_xml.attribute(QStringLiteral("uuid")))); + if (built) { + added_elements << built; + continue; } - ItemGroups::setGroup(nvel_elmt, ItemGroups::read(element_xml)); - added_elements << nvel_elmt; } + if (Element *nvel_elmt = buildElement(element_xml, table_adr_id, consider_informations)) + added_elements << nvel_elmt; } //Texts, images and shapes written before they carried a uuid (or diff --git a/sources/diagram.h b/sources/diagram.h index a73f9938c..d79439795 100644 --- a/sources/diagram.h +++ b/sources/diagram.h @@ -143,6 +143,11 @@ class Diagram : public QGraphicsScene /// The XML of this folio's items, while they are not built yet QDomElement m_deferred; QString m_deferred_part; + QHash m_partial_elements; + QHash m_partial_table; + bool m_partial_changed = false; + Element *buildElement(QDomElement element_xml, QHash &table_adr_id, + bool consider_informations); QString m_opened_header; QByteArray m_kept_text; bool m_loading_deferred = false; @@ -220,6 +225,10 @@ class Diagram : public QGraphicsScene /// (stage D2) void setDeferredPart(const QString &file) {m_deferred = QDomElement(); m_deferred_part = file;} QString deferredPart() const {return m_deferred_part;} + QList buildSymbols(const QSet &uuids); + /// Symbols built as partners (buildSymbols()) changed since: + /// the folio must be built to be saved + bool partialChanged() const {return m_partial_changed;} QString headerXml(); /// The folio's text in the file it was read from, while it is /// not built (QETProject::toXml()) diff --git a/sources/qetgraphicsitem/element.cpp b/sources/qetgraphicsitem/element.cpp index 43511ae37..ae497bafb 100644 --- a/sources/qetgraphicsitem/element.cpp +++ b/sources/qetgraphicsitem/element.cpp @@ -1654,10 +1654,11 @@ void Element::initLink(QETProject *prj) // if nothing to link return now if (tmp_uuids_link.isEmpty()) return; - //The built folios only: a partner on a folio not built yet - //(QET_LAZY_FOLIOS) is linked when its folio is, and the saved list - //is kept until then, for the order it gives - ElementProvider ep(prj->builtFolios()); + //The symbols built: on the folios built, and those built alone as + //partners on folios not built (Diagram::buildSymbols()). A partner + //not built yet (QET_LAZY_FOLIOS) is linked when it is, and the + //saved list is kept until then, for the order it gives + ElementProvider ep(prj->folios()); QList uuids; for (const auto &linkInfo : tmp_uuids_link) { uuids.append(linkInfo.uuid); diff --git a/sources/qetgraphicsitem/terminal.cpp b/sources/qetgraphicsitem/terminal.cpp index 0e3684cea..5cade6949 100644 --- a/sources/qetgraphicsitem/terminal.cpp +++ b/sources/qetgraphicsitem/terminal.cpp @@ -1102,6 +1102,14 @@ QList relatedPotentialTerminal ( project->buildFoliosHoldingSymbols(waiting); } QList elmt_list = terminal -> parentElement() -> linkedElements(); + //A partner built alone (stage D5b) has no wires + //yet: its folio is built + if (!elmt_list.isEmpty() && project && elmt_list.first()->diagram() + && !elmt_list.first()->diagram()->isLoaded() + && project->state() != QETProject::ProjectParsingRunning) { + project->buildFolios({elmt_list.first()->diagram()}); + elmt_list = terminal->parentElement()->linkedElements(); + } if (!elmt_list.isEmpty()) { return (elmt_list.first()->terminals()); diff --git a/sources/qetproject.cpp b/sources/qetproject.cpp index bfc52f86a..9937d7099 100644 --- a/sources/qetproject.cpp +++ b/sources/qetproject.cpp @@ -32,6 +32,7 @@ #include "diagram.h" #include "qetgraphicsitem/element.h" #include "qetgraphicsitem/conductor.h" +#include "qetgraphicsitem/dynamicelementtextitem.h" #include "qetgraphicsitem/ViewItem/qetgraphicstableitem.h" #include "qetgraphicsitem/ViewItem/projectdbmodel.h" #include "qetapp.h" @@ -487,7 +488,7 @@ QETProject::ProjectState QETProject::openFile(QFile *file) if (!index.isEmpty() && index.contains("folio_header") && QetContainer::join(entries, &document, &why, &folio_files)) { - if (m_data_base.fillFromIndex(index, &m_folio_headers, &why) + if (m_data_base.fillFromIndex(index, &m_folio_headers, &why, &m_index_tables) && m_folio_headers.size() == folio_files.size()) { xml_project = document; m_container_files = files; @@ -496,6 +497,7 @@ QETProject::ProjectState QETProject::openFile(QFile *file) } else { m_data_base.discardFill(); m_folio_headers.clear(); + m_index_tables.reset(); qInfo().noquote() << "Not opened from the index of" << fi.fileName() << ":" << why; } } @@ -728,6 +730,21 @@ void QETProject::buildFolios(const QList &folios) const for (Element *elmt : d->elements()) if (elmt->keepsSavedLinks()) elmt->initLink(self); + //A linked folio report's texts show a wire's properties: the + //report's own wire, else its partner's. A report built alone + //first, as a partner (buildPartnerSymbols()), linked before it had + //wires, and a report elsewhere may watch that partner: both look + //again now, as opening has them do once every wire exists (an + //unlinked report keeps its texts, as opening leaves them) + for (Diagram *d : std::as_const(built)) + for (Element *elmt : d->elements()) + if ((elmt->linkType() & Element::AllReport) && !elmt->linkedElements().isEmpty()) + { + emit elmt->linkedElementChanged(); + for (Element *partner : elmt->linkedElements()) + if (!built.contains(partner->diagram())) + emit partner->linkedElementChanged(); + } //Likewise a table whose previous table was on a folio not built for (Diagram *d : builtFolios()) for (QGraphicsItem *item : d->items()) @@ -913,6 +930,16 @@ QSet QETProject::tableChainFolios(Diagram *folio) const } } continue; + } + //Opened from an index: its tables, not the folio's part + if (m_index_tables) { + for (const projectDataBase::FolioTable &table : std::as_const(*m_index_tables)) { + if (table.folio == d->uuid()) { + folio_of.insert(table.table, d); + previous_of.insert(table.table, table.previous); + } + } + continue; } const QDomNodeList tables = d->deferredXml().elementsByTagName( QetGraphicsTableItem::xmlTagName()); @@ -983,11 +1010,43 @@ void QETProject::buildFolioToShow(Diagram *folio) return; } if (qEnvironmentVariableIntValue("QET_LAZY_FOLIOS_TRACE") == 1) { - qWarning().noquote() << "lazy folios:" << to_build.size() - << "folios built for the partners of folio" - << folio->folioIndex() + 1; + qWarning().noquote() << "lazy folios: the partners of folio" << folio->folioIndex() + 1 + << "built alone, on" << to_build.size() << "folios"; } - buildFolios(to_build); + //The partners alone, not their folios (stage D5b) + buildPartnerSymbols(to_build, partners); +} + +/** + @brief QETProject::buildPartnerSymbols + Build on the folios @p folios, not built, the symbols @p partners alone + (Diagram::buildSymbols()), as opening builds them: linked to the symbols + built already, their texts refreshed, in the state opening has. +*/ +void QETProject::buildPartnerSymbols(const QList &folios, const QSet &partners) +{ + const ProjectState state = m_state; + m_state = ProjectParsingRunning; + QList built; + for (Diagram *d : folios) { + const QList symbols = d->buildSymbols(partners); + if (!symbols.isEmpty()) { + built += symbols; + linkElementsToElementAutoNums(d); + } + } + for (Element *element : std::as_const(built)) { + element->initLink(this); + for (auto &text : element->dynamicTextItems()) + text->refreshLabelConnection(); + } + //A built symbol linked to one of them puts its partners back in + //their saved order + for (Diagram *d : builtFolios()) + for (Element *element : d->elements()) + if (element->keepsSavedLinks()) + element->initLink(this); + m_state = state; } /** @@ -2316,6 +2375,11 @@ QDomDocument QETProject::toXml(QHash *kept_folios, QStringList const bool keep = unloadedFolioCount() && !m_kept_folios_stale && ((kept_folios && m_kept_folios_trusted) || (kept_parts && m_kept_parts_trusted)); if (keep) { + //Partners built alone and linked or unlinked since: their folio + //is not what was read + for (Diagram *diagram : std::as_const(m_diagrams_list)) + if (!diagram->isLoaded() && diagram->partialChanged()) + buildFolios({diagram}); for (Diagram *diagram : std::as_const(m_diagrams_list)) if (diagram->headerChanged()) buildFolioToShow(diagram); @@ -2588,10 +2652,27 @@ QByteArray QETProject::containerIndex() seen.insert(uuid); return true; }; + //The drawn tables and the tables they continue: which folios a + //folio's tables are built with (tableChainFolios()) + QList tables; for (Diagram *diagram : std::as_const(m_diagrams_list)) { if (!unique(diagram->uuid())) return QByteArray(); if (!diagram->isLoaded()) { headers.append({diagram->uuid(), diagram->headerXml()}); + if (m_index_tables) { + for (const projectDataBase::FolioTable &table : std::as_const(*m_index_tables)) + if (table.folio == diagram->uuid()) + tables.append(table); + continue; + } + const QDomNodeList items = diagram->deferredXml().elementsByTagName( + QetGraphicsTableItem::xmlTagName()); + for (int i = 0 ; i < items.size() ; ++i) { + const QDomElement table = items.at(i).toElement(); + tables.append({QUuid(table.attribute(QStringLiteral("uuid"))), diagram->uuid(), + QUuid(table.firstChildElement(QStringLiteral("previous_table")) + .attribute(QStringLiteral("uuid")))}); + } continue; } for (Element *element : diagram->elements()) @@ -2599,8 +2680,14 @@ QByteArray QETProject::containerIndex() for (Conductor *conductor : diagram->conductors()) if (!unique(conductor->uuid())) return QByteArray(); headers.append({diagram->uuid(), diagram->headerXml()}); + for (QGraphicsItem *item : diagram->items()) { + if (item->type() == QetGraphicsTableItem::Type) { + auto table = static_cast(item); + tables.append({table->uuid(), diagram->uuid(), table->previousTableUuid()}); + } + } } - return m_data_base.indexFile(headers); + return m_data_base.indexFile(headers, tables); } /** @@ -3323,7 +3410,10 @@ QDomElement QETProject::readFolioPart(const QString &file) { QDomDocument folio; QString error; - if (!QetContainer::readFolio(m_container_files, file, &folio, &error)) { + if (!m_folio_reader) + m_folio_reader = std::make_unique( + m_container_files.value(QStringLiteral("project.sqlite"))); + if (!QetContainer::readFolio(m_container_files, file, &folio, &error, m_folio_reader.get())) { qWarning().noquote() << "Unable to read the folio" << file << ":" << error; return QDomElement(); } diff --git a/sources/qetproject.h b/sources/qetproject.h index 3a51c0d1f..aad50ad7e 100644 --- a/sources/qetproject.h +++ b/sources/qetproject.h @@ -32,6 +32,7 @@ #include "wiringrules.h" #include "diagram.h" #include "autoNum/assignvariables.h" +#include "container/qetcontainerdb.h" #ifdef BUILD_WITHOUT_KF # include "ui/nokde/kautosavefile.h" #else @@ -45,6 +46,7 @@ #include #include +#include #include class Diagram; @@ -130,6 +132,7 @@ class QETProject : public QObject QList cableLabelFolios(); QList foliosWithWiresNotReconnected(); void buildFolioToShow(Diagram *folio); + void buildPartnerSymbols(const QList &folios, const QSet &partners); void buildFoliosHolding(ElementData::Types kinds, bool free_only); void buildFoliosHoldingSymbols(const QSet &symbols); QSet symbolsOnUnbuiltFolios() const; @@ -545,8 +548,12 @@ class QETProject : public QObject bool m_kept_parts_trusted = false; QSet m_waiting_link_targets; QHash m_container_files; + ///Reads the folio parts' engineering data, opened once + std::unique_ptr m_folio_reader; QStringList m_folio_files; QList> m_folio_headers; + ///The drawn tables of the index opened from, if it has them + std::optional> m_index_tables; void readFoliosFromIndex(); bool m_kept_folios_stale = false; /// Something was edited since the project was opened: a table diff --git a/tests/qttest/lazy_folios_probe.cpp b/tests/qttest/lazy_folios_probe.cpp index 1704a0a9d..e0bcf3d4c 100644 --- a/tests/qttest/lazy_folios_probe.cpp +++ b/tests/qttest/lazy_folios_probe.cpp @@ -34,6 +34,7 @@ #include "../../sources/diagramevent/diagrameventaddpaste.h" #include #include "../../sources/qetgraphicsitem/conductor.h" +#include "../../sources/qetgraphicsitem/ViewItem/qetgraphicstableitem.h" #include "../../sources/qetgraphicsitem/terminal.h" #include "../../sources/undocommand/changeelementinformationcommand.h" #include "../../sources/undocommand/changetitleblockcommand.h" @@ -678,6 +679,22 @@ int main(int argc, char **argv) for (int i = 0; i < folios.size(); ++i) same(indexed, i, "shown from the .qetz"); check(indexed.toXml().toString() == eager.toXml().toString(), "the .qetz saves as when built"); + //A folio whose table continues one on another folio, shown + //first: the index, not the folios' parts, says which folios + //its table is built with (D5) + int continued = 0; + for (int i = 0; i < folios.size(); ++i) { + bool continues = false; + for (QGraphicsItem *item : folios.at(i)->items()) + if (item->type() == QetGraphicsTableItem::Type + && !static_cast(item)->previousTableUuid().isNull()) + continues = true; + if (!continues) continue; + QETProject fresh(zipped); + same(fresh, i, "a continued table shown first from the .qetz"); + ++continued; + } + check(continued > 0, "a folio whose table continues another"); //Saved as a .qetz without building the folios not touched //(D4): their parts and rows copied as they were From 8ae937007c2ce88214ff989d33d4d9553ccf93c0 Mon Sep 17 00:00:00 2001 From: ispyisail Date: Mon, 12 Oct 2026 07:37:26 +1300 Subject: [PATCH 08/26] Answer numbering, searches and unused symbols from the database (stage D5) The project database's element table now holds each symbol's definition (the path its file names it by) and its sequential numbers, filled from the document and from built folios alike. Numbering, the searches for symbols of a kind (a link tab's candidates) and the list of unused symbols read them for the folios not built, instead of reading those folios' files. Saving looks up a folio's tables in the index it was opened from. A .qetz asked its numbering, searches and unused symbols reads only the files of the folios it builds (tst_lazyfolios, which fails when the unused symbols are read from the folio files: 50 read, 22 built). ctest 96/96 eager and lazy; 342 corpus projects save the same in all four arms. Co-Authored-By: Claude Opus 5.5 --- sources/dataBase/projectdatabase.cpp | 124 ++++++++++++++++++++++++++- sources/dataBase/projectdatabase.h | 13 +++ sources/qetproject.cpp | 123 +++++++++++++------------- sources/qetproject.h | 4 + tests/qttest/lazy_folios_probe.cpp | 29 +++++++ 5 files changed, 234 insertions(+), 59 deletions(-) diff --git a/sources/dataBase/projectdatabase.cpp b/sources/dataBase/projectdatabase.cpp index f43af8179..80c5d1ec5 100644 --- a/sources/dataBase/projectdatabase.cpp +++ b/sources/dataBase/projectdatabase.cpp @@ -50,6 +50,8 @@ #include #include #include +#include +#include #include #include #include @@ -359,6 +361,8 @@ struct DocumentElement QString type; QString sub_type; QVariant group; + QString location; + autonum::sequentialNumbers sequence; DiagramContext informations; QString label; QHash terminals; @@ -379,6 +383,27 @@ bool readSequence(const QDomElement &item, autonum::sequentialNumbers *sequence) return true; } + //The sequential values an element was saved with, as Element::fromXml() + //reads them, the older attributes included +autonum::sequentialNumbers elementSequence(const QDomElement &item) +{ + autonum::sequentialNumbers sequence; + if (readSequence(item, &sequence)) { + return sequence; + } + auto load = [&item](const QString &prefix, QStringList *list) { + for (int i = 1 ; !item.attribute(prefix + QString::number(i)).isEmpty() ; ++i) + list->append(item.attribute(prefix + QString::number(i))); + }; + load(QStringLiteral("sequ_"), &sequence.unit); + load(QStringLiteral("sequf_"), &sequence.unit_folio); + load(QStringLiteral("seqt_"), &sequence.ten); + load(QStringLiteral("seqtf_"), &sequence.ten_folio); + load(QStringLiteral("seqh_"), &sequence.hundred); + load(QStringLiteral("seqhf_"), &sequence.hundred_folio); + return sequence; +} + struct DocumentConductor { QString uuid; @@ -609,6 +634,8 @@ bool projectDataBase::populateFromDocument(const QDomDocument &document, QString element.type = known->type; element.sub_type = known->sub_type; element.terminals = known->terminals; + element.location = element_xml.attribute(QStringLiteral("type")); + element.sequence = elementSequence(element_xml); const QUuid group = ItemGroups::read(element_xml); element.group = group.isNull() ? QVariant() : QVariant(group.toString()); element.informations.fromXml( @@ -810,6 +837,8 @@ bool projectDataBase::populateFromDocument(const QDomDocument &document, QString m_insert_elements_query.bindValue(QStringLiteral(":type"), element.type); m_insert_elements_query.bindValue(QStringLiteral(":sub_type"), element.sub_type); m_insert_elements_query.bindValue(QStringLiteral(":group_uuid"), element.group); + m_insert_elements_query.bindValue(QStringLiteral(":definition"), element.location); + m_insert_elements_query.bindValue(QStringLiteral(":sequence"), sequenceValue(element.sequence)); if (!m_insert_elements_query.exec()) { qDebug() << "projectDataBase::populateFromDocument element insert error : " << m_insert_elements_query.lastError(); } @@ -1207,6 +1236,94 @@ QHash projectDataBase::elementTypesOnFolios(const QSet &f return types; } +/** + @brief projectDataBase::symbolsOnFolios + @return the symbols on the folios @p folios, folio by folio in that + order and on each in the order of its rows -- the order the folio + places them -- read from the element table as it is (see + elementTypesOnFolios()). A symbol whose definition was not found has + no row. +*/ +QList projectDataBase::symbolsOnFolios(const QList &folios) const +{ + QList symbols; + QSqlQuery query(m_data_base); + query.prepare(QStringLiteral("SELECT uuid, type, definition, sequence FROM element " + "WHERE diagram_uuid = :folio ORDER BY rowid")); + for (const QUuid &folio : folios) { + query.bindValue(QStringLiteral(":folio"), folio.toString()); + if (!query.exec()) { + continue; + } + while (query.next()) { + symbols.append({QUuid(query.value(0).toString()), folio, query.value(1).toString(), + query.value(2).toString(), sequenceFromValue(query.value(3))}); + } + } + return symbols; +} + +/** + @brief projectDataBase::linkedSymbols + @return the symbols @p element is linked to, read from the link table + as it is (see elementTypesOnFolios()) +*/ +QList projectDataBase::linkedSymbols(const QUuid &element) const +{ + QList linked; + QSqlQuery query(m_data_base); + query.prepare(QStringLiteral("SELECT linked_uuid FROM link WHERE element_uuid = :uuid ORDER BY rowid")); + query.bindValue(QStringLiteral(":uuid"), element.toString()); + if (query.exec()) { + while (query.next()) + linked.append(QUuid(query.value(0).toString())); + } + return linked; +} + +/** + @brief projectDataBase::sequenceValue + @return @p sequence as the element table's sequence column holds it: + each part the file would save (sequentialNumbers::toXml()), by its tag, + as JSON; null when the file would save none. sequenceFromValue() reads + it back as reading the file would (sequentialNumbers::fromXml()). +*/ +QVariant projectDataBase::sequenceValue(const autonum::sequentialNumbers &sequence) +{ + QDomDocument document; + const QDomElement xml = sequence.toXml(document); + if (!xml.hasChildNodes()) { + return QVariant(); + } + QJsonObject parts; + for (QDomElement part = xml.firstChildElement() ; !part.isNull() ; + part = part.nextSiblingElement()) + parts.insert(part.tagName(), part.text()); + return QString::fromUtf8(QJsonDocument(parts).toJson(QJsonDocument::Compact)); +} + +/** + @brief projectDataBase::sequenceFromValue + @return the sequential numbers held by @p value (sequenceValue()) +*/ +autonum::sequentialNumbers projectDataBase::sequenceFromValue(const QVariant &value) +{ + autonum::sequentialNumbers sequence; + if (value.isNull()) { + return sequence; + } + const QJsonObject parts = QJsonDocument::fromJson(value.toString().toUtf8()).object(); + QDomDocument document; + QDomElement xml = document.createElement(QStringLiteral("sequentialNumbers")); + for (auto it = parts.constBegin() ; it != parts.constEnd() ; ++it) { + QDomElement part = document.createElement(it.key()); + part.appendChild(document.createTextNode(it.value().toString())); + xml.appendChild(part); + } + sequence.fromXml(xml); + return sequence; +} + /** @brief projectDataBase::folioUuidsOfElements @return the uuids of the folios the symbols @p elements are on, read @@ -2322,6 +2439,8 @@ bool projectDataBase::createDataBase() "type VARCHAR(50)," "sub_type VARCHAR(50)," "group_uuid VARCHAR(50)," + "definition TEXT," + "sequence TEXT," "FOREIGN KEY (diagram_uuid) REFERENCES diagram (uuid)" ")"); if (!query_.exec(element_table)) { @@ -3985,7 +4104,8 @@ void projectDataBase::prepareQuery() m_diagram_info_order_changed.prepare("UPDATE diagram_info SET folio = :folio WHERE diagram_uuid = :uuid"); //INSERT ELEMENT - QString insert_element_query("INSERT INTO element (uuid, diagram_uuid, pos, type, sub_type, group_uuid) VALUES (:uuid, :diagram_uuid, :pos, :type, :sub_type, :group_uuid)"); + QString insert_element_query("INSERT INTO element (uuid, diagram_uuid, pos, type, sub_type, group_uuid, definition, sequence) " + "VALUES (:uuid, :diagram_uuid, :pos, :type, :sub_type, :group_uuid, :definition, :sequence)"); m_insert_elements_query = QSqlQuery(m_data_base); m_insert_elements_query.prepare(insert_element_query); @@ -4100,6 +4220,8 @@ void projectDataBase::bindElementValues(QSqlQuery &query, Element *element, Diag query.bindValue(QStringLiteral(":type"), element_data.typeToString()); query.bindValue(QStringLiteral(":sub_type"), element_data.masterTypeToString()); query.bindValue(QStringLiteral(":group_uuid"), groupValue(element)); + query.bindValue(QStringLiteral(":definition"), element->location().path()); + query.bindValue(QStringLiteral(":sequence"), sequenceValue(element->sequenceStruct())); } /** diff --git a/sources/dataBase/projectdatabase.h b/sources/dataBase/projectdatabase.h index ec0ac2e88..489880ecb 100644 --- a/sources/dataBase/projectdatabase.h +++ b/sources/dataBase/projectdatabase.h @@ -34,6 +34,7 @@ #include "../diagramcontext.h" #include "../titleblockproperties.h" #include "../conductorproperties.h" +#include "../autoNum/assignvariables.h" class Element; class QETProject; @@ -180,6 +181,18 @@ class projectDataBase : public QObject QSet foliosWithUnjoinedWires() const; QSet folioUuidsOfElements(const QSet &elements) const; QHash elementTypesOnFolios(const QSet &folios) const; + /// A symbol placed on a folio, as the element table has it + struct FolioSymbol { + QUuid uuid; + QUuid folio; + QString kind; ///< ElementData::typeToString() + QString location; ///< its definition, ElementsLocation::path() + autonum::sequentialNumbers sequence; + }; + QList symbolsOnFolios(const QList &folios) const; + QList linkedSymbols(const QUuid &element) const; + static QVariant sequenceValue(const autonum::sequentialNumbers &sequence); + static autonum::sequentialNumbers sequenceFromValue(const QVariant &value); void addElement (Element *element); void removeElement (Element *element); diff --git a/sources/qetproject.cpp b/sources/qetproject.cpp index 9937d7099..a63de72a7 100644 --- a/sources/qetproject.cpp +++ b/sources/qetproject.cpp @@ -796,29 +796,22 @@ void QETProject::buildFoliosHolding(ElementData::Types kinds, bool free_only) for (auto it = types.cbegin() ; it != types.cend() ; ++it) placed.insert(it.key()); } + //The symbols and their links as the database has them QList to_build; for (Diagram *d : std::as_const(m_diagrams_list)) { if (d->isLoaded()) { continue; } - for (const QDomElement &element_xml : - QET::findInDomElement(d->deferredXml(), - QStringLiteral("elements"), - QStringLiteral("element"))) + for (const projectDataBase::FolioSymbol &symbol : m_data_base.symbolsOnFolios({d->uuid()})) { - const auto type = types.constFind(QUuid(element_xml.attribute(QStringLiteral("uuid")))); - if (type == types.constEnd() - || !(kinds & ElementData::typeFromString(*type))) { + if (!(kinds & ElementData::typeFromString(symbol.kind))) { continue; } bool linked = false; if (free_only) { - for (const QDomElement &link : - QET::findInDomElement(element_xml, - QStringLiteral("links_uuids"), - QStringLiteral("link_uuid"))) { - if (placed.contains(QUuid(link.attribute(QStringLiteral("uuid"))))) { + for (const QUuid &link : m_data_base.linkedSymbols(symbol.uuid)) { + if (placed.contains(link)) { linked = true; break; } @@ -844,7 +837,8 @@ void QETProject::buildFoliosHolding(ElementData::Types kinds, bool free_only) /** @brief QETProject::buildFoliosHoldingSymbols Build the folios not built yet (QET_LAZY_FOLIOS) holding one of the - symbols @p symbols, found in the XML kept for them: what needs those + symbols @p symbols, found in the project database once it holds the + folios not built, else in the XML kept for them: what needs those symbols while the project is being read, before the database knows where they are (a terminal strip's terminals) */ @@ -856,7 +850,7 @@ void QETProject::buildFoliosHoldingSymbols(const QSet &symbols) QList to_build; //Opened from an index, the database knows where every symbol is //without reading the folios' parts - if (m_opened_from_index) { + if (m_opened_from_index || m_data_base.rowsFromDocument()) { const QSet folios = m_data_base.folioUuidsOfElements(symbols); for (Diagram *d : std::as_const(m_diagrams_list)) if (!d->isLoaded() && folios.contains(d->uuid())) @@ -1892,7 +1886,7 @@ std::optional QETProject::elementAutoNumIdToLink( the symbol on a folio built and, on a folio not built yet (QET_LAZY_FOLIOS), from what building it would give: its information in the project database's store, the scheme it would be linked to, - its sequential numbers and its kind as the file has them. + its sequential numbers and its kind as the database has them. */ QList QETProject::symbolNumbering() const { @@ -1923,38 +1917,27 @@ QList QETProject::symbolNumbering() const return list; } - //A symbol the database has no row for is one building leaves out - //(its definition is missing) - const QHash types = m_data_base.elementTypesOnFolios(unbuilt); - const QHash titles_by_formula = elementAutoNumTitlesByFormula(); + //The symbols of the folios not built, from the database. A symbol + //it has no row for is one building leaves out (its definition is + //missing) + QList unbuilt_in_order; for (Diagram *d : m_diagrams_list) + if (!d->isLoaded()) unbuilt_in_order << d->uuid(); + const QHash titles_by_formula = elementAutoNumTitlesByFormula(); + for (const projectDataBase::FolioSymbol &row : m_data_base.symbolsOnFolios(unbuilt_in_order)) { - if (d->isLoaded()) { - continue; - } - for (const QDomElement &element_xml : - QET::findInDomElement(d->deferredXml(), - QStringLiteral("elements"), - QStringLiteral("element"))) - { - const QUuid uuid(element_xml.attribute(QStringLiteral("uuid"))); - const auto type = types.constFind(uuid); - if (type == types.constEnd()) { - continue; - } - const DiagramContext info = m_data_base.elementInformation(uuid); - SymbolNumbering symbol; - symbol.label = info.value(QETInformation::ELMT_LABEL).toString(); - symbol.formula = info.value(QETInformation::ELMT_FORMULA).toString(); - const auto linked = elementAutoNumIdToLink(info, titles_by_formula); - symbol.scheme = linked ? *linked - : QUuid(info.value(QETInformation::ELMT_FORMULA_ID).toString()); - symbol.sequence.fromXml(element_xml.firstChildElement(QStringLiteral("sequentialNumbers"))); - const ElementData::Type kind = ElementData::typeFromString(*type); - symbol.numbered = kind != ElementData::Slave - && !(kind & ElementData::AllReport); - list << symbol; - } + const DiagramContext info = m_data_base.elementInformation(row.uuid); + SymbolNumbering symbol; + symbol.label = info.value(QETInformation::ELMT_LABEL).toString(); + symbol.formula = info.value(QETInformation::ELMT_FORMULA).toString(); + const auto linked = elementAutoNumIdToLink(info, titles_by_formula); + symbol.scheme = linked ? *linked + : QUuid(info.value(QETInformation::ELMT_FORMULA_ID).toString()); + symbol.sequence = row.sequence; + const ElementData::Type kind = ElementData::typeFromString(row.kind); + symbol.numbered = kind != ElementData::Slave + && !(kind & ElementData::AllReport); + list << symbol; } return list; } @@ -2387,8 +2370,7 @@ QDomDocument QETProject::toXml(QHash *kept_folios, QStringList //give it, which any edit can change if (m_edited_since_open) for (Diagram *diagram : std::as_const(m_diagrams_list)) - if (!diagram->isLoaded() && !diagram->deferredXml().elementsByTagName( - QetGraphicsTableItem::xmlTagName()).isEmpty()) + if (!diagram->isLoaded() && unbuiltFolioHasTables(diagram)) buildFolioToShow(diagram); //A built folio is written from its items, which need their //partners: a table saves the table before it, and in a .qet a @@ -2996,24 +2978,48 @@ bool QETProject::usesElement(const ElementsLocation &location) const return(unbuiltFolioElements().contains(location)); } +/** + @brief QETProject::unbuiltFolioHasTables + @return true if @p folio, not built yet (QET_LAZY_FOLIOS), has a drawn + table: from the index it was opened from, else from its XML +*/ +bool QETProject::unbuiltFolioHasTables(Diagram *folio) const +{ + if (m_index_tables) { + for (const projectDataBase::FolioTable &table : std::as_const(*m_index_tables)) + if (table.folio == folio->uuid()) return true; + return false; + } + return !folio->deferredXml().elementsByTagName(QetGraphicsTableItem::xmlTagName()).isEmpty(); +} + /** @brief QETProject::unbuiltFolioElements @return the locations of the symbols placed on the folios not built yet - (QET_LAZY_FOLIOS), each once, read from the XML kept for them as - building them reads it (Diagram::fromXml()) + (QET_LAZY_FOLIOS), each once, read from the project database once it + holds those folios, else from the XML kept for them as building them + reads it (Diagram::fromXml()) */ QList QETProject::unbuiltFolioElements() const { QSet types; - for (Diagram *diagram : m_diagrams_list) { - if (diagram->isLoaded()) { - continue; - } - for (const QDomElement &element_xml : - QET::findInDomElement(diagram->deferredXml(), - QStringLiteral("elements"), - QStringLiteral("element"))) { - types.insert(element_xml.attribute(QStringLiteral("type"))); + if (m_data_base.rowsFromDocument()) { + QList unbuilt; + for (Diagram *diagram : m_diagrams_list) + if (!diagram->isLoaded()) unbuilt << diagram->uuid(); + for (const projectDataBase::FolioSymbol &symbol : m_data_base.symbolsOnFolios(unbuilt)) + types.insert(symbol.location); + } else { + for (Diagram *diagram : m_diagrams_list) { + if (diagram->isLoaded()) { + continue; + } + for (const QDomElement &element_xml : + QET::findInDomElement(diagram->deferredXml(), + QStringLiteral("elements"), + QStringLiteral("element"))) { + types.insert(element_xml.attribute(QStringLiteral("type"))); + } } } QList locations; @@ -3413,6 +3419,7 @@ QDomElement QETProject::readFolioPart(const QString &file) if (!m_folio_reader) m_folio_reader = std::make_unique( m_container_files.value(QStringLiteral("project.sqlite"))); + ++m_folio_parts_read; if (!QetContainer::readFolio(m_container_files, file, &folio, &error, m_folio_reader.get())) { qWarning().noquote() << "Unable to read the folio" << file << ":" << error; return QDomElement(); diff --git a/sources/qetproject.h b/sources/qetproject.h index aad50ad7e..61f9729d7 100644 --- a/sources/qetproject.h +++ b/sources/qetproject.h @@ -332,6 +332,7 @@ class QETProject : public QObject bool usesElement(const ElementsLocation &) const; QList unusedElements() const; QList unbuiltFolioElements() const; + bool unbuiltFolioHasTables(Diagram *folio) const; bool usesTitleBlockTemplate(const TitleBlockTemplateLocation &); bool projectWasModified(); bool projectOptionsWereModified(); @@ -455,6 +456,8 @@ class QETProject : public QObject bool writeContainerFile(const QString &path, QString *error = nullptr); QByteArray containerIndex(); QDomElement readFolioPart(const QString &file); + /// How many folio parts of the .qetz were read so far + int folioPartsRead() const {return m_folio_parts_read;} bool openedFromIndex() const {return m_opened_from_index;} private: static QUuid derivedUuid(const QByteArray &content); @@ -550,6 +553,7 @@ class QETProject : public QObject QHash m_container_files; ///Reads the folio parts' engineering data, opened once std::unique_ptr m_folio_reader; + int m_folio_parts_read = 0; QStringList m_folio_files; QList> m_folio_headers; ///The drawn tables of the index opened from, if it has them diff --git a/tests/qttest/lazy_folios_probe.cpp b/tests/qttest/lazy_folios_probe.cpp index e0bcf3d4c..24d71f743 100644 --- a/tests/qttest/lazy_folios_probe.cpp +++ b/tests/qttest/lazy_folios_probe.cpp @@ -696,6 +696,35 @@ int main(int argc, char **argv) } check(continued > 0, "a folio whose table continues another"); + //Numbering, the searches for symbols and the unused symbols + //are answered from the database: no folio file is read + //but those of the folios built (D5) + { + QETProject asked(zipped); + check(numbering(asked) == numbering(eager), "numbering of a .qetz as when built"); + for (const QString &title : eager.elementAutoNum().keys()) + check(gaps(asked, title) == gaps(eager, title), "a .qetz scheme's gaps as when built"); + auto locations = [](const QList &list) { + QStringList strings; + for (const ElementsLocation &location : list) strings << location.toString(); + strings.sort(); + return strings; + }; + check(locations(asked.unusedElements()) == locations(eager.unusedElements()), + "a .qetz's unused symbols as when built"); + const ElementProvider usual_search(&eager), search(&asked); + for (const ElementData::Types kinds : {ElementData::Types(ElementData::Slave), + ElementData::Types(ElementData::Master), + ElementData::Types(ElementData::AllReport)}) + check(found(search.freeElement(kinds)) == found(usual_search.freeElement(kinds)), + "free symbols of a .qetz found as when built"); + const int built = int(folios.size()) - asked.unloadedFolioCount(); + std::printf(".qetz asked numbering, searches, unused symbols: %d folio files read, %d folios built\n", + asked.folioPartsRead(), built); + check(asked.unloadedFolioCount() > 0, "asking a .qetz does not build every folio"); + check(asked.folioPartsRead() == built, "asking a .qetz reads no file of a folio not built"); + } + //Saved as a .qetz without building the folios not touched //(D4): their parts and rows copied as they were QETProject untouched(zipped); From e7fd820db6660b1bcac4c6850a87d765e9acb600 Mon Sep 17 00:00:00 2001 From: ispyisail Date: Mon, 12 Oct 2026 08:46:31 +1300 Subject: [PATCH 09/26] Add, move or remove a folio without building the others (stage D5) Adding, moving or removing a folio rewrote every folio's position in the project database through diagrams(), which builds every folio: 3 s on a 50-folio project opened lazily. A folio not built has its position and title block already, so folios() is enough. The list of tables a table can continue builds only the folios holding a table (from the index of a .qetz), and the wiring list's count of wires left out looks at built folios only: reading a wire drops it unless both its ends are found. tst_lazyfolios: a folio added, moved and removed builds no other, with the folios' rows as when every folio is built (fails with diagrams(): 47 built); the tables listed as when built, building the 12 folios that hold one (fails when every folio is built). ctest 96/96 eager and lazy; 342 corpus projects save the same in all four arms. Co-Authored-By: Claude Opus 5.5 --- sources/dataBase/projectdatabase.cpp | 9 +++-- sources/elementprovider.cpp | 9 ++++- sources/qetproject.cpp | 21 +++++++++++ sources/qetproject.h | 1 + tests/qttest/lazy_folios_probe.cpp | 55 ++++++++++++++++++++++++++++ 5 files changed, 91 insertions(+), 4 deletions(-) diff --git a/sources/dataBase/projectdatabase.cpp b/sources/dataBase/projectdatabase.cpp index 80c5d1ec5..007bad900 100644 --- a/sources/dataBase/projectdatabase.cpp +++ b/sources/dataBase/projectdatabase.cpp @@ -1536,8 +1536,10 @@ int projectDataBase::excludedConductorCount() const return 0; } + //A folio not built (QET_LAZY_FOLIOS) has none: reading a wire + //drops it unless both its ends are found (Conductor::fromXml()) int count = 0; - for (auto *diagram : m_project->diagrams()) + for (auto *diagram : m_project->builtFolios()) { const auto conductor_list = diagram->conductors(); for (auto *conductor : conductor_list) @@ -1722,11 +1724,12 @@ void projectDataBase::flushElementPositions() /** @brief projectDataBase::updateFolioPositions Write every folio's position and folio number again, after a folio was - added, removed or moved. + added, removed or moved. A folio not built yet (QET_LAZY_FOLIOS) has + its position and title block already: it is not built for this. */ void projectDataBase::updateFolioPositions() { - for (auto diagram : m_project->diagrams()) + for (auto diagram : m_project->folios()) { m_diagram_order_changed.bindValue(":pos", m_project->folioIndex(diagram)+1); m_diagram_order_changed.bindValue(":uuid", diagram->uuid().toString()); diff --git a/sources/elementprovider.cpp b/sources/elementprovider.cpp index a493fd7fe..174151811 100644 --- a/sources/elementprovider.cpp +++ b/sources/elementprovider.cpp @@ -199,7 +199,14 @@ QVector ElementProvider::table( } } - for (auto d : allFolios()) { + //Only the folios holding a table are built for it + QList folios = m_diagram_list; + if (m_project) { + m_project->buildFoliosHoldingTables(); + folios = m_project->builtFolios(); + folios.removeOne(m_excluded); + } + for (auto d : std::as_const(folios)) { for (auto item_ : d->items()) { if(item_->type() == QetGraphicsTableItem::Type) diff --git a/sources/qetproject.cpp b/sources/qetproject.cpp index a63de72a7..34e46e67d 100644 --- a/sources/qetproject.cpp +++ b/sources/qetproject.cpp @@ -2978,6 +2978,27 @@ bool QETProject::usesElement(const ElementsLocation &location) const return(unbuiltFolioElements().contains(location)); } +/** + @brief QETProject::buildFoliosHoldingTables + Build the folios not built yet (QET_LAZY_FOLIOS) that hold a drawn + table, found from the index the project was opened from when it has + one: what a list of the project's tables needs, and no more +*/ +void QETProject::buildFoliosHoldingTables() +{ + QList to_build; + for (Diagram *d : std::as_const(m_diagrams_list)) + if (!d->isLoaded() && unbuiltFolioHasTables(d)) + to_build << d; + if (to_build.isEmpty()) { + return; + } + if (qEnvironmentVariableIntValue("QET_LAZY_FOLIOS_TRACE") == 1) { + qWarning().noquote() << "lazy folios:" << to_build.size() << "folios built for their tables"; + } + buildFolios(to_build); +} + /** @brief QETProject::unbuiltFolioHasTables @return true if @p folio, not built yet (QET_LAZY_FOLIOS), has a drawn diff --git a/sources/qetproject.h b/sources/qetproject.h index 61f9729d7..509a532a8 100644 --- a/sources/qetproject.h +++ b/sources/qetproject.h @@ -135,6 +135,7 @@ class QETProject : public QObject void buildPartnerSymbols(const QList &folios, const QSet &partners); void buildFoliosHolding(ElementData::Types kinds, bool free_only); void buildFoliosHoldingSymbols(const QSet &symbols); + void buildFoliosHoldingTables(); QSet symbolsOnUnbuiltFolios() const; /// While saving: the symbols on folios not built a built symbol /// may be saved linked to (Element::toXml()) diff --git a/tests/qttest/lazy_folios_probe.cpp b/tests/qttest/lazy_folios_probe.cpp index 24d71f743..2947e6b37 100644 --- a/tests/qttest/lazy_folios_probe.cpp +++ b/tests/qttest/lazy_folios_probe.cpp @@ -725,6 +725,61 @@ int main(int argc, char **argv) check(asked.folioPartsRead() == built, "asking a .qetz reads no file of a folio not built"); } + //A folio added, moved and removed: no other folio built, the + //folios' rows as when every folio is built (D5) + { + auto shuffle = [](QETProject &project) { + Diagram *added = project.addNewDiagram(); + project.diagramOrderChanged(1, 0); + project.removeDiagram(added); + }; + auto folioRows = [](QETProject &project) { + QStringList rows; + for (const QString &row : tables(project)) + if (row.startsWith("diagram")) rows << row; + return rows; + }; + QETProject shuffled(zipped); + const int unbuilt = shuffled.unloadedFolioCount(); + shuffle(shuffled); + std::printf(".qetz folio added, moved, removed: %d folios built\n", + unbuilt - shuffled.unloadedFolioCount()); + check(shuffled.unloadedFolioCount() == unbuilt, "adding, moving and removing a folio builds no other"); + qunsetenv("QET_LAZY_FOLIOS"); + QETProject usual_shuffled(zipped); + qputenv("QET_LAZY_FOLIOS", "1"); + shuffle(usual_shuffled); + check(folioRows(shuffled) == folioRows(usual_shuffled), "folios' rows after a move as when built"); + } + + //The tables a table can continue (its properties editor) and + //the wiring list's count of wires left out: only the folios + //holding a table built (D5) + { + QETProject listed(zipped); + const int unbuilt = listed.unloadedFolioCount(); + auto tableUuids = [](const QVector &list) { + QStringList uuids; + for (QetGraphicsTableItem *table : list) uuids << table->uuid().toString(); + uuids.sort(); + return uuids; + }; + const QStringList lazy_tables = tableUuids(ElementProvider(&listed).table(nullptr, nullptr)); + check(!lazy_tables.isEmpty(), "the project has tables"); + check(lazy_tables == tableUuids(ElementProvider(&eager).table(nullptr, nullptr)), + "the project's tables listed as when built"); + check(listed.dataBase()->excludedConductorCount() == eager.dataBase()->excludedConductorCount(), + "wires left out of the wiring list counted as when built"); + int with_tables = 0; + for (Diagram *folio : folios) { + for (QGraphicsItem *item : folio->items()) + if (item->type() == QetGraphicsTableItem::Type) { ++with_tables; break; } + } + std::printf(".qetz tables listed: %d folios built, %d hold a table\n", + unbuilt - listed.unloadedFolioCount(), with_tables); + check(listed.unloadedFolioCount() > 0, "listing the tables does not build every folio"); + } + //Saved as a .qetz without building the folios not touched //(D4): their parts and rows copied as they were QETProject untouched(zipped); From b972dbec4b4f8950218bd5e52aa9634aeb294f1f Mon Sep 17 00:00:00 2001 From: ispyisail Date: Mon, 12 Oct 2026 10:37:27 +1300 Subject: [PATCH 10/26] Number terminals and export wire numbers without building every folio (stage D5) Terminal numbering builds only the folios holding a terminal. The wire numbers export reads the wires of the folios not built from the project database, and walks a built folio's own wires instead of a DiagramContent, which followed their potentials through folio reports and built the folios at the other end. The wiring list export names a folio from its title block, which a folio not built has; it still builds every folio, writing the whole project out (toXml()). tst_lazyfolios: wire numbers exported as when built, from a .qet and a .qetz, building no folio (fails with DiagramContent: 27 built); terminal numbering builds 7 folios, those with a terminal (fails with diagrams(): 47); the wiring list as when built. ctest 96/96 eager and lazy; 342 corpus projects save the same in all four arms. Co-Authored-By: Claude Opus 5.5 --- sources/conductornumexport.cpp | 29 ++++++++++++++++--- sources/dataBase/projectdatabase.cpp | 26 +++++++++++++++++ sources/dataBase/projectdatabase.h | 4 +++ sources/ui/terminalnumberingdialog.cpp | 14 +++++++-- sources/wiringlistexport.cpp | 4 +-- tests/qttest/lazy_folios_probe.cpp | 39 ++++++++++++++++++++++++++ 6 files changed, 108 insertions(+), 8 deletions(-) diff --git a/sources/conductornumexport.cpp b/sources/conductornumexport.cpp index f969a2015..f44e5b1bf 100644 --- a/sources/conductornumexport.cpp +++ b/sources/conductornumexport.cpp @@ -19,11 +19,12 @@ #include "qetproject.h" #include "qetapp.h" #include "diagram.h" -#include "diagramcontent.h" #include "qetgraphicsitem/conductor.h" #include "qetgraphicsitem/conductortextitem.h" #include "qetgraphicsitem/element.h" #include "qetgraphicsitem/terminal.h" +#include "dataBase/projectdatabase.h" +#include "properties/elementdata.h" #include @@ -114,10 +115,30 @@ void ConductorNumExport::fillHash() { //We used this rx to avoid insert num composed only withe white space. QRegularExpression rx("^ *$"); - for (Diagram *d : m_project->diagrams()) + auto count = [this](const QString &num) { + m_hash.insert(num, m_hash.value(num, 0) + 1); + }; + //The folios not built (QET_LAZY_FOLIOS): their wires from the + //project database, by the same rules + QList unbuilt; + for (Diagram *d : m_project->folios()) + if (!d->isLoaded()) unbuilt << d->uuid(); + for (const auto &wire : m_project->dataBase()->wiresOnFolios(unbuilt)) { - DiagramContent dc(d, false); - for (Conductor *c : dc.conductors()) + if (wire.text.isEmpty() || wire.text.contains(rx)) { + continue; + } + const bool report1 = ElementData::typeFromString(wire.kind1) & ElementData::AllReport; + const bool report2 = ElementData::typeFromString(wire.kind2) & ElementData::AllReport; + if (!(report1 && report2)) { + count(wire.text); + } + } + for (Diagram *d : m_project->builtFolios()) + { + //Its wires only: a DiagramContent would follow their + //potentials onto other folios, building those not built + for (Conductor *c : d->conductors()) { QString num = c->textItem()->toPlainText(); if (num.isEmpty() || num.contains(rx)) { diff --git a/sources/dataBase/projectdatabase.cpp b/sources/dataBase/projectdatabase.cpp index 007bad900..460e68b29 100644 --- a/sources/dataBase/projectdatabase.cpp +++ b/sources/dataBase/projectdatabase.cpp @@ -1263,6 +1263,32 @@ QList projectDataBase::symbolsOnFolios(const QList return symbols; } +/** + @brief projectDataBase::wiresOnFolios + @return the wires on the folios @p folios, folio by folio in that order, + read from the conductor table as it is (see elementTypesOnFolios()) +*/ +QList projectDataBase::wiresOnFolios(const QList &folios) const +{ + QList wires; + QSqlQuery query(m_data_base); + query.prepare(QStringLiteral("SELECT c.text, e1.type, e2.type FROM conductor c " + "LEFT JOIN element e1 ON e1.uuid = c.terminal1_element_uuid " + "LEFT JOIN element e2 ON e2.uuid = c.terminal2_element_uuid " + "WHERE c.diagram_uuid = :folio ORDER BY c.rowid")); + for (const QUuid &folio : folios) { + query.bindValue(QStringLiteral(":folio"), folio.toString()); + if (!query.exec()) { + continue; + } + while (query.next()) { + wires.append({query.value(0).toString(), query.value(1).toString(), + query.value(2).toString()}); + } + } + return wires; +} + /** @brief projectDataBase::linkedSymbols @return the symbols @p element is linked to, read from the link table diff --git a/sources/dataBase/projectdatabase.h b/sources/dataBase/projectdatabase.h index 489880ecb..c8583a4cc 100644 --- a/sources/dataBase/projectdatabase.h +++ b/sources/dataBase/projectdatabase.h @@ -191,6 +191,10 @@ class projectDataBase : public QObject }; QList symbolsOnFolios(const QList &folios) const; QList linkedSymbols(const QUuid &element) const; + /// A wire placed on a folio: its text and the kinds of the + /// symbols at its ends (ElementData::typeToString()) + struct FolioWire { QString text; QString kind1; QString kind2; }; + QList wiresOnFolios(const QList &folios) const; static QVariant sequenceValue(const autonum::sequentialNumbers &sequence); static autonum::sequentialNumbers sequenceFromValue(const QVariant &value); diff --git a/sources/ui/terminalnumberingdialog.cpp b/sources/ui/terminalnumberingdialog.cpp index 6e30d32cc..2e180a2f5 100644 --- a/sources/ui/terminalnumberingdialog.cpp +++ b/sources/ui/terminalnumberingdialog.cpp @@ -13,6 +13,16 @@ #include #include +namespace { + //The folios holding a terminal, built: a folio not built yet + //(QET_LAZY_FOLIOS) holding none is not built for this +QList terminalFolios(QETProject *project) +{ + project->buildFoliosHolding(ElementData::Terminal, false); + return project->builtFolios(); +} +} + /** * @brief TerminalNumberingDialog::TerminalNumberingDialog * Constructor @@ -31,7 +41,7 @@ TerminalNumberingDialog::TerminalNumberingDialog(QWidget *parent, QETProject *pr // Collect all unique terminal strip prefixes from the project if (project) { QSet prefixes; - foreach (Diagram *diagram, project->diagrams()) { + foreach (Diagram *diagram, terminalFolios(project)) { foreach (QGraphicsItem *qgi, diagram->items()) { if (Element *elmt = qgraphicsitem_cast(qgi)) { if (elmt->elementData().m_type == ElementData::Terminal) { @@ -153,7 +163,7 @@ QUndoCommand* TerminalNumberingDialog::getUndoCommand(QETProject *project) const QList termList; // 2. Collect all terminals from all folios in the project - foreach (Diagram *diagram, project->diagrams()) { + foreach (Diagram *diagram, terminalFolios(project)) { int fIndex = diagram->folioIndex(); foreach (QGraphicsItem *qgi, diagram->items()) { if (Element *elmt = qgraphicsitem_cast(qgi)) { diff --git a/sources/wiringlistexport.cpp b/sources/wiringlistexport.cpp index c56b6fc76..39282ac69 100644 --- a/sources/wiringlistexport.cpp +++ b/sources/wiringlistexport.cpp @@ -28,8 +28,8 @@ QString WiringListExport::findDiagramFolio(const QDomElement &diagramElem) const // The folio number as the folio shows it ("3/12"). The saved "folio" // attribute is its template ("%id/%total"). const int index = diagramElem.attribute("order").toInt(); - if (m_project && index >= 1 && index <= m_project->diagrams().size()) { - return m_project->diagrams().at(index - 1)->border_and_titleblock + if (m_project && index >= 1 && index <= m_project->folios().size()) { + return m_project->folios().at(index - 1)->border_and_titleblock .titleblockInformation().value("folio").toString(); } if (diagramElem.hasAttribute("folio")) return diagramElem.attribute("folio"); diff --git a/tests/qttest/lazy_folios_probe.cpp b/tests/qttest/lazy_folios_probe.cpp index 2947e6b37..2a809693c 100644 --- a/tests/qttest/lazy_folios_probe.cpp +++ b/tests/qttest/lazy_folios_probe.cpp @@ -29,6 +29,9 @@ #include "../../sources/TerminalStrip/terminalstrip.h" #include "../../sources/ui/elementpropertieswidget.h" #include "../../sources/conductorautonumerotation.h" +#include "../../sources/conductornumexport.h" +#include "../../sources/wiringlistexport.h" +#include "../../sources/ui/terminalnumberingdialog.h" #include "../../sources/diagramcontent.h" #include "../../sources/diagramview.h" #include "../../sources/diagramevent/diagrameventaddpaste.h" @@ -376,6 +379,15 @@ int main(int argc, char **argv) for (const QString &title : titles) check(gaps(numbered, title) == gaps(eager, title), "a scheme's gaps as when built"); check(numbered.unloadedFolioCount() == folios.size() - strip_folios.size(), "numbering read without building"); + { + //Wire numbers exported from the database filled from the + //document, as when built (D5) + QETProject exported(path); + check(ConductorNumExport(&exported).wiresNum() == ConductorNumExport(&eager).wiresNum(), + "wire numbers of a .qet exported as when built"); + check(exported.unloadedFolioCount() == folios.size() - strip_folios.size(), + "exporting a .qet's wire numbers builds no folio"); + } //A symbol placed on a folio, numbered by a scheme others follow QString location, title; int folio = -1; @@ -695,6 +707,13 @@ int main(int argc, char **argv) ++continued; } check(continued > 0, "a folio whose table continues another"); + { + //The wiring list export writes the whole project out + //(toXml()), building every folio: as when built + QETProject exported(zipped); + check(WiringListExport(&exported).toCsvString() == WiringListExport(&eager).toCsvString(), + "wiring list exported as when built"); + } //Numbering, the searches for symbols and the unused symbols //are answered from the database: no folio file is read @@ -780,6 +799,26 @@ int main(int argc, char **argv) check(listed.unloadedFolioCount() > 0, "listing the tables does not build every folio"); } + //The wire numbers export and the terminal + //numbering dialog: as when built, building only the folios + //holding a terminal (D5) + { + QETProject exported(zipped); + const int unbuilt = exported.unloadedFolioCount(); + check(ConductorNumExport(&exported).wiresNum() == ConductorNumExport(&eager).wiresNum(), + "wire numbers exported as when built"); + check(exported.unloadedFolioCount() == unbuilt, "exporting wire numbers builds no folio"); + { TerminalNumberingDialog dialog(nullptr, &exported); } + int with_terminals = 0; + for (Diagram *folio : folios) + for (Element *element : folio->elements()) + if (element->elementData().m_type == ElementData::Terminal) { ++with_terminals; break; } + std::printf(".qetz terminal numbering: %d folios built, %d hold a terminal\n", + unbuilt - exported.unloadedFolioCount(), with_terminals); + check(exported.folios().size() - exported.unloadedFolioCount() <= with_terminals + (folios.size() - unbuilt), + "terminal numbering builds only the folios holding a terminal"); + } + //Saved as a .qetz without building the folios not touched //(D4): their parts and rows copied as they were QETProject untouched(zipped); From 65b10b01d9d8a0c708bebce40fc07ecd6f227b39 Mon Sep 17 00:00:00 2001 From: ispyisail Date: Mon, 12 Oct 2026 12:20:46 +1300 Subject: [PATCH 11/26] Export the wiring list without building every folio (stage D5) The wiring list export read the project as QETProject::toXml() writes it, which builds every folio. It now reads a document made without building: a built folio written as saving writes it (QETProject::folioXml(), which keeps its symbols' links to partners not built), a folio not built copied from its XML, and what QElectroTech computes for a wire's ends -- its symbols' labels, their last partner's label, the terminals' names -- filled in from the symbols built and from the project database. A symbol still waiting for a partner not built takes its partners from the database, saved order. 136-folio .qetz, wiring list from the command line: 30.1 s with every folio built, 4.5 s lazily, same file. tst_lazyfolios: the wiring list of a .qet and of a .qetz as when built, building no folio. New corpus gate (it fails on all 6 projects tried with terminal names broken): --export-cables and --export-wires lazily from the .qet and the .qetz = eager on 342 projects; one (forum f3386) differed until waiting symbols took their partners from the database. ctest 96/96 eager and lazy (one earlier run had modal_quit_regression and tst_resaveunchanged fail once each, both passing on rerun in both modes); 342 corpus projects save the same in all four arms. Co-Authored-By: Claude Opus 5.5 --- sources/dataBase/projectdatabase.cpp | 28 +++++++++ sources/dataBase/projectdatabase.h | 2 + sources/diagram.h | 2 + sources/qetproject.cpp | 14 +++++ sources/qetproject.h | 1 + sources/wiringlistexport.cpp | 88 +++++++++++++++++++++++++++- sources/wiringlistexport.h | 1 + tests/qttest/lazy_folios_probe.cpp | 28 +++++---- 8 files changed, 153 insertions(+), 11 deletions(-) diff --git a/sources/dataBase/projectdatabase.cpp b/sources/dataBase/projectdatabase.cpp index 460e68b29..74e391b5f 100644 --- a/sources/dataBase/projectdatabase.cpp +++ b/sources/dataBase/projectdatabase.cpp @@ -1289,6 +1289,34 @@ QList projectDataBase::wiresOnFolios(const QList wiresOnFolios(const QList &folios) const; + QString elementLabel(const QUuid &element) const; + QString terminalName(const QUuid &terminal, const QUuid &element) const; static QVariant sequenceValue(const autonum::sequentialNumbers &sequence); static autonum::sequentialNumbers sequenceFromValue(const QVariant &value); diff --git a/sources/diagram.h b/sources/diagram.h index d79439795..b29730aa4 100644 --- a/sources/diagram.h +++ b/sources/diagram.h @@ -229,6 +229,8 @@ class Diagram : public QGraphicsScene /// Symbols built as partners (buildSymbols()) changed since: /// the folio must be built to be saved bool partialChanged() const {return m_partial_changed;} + /// The symbols built as partners (buildSymbols()), by uuid + QHash partialElements() const {return m_partial_elements;} QString headerXml(); /// The folio's text in the file it was read from, while it is /// not built (QETProject::toXml()) diff --git a/sources/qetproject.cpp b/sources/qetproject.cpp index 34e46e67d..b788313c9 100644 --- a/sources/qetproject.cpp +++ b/sources/qetproject.cpp @@ -2978,6 +2978,20 @@ bool QETProject::usesElement(const ElementsLocation &location) const return(unbuiltFolioElements().contains(location)); } +/** + @brief QETProject::folioXml + @return the built folio @p folio written as saving writes it: its + symbols keep the links to partners on folios not built yet + (QET_LAZY_FOLIOS), which Diagram::toXml() alone leaves out +*/ +QDomDocument QETProject::folioXml(Diagram *folio) +{ + m_waiting_link_targets = unloadedFolioCount() ? symbolsOnUnbuiltFolios() : QSet(); + const QDomDocument xml = folio->toXml(); + m_waiting_link_targets.clear(); + return xml; +} + /** @brief QETProject::buildFoliosHoldingTables Build the folios not built yet (QET_LAZY_FOLIOS) that hold a drawn diff --git a/sources/qetproject.h b/sources/qetproject.h index 509a532a8..8d8d8bded 100644 --- a/sources/qetproject.h +++ b/sources/qetproject.h @@ -140,6 +140,7 @@ class QETProject : public QObject /// While saving: the symbols on folios not built a built symbol /// may be saved linked to (Element::toXml()) QSet waitingLinkTargets() const {return m_waiting_link_targets;} + QDomDocument folioXml(Diagram *folio); QSet tableChainFolios(Diagram *folio) const; void loadFolios() const; void buildFolios(const QList &folios) const; diff --git a/sources/wiringlistexport.cpp b/sources/wiringlistexport.cpp index 39282ac69..c69df19f8 100644 --- a/sources/wiringlistexport.cpp +++ b/sources/wiringlistexport.cpp @@ -1,6 +1,9 @@ #include "wiringlistexport.h" #include "qetproject.h" #include "diagram.h" +#include "dataBase/projectdatabase.h" +#include "ElementsCollection/xmlelementcollection.h" +#include "qetgraphicsitem/element.h" #include #include #include @@ -48,6 +51,89 @@ QDomElement WiringListExport::climbToDiagram(QDomNode node) const return QDomElement(); } +/** + The project as the export reads it: its symbol collection and its folios + in order, each with its "order". A folio not built (QET_LAZY_FOLIOS) is + not built for it: its XML is copied, and what QElectroTech computes for + its wires' ends -- the labels of the symbols they end on and of their + partners, the terminals' names -- is filled in from the symbols built + (on built folios or alone, as partners) and from the project database. +*/ +QDomDocument WiringListExport::projectDocument() const +{ + QDomDocument document; + QDomElement root = document.createElement(QStringLiteral("project")); + document.appendChild(root); + if (XmlElementCollection *collection = m_project->embeddedElementCollection()) + root.appendChild(document.importNode(collection->root(), true)); + + //A folio whose partners built alone were linked or unlinked since + //is not what its XML says: built, as saving builds it + for (Diagram *diagram : m_project->folios()) + if (!diagram->isLoaded() && diagram->partialChanged()) + m_project->buildFolios({diagram}); + + QHash built; + for (Diagram *diagram : m_project->folios()) { + if (diagram->isLoaded()) { + for (Element *element : diagram->elements()) built.insert(element->uuid(), element); + } else { + const QHash partial = diagram->partialElements(); + for (auto it = partial.cbegin() ; it != partial.cend() ; ++it) built.insert(it.key(), it.value()); + } + } + projectDataBase *database = m_project->dataBase(); + auto label = [&built, database](const QUuid &uuid) { + Element *element = built.value(uuid); + return element ? element->actualLabel() : database->elementLabel(uuid); + }; + //Conductor::toXml(): the label of the last of the symbol's partners. + //A symbol still waiting for a partner not built has its links as + //saved, in the database + auto linked = [&built, database, &label](const QUuid &uuid) -> std::optional { + Element *element = built.value(uuid); + if (element && element->pendingLinks().isEmpty()) { + if (element->linkedElements().isEmpty()) return std::nullopt; + return element->linkedElements().last()->actualLabel(); + } + const QList links = database->linkedSymbols(uuid); + if (links.isEmpty()) return std::nullopt; + return label(links.last()); + }; + + const QList folios = m_project->folios(); + for (int i = 0 ; i < folios.size() ; ++i) { + Diagram *diagram = folios.at(i); + QDomElement folio; + const bool is_built = diagram->isLoaded(); + folio = document.importNode(is_built ? m_project->folioXml(diagram).documentElement() + : diagram->deferredXml(), true).toElement(); + //A built folio's wires describe their ends but a partner not + //built yet; a folio not built has none of it in its part (.qetz) + //or as read, before any change (.qet) + QDomNodeList wires = folio.elementsByTagName(QStringLiteral("conductor")); + for (int w = 0 ; w < wires.size() ; ++w) { + QDomElement wire = wires.at(w).toElement(); + for (const QString end : {QStringLiteral("1"), QStringLiteral("2")}) { + const QUuid element(wire.attribute(QStringLiteral("element") + end)); + if (element.isNull()) continue; //an end saved the older way + if (const auto partner = linked(element)) + wire.setAttribute(QStringLiteral("element") + end + QStringLiteral("_linked"), *partner); + else + wire.removeAttribute(QStringLiteral("element") + end + QStringLiteral("_linked")); + if (is_built) continue; + wire.setAttribute(QStringLiteral("element") + end + QStringLiteral("_label"), label(element)); + wire.setAttribute(QStringLiteral("terminalname") + end, + database->terminalName(QUuid(wire.attribute(QStringLiteral("terminal") + end)), + element)); + } + } + folio.setAttribute(QStringLiteral("order"), i + 1); + root.appendChild(folio); + } + return document; +} + QMap WiringListExport::collectElementsInfo(const QDomElement &root) const { QMap infoMap; @@ -191,7 +277,7 @@ QString WiringListExport::toCsvString() const { if (!m_project) return QString(); - QDomDocument doc = m_project->toXml(); + QDomDocument doc = projectDocument(); if (doc.isNull()) return QString(); QSet conductorDefinitionTypes; diff --git a/sources/wiringlistexport.h b/sources/wiringlistexport.h index 8f1696b29..759e099a2 100644 --- a/sources/wiringlistexport.h +++ b/sources/wiringlistexport.h @@ -60,6 +60,7 @@ private: QString normalizeUuid(const QString &u) const; QString findDiagramFolio(const QDomElement &diagramElem) const; QDomElement climbToDiagram(QDomNode node) const; + QDomDocument projectDocument() const; QMap collectElementsInfo(const QDomElement &root) const; QList collectConductors(const QDomElement &root) const; diff --git a/tests/qttest/lazy_folios_probe.cpp b/tests/qttest/lazy_folios_probe.cpp index 2a809693c..4c1a24e96 100644 --- a/tests/qttest/lazy_folios_probe.cpp +++ b/tests/qttest/lazy_folios_probe.cpp @@ -385,8 +385,16 @@ int main(int argc, char **argv) QETProject exported(path); check(ConductorNumExport(&exported).wiresNum() == ConductorNumExport(&eager).wiresNum(), "wire numbers of a .qet exported as when built"); + const QString wiring = WiringListExport(&exported).toCsvString(), usual_wiring = WiringListExport(&eager).toCsvString(); + if (wiring != usual_wiring) { + const QStringList a = wiring.split('\n'), b = usual_wiring.split('\n'); + for (const QString &line : b) if (!a.contains(line)) std::printf(" wiring when built: %s\n", qPrintable(line)); + for (const QString &line : a) if (!b.contains(line)) std::printf(" wiring lazily: %s\n", qPrintable(line)); + std::printf(" %d lines lazily, %d when built\n", int(a.size()), int(b.size())); + } + check(wiring == usual_wiring, "wiring list of a .qet exported as when built"); check(exported.unloadedFolioCount() == folios.size() - strip_folios.size(), - "exporting a .qet's wire numbers builds no folio"); + "exporting a .qet's wire numbers and wiring list builds no folio"); } //A symbol placed on a folio, numbered by a scheme others follow QString location, title; @@ -707,13 +715,6 @@ int main(int argc, char **argv) ++continued; } check(continued > 0, "a folio whose table continues another"); - { - //The wiring list export writes the whole project out - //(toXml()), building every folio: as when built - QETProject exported(zipped); - check(WiringListExport(&exported).toCsvString() == WiringListExport(&eager).toCsvString(), - "wiring list exported as when built"); - } //Numbering, the searches for symbols and the unused symbols //are answered from the database: no folio file is read @@ -799,7 +800,7 @@ int main(int argc, char **argv) check(listed.unloadedFolioCount() > 0, "listing the tables does not build every folio"); } - //The wire numbers export and the terminal + //The wire numbers and wiring list exports, and the terminal //numbering dialog: as when built, building only the folios //holding a terminal (D5) { @@ -807,7 +808,14 @@ int main(int argc, char **argv) const int unbuilt = exported.unloadedFolioCount(); check(ConductorNumExport(&exported).wiresNum() == ConductorNumExport(&eager).wiresNum(), "wire numbers exported as when built"); - check(exported.unloadedFolioCount() == unbuilt, "exporting wire numbers builds no folio"); + const QString wiring = WiringListExport(&exported).toCsvString(), usual_wiring = WiringListExport(&eager).toCsvString(); + if (wiring != usual_wiring) { + const QStringList a = wiring.split('\n'), b = usual_wiring.split('\n'); + for (const QString &line : b) if (!a.contains(line)) std::printf(" wiring when built: %s\n", qPrintable(line)); + for (const QString &line : a) if (!b.contains(line)) std::printf(" wiring lazily: %s\n", qPrintable(line)); + } + check(wiring == usual_wiring, "wiring list exported as when built"); + check(exported.unloadedFolioCount() == unbuilt, "exporting wire numbers and the wiring list builds no folio"); { TerminalNumberingDialog dialog(nullptr, &exported); } int with_terminals = 0; for (Diagram *folio : folios) From 9078e4e3216e590f994f2b83d40283741e121b63 Mon Sep 17 00:00:00 2001 From: ispyisail Date: Mon, 12 Oct 2026 14:20:17 +1300 Subject: [PATCH 12/26] Answer scripts from the folio asked for, without building every folio (stage D5) The scripting API (scripts and the assistant's live mode) reached a folio through diagrams(), which builds every folio, for every call. A folio's own data -- count, title, uuid, title block, border, wire defaults -- now comes from folios(), built or not; a call reading or changing a folio's items builds that folio, as showing it does (built()); selecting a symbol by uuid builds the folio the project database places it on. Search and replace and the continuity check still build every folio. DiagramContent worked out one wire of each potential when made, which follows potentials through folio reports and built the folios at the other end, for every caller; it is now worked out when first asked (Search and replace asks; a merge asks both sides, as before). tst_lazyfolios: on a .qetz, a script reads every folio's title, uuid and border building none (fails with diagrams()), and the symbols and wires of the folio with the most links to other folios as when built, building that folio only (27 before potentials were lazy). ctest 96/96 but modal_quit_regression in both modes: a race in that test (gdb interrupts the main thread inside malloc while the folio view repaints during the dialog, master too), 2 of 23 runs here, 0 of 10 on master, passing alone. 342 corpus projects save the same in all four arms, and export the same wiring list and wire numbers lazily. Co-Authored-By: Claude Opus 5.5 --- .../ui/searchandreplacewidget.cpp | 2 +- sources/diagramcontent.cpp | 39 ++-- sources/diagramcontent.h | 9 +- sources/scripting/qetscriptapi.cpp | 169 ++++++++++-------- sources/scripting/qetscriptapi.h | 2 + tests/qttest/lazy_folios_probe.cpp | 29 +++ 6 files changed, 164 insertions(+), 86 deletions(-) diff --git a/sources/SearchAndReplace/ui/searchandreplacewidget.cpp b/sources/SearchAndReplace/ui/searchandreplacewidget.cpp index 3a0e656ef..929710662 100644 --- a/sources/SearchAndReplace/ui/searchandreplacewidget.cpp +++ b/sources/SearchAndReplace/ui/searchandreplacewidget.cpp @@ -307,7 +307,7 @@ void SearchAndReplaceWidget::fillItemsList() m_indi_text_qtwi->sortChildren(0, Qt::AscendingOrder); - for (Conductor *c : dc.m_potential_conductors) + for (Conductor *c : dc.potentialConductors()) { QTreeWidgetItem *qtwi = new QTreeWidgetItem(m_conductor_qtwi); qtwi->setText(0, c->properties().text); diff --git a/sources/diagramcontent.cpp b/sources/diagramcontent.cpp index fc54e83c9..af6d66dbe 100644 --- a/sources/diagramcontent.cpp +++ b/sources/diagramcontent.cpp @@ -79,13 +79,7 @@ DiagramContent::DiagramContent(Diagram *diagram, bool selected) : m_other_conductors << c; } - if (m_potential_conductors.isEmpty()) { - m_potential_conductors << c; - } else { - if (!potentialIsManaged(c->relatedPotentialConductors(true).values())) { - m_potential_conductors << c; - } - } + m_potential_candidates << c; break; } case DiagramImageItem::Type: { m_images << qgraphicsitem_cast(item); break;} @@ -379,12 +373,14 @@ DiagramContent &DiagramContent::operator+=(const DiagramContent &other) if(!m_selected_items.contains(qgi)) m_selected_items << qgi; - for (Conductor *c : other.m_potential_conductors) + QList potentials = potentialConductors(); + for (Conductor *c : other.potentialConductors()) { QList c_list = c->relatedPotentialConductors(true).values(); c_list << c; if (!potentialIsManaged(c_list)) { - m_potential_conductors << c; + potentials << c; + m_potential_conductors = potentials; } } @@ -402,17 +398,40 @@ DiagramContent &DiagramContent::operator+=(const DiagramContent &other) */ bool DiagramContent::potentialIsManaged(QList conductors) { + const QList potentials = potentialConductors(); bool b = false; for (Conductor *c : conductors) { - if (m_potential_conductors.contains(c)) + if (potentials.contains(c)) b = true; } return b; } +/** + @brief DiagramContent::potentialConductors + @return one wire of each potential of the wires of this content, worked + out the first time it is asked (see m_potential_conductors) +*/ +QList DiagramContent::potentialConductors() const +{ + if (!m_potential_conductors) { + QList potentials; + for (Conductor *c : m_potential_candidates) { + bool managed = false; + if (!potentials.isEmpty()) + for (Conductor *related : c->relatedPotentialConductors(true)) + if (potentials.contains(related)) { managed = true; break; } + if (!managed) + potentials << c; + } + m_potential_conductors = potentials; + } + return *m_potential_conductors; +} + /** @brief DiagramContent::hasTextEditing @return true if handle a text currently in editing interaction diff --git a/sources/diagramcontent.h b/sources/diagramcontent.h index 92d7c1caf..0afcf4516 100644 --- a/sources/diagramcontent.h +++ b/sources/diagramcontent.h @@ -19,6 +19,7 @@ #define DIAGRAM_CONTENT_H #include +#include #include #include "../qetgraphicsitem/terminalelement.h" @@ -77,7 +78,6 @@ class DiagramContent QList m_conductors_to_update; QList m_conductors_to_move; QList m_other_conductors; - QList m_potential_conductors; QSet m_element_texts; QSet m_texts_groups; QList m_selected_items; @@ -89,6 +89,7 @@ class DiagramContent QList selectedTexts() const; QList selectedTextsGroup() const; QList conductors(int = AnyConductor) const; + QList potentialConductors() const; bool hasDeletableItems() const; bool hasCopiableItems() const; QList items(int = All) const; @@ -103,6 +104,12 @@ class DiagramContent private: int removePinnedGroups(); + /// The wires potentialConductors() is made from, in order + QList m_potential_candidates; + /// One wire of each potential, worked out when first asked: + /// following a potential goes through folio reports to other + /// folios, building those not built yet (QET_LAZY_FOLIOS) + mutable std::optional> m_potential_conductors; }; QDebug &operator<<(QDebug, DiagramContent &); #endif diff --git a/sources/scripting/qetscriptapi.cpp b/sources/scripting/qetscriptapi.cpp index be97fc618..953e355b8 100644 --- a/sources/scripting/qetscriptapi.cpp +++ b/sources/scripting/qetscriptapi.cpp @@ -126,7 +126,7 @@ QString QetScriptApi::filePath() const int QetScriptApi::folioCount() const { - return m_project ? m_project->diagrams().count() : 0; + return m_project ? m_project->folios().count() : 0; } /** @@ -183,7 +183,7 @@ QStringList QetScriptApi::signatures() int QetScriptApi::currentFolio() const { if (!m_project) return -1; - const QList diagrams = m_project->diagrams(); + const QList diagrams = m_project->folios(); if (m_view && m_view->diagram()) { return diagrams.indexOf(m_view->diagram()); } @@ -193,7 +193,7 @@ int QetScriptApi::currentFolio() const QString QetScriptApi::folioTitle(int index) const { if (!m_project) return QString(); - const QList diagrams = m_project->diagrams(); + const QList diagrams = m_project->folios(); if (index < 0 || index >= diagrams.count()) return QString(); return diagrams.at(index)->title(); } @@ -207,7 +207,7 @@ QString QetScriptApi::folioTitle(int index) const QString QetScriptApi::folioUuid(int index) const { if (!m_project) return QString(); - const QList diagrams = m_project->diagrams(); + const QList diagrams = m_project->folios(); if (index < 0 || index >= diagrams.count()) return QString(); return diagrams.at(index)->uuid().toString(); } @@ -222,7 +222,7 @@ int QetScriptApi::folioIndex(const QString &uuid) const if (!m_project) return -1; const QUuid wanted(uuid); if (wanted.isNull()) return -1; - const QList diagrams = m_project->diagrams(); + const QList diagrams = m_project->folios(); for (int i = 0 ; i < diagrams.count() ; ++i) { if (diagrams.at(i)->uuid() == wanted) return i; } @@ -232,18 +232,18 @@ int QetScriptApi::folioIndex(const QString &uuid) const int QetScriptApi::elementCount(int folioIndex) const { if (!m_project) return 0; - const QList diagrams = m_project->diagrams(); + const QList diagrams = m_project->folios(); if (folioIndex < 0 || folioIndex >= diagrams.count()) return 0; - DiagramContent content(diagrams.at(folioIndex), false); + DiagramContent content(built(diagrams.at(folioIndex)), false); return content.m_elements.count(); } int QetScriptApi::conductorCount(int folioIndex) const { if (!m_project) return 0; - const QList diagrams = m_project->diagrams(); + const QList diagrams = m_project->folios(); if (folioIndex < 0 || folioIndex >= diagrams.count()) return 0; - DiagramContent content(diagrams.at(folioIndex), false); + DiagramContent content(built(diagrams.at(folioIndex)), false); return content.conductors(DiagramContent::AnyConductor).count(); } @@ -462,6 +462,19 @@ void QetScriptApi::setLive(QStringList *log) m_live_log = log; } +/** + @brief QetScriptApi::built + @return @p folio, built if it was not yet (QET_LAZY_FOLIOS), as showing + it builds it: what a call reading or changing its items needs. A + folio's own data (title block, border, wire defaults) needs no build. +*/ +Diagram *QetScriptApi::built(Diagram *folio) const +{ + if (folio && m_project) + m_project->buildFolioToShow(folio); + return folio; +} + /** @brief QetScriptApi::findElement @param folioIndex @@ -472,9 +485,9 @@ void QetScriptApi::setLive(QStringList *log) Element *QetScriptApi::findElement(int folioIndex, const QString &elementUuid) const { if (!m_project) return nullptr; - const QList diagrams = m_project->diagrams(); + const QList diagrams = m_project->folios(); if (folioIndex < 0 || folioIndex >= diagrams.count()) return nullptr; - DiagramContent content(diagrams.at(folioIndex), false); + DiagramContent content(built(diagrams.at(folioIndex)), false); for (Element *elmt : std::as_const(content.m_elements)) { if (elmt->uuid().toString() == elementUuid) return elmt; @@ -705,9 +718,9 @@ QString QetScriptApi::addElement(int folioIndex, const QString &locationPath, do log(QStringLiteral("qet.addElement: project is read-only")); return QString(); } - const QList diagrams = m_project->diagrams(); + const QList diagrams = m_project->folios(); if (folioIndex < 0 || folioIndex >= diagrams.count()) return QString(); - Diagram *diagram = diagrams.at(folioIndex); + Diagram *diagram = built(diagrams.at(folioIndex)); // ElementsLocation::setPath() forces ANY path to embed:// as soon as a // non-null project is given, common://custom:// included -- passing @@ -817,7 +830,7 @@ bool QetScriptApi::deleteElement(int folioIndex, const QString &elementUuid) } Element *element = findElement(folioIndex, elementUuid); if (!element) return false; - Diagram *diagram = m_project->diagrams().at(folioIndex); + Diagram *diagram = built(m_project->folios().at(folioIndex)); DiagramContent content; content.m_elements << element; @@ -906,9 +919,9 @@ QStringList QetScriptApi::elementUuids(int folioIndex) const { QStringList uuids; if (!m_project) return uuids; - const QList diagrams = m_project->diagrams(); + const QList diagrams = m_project->folios(); if (folioIndex < 0 || folioIndex >= diagrams.count()) return uuids; - DiagramContent content(diagrams.at(folioIndex), false); + DiagramContent content(built(diagrams.at(folioIndex)), false); for (Element *elmt : std::as_const(content.m_elements)) { uuids << elmt->uuid().toString(); } @@ -1077,7 +1090,7 @@ bool QetScriptApi::addConductor(int folioIndex, return false; } - Diagram *diagram = m_project->diagrams().at(folioIndex); + Diagram *diagram = built(m_project->folios().at(folioIndex)); ConductorCreator creator(diagram, terminals); Q_UNUSED(creator) @@ -1110,10 +1123,10 @@ QStringList QetScriptApi::conductors(int folioIndex) const { QStringList list; if (!m_project) return list; - const QList diagrams = m_project->diagrams(); + const QList diagrams = m_project->folios(); if (folioIndex < 0 || folioIndex >= diagrams.count()) return list; - DiagramContent content(diagrams.at(folioIndex), false); + DiagramContent content(built(diagrams.at(folioIndex)), false); const QList all = content.conductors(DiagramContent::AnyConductor); for (Conductor *c : all) { @@ -1132,10 +1145,10 @@ QStringList QetScriptApi::conductorUuids(int folioIndex) const { QStringList list; if (!m_project) return list; - const QList diagrams = m_project->diagrams(); + const QList diagrams = m_project->folios(); if (folioIndex < 0 || folioIndex >= diagrams.count()) return list; - DiagramContent content(diagrams.at(folioIndex), false); + DiagramContent content(built(diagrams.at(folioIndex)), false); for (Conductor *c : content.conductors(DiagramContent::AnyConductor)) list << c->uuid().toString(); return list; @@ -1152,12 +1165,12 @@ QStringList QetScriptApi::conductorUuids(int folioIndex) const QStringList QetScriptApi::conductorEnds(int folioIndex, const QString &uuid) const { if (!m_project) return {}; - const QList diagrams = m_project->diagrams(); + const QList diagrams = m_project->folios(); if (folioIndex < 0 || folioIndex >= diagrams.count()) return {}; const QUuid wanted(uuid); if (wanted.isNull()) return {}; - DiagramContent content(diagrams.at(folioIndex), false); + DiagramContent content(built(diagrams.at(folioIndex)), false); for (Conductor *c : content.conductors(DiagramContent::AnyConductor)) if (c->uuid() == wanted) return {describeEnd(c->terminal1), describeEnd(c->terminal2)}; @@ -1787,18 +1800,18 @@ QList sortedByPosition(const QSet &items) QList QetScriptApi::sortedTexts(int folioIndex) const { if (!m_project) return {}; - const QList diagrams = m_project->diagrams(); + const QList diagrams = m_project->folios(); if (folioIndex < 0 || folioIndex >= diagrams.count()) return {}; - DiagramContent content(diagrams.at(folioIndex), false); + DiagramContent content(built(diagrams.at(folioIndex)), false); return sortedByPosition(content.m_text_fields); } QList QetScriptApi::sortedShapes(int folioIndex) const { if (!m_project) return {}; - const QList diagrams = m_project->diagrams(); + const QList diagrams = m_project->folios(); if (folioIndex < 0 || folioIndex >= diagrams.count()) return {}; - DiagramContent content(diagrams.at(folioIndex), false); + DiagramContent content(built(diagrams.at(folioIndex)), false); return sortedByPosition(content.m_shapes); } @@ -1843,9 +1856,9 @@ int QetScriptApi::addText(int folioIndex, const QString &text, double x, double log(QStringLiteral("qet.addText: project is read-only")); return -1; } - const QList diagrams = m_project->diagrams(); + const QList diagrams = m_project->folios(); if (folioIndex < 0 || folioIndex >= diagrams.count()) return -1; - Diagram *diagram = diagrams.at(folioIndex); + Diagram *diagram = built(diagrams.at(folioIndex)); auto *item = new IndependentTextItem(); item->setPlainText(text); @@ -1929,7 +1942,7 @@ bool QetScriptApi::deleteText(int folioIndex, int textIndex) } IndependentTextItem *item = findText(folioIndex, textIndex, QStringLiteral("deleteText")); if (!item) return false; - Diagram *diagram = m_project->diagrams().at(folioIndex); + Diagram *diagram = built(m_project->folios().at(folioIndex)); DiagramContent content; content.m_text_fields << item; @@ -1969,7 +1982,7 @@ int QetScriptApi::addShape(int folioIndex, const QString &type, log(QStringLiteral("qet.addShape: project is read-only")); return -1; } - const QList diagrams = m_project->diagrams(); + const QList diagrams = m_project->folios(); if (folioIndex < 0 || folioIndex >= diagrams.count()) return -1; QetShapeItem::ShapeType shape_type; @@ -1984,7 +1997,7 @@ int QetScriptApi::addShape(int folioIndex, const QString &type, return -1; } - Diagram *diagram = diagrams.at(folioIndex); + Diagram *diagram = built(diagrams.at(folioIndex)); auto *shape = new QetShapeItem(QPointF(x1, y1), QPointF(x2, y2), shape_type); diagram->undoStack().push(new AddGraphicsObjectCommand(shape, diagram, QPointF(0, 0))); return sortedShapes(folioIndex).indexOf(shape); @@ -2116,7 +2129,7 @@ bool QetScriptApi::deleteShape(int folioIndex, int shapeIndex) .arg(folioIndex).arg(list.count()).arg(shapeIndex)); return false; } - Diagram *diagram = m_project->diagrams().at(folioIndex); + Diagram *diagram = built(m_project->folios().at(folioIndex)); DiagramContent content; content.m_shapes << list.at(shapeIndex); @@ -2223,7 +2236,7 @@ int QetScriptApi::addPolygon(int folioIndex, const QVariantList &points, bool cl log(QStringLiteral("qet.addPolygon: project is read-only")); return -1; } - const QList diagrams = m_project->diagrams(); + const QList diagrams = m_project->folios(); if (folioIndex < 0 || folioIndex >= diagrams.count()) return -1; if (points.count() < 2) { log(QStringLiteral("qet.addPolygon: at least 2 points are required, got %1") @@ -2237,7 +2250,7 @@ int QetScriptApi::addPolygon(int folioIndex, const QVariantList &points, bool cl return -1; } - Diagram *diagram = diagrams.at(folioIndex); + Diagram *diagram = built(diagrams.at(folioIndex)); auto *shape = new QetShapeItem(poly.first(), poly.last(), QetShapeItem::Polygon); shape->setPolygon(poly); shape->setClosed(closed); @@ -2321,7 +2334,7 @@ int QetScriptApi::addPath(int folioIndex, const QVariantList &nodes, bool closed log(QStringLiteral("qet.addPath: project is read-only")); return -1; } - const QList diagrams = m_project->diagrams(); + const QList diagrams = m_project->folios(); if (folioIndex < 0 || folioIndex >= diagrams.count()) return -1; if (nodes.count() < 2) { log(QStringLiteral("qet.addPath: at least 2 nodes are required, got %1").arg(nodes.count())); @@ -2338,7 +2351,7 @@ int QetScriptApi::addPath(int folioIndex, const QVariantList &nodes, bool closed path_nodes << node; } - Diagram *diagram = diagrams.at(folioIndex); + Diagram *diagram = built(diagrams.at(folioIndex)); auto *shape = new QetShapeItem(path_nodes.first().anchor, path_nodes.last().anchor, QetShapeItem::Path); shape->setPathNodes(path_nodes); @@ -2582,7 +2595,7 @@ bool QetScriptApi::deleteConductor(int folioIndex, const QString &elementUuid, i } Conductor *conductor = findConductor(folioIndex, elementUuid, terminalIndex, caller); if (!conductor) return false; - Diagram *diagram = m_project->diagrams().at(folioIndex); + Diagram *diagram = built(m_project->folios().at(folioIndex)); DiagramContent content; content.m_conductors_to_move << conductor; @@ -2603,10 +2616,10 @@ bool QetScriptApi::removeFolio(int folioIndex) log(QStringLiteral("qet.removeFolio: project is read-only")); return false; } - const QList diagrams = m_project->diagrams(); + const QList diagrams = m_project->folios(); if (folioIndex < 0 || folioIndex >= diagrams.count()) return false; m_project->undoStack()->push(new RemoveDiagramCommand(m_project, diagrams.at(folioIndex))); - return m_project->diagrams().count() == diagrams.count() - 1; + return m_project->folios().count() == diagrams.count() - 1; } namespace { @@ -2640,7 +2653,7 @@ const QStringList &titleBlockFieldNames() QString QetScriptApi::folioProperty(int folioIndex, const QString &property) const { if (!m_project) return QString(); - const QList diagrams = m_project->diagrams(); + const QList diagrams = m_project->folios(); if (folioIndex < 0 || folioIndex >= diagrams.count()) return QString(); Diagram *diagram = diagrams.at(folioIndex); if (property == QLatin1String("template")) @@ -2665,7 +2678,7 @@ bool QetScriptApi::setFolioProperty(int folioIndex, const QString &property, con log(QStringLiteral("qet.setFolioProperty: project is read-only")); return false; } - const QList diagrams = m_project->diagrams(); + const QList diagrams = m_project->folios(); if (folioIndex < 0 || folioIndex >= diagrams.count()) return false; Diagram *diagram = diagrams.at(folioIndex); @@ -2958,9 +2971,9 @@ bool QetScriptApi::sortTerminalStrip(int stripIndex) QList QetScriptApi::sortedTables(int folioIndex) const { if (!m_project) return {}; - const QList diagrams = m_project->diagrams(); + const QList diagrams = m_project->folios(); if (folioIndex < 0 || folioIndex >= diagrams.count()) return {}; - DiagramContent content(diagrams.at(folioIndex), false); + DiagramContent content(built(diagrams.at(folioIndex)), false); return sortedByPosition(content.m_tables); } @@ -3020,7 +3033,7 @@ int QetScriptApi::addTable(int folioIndex, const QString &kind, const QString &n log(QStringLiteral("qet.%1: project is read-only").arg(caller)); return -1; } - const QList diagrams = m_project->diagrams(); + const QList diagrams = m_project->folios(); if (folioIndex < 0 || folioIndex >= diagrams.count()) { log(QStringLiteral("qet.%1: no folio at index %2").arg(caller).arg(folioIndex)); return -1; @@ -3089,7 +3102,7 @@ int QetScriptApi::addTable(int folioIndex, const QString &kind, const QString &n dialog.setAddNewTableToNewDiagram(false); const QList before = sortedTables(folioIndex); - QetGraphicsTableFactory::create(diagrams.at(folioIndex), &dialog); + QetGraphicsTableFactory::create(built(diagrams.at(folioIndex)), &dialog); const QList after = sortedTables(folioIndex); for (QetGraphicsTableItem *t : after) { if (!before.contains(t)) return after.indexOf(t); @@ -3105,9 +3118,9 @@ bool QetScriptApi::deleteTable(int folioIndex, int tableIndex) log(QStringLiteral("qet.deleteTable: project is read-only")); return false; } - const QList diagrams = m_project->diagrams(); + const QList diagrams = m_project->folios(); if (folioIndex < 0 || folioIndex >= diagrams.count()) return false; - Diagram *diagram = diagrams.at(folioIndex); + Diagram *diagram = built(diagrams.at(folioIndex)); const QList all = sortedTables(folioIndex); if (tableIndex < 0 || tableIndex >= all.count()) { @@ -3381,7 +3394,7 @@ bool QetScriptApi::useConductorAutoNum(int folioIndex, const QString &name) log(QStringLiteral("qet.useConductorAutoNum: project is read-only")); return false; } - const QList diagrams = m_project->diagrams(); + const QList diagrams = m_project->folios(); if (folioIndex < 0 || folioIndex >= diagrams.count()) return false; if (!name.isEmpty() && !m_project->conductorAutoNum().contains(name)) { log(QStringLiteral("qet.useConductorAutoNum: no conductor auto-numbering named '%1'").arg(name)); @@ -3395,9 +3408,9 @@ bool QetScriptApi::useConductorAutoNum(int folioIndex, const QString &name) QList QetScriptApi::sortedImages(int folioIndex) const { if (!m_project) return {}; - const QList diagrams = m_project->diagrams(); + const QList diagrams = m_project->folios(); if (folioIndex < 0 || folioIndex >= diagrams.count()) return {}; - DiagramContent content(diagrams.at(folioIndex), false); + DiagramContent content(built(diagrams.at(folioIndex)), false); return sortedByPosition(content.m_images); } @@ -3491,7 +3504,7 @@ int QetScriptApi::addImage(int folioIndex, const QString &filePath, double x, do log(QStringLiteral("qet.addImage: project is read-only")); return -1; } - const QList diagrams = m_project->diagrams(); + const QList diagrams = m_project->folios(); if (folioIndex < 0 || folioIndex >= diagrams.count()) return -1; // Same checks as a picture dropped on a folio: a regular file of at @@ -3503,7 +3516,7 @@ int QetScriptApi::addImage(int folioIndex, const QString &filePath, double x, do return -1; } - Diagram *diagram = diagrams.at(folioIndex); + Diagram *diagram = built(diagrams.at(folioIndex)); auto *item = new DiagramImageItem(QPixmap::fromImage(image)); diagram->undoStack().push(new AddGraphicsObjectCommand(item, diagram, QPointF(x, y))); return sortedImages(folioIndex).indexOf(item); @@ -3619,7 +3632,7 @@ bool QetScriptApi::deleteImage(int folioIndex, int imageIndex) .arg(folioIndex).arg(list.count()).arg(imageIndex)); return false; } - Diagram *diagram = m_project->diagrams().at(folioIndex); + Diagram *diagram = built(m_project->folios().at(folioIndex)); DiagramContent content; content.m_images << list.at(imageIndex); diagram->undoStack().push(new DeleteQGraphicsItemCommand(diagram, content)); @@ -3675,7 +3688,7 @@ int QetScriptApi::addPdfPage(int folioIndex, const QString &pdfPath, int pageNum log(QStringLiteral("qet.addPdfPage: project is read-only")); return -1; } - const QList diagrams = m_project->diagrams(); + const QList diagrams = m_project->folios(); if (folioIndex < 0 || folioIndex >= diagrams.count()) return -1; if (dpi <= 0) { log(QStringLiteral("qet.addPdfPage: dpi must be positive, got %1").arg(dpi)); @@ -3724,7 +3737,7 @@ int QetScriptApi::addPdfPage(int folioIndex, const QString &pdfPath, int pageNum painter.drawImage(0, 0, rendered); painter.end(); - Diagram *diagram = diagrams.at(folioIndex); + Diagram *diagram = built(diagrams.at(folioIndex)); auto *item = new DiagramImageItem(QPixmap::fromImage(background)); diagram->undoStack().push(new AddGraphicsObjectCommand(item, diagram, QPointF(x, y))); return sortedImages(folioIndex).indexOf(item); @@ -3948,7 +3961,7 @@ bool QetScriptApi::deleteElementText(int folioIndex, const QString &elementUuid, DynamicElementTextItem *t = findElementText(folioIndex, elementUuid, textIndex, QStringLiteral("deleteElementText")); if (!t) return false; - Diagram *diagram = m_project->diagrams().at(folioIndex); + Diagram *diagram = built(m_project->folios().at(folioIndex)); DiagramContent content; content.m_element_texts << t; diagram->undoStack().push(new DeleteQGraphicsItemCommand(diagram, content)); @@ -4178,7 +4191,7 @@ QStringList QetScriptApi::duplicateElements(int fromFolioIndex, const QStringLis log(QStringLiteral("qet.%1: project is read-only").arg(caller)); return created; } - const QList diagrams = m_project->diagrams(); + const QList diagrams = m_project->folios(); if (fromFolioIndex < 0 || fromFolioIndex >= diagrams.count() || toFolioIndex < 0 || toFolioIndex >= diagrams.count()) { log(QStringLiteral("qet.%1: folio index out of range").arg(caller)); @@ -4188,8 +4201,8 @@ QStringList QetScriptApi::duplicateElements(int fromFolioIndex, const QStringLis log(QStringLiteral("qet.%1: no elements named").arg(caller)); return created; } - Diagram *source = diagrams.at(fromFolioIndex); - Diagram *target = diagrams.at(toFolioIndex); + Diagram *source = built(diagrams.at(fromFolioIndex)); + Diagram *target = built(diagrams.at(toFolioIndex)); QList chosen; for (const QString &uuid : elementUuids) { @@ -4273,7 +4286,7 @@ const QStringList &folioBorderNames() QString QetScriptApi::folioBorder(int folioIndex, const QString &property) const { if (!m_project) return QString(); - const QList diagrams = m_project->diagrams(); + const QList diagrams = m_project->folios(); if (folioIndex < 0 || folioIndex >= diagrams.count()) return QString(); const BorderProperties b = diagrams.at(folioIndex)->border_and_titleblock.exportBorder(); if (property == QLatin1String("columns")) return QString::number(b.columns_count); @@ -4412,7 +4425,7 @@ bool QetScriptApi::setFolioBorder(int folioIndex, const QString &property, const .arg(caller, property, folioBorderNames().join(QStringLiteral(", ")))); return false; } - const QList diagrams = m_project->diagrams(); + const QList diagrams = m_project->folios(); if (folioIndex < 0 || folioIndex >= diagrams.count()) return false; Diagram *diagram = diagrams.at(folioIndex); @@ -4498,7 +4511,7 @@ QString QetScriptApi::conductorDefault(int folioIndex, const QString &property) if (folioIndex == -1) { p = m_project->defaultConductorProperties(); } else { - const QList diagrams = m_project->diagrams(); + const QList diagrams = m_project->folios(); if (folioIndex < 0 || folioIndex >= diagrams.count()) return QString(); p = diagrams.at(folioIndex)->defaultConductorProperties; } @@ -4524,7 +4537,7 @@ bool QetScriptApi::setConductorDefault(int folioIndex, const QString &property, } Diagram *diagram = nullptr; if (folioIndex != -1) { - const QList diagrams = m_project->diagrams(); + const QList diagrams = m_project->folios(); if (folioIndex < 0 || folioIndex >= diagrams.count()) return false; diagram = diagrams.at(folioIndex); } @@ -4633,12 +4646,12 @@ QVariantMap QetScriptApi::terminalPosition(int folioIndex, const QString &elemen QVariantList QetScriptApi::conductorPath(int folioIndex, const QString &conductorUuid) const { if (!m_project) return {}; - const QList diagrams = m_project->diagrams(); + const QList diagrams = m_project->folios(); if (folioIndex < 0 || folioIndex >= diagrams.count()) return {}; const QUuid wanted(conductorUuid); if (wanted.isNull()) return {}; - DiagramContent content(diagrams.at(folioIndex), false); + DiagramContent content(built(diagrams.at(folioIndex)), false); for (Conductor *c : content.conductors(DiagramContent::AnyConductor)) { if (c->uuid() != wanted) continue; QVariantList points; @@ -4673,14 +4686,14 @@ int QetScriptApi::insertFolio(int position) log(QStringLiteral("qet.insertFolio: project is read-only")); return -1; } - if (position < 0 || position > m_project->diagrams().count()) { + if (position < 0 || position > m_project->folios().count()) { log(QStringLiteral("qet.insertFolio: position %1 is outside 0..%2") - .arg(position).arg(m_project->diagrams().count())); + .arg(position).arg(m_project->folios().count())); return -1; } Diagram *diagram = m_project->addNewDiagram(position); if (!diagram) return -1; - return m_project->diagrams().indexOf(diagram); + return m_project->folios().indexOf(diagram); } /** @@ -4760,7 +4773,7 @@ int QetScriptApi::addFolio() } Diagram *diagram = m_project->addNewDiagram(); if (!diagram) return -1; - return m_project->diagrams().indexOf(diagram); + return m_project->folios().indexOf(diagram); } bool QetScriptApi::setFolioTitle(int folioIndex, const QString &title) @@ -4770,7 +4783,7 @@ bool QetScriptApi::setFolioTitle(int folioIndex, const QString &title) log(QStringLiteral("qet.setFolioTitle: project is read-only")); return false; } - const QList diagrams = m_project->diagrams(); + const QList diagrams = m_project->folios(); if (folioIndex < 0 || folioIndex >= diagrams.count()) return false; Diagram *diagram = diagrams.at(folioIndex); @@ -5202,7 +5215,15 @@ QVariantList QetScriptApi::checkContinuity(int folioIndex) bool QetScriptApi::selectElement(const QString &elementUuid) { if (!m_project) return false; - for (Diagram *diagram : m_project->diagrams()) { + //The folio the project database places it on, built; every folio + //when it places it on none + QList folios; + const QSet on = m_project->dataBase()->folioUuidsOfElements({QUuid(elementUuid)}); + for (Diagram *diagram : m_project->folios()) + if (on.contains(diagram->uuid())) folios << built(diagram); + if (folios.isEmpty()) + folios = m_project->diagrams(); + for (Diagram *diagram : std::as_const(folios)) { DiagramContent content(diagram, false); for (Element *elmt : std::as_const(content.m_elements)) { if (elmt->uuid().toString() == elementUuid) { @@ -5218,7 +5239,7 @@ QStringList QetScriptApi::selectedElements(int folioIndex) const { QStringList list; if (!m_project) return list; - const QList diagrams = m_project->diagrams(); + const QList diagrams = m_project->folios(); if (folioIndex < 0 || folioIndex >= diagrams.count()) return list; for (QGraphicsItem *item : diagrams.at(folioIndex)->selectedItems()) { if (item->type() == Element::Type) { @@ -5232,7 +5253,7 @@ QStringList QetScriptApi::selectedElements(int folioIndex) const void QetScriptApi::deselectAll(int folioIndex) { if (!m_project) return; - const QList diagrams = m_project->diagrams(); + const QList diagrams = m_project->folios(); if (folioIndex < 0 || folioIndex >= diagrams.count()) return; diagrams.at(folioIndex)->clearSelection(); } diff --git a/sources/scripting/qetscriptapi.h b/sources/scripting/qetscriptapi.h index f6b8b0c0e..87f30aca5 100644 --- a/sources/scripting/qetscriptapi.h +++ b/sources/scripting/qetscriptapi.h @@ -25,6 +25,7 @@ #include class QETProject; +class Diagram; class DiagramView; class Element; class Terminal; @@ -642,6 +643,7 @@ class QetScriptApi : public QObject private: bool runFlag(const QString &flag, const QStringList &args); + Diagram *built(Diagram *folio) const; Element *findElement(int folioIndex, const QString &elementUuid) const; Terminal *findTerminal(int folioIndex, const QString &elementUuid, int terminalIndex, const QString &caller); diff --git a/tests/qttest/lazy_folios_probe.cpp b/tests/qttest/lazy_folios_probe.cpp index 4c1a24e96..39b69751b 100644 --- a/tests/qttest/lazy_folios_probe.cpp +++ b/tests/qttest/lazy_folios_probe.cpp @@ -30,6 +30,7 @@ #include "../../sources/ui/elementpropertieswidget.h" #include "../../sources/conductorautonumerotation.h" #include "../../sources/conductornumexport.h" +#include "../../sources/scripting/qetscriptapi.h" #include "../../sources/wiringlistexport.h" #include "../../sources/ui/terminalnumberingdialog.h" #include "../../sources/diagramcontent.h" @@ -800,6 +801,34 @@ int main(int argc, char **argv) check(listed.unloadedFolioCount() > 0, "listing the tables does not build every folio"); } + //A script (and the assistant's live mode) asking about the + //folios and one folio's items: that folio built, no other (D5) + { + QETProject scripted(zipped); + const int unbuilt = scripted.unloadedFolioCount(); + QetScriptApi api(&scripted), usual_api(&eager); + check(api.folioCount() == usual_api.folioCount(), "a script counts the folios as when built"); + for (int i = 0 ; i < folios.size() ; ++i) { + check(api.folioTitle(i) == usual_api.folioTitle(i) && api.folioUuid(i) == usual_api.folioUuid(i) + && api.folioBorder(i, "columns") == usual_api.folioBorder(i, "columns"), + "a script reads a folio's own data as when built"); + } + check(scripted.unloadedFolioCount() == unbuilt, "reading the folios' own data builds none"); + //The folio with the most links to other folios + const int target = busiest; + check(!scripted.folios().at(target)->isLoaded(), "a folio not built"); + QStringList lazy_uuids = api.elementUuids(target), usual_uuids = usual_api.elementUuids(target); + lazy_uuids.sort(); usual_uuids.sort(); + check(lazy_uuids == usual_uuids, "a script lists a folio's symbols as when built"); + QStringList lazy_wires = api.conductors(target), usual_wires = usual_api.conductors(target); + lazy_wires.sort(); usual_wires.sort(); + check(lazy_wires == usual_wires, "a script lists a folio's wires as when built"); + check(scripted.folios().at(target)->isLoaded(), "the folio a script reads is built"); + std::printf(".qetz script on folio %d: %d folios built\n", target + 1, + unbuilt - scripted.unloadedFolioCount()); + check(unbuilt - scripted.unloadedFolioCount() == 1, "a script reading one folio builds that folio only"); + } + //The wire numbers and wiring list exports, and the terminal //numbering dialog: as when built, building only the folios //holding a terminal (D5) From 244de054215d78e97c6178cc6f42091b6e274131 Mon Sep 17 00:00:00 2001 From: ispyisail Date: Mon, 12 Oct 2026 16:36:56 +1300 Subject: [PATCH 13/26] Change a symbol's information without building its folio (stage D5) A script changing the information of a symbol on a folio not built, when nothing else shows that information (no symbol built holds it, no partner is linked to it, no label formula), changes it through the project database: the store and the symbol's element info row, undoable by uuid (ChangeStoredElementInformationCommand). Reading it back reads the store. Building the symbol later takes the change (projectDataBase::takeUnbuiltEdit(), in Element::fromXml()), the label and texts following as for any change. Saving a .qetz writes the changed information in the database in place of the rows the folio was read with, its file copied, the folio still not built; a symbol whose file holds no information block, or a .qet, builds the folio to save. 136-folio .qetz, "comment" set on the 3 513 symbols the database can change: 0.9 s, no folio built, saved in 6.1 s, none built (27 s and 116 folios when saving built them). The same edits on the 4 039 symbols with no partner and no formula, lazily and with every folio built: the two saved projects are the same file. tst_lazyfolios: a script changes a symbol not built, building none; saved as when built (fails when building ignores the change, and when the .qetz writer does); undone; the folio built afterwards shows it. ctest 96/96 eager and lazy; 342 corpus projects save the same in all four arms and export the same. Co-Authored-By: Claude Opus 5.5 --- sources/container/qetcontainerdb.cpp | 22 +++ sources/dataBase/projectdatabase.cpp | 140 ++++++++++++++++++ sources/dataBase/projectdatabase.h | 11 ++ sources/qetgraphicsitem/element.cpp | 5 + sources/qetproject.cpp | 28 +++- sources/scripting/qetscriptapi.cpp | 37 +++++ sources/scripting/qetscriptapi.h | 5 + .../changeelementinformationcommand.cpp | 24 +++ .../changeelementinformationcommand.h | 25 ++++ tests/qttest/lazy_folios_probe.cpp | 51 +++++++ 10 files changed, 345 insertions(+), 3 deletions(-) diff --git a/sources/container/qetcontainerdb.cpp b/sources/container/qetcontainerdb.cpp index 541b1d00f..f455f945a 100644 --- a/sources/container/qetcontainerdb.cpp +++ b/sources/container/qetcontainerdb.cpp @@ -749,6 +749,28 @@ QByteArray QetContainerDb::extract(QDomElement project, QString *error, const QB QStringLiteral("INSERT INTO conductor SELECT * FROM old.conductor WHERE folio_id = ? ORDER BY rowid")}) { Statement rows(db, sql); ok = ok && rows.run({id}); + } + //Its symbols whose information changed since (QETProject::toXml()): + //their information rows written as a symbol's block writes them + for (const QDomElement &edited : children(d, QStringLiteral("qetz-edited-element"))) { + Statement find(db, QStringLiteral("SELECT key FROM element WHERE folio_id = ? AND uuid = ? AND info_at IS NOT NULL")); + find.bind({id, edited.attribute(QStringLiteral("uuid"))}); + if (!find.next()) { + setError(error, QStringLiteral("a symbol changed on folio %1 kept as it was is not in its database").arg(id)); + return {}; + } + const QString key = find.value(0).toString(); + Statement drop(db, QStringLiteral("DELETE FROM element_info WHERE element_key = ?")); + ok = ok && drop.run({key}); + int ord = 0; + for (QDomElement i : children(edited.firstChildElement(QStringLiteral("elementInformations")), + QStringLiteral("elementInformation"))) { + QVariant value; + simpleValue(i, &value); + ok = ok && element_info.run({key, ord++, i.hasAttribute(QStringLiteral("name")) + ? QVariant(i.attribute(QStringLiteral("name"))) : QVariant(), + value, attributesJson(i, QStringLiteral("name"), true)}); + } } if (!ok) { setError(error, QStringLiteral("folio %1 kept as it was is not in its database").arg(id)); diff --git a/sources/dataBase/projectdatabase.cpp b/sources/dataBase/projectdatabase.cpp index 74e391b5f..b076e92a0 100644 --- a/sources/dataBase/projectdatabase.cpp +++ b/sources/dataBase/projectdatabase.cpp @@ -1378,6 +1378,146 @@ autonum::sequentialNumbers projectDataBase::sequenceFromValue(const QVariant &va return sequence; } +/** + @brief projectDataBase::canEditUnbuilt + @return true if the information of the symbol @p element can be changed + through the store alone, its folio not built (QET_LAZY_FOLIOS): it is + placed once, on a folio not built, and nothing else shows its + information -- no symbol built (alone or with its folio) holds it, no + partner is linked to it, and it has no label formula, which could read + what changes +*/ +bool projectDataBase::canEditUnbuilt(const QUuid &element) const +{ + if (!hasElementInformation(element) || placedElementCount(element) != 0 + || !linkedSymbols(element).isEmpty() + || !m_element_information.value(element).value(QStringLiteral("formula")).toString().isEmpty()) { + return false; + } + QSqlQuery query(m_data_base); + query.prepare(QStringLiteral("SELECT COUNT(*) FROM element WHERE uuid = :uuid")); + query.bindValue(QStringLiteral(":uuid"), element.toString()); + return query.exec() && query.next() && query.value(0).toInt() == 1; +} + +/** + @brief projectDataBase::editUnbuiltElementInformation + Change the information of the symbol @p element, on a folio not built + (canEditUnbuilt()), to @p information, as Element::setElementInformations() + would: the store and the symbol's element info row. Building the symbol + takes it (takeUnbuiltEdit()); saving builds its folio first + (foliosWithUnbuiltEdits()). +*/ +void projectDataBase::editUnbuiltElementInformation(const QUuid &element, DiagramContext information) +{ + if (information.contains(QStringLiteral("formula_id")) + && information.value(QStringLiteral("formula")).toString().isEmpty()) { + information.remove(QStringLiteral("formula_id")); + } + QUuid folio; + { + QSqlQuery query(m_data_base); + query.prepare(QStringLiteral("SELECT diagram_uuid FROM element WHERE uuid = :uuid")); + query.bindValue(QStringLiteral(":uuid"), element.toString()); + if (query.exec() && query.next()) folio = QUuid(query.value(0).toString()); + } + //Whether the file has a block of information for it: a symbol + //saves one when it has any (Element::toXml()) + const bool had_information = m_unbuilt_edits.contains(element) + ? m_unbuilt_edits.value(element).had_information + : !m_element_information.value(element).keys().isEmpty(); + m_unbuilt_edits.insert(element, {information, folio, had_information}); + if (!storeElementInformation(element, information)) { + return; + } + m_content_changed = true; + if (!m_update_blocked) { + bindElementInfoValues(m_update_element_query, element.toString(), information, + information.value(QStringLiteral("label")).toString()); + if (!m_update_element_query.exec()) { + qDebug() << "projectDataBase::editUnbuiltElementInformation update error : " << m_update_element_query.lastError(); + } + } + emit dataBaseUpdated(); +} + +/** + @brief projectDataBase::takeUnbuiltEdit + @return the information @p element was given while its folio was not + built (editUnbuiltElementInformation()), forgotten here: the symbol + being built takes it. Nothing if it was given none. +*/ +std::optional projectDataBase::takeUnbuiltEdit(const QUuid &element) +{ + const auto edit = m_unbuilt_edits.constFind(element); + if (edit == m_unbuilt_edits.constEnd()) { + return std::nullopt; + } + const DiagramContext information = edit->information; + m_unbuilt_edits.erase(edit); + return information; +} + +/** + @brief projectDataBase::foliosWithUnbuiltEdits + @return the folios holding a symbol whose information was changed while + they were not built (editUnbuiltElementInformation()) +*/ +QSet projectDataBase::foliosWithUnbuiltEdits() const +{ + QSet folios; + for (const auto &edit : m_unbuilt_edits) folios.insert(edit.folio); + return folios; +} + +/** + @brief projectDataBase::unbuiltEditsOnFolio + @return the information changed through the store of the symbols on the + folio @p folio, not built (editUnbuiltElementInformation()), by symbol. + @p need_build is set when a .qetz cannot save one in place of the rows it + was read with: the file has no block of information for the symbol, or + the change empties it (a symbol with none saves no block) +*/ +QHash projectDataBase::unbuiltEditsOnFolio(const QUuid &folio, bool *need_build) const +{ + QHash edits; + if (need_build) *need_build = false; + for (auto it = m_unbuilt_edits.cbegin() ; it != m_unbuilt_edits.cend() ; ++it) { + if (it->folio != folio) continue; + edits.insert(it.key(), it->information); + if (need_build && (!it->had_information || it->information.keys().isEmpty())) + *need_build = true; + } + return edits; +} + +/** + @brief projectDataBase::setPlacedElementInformation + Give the symbol @p element its information @p information: through the + symbol when it is built, through the store when it is not + (canEditUnbuilt()). What undoing and redoing a change of a symbol not + built does, the symbol having perhaps been built since. + @return false if neither can +*/ +bool projectDataBase::setPlacedElementInformation(const QUuid &element, const DiagramContext &information) +{ + const QList> placed = m_placed_elements.values(element); + if (placed.size() == 1 && placed.first()) { + placed.first()->setElementInformations(information); + elementInfoChanged(placed.first().data()); + return true; + } + if (placed.isEmpty() && m_unbuilt_edits.contains(element)) { + editUnbuiltElementInformation(element, information); + return true; + } + if (placed.isEmpty() && canEditUnbuilt(element)) { + editUnbuiltElementInformation(element, information); + return true; + } + return false; +} + /** @brief projectDataBase::folioUuidsOfElements @return the uuids of the folios the symbols @p elements are on, read diff --git a/sources/dataBase/projectdatabase.h b/sources/dataBase/projectdatabase.h index 4170a288c..3448346ac 100644 --- a/sources/dataBase/projectdatabase.h +++ b/sources/dataBase/projectdatabase.h @@ -180,6 +180,12 @@ class projectDataBase : public QObject QString prefillReport() const; QSet foliosWithUnjoinedWires() const; QSet folioUuidsOfElements(const QSet &elements) const; + bool canEditUnbuilt(const QUuid &element) const; + void editUnbuiltElementInformation(const QUuid &element, DiagramContext information); + std::optional takeUnbuiltEdit(const QUuid &element); + QSet foliosWithUnbuiltEdits() const; + QHash unbuiltEditsOnFolio(const QUuid &folio, bool *need_build = nullptr) const; + bool setPlacedElementInformation(const QUuid &element, const DiagramContext &information); QHash elementTypesOnFolios(const QSet &folios) const; /// A symbol placed on a folio, as the element table has it struct FolioSymbol { @@ -335,6 +341,11 @@ class projectDataBase : public QObject //briefly gives a copy its original's uuid, so one uuid can have //two symbols until the copy's is renewed. QMultiHash> m_placed_elements; + /// Information changed through the store for symbols on folios + /// not built (editUnbuiltElementInformation()), with the folio + /// each is on: building the symbol takes it (takeUnbuiltEdit()) + struct UnbuiltEdit { DiagramContext information; QUuid folio; bool had_information; }; + QHash m_unbuilt_edits; QHash m_folio_titleblocks; QHash> m_placed_folios; QHash m_conductor_properties; diff --git a/sources/qetgraphicsitem/element.cpp b/sources/qetgraphicsitem/element.cpp index ae497bafb..87fc394b5 100644 --- a/sources/qetgraphicsitem/element.cpp +++ b/sources/qetgraphicsitem/element.cpp @@ -1115,6 +1115,11 @@ bool Element::fromXml(QDomElement &e, DiagramContext dc; dc.fromXml(e.firstChildElement(QStringLiteral("elementInformations")), QStringLiteral("elementInformation")); + //Changed through the project database while its folio was not + //built (QET_LAZY_FOLIOS): the change, not what the file says + if (diagram() && diagram()->project()) + if (const auto edited = diagram()->project()->dataBase()->takeUnbuiltEdit(m_uuid)) + dc = *edited; //Load override properties (For now, only used when the element is a terminal) if (m_data.m_type == ElementData::Terminal) diff --git a/sources/qetproject.cpp b/sources/qetproject.cpp index b788313c9..63d6a1239 100644 --- a/sources/qetproject.cpp +++ b/sources/qetproject.cpp @@ -2360,9 +2360,20 @@ QDomDocument QETProject::toXml(QHash *kept_folios, QStringList if (keep) { //Partners built alone and linked or unlinked since: their folio //is not what was read - for (Diagram *diagram : std::as_const(m_diagrams_list)) - if (!diagram->isLoaded() && diagram->partialChanged()) - buildFolios({diagram}); + //and symbols whose information was changed through the project + //database while their folio was not built: built together + //-- a .qetz writes those in its database instead, when it can + const QSet edited = m_data_base.foliosWithUnbuiltEdits(); + QList changed; + for (Diagram *diagram : std::as_const(m_diagrams_list)) { + if (diagram->isLoaded()) continue; + bool need_build = !kept_parts; + if (edited.contains(diagram->uuid()) && kept_parts) + m_data_base.unbuiltEditsOnFolio(diagram->uuid(), &need_build); + if (diagram->partialChanged() || (edited.contains(diagram->uuid()) && need_build)) + changed << diagram; + } + buildFolios(changed); for (Diagram *diagram : std::as_const(m_diagrams_list)) if (diagram->headerChanged()) buildFolioToShow(diagram); @@ -2483,6 +2494,17 @@ QDomDocument QETProject::toXml(QHash *kept_folios, QStringList auto placeholder = xml_doc.createElement(QStringLiteral("diagram")); placeholder.setAttribute(QStringLiteral("qetz-kept-part"), diagram->deferredPart()); placeholder.setAttribute(QStringLiteral("uuid"), diagram->uuid().toString()); + //Symbols whose information changed while the folio was + //not built: the rows QetContainer::split() writes for them + const QHash edits = m_data_base.unbuiltEditsOnFolio(diagram->uuid()); + for (auto it = edits.cbegin() ; it != edits.cend() ; ++it) { + auto symbol = xml_doc.createElement(QStringLiteral("qetz-edited-element")); + symbol.setAttribute(QStringLiteral("uuid"), it.key().toString()); + auto information = xml_doc.createElement(QStringLiteral("elementInformations")); + it.value().toXml(information, QStringLiteral("elementInformation")); + symbol.appendChild(information); + placeholder.appendChild(symbol); + } kept_parts->append(diagram->deferredPart()); project_root.appendChild(placeholder); ++order_num; diff --git a/sources/scripting/qetscriptapi.cpp b/sources/scripting/qetscriptapi.cpp index 953e355b8..32247aed9 100644 --- a/sources/scripting/qetscriptapi.cpp +++ b/sources/scripting/qetscriptapi.cpp @@ -462,6 +462,28 @@ void QetScriptApi::setLive(QStringList *log) m_live_log = log; } +/** + @brief QetScriptApi::unbuiltInformation + @return the information of the symbol @p elementUuid on the folio + @p folioIndex when that folio is not built (QET_LAZY_FOLIOS) and the + symbol's information can be read and changed through the project + database alone (projectDataBase::canEditUnbuilt()); nothing otherwise, + the folio to be built +*/ +std::optional QetScriptApi::unbuiltInformation(int folioIndex, const QString &elementUuid) const +{ + if (!m_project || folioIndex < 0 || folioIndex >= m_project->folios().count()) return std::nullopt; + Diagram *folio = m_project->folios().at(folioIndex); + const QUuid uuid(elementUuid); + projectDataBase *database = m_project->dataBase(); + if (folio->isLoaded() || !database->rowsFromDocument() + || !database->folioUuidsOfElements({uuid}).contains(folio->uuid()) + || !database->canEditUnbuilt(uuid)) { + return std::nullopt; + } + return database->elementInformation(uuid); +} + /** @brief QetScriptApi::built @return @p folio, built if it was not yet (QET_LAZY_FOLIOS), as showing @@ -666,6 +688,19 @@ bool QetScriptApi::setInfoKey(int folioIndex, const QString &elementUuid, log(QStringLiteral("qet.%1: \"%2\" is internal, follow a numbering with numberElement()") .arg(caller, key)); return false; + } + //A symbol on a folio not built whose information nothing else + //shows is changed through the project database, its folio left + //unbuilt (stage D5) + if (key != QETInformation::ELMT_FORMULA) { + if (const auto stored = unbuiltInformation(folioIndex, elementUuid)) { + if (stored->value(key).toString() == value) return true; // nothing to push + DiagramContext new_info = *stored; + new_info.addValue(key, value); + m_project->undoStack()->push(new ChangeStoredElementInformationCommand( + m_project, QUuid(elementUuid), *stored, new_info)); + return true; + } } Element *element = findElement(folioIndex, elementUuid); if (!element) return false; @@ -1009,6 +1044,8 @@ int QetScriptApi::terminalIndex(int folioIndex, const QString &elementUuid, QString QetScriptApi::elementInfo(int folioIndex, const QString &elementUuid, const QString &key) const { + if (const auto stored = unbuiltInformation(folioIndex, elementUuid)) + return stored->value(key).toString(); Element *element = findElement(folioIndex, elementUuid); if (!element) return QString(); return element->elementInformations().value(key).toString(); diff --git a/sources/scripting/qetscriptapi.h b/sources/scripting/qetscriptapi.h index 87f30aca5..6a954d5a6 100644 --- a/sources/scripting/qetscriptapi.h +++ b/sources/scripting/qetscriptapi.h @@ -24,6 +24,10 @@ #include #include +#include + +#include "../diagramcontext.h" + class QETProject; class Diagram; class DiagramView; @@ -644,6 +648,7 @@ class QetScriptApi : public QObject private: bool runFlag(const QString &flag, const QStringList &args); Diagram *built(Diagram *folio) const; + std::optional unbuiltInformation(int folioIndex, const QString &elementUuid) const; Element *findElement(int folioIndex, const QString &elementUuid) const; Terminal *findTerminal(int folioIndex, const QString &elementUuid, int terminalIndex, const QString &caller); diff --git a/sources/undocommand/changeelementinformationcommand.cpp b/sources/undocommand/changeelementinformationcommand.cpp index bf7045b9d..7a76220e5 100644 --- a/sources/undocommand/changeelementinformationcommand.cpp +++ b/sources/undocommand/changeelementinformationcommand.cpp @@ -119,3 +119,27 @@ void ChangeElementInformationCommand::updateProjectDB() elmt->diagram()->project()->dataBase()->elementInfoChanged(list_); } } + +ChangeStoredElementInformationCommand::ChangeStoredElementInformationCommand( + QETProject *project, const QUuid &element, const DiagramContext &old_info, + const DiagramContext &new_info, QUndoCommand *parent) : + QUndoCommand(parent), + m_project(project), + m_element(element), + m_old_info(old_info), + m_new_info(new_info) +{ + setText(QObject::tr("Change the information of the element")); +} + +void ChangeStoredElementInformationCommand::undo() +{ + if (m_project) + m_project->dataBase()->setPlacedElementInformation(m_element, m_old_info); +} + +void ChangeStoredElementInformationCommand::redo() +{ + if (m_project) + m_project->dataBase()->setPlacedElementInformation(m_element, m_new_info); +} diff --git a/sources/undocommand/changeelementinformationcommand.h b/sources/undocommand/changeelementinformationcommand.h index 6e103ebf6..1d7d2cf70 100644 --- a/sources/undocommand/changeelementinformationcommand.h +++ b/sources/undocommand/changeelementinformationcommand.h @@ -21,8 +21,11 @@ #include "../diagramcontext.h" #include +#include +#include class Element; +class QETProject; /** @brief The ChangeElementInformationCommand class @@ -52,4 +55,26 @@ class ChangeElementInformationCommand : public QUndoCommand QMap, QPair> m_map; }; +/** + @brief The ChangeStoredElementInformationCommand class + Change the information of a symbol given by its uuid, built or not + (QET_LAZY_FOLIOS): through the symbol when it is built, through the + project database when it is not (projectDataBase::canEditUnbuilt()). +*/ +class ChangeStoredElementInformationCommand : public QUndoCommand +{ + public: + ChangeStoredElementInformationCommand(QETProject *project, const QUuid &element, + const DiagramContext &old_info, + const DiagramContext &new_info, + QUndoCommand *parent = nullptr); + void undo() override; + void redo() override; + + private: + QPointer m_project; + QUuid m_element; + DiagramContext m_old_info, m_new_info; +}; + #endif // CHANGEELEMENTINFORMATIONCOMMAND_H diff --git a/tests/qttest/lazy_folios_probe.cpp b/tests/qttest/lazy_folios_probe.cpp index 39b69751b..0375b24f5 100644 --- a/tests/qttest/lazy_folios_probe.cpp +++ b/tests/qttest/lazy_folios_probe.cpp @@ -829,6 +829,57 @@ int main(int argc, char **argv) check(unbuilt - scripted.unloadedFolioCount() == 1, "a script reading one folio builds that folio only"); } + //A script changes the information of a symbol on a folio not + //built, which nothing else shows: through the project + //database, no folio built; saved as when built; undone (D5) + { + QETProject changed(zipped); + const int unbuilt = changed.unloadedFolioCount(); + int folio_index = -1; + QString uuid; + for (int i = 0 ; i < folios.size() && uuid.isEmpty() ; ++i) { + if (changed.folios().at(i)->isLoaded()) continue; + for (Element *element : folios.at(i)->elements()) + if (changed.dataBase()->canEditUnbuilt(element->uuid())) { + folio_index = i; uuid = element->uuid().toString(); break; + } + } + check(!uuid.isEmpty(), "a symbol to change through the database"); + QetScriptApi api(&changed); + const QString before = api.elementInfo(folio_index, uuid, "comment"); + const QString value = QStringLiteral("changed through the database"); + check(api.setElementInfo(folio_index, uuid, "comment", value), "a script changes a symbol not built"); + check(api.elementInfo(folio_index, uuid, "comment") == value, "the change read back"); + check(changed.unloadedFolioCount() == unbuilt, "changing a symbol not built builds no folio"); + const QString changed_path = output + "/changed.qetz"; + check(changed.writeContainerFile(changed_path, &error), "saved after the change"); + std::printf(".qetz symbol on folio %d changed by a script: %d folios built, %d by saving\n", + folio_index + 1, 0, unbuilt - changed.unloadedFolioCount()); + qunsetenv("QET_LAZY_FOLIOS"); + const QString usual_path = output + "/changed-usual.qetz"; + { + QETProject usual_changed(zipped); + QetScriptApi usual_api(&usual_changed); + check(usual_api.setElementInfo(folio_index, uuid, "comment", value), "the same change when built"); + check(usual_changed.writeContainerFile(usual_path, &error), "saved after the change when built"); + } + QETProject changed_back(changed_path), usual_back(usual_path); + qputenv("QET_LAZY_FOLIOS", "1"); + check(sortedLines(changed_back.toXml().toString(), path) == sortedLines(usual_back.toXml().toString(), path), + "a symbol changed through the database saved as when built"); + check(QetScriptApi(&changed_back).elementInfo(folio_index, uuid, "comment") == value, "the change saved"); + //Undone, then redone and the folio built: the symbol shows it + changed.undoStack()->undo(); + check(api.elementInfo(folio_index, uuid, "comment") == before, "the change undone"); + changed.undoStack()->redo(); + changed.buildFolioToShow(changed.folios().at(folio_index)); + Element *built_symbol = nullptr; + for (Element *element : changed.folios().at(folio_index)->elements()) + if (element->uuid().toString() == uuid) built_symbol = element; + check(built_symbol && built_symbol->elementInformations().value("comment").toString() == value, + "a symbol changed before its folio was built shows the change"); + } + //The wire numbers and wiring list exports, and the terminal //numbering dialog: as when built, building only the folios //holding a terminal (D5) From b151f52428bb16844a678e9a22f780a40d1fa299 Mon Sep 17 00:00:00 2001 From: ispyisail Date: Mon, 12 Oct 2026 17:54:37 +1300 Subject: [PATCH 14/26] Save a new title, author or revision without building the folio (stage D5) A folio not built whose own data changed was built to save, with the folios its links reach. When what changed is only title block fields nothing on a folio is computed from -- its title, author, date, revision index, version, file name, template; a formula reads the folio's label, plant, location, extra fields and grid -- its items as read are what building it would write: it is saved from them with its own data as it is now (Diagram::deferredXmlWithHeader()), .qet and .qetz alike, with the information of its symbols changed through the project database (or built, when one of those needs it). Any other change still builds it. 136-folio .qetz, revision index and author set on every folio by a script: 0.8 s, saved in 6.8 s, no folio built; with every folio built, the same file. tst_lazyfolios: a new revision on every folio of a .qet and of a .qetz saved as when built, building only the folios holding a table (fails, 47 built, when every change counts); a folio whose label changed is built (fails when none counts). ctest 96/96 eager and lazy; 342 corpus projects save the same in all four arms and export the same. Co-Authored-By: Claude Opus 5.5 --- sources/diagram.cpp | 51 +++++++++++++++++++++ sources/diagram.h | 2 + sources/qetproject.cpp | 30 +++++++++++- tests/qttest/lazy_folios_probe.cpp | 73 ++++++++++++++++++++++++++++++ 4 files changed, 154 insertions(+), 2 deletions(-) diff --git a/sources/diagram.cpp b/sources/diagram.cpp index 7dcbf2427..316593e78 100644 --- a/sources/diagram.cpp +++ b/sources/diagram.cpp @@ -830,6 +830,57 @@ QList Diagram::buildSymbols(const QSet &uuids) return built; } +namespace { + //The title block fields nothing on a folio is computed from: a formula + //reads the folio's label, plant, location, extra fields and grid + //(autonum::FormulaContext), none of these +const QStringList ShownOnlyFields = { + QStringLiteral("title"), QStringLiteral("author"), QStringLiteral("date"), + QStringLiteral("indexrev"), QStringLiteral("version"), QStringLiteral("filename"), + QStringLiteral("titleblocktemplate"), QStringLiteral("titleblocktemplateCollection")}; + +QString headerWithoutShownOnly(const QString &header) +{ + QDomDocument document; + document.setContent(header); + QDomElement root = document.documentElement(); + for (const QString &field : ShownOnlyFields) root.removeAttribute(field); + return document.toString(); +} +} + +/** + @brief Diagram::headerChangeShowsOnly + @return true if what changed in this folio's own data since it was read + (headerChanged()) is only title block fields nothing on the folio is + computed from (its title, author, date, revision, file name, template): + its items, as read, are what building it would write +*/ +bool Diagram::headerChangeShowsOnly() +{ + return headerWithoutShownOnly(headerXml()) == headerWithoutShownOnly(m_opened_header); +} + +/** + @brief Diagram::deferredXmlWithHeader + @return this folio, not built, as it saves: its own data as it is now + (writeHeader()) and its items as read (deferredXml()), in @p document -- + when its items are what building it would write (headerChangeShowsOnly()) +*/ +QDomElement Diagram::deferredXmlWithHeader(QDomDocument &document) +{ + QDomElement folio = document.createElement(QStringLiteral("diagram")); + writeHeader(document, folio); + QSet header_children; + for (QDomElement child = folio.firstChildElement() ; !child.isNull() ; child = child.nextSiblingElement()) + header_children.insert(child.tagName()); + const QDomElement items = deferredXml(); + for (QDomElement child = items.firstChildElement() ; !child.isNull() ; child = child.nextSiblingElement()) + if (!header_children.contains(child.tagName())) + folio.appendChild(document.importNode(child, true)); + return folio; +} + /** @brief Diagram::deferredXml @return the XML of this folio's items while they are not built: kept diff --git a/sources/diagram.h b/sources/diagram.h index b29730aa4..4117bd8e6 100644 --- a/sources/diagram.h +++ b/sources/diagram.h @@ -240,6 +240,8 @@ class Diagram : public QGraphicsScene /// headerChanged() (QETProject::toXml()) void rememberHeader() {m_opened_header = headerXml();} bool headerChanged() {return headerXml() != m_opened_header;} + bool headerChangeShowsOnly(); + QDomElement deferredXmlWithHeader(QDomDocument &document); void ensureLoaded(); bool buildDeferred(); //The last argument is only for a copy fragment: when it is diff --git a/sources/qetproject.cpp b/sources/qetproject.cpp index 63d6a1239..3a640c798 100644 --- a/sources/qetproject.cpp +++ b/sources/qetproject.cpp @@ -2374,9 +2374,17 @@ QDomDocument QETProject::toXml(QHash *kept_folios, QStringList changed << diagram; } buildFolios(changed); - for (Diagram *diagram : std::as_const(m_diagrams_list)) - if (diagram->headerChanged()) + //-- unless it changed only in what nothing is computed from (its + //title, date, revision...): written from its items as read + //(Diagram::deferredXmlWithHeader()) + for (Diagram *diagram : std::as_const(m_diagrams_list)) { + if (!diagram->headerChanged()) continue; + bool edits_need_build = false; + if (!diagram->isLoaded()) + m_data_base.unbuiltEditsOnFolio(diagram->uuid(), &edits_need_build); + if (diagram->isLoaded() || !diagram->headerChangeShowsOnly() || edits_need_build) buildFolioToShow(diagram); + } //A table filled from the database saves the size its rows //give it, which any edit can change if (m_edited_since_open) @@ -2490,6 +2498,24 @@ QDomDocument QETProject::toXml(QHash *kept_folios, QStringList << "[" << diagram << "]"; + if (keep && !diagram->isLoaded() && diagram->headerChanged()) { + QDomElement xml_diagram = diagram->deferredXmlWithHeader(xml_doc); + //Its symbols whose information changed while it was not built + const QHash edits = m_data_base.unbuiltEditsOnFolio(diagram->uuid()); + for (const QDomElement &list : QET::findInDomElement(xml_diagram, QStringLiteral("elements"))) + for (QDomElement element = list.firstChildElement(QStringLiteral("element")) ; !element.isNull() ; + element = element.nextSiblingElement(QStringLiteral("element"))) { + const auto edit = edits.constFind(QUuid(element.attribute(QStringLiteral("uuid")))); + if (edit == edits.constEnd()) continue; + QDomElement information = xml_doc.createElement(QStringLiteral("elementInformations")); + edit->toXml(information, QStringLiteral("elementInformation")); + const QDomElement old_information = element.firstChildElement(QStringLiteral("elementInformations")); + element.replaceChild(information, old_information); //it has one (unbuiltEditsOnFolio()) + } + xml_diagram.setAttribute("order", order_num ++); + project_root.appendChild(xml_diagram); + continue; + } if (keep && !diagram->isLoaded() && kept_parts) { auto placeholder = xml_doc.createElement(QStringLiteral("diagram")); placeholder.setAttribute(QStringLiteral("qetz-kept-part"), diagram->deferredPart()); diff --git a/tests/qttest/lazy_folios_probe.cpp b/tests/qttest/lazy_folios_probe.cpp index 0375b24f5..3389782b8 100644 --- a/tests/qttest/lazy_folios_probe.cpp +++ b/tests/qttest/lazy_folios_probe.cpp @@ -49,6 +49,73 @@ #include static void check(bool value, const char *message) { if (!value) throw std::runtime_error(message); } +static QStringList sortedLines(QString xml, const QString &path); + + //A new revision on every folio (its title block's revision index and + //author), saved lazily and with every folio built: what saving builds, + //and whether the two saves hold the same. A change to folio + //@p label_folio's label (what formulas read) when it is not -1 +static void revisionEveryFolio(const QString &source, const QString &output, const QString &name, + const QString &path, int label_folio = -1) +{ + QString error; + qputenv("QET_LAZY_FOLIOS", "1"); + QETProject revised(source); + const int unbuilt = revised.unloadedFolioCount(); + QetScriptApi api(&revised); + for (int i = 0 ; i < api.folioCount() ; ++i) { + check(api.setFolioProperty(i, "indexrev", "B"), "a folio's revision set"); + check(api.setFolioProperty(i, "author", "Revised"), "a folio's author set"); + } + if (label_folio >= 0) + check(api.setFolioProperty(label_folio, "folio", "X%id"), "a folio's label set"); + check(revised.unloadedFolioCount() == unbuilt, "setting the folios' title blocks builds none"); + int with_tables = 0; + for (Diagram *folio : revised.folios()) + if (!folio->isLoaded() && revised.unbuiltFolioHasTables(folio)) ++with_tables; + const bool qetz = QETProject::isContainerPath(source); + const QString lazy_path = output + "/" + name + (qetz ? ".qetz" : ".qet"); + if (qetz) check(revised.writeContainerFile(lazy_path, &error), "revised project saved"); + else { revised.setFilePath(lazy_path); check(revised.write().isOk(), "revised project saved"); } + const int built = unbuilt - revised.unloadedFolioCount(); + std::printf("%s: new revision on every folio, %d folios built by saving, %d hold a table%s\n", + qPrintable(name), built, with_tables, + label_folio >= 0 ? ", one label changed" : ""); + if (label_folio < 0) + check(built <= with_tables, "saving a new revision builds only the folios holding a table"); + else + check(revised.folios().at(label_folio)->isLoaded(), "a folio whose label changed is built to save"); + qunsetenv("QET_LAZY_FOLIOS"); + const QString usual_path = output + "/" + name + "-usual" + (qetz ? ".qetz" : ".qet"); + { + QETProject usual(source); + QetScriptApi usual_api(&usual); + for (int i = 0 ; i < usual_api.folioCount() ; ++i) { + usual_api.setFolioProperty(i, "indexrev", "B"); + usual_api.setFolioProperty(i, "author", "Revised"); + } + if (label_folio >= 0) usual_api.setFolioProperty(label_folio, "folio", "X%id"); + if (qetz) check(usual.writeContainerFile(usual_path, &error), "revision saved when built"); + else { usual.setFilePath(usual_path); check(usual.write().isOk(), "revision saved when built"); } + } + QETProject lazy_back(lazy_path), usual_back(usual_path); + qputenv("QET_LAZY_FOLIOS", "1"); + //What a save says of itself (file name, path, time) left out + auto lines = [&path](QETProject &project) { + QStringList list = sortedLines(project.toXml().toString(), path); + list.removeIf([](const QString &line) { return line.contains(QLatin1String("name=\"saved")); }); + return list; + }; + const QStringList lazy_lines = lines(lazy_back), usual_lines = lines(usual_back); + if (lazy_lines != usual_lines) { + int shown = 0; + for (const QString &line : usual_lines) + if (!lazy_lines.contains(line) && shown++ < 8) std::printf(" when built: %s\n", qPrintable(line.left(300))); + for (const QString &line : lazy_lines) + if (!usual_lines.contains(line) && shown++ < 16) std::printf(" lazily: %s\n", qPrintable(line.left(300))); + } + check(lazy_lines == usual_lines, "a new revision saved as when built"); +} //Every symbol's numbering, in a stable order static QStringList numbering(const QETProject &project) @@ -380,6 +447,7 @@ int main(int argc, char **argv) for (const QString &title : titles) check(gaps(numbered, title) == gaps(eager, title), "a scheme's gaps as when built"); check(numbered.unloadedFolioCount() == folios.size() - strip_folios.size(), "numbering read without building"); + revisionEveryFolio(path, output, "revised-qet", path); { //Wire numbers exported from the database filled from the //document, as when built (D5) @@ -880,6 +948,11 @@ int main(int argc, char **argv) "a symbol changed before its folio was built shows the change"); } + //A new revision on every folio: saved from the folios as read, + //not built; a label change builds its folio (D5) + revisionEveryFolio(zipped, output, "revised", path); + revisionEveryFolio(zipped, output, "relabelled", path, busiest); + //The wire numbers and wiring list exports, and the terminal //numbering dialog: as when built, building only the folios //holding a terminal (D5) From 5c4fa8fadfa138639d6518231cc16eb605813ff5 Mon Sep 17 00:00:00 2001 From: ispyisail Date: Mon, 12 Oct 2026 19:27:22 +1300 Subject: [PATCH 15/26] Build folios only when needed, by default (stage D6) Every project now opens without building its folios; a folio is built when it is shown or something needs its items (QETProject::lazyFolios()). QET_LAZY_FOLIOS is gone; QET_EAGER_FOLIOS=1 builds every folio on opening, as before, for comparisons and troubleshooting. Comments that named the old variable now say "lazy folios"; the tests switch modes with the new one. The first save of a .qet as .qetz still builds every folio: a .qet does not say which program wrote it, so its folios are written again rather than copied. A .qetz opens and saves without building. ctest 96/96 by default and with QET_EAGER_FOLIOS=1; 342 corpus projects save the same in all four arms and export the same. In the GUI (qet-gui-dialog): a 136-folio .qetz opens with no folio built, shows folios 1 to 4, saves building none, the save = the original but the time spent in it; a .qet opens lazily, saves as a .qetz beside it after the notice, the .qet unchanged. Co-Authored-By: Claude Opus 5.5 --- sources/TerminalStrip/terminalstrip.cpp | 2 +- sources/cli_export.cpp | 6 +-- sources/conductornumexport.cpp | 2 +- sources/dataBase/projectdatabase.cpp | 22 ++++---- sources/dataBase/projectdatabase.h | 4 +- sources/diagram.cpp | 10 ++-- sources/diagram.h | 2 +- sources/diagramcontent.h | 2 +- sources/diagramview.cpp | 2 +- sources/elementprovider.cpp | 4 +- sources/elementprovider.h | 2 +- sources/projectview.cpp | 2 +- .../ViewItem/projectdbmodel.cpp | 4 +- .../ViewItem/qetgraphicstableitem.cpp | 4 +- sources/qetgraphicsitem/element.cpp | 8 +-- sources/qetgraphicsitem/terminal.cpp | 2 +- sources/qetproject.cpp | 41 +++++++++------ sources/qetproject.h | 10 +++- sources/scripting/qetscriptapi.cpp | 4 +- sources/ui/terminalnumberingdialog.cpp | 2 +- .../changeelementinformationcommand.h | 2 +- sources/wiringlistexport.cpp | 2 +- tests/qttest/lazy_folios_probe.cpp | 50 +++++++++---------- tests/qttest/tst_resaveunchanged.cpp | 2 +- 24 files changed, 104 insertions(+), 87 deletions(-) diff --git a/sources/TerminalStrip/terminalstrip.cpp b/sources/TerminalStrip/terminalstrip.cpp index cbb537cdb..5bbcf5cee 100644 --- a/sources/TerminalStrip/terminalstrip.cpp +++ b/sources/TerminalStrip/terminalstrip.cpp @@ -1011,7 +1011,7 @@ bool TerminalStrip::fromXml(QDomElement &xml_element) auto xml_layout = xml_element.firstChildElement(QStringLiteral("layout")); if (!xml_layout.isNull()) { - //The folios holding its terminals are built (QET_LAZY_FOLIOS): + //The folios holding its terminals are built (lazy folios): //a strip is read with the project, before the database could //tell where any symbol is QSet terminals; diff --git a/sources/cli_export.cpp b/sources/cli_export.cpp index cdb00096f..8214d4eea 100644 --- a/sources/cli_export.cpp +++ b/sources/cli_export.cpp @@ -878,7 +878,7 @@ int resaveProject(QETProject &project, const QString &output) out << "Re-saved project -> " << output << "\n"; return 0; } - //Folios not built (QET_LAZY_FOLIOS) written as read when that is + //Folios not built (lazy folios) written as read when that is //what building them would write (QETProject::toXml()) QHash kept_folios; const QDomDocument doc = project.toXml(&kept_folios); @@ -973,8 +973,8 @@ int setTitleBlock(QETProject &project, const QString &output, return 1; } } else { - //Folios not built (QET_LAZY_FOLIOS) written as read when that is - //what building them would write (QETProject::toXml()) + //Folios not built (lazy folios) written as read when that + //is what building them would write (QETProject::toXml()) QHash kept_folios; const QDomDocument doc = project.toXml(&kept_folios); QFile file(output); diff --git a/sources/conductornumexport.cpp b/sources/conductornumexport.cpp index f44e5b1bf..ea43be511 100644 --- a/sources/conductornumexport.cpp +++ b/sources/conductornumexport.cpp @@ -118,7 +118,7 @@ void ConductorNumExport::fillHash() auto count = [this](const QString &num) { m_hash.insert(num, m_hash.value(num, 0) + 1); }; - //The folios not built (QET_LAZY_FOLIOS): their wires from the + //The folios not built (lazy folios): their wires from the //project database, by the same rules QList unbuilt; for (Diagram *d : m_project->folios()) diff --git a/sources/dataBase/projectdatabase.cpp b/sources/dataBase/projectdatabase.cpp index b076e92a0..15da3aeae 100644 --- a/sources/dataBase/projectdatabase.cpp +++ b/sources/dataBase/projectdatabase.cpp @@ -141,7 +141,7 @@ void projectDataBase::updateDB() @brief projectDataBase::keepUnbuiltFolioRows Before the tables are rebuilt from the folios: keep the element, element_info, terminal, conductor and link rows of the folios not built - yet (QET_LAZY_FOLIOS). They came from the document, nothing of those + yet (lazy folios). They came from the document, nothing of those folios can have changed since, and the rebuild puts them back where each folio comes (restoreKeptRows()). Nothing to do when every folio is built. @@ -455,7 +455,7 @@ bool projectDataBase::populateFromDocument(const QDomDocument &document, QString //The folios, in the order the project read them (readDiagramsXml()), //each with its saved uuid -- the one the built folio has. const QDomNodeList diagram_nodes = document.elementsByTagName(QStringLiteral("diagram")); - //Built or not (QET_LAZY_FOLIOS): only their uuids are read, and the + //Built or not (lazy folios): only their uuids are read, and the //conductors of those built const QList diagrams = m_project->folios(); if (diagram_nodes.size() != diagrams.size()) { @@ -1215,7 +1215,7 @@ QSet projectDataBase::foliosWithUnjoinedWires() const @brief projectDataBase::elementTypesOnFolios @return the kind (ElementData::typeToString()) of every symbol on the folios @p folios, by uuid, read from the element table as it is: for - folios not built yet (QET_LAZY_FOLIOS) its rows came from the document + folios not built yet (lazy folios) its rows came from the document and nothing can have changed them. A symbol whose definition was not found has no row. */ @@ -1381,7 +1381,7 @@ autonum::sequentialNumbers projectDataBase::sequenceFromValue(const QVariant &va /** @brief projectDataBase::canEditUnbuilt @return true if the information of the symbol @p element can be changed - through the store alone, its folio not built (QET_LAZY_FOLIOS): it is + through the store alone, its folio not built (lazy folios): it is placed once, on a folio not built, and nothing else shows its information -- no symbol built (alone or with its folio) holds it, no partner is linked to it, and it has no label formula, which could read @@ -1522,7 +1522,7 @@ bool projectDataBase::setPlacedElementInformation(const QUuid &element, const Di @brief projectDataBase::folioUuidsOfElements @return the uuids of the folios the symbols @p elements are on, read from the element table as it is, without bringing it up to date: for a - symbol on a folio not built yet (QET_LAZY_FOLIOS) its row came from the + symbol on a folio not built yet (lazy folios) its row came from the document and nothing can have changed it. */ QSet projectDataBase::folioUuidsOfElements(const QSet &elements) const @@ -1660,7 +1660,7 @@ QSqlQuery projectDataBase::newQuery(const QString &query, QString *error) { //storeConductorProperties(). //The drawing tables hold what only built items know (a text's //laid-out size, a picture's pixels): with folios not built yet - //(QET_LAZY_FOLIOS) a read of them builds those first. Every other + //(lazy folios) a read of them builds those first. Every other //table was filled from the document. static const QRegularExpression drawing_tables( QStringLiteral("\\b(shape|independent_text|image)\\b"), @@ -1730,7 +1730,7 @@ int projectDataBase::excludedConductorCount() const return 0; } - //A folio not built (QET_LAZY_FOLIOS) has none: reading a wire + //A folio not built (lazy folios) has none: reading a wire //drops it unless both its ends are found (Conductor::fromXml()) int count = 0; for (auto *diagram : m_project->builtFolios()) @@ -1918,7 +1918,7 @@ void projectDataBase::flushElementPositions() /** @brief projectDataBase::updateFolioPositions Write every folio's position and folio number again, after a folio was - added, removed or moved. A folio not built yet (QET_LAZY_FOLIOS) has + added, removed or moved. A folio not built yet (lazy folios) has its position and title block already: it is not built for this. */ void projectDataBase::updateFolioPositions() @@ -2583,7 +2583,7 @@ void projectDataBase::populateDrawingItemTables() //Queued directly, not through addDrawingItem(): every one of them //came onto its folio through Diagram::addItem(), which connected it //already, and a full rebuild runs on every load. A folio not built - //yet (QET_LAZY_FOLIOS) has none: its items write their rows as it + //yet (lazy folios) has none: its items write their rows as it //is built. for (auto diagram : m_project->folios()) { @@ -3887,7 +3887,7 @@ void projectDataBase::prefillFromDocument(const QDomDocument &document) @brief projectDataBase::endPrefill The folios are built: compare what prefillFromDocument() stored with what building stored, and forget what nothing was placed for -- unless - @p keep_unplaced, when some folios are not built yet (QET_LAZY_FOLIOS): + @p keep_unplaced, when some folios are not built yet (lazy folios): the store then answers for their items until they are. */ void projectDataBase::endPrefill(bool keep_unplaced) @@ -4099,7 +4099,7 @@ void projectDataBase::flushLinks() qDebug() << "projectDataBase::flushLinks insert error : " << insert.lastError(); } } - //Partners on a folio not built (QET_LAZY_FOLIOS), which it will + //Partners on a folio not built (lazy folios), which it will //be linked to once that folio is for (const auto &waiting : element->waitingLinks()) { diff --git a/sources/dataBase/projectdatabase.h b/sources/dataBase/projectdatabase.h index 3448346ac..8d95fe0c5 100644 --- a/sources/dataBase/projectdatabase.h +++ b/sources/dataBase/projectdatabase.h @@ -160,7 +160,7 @@ class projectDataBase : public QObject //they are, say how many of those values building agreed with. void prefillFromDocument(const QDomDocument &document); void endPrefill(bool keep_unplaced = false); - //A folio of a project opened with QET_LAZY_FOLIOS is being built: + //A folio of a project opened with lazy folios is being built: //its rows came from the document already, so none are written //and the database is not marked changed void setBuildingFolio(bool building); @@ -357,7 +357,7 @@ class projectDataBase : public QObject //The folios whose rows a rebuild keeps (keepUnbuiltFolioRows()) QSet m_kept_folios; //The tables' rows were filled from the document: those of a - //folio not built yet are right (QET_LAZY_FOLIOS) + //folio not built yet are right (lazy folios) bool m_rows_from_document = false; void keepUnbuiltFolioRows(); void restoreKeptRows(const QString &table, Diagram *folio, const QString &where); diff --git a/sources/diagram.cpp b/sources/diagram.cpp index 316593e78..c47406143 100644 --- a/sources/diagram.cpp +++ b/sources/diagram.cpp @@ -772,7 +772,7 @@ void Diagram::keyReleaseEvent(QKeyEvent *e) /** @brief Diagram::ensureLoaded Build this folio's items from the XML kept when the project was - opened with QET_LAZY_FOLIOS, and do for them what opening does for + opened with lazy folios, and do for them what opening does for every folio's (QETProject::folioLoaded()). Nothing for a folio built already. */ @@ -790,7 +790,7 @@ void Diagram::ensureLoaded() /** @brief Diagram::buildSymbols - Build, of this folio not built yet (QET_LAZY_FOLIOS), the symbols + Build, of this folio not built yet (lazy folios), the symbols @p uuids only -- partners a shown folio draws from -- and keep them for when the folio is built (stage D5b). They belong to the folio's scene; the folio stays not built. @@ -1247,7 +1247,7 @@ QList < QSet > Diagram::potentials() \~French Un Document XML (QDomDocument) */ QDomDocument Diagram::toXml(bool whole_content, bool is_copy_command) { - //A folio of a project opened with QET_LAZY_FOLIOS is written from + //A folio of a project opened with lazy folios is written from //its built items (QETProject::toXml() may write one not built as //it was read instead) ensureLoaded(); @@ -1360,7 +1360,7 @@ QDomDocument Diagram::toXml(bool whole_content, bool is_copy_command) { //Wires that tie on their ends (common: several from one terminal) //keep the order they were made in, which for a project's is its //file's -- the order the scene gave them so far, except for a folio - //built late (QET_LAZY_FOLIOS), whose scene order differed + //built late (lazy folios), whose scene order differed std::stable_sort(list_conductors.begin(), list_conductors.end(), [](Conductor *a, Conductor *b) { const QString ka = conductorSortKey(a), kb = conductorSortKey(b); @@ -2042,7 +2042,7 @@ bool Diagram::fromXml(QDomElement &document, return(true); } - //A project opened with QET_LAZY_FOLIOS builds a folio's items when + //A project opened with lazy folios builds a folio's items when //they are first needed: until then the folio has what it says //about itself (uuid, title block, border, numbering) and keeps its //XML, which belongs to the project's document. diff --git a/sources/diagram.h b/sources/diagram.h index 4117bd8e6..5785b7efd 100644 --- a/sources/diagram.h +++ b/sources/diagram.h @@ -217,7 +217,7 @@ class Diagram : public QGraphicsScene bool = true, DiagramContent * = nullptr); /// false while this folio's items wait in its XML to be - /// built (QET_LAZY_FOLIOS) + /// built (lazy folios) bool isLoaded() const {return m_deferred.isNull() && m_deferred_part.isEmpty();} QDomElement deferredXml(); /// The folio's items are in the part @p file of the .qetz the diff --git a/sources/diagramcontent.h b/sources/diagramcontent.h index 0afcf4516..a693cfdf1 100644 --- a/sources/diagramcontent.h +++ b/sources/diagramcontent.h @@ -108,7 +108,7 @@ class DiagramContent QList m_potential_candidates; /// One wire of each potential, worked out when first asked: /// following a potential goes through folio reports to other - /// folios, building those not built yet (QET_LAZY_FOLIOS) + /// folios, building those not built yet (lazy folios) mutable std::optional> m_potential_conductors; }; QDebug &operator<<(QDebug, DiagramContent &); diff --git a/sources/diagramview.cpp b/sources/diagramview.cpp index 5a1bc6590..7ece3ee4a 100644 --- a/sources/diagramview.cpp +++ b/sources/diagramview.cpp @@ -1649,7 +1649,7 @@ void DiagramView::editDiagramProperties() @brief DiagramView::buildFolio Build the items of the folio this view shows, and those of the folios its cross-references need, if the project was opened without them - (QET_LAZY_FOLIOS); then fit the view to them. + (lazy folios); then fit the view to them. */ void DiagramView::buildFolio() { diff --git a/sources/elementprovider.cpp b/sources/elementprovider.cpp index 174151811..3bb4c4165 100644 --- a/sources/elementprovider.cpp +++ b/sources/elementprovider.cpp @@ -55,7 +55,7 @@ ElementProvider::ElementProvider(const QList &diagrams) : @brief ElementProvider::folios @return the folios to search for symbols of the kinds @p kinds (free ones only if @p free_only). For a project, the folios not built yet - (QET_LAZY_FOLIOS) holding such a symbol are built first, and only them + (lazy folios) holding such a symbol are built first, and only them (QETProject::buildFoliosHolding()). */ QList ElementProvider::folios(ElementData::Types kinds, bool free_only) const @@ -98,7 +98,7 @@ QVector > ElementProvider::freeElement(ElementData::Types filt QList elmt_list; const QList searched = folios(filter, true); //A symbol whose saved partner is on a folio not built yet - //(QET_LAZY_FOLIOS) is not linked yet, but is not free: it will be + //(lazy folios) is not linked yet, but is not free: it will be const QSet waiting = m_project ? m_project->symbolsOnUnbuiltFolios() : QSet(); diff --git a/sources/elementprovider.h b/sources/elementprovider.h index 5f839c38c..592088d83 100644 --- a/sources/elementprovider.h +++ b/sources/elementprovider.h @@ -55,7 +55,7 @@ class ElementProvider QList m_diagram_list; //Searching a project: its folios are asked for when searched, - //so that a project opened with QET_LAZY_FOLIOS builds only + //so that a project opened with lazy folios builds only //those a search can find something on QETProject *m_project = nullptr; Diagram *m_excluded = nullptr; diff --git a/sources/projectview.cpp b/sources/projectview.cpp index ab62a3591..158cc2ae5 100644 --- a/sources/projectview.cpp +++ b/sources/projectview.cpp @@ -1257,7 +1257,7 @@ void ProjectView::rebuildDiagramsMap() */ void ProjectView::tabChanged(int tab_id) { - //A folio opened without its items (QET_LAZY_FOLIOS) is built when + //A folio opened without its items (lazy folios) is built when //its tab is first shown. Read the tab itself: the first tab is //shown while loadDiagrams() is still filling m_diagram_ids if (auto dv = qobject_cast(m_tab->widget(tab_id))) diff --git a/sources/qetgraphicsitem/ViewItem/projectdbmodel.cpp b/sources/qetgraphicsitem/ViewItem/projectdbmodel.cpp index cf9525cc2..eb76c5662 100644 --- a/sources/qetgraphicsitem/ViewItem/projectdbmodel.cpp +++ b/sources/qetgraphicsitem/ViewItem/projectdbmodel.cpp @@ -323,7 +323,7 @@ void ProjectDBModel::setIdentifier(const QString &identifier) { /** @brief ProjectDBModel::folioBuilt - The folio of a table made while it was being built (QET_LAZY_FOLIOS) + The folio of a table made while it was being built (lazy folios) is built: read the database, as opening does once it has filled it. Nothing for any other model. */ @@ -391,7 +391,7 @@ void ProjectDBModel::fillValue() { m_record.clear(); //Opening builds a table before the database is filled, and fills - //it after: a table on a folio built late (QET_LAZY_FOLIOS) does + //it after: a table on a folio built late (lazy folios) does //the same, so that it is laid out as it would have been on //opening -- empty until its folio is built (folioBuilt()) if (m_fill_when_built) { diff --git a/sources/qetgraphicsitem/ViewItem/qetgraphicstableitem.cpp b/sources/qetgraphicsitem/ViewItem/qetgraphicstableitem.cpp index 0b6922ebf..6fe42bfcd 100644 --- a/sources/qetgraphicsitem/ViewItem/qetgraphicstableitem.cpp +++ b/sources/qetgraphicsitem/ViewItem/qetgraphicstableitem.cpp @@ -566,7 +566,7 @@ void QetGraphicsTableItem::initLink() if (!m_pending_previous_table_uuid.isNull()) { //The built folios only: a previous table on a folio not built - //yet (QET_LAZY_FOLIOS) is linked when that folio is + //yet (lazy folios) is linked when that folio is QETProject *project = this->diagram()->project(); ElementProvider provider_(project->builtFolios()); auto previous_table = provider_.tableFromUuid(m_pending_previous_table_uuid); @@ -583,7 +583,7 @@ void QetGraphicsTableItem::initLink() /** @brief QetGraphicsTableItem::previousTableUuid @return the uuid of the table before this one in its chain, linked or - still waiting for its folio to be built (QET_LAZY_FOLIOS); null if none + still waiting for its folio to be built (lazy folios); null if none */ QUuid QetGraphicsTableItem::previousTableUuid() const { diff --git a/sources/qetgraphicsitem/element.cpp b/sources/qetgraphicsitem/element.cpp index 87fc394b5..03e600436 100644 --- a/sources/qetgraphicsitem/element.cpp +++ b/sources/qetgraphicsitem/element.cpp @@ -1116,7 +1116,7 @@ bool Element::fromXml(QDomElement &e, dc.fromXml(e.firstChildElement(QStringLiteral("elementInformations")), QStringLiteral("elementInformation")); //Changed through the project database while its folio was not - //built (QET_LAZY_FOLIOS): the change, not what the file says + //built (lazy folios): the change, not what the file says if (diagram() && diagram()->project()) if (const auto edited = diagram()->project()->dataBase()->takeUnbuiltEdit(m_uuid)) dc = *edited; @@ -1255,7 +1255,7 @@ QDomElement Element::toXml( //if this element is linked to other elements, //save the uuid of each other elements - //A partner on a folio not built yet (QET_LAZY_FOLIOS), which this + //A partner on a folio not built yet (lazy folios), which this //symbol will be linked to, is saved too, in the order the symbol was //saved with: saving needs no partner built (stage D5a) QList> links; @@ -1661,7 +1661,7 @@ void Element::initLink(QETProject *prj) //The symbols built: on the folios built, and those built alone as //partners on folios not built (Diagram::buildSymbols()). A partner - //not built yet (QET_LAZY_FOLIOS) is linked when it is, and the + //not built yet (lazy folios) is linked when it is, and the //saved list is kept until then, for the order it gives ElementProvider ep(prj->folios()); QList uuids; @@ -1689,7 +1689,7 @@ void Element::initLink(QETProject *prj) /** @brief Element::pendingLinks @return the uuids of the partners this symbol was saved linked to and - is not linked to yet: those on a folio not built yet (QET_LAZY_FOLIOS), + is not linked to yet: those on a folio not built yet (lazy folios), until initLink() runs again once it is */ QSet Element::pendingLinks() const diff --git a/sources/qetgraphicsitem/terminal.cpp b/sources/qetgraphicsitem/terminal.cpp index 5cade6949..b232d252d 100644 --- a/sources/qetgraphicsitem/terminal.cpp +++ b/sources/qetgraphicsitem/terminal.cpp @@ -1090,7 +1090,7 @@ QList relatedPotentialTerminal ( // If terminal parent element is a folio report. if (all_diagram && terminal -> parentElement() -> linkType() & Element::AllReport) { - //The report's partner on a folio not built yet (QET_LAZY_FOLIOS) + //The report's partner on a folio not built yet (lazy folios) //is built, so that a potential is followed as far as when every //folio is; not while folios are being read or built, which //would build whole report chains diff --git a/sources/qetproject.cpp b/sources/qetproject.cpp index 3a640c798..1bf67e8a9 100644 --- a/sources/qetproject.cpp +++ b/sources/qetproject.cpp @@ -327,7 +327,7 @@ void QETProject::init() connect(&m_titleblocks_collection, &TitleBlockTemplatesCollection::changed, this, &QETProject::updateDiagramsTitleBlockTemplate); connect(&m_titleblocks_collection, &TitleBlockTemplatesCollection::aboutToRemove, this, &QETProject::removeDiagramsTitleBlockTemplate); - //What changes the saved items of folios not built (QET_LAZY_FOLIOS): + //What changes the saved items of folios not built (lazy folios): //a folio's position, number or the folio total in their texts, //the report and cross-reference settings, a symbol's definition. //A folio not built is then built to be saved (writesKeptFolios()) @@ -469,7 +469,7 @@ QETProject::ProjectState QETProject::openFile(QFile *file) //folios' parts read when they are built (stage D2); //any other, or one whose index cannot be used, is read whole m_opened_from_index = false; - if (QetContainer::isZip(content) && qEnvironmentVariableIntValue("QET_LAZY_FOLIOS") == 1) + if (QetContainer::isZip(content) && lazyFolios()) { QList entries; QString why; @@ -599,7 +599,7 @@ QList QETProject::diagrams() const /** @brief QETProject::folios @return the folios of this project, whether their items are built yet - or not (a project opened with QET_LAZY_FOLIOS); see Diagram::isLoaded() + or not (a project opened with lazy folios); see Diagram::isLoaded() */ QList QETProject::folios() const { @@ -637,7 +637,7 @@ QList QETProject::cableLabelFolios() /** @brief QETProject::builtFolios @return the folios of this project whose items are built: all of them, - but for a project opened with QET_LAZY_FOLIOS + but for a project opened with lazy folios */ QList QETProject::builtFolios() const { @@ -769,7 +769,7 @@ void QETProject::buildFolios(const QList &folios) const /** @brief QETProject::buildFoliosHolding - Build the folios not built yet (QET_LAZY_FOLIOS) holding a symbol of + Build the folios not built yet (lazy folios) holding a symbol of one of the kinds @p kinds -- if @p free_only, one which may be linked to nothing once built: none of the partners it was saved linked to is in the project. What a search for such symbols must build to find @@ -836,7 +836,7 @@ void QETProject::buildFoliosHolding(ElementData::Types kinds, bool free_only) /** @brief QETProject::buildFoliosHoldingSymbols - Build the folios not built yet (QET_LAZY_FOLIOS) holding one of the + Build the folios not built yet (lazy folios) holding one of the symbols @p symbols, found in the project database once it holds the folios not built, else in the XML kept for them: what needs those symbols while the project is being read, before the database knows @@ -886,7 +886,7 @@ void QETProject::buildFoliosHoldingSymbols(const QSet &symbols) /** @brief QETProject::symbolsOnUnbuiltFolios @return the uuids of the symbols on the folios not built yet - (QET_LAZY_FOLIOS) that building them places + (lazy folios) that building them places */ QSet QETProject::symbolsOnUnbuiltFolios() const { @@ -1884,7 +1884,7 @@ std::optional QETProject::elementAutoNumIdToLink( @brief QETProject::symbolNumbering @return the numbering of every symbol placed on the folios, read from the symbol on a folio built and, on a folio not built yet - (QET_LAZY_FOLIOS), from what building it would give: its information + (lazy folios), from what building it would give: its information in the project database's store, the scheme it would be linked to, its sequential numbers and its kind as the database has them. */ @@ -2342,7 +2342,7 @@ QDomDocument QETProject::toXml(QHash *kept_folios, QStringList //folio not built is written as , its //part and its rows copied by QetContainer::split() //(stage D4) - //With @p kept_folios, a folio not built (QET_LAZY_FOLIOS) is + //With @p kept_folios, a folio not built (lazy folios) is //written as a placeholder and its text //as read from the file goes in kept_folios, for //spliceKeptFolios() -- when building it would write that same @@ -2786,6 +2786,8 @@ QETResult QETProject::write() } else { + //Folios not built (lazy folios) written as read when that + //is what building them would write (toXml()) QHash kept_folios; QDomDocument xml_project(toXml(&kept_folios)); if (!QET::writeXmlText(spliceKeptFolios(xml_project.toString(4), kept_folios), @@ -3030,7 +3032,7 @@ bool QETProject::usesElement(const ElementsLocation &location) const @brief QETProject::folioXml @return the built folio @p folio written as saving writes it: its symbols keep the links to partners on folios not built yet - (QET_LAZY_FOLIOS), which Diagram::toXml() alone leaves out + (lazy folios), which Diagram::toXml() alone leaves out */ QDomDocument QETProject::folioXml(Diagram *folio) { @@ -3042,7 +3044,7 @@ QDomDocument QETProject::folioXml(Diagram *folio) /** @brief QETProject::buildFoliosHoldingTables - Build the folios not built yet (QET_LAZY_FOLIOS) that hold a drawn + Build the folios not built yet (lazy folios) that hold a drawn table, found from the index the project was opened from when it has one: what a list of the project's tables needs, and no more */ @@ -3063,7 +3065,7 @@ void QETProject::buildFoliosHoldingTables() /** @brief QETProject::unbuiltFolioHasTables - @return true if @p folio, not built yet (QET_LAZY_FOLIOS), has a drawn + @return true if @p folio, not built yet (lazy folios), has a drawn table: from the index it was opened from, else from its XML */ bool QETProject::unbuiltFolioHasTables(Diagram *folio) const @@ -3079,7 +3081,7 @@ bool QETProject::unbuiltFolioHasTables(Diagram *folio) const /** @brief QETProject::unbuiltFolioElements @return the locations of the symbols placed on the folios not built yet - (QET_LAZY_FOLIOS), each once, read from the project database once it + (lazy folios), each once, read from the project database once it holds those folios, else from the XML kept for them as building them reads it (Diagram::fromXml()) */ @@ -3362,11 +3364,11 @@ void QETProject::readProjectXml(QDomDocument &xml_project) m_data_base.prefillFromDocument(xml_project); } - //QET_LAZY_FOLIOS leaves the folios to be built when needed only + //Folios are left to be built when needed (lazyFolios()) only //when the database can be filled from the document: otherwise //filling it from the folios builds every one of them anyway, late //and more slowly than reading them now - m_lazy_open = qEnvironmentVariableIntValue("QET_LAZY_FOLIOS") == 1; + m_lazy_open = lazyFolios(); if (m_lazy_open) { const QString why = projectDataBase::documentRefusal(xml_project); if (!why.isEmpty()) { @@ -3489,6 +3491,15 @@ void QETProject::readFoliosFromIndex() updateDiagramsFolioData(); } +/** + @brief QETProject::lazyFolios + @see the declaration +*/ +bool QETProject::lazyFolios() +{ + return qEnvironmentVariableIntValue("QET_EAGER_FOLIOS") != 1; +} + /** @brief QETProject::readFolioPart @return the folio part @p file of the .qetz this project was opened diff --git a/sources/qetproject.h b/sources/qetproject.h index 8d8d8bded..cd8f12335 100644 --- a/sources/qetproject.h +++ b/sources/qetproject.h @@ -124,7 +124,7 @@ class QETProject : public QObject ProjectState state() const; QList diagrams() const; //The folios whether their items are built or not - //(QET_LAZY_FOLIOS); diagrams() builds them + //(lazy folios); diagrams() builds them QList folios() const; bool deferringFolios() const {return m_deferring_folios;} int unloadedFolioCount() const; @@ -162,6 +162,12 @@ class QETProject : public QObject /// process can destroy the project before the write finishes (crash). static void setBackupEnabled(bool enabled); + /// Whether a folio is built only when it is needed, not all on + /// opening (stage D6): yes, unless the + /// environment variable QET_EAGER_FOLIOS=1 asks for every folio + /// built on opening, as QElectroTech did before + static bool lazyFolios(); + /// Number of crash-recovery snapshots kept per project, written in /// turn by writeBackup(), so one bad write cannot replace the only copy static constexpr int BackupGenerations = 3; @@ -267,7 +273,7 @@ class QETProject : public QObject static QString normalizedAutoNumName(const QString &name); QVector elementsUsingElementAutoNum(const QString &title) const; /// A placed symbol's numbering, whether its folio is built or - /// not (QET_LAZY_FOLIOS): what is asked of every symbol of the + /// not (lazy folios): what is asked of every symbol of the /// project when one is numbered struct SymbolNumbering { const Element *element = nullptr; ///< null on a folio not built diff --git a/sources/scripting/qetscriptapi.cpp b/sources/scripting/qetscriptapi.cpp index 32247aed9..3f372bdfb 100644 --- a/sources/scripting/qetscriptapi.cpp +++ b/sources/scripting/qetscriptapi.cpp @@ -465,7 +465,7 @@ void QetScriptApi::setLive(QStringList *log) /** @brief QetScriptApi::unbuiltInformation @return the information of the symbol @p elementUuid on the folio - @p folioIndex when that folio is not built (QET_LAZY_FOLIOS) and the + @p folioIndex when that folio is not built (lazy folios) and the symbol's information can be read and changed through the project database alone (projectDataBase::canEditUnbuilt()); nothing otherwise, the folio to be built @@ -486,7 +486,7 @@ std::optional QetScriptApi::unbuiltInformation(int folioIndex, c /** @brief QetScriptApi::built - @return @p folio, built if it was not yet (QET_LAZY_FOLIOS), as showing + @return @p folio, built if it was not yet (lazy folios), as showing it builds it: what a call reading or changing its items needs. A folio's own data (title block, border, wire defaults) needs no build. */ diff --git a/sources/ui/terminalnumberingdialog.cpp b/sources/ui/terminalnumberingdialog.cpp index 2e180a2f5..89811db79 100644 --- a/sources/ui/terminalnumberingdialog.cpp +++ b/sources/ui/terminalnumberingdialog.cpp @@ -15,7 +15,7 @@ namespace { //The folios holding a terminal, built: a folio not built yet - //(QET_LAZY_FOLIOS) holding none is not built for this + //(lazy folios) holding none is not built for this QList terminalFolios(QETProject *project) { project->buildFoliosHolding(ElementData::Terminal, false); diff --git a/sources/undocommand/changeelementinformationcommand.h b/sources/undocommand/changeelementinformationcommand.h index 1d7d2cf70..9f6bcee24 100644 --- a/sources/undocommand/changeelementinformationcommand.h +++ b/sources/undocommand/changeelementinformationcommand.h @@ -58,7 +58,7 @@ class ChangeElementInformationCommand : public QUndoCommand /** @brief The ChangeStoredElementInformationCommand class Change the information of a symbol given by its uuid, built or not - (QET_LAZY_FOLIOS): through the symbol when it is built, through the + (lazy folios): through the symbol when it is built, through the project database when it is not (projectDataBase::canEditUnbuilt()). */ class ChangeStoredElementInformationCommand : public QUndoCommand diff --git a/sources/wiringlistexport.cpp b/sources/wiringlistexport.cpp index c69df19f8..8de281782 100644 --- a/sources/wiringlistexport.cpp +++ b/sources/wiringlistexport.cpp @@ -53,7 +53,7 @@ QDomElement WiringListExport::climbToDiagram(QDomNode node) const /** The project as the export reads it: its symbol collection and its folios - in order, each with its "order". A folio not built (QET_LAZY_FOLIOS) is + in order, each with its "order". A folio not built (lazy folios) is not built for it: its XML is copied, and what QElectroTech computes for its wires' ends -- the labels of the symbols they end on and of their partners, the terminals' names -- is filled in from the symbols built diff --git a/tests/qttest/lazy_folios_probe.cpp b/tests/qttest/lazy_folios_probe.cpp index 3389782b8..09630399c 100644 --- a/tests/qttest/lazy_folios_probe.cpp +++ b/tests/qttest/lazy_folios_probe.cpp @@ -1,5 +1,5 @@ // SPDX-License-Identifier: GPL-2.0-or-later -// Lazy folios (QET_LAZY_FOLIOS), +// Lazy folios (on by default, QET_EAGER_FOLIOS=1 for every folio built), // linked against the application's objects: a folio shown in a project // opened without building its folios draws and links as in the project // opened as usual, building only it and the folios its links and tables @@ -59,7 +59,7 @@ static void revisionEveryFolio(const QString &source, const QString &output, con const QString &path, int label_folio = -1) { QString error; - qputenv("QET_LAZY_FOLIOS", "1"); + qunsetenv("QET_EAGER_FOLIOS"); QETProject revised(source); const int unbuilt = revised.unloadedFolioCount(); QetScriptApi api(&revised); @@ -85,7 +85,7 @@ static void revisionEveryFolio(const QString &source, const QString &output, con check(built <= with_tables, "saving a new revision builds only the folios holding a table"); else check(revised.folios().at(label_folio)->isLoaded(), "a folio whose label changed is built to save"); - qunsetenv("QET_LAZY_FOLIOS"); + qputenv("QET_EAGER_FOLIOS", "1"); const QString usual_path = output + "/" + name + "-usual" + (qetz ? ".qetz" : ".qet"); { QETProject usual(source); @@ -99,7 +99,7 @@ static void revisionEveryFolio(const QString &source, const QString &output, con else { usual.setFilePath(usual_path); check(usual.write().isOk(), "revision saved when built"); } } QETProject lazy_back(lazy_path), usual_back(usual_path); - qputenv("QET_LAZY_FOLIOS", "1"); + qunsetenv("QET_EAGER_FOLIOS"); //What a save says of itself (file name, path, time) left out auto lines = [&path](QETProject &project) { QStringList list = sortedLines(project.toXml().toString(), path); @@ -364,7 +364,7 @@ int main(int argc, char **argv) qputenv("QET_LAZY_SAVE_KEPT", "1"); try { const QString output = QString::fromLocal8Bit(argv[3]); - qunsetenv("QET_LAZY_FOLIOS"); + qputenv("QET_EAGER_FOLIOS", "1"); //A file saved by this version: its folios carry their uuids, //which opening without building them needs const QString path = output + "/lazy_folios.qet"; @@ -391,9 +391,9 @@ int main(int argc, char **argv) QETProject eager(path); check(eager.state() == QETProject::Ok, "project opens"); { - qputenv("QET_LAZY_FOLIOS", "1"); + qunsetenv("QET_EAGER_FOLIOS"); QETProject lazy_strips(path); - qunsetenv("QET_LAZY_FOLIOS"); + qputenv("QET_EAGER_FOLIOS", "1"); auto strip_of = [](const QETProject &project) { for (TerminalStrip *strip : project.terminalStrip()) if (strip->name() == QStringLiteral("X99")) return strip->physicalTerminalCount(); @@ -414,7 +414,7 @@ int main(int argc, char **argv) } check(linked > 0, "the project has linked symbols"); - qputenv("QET_LAZY_FOLIOS", "1"); + qunsetenv("QET_EAGER_FOLIOS"); auto same = [&](QETProject &lazy, int i, const char *what) { Diagram *folio = lazy.folios().at(i); lazy.buildFolioToShow(folio); @@ -542,9 +542,9 @@ int main(int argc, char **argv) } { QETProject edited(path), usual(path); - qunsetenv("QET_LAZY_FOLIOS"); + qputenv("QET_EAGER_FOLIOS", "1"); QETProject usual_edited(path); - qputenv("QET_LAZY_FOLIOS", "1"); + qunsetenv("QET_EAGER_FOLIOS"); edit(edited, busiest); std::printf("edits on folio %d (%d links to other folios): %d folios not built\n", busiest + 1, most, edited.unloadedFolioCount()); @@ -624,9 +624,9 @@ int main(int argc, char **argv) //After everyday edits on a folio: as when built, others not built { QETProject edited(path), usual(path); - qunsetenv("QET_LAZY_FOLIOS"); + qputenv("QET_EAGER_FOLIOS", "1"); QETProject usual_edited(path); - qputenv("QET_LAZY_FOLIOS", "1"); + qunsetenv("QET_EAGER_FOLIOS"); edit(edited, busiest); edit(usual_edited, busiest); QFile file(path); file.open(QIODevice::ReadOnly); @@ -656,9 +656,9 @@ int main(int argc, char **argv) //links reach are built to be saved, the others not { QETProject renamed(path); - qunsetenv("QET_LAZY_FOLIOS"); + qputenv("QET_EAGER_FOLIOS", "1"); QETProject usual_renamed(path); - qputenv("QET_LAZY_FOLIOS", "1"); + qunsetenv("QET_EAGER_FOLIOS"); for (QETProject *project : {&renamed, &usual_renamed}) { Diagram *folio = project->folios().at(busiest); TitleBlockProperties block = folio->border_and_titleblock.exportTitleBlock(); @@ -673,9 +673,9 @@ int main(int argc, char **argv) //folio is built to be saved { QETProject changed(path); - qunsetenv("QET_LAZY_FOLIOS"); + qputenv("QET_EAGER_FOLIOS", "1"); QETProject usual_changed(path); - qputenv("QET_LAZY_FOLIOS", "1"); + qunsetenv("QET_EAGER_FOLIOS"); for (QETProject *project : {&changed, &usual_changed}) { DiagramContext properties = project->projectProperties(); properties.addValue(QStringLiteral("lazy_test"), QStringLiteral("1")); @@ -736,7 +736,7 @@ int main(int argc, char **argv) check(out.open(QIODevice::WriteOnly), "copy written"); out.write(document.toByteArray(4)); out.close(); - qputenv("QET_LAZY_FOLIOS", "1"); + qunsetenv("QET_EAGER_FOLIOS"); qputenv("QET_LAZY_FOLIOS_TRACE", "1"); static int built_late; built_late = 0; const QtMessageHandler previous = qInstallMessageHandler( @@ -834,9 +834,9 @@ int main(int argc, char **argv) std::printf(".qetz folio added, moved, removed: %d folios built\n", unbuilt - shuffled.unloadedFolioCount()); check(shuffled.unloadedFolioCount() == unbuilt, "adding, moving and removing a folio builds no other"); - qunsetenv("QET_LAZY_FOLIOS"); + qputenv("QET_EAGER_FOLIOS", "1"); QETProject usual_shuffled(zipped); - qputenv("QET_LAZY_FOLIOS", "1"); + qunsetenv("QET_EAGER_FOLIOS"); shuffle(usual_shuffled); check(folioRows(shuffled) == folioRows(usual_shuffled), "folios' rows after a move as when built"); } @@ -923,7 +923,7 @@ int main(int argc, char **argv) check(changed.writeContainerFile(changed_path, &error), "saved after the change"); std::printf(".qetz symbol on folio %d changed by a script: %d folios built, %d by saving\n", folio_index + 1, 0, unbuilt - changed.unloadedFolioCount()); - qunsetenv("QET_LAZY_FOLIOS"); + qputenv("QET_EAGER_FOLIOS", "1"); const QString usual_path = output + "/changed-usual.qetz"; { QETProject usual_changed(zipped); @@ -932,7 +932,7 @@ int main(int argc, char **argv) check(usual_changed.writeContainerFile(usual_path, &error), "saved after the change when built"); } QETProject changed_back(changed_path), usual_back(usual_path); - qputenv("QET_LAZY_FOLIOS", "1"); + qunsetenv("QET_EAGER_FOLIOS"); check(sortedLines(changed_back.toXml().toString(), path) == sortedLines(usual_back.toXml().toString(), path), "a symbol changed through the database saved as when built"); check(QetScriptApi(&changed_back).elementInfo(folio_index, uuid, "comment") == value, "the change saved"); @@ -989,9 +989,9 @@ int main(int argc, char **argv) std::printf(".qetz saved untouched: %d of %d folios not built before, %d after\n", unbuilt, int(folios.size()), untouched.unloadedFolioCount()); check(untouched.unloadedFolioCount() > 0, "saving an untouched .qetz does not build every folio"); - qunsetenv("QET_LAZY_FOLIOS"); + qputenv("QET_EAGER_FOLIOS", "1"); QETProject kept_back(kept_path); - qputenv("QET_LAZY_FOLIOS", "1"); + qunsetenv("QET_EAGER_FOLIOS"); check(kept_back.toXml().toString() == eager.toXml().toString(), "a .qetz saved without building holds the project"); @@ -1006,14 +1006,14 @@ int main(int argc, char **argv) //read back too: what a save and a reading do to a project is //not what is checked here const QString usual_path = output + "/edited-usual.qetz"; - qunsetenv("QET_LAZY_FOLIOS"); + qputenv("QET_EAGER_FOLIOS", "1"); { QETProject edited_usual(path); edit(edited_usual, busiest); check(edited_usual.writeContainerFile(usual_path, &error), "edits saved with every folio built"); } QETProject edited_back(edited_path), usual_back(usual_path); - qputenv("QET_LAZY_FOLIOS", "1"); + qunsetenv("QET_EAGER_FOLIOS"); check(sortedLines(edited_back.toXml().toString(), path) == sortedLines(usual_back.toXml().toString(), path), "an edited .qetz saved without building holds what the edits made"); //Each symbol's links as saved, partners on folios not built diff --git a/tests/qttest/tst_resaveunchanged.cpp b/tests/qttest/tst_resaveunchanged.cpp index c7d7bc409..9cceb2b50 100644 --- a/tests/qttest/tst_resaveunchanged.cpp +++ b/tests/qttest/tst_resaveunchanged.cpp @@ -328,7 +328,7 @@ private slots: env.insert(QStringLiteral("QET_ENABLE_SCRIPTING"), QStringLiteral("1")); env.insert(QStringLiteral("QET_CHECK_ELEMENT_INFO_STORE"), QStringLiteral("1")); if (eager) { - env.remove(QStringLiteral("QET_LAZY_FOLIOS")); //every folio built on opening + env.insert(QStringLiteral("QET_EAGER_FOLIOS"), QStringLiteral("1")); //every folio built on opening } env.insert(QStringLiteral("HOME"), home); env.insert(QStringLiteral("XDG_CONFIG_HOME"), home + QStringLiteral("/config")); From 60489c98938624cbe31064bfc056f9c2d7fdc61a Mon Sep 17 00:00:00 2001 From: ispyisail Date: Mon, 12 Oct 2026 20:43:57 +1300 Subject: [PATCH 16/26] Fix two faults the end-state gates found (stage D7) A .qetz re-saved after opening from its index saved wrong folio numbers in its database: reading each folio's header wrote its row with a number counted from the folios read so far ("1/1", "2/2" for "1/3", "2/3"), and an index-opened project keeps the rows it was saved with, so nothing wrote them again. The folio table's rows are no longer written while the project is being read (projectDataBase:: diagramInfoChanged()). Folio list tables showed it. A .qetz whose project.sqlite is not a database crashed QElectroTech: the first statement could not be prepared and SQLite was handed a null statement. A statement that cannot be prepared now binds nothing and steps to no row, and the file is refused. Gates: 342 corpus projects, written by QElectroTech 0.3 to 0.200, all open; .qet -> .qetz -> .qet equals the plain resave with every folio built; a .qetz re-saved twice holds the same bytes entry by entry but the save's own name, path and time (the first fault made the first re-save differ); hostile .qetz (truncated, garbage database, view for a table, trigger, escaping and repeated names, later format, missing folio, 2 GiB zip bomb) neither crash nor hang and none opens; an unknown extra entry still opens. tst_lazyfolios: folio numbers with totals from the index as when built (fails without the fix); tst_qetcontainer: garbage database refused (segfaults without it). ctest 96/96 both ways; save and export gates same on 342. Co-Authored-By: Claude Opus 5.5 --- sources/container/qetcontainerdb.cpp | 8 ++++++- sources/dataBase/projectdatabase.cpp | 7 +++++++ tests/qttest/lazy_folios_probe.cpp | 31 ++++++++++++++++++++++++++++ tests/qttest/tst_qetcontainer.cpp | 19 +++++++++++++++++ 4 files changed, 64 insertions(+), 1 deletion(-) diff --git a/sources/container/qetcontainerdb.cpp b/sources/container/qetcontainerdb.cpp index f455f945a..3d706e975 100644 --- a/sources/container/qetcontainerdb.cpp +++ b/sources/container/qetcontainerdb.cpp @@ -149,7 +149,11 @@ class Statement } ~Statement() { sqlite3_finalize(m_stmt); } bool ok() const { return m_stmt; } + //A statement that could not be prepared (the database is not + //one, a table is missing) binds nothing and steps to no row: + //SQLite's own calls are not all safe on a null statement void bind(const QVariantList &values) { + if (!m_stmt) return; sqlite3_reset(m_stmt); sqlite3_clear_bindings(m_stmt); for (int i = 0 ; i < values.size() ; ++i) { @@ -165,11 +169,13 @@ class Statement } } bool run(const QVariantList &values) { + if (!m_stmt) return false; bind(values); return sqlite3_step(m_stmt) == SQLITE_DONE; } - bool next() { return sqlite3_step(m_stmt) == SQLITE_ROW; } + bool next() { return m_stmt && sqlite3_step(m_stmt) == SQLITE_ROW; } QVariant value(int column) const { + if (!m_stmt) return QVariant(); switch (sqlite3_column_type(m_stmt, column)) { case SQLITE_NULL: return QVariant(); case SQLITE_INTEGER: return QVariant(sqlite3_column_int(m_stmt, column)); diff --git a/sources/dataBase/projectdatabase.cpp b/sources/dataBase/projectdatabase.cpp index 15da3aeae..d73675c59 100644 --- a/sources/dataBase/projectdatabase.cpp +++ b/sources/dataBase/projectdatabase.cpp @@ -2052,6 +2052,13 @@ void projectDataBase::removeDiagram(Diagram *diagram) void projectDataBase::diagramInfoChanged(Diagram *diagram) { m_content_changed = true; + //Not while the project is being read: a folio's header read gives + //it a folio number counted from the folios read so far ("1/1" for + //the first of three); the rows are filled once every folio is + //read, from the document or from the index a .qetz was saved with + if (m_update_blocked) { + return; + } bindDiagramInfoValues(m_update_diagram_info_query, diagram); if (!m_update_diagram_info_query.exec()) { diff --git a/tests/qttest/lazy_folios_probe.cpp b/tests/qttest/lazy_folios_probe.cpp index 09630399c..2a3349ad5 100644 --- a/tests/qttest/lazy_folios_probe.cpp +++ b/tests/qttest/lazy_folios_probe.cpp @@ -765,6 +765,37 @@ int main(int argc, char **argv) check(indexed.unloadedFolioCount() == folios.size() - strip_folios.size(), "the .qetz opens building only the folios of the terminal strip's terminals"); check(numbering(indexed) == numbering(eager), "symbols' numbering from the index as when built"); + //The folios' rows as saved, not as reading their headers one + //by one numbers them ("1/1" for the first of three, D7) + { + auto folioRows = [](QETProject &project) { + QStringList rows; + for (const QString &row : tables(project)) + if (row.startsWith("diagram")) rows << row; + return rows; + }; + check(folioRows(indexed) == folioRows(eager), "the folios' rows from the index as when built"); + //Folio numbers that count the folios ("%id/%total") + const QString totals = output + "/totals.qetz"; + qputenv("QET_EAGER_FOLIOS", "1"); + { + QETProject numbered(zipped); + QetScriptApi api(&numbered); + for (int i = 0 ; i < api.folioCount() ; ++i) + api.setFolioProperty(i, "folio", "%id/%total"); + check(numbered.writeContainerFile(totals, &error), "folio numbers with totals saved"); + } + QETProject usual_totals(totals); + qunsetenv("QET_EAGER_FOLIOS"); + QETProject lazy_totals(totals); + check(lazy_totals.openedFromIndex(), "opened from its index"); + const QStringList lazy_rows = folioRows(lazy_totals), usual_rows = folioRows(usual_totals); + if (lazy_rows != usual_rows) + for (int i = 0 ; i < lazy_rows.size() && i < 3 ; ++i) + std::printf(" lazily %s\n built %s\n", qPrintable(lazy_rows.at(i).left(140)), + qPrintable(usual_rows.value(i).left(140))); + check(lazy_rows == usual_rows, "folio numbers with totals from the index as when built"); + } for (int i = 0; i < folios.size(); ++i) same(indexed, i, "shown from the .qetz"); check(indexed.toXml().toString() == eager.toXml().toString(), "the .qetz saves as when built"); diff --git a/tests/qttest/tst_qetcontainer.cpp b/tests/qttest/tst_qetcontainer.cpp index 1bccb0ce0..592d011e6 100644 --- a/tests/qttest/tst_qetcontainer.cpp +++ b/tests/qttest/tst_qetcontainer.cpp @@ -319,6 +319,25 @@ private slots: // A crafted project.sqlite whose "element" is a view, not a table, is // refused: nothing in a file's schema is trusted. + void garbageDatabaseRefused() + { + //Bytes that are no database: refused, not read (crashed reading + //the first statement until stage D7) + QDomDocument original; + QList entries = splitFile(QStringLiteral(QET_EXAMPLES_DIR "/perceuse.qet"), &original); + bool replaced = false; + for (QetZip::Entry &entry : entries) + if (entry.name == QLatin1String("project.sqlite")) { + entry.data = QByteArray("not a database").repeated(100); + replaced = true; + } + QVERIFY(replaced); + QDomDocument back; + QString error; + QVERIFY(!QetContainer::join(entries, &back, &error)); + QVERIFY(!error.isEmpty()); + } + void viewInsteadOfTableRefused() { QDomDocument original; From 9d3b9ae78981ba22aa96bd02c416b5cb32be24e0 Mon Sep 17 00:00:00 2001 From: ispyisail Date: Mon, 12 Oct 2026 21:24:10 +1300 Subject: [PATCH 17/26] Translate the .qetz strings into French MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The strings the zipped project format added (Save a copy for an older QElectroTech, the notice when a .qet is saved as .qetz beside it, the .qetz file filters) and the undo text of a symbol's information changed through the project database. Only French, by hand, in the wording the file already uses ("Projets QElectroTech (*.qet)", "Modifier les informations de l'élément", "Fichier"); the other languages are left for their translators. lrelease: 0 unfinished; lupdate over sources keeps every entry finished. Co-Authored-By: Claude Opus 5.5 --- lang/qet_fr.ts | 48 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/lang/qet_fr.ts b/lang/qet_fr.ts index 4f4b125bf..440ebec99 100644 --- a/lang/qet_fr.ts +++ b/lang/qet_fr.ts @@ -10898,6 +10898,10 @@ Donnez-leur une autre numérotation d'abord. Management Management + + Project QElectroTech (*.qet *.qetz) + Projets QElectroTech (*.qet *.qetz) + ProjectDBModel @@ -11409,6 +11413,33 @@ Voulez-vous enregistrer les modifications ? window title for a project-less ProjectView Projet + + Zipped project QElectroTech (*.qetz) + filetypes allowed when saving a project file + Projet QElectroTech compressé (*.qetz) + + + Save a copy for an older QElectroTech + dialog title + Enregistrer une copie pour une version antérieure de QElectroTech + + + This project was opened from %1. It is saved as %2, beside it, in the zipped project format; %1 stays as it is. + +To write a file for a version of QElectroTech that cannot open %2, use File > Save a copy for an older QElectroTech. + Ce projet a été ouvert depuis %1. Il est enregistré sous %2, à côté, au format de projet compressé ; %1 reste tel quel. + +Pour écrire un fichier lisible par une version de QElectroTech qui ne peut pas ouvrir %2, utilisez Fichier > Enregistrer une copie pour une version antérieure de QElectroTech. + + + Save as a zipped project + message box title + Enregistrement au format de projet compressé + + + %1 already exists. Replace it? + %1 existe déjà. Le remplacer ? + PropertiesEditorDockWidget @@ -13692,6 +13723,19 @@ Déponter et/ou supprimer les niveaux des bornes concerné afin de pouvoir les s message box title Erreur + + Save a copy for an older QElectroTech + Enregistrer une copie pour une version antérieure de QElectroTech + + + Writes a copy of the current project as a .qet file, which versions of QElectroTech that cannot open a .qetz open + status bar tip + Enregistre une copie du projet courant au format .qet, que savent ouvrir les versions de QElectroTech qui ne peuvent pas ouvrir un .qetz + + + QElectroTech project(*.qet *.qetz);;XML Files (*.xml);;All files (*) + Projets QElectroTech (*.qet *.qetz);;Fichiers XML (*.xml);;Tous les fichiers (*) + QETElementEditor @@ -17242,6 +17286,10 @@ Pour l'activer : Configurer QElectroTech > Général > Projets, ou d Ungroup a set of terminals Dégrouper un ensemble de bornes + + Change the information of the element + Modifier les informations de l'élément + QTextOrientationWidget From 975ec57c74c027aed1563a4d8c15b438ce96abbb Mon Sep 17 00:00:00 2001 From: ispyisail Date: Mon, 12 Oct 2026 21:46:37 +1300 Subject: [PATCH 18/26] Write a .qetz's database without keys or indexes (stage D3b) The database a .qetz holds is read whole: its tables are now written without their primary keys and indexes, in the order of their rows, and the file compacted; the reader indexes what it looks folios up by once it has read the file. Keys and indexes on text uuids were half of the database once zipped. A file written before reads as before. f2099 (136 folios) 5.14 MB -> 3.60 MB, industrial.qet 1.03 -> 0.71 MB; opening and showing a folio take as long as before (2.5 s, 215 ms). Gates on 342 corpus projects: saves, exports, round trip, re-save byte-stable, all open; hostile .qetz all refused; ctest 96/96 both ways. Co-Authored-By: Claude Opus 5.5 --- sources/container/qetcontainerdb.cpp | 52 ++++++++++++++++++++++++++-- 1 file changed, 50 insertions(+), 2 deletions(-) diff --git a/sources/container/qetcontainerdb.cpp b/sources/container/qetcontainerdb.cpp index 3d706e975..340a22b47 100644 --- a/sources/container/qetcontainerdb.cpp +++ b/sources/container/qetcontainerdb.cpp @@ -468,6 +468,36 @@ QStringList keys(const QList &items, const QString &prefix) namespace { //@p database opened read only in @p db, its tables checked to be plain //tables: a file is untrusted + //Every table of @p db written again without its keys and indexes, in + //the order of its rows, and the file compacted: the file a .qetz holds + //is read whole, its lookups indexed once read (openDatabase(), + //projectDataBase::fillFromIndex()), and keys and indexes of text uuids + //were half of it (stage D3b) +bool plainTables(sqlite3 *db) +{ + QStringList tables; + { + sqlite3_stmt *list = nullptr; + sqlite3_prepare_v2(db, "SELECT name FROM main.sqlite_master WHERE type = 'table' " + "AND name NOT LIKE 'sqlite_%' ORDER BY name", -1, &list, nullptr); + while (list && sqlite3_step(list) == SQLITE_ROW) + tables << QString::fromUtf8(reinterpret_cast(sqlite3_column_text(list, 0))); + sqlite3_finalize(list); + } + auto run = [db](const QString &sql) { + return sqlite3_exec(db, sql.toUtf8().constData(), nullptr, nullptr, nullptr) == SQLITE_OK; + }; + bool ok = run(QStringLiteral("BEGIN")); + for (const QString &table : std::as_const(tables)) { + const QString plain = table + QStringLiteral("__plain"); + ok = ok && run(QStringLiteral("CREATE TABLE \"%1\" AS SELECT * FROM \"%2\" ORDER BY rowid").arg(plain, table)) + && run(QStringLiteral("DROP TABLE \"%1\"").arg(table)) + && run(QStringLiteral("ALTER TABLE \"%1\" RENAME TO \"%2\"").arg(plain, table)); + } + ok = ok && run(QStringLiteral("COMMIT")); + return ok && run(QStringLiteral("VACUUM")); +} + bool openDatabase(Database &db, const QByteArray &database, QString *error) { if (sqlite3_open(":memory:", &db.m_db) != SQLITE_OK) { @@ -477,15 +507,15 @@ bool openDatabase(Database &db, const QByteArray &database, QString *error) unsigned char *copy = static_cast(sqlite3_malloc64(sqlite3_uint64(database.size()))); if (!copy) { setError(error, QStringLiteral("out of memory")); return false; } memcpy(copy, database.constData(), size_t(database.size())); + //A copy of its own, writable: its lookups are indexed below if (sqlite3_deserialize(db.m_db, "main", copy, database.size(), database.size(), - SQLITE_DESERIALIZE_FREEONCLOSE | SQLITE_DESERIALIZE_READONLY) != SQLITE_OK) { + SQLITE_DESERIALIZE_FREEONCLOSE | SQLITE_DESERIALIZE_RESIZEABLE) != SQLITE_OK) { setError(error, QStringLiteral("the project database cannot be read")); return false; } //A file is untrusted: nothing in its schema may run, and every name //read must be a plain table. db.exec(QStringLiteral("PRAGMA trusted_schema = OFF")); - db.exec(QStringLiteral("PRAGMA query_only = ON")); //Format 3 names the document's tables doc_, format 2 not { Statement doc(db, QStringLiteral("SELECT count(*) FROM sqlite_master WHERE name = 'doc_meta'")); @@ -501,6 +531,17 @@ bool openDatabase(Database &db, const QByteArray &database, QString *error) return false; } } + //What the folios are read by: a file holds no index (plainTables()) + for (const char *index : {"folio(id)", "folio_property(folio_id)", "element(key)", "element(folio_id)", + "element_info(element_key)", "link(element_key)", "conductor(key)", "meta(key)"}) { + const QString on = QString::fromLatin1(index); + const QString name = QStringLiteral("qetz_read_") + QString(on).replace(QLatin1Char('('), QLatin1Char('_')).remove(QLatin1Char(')')); + if (!db.exec(QStringLiteral("CREATE INDEX IF NOT EXISTS %1 ON %2").arg(name, on))) { + setError(error, QStringLiteral("the project database cannot be read: %1").arg(db.error())); + return false; + } + } + db.exec(QStringLiteral("PRAGMA query_only = ON")); return true; } @@ -866,6 +907,13 @@ QByteArray QetContainerDb::extract(QDomElement project, QString *error, const QB setError(error, QStringLiteral("cannot fill the project database: %1").arg(db.error())); return {}; } + if (!kept_from.isEmpty()) { + sqlite3_exec(db.m_db, "DETACH DATABASE old", nullptr, nullptr, nullptr); + } + if (!plainTables(db.m_db)) { + setError(error, QStringLiteral("cannot write the project database: %1").arg(db.error())); + return {}; + } sqlite3_int64 size = 0; unsigned char *bytes = sqlite3_serialize(db.m_db, "main", &size, 0); From a6d81a21a865281950f1b66193424711c0b35e2b Mon Sep 17 00:00:00 2001 From: ispyisail Date: Mon, 12 Oct 2026 21:46:37 +1300 Subject: [PATCH 19/26] Build the folios before reading the drawn items' view (stage D6) A query of drawing_item_view, which is made of the shape, text and picture tables, built no folio, and with folios built only when needed (the default since D6) it found nothing: the drawn items are written to the database by built folios. A query naming the view builds them, as one naming the tables does. Found by the qet MCP server's suite (four drawn-item tests failed lazily, passed with every folio built). tst_lazyfolios: the view's rows from a .qetz as when built (5 = 5). Co-Authored-By: Claude Opus 5.5 --- sources/dataBase/projectdatabase.cpp | 7 ++++--- tests/qttest/lazy_folios_probe.cpp | 13 +++++++++++++ 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/sources/dataBase/projectdatabase.cpp b/sources/dataBase/projectdatabase.cpp index d73675c59..c7c84c4a8 100644 --- a/sources/dataBase/projectdatabase.cpp +++ b/sources/dataBase/projectdatabase.cpp @@ -1660,10 +1660,11 @@ QSqlQuery projectDataBase::newQuery(const QString &query, QString *error) { //storeConductorProperties(). //The drawing tables hold what only built items know (a text's //laid-out size, a picture's pixels): with folios not built yet - //(lazy folios) a read of them builds those first. Every other - //table was filled from the document. + //(lazy folios) a read of them builds those first, and so does a + //read of the view made of them. Every other table was filled from + //the document. static const QRegularExpression drawing_tables( - QStringLiteral("\\b(shape|independent_text|image)\\b"), + QStringLiteral("\\b(shape|independent_text|image|drawing_item_view)\\b"), QRegularExpression::CaseInsensitiveOption); if (m_project && m_project->unloadedFolioCount() && drawing_tables.match(query).hasMatch()) { m_project->loadFolios(); diff --git a/tests/qttest/lazy_folios_probe.cpp b/tests/qttest/lazy_folios_probe.cpp index 2a3349ad5..2bc143b93 100644 --- a/tests/qttest/lazy_folios_probe.cpp +++ b/tests/qttest/lazy_folios_probe.cpp @@ -765,6 +765,19 @@ int main(int argc, char **argv) check(indexed.unloadedFolioCount() == folios.size() - strip_folios.size(), "the .qetz opens building only the folios of the terminal strip's terminals"); check(numbering(indexed) == numbering(eager), "symbols' numbering from the index as when built"); + //The drawn items, through the view made of their tables: the + //folios built for it, as for the tables (D6: none were) + { + QETProject drawn(zipped); + auto count = [](QETProject &project) { + QSqlQuery query = project.dataBase()->newQuery(QStringLiteral("SELECT COUNT(*) FROM drawing_item_view")); + return query.exec() && query.next() ? query.value(0).toInt() : -1; + }; + const int lazily = count(drawn), built = count(eager); + std::printf(".qetz drawn items through their view: %d lazily, %d when built\n", lazily, built); + check(built > 0, "the project has drawn items"); + check(lazily == built, "the drawn items' view as when built"); + } //The folios' rows as saved, not as reading their headers one //by one numbers them ("1/1" for the first of three, D7) { From b8390c853c73858533ddec96c44641c222da5eb4 Mon Sep 17 00:00:00 2001 From: ispyisail Date: Mon, 12 Oct 2026 21:46:37 +1300 Subject: [PATCH 20/26] Read and write .qetz in the qet MCP server Every tool now takes a .qetz as it takes a .qet. The tools that read a project's XML read the .qet the configured QElectroTech saves from the .qetz, made once per version of the file (path, size, modification time); without a QElectroTech, they say to set QET_BINARY. The tools that launch QElectroTech hand it the .qetz, and write one when the output path ends in .qetz. qet_scan counts .qetz files too. The server's description and README say so. Reading through QElectroTech instead of the file's own layout keeps the server right while the zipped format still changes; the first read of a big .qetz takes as long as QElectroTech needs to save it. test_qet_mcp.py: the read tools answer the same for a .qetz and its .qet, qet_diff finds nothing between them, qet_scan counts both, a .qetz is converted once, qet_query reads it, qet_edit writes one (four fail when _root() parses the file itself); without QElectroTech the answer names QET_BINARY. Whole suite: 430 tests pass with QET_ENABLE_SCRIPTING=1. Co-Authored-By: Claude Opus 5.5 --- misc/qet-mcp/README.md | 7 +++ misc/qet-mcp/qet_mcp.py | 62 ++++++++++++++++++++++--- misc/qet-mcp/test_qet_mcp.py | 89 ++++++++++++++++++++++++++++++++++++ 3 files changed, 151 insertions(+), 7 deletions(-) diff --git a/misc/qet-mcp/README.md b/misc/qet-mcp/README.md index 98586dc52..1de4752d1 100644 --- a/misc/qet-mcp/README.md +++ b/misc/qet-mcp/README.md @@ -25,6 +25,13 @@ here read the model. ## Tools +Every tool takes a `.qet` or a `.qetz` (the same project, zipped). The +tools that read a project's XML read the `.qet` QElectroTech saves from a +`.qetz`, made once per version of the file with the configured +QElectroTech (`QET_BINARY`); the first read of a big `.qetz` takes as long +as QElectroTech needs to save it. The tools that launch QElectroTech hand +it the `.qetz` itself, and write one when the output path ends in `.qetz`. + | Tool | What it answers | |---|---| | `qet_project_info` | title, format version, folios with their uuids, element and conductor counts | diff --git a/misc/qet-mcp/qet_mcp.py b/misc/qet-mcp/qet_mcp.py index f1157ea3c..58177e01a 100755 --- a/misc/qet-mcp/qet_mcp.py +++ b/misc/qet-mcp/qet_mcp.py @@ -97,11 +97,56 @@ def _root(path: str) -> ET.Element: if not p.is_file(): raise ValueError(f"no such file: {p}") try: - return ET.parse(p).getroot() + return ET.parse(_as_qet(p)).getroot() except ET.ParseError as exc: raise ValueError(f"{p.name} is not parseable XML: {exc}") from exc +# -------------------------------------------------------------------------- +# zipped projects (.qetz) +# -------------------------------------------------------------------------- + +# A .qetz is QElectroTech's zipped project: a database and one file per +# folio. The tools that read a project's XML read the .qet QElectroTech +# saves from it, so they read what QElectroTech reads, whatever the zipped +# format holds; one copy per version of the file (path, size, time). +_QETZ_COPIES: dict = {} + + +def _is_qetz(path) -> bool: + return str(path).lower().endswith(".qetz") + + +def _as_qet(path) -> Path: + """The file the XML tools read for `path`: itself, or for a .qetz the + .qet QElectroTech saves from it (made with the configured binary).""" + p = Path(path).expanduser() + if not _is_qetz(p): + return p + if not p.is_file(): + raise ValueError(f"no such file: {p}") + st = p.stat() + key = (str(p.resolve()), st.st_size, st.st_mtime_ns) + cached = _QETZ_COPIES.get(key) + if cached is not None and cached.is_file(): + return cached + binary = resolve_binary() + if not binary: + raise ValueError(f"{p.name} is a zipped project (.qetz): reading it needs " + "QElectroTech; set QET_BINARY") + folder = Path(tempfile.mkdtemp(prefix="qet-mcp-qetz-")) + out = folder / (p.stem + ".qet") + # Saving as .qet builds every folio: allow a big project its time + result = _run_qet(str(binary), ["--resave", str(p), str(out)], timeout=900) + if not out.is_file() or out.stat().st_size == 0: + shutil.rmtree(folder, ignore_errors=True) + detail = (result.get("stderr") or result.get("stdout") or "").strip()[-400:] + raise ValueError(f"QElectroTech could not read {p.name}" + + (f": {detail}" if detail else "")) + _QETZ_COPIES[key] = out + return out + + def _element_info(el: ET.Element) -> dict: """The bag, as a plain dict.""" out = {} @@ -796,7 +841,7 @@ def _num(v) -> float: def tool_scan(directory: str, tag: str = "conductor", attribute: str = "cable", recursive: bool = True) -> dict: - """Sweep every .qet in a directory, counting how many carry a + """Sweep every .qet and .qetz in a directory, counting how many carry a non-empty `attribute`. This is the corpus question: "3190 conductors, 0 cable values across @@ -805,13 +850,14 @@ def tool_scan(directory: str, tag: str = "conductor", d = Path(directory).expanduser() if not d.is_dir(): raise ValueError(f"not a directory: {d}") - files = sorted(d.rglob("*.qet") if recursive else d.glob("*.qet")) + files = sorted([f for pattern in ("*.qet", "*.qetz") + for f in (d.rglob(pattern) if recursive else d.glob(pattern))]) total = non_empty = 0 per_file, values, unreadable = [], {}, [] for f in files: try: - root = ET.parse(f).getroot() - except ET.ParseError as exc: + root = ET.parse(_as_qet(f)).getroot() + except (ET.ParseError, ValueError) as exc: unreadable.append({"file": f.name, "error": str(exc)}) continue n = ne = 0 @@ -3919,8 +3965,10 @@ def scripts_dir() -> Path: SERVER_INSTRUCTIONS = ( "This server works with QElectroTech, a free editor for electrical " - "diagrams. A project (.qet) holds folios (sheets) of symbols " - "(elements) joined by wires (conductors). Folio indexes count from 0.\n" + "diagrams. A project (.qet, or .qetz: the same project zipped) holds " + "folios (sheets) of symbols (elements) joined by wires (conductors). " + "Every tool takes either; a .qetz is read through QElectroTech, so the " + "first read of a big one takes a while. Folio indexes count from 0.\n" "Start with qet_about: where QElectroTech keeps things, what is " "switched on, the stored scripts, and house_style -- this " "installation's own drawing conventions if the user set any. Follow " diff --git a/misc/qet-mcp/test_qet_mcp.py b/misc/qet-mcp/test_qet_mcp.py index ea85bf798..30492aa4b 100644 --- a/misc/qet-mcp/test_qet_mcp.py +++ b/misc/qet-mcp/test_qet_mcp.py @@ -6476,6 +6476,95 @@ class CorpusIntegration(unittest.TestCase): sb.close() +class ZippedProjectWithoutBinary(unittest.TestCase): + def test_reading_a_qetz_without_qelectrotech_says_what_to_set(self): + """A .qetz is read through QElectroTech (m._as_qet()): with none + configured, the answer names QET_BINARY rather than failing to + parse a zip as XML.""" + with tempfile.TemporaryDirectory() as tmp: + z = Path(tmp) / "p.qetz" + z.write_bytes(b"PK\x03\x04 not read") + with mock.patch.object(m, "resolve_binary", return_value=None): + with self.assertRaises(ValueError) as caught: + m.tool_project_info(str(z)) + self.assertIn("QET_BINARY", str(caught.exception)) + + +@needs_examples +class ZippedProjects(unittest.TestCase): + """Every tool reads and writes QElectroTech's zipped project (.qetz) as + it does the .qet: the XML tools read the .qet QElectroTech saves from + it, the tools that launch QElectroTech hand it the .qetz.""" + + @classmethod + def setUpClass(cls): + cls.tmp = tempfile.TemporaryDirectory() + root = Path(cls.tmp.name) + cls.env = mock.patch.dict(os.environ, {"QET_BINARY": BINARY, "QET_ENABLE_SCRIPTING": "1", + "QET_MCP_SCRIPTS_DIR": str(root / "scripts"), + "QET_MCP_INFO_FILE": str(root / "none.json")}) + cls.env.start() + cls.qet, cls.qetz = root / "p.qet", root / "p.qetz" + source = str(Path(EXAMPLES) / "industrial.qet") + for out in (cls.qet, cls.qetz): + r = m._run_qet(BINARY, ["--resave", source, str(out)], timeout=600) + assert out.is_file(), r + assert cls.qetz.read_bytes()[:2] == b"PK" + + @classmethod + def tearDownClass(cls): + cls.env.stop() + cls.tmp.cleanup() + + def test_read_tools_answer_as_for_the_qet(self): + def strip(d): + return {k: v for k, v in d.items() if k not in ("path", "file")} + for tool in (m.tool_project_info, m.tool_elements, m.tool_conductors, m.tool_items): + self.assertEqual(strip(tool(str(self.qetz))), strip(tool(str(self.qet))), tool.__name__) + + def test_diff_finds_nothing_between_the_qet_and_the_qetz(self): + d = m.tool_diff(str(self.qet), str(self.qetz)) + self.assertTrue(d.get("identical") or not any( + v.get("added") or v.get("removed") or v.get("changed") + for v in d.values() if isinstance(v, dict)), d) + + def test_scan_counts_a_qetz(self): + folder = Path(self.tmp.name) / "scan" + folder.mkdir(exist_ok=True) + for f in (self.qet, self.qetz): + shutil.copy2(f, folder / f.name) + scan = m.tool_scan(str(folder), "conductor", "cable", recursive=False) + self.assertEqual(scan["files"], 2) + self.assertEqual(scan["unreadable"], []) + self.assertEqual(scan["per_file"][0]["conductor"], scan["per_file"][1]["conductor"]) + + def test_a_qetz_is_converted_once_while_it_does_not_change(self): + m._QETZ_COPIES.clear() + real = m._run_qet + with mock.patch.object(m, "_run_qet", side_effect=real) as run: + m.tool_project_info(str(self.qetz)) + m.tool_elements(str(self.qetz)) + self.assertEqual(run.call_count, 1) + + def test_query_reads_the_qetz_itself(self): + sql = "SELECT COUNT(*) AS n FROM element" + self.assertEqual(m.tool_query(BINARY, str(self.qetz), sql)["rows"], + m.tool_query(BINARY, str(self.qet), sql)["rows"]) + + def test_edit_writes_a_qetz(self): + first = m.tool_elements(str(self.qetz))["elements"][0] + out = Path(self.tmp.name) / "edited.qetz" + r = m.tool_edit(BINARY, str(self.qetz), + [{"op": "rotate_element", "folio": first["folio"] - 1, + "element": first["uuid"], "angle": 90}], + str(out), ELEMENTS or None) + self.assertTrue(r.get("ok"), r) + self.assertEqual(out.read_bytes()[:2], b"PK") + # The rotation is what the diff of the two .qetz finds + diff = json.dumps(m.tool_diff(str(self.qetz), str(out))) + self.assertIn(m._uuid_key(first["uuid"]) if hasattr(m, "_uuid_key") else first["uuid"], diff.lower()) + + if __name__ == "__main__": print(f"binary : {BINARY or '(none: integration tests skipped)'}") print(f"elements : {ELEMENTS or '(none)'}") From 798010cc833c846e9627869f46e064060864ffb5 Mon Sep 17 00:00:00 2001 From: ispyisail Date: Mon, 12 Oct 2026 23:26:39 +1300 Subject: [PATCH 21/26] Change a wire's properties by uuid without building its folios (stage D5) Scripts get conductorPropertyByUuid() and setConductorPropertyByUuid(), the wire named by its uuid (conductorUuids()) rather than by a symbol's terminal. A change goes to every wire at the wire's potential, as setConductorProperty() does. When those wires are all on folios not built, none is numbered by a formula, and what changes in each is what a .qetz database holds in its own columns (number, function, voltage, colour and section of the wire, cable, bus), it goes through the project database: the store and the conductor rows, undoable by uuid. The potential is followed in the database (projectDataBase::potentialWires(): through wire ends, folio reports and terminal blocks, as Conductor::relatedPotentialConductors() follows it; a terminal block that isolates its terminals answers nothing). Building a wire takes its change (Conductor::fromXml()); a .qetz save writes it in the database, the folio file copied; a folio written from its XML gets it there; a .qet builds. Anything else builds the folio, as before. tst_lazyfolios: every wire's potential from the database = as followed, 1 342 of 1 342 (industrial, eager and from a .qetz); a cable set by uuid on a 14-wire potential builds no folio, reaches every wire, saves as when built, undoes, shows once built (fails when building ignores the change, and when the .qetz writer does). The test's run limit is raised (500 s -> 1 500 s, ctest 1 800 s): under load it ran past 500 s. ctest 96/96 both ways. Co-Authored-By: Claude Opus 5.5 --- sources/container/qetcontainerdb.cpp | 17 +++ sources/dataBase/projectdatabase.cpp | 173 +++++++++++++++++++++++++ sources/dataBase/projectdatabase.h | 9 ++ sources/qetgraphicsitem/conductor.cpp | 5 + sources/qetproject.cpp | 21 +++ sources/scripting/qetscriptapi.cpp | 178 ++++++++++++++++++++++++++ sources/scripting/qetscriptapi.h | 10 ++ tests/qttest/CMakeLists.txt | 2 +- tests/qttest/lazy_folios_probe.cpp | 92 +++++++++++++ tests/qttest/run_lazy_folios_probe.py | 2 +- 10 files changed, 507 insertions(+), 2 deletions(-) diff --git a/sources/container/qetcontainerdb.cpp b/sources/container/qetcontainerdb.cpp index 340a22b47..85349e9fc 100644 --- a/sources/container/qetcontainerdb.cpp +++ b/sources/container/qetcontainerdb.cpp @@ -818,6 +818,23 @@ QByteArray QetContainerDb::extract(QDomElement project, QString *error, const QB ? QVariant(i.attribute(QStringLiteral("name"))) : QVariant(), value, attributesJson(i, QStringLiteral("name"), true)}); } + } + //Its wires whose properties changed since: their columns + for (const QDomElement &edited : children(d, QStringLiteral("qetz-edited-conductor"))) { + QStringList set; + QVariantList values; + for (const QString &column : ConductorData) { + if (column == QLatin1String("uuid") || column.startsWith(QLatin1String("element")) + || column.startsWith(QLatin1String("terminal")) || column == QLatin1String("freezeLabel") + || !edited.hasAttribute(column)) + continue; + set << QStringLiteral("\"%1\" = ?").arg(column); + values << edited.attribute(column); + } + values << id << edited.attribute(QStringLiteral("uuid")); + Statement update(db, QStringLiteral("UPDATE conductor SET %1 WHERE folio_id = ? AND uuid = ?") + .arg(set.join(QStringLiteral(", ")))); + ok = ok && !set.isEmpty() && update.run(values) && sqlite3_changes(db.m_db) == 1; } if (!ok) { setError(error, QStringLiteral("folio %1 kept as it was is not in its database").arg(id)); diff --git a/sources/dataBase/projectdatabase.cpp b/sources/dataBase/projectdatabase.cpp index c7c84c4a8..32085c9b3 100644 --- a/sources/dataBase/projectdatabase.cpp +++ b/sources/dataBase/projectdatabase.cpp @@ -1317,6 +1317,77 @@ QString projectDataBase::terminalName(const QUuid &terminal, const QUuid &elemen return query.exec() && query.next() ? query.value(0).toString() : QString(); } +/** + @brief projectDataBase::potentialWires + @return the wires at the potential of @p wire, @p wire included, read + from the conductor, element and link tables as they are (see + elementTypesOnFolios()), followed as Conductor::relatedPotentialConductors() + follows them over every folio: through a wire's ends, from a folio + report to the terminals of the first symbol it is linked to, and from a + terminal block's terminal to its others. Nothing when a terminal block + on the way isolates its terminals (their groups are in its definition, + not in the database) or @p wire has no row. +*/ +std::optional> projectDataBase::potentialWires(const QUuid &wire) const +{ + using Node = std::pair; //element uuid, terminal uuid + QHash> ends; + QHash wires_at; + QHash> nodes_of; + { + QSqlQuery query(m_data_base); + query.exec(QStringLiteral("SELECT uuid, terminal1_element_uuid, terminal1_uuid, " + "terminal2_element_uuid, terminal2_uuid FROM conductor")); + while (query.next()) { + const Node a{query.value(1).toString(), query.value(2).toString()}; + const Node b{query.value(3).toString(), query.value(4).toString()}; + const QString uuid = query.value(0).toString(); + ends.insert(uuid, {a, b}); + for (const Node &n : {a, b}) { + wires_at[n] << uuid; + if (!nodes_of.value(n.first).contains(n)) nodes_of[n.first] << n; + } + } + } + const auto start = ends.constFind(wire.toString()); + if (start == ends.constEnd()) { + return std::nullopt; + } + QHash kinds; + { + QSqlQuery query(m_data_base); + query.exec(QStringLiteral("SELECT uuid, type FROM element")); + while (query.next()) + kinds.insert(query.value(0).toString(), ElementData::typeFromString(query.value(1).toString())); + } + QSet wires; + QSet seen; + QList todo{start->first, start->second}; + while (!todo.isEmpty()) { + const Node node = todo.takeLast(); + if (seen.contains(node)) continue; + seen.insert(node); + for (const QString &w : wires_at.value(node)) { + if (wires.contains(QUuid(w))) continue; + wires.insert(QUuid(w)); + const auto e = ends.value(w); + todo << e.first << e.second; + } + const ElementData::Type kind = kinds.value(node.first, ElementData::Simple); + if (kind & ElementData::AllReport) { + const QList partners = linkedSymbols(QUuid(node.first)); + if (!partners.isEmpty()) + todo << nodes_of.value(partners.first().toString()); + } else if (kind & ElementData::Terminal) { + if (QET::infoFlagIsTrue(m_element_information.value(QUuid(node.first)) + .value(QStringLiteral("potential_isolating")).toString())) + return std::nullopt; + todo << nodes_of.value(node.first); + } + } + return wires; +} + /** @brief projectDataBase::linkedSymbols @return the symbols @p element is linked to, read from the link table @@ -1467,9 +1538,111 @@ QSet projectDataBase::foliosWithUnbuiltEdits() const { QSet folios; for (const auto &edit : m_unbuilt_edits) folios.insert(edit.folio); + for (const auto &edit : m_unbuilt_wire_edits) folios.insert(edit.folio); return folios; } +/** + @brief projectDataBase::unbuiltWire + @return true if the wire @p wire is on a folio not built (lazy folios): + it has a row and its properties are stored, and no wire built holds its + uuid. @p folio is set to its folio. +*/ +bool projectDataBase::unbuiltWire(const QUuid &wire, QUuid *folio) const +{ + if (placedConductorCount(wire) != 0 || !hasConductorProperties(wire)) { + return false; + } + QSqlQuery query(m_data_base); + query.prepare(QStringLiteral("SELECT diagram_uuid FROM conductor WHERE uuid = :uuid")); + query.bindValue(QStringLiteral(":uuid"), wire.toString()); + if (!query.exec() || !query.next()) { + return false; + } + if (folio) *folio = QUuid(query.value(0).toString()); + return !query.next(); +} + +/** + @brief projectDataBase::editUnbuiltConductorProperties + Give the wire @p wire, on a folio not built (unbuiltWire()), the + properties @p properties as Conductor::setProperties() would: the store, + its rows and its conductor row's text. Building it takes them + (takeUnbuiltWireEdit()); a .qetz saves them in its database (QETProject:: + toXml()), anything else builds its folio to save. +*/ +void projectDataBase::editUnbuiltConductorProperties(const QUuid &wire, const ConductorProperties &properties) +{ + QUuid folio; + if (!unbuiltWire(wire, &folio)) { + return; + } + m_unbuilt_wire_edits.insert(wire, {properties, folio}); + if (!storeConductorProperties(wire, properties)) { + return; + } + m_content_changed = true; + if (!m_update_blocked) { + m_update_conductor_query.bindValue(QStringLiteral(":uuid"), wire.toString()); + m_update_conductor_query.bindValue(QStringLiteral(":text"), properties.text); + if (!m_update_conductor_query.exec()) { + qDebug() << "projectDataBase::editUnbuiltConductorProperties update error : " << m_update_conductor_query.lastError(); + } + } + emit dataBaseUpdated(); +} + +/** + @brief projectDataBase::takeUnbuiltWireEdit + @return the properties @p wire was given while its folio was not built, + forgotten here: the wire being built takes them +*/ +std::optional projectDataBase::takeUnbuiltWireEdit(const QUuid &wire) +{ + const auto edit = m_unbuilt_wire_edits.constFind(wire); + if (edit == m_unbuilt_wire_edits.constEnd()) { + return std::nullopt; + } + const ConductorProperties properties = edit->properties; + m_unbuilt_wire_edits.erase(edit); + return properties; +} + +/** + @brief projectDataBase::unbuiltWireEditsOnFolio + @return the properties given to the wires of the folio @p folio while it + was not built, by wire +*/ +QHash projectDataBase::unbuiltWireEditsOnFolio(const QUuid &folio) const +{ + QHash edits; + for (auto it = m_unbuilt_wire_edits.cbegin() ; it != m_unbuilt_wire_edits.cend() ; ++it) + if (it->folio == folio) edits.insert(it.key(), it->properties); + return edits; +} + +/** + @brief projectDataBase::setPlacedConductorProperties + Give the wire @p wire its properties @p properties: through the wire when + it is built, through the store when it is not (unbuiltWire()). What + undoing and redoing a change of wires not built does, the wires having + perhaps been built since. + @return false if neither can +*/ +bool projectDataBase::setPlacedConductorProperties(const QUuid &wire, const ConductorProperties &properties) +{ + const QList> placed = m_placed_conductors.values(wire); + if (placed.size() == 1 && placed.first()) { + placed.first()->setProperties(properties); + return true; + } + if (placed.isEmpty() && unbuiltWire(wire)) { + editUnbuiltConductorProperties(wire, properties); + return true; + } + return false; +} + /** @brief projectDataBase::unbuiltEditsOnFolio @return the information changed through the store of the symbols on the diff --git a/sources/dataBase/projectdatabase.h b/sources/dataBase/projectdatabase.h index 8d95fe0c5..ceaeec512 100644 --- a/sources/dataBase/projectdatabase.h +++ b/sources/dataBase/projectdatabase.h @@ -186,6 +186,11 @@ class projectDataBase : public QObject QSet foliosWithUnbuiltEdits() const; QHash unbuiltEditsOnFolio(const QUuid &folio, bool *need_build = nullptr) const; bool setPlacedElementInformation(const QUuid &element, const DiagramContext &information); + bool unbuiltWire(const QUuid &wire, QUuid *folio = nullptr) const; + void editUnbuiltConductorProperties(const QUuid &wire, const ConductorProperties &properties); + std::optional takeUnbuiltWireEdit(const QUuid &wire); + QHash unbuiltWireEditsOnFolio(const QUuid &folio) const; + bool setPlacedConductorProperties(const QUuid &wire, const ConductorProperties &properties); QHash elementTypesOnFolios(const QSet &folios) const; /// A symbol placed on a folio, as the element table has it struct FolioSymbol { @@ -203,6 +208,7 @@ class projectDataBase : public QObject QList wiresOnFolios(const QList &folios) const; QString elementLabel(const QUuid &element) const; QString terminalName(const QUuid &terminal, const QUuid &element) const; + std::optional> potentialWires(const QUuid &wire) const; static QVariant sequenceValue(const autonum::sequentialNumbers &sequence); static autonum::sequentialNumbers sequenceFromValue(const QVariant &value); @@ -346,6 +352,9 @@ class projectDataBase : public QObject /// each is on: building the symbol takes it (takeUnbuiltEdit()) struct UnbuiltEdit { DiagramContext information; QUuid folio; bool had_information; }; QHash m_unbuilt_edits; + /// The same for wires (editUnbuiltConductorProperties()) + struct UnbuiltWireEdit { ConductorProperties properties; QUuid folio; }; + QHash m_unbuilt_wire_edits; QHash m_folio_titleblocks; QHash> m_placed_folios; QHash m_conductor_properties; diff --git a/sources/qetgraphicsitem/conductor.cpp b/sources/qetgraphicsitem/conductor.cpp index 339c4a243..f6d3b8ae4 100644 --- a/sources/qetgraphicsitem/conductor.cpp +++ b/sources/qetgraphicsitem/conductor.cpp @@ -1154,6 +1154,11 @@ bool Conductor::fromXml(QDomElement &dom_element) m_freeze_label = dom_element.attribute("freezeLabel") == "true" ? true : false; + //Changed through the project database while its folio was not + //built (lazy folios): the change, not what the file says + if (diagram() && diagram()->project()) + if (const auto edited = diagram()->project()->dataBase()->takeUnbuiltWireEdit(m_uuid)) + pr = *edited; setProperties(pr); return retval; diff --git a/sources/qetproject.cpp b/sources/qetproject.cpp index 1bf67e8a9..d03f15633 100644 --- a/sources/qetproject.cpp +++ b/sources/qetproject.cpp @@ -2512,6 +2512,17 @@ QDomDocument QETProject::toXml(QHash *kept_folios, QStringList const QDomElement old_information = element.firstChildElement(QStringLiteral("elementInformations")); element.replaceChild(information, old_information); //it has one (unbuiltEditsOnFolio()) } + //Its wires whose properties changed + const QHash wire_edits = m_data_base.unbuiltWireEditsOnFolio(diagram->uuid()); + if (!wire_edits.isEmpty()) + for (const QDomElement &list : QET::findInDomElement(xml_diagram, QStringLiteral("conductors"))) + for (QDomElement wire = list.firstChildElement(QStringLiteral("conductor")) ; !wire.isNull() ; + wire = wire.nextSiblingElement(QStringLiteral("conductor"))) { + const auto edit = wire_edits.constFind(QUuid(wire.attribute(QStringLiteral("uuid")))); + if (edit == wire_edits.constEnd()) continue; + for (const auto &attribute : edit->attributes()) + wire.setAttribute(attribute.first, attribute.second); + } xml_diagram.setAttribute("order", order_num ++); project_root.appendChild(xml_diagram); continue; @@ -2530,6 +2541,16 @@ QDomDocument QETProject::toXml(QHash *kept_folios, QStringList it.value().toXml(information, QStringLiteral("elementInformation")); symbol.appendChild(information); placeholder.appendChild(symbol); + } + //Its wires whose properties changed: what the .qetz database + //holds of them (only that can change, QetScriptApi) + const QHash wire_edits = m_data_base.unbuiltWireEditsOnFolio(diagram->uuid()); + for (auto it = wire_edits.cbegin() ; it != wire_edits.cend() ; ++it) { + auto wire = xml_doc.createElement(QStringLiteral("qetz-edited-conductor")); + wire.setAttribute(QStringLiteral("uuid"), it.key().toString()); + for (const auto &attribute : it.value().attributes()) + wire.setAttribute(attribute.first, attribute.second); + placeholder.appendChild(wire); } kept_parts->append(diagram->deferredPart()); project_root.appendChild(placeholder); diff --git a/sources/scripting/qetscriptapi.cpp b/sources/scripting/qetscriptapi.cpp index 3f372bdfb..e8b7f70e5 100644 --- a/sources/scripting/qetscriptapi.cpp +++ b/sources/scripting/qetscriptapi.cpp @@ -1265,6 +1265,16 @@ bool QetScriptApi::setConductorProperty(int folioIndex, const QString &elementUu .arg(caller, value, property)); return false; } + return applyConductorProperties(conductor, properties); +} + +/** + @brief QetScriptApi::applyConductorProperties + Give @p conductor and every wire at its potential the properties + @p properties, one undo step +*/ +bool QetScriptApi::applyConductorProperties(Conductor *conductor, const ConductorProperties &properties) +{ if (properties == conductor->properties()) return true; // already so QSet potential = conductor->relatedPotentialConductors(true); @@ -1282,6 +1292,174 @@ bool QetScriptApi::setConductorProperty(int folioIndex, const QString &elementUu return true; } +namespace { + //The wire properties a .qetz's database holds in its own columns, which + //nothing on a folio computes from but a wire's number formula +const QStringList &storedConductorPropertyNames() +{ + static const QStringList names { + QStringLiteral("num"), QStringLiteral("function"), QStringLiteral("bus"), + QStringLiteral("cable"), QStringLiteral("tension_protocol"), + QStringLiteral("conductor_color"), QStringLiteral("conductor_section")}; + return names; +} + + //true if @p a and @p b differ only in storedConductorPropertyNames() +bool differOnlyInStored(const ConductorProperties &a, const ConductorProperties &b) +{ + QHash left, right; + for (const auto &attribute : a.attributes()) left.insert(attribute.first, attribute.second); + for (const auto &attribute : b.attributes()) right.insert(attribute.first, attribute.second); + QSet names(left.keyBegin(), left.keyEnd()); + for (const QString &name : right.keys()) names.insert(name); + for (const QString &name : std::as_const(names)) + if (left.value(name) != right.value(name) && !storedConductorPropertyNames().contains(name)) + return false; + return true; +} + + //Wires given their properties by uuid, built or not (lazy folios): + //projectDataBase::setPlacedConductorProperties() +class ChangeStoredConductorPropertiesCommand : public QUndoCommand +{ + public: + ChangeStoredConductorPropertiesCommand(QETProject *project, + const QHash> &changes) : + m_project(project), m_changes(changes) + { setText(QObject::tr("Edit the conductor properties")); } + void undo() override { apply(false); } + void redo() override { apply(true); } + private: + void apply(bool forward) { + if (!m_project) return; + for (auto it = m_changes.cbegin() ; it != m_changes.cend() ; ++it) + m_project->dataBase()->setPlacedConductorProperties( + it.key(), forward ? it->second : it->first); + } + QPointer m_project; + QHash> m_changes; +}; +} + +/** + @brief QetScriptApi::findConductorByUuid + @return the wire @p conductorUuid of the folio @p folioIndex, the folio + built, or nullptr (said in the log) +*/ +Conductor *QetScriptApi::findConductorByUuid(int folioIndex, const QString &conductorUuid, + const QString &caller) +{ + if (!m_project || folioIndex < 0 || folioIndex >= m_project->folios().count()) { + log(QStringLiteral("qet.%1: no folio at index %2").arg(caller).arg(folioIndex)); + return nullptr; + } + const QUuid wanted(conductorUuid); + for (Conductor *c : built(m_project->folios().at(folioIndex))->conductors()) + if (c->uuid() == wanted) return c; + log(QStringLiteral("qet.%1: no conductor %2 on folio %3").arg(caller, conductorUuid).arg(folioIndex)); + return nullptr; +} + +/** + @brief QetScriptApi::unbuiltConductorChange + Change the property @p property of the wire @p wire, on the folio + @p folioIndex not built (lazy folios), and of every wire at its potential, + through the project database, as applyConductorProperties() would: when + they are all on folios not built, none is numbered by a formula, and what + changes in each is only what the .qetz database holds + (storedConductorPropertyNames()) -- nothing then is computed from it. + @return true if done; false for the folio to be built, @p refused set when + the value itself is not one +*/ +bool QetScriptApi::unbuiltConductorChange(int folioIndex, const QUuid &wire, const QString &property, + const QString &value, bool *refused) +{ + *refused = false; + projectDataBase *database = m_project->dataBase(); + Diagram *folio = m_project->folios().at(folioIndex); + QUuid on; + if (folio->isLoaded() || !database->rowsFromDocument() || !storedConductorPropertyNames().contains(property) + || !database->unbuiltWire(wire, &on) || on != folio->uuid()) { + return false; + } + ConductorProperties properties = database->conductorProperties(wire); + if (!setConductorPropertyValue(properties, property, value)) { + *refused = true; + return false; + } + if (!properties.m_formula.isEmpty()) { + return false; + } + const auto potential = database->potentialWires(wire); + if (!potential) { + return false; + } + QHash> changes; + for (const QUuid &other : *potential) { + if (!database->unbuiltWire(other)) return false; + const ConductorProperties old = database->conductorProperties(other); + if (!differOnlyInStored(old, properties)) return false; + if (!(old == properties)) changes.insert(other, {old, properties}); + } + if (!changes.isEmpty()) + m_project->undoStack()->push(new ChangeStoredConductorPropertiesCommand(m_project, changes)); + return true; +} + +QString QetScriptApi::conductorPropertyByUuid(int folioIndex, const QString &conductorUuid, + const QString &property) const +{ + if (!m_project || folioIndex < 0 || folioIndex >= m_project->folios().count()) return QString(); + Diagram *folio = m_project->folios().at(folioIndex); + projectDataBase *database = m_project->dataBase(); + QUuid on; + if (!folio->isLoaded() && database->rowsFromDocument() + && database->unbuiltWire(QUuid(conductorUuid), &on) && on == folio->uuid()) + return conductorPropertyValue(database->conductorProperties(QUuid(conductorUuid)), property); + auto *self = const_cast(this); + Conductor *conductor = self->findConductorByUuid(folioIndex, conductorUuid, + QStringLiteral("conductorPropertyByUuid")); + return conductor ? conductorPropertyValue(conductor->properties(), property) : QString(); +} + +bool QetScriptApi::setConductorPropertyByUuid(int folioIndex, const QString &conductorUuid, + const QString &property, const QString &value) +{ + if (!m_project) return false; + const QString caller = QStringLiteral("setConductorPropertyByUuid"); + if (m_project->isReadOnly()) { + log(QStringLiteral("qet.%1: project is read-only").arg(caller)); + return false; + } + if (!conductorPropertyNames().contains(property)) { + log(QStringLiteral("qet.%1: unknown property '%2'; expected one of %3") + .arg(caller, property, conductorPropertyNames().join(QStringLiteral(", ")))); + return false; + } + if (folioIndex < 0 || folioIndex >= m_project->folios().count()) { + log(QStringLiteral("qet.%1: no folio at index %2").arg(caller).arg(folioIndex)); + return false; + } + //The folio left unbuilt when the change can go through the project + //database (stage D5) + bool refused = false; + if (unbuiltConductorChange(folioIndex, QUuid(conductorUuid), property, value, &refused)) + return true; + if (refused) { + log(QStringLiteral("qet.%1: '%2' is not a valid value for %3").arg(caller, value, property)); + return false; + } + Conductor *conductor = findConductorByUuid(folioIndex, conductorUuid, caller); + if (!conductor) return false; + ConductorProperties properties = conductor->properties(); + if (!setConductorPropertyValue(properties, property, value)) { + log(QStringLiteral("qet.%1: '%2' is not a valid value for %3") + .arg(caller, value, property)); + return false; + } + return applyConductorProperties(conductor, properties); +} + /** @brief QetScriptApi::conductorSegments List the drawn path of the conductor on a terminal, one line per diff --git a/sources/scripting/qetscriptapi.h b/sources/scripting/qetscriptapi.h index 6a954d5a6..9d0c9c7b7 100644 --- a/sources/scripting/qetscriptapi.h +++ b/sources/scripting/qetscriptapi.h @@ -34,6 +34,7 @@ class DiagramView; class Element; class Terminal; class Conductor; +class ConductorProperties; class IndependentTextItem; class QetShapeItem; class DiagramImageItem; @@ -440,6 +441,11 @@ class QetScriptApi : public QObject Q_INVOKABLE bool setConductorProperty(int folioIndex, const QString &elementUuid, int terminalIndex, const QString &property, const QString &value); + // The same, the wire named by its uuid (conductorUuids()) + Q_INVOKABLE QString conductorPropertyByUuid(int folioIndex, const QString &conductorUuid, + const QString &property) const; + Q_INVOKABLE bool setConductorPropertyByUuid(int folioIndex, const QString &conductorUuid, + const QString &property, const QString &value); // -- a conductor's own drawn path, not the whole potential's // properties above -- one conductor only, addressed the same way -- @@ -648,6 +654,10 @@ class QetScriptApi : public QObject private: bool runFlag(const QString &flag, const QStringList &args); Diagram *built(Diagram *folio) const; + Conductor *findConductorByUuid(int folioIndex, const QString &conductorUuid, const QString &caller); + bool applyConductorProperties(Conductor *conductor, const ConductorProperties &properties); + bool unbuiltConductorChange(int folioIndex, const QUuid &wire, const QString &property, + const QString &value, bool *refused); std::optional unbuiltInformation(int folioIndex, const QString &elementUuid) const; Element *findElement(int folioIndex, const QString &elementUuid) const; Terminal *findTerminal(int folioIndex, const QString &elementUuid, int terminalIndex, diff --git a/tests/qttest/CMakeLists.txt b/tests/qttest/CMakeLists.txt index b5be0cf96..2418a9ef0 100644 --- a/tests/qttest/CMakeLists.txt +++ b/tests/qttest/CMakeLists.txt @@ -92,7 +92,7 @@ if(Python3_Interpreter_FOUND AND CMAKE_GENERATOR STREQUAL "Ninja") --build ${CMAKE_BINARY_DIR} --binary $ --source ${CMAKE_CURRENT_SOURCE_DIR}/lazy_folios_probe.cpp --fixture ${CMAKE_CURRENT_SOURCE_DIR}/../../examples/industrial.qet) - set_tests_properties(tst_lazyfolios PROPERTIES TIMEOUT 600 RUN_SERIAL TRUE + set_tests_properties(tst_lazyfolios PROPERTIES TIMEOUT 1800 RUN_SERIAL TRUE ENVIRONMENT "QT_QPA_PLATFORM=offscreen") # Deleting a cable, undoing the delete twice and then editing anything # has both cable commands tearing down the same cable. Runs under the diff --git a/tests/qttest/lazy_folios_probe.cpp b/tests/qttest/lazy_folios_probe.cpp index 2bc143b93..630dca6db 100644 --- a/tests/qttest/lazy_folios_probe.cpp +++ b/tests/qttest/lazy_folios_probe.cpp @@ -777,6 +777,28 @@ int main(int argc, char **argv) std::printf(".qetz drawn items through their view: %d lazily, %d when built\n", lazily, built); check(built > 0, "the project has drawn items"); check(lazily == built, "the drawn items' view as when built"); + } + //Every wire's potential from the database as QElectroTech + //follows it with every folio built + { + QETProject indexed_potentials(zipped); + int same = 0, differ = 0, unknown = 0; + for (Diagram *folio : eager.folios()) + for (Conductor *wire : folio->conductors()) { + QSet live{wire->uuid()}; + for (Conductor *c : wire->relatedPotentialConductors(true)) live.insert(c->uuid()); + for (QETProject *project : {&eager, &indexed_potentials}) { + const auto stored = project->dataBase()->potentialWires(wire->uuid()); + if (!stored) { ++unknown; continue; } + if (*stored == live) { ++same; continue; } + if (differ++ < 3) + std::printf(" potential of %s: %d wires followed, %d from the database\n", + qPrintable(wire->uuid().toString()), int(live.size()), int(stored->size())); + } + } + std::printf("potentials from the database: %d same, %d differ, %d not answered\n", same, differ, unknown); + check(differ == 0 && same > 0, "every wire's potential from the database as followed"); + check(indexed_potentials.unloadedFolioCount() > 0, "potentials read without building"); } //The folios' rows as saved, not as reading their headers one //by one numbers them ("1/1" for the first of three, D7) @@ -997,6 +1019,76 @@ int main(int argc, char **argv) revisionEveryFolio(zipped, output, "revised", path); revisionEveryFolio(zipped, output, "relabelled", path, busiest); + //A script changes a wire's cable by its uuid, on folios not + //built, the wire and its potential numbered by hand: through + //the project database, no folio built; saved as when built; + //undone; the folio built afterwards shows it (D5) + { + QETProject cabled(zipped); + const int unbuilt = cabled.unloadedFolioCount(); + int folio_index = -1; + QString uuid; + QSet potential; + for (int i = 0 ; i < folios.size() && uuid.isEmpty() ; ++i) { + if (cabled.folios().at(i)->isLoaded()) continue; + for (Conductor *wire : folios.at(i)->conductors()) { + QSet wires = wire->relatedPotentialConductors(true); + wires << wire; + bool usable = true; + for (Conductor *c : std::as_const(wires)) + if (!c->properties().m_formula.isEmpty() + || cabled.folios().at(eager.folioIndex(c->diagram()))->isLoaded()) + usable = false; + if (!usable || wires.size() < 2) continue; + folio_index = i; + uuid = wire->uuid().toString(); + for (Conductor *c : std::as_const(wires)) potential.insert(c->uuid()); + break; + } + } + check(!uuid.isEmpty(), "a wire numbered by hand, its potential on folios not built"); + QetScriptApi api(&cabled); + const QString before = api.conductorPropertyByUuid(folio_index, uuid, "cable"); + const QString value = QStringLiteral("W-42"); + check(api.setConductorPropertyByUuid(folio_index, uuid, "cable", value), "a script changes a wire not built"); + check(api.conductorPropertyByUuid(folio_index, uuid, "cable") == value, "the wire's change read back"); + for (const QUuid &wire : std::as_const(potential)) + check(cabled.dataBase()->conductorProperties(wire).m_cable == value, "every wire of the potential changed"); + check(cabled.unloadedFolioCount() == unbuilt, "changing wires not built builds no folio"); + const QString cabled_path = output + "/cabled.qetz"; + check(cabled.writeContainerFile(cabled_path, &error), "saved after the wire change"); + std::printf(".qetz wire on folio %d, potential of %d wires, cable changed by uuid: 0 folios built, %d by saving\n", + folio_index + 1, int(potential.size()), unbuilt - cabled.unloadedFolioCount()); + qputenv("QET_EAGER_FOLIOS", "1"); + const QString usual_path = output + "/cabled-usual.qetz"; + { + QETProject usual_cabled(zipped); + check(QetScriptApi(&usual_cabled).setConductorPropertyByUuid(folio_index, uuid, "cable", value), + "the same wire change when built"); + check(usual_cabled.writeContainerFile(usual_path, &error), "saved after the wire change when built"); + } + QETProject cabled_back(cabled_path), usual_back(usual_path); + qunsetenv("QET_EAGER_FOLIOS"); + auto lines = [&path](QETProject &project) { + QStringList list = sortedLines(project.toXml().toString(), path); + list.removeIf([](const QString &line) { return line.contains(QLatin1String("name=\"saved")); }); + return list; + }; + check(lines(cabled_back) == lines(usual_back), "a wire changed through the database saved as when built"); + cabled.undoStack()->undo(); + check(api.conductorPropertyByUuid(folio_index, uuid, "cable") == before, "the wire change undone"); + cabled.undoStack()->redo(); + Conductor *built_wire = nullptr; + for (Conductor *wire : cabled.folios().at(folio_index)->conductors()) + if (wire->uuid().toString() == uuid) built_wire = wire; + check(!built_wire, "the folio not built yet"); + cabled.buildFolioToShow(cabled.folios().at(folio_index)); + for (Conductor *wire : cabled.folios().at(folio_index)->conductors()) + if (wire->uuid().toString() == uuid) built_wire = wire; + check(built_wire && built_wire->properties().m_cable == value, + "a wire changed before its folio was built shows the change"); + } + //The wire numbers and wiring list exports, and the terminal //numbering dialog: as when built, building only the folios //holding a terminal (D5) diff --git a/tests/qttest/run_lazy_folios_probe.py b/tests/qttest/run_lazy_folios_probe.py index 2c7f881cb..af41a0fcc 100644 --- a/tests/qttest/run_lazy_folios_probe.py +++ b/tests/qttest/run_lazy_folios_probe.py @@ -54,7 +54,7 @@ subprocess.run([ninja, '-C', str(build), '-f', probe_manifest.name, target], che output = build / 'lazy-folios' output.mkdir(exist_ok=True) report = output / 'results.txt' -result = subprocess.run([str(probe_binary), str(fixture), str(report), str(output)], timeout=500) +result = subprocess.run([str(probe_binary), str(fixture), str(report), str(output)], timeout=1500) if report.exists(): print(report.read_text(encoding='utf-8', errors='replace')) sys.exit(result.returncode) From 4104bb8d537b02fb67b39d58200fdb5814e74507 Mon Sep 17 00:00:00 2001 From: ispyisail Date: Tue, 13 Oct 2026 07:05:27 +1300 Subject: [PATCH 22/26] Write only the wire attributes reading needs in a .qetz (stage D3b) A wire's properties are one row per attribute in the project database, 45 071 rows for f2099's 2 139 wires. The database a .qetz holds now keeps only the attributes reading would not give anyway: each is left out in turn when the properties read without it are the same (ConductorProperties ::fromXml(), its defaults); rows by wire and name, so the file does not depend on memory order. Reading writes every attribute back into the table, so queries and saves see what they saw before. f2099: 9 387 rows, the file 3.60 MB -> 3.22 MB (5.14 MB before D3b); industrial.qet 0.71 -> 0.61 MB. Gates with the master merge and the wire edits by uuid: ctest 99/99 lazily, 98/99 with every folio built (modal_quit_regression, the race fixed next); hostile .qetz all refused; 342 corpus projects save, export, round trip and re-save byte-stable. Co-Authored-By: Claude Opus 5.5 --- sources/dataBase/projectdatabase.cpp | 90 ++++++++++++++++++++++++---- 1 file changed, 78 insertions(+), 12 deletions(-) diff --git a/sources/dataBase/projectdatabase.cpp b/sources/dataBase/projectdatabase.cpp index 32085c9b3..6dd82ea82 100644 --- a/sources/dataBase/projectdatabase.cpp +++ b/sources/dataBase/projectdatabase.cpp @@ -931,6 +931,38 @@ QStringList indexTables(const QSqlDatabase &db) } } +namespace { + //A wire's properties read from @p attributes, as a saved wire's are +ConductorProperties propertiesFrom(const QHash &attributes) +{ + QDomDocument scratch; + QDomElement e = scratch.createElement(QStringLiteral("conductor")); + for (auto it = attributes.cbegin() ; it != attributes.cend() ; ++it) + e.setAttribute(it.key(), it.value()); + ConductorProperties properties; + properties.fromXml(e); + return properties; +} + + //@p attributes without those reading them would give anyway: each + //left out in turn when the properties read without it are the same + //(stage D3b) +QHash fewestAttributes(QHash attributes) +{ + const ConductorProperties full = propertiesFrom(attributes); + QStringList names = attributes.keys(); + names.sort(); + for (const QString &name : std::as_const(names)) { + QHash without = attributes; + without.remove(name); + const ConductorProperties read = propertiesFrom(without); + if (read == full && read.attributes() == full.attributes()) + attributes = without; + } + return attributes; +} +} + /** @brief projectDataBase::indexFile This database written to an SQLite file, for a .qetz to be opened from @@ -1011,6 +1043,29 @@ QByteArray projectDataBase::indexFile(const QList> &fo query.addBindValue(table.folio.toString()); query.addBindValue(table.previous.isNull() ? QVariant() : QVariant(table.previous.toString())); ok = ok && query.exec(); + } + //A wire's properties: only the attributes reading them needs, + //by wire and name (fillFromIndex() writes the others back) + { + QMap> wires; + QSqlQuery rows(file); + rows.exec(QStringLiteral("SELECT conductor_uuid, name, value FROM conductor_properties")); + while (rows.next()) + wires[rows.value(0).toString()].insert(rows.value(1).toString(), rows.value(2).toString()); + ok = ok && query.exec(QStringLiteral("DELETE FROM conductor_properties")); + query.prepare(QStringLiteral("INSERT INTO conductor_properties (conductor_uuid, name, value) " + "VALUES (?, ?, ?)")); + for (auto wire = wires.cbegin() ; wire != wires.cend() ; ++wire) { + const QHash fewest = fewestAttributes(wire.value()); + QStringList names = fewest.keys(); + names.sort(); + for (const QString &name : std::as_const(names)) { + query.addBindValue(wire.key()); + query.addBindValue(name); + query.addBindValue(fewest.value(name)); + ok = ok && query.exec(); + } + } } ok = file.commit() && ok; ok = query.exec(QStringLiteral("VACUUM")) && ok; @@ -1164,18 +1219,29 @@ bool projectDataBase::fillFromIndex(const QByteArray &index, query.value(3).toInt() != 0); } QHash> attributes; - query.exec(QStringLiteral("SELECT conductor_uuid, name, value FROM conductor_properties")); - while (query.next()) - attributes[QUuid(query.value(0).toString())].insert(query.value(1).toString(), - query.value(2).toString()); - QDomDocument scratch; - for (auto it = attributes.cbegin() ; it != attributes.cend() ; ++it) { - QDomElement e = scratch.createElement(QStringLiteral("conductor")); - for (auto a = it.value().cbegin() ; a != it.value().cend() ; ++a) - e.setAttribute(a.key(), a.value()); - ConductorProperties properties; - properties.fromXml(e); - m_prefilled_conductors.insert(it.key(), properties); + QList wire_order; + query.exec(QStringLiteral("SELECT conductor_uuid, name, value FROM conductor_properties ORDER BY rowid")); + while (query.next()) { + const QUuid wire(query.value(0).toString()); + if (!attributes.contains(wire)) wire_order << wire; + attributes[wire].insert(query.value(1).toString(), query.value(2).toString()); + } + //A file may hold only the attributes reading needs (indexFile()): + //the table gets every attribute a saved wire carries, as before + query.exec(QStringLiteral("DELETE FROM conductor_properties")); + query.prepare(QStringLiteral("INSERT INTO conductor_properties (conductor_uuid, name, value) VALUES (?, ?, ?)")); + for (const QUuid &wire : std::as_const(wire_order)) { + const ConductorProperties properties = propertiesFrom(attributes.value(wire)); + m_prefilled_conductors.insert(wire, properties); + QHash &full = attributes[wire]; + full.clear(); + for (const auto &attribute : properties.attributes()) { + full.insert(attribute.first, attribute.second); + query.addBindValue(wire.toString()); + query.addBindValue(attribute.first); + query.addBindValue(attribute.second); + query.exec(); + } } for (auto it = m_prefilled_elements.cbegin() ; it != m_prefilled_elements.cend() ; ++it) m_element_information.insert(it.key(), it.value()); From bfa9912c5f71621c94a0f5d42c11182c841a8e4d Mon Sep 17 00:00:00 2001 From: ispyisail Date: Tue, 13 Oct 2026 07:05:27 +1300 Subject: [PATCH 23/26] Make modal_quit_regression deterministic The test interrupted QElectroTech at a random moment inside the dialog's loop and called quitQET() there. Landing inside malloc(), the call waited on the allocator's lock for ever: 1 run in 20 timed out at 120 s, on master too. It now lets the program run on to the start of the main thread's next QEventLoop::processEvents(), between two events, and calls from there; the event loop watchdog wakes the loop every 50 ms. A run could also stop before any dialog: SingleApplication keys a System V shared memory block from a file in the temporary folder, 16 bits of the key varying, and each run killed by gdb leaves its block; once a new run's key fell on one of them (976 here), it refused to start. The run now removes the blocks it leaves, attached to no process, and is tried again (a new binary path, a new key, up to three times) when SingleApplication stops it that way: the scenario never ran. The "dialog entered" marker no longer prints when the program had exited. 60 runs under load, 976 old blocks present: 60 passed, one try repeated. With refuseCloseWhileModal() disabled: 3 of 3 FAIL (signal 6). Co-Authored-By: Claude Opus 5.5 --- tests/modal-quit-regression/run.sh | 68 +++++++++++++++++++++++++----- 1 file changed, 57 insertions(+), 11 deletions(-) diff --git a/tests/modal-quit-regression/run.sh b/tests/modal-quit-regression/run.sh index 72638b9b9..2e7da7f10 100755 --- a/tests/modal-quit-regression/run.sh +++ b/tests/modal-quit-regression/run.sh @@ -33,7 +33,13 @@ # 2. let it run for a moment, then interrupt it -- the main thread is now # inside the dialog's nested loop, which is the only place the bug lives # (a deleteLater() posted before exec() started is not processed by it); -# 3. call QETApp::quitQET(), which closes every editor; +# 3. let it run on to the start of the loop's next QEventLoop::processEvents() +# on the main thread, and call QETApp::quitQET() there, which closes +# every editor. Not where the interrupt lands: that can be inside +# malloc(), whose lock a call into the program then waits on for ever +# (1 run in 20 timed out that way, on master too). Between two +# events the loop holds nothing, and QElectroTech's event loop +# watchdog wakes it every 50 ms, so the stop comes at once; # 4. let it run again: the unfixed build aborts within a second, the fixed # one keeps running until the test interrupts it. # @@ -104,14 +110,32 @@ if command -v nm >/dev/null; then fi SANDBOX="$(mktemp -d /tmp/qet-modal-quit.XXXXXX)" -cleanup() { [ "${KEEP_LOGS:-0}" = "1" ] || rm -rf "$SANDBOX"; } + +# SingleApplication keeps a System V shared memory block, keyed from a file in +# the temporary folder: 16 bits of the key vary. This run ends QElectroTech +# with gdb's kill, which leaves its block behind; blocks left so pile up, and +# once a new run's key falls on one, SingleApplication cannot start ("Unable to +# attach to shared memory block") -- 1 run in 40 with 976 of them. So the +# blocks this run leaves, attached to no process, are removed at the end, and +# a run that meets someone else's is tried again (below). +orphan_blocks() { + command -v ipcs >/dev/null || return 0 + ipcs -m 2>/dev/null | awk -v me="$(id -un)" 'NR > 3 && $3 == me && $6 == 0 { print $2 }' +} +BLOCKS_BEFORE=" $(orphan_blocks | tr '\n' ' ') " +cleanup() { + if command -v ipcrm >/dev/null; then + for id in $(orphan_blocks); do + case "$BLOCKS_BEFORE" in *" $id "*) ;; *) ipcrm -m "$id" 2>/dev/null ;; esac + done + fi + [ "${KEEP_LOGS:-0}" = "1" ] || rm -rf "$SANDBOX" +} trap cleanup EXIT # A unique binary path gives this run its own SingleApplication socket, so it # can neither be captured by nor capture a QElectroTech already running. A # symlink would not do: applicationFilePath() resolves it back to the real path. -TEST_BINARY="$SANDBOX/qelectrotech-modalquit" -cp "$BINARY" "$TEST_BINARY" || { echo "could not copy binary" >&2; exit 2; } # Keep the original file name. QElectroTech decides what to open from the # extension, so copying a .elmt to "project.qet" would quietly turn an @@ -141,9 +165,16 @@ end break QDialog::exec run delete +if \$_isvoid(\$_exitcode) printf "QET_TEST: dialog exec() entered\n" +else +printf "QET_TEST: exited before any dialog, code %d\n", \$_exitcode +end python qet_interrupt_later($SETTLE) continue +tbreak QEventLoop::processEvents thread 1 +continue +printf "QET_TEST: at the dialog's event loop\n" thread 1 set \$app = ((void* (*)(void))'QETApp::instance()')() call ((void (*)(void*))'QETApp::quitQET()')(\$app) @@ -162,14 +193,29 @@ EOF # the full watchdog period no matter how fast gdb finished: ten seconds of # work reported as two minutes, a stone's throw from the CTest timeout. GDB_TIMEOUT=120 -if command -v timeout >/dev/null; then - timeout --signal=KILL "$GDB_TIMEOUT" \ +# Each try under a binary path of its own: a new SingleApplication key. Tried +# again only when QElectroTech stopped before any dialog because +# SingleApplication met a block it could not use -- the scenario never ran, +# the code under test played no part. +for attempt in 1 2 3; do + TEST_BINARY="$SANDBOX/qelectrotech-modalquit-$attempt" + cp "$BINARY" "$TEST_BINARY" || { echo "could not copy binary" >&2; exit 2; } + if command -v timeout >/dev/null; then + timeout --signal=KILL "$GDB_TIMEOUT" \ + gdb -batch -x "$SANDBOX/scenario.gdb" --args "$TEST_BINARY" "$SANDBOX_PROJECT" \ + > "$SANDBOX/gdb.log" 2>&1 + else gdb -batch -x "$SANDBOX/scenario.gdb" --args "$TEST_BINARY" "$SANDBOX_PROJECT" \ - > "$SANDBOX/gdb.log" 2>&1 -else - gdb -batch -x "$SANDBOX/scenario.gdb" --args "$TEST_BINARY" "$SANDBOX_PROJECT" \ - > "$SANDBOX/gdb.log" 2>&1 -fi + > "$SANDBOX/gdb.log" 2>&1 + fi + if grep -q "QET_TEST: exited before any dialog" "$SANDBOX/gdb.log" \ + && grep -q "SingleApplication: Unable to" "$SANDBOX/gdb.log"; then + echo "note: try $attempt: SingleApplication could not set up its shared memory (a block left by another run); trying again" + cp "$SANDBOX/gdb.log" "$SANDBOX/gdb-try$attempt.log" + continue + fi + break +done log="$SANDBOX/gdb.log" if ! grep -q "QET_TEST: dialog exec() entered" "$log"; then From f7e0283e6176adcd7ae0edb063e3d77bbc75a5e3 Mon Sep 17 00:00:00 2001 From: ispyisail Date: Tue, 13 Oct 2026 10:29:32 +1300 Subject: [PATCH 24/26] Include QDomDocument in wiringlistexport.h The header declares projectDocument() returning a QDomDocument but did not include it; it only built where the precompiled header pulled it in. CI builds without one and failed. Co-Authored-By: Claude Opus 5.5 --- sources/wiringlistexport.h | 1 + 1 file changed, 1 insertion(+) diff --git a/sources/wiringlistexport.h b/sources/wiringlistexport.h index 759e099a2..025249e55 100644 --- a/sources/wiringlistexport.h +++ b/sources/wiringlistexport.h @@ -6,6 +6,7 @@ #include #include #include +#include class QETProject; class QWidget; From fbda162e341df4b89be0d25c23a007699a79e2d5 Mon Sep 17 00:00:00 2001 From: ispyisail Date: Tue, 13 Oct 2026 23:00:49 +1300 Subject: [PATCH 25/26] Find a .qetz's folios with cable wires in its database Opened from its database, a .qetz's folios not built have no XML in memory: reading each one's part only to look for wires referring to a cable read every folio. The project database already knows which wires carry a cable reference, and so which folios to build for the label pass. Co-Authored-By: Claude Opus 5.5 --- sources/qetproject.cpp | 34 +++++++++++++++++++++++----------- 1 file changed, 23 insertions(+), 11 deletions(-) diff --git a/sources/qetproject.cpp b/sources/qetproject.cpp index d03f15633..bd75b4eeb 100644 --- a/sources/qetproject.cpp +++ b/sources/qetproject.cpp @@ -611,9 +611,10 @@ QList QETProject::folios() const @return the folios CableManager::refreshLabels() works on, built: every folio when the project has cables (a core is followed through folio reports to any folio); without cables, the folios built and those not - built yet (QET_LAZY_FOLIOS) holding a wire that still refers to a - cable -- the only wires the pass changes then, taking the reference - away -- found in the XML kept for them + built yet holding a wire that still refers to a cable -- the only + wires the pass changes then, taking the reference away. Read from the + project database for a .qetz opened from it, from the folios' XML + otherwise. */ QList QETProject::cableLabelFolios() { @@ -621,14 +622,25 @@ QList QETProject::cableLabelFolios() return diagrams(); } QList referring; - for (Diagram *d : std::as_const(m_diagrams_list)) { - if (d->isLoaded()) continue; - for (const QDomElement &wire : QET::findInDomElement(d->deferredXml(), QStringLiteral("conductors"), - QStringLiteral("conductor"))) - if (!QUuid(wire.attribute(QStringLiteral("cable_uuid"))).isNull()) { - referring << d; - break; - } + if (m_opened_from_index) { + QSet folios; + QSqlQuery query = m_data_base.newQuery(QStringLiteral( + "SELECT DISTINCT c.diagram_uuid FROM conductor c JOIN conductor_properties p " + "ON p.conductor_uuid = c.uuid WHERE p.name = 'cable_uuid' AND p.value != ''")); + if (query.exec()) + while (query.next()) folios.insert(QUuid(query.value(0).toString())); + for (Diagram *d : std::as_const(m_diagrams_list)) + if (!d->isLoaded() && folios.contains(d->uuid())) referring << d; + } else { + for (Diagram *d : std::as_const(m_diagrams_list)) { + if (d->isLoaded()) continue; + for (const QDomElement &wire : QET::findInDomElement(d->deferredXml(), QStringLiteral("conductors"), + QStringLiteral("conductor"))) + if (!QUuid(wire.attribute(QStringLiteral("cable_uuid"))).isNull()) { + referring << d; + break; + } + } } buildFolios(referring); return builtFolios(); From c259cae4295b5efebf2a7f18cf2ad44c442b0c1b Mon Sep 17 00:00:00 2001 From: ispyisail Date: Wed, 14 Oct 2026 07:35:34 +1300 Subject: [PATCH 26/26] Keep each fact once in a .qetz: drop the stored project database A .qetz held the engineering data twice: the document's tables, and beside them QElectroTech's own project database (stage D2/D3), which QElectroTech also builds from the document. Two copies of one fact can disagree, and the copy cost a third of the file. Now project.sqlite holds the document's tables only; QElectroTech builds its own project database from them when the project opens, as it does for a .qet. Measured (this build, lazy folios): industrial (50 folios): 620 -> 374 KB, open 0.37 -> 0.65 s f2099/nuri (136 folios): 3.27 -> 2.20 MB, open 2.2 -> 3.5 s The same projects as .qet open in 3.3 s and 23 s. Saving still copies the file of every folio not built, with its rows (stage D4): the project keeps the part each folio was read from. nuri opened and saved again: 6.3 s, no folio built, the same bytes. A symbol's terminal list is left out of the folio file when every folio, symbol and wire has a uuid of its own, now checked on the document (QetContainer::ownUuids()) rather than by the stored database being written. Checked: 342 corpus projects open, .qetz -> .qet equals the plain resave, a .qetz saved twice gives the same bytes; tst_lazyfolios, tst_qetcontainer, tst_resaveunchanged (which now checks project.sqlite holds only the document's tables). Co-Authored-By: Claude Opus 5.5 --- sources/container/qetcontainer.cpp | 91 +++---- sources/container/qetcontainer.h | 12 +- sources/container/qetcontainerdb.cpp | 115 +-------- sources/container/qetcontainerdb.h | 15 -- sources/dataBase/projectdatabase.cpp | 371 --------------------------- sources/dataBase/projectdatabase.h | 11 - sources/diagram.cpp | 21 +- sources/diagram.h | 16 +- sources/qetproject.cpp | 260 ++++--------------- sources/qetproject.h | 21 +- tests/qttest/CMakeLists.txt | 7 +- tests/qttest/lazy_folios_probe.cpp | 27 +- tests/qttest/tst_qetcontainer.cpp | 7 +- tests/qttest/tst_resaveunchanged.cpp | 33 +++ 14 files changed, 165 insertions(+), 842 deletions(-) diff --git a/sources/container/qetcontainer.cpp b/sources/container/qetcontainer.cpp index 4b8f18a14..15ffe995d 100644 --- a/sources/container/qetcontainer.cpp +++ b/sources/container/qetcontainer.cpp @@ -23,6 +23,7 @@ #include #include #include +#include const char QetContainer::MimeType[] = "application/x-qelectrotech-project+zip"; @@ -107,6 +108,7 @@ void removeElement(QDomElement e) } } + //The collection's symbol definitions, each to elements// void splitCollection(QDomDocument &document, const QDomElement &category, const QString &path, QList *parts, QSet *taken) @@ -184,6 +186,33 @@ QString QetContainer::pictureFile(const QDomElement &image) return image.attribute(ImageFileAttribute); } +/** + @brief QetContainer::ownUuids + @return true if every folio, symbol and wire of @p project has a uuid + no other has: a wire then finds its ends without the symbols' terminal + lists, which split() leaves out +*/ +bool QetContainer::ownUuids(const QDomElement &project) +{ + QSet seen; + auto own = [&seen](const QDomElement &item) { + const QString uuid = item.attribute(QStringLiteral("uuid")); + if (QUuid(uuid).isNull() || seen.contains(uuid)) return false; + seen.insert(uuid); + return true; + }; + for (const QDomElement &diagram : childElements(project, QStringLiteral("diagram"))) { + if (!own(diagram)) return false; + for (const QDomElement &list : childElements(diagram, QStringLiteral("elements"))) + for (const QDomElement &element : childElements(list, QStringLiteral("element"))) + if (!own(element)) return false; + for (const QDomElement &list : childElements(diagram, QStringLiteral("conductors"))) + for (const QDomElement &wire : childElements(list, QStringLiteral("conductor"))) + if (!own(wire)) return false; + } + return true; +} + /** @brief QetContainer::dropComputed Take out of the folio @p diagram what QElectroTech computes when it @@ -235,8 +264,7 @@ void QetContainer::dropComputed(QDomElement diagram, bool terminal_lists) set if @p project is not a project */ QList QetContainer::split(const QDomDocument &project, const QString &writer, - QString *error, const QByteArray &index, - const QHash &kept_from) + QString *error, const QHash &kept_from) { QDomDocument document = project.cloneNode(true).toDocument(); QDomElement root = document.documentElement(); @@ -250,7 +278,7 @@ QList QetContainer::split(const QDomDocument &project, const QStr //stage D4) keeps its part and, through //@p kept_from's project.sqlite, its rows const QByteArray database = QetContainerDb::extract( - root, error, index, kept_from.value(QStringLiteral("project.sqlite"))); + root, error, kept_from.value(QStringLiteral("project.sqlite"))); if (database.isEmpty()) return {}; @@ -259,9 +287,10 @@ QList QetContainer::split(const QDomDocument &project, const QStr //Folios, each with its pictures taken out int folio = 0; - //Nothing QElectroTech computes is kept (format 3) + //Nothing QElectroTech computes is kept (format 4) + const bool own_uuids = ownUuids(root); for (QDomElement diagram : childElements(root, QStringLiteral("diagram"))) - dropComputed(diagram, !index.isEmpty()); + dropComputed(diagram, own_uuids); //The parts of the folios kept as they were, and their pictures QMap pictures; //name -> png, once each @@ -354,46 +383,6 @@ QList QetContainer::split(const QDomDocument &project, const QStr return entries; } -/** - @brief QetContainer::readFolio - The folio part @p file of the .qetz whose entries are @p files, as - join() puts it in the project: its pictures back as base64 and its - engineering data back from project.sqlite. -*/ -bool QetContainer::readFolio(const QHash &files, const QString &file, - QDomDocument *folio, QString *error, - QetContainerDb::FolioReader *reader) -{ - if (!files.contains(file)) { - setError(error, QStringLiteral("part \"%1\" is missing").arg(file)); - return false; - } - QString part_error; - if (!parse(files.value(file), folio, &part_error)) { - setError(error, QStringLiteral("%1: %2").arg(file, part_error)); - return false; - } - const QDomNodeList images = folio->elementsByTagName(QStringLiteral("image")); - for (int i = 0 ; i < images.size() ; ++i) - { - QDomElement image = images.at(i).toElement(); - if (!image.hasAttribute(ImageFileAttribute)) continue; - const QString picture = image.attribute(ImageFileAttribute); - if (!files.contains(picture)) { - setError(error, QStringLiteral("picture \"%1\" is missing").arg(picture)); - return false; - } - image.removeAttribute(ImageFileAttribute); - image.insertBefore(folio->createTextNode(QString::fromLatin1(files.value(picture).toBase64())), - image.firstChild()); - } - if (!files.contains(QStringLiteral("project.sqlite"))) - return true; - return reader ? reader->restore(folio->documentElement(), error) - : QetContainerDb::restoreFolio(folio->documentElement(), - files.value(QStringLiteral("project.sqlite")), error); -} - /** @brief QetContainer::join Rebuild in @p project the document split() made @p entries from. @@ -401,7 +390,8 @@ bool QetContainer::readFolio(const QHash &files, const QStr (the manifest's min-reader), or a part missing or unreadable. Entries nothing names are ignored, so a later format may add some. With @p pictures, the pictures stay out of the document: see the - namespace. + namespace. With @p folio_files, the part each folio was read from, in + order. */ bool QetContainer::join(const QList &entries, QDomDocument *project, QString *error, QStringList *folio_files, Pictures *pictures) @@ -436,16 +426,9 @@ bool QetContainer::join(const QList &entries, QDomDocument *proje for (QDomElement part : placeholders) { const QString file = part.attribute(FileAttribute); - //With @p folio_files, the folios stay parts, read when built if (folio_files && part.parentNode() == project->documentElement() - && file.startsWith(QLatin1String("folios/"))) { - if (!files.contains(file)) { - setError(error, QStringLiteral("part \"%1\" is missing").arg(file)); - return false; - } + && file.startsWith(QLatin1String("folios/"))) folio_files->append(file); - continue; - } if (!files.contains(file)) { setError(error, QStringLiteral("part \"%1\" is missing").arg(file)); return false; diff --git a/sources/container/qetcontainer.h b/sources/container/qetcontainer.h index 33740306f..54d75a693 100644 --- a/sources/container/qetcontainer.h +++ b/sources/container/qetcontainer.h @@ -18,7 +18,6 @@ #ifndef QETCONTAINER_H #define QETCONTAINER_H -#include "qetcontainerdb.h" #include "qetzip.h" #include @@ -61,27 +60,24 @@ namespace QetContainer /// 1/0, its PRAGMA user_version the format. /// 4: what QElectroTech computes left out of the folios /// (dropComputed()), the tables written without keys or indexes. + /// project.sqlite holds the engineering data once: QElectroTech's + /// own project database is built from it when the project opens. /// Formats 2 and 3 are still read. QList split(const QDomDocument &project, const QString &writer = QString(), QString *error = nullptr, - const QByteArray &index = QByteArray(), const QHash &kept_from = QHash()); bool join(const QList &entries, QDomDocument *project, QString *error = nullptr, QStringList *folio_files = nullptr, Pictures *pictures = nullptr); - /// A folio part of @p files, which join() left out, with its - /// pictures and its engineering data back (stage D2) - /// @p reader, if any, reads the engineering data of @p files - bool readFolio(const QHash &files, const QString &file, - QDomDocument *folio, QString *error = nullptr, - QetContainerDb::FolioReader *reader = nullptr); /// The file join() left the picture of @p image in, or an empty string QString pictureFile(const QDomElement &image); /// Take what QElectroTech computes out of a folio (format 4) void dropComputed(QDomElement diagram, bool terminal_lists); + /// true if every folio, symbol and wire has a uuid of its own + bool ownUuids(const QDomElement &project); /// Parse @p xml as a .qet is read (QETProject::openFile) bool parse(const QByteArray &xml, QDomDocument *document, QString *error = nullptr); /// true if @p content is a zip, so a .qetz rather than a .qet diff --git a/sources/container/qetcontainerdb.cpp b/sources/container/qetcontainerdb.cpp index aea1b4768..059fe41cf 100644 --- a/sources/container/qetcontainerdb.cpp +++ b/sources/container/qetcontainerdb.cpp @@ -19,7 +19,6 @@ #include "qetcontainer.h" -#include #include #include #include @@ -134,9 +133,7 @@ QString schema() .arg(columns(FolioData), columns(ElementData), columns(ConductorData)); } - //The tables of a document's engineering data. While the project - //database's own tables were in the same file they were named - //doc_ (stage D3) + //The tables of a document's engineering data const QStringList DocTables = { QStringLiteral("strip_bridge_terminal"), QStringLiteral("strip_bridge"), QStringLiteral("strip_terminal"), QStringLiteral("terminal_strip"), @@ -144,26 +141,16 @@ const QStringList DocTables = { QStringLiteral("element_info"), QStringLiteral("element"), QStringLiteral("link"), QStringLiteral("conductor"), QStringLiteral("meta")}; - //Owns an sqlite3 handle; its SQL names the document's tables by their - //plain names, given their prefix here, and its columns as the format of - //the database read names them + //Owns an sqlite3 handle; names its columns as the format of the + //database read names them class Database { public: ~Database() { if (m_db) sqlite3_close(m_db); } sqlite3 *m_db = nullptr; - QString m_prefix; int m_format = QetContainer::Format; - QString sql(const QString &text) const { - if (m_prefix.isEmpty()) return text; - static const QRegularExpression names(QStringLiteral("(? - { - Statement doc(db, QStringLiteral("SELECT count(*) FROM sqlite_master WHERE name = 'doc_meta'")); - if (doc.next() && doc.value(0).toInt() > 0) db.m_prefix = QStringLiteral("doc_"); - } - for (const char *table : {"meta", "project_property", "folio", "folio_property", - "element", "element_info", "link", "conductor", "terminal_strip", - "strip_terminal", "strip_bridge", "strip_bridge_terminal"}) { + for (const QString &table : DocTables) { Statement kind(db, QStringLiteral("SELECT type FROM sqlite_master WHERE name = ?")); - kind.bind({db.table(QString::fromLatin1(table))}); + kind.bind({table}); if (!kind.next() || kind.value(0).toString() != QLatin1String("table")) { - setError(error, QStringLiteral("the project database has no table %1").arg(QLatin1String(table))); + setError(error, QStringLiteral("the project database has no table %1").arg(table)); return false; } } @@ -755,28 +734,10 @@ bool restoreDiagram(Database &db, QDomElement d, QString *error) folios still in it) into a new database. @return the database file, or an empty array with @p error set */ -QByteArray QetContainerDb::extract(QDomElement project, QString *error, const QByteArray &index, - const QByteArray &kept_from) +QByteArray QetContainerDb::extract(QDomElement project, QString *error, const QByteArray &kept_from) { Database db; - if (sqlite3_open(":memory:", &db.m_db) != SQLITE_OK) { - setError(error, QStringLiteral("cannot create the project database")); - return {}; - } - //The project database's own tables first, when given: one file - //holds both (stage D3), the document's named doc_ - if (!index.isEmpty()) { - db.m_prefix = QStringLiteral("doc_"); - unsigned char *copy = static_cast(sqlite3_malloc64(sqlite3_uint64(index.size()))); - if (!copy) { setError(error, QStringLiteral("out of memory")); return {}; } - memcpy(copy, index.constData(), size_t(index.size())); - if (sqlite3_deserialize(db.m_db, "main", copy, index.size(), index.size(), - SQLITE_DESERIALIZE_FREEONCLOSE | SQLITE_DESERIALIZE_RESIZEABLE) != SQLITE_OK) { - setError(error, QStringLiteral("cannot read the project database's index")); - return {}; - } - } - if (!db.exec(schema())) { + if (sqlite3_open(":memory:", &db.m_db) != SQLITE_OK || !db.exec(schema())) { setError(error, QStringLiteral("cannot create the project database")); return {}; } @@ -795,7 +756,7 @@ QByteArray QetContainerDb::extract(QDomElement project, QString *error, const QB db.exec(QStringLiteral("PRAGMA trusted_schema = OFF")); for (const QString &table : DocTables) { Statement kind(db, QStringLiteral("SELECT type FROM old.sqlite_master WHERE name = ?")); - kind.bind({db.table(table)}); + kind.bind({table}); if (!kind.next() || kind.value(0).toString() != QLatin1String("table")) { setError(error, QStringLiteral("the kept folios' database has no table %1").arg(table)); return {}; @@ -1105,58 +1066,6 @@ bool QetContainerDb::restore(QDomElement project, const QByteArray &database, QS return true; } -/** - @brief QetContainerDb::restoreFolio - restore() for the one folio @p diagram, read from its own part (a folio - built after its project was opened from a .qetz, stage D2) -*/ -bool QetContainerDb::restoreFolio(QDomElement diagram, const QByteArray &database, QString *error) -{ - Database db; - return openDatabase(db, database, error) && restoreDiagram(db, diagram, error); -} - -struct QetContainerDb::FolioReader::Private -{ - QByteArray database; - Database db; - bool opened = false; - QString open_error; -}; - -/** - @brief QetContainerDb::FolioReader::FolioReader - Reads folios' engineering data from @p database, opened the first time - one is read and kept open: a folio built after the project was opened - from a .qetz no longer copies the whole database (stage D5) -*/ -QetContainerDb::FolioReader::FolioReader(const QByteArray &database) : - d(std::make_unique()) -{ - d->database = database; -} - -QetContainerDb::FolioReader::~FolioReader() = default; - -/** - @brief QetContainerDb::FolioReader::restore - restoreFolio() for @p diagram -*/ -bool QetContainerDb::FolioReader::restore(QDomElement diagram, QString *error) -{ - if (!d->opened) { - d->opened = true; - if (!openDatabase(d->db, d->database, &d->open_error) && d->open_error.isEmpty()) - d->open_error = QStringLiteral("the project database cannot be read"); - d->database = QByteArray(); - } - if (!d->open_error.isEmpty()) { - setError(error, d->open_error); - return false; - } - return restoreDiagram(d->db, diagram, error); -} - /** @brief QetContainerDb::currentFormat @return true if @p database, a project.sqlite, is of the format this diff --git a/sources/container/qetcontainerdb.h b/sources/container/qetcontainerdb.h index 5e34f307d..4f4b31408 100644 --- a/sources/container/qetcontainerdb.h +++ b/sources/container/qetcontainerdb.h @@ -21,8 +21,6 @@ #include #include -#include - /** @brief The QetContainerDb namespace The engineering data of a project, moved between its document and the @@ -42,21 +40,8 @@ namespace QetContainerDb { QByteArray extract(QDomElement project, QString *error = nullptr, - const QByteArray &index = QByteArray(), const QByteArray &kept_from = QByteArray()); bool restore(QDomElement project, const QByteArray &database, QString *error = nullptr); - bool restoreFolio(QDomElement diagram, const QByteArray &database, QString *error = nullptr); - /// restoreFolio() for the folios of one database, opened once - class FolioReader - { - public: - explicit FolioReader(const QByteArray &database); - ~FolioReader(); - bool restore(QDomElement diagram, QString *error = nullptr); - private: - struct Private; - std::unique_ptr d; - }; /// true if @p database is of the format this writes bool currentFormat(const QByteArray &database); } diff --git a/sources/dataBase/projectdatabase.cpp b/sources/dataBase/projectdatabase.cpp index 6dd82ea82..b688865b9 100644 --- a/sources/dataBase/projectdatabase.cpp +++ b/sources/dataBase/projectdatabase.cpp @@ -49,10 +49,8 @@ #include #include #include -#include #include #include -#include #include #include #include @@ -896,375 +894,6 @@ bool projectDataBase::populateFromDocument(const QDomDocument &document, QString return true; } -/** - @brief projectDataBase::schemaStamp - @return a stamp of the tables of this database as created: an index - file (indexFile()) is read back only into a database with the same -*/ -QString projectDataBase::schemaStamp() const -{ - QSqlQuery query(m_data_base); - query.exec(QStringLiteral("SELECT sql FROM sqlite_master WHERE type IN ('table', 'index') " - "AND name NOT LIKE 'sqlite_%' ORDER BY name")); - QCryptographicHash hash(QCryptographicHash::Sha256); - while (query.next()) - hash.addData(query.value(0).toString().toUtf8() + ';'); - return QString::fromLatin1(hash.result().toHex()); -} - -namespace { - //The tables an index file carries, all of this database's but the - //drawing items, which exist for built folios only -QStringList indexTables(const QSqlDatabase &db) -{ - QStringList tables; - QSqlQuery query(db); - query.exec(QStringLiteral("SELECT name FROM sqlite_master WHERE type = 'table' " - "AND name NOT LIKE 'sqlite_%' ORDER BY name")); - while (query.next()) { - const QString name = query.value(0).toString(); - if (name != QLatin1String("shape") && name != QLatin1String("independent_text") - && name != QLatin1String("image")) - tables << name; - } - return tables; -} -} - -namespace { - //A wire's properties read from @p attributes, as a saved wire's are -ConductorProperties propertiesFrom(const QHash &attributes) -{ - QDomDocument scratch; - QDomElement e = scratch.createElement(QStringLiteral("conductor")); - for (auto it = attributes.cbegin() ; it != attributes.cend() ; ++it) - e.setAttribute(it.key(), it.value()); - ConductorProperties properties; - properties.fromXml(e); - return properties; -} - - //@p attributes without those reading them would give anyway: each - //left out in turn when the properties read without it are the same - //(stage D3b) -QHash fewestAttributes(QHash attributes) -{ - const ConductorProperties full = propertiesFrom(attributes); - QStringList names = attributes.keys(); - names.sort(); - for (const QString &name : std::as_const(names)) { - QHash without = attributes; - without.remove(name); - const ConductorProperties read = propertiesFrom(without); - if (read == full && read.attributes() == full.attributes()) - attributes = without; - } - return attributes; -} -} - -/** - @brief projectDataBase::indexFile - This database written to an SQLite file, for a .qetz to be opened from - (stage D2): every table but the drawing items, brought - up to date first, with the folios' own data @p folio_headers (in folio - order, each as Diagram::headerXml() writes it), the drawn tables - @p tables and how they continue one another (which folios a folio's - tables must be built with), and the stamp of the schema it was written - with. - @return the file, or nothing if it could not be written -*/ -QByteArray projectDataBase::indexFile(const QList> &folio_headers, - const QList &tables) -{ - if (m_content_changed) { - updateDB(); - } - flushLinks(); - flushElementPositions(); - flushConductorProperties(); - - QTemporaryDir dir; - if (!dir.isValid()) { - return QByteArray(); - } - const QString path = dir.filePath(QStringLiteral("index.sqlite")); - QString escaped = path; - escaped.replace(QLatin1Char('\''), QStringLiteral("''")); - QSqlQuery vacuum(m_data_base); - if (!vacuum.exec(QStringLiteral("VACUUM INTO '%1'").arg(escaped))) { - qWarning() << "projectDataBase::indexFile:" << vacuum.lastError().text(); - return QByteArray(); - } - const QString stamp = schemaStamp(); - const QString connection = QStringLiteral("qet_index_") + QUuid::createUuid().toString(); - bool ok = false; - { - QSqlDatabase file = QSqlDatabase::addDatabase(QStringLiteral("QSQLITE"), connection); - file.setDatabaseName(path); - if (file.open()) - { - QSqlQuery query(file); - //Its views read the drawing items too: an index holds tables - QStringList views; - query.exec(QStringLiteral("SELECT name FROM sqlite_master WHERE type = 'view'")); - while (query.next()) views << query.value(0).toString(); - for (const QString &view : std::as_const(views)) - query.exec(QStringLiteral("DROP VIEW \"%1\"").arg(view)); - ok = query.exec(QStringLiteral("DROP TABLE shape")) - && query.exec(QStringLiteral("DROP TABLE independent_text")) - && query.exec(QStringLiteral("DROP TABLE image")) - && query.exec(QStringLiteral("CREATE TABLE index_meta (key TEXT PRIMARY KEY, value TEXT)")) - && query.exec(QStringLiteral("CREATE TABLE folio_header (pos INTEGER PRIMARY KEY, " - "uuid TEXT NOT NULL, xml TEXT NOT NULL)")) - && query.exec(QStringLiteral("CREATE TABLE folio_table (uuid TEXT NOT NULL, " - "folio_uuid TEXT NOT NULL, previous_uuid TEXT)")); - file.transaction(); - query.prepare(QStringLiteral("INSERT INTO index_meta (key, value) VALUES (?, ?)")); - for (const auto &meta : {std::pair{QStringLiteral("schema"), stamp}, - std::pair{QStringLiteral("writer"), - QStringLiteral("QElectroTech ") + QetVersion::currentVersion().toString()}}) { - query.addBindValue(meta.first); - query.addBindValue(meta.second); - ok = ok && query.exec(); - } - query.prepare(QStringLiteral("INSERT INTO folio_header (pos, uuid, xml) VALUES (?, ?, ?)")); - int pos = 0; - for (const auto &header : folio_headers) { - query.addBindValue(pos++); - query.addBindValue(header.first.toString()); - query.addBindValue(header.second); - ok = ok && query.exec(); - } - query.prepare(QStringLiteral("INSERT INTO folio_table (uuid, folio_uuid, previous_uuid) " - "VALUES (?, ?, ?)")); - for (const FolioTable &table : tables) { - query.addBindValue(table.table.toString()); - query.addBindValue(table.folio.toString()); - query.addBindValue(table.previous.isNull() ? QVariant() : QVariant(table.previous.toString())); - ok = ok && query.exec(); - } - //A wire's properties: only the attributes reading them needs, - //by wire and name (fillFromIndex() writes the others back) - { - QMap> wires; - QSqlQuery rows(file); - rows.exec(QStringLiteral("SELECT conductor_uuid, name, value FROM conductor_properties")); - while (rows.next()) - wires[rows.value(0).toString()].insert(rows.value(1).toString(), rows.value(2).toString()); - ok = ok && query.exec(QStringLiteral("DELETE FROM conductor_properties")); - query.prepare(QStringLiteral("INSERT INTO conductor_properties (conductor_uuid, name, value) " - "VALUES (?, ?, ?)")); - for (auto wire = wires.cbegin() ; wire != wires.cend() ; ++wire) { - const QHash fewest = fewestAttributes(wire.value()); - QStringList names = fewest.keys(); - names.sort(); - for (const QString &name : std::as_const(names)) { - query.addBindValue(wire.key()); - query.addBindValue(name); - query.addBindValue(fewest.value(name)); - ok = ok && query.exec(); - } - } - } - ok = file.commit() && ok; - ok = query.exec(QStringLiteral("VACUUM")) && ok; - file.close(); - } - } - QSqlDatabase::removeDatabase(connection); - if (!ok) { - return QByteArray(); - } - QFile file(path); - if (!file.open(QIODevice::ReadOnly)) { - return QByteArray(); - } - return file.readAll(); -} - -/** - @brief projectDataBase::discardFill - Undo fillFromIndex(): the project is read from its document instead -*/ -void projectDataBase::discardFill() -{ - QSqlQuery query(m_data_base); - for (const QString &table : indexTables(m_data_base)) - query.exec(QStringLiteral("DELETE FROM \"%1\"").arg(table)); - m_element_information.clear(); - m_folio_titleblocks.clear(); - m_conductor_properties.clear(); - m_conductor_attributes.clear(); - m_prefilled_elements.clear(); - m_prefilled_folios.clear(); - m_prefilled_conductors.clear(); - m_rows_from_document = false; - m_content_changed = true; -} - -/** - @brief projectDataBase::fillFromIndex - Fill this database from @p index, a file indexFile() wrote, instead of - from the project's document: its tables, and the stores of symbol - information and wire properties (the folios' title blocks are filled - from @p folio_headers by the folios). The file is untrusted: read only - when written with this database's schema, its tables plain tables, and - nothing in its schema run. - @param folio_headers : the folios' own data, in folio order - @param folio_tables : if not null, the drawn tables indexFile() wrote, or - nothing if the index predates them - @return false, with @p why set and nothing changed in the tables, if - the file cannot be read -*/ -bool projectDataBase::fillFromIndex(const QByteArray &index, - QList> *folio_headers, - QString *why, - std::optional> *folio_tables) -{ - auto refuse = [why](const QString &reason) { - if (why) *why = reason; - return false; - }; - QTemporaryDir dir; - if (!dir.isValid()) { - return refuse(QStringLiteral("no temporary folder")); - } - const QString path = dir.filePath(QStringLiteral("index.sqlite")); - { - QFile file(path); - if (!file.open(QIODevice::WriteOnly) || file.write(index) != index.size()) { - return refuse(QStringLiteral("cannot write the index to a temporary file")); - } - } - QString escaped = path; - escaped.replace(QLatin1Char('\''), QStringLiteral("''")); - QSqlQuery query(m_data_base); - query.exec(QStringLiteral("PRAGMA trusted_schema = OFF")); - if (!query.exec(QStringLiteral("ATTACH DATABASE '%1' AS idx").arg(escaped))) { - return refuse(QStringLiteral("the index cannot be read")); - } - auto detach = [this]() { - QSqlQuery q(m_data_base); - q.exec(QStringLiteral("DETACH DATABASE idx")); - }; - - const QStringList tables = indexTables(m_data_base); - auto isTable = [this](const QString &name) { - QSqlQuery kind(m_data_base); - kind.prepare(QStringLiteral("SELECT type FROM idx.sqlite_master WHERE name = ?")); - kind.addBindValue(name); - return kind.exec() && kind.next() && kind.value(0).toString() == QLatin1String("table"); - }; - for (const QString &table : tables + QStringList{QStringLiteral("index_meta"), - QStringLiteral("folio_header")}) { - if (!isTable(table)) { - detach(); - return refuse(QStringLiteral("the index has no table %1").arg(table)); - } - } - query.exec(QStringLiteral("SELECT value FROM idx.index_meta WHERE key = 'schema'")); - if (!query.next() || query.value(0).toString() != schemaStamp()) { - detach(); - return refuse(QStringLiteral("the index was written with another database schema")); - } - - m_data_base.transaction(); - bool ok = true; - for (const QString &table : tables) { - QSqlQuery columns(m_data_base); - columns.exec(QStringLiteral("PRAGMA main.table_info(\"%1\")").arg(table)); - QStringList names; - while (columns.next()) - names << QStringLiteral("\"%1\"").arg(columns.value(1).toString()); - const QString list = names.join(QLatin1Char(',')); - ok = ok && query.exec(QStringLiteral("DELETE FROM main.\"%1\"").arg(table)) - && query.exec(QStringLiteral("INSERT INTO main.\"%1\" (%2) SELECT %2 FROM idx.\"%1\"") - .arg(table, list)); - } - if (!ok) { - m_data_base.rollback(); - detach(); - return refuse(QStringLiteral("the index's rows cannot be read")); - } - m_data_base.commit(); - - folio_headers->clear(); - query.exec(QStringLiteral("SELECT uuid, xml FROM idx.folio_header ORDER BY pos")); - while (query.next()) - folio_headers->append({QUuid(query.value(0).toString()), query.value(1).toString()}); - if (folio_tables) { - folio_tables->reset(); - if (isTable(QStringLiteral("folio_table"))) { - QList read; - query.exec(QStringLiteral("SELECT uuid, folio_uuid, previous_uuid FROM idx.folio_table")); - while (query.next()) - read.append({QUuid(query.value(0).toString()), QUuid(query.value(1).toString()), - QUuid(query.value(2).toString())}); - *folio_tables = read; - } - } - detach(); - - //The stores, from their tables, as the document would fill them - m_prefilled_elements.clear(); - m_prefilled_folios.clear(); - m_prefilled_conductors.clear(); - for (PrefillTally &tally : m_prefill_tally) tally = PrefillTally(); - query.exec(QStringLiteral("SELECT element_uuid, name, value, show FROM element_information " - "ORDER BY element_uuid, ord")); - while (query.next()) { - const QUuid uuid(query.value(0).toString()); - m_prefilled_elements[uuid].addValue(query.value(1).toString(), query.value(2).toString(), - query.value(3).toInt() != 0); - } - QHash> attributes; - QList wire_order; - query.exec(QStringLiteral("SELECT conductor_uuid, name, value FROM conductor_properties ORDER BY rowid")); - while (query.next()) { - const QUuid wire(query.value(0).toString()); - if (!attributes.contains(wire)) wire_order << wire; - attributes[wire].insert(query.value(1).toString(), query.value(2).toString()); - } - //A file may hold only the attributes reading needs (indexFile()): - //the table gets every attribute a saved wire carries, as before - query.exec(QStringLiteral("DELETE FROM conductor_properties")); - query.prepare(QStringLiteral("INSERT INTO conductor_properties (conductor_uuid, name, value) VALUES (?, ?, ?)")); - for (const QUuid &wire : std::as_const(wire_order)) { - const ConductorProperties properties = propertiesFrom(attributes.value(wire)); - m_prefilled_conductors.insert(wire, properties); - QHash &full = attributes[wire]; - full.clear(); - for (const auto &attribute : properties.attributes()) { - full.insert(attribute.first, attribute.second); - query.addBindValue(wire.toString()); - query.addBindValue(attribute.first); - query.addBindValue(attribute.second); - query.exec(); - } - } - for (auto it = m_prefilled_elements.cbegin() ; it != m_prefilled_elements.cend() ; ++it) - m_element_information.insert(it.key(), it.value()); - for (auto it = m_prefilled_conductors.cbegin() ; it != m_prefilled_conductors.cend() ; ++it) - m_conductor_properties.insert(it.key(), it.value()); - m_conductor_attributes = attributes; - for (const auto &header : std::as_const(*folio_headers)) { - QDomDocument document; - if (!document.setContent(header.second)) continue; - TitleBlockProperties properties; - properties.fromXml(document.documentElement()); - properties.version = QetVersion::displayedVersion(); - properties.collection = QET::QetCollection::Embedded; - m_prefilled_folios.insert(header.first, properties); - m_folio_titleblocks.insert(header.first, properties); - } - - m_rows_from_document = true; - m_content_changed = false; - emit dataBaseUpdated(); - return true; -} - /** @brief projectDataBase::foliosWithUnjoinedWires @return the uuids of the folios whose document has a wire that building diff --git a/sources/dataBase/projectdatabase.h b/sources/dataBase/projectdatabase.h index ceaeec512..6b5fcc38c 100644 --- a/sources/dataBase/projectdatabase.h +++ b/sources/dataBase/projectdatabase.h @@ -166,17 +166,6 @@ class projectDataBase : public QObject void setBuildingFolio(bool building); bool isBuildingFolio() const {return m_building_folios > 0;} bool rowsFromDocument() const {return m_rows_from_document;} - QString schemaStamp() const; - /// A drawn table of a folio and the table it continues - struct FolioTable { QUuid table; QUuid folio; QUuid previous; }; - QByteArray indexFile(const QList> &folio_headers, - const QList &tables = QList()); - void discardFill(); - /// The tables are right as they are (filled from an index) - void markCurrent() {m_content_changed = false;} - bool fillFromIndex(const QByteArray &index, QList> *folio_headers, - QString *why = nullptr, - std::optional> *folio_tables = nullptr); QString prefillReport() const; QSet foliosWithUnjoinedWires() const; QSet folioUuidsOfElements(const QSet &elements) const; diff --git a/sources/diagram.cpp b/sources/diagram.cpp index 483f2dd62..1eb549fd8 100644 --- a/sources/diagram.cpp +++ b/sources/diagram.cpp @@ -882,20 +882,6 @@ QDomElement Diagram::deferredXmlWithHeader(QDomDocument &document) return folio; } -/** - @brief Diagram::deferredXml - @return the XML of this folio's items while they are not built: kept - from the project's document, or read now from the folio's part of the - .qetz the project was opened from -*/ -QDomElement Diagram::deferredXml() -{ - if (m_deferred.isNull() && !m_deferred_part.isEmpty() && m_project) { - m_deferred = m_project->readFolioPart(m_deferred_part); - } - return m_deferred; -} - /** @brief Diagram::buildDeferred Build this folio's items from the XML kept for later, and nothing else. @@ -908,12 +894,9 @@ bool Diagram::buildDeferred() if (isLoaded()) { return false; } - QDomElement xml = deferredXml(); + QDomElement xml = m_deferred; m_deferred = QDomElement(); - m_deferred_part.clear(); - if (xml.isNull()) { - return true; //its part could not be read: QETProject::readFolioPart() said why - } + m_kept_part.clear(); m_kept_text.clear(); projectDataBase *db = m_project ? m_project->dataBase() : nullptr; if (db) db->setBuildingFolio(true); diff --git a/sources/diagram.h b/sources/diagram.h index 5785b7efd..233ba48d9 100644 --- a/sources/diagram.h +++ b/sources/diagram.h @@ -142,7 +142,7 @@ class Diagram : public QGraphicsScene QUuid m_uuid = QUuid::createUuid(); /// The XML of this folio's items, while they are not built yet QDomElement m_deferred; - QString m_deferred_part; + QString m_kept_part; QHash m_partial_elements; QHash m_partial_table; bool m_partial_changed = false; @@ -218,13 +218,13 @@ class Diagram : public QGraphicsScene DiagramContent * = nullptr); /// false while this folio's items wait in its XML to be /// built (lazy folios) - bool isLoaded() const {return m_deferred.isNull() && m_deferred_part.isEmpty();} - QDomElement deferredXml(); - /// The folio's items are in the part @p file of the .qetz the - /// project was opened from, read when they are first needed - /// (stage D2) - void setDeferredPart(const QString &file) {m_deferred = QDomElement(); m_deferred_part = file;} - QString deferredPart() const {return m_deferred_part;} + bool isLoaded() const {return m_deferred.isNull();} + QDomElement deferredXml() const {return m_deferred;} + /// The part @p file of the .qetz the project was opened from + /// holds this folio as it is while not built: saving may copy it + /// (QETProject::toXml(), stage D4) + void setKeptPart(const QString &file) {m_kept_part = file;} + QString keptPart() const {return m_kept_part;} QList buildSymbols(const QSet &uuids); /// Symbols built as partners (buildSymbols()) changed since: /// the folio must be built to be saved diff --git a/sources/qetproject.cpp b/sources/qetproject.cpp index e5600445c..afd94ab65 100644 --- a/sources/qetproject.cpp +++ b/sources/qetproject.cpp @@ -479,48 +479,27 @@ QETProject::ProjectState QETProject::openFile(QFile *file) // QetContainer::parse() does exactly that, for a .qet and for each part // of a .qetz alike. QString read_error; - //A .qetz holding the index of its database opens from it, the - //folios' parts read when they are built (stage D2); - //any other, or one whose index cannot be used, is read whole - m_opened_from_index = false; - if (QetContainer::isZip(content) && lazyFolios()) - { + //A .qetz: its entries are kept, for a save to copy the parts of the + //folios not built (stage D4) + m_container_files.clear(); + m_folio_parts.clear(); + bool read = false; + if (QetContainer::isZip(content)) { QList entries; - QString why; - if (QetZip::fromBytes(content, &entries, &why)) - { - QHash files; + read = QetZip::fromBytes(content, &entries, &read_error) + && QetContainer::join(entries, &xml_project, &read_error, &m_folio_parts, + &m_container_pictures); + if (read) for (const QetZip::Entry &entry : std::as_const(entries)) - files.insert(entry.name, entry.data); - QStringList folio_files; - QDomDocument document; - //The index: in project.sqlite from format 3, its own entry - //before - const QByteArray index = files.contains(QStringLiteral("index.sqlite")) - ? files.value(QStringLiteral("index.sqlite")) - : files.value(QStringLiteral("project.sqlite")); - if (!index.isEmpty() && index.contains("folio_header") - && QetContainer::join(entries, &document, &why, &folio_files)) - { - if (m_data_base.fillFromIndex(index, &m_folio_headers, &why, &m_index_tables) - && m_folio_headers.size() == folio_files.size()) { - xml_project = document; - m_container_files = files; - m_folio_files = folio_files; - m_opened_from_index = true; - } else { - m_data_base.discardFill(); - m_folio_headers.clear(); - m_index_tables.reset(); - qInfo().noquote() << "Not opened from the index of" << fi.fileName() << ":" << why; - } - } - } + m_container_files.insert(entry.name, entry.data); + } else { + read = QetContainer::readDocument(content, &xml_project, &read_error); } - if (!m_opened_from_index && !QetContainer::readDocument(content, &xml_project, &read_error, &m_container_pictures)) + if (!read) { qWarning().noquote() << "Unable to read" << fi.fileName() << ":" << read_error; m_container_pictures.clear(); + m_folio_parts.clear(); if(opened_here) { file->close(); } @@ -627,9 +606,8 @@ QList QETProject::folios() const folio when the project has cables (a core is followed through folio reports to any folio); without cables, the folios built and those not built yet holding a wire that still refers to a cable -- the only - wires the pass changes then, taking the reference away. Read from the - project database for a .qetz opened from it, from the folios' XML - otherwise. + wires the pass changes then, taking the reference away, read from the + folios' XML. */ QList QETProject::cableLabelFolios() { @@ -637,25 +615,14 @@ QList QETProject::cableLabelFolios() return diagrams(); } QList referring; - if (m_opened_from_index) { - QSet folios; - QSqlQuery query = m_data_base.newQuery(QStringLiteral( - "SELECT DISTINCT c.diagram_uuid FROM conductor c JOIN conductor_properties p " - "ON p.conductor_uuid = c.uuid WHERE p.name = 'cable_uuid' AND p.value != ''")); - if (query.exec()) - while (query.next()) folios.insert(QUuid(query.value(0).toString())); - for (Diagram *d : std::as_const(m_diagrams_list)) - if (!d->isLoaded() && folios.contains(d->uuid())) referring << d; - } else { - for (Diagram *d : std::as_const(m_diagrams_list)) { - if (d->isLoaded()) continue; - for (const QDomElement &wire : QET::findInDomElement(d->deferredXml(), QStringLiteral("conductors"), - QStringLiteral("conductor"))) - if (!QUuid(wire.attribute(QStringLiteral("cable_uuid"))).isNull()) { - referring << d; - break; - } - } + for (Diagram *d : std::as_const(m_diagrams_list)) { + if (d->isLoaded()) continue; + for (const QDomElement &wire : QET::findInDomElement(d->deferredXml(), QStringLiteral("conductors"), + QStringLiteral("conductor"))) + if (!QUuid(wire.attribute(QStringLiteral("cable_uuid"))).isNull()) { + referring << d; + break; + } } buildFolios(referring); return builtFolios(); @@ -875,9 +842,8 @@ void QETProject::buildFoliosHoldingSymbols(const QSet &symbols) return; } QList to_build; - //Opened from an index, the database knows where every symbol is - //without reading the folios' parts - if (m_opened_from_index || m_data_base.rowsFromDocument()) { + //Filled from the document, the database knows where every symbol is + if (m_data_base.rowsFromDocument()) { const QSet folios = m_data_base.folioUuidsOfElements(symbols); for (Diagram *d : std::as_const(m_diagrams_list)) if (!d->isLoaded() && folios.contains(d->uuid())) @@ -951,16 +917,6 @@ QSet QETProject::tableChainFolios(Diagram *folio) const } } continue; - } - //Opened from an index: its tables, not the folio's part - if (m_index_tables) { - for (const projectDataBase::FolioTable &table : std::as_const(*m_index_tables)) { - if (table.folio == d->uuid()) { - folio_of.insert(table.table, d); - previous_of.insert(table.table, table.previous); - } - } - continue; } const QDomNodeList tables = d->deferredXml().elementsByTagName( QetGraphicsTableItem::xmlTagName()); @@ -2365,7 +2321,7 @@ void QETProject::autoFolioNumberingSelectedFolios(int from, */ QDomDocument QETProject::toXml(QHash *kept_folios, QStringList *kept_parts) { - //With @p kept_parts, the same for a .qetz opened from its index: a + //With @p kept_parts, for a .qetz: a //folio not built is written as , its //part and its rows copied by QetContainer::split() //(stage D4) @@ -2556,7 +2512,7 @@ QDomDocument QETProject::toXml(QHash *kept_folios, QStringList } if (keep && !diagram->isLoaded() && kept_parts) { auto placeholder = xml_doc.createElement(QStringLiteral("diagram")); - placeholder.setAttribute(QStringLiteral("qetz-kept-part"), diagram->deferredPart()); + placeholder.setAttribute(QStringLiteral("qetz-kept-part"), diagram->keptPart()); placeholder.setAttribute(QStringLiteral("uuid"), diagram->uuid().toString()); //Symbols whose information changed while the folio was //not built: the rows QetContainer::split() writes for them @@ -2579,7 +2535,7 @@ QDomDocument QETProject::toXml(QHash *kept_folios, QStringList wire.setAttribute(attribute.first, attribute.second); placeholder.appendChild(wire); } - kept_parts->append(diagram->deferredPart()); + kept_parts->append(diagram->keptPart()); project_root.appendChild(placeholder); ++order_num; continue; @@ -2682,24 +2638,18 @@ bool QETProject::isContainerPath(const QString &path) the whole archive is written. */ bool QETProject::writeContainer(const QDomDocument &project, const QString &path, QString *error, - const QByteArray &index, const QHash &kept_from) + const QHash &kept_from) { - //The project database, for opening without reading the folios - //(stage D2), is in project.sqlite with the - //document's engineering data (format 3, D3) QList entries = QetContainer::split( project, QStringLiteral("QElectroTech %1").arg(QetVersion::currentVersion().toString()), - error, index, kept_from); + error, kept_from); return !entries.isEmpty() && QetZip::write(path, entries, error); } /** @brief QETProject::writeContainerFile - Write this project to @p path as a .qetz, with the index of its - database a .qetz is opened from (stage D2) when every - folio, symbol and wire has a uuid of its own -- what opening from the - index relies on, as opening without building folios does. + Write this project to @p path as a .qetz */ bool QETProject::writeContainerFile(const QString &path, QString *error) { @@ -2707,67 +2657,10 @@ bool QETProject::writeContainerFile(const QString &path, QString *error) //what building them would write (toXml()); every other folio built QStringList kept; const QDomDocument document = toXml(nullptr, &kept); - return writeContainer(document, path, error, containerIndex(), + return writeContainer(document, path, error, kept.isEmpty() ? QHash() : m_container_files); } -/** - @brief QETProject::containerIndex - @return the index of this project's database for a .qetz, empty when a - folio, symbol or wire has no uuid of its own -*/ -QByteArray QETProject::containerIndex() -{ - //Opened from an index, the folios not built have their rows in the - //database, whose keys are unique; the built ones are looked at - if (!m_kept_parts_trusted) { - loadFolios(); - } - QSet seen; - QList> headers; - auto unique = [&seen](const QUuid &uuid) { - if (uuid.isNull() || seen.contains(uuid)) return false; - seen.insert(uuid); - return true; - }; - //The drawn tables and the tables they continue: which folios a - //folio's tables are built with (tableChainFolios()) - QList tables; - for (Diagram *diagram : std::as_const(m_diagrams_list)) { - if (!unique(diagram->uuid())) return QByteArray(); - if (!diagram->isLoaded()) { - headers.append({diagram->uuid(), diagram->headerXml()}); - if (m_index_tables) { - for (const projectDataBase::FolioTable &table : std::as_const(*m_index_tables)) - if (table.folio == diagram->uuid()) - tables.append(table); - continue; - } - const QDomNodeList items = diagram->deferredXml().elementsByTagName( - QetGraphicsTableItem::xmlTagName()); - for (int i = 0 ; i < items.size() ; ++i) { - const QDomElement table = items.at(i).toElement(); - tables.append({QUuid(table.attribute(QStringLiteral("uuid"))), diagram->uuid(), - QUuid(table.firstChildElement(QStringLiteral("previous_table")) - .attribute(QStringLiteral("uuid")))}); - } - continue; - } - for (Element *element : diagram->elements()) - if (!unique(element->uuid())) return QByteArray(); - for (Conductor *conductor : diagram->conductors()) - if (!unique(conductor->uuid())) return QByteArray(); - headers.append({diagram->uuid(), diagram->headerXml()}); - for (QGraphicsItem *item : diagram->items()) { - if (item->type() == QetGraphicsTableItem::Type) { - auto table = static_cast(item); - tables.append({table->uuid(), diagram->uuid(), table->previousTableUuid()}); - } - } - } - return m_data_base.indexFile(headers, tables); -} - /** @brief QETProject::besideContainerPath @return the .qetz a project read from a .qet is saved to: the same @@ -3093,8 +2986,7 @@ QDomDocument QETProject::folioXml(Diagram *folio) /** @brief QETProject::buildFoliosHoldingTables Build the folios not built yet (lazy folios) that hold a drawn - table, found from the index the project was opened from when it has - one: what a list of the project's tables needs, and no more + table: what a list of the project's tables needs, and no more */ void QETProject::buildFoliosHoldingTables() { @@ -3114,15 +3006,10 @@ void QETProject::buildFoliosHoldingTables() /** @brief QETProject::unbuiltFolioHasTables @return true if @p folio, not built yet (lazy folios), has a drawn - table: from the index it was opened from, else from its XML + table, read from its XML */ bool QETProject::unbuiltFolioHasTables(Diagram *folio) const { - if (m_index_tables) { - for (const projectDataBase::FolioTable &table : std::as_const(*m_index_tables)) - if (table.folio == folio->uuid()) return true; - return false; - } return !folio->deferredXml().elementsByTagName(QetGraphicsTableItem::xmlTagName()).isEmpty(); } @@ -3408,9 +3295,7 @@ void QETProject::readProjectXml(QDomDocument &xml_project) //The stores of symbol information, folio title blocks and wire //properties are filled from the document before the folios are //built - if (!m_opened_from_index) { - m_data_base.prefillFromDocument(xml_project); - } + m_data_base.prefillFromDocument(xml_project); //Folios are left to be built when needed (lazyFolios()) only //when the database can be filled from the document: otherwise @@ -3426,11 +3311,7 @@ void QETProject::readProjectXml(QDomDocument &xml_project) } //Load the diagrams - if (m_opened_from_index) { - readFoliosFromIndex(); - } else { - readDiagramsXml(xml_project); - } + readDiagramsXml(xml_project); const qint64 diagrams_ms = phase_timer.restart(); //Tie the elements to the numbering schemes they follow @@ -3459,14 +3340,7 @@ void QETProject::readProjectXml(QDomDocument &xml_project) m_data_base.blockSignals(false); m_data_base.setUpdateBlocked(false); - if (m_opened_from_index) { - //Filled from the index already: making the folios changed - //nothing their rows say - m_data_base.markCurrent(); - m_data_base.updateDB(); - } else { - m_data_base.updateDB(xml_project); - } + m_data_base.updateDB(xml_project); m_data_base.endPrefill(unloadedFolioCount() > 0); //Saving may write the folios not built as they were read, when //building them would write the same: a file written by this @@ -3482,10 +3356,13 @@ void QETProject::readProjectXml(QDomDocument &xml_project) for (Diagram *diagram : std::as_const(m_diagrams_list)) if (!diagram->isLoaded() && diagram->keptText().isEmpty()) m_kept_folios_trusted = false; - m_kept_parts_trusted = m_opened_from_index && unloadedFolioCount() > 0 + m_kept_parts_trusted = !m_folio_parts.isEmpty() && unloadedFolioCount() > 0 && m_project_qet_version == QetVersion::currentVersion() && m_data_base.rowsFromDocument() && QetContainerDb::currentFormat(m_container_files.value(QStringLiteral("project.sqlite"))); + for (Diagram *diagram : std::as_const(m_diagrams_list)) + if (!diagram->isLoaded() && diagram->keptPart().isEmpty()) + m_kept_parts_trusted = false; m_kept_folios_stale = false; m_edited_since_open = false; if (m_kept_folios_trusted || m_kept_parts_trusted) @@ -3511,34 +3388,6 @@ void QETProject::readProjectXml(QDomDocument &xml_project) m_state = Ok; } -/** - @brief QETProject::readFoliosFromIndex - readDiagramsXml() for a .qetz opened from its index: each folio made - from its own data as the index has it, its items left in its part - (stage D2) -*/ -void QETProject::readFoliosFromIndex() -{ - for (int i = 0 ; i < m_folio_headers.size() ; ++i) - { - QDomDocument header; - header.setContent(m_folio_headers.at(i).second); - auto diagram = new Diagram(this); - m_diagrams_list << diagram; - connect(&diagram->border_and_titleblock, &BorderTitleBlock::needFolioData, - this, &QETProject::updateDiagramsFolioData); - connect(diagram, &Diagram::usedTitleBlockTemplateChanged, - this, &QETProject::usedTitleBlockTemplateChanged); - m_deferring_folios = true; - QDomElement header_element = header.documentElement(); - diagram->initFromXml(header_element); - m_deferring_folios = false; - diagram->setDeferredPart(m_folio_files.at(i)); - m_data_base.placeFolio(diagram); - } - updateDiagramsFolioData(); -} - /** @brief QETProject::lazyFolios @see the declaration @@ -3548,27 +3397,6 @@ bool QETProject::lazyFolios() return qEnvironmentVariableIntValue("QET_EAGER_FOLIOS") != 1; } -/** - @brief QETProject::readFolioPart - @return the folio part @p file of the .qetz this project was opened - from, read now, its engineering data back from project.sqlite; null - if it cannot be read (said in the log) -*/ -QDomElement QETProject::readFolioPart(const QString &file) -{ - QDomDocument folio; - QString error; - if (!m_folio_reader) - m_folio_reader = std::make_unique( - m_container_files.value(QStringLiteral("project.sqlite"))); - ++m_folio_parts_read; - if (!QetContainer::readFolio(m_container_files, file, &folio, &error, m_folio_reader.get())) { - qWarning().noquote() << "Unable to read the folio" << file << ":" << error; - return QDomElement(); - } - return folio.documentElement(); -} - /** @brief QETProject::readDiagramsXml Load the diagrams from the xml description of the project. @@ -3665,6 +3493,8 @@ void QETProject::readDiagramsXml(QDomDocument &xml_project) m_deferring_folios = false; if (!diagram->isLoaded() && !line_starts.isEmpty()) diagram->setKeptText(folio_text(diagram_xml_element)); + if (!diagram->isLoaded() && m_folio_parts.size() == diagram_nodes.length()) + diagram->setKeptPart(m_folio_parts.at(i)); m_data_base.placeFolio(diagram); if(dlgWaiting) dlgWaiting->setDetail(diagram->title()); diff --git a/sources/qetproject.h b/sources/qetproject.h index a76a5d60f..b98e9cf0a 100644 --- a/sources/qetproject.h +++ b/sources/qetproject.h @@ -460,14 +460,9 @@ class QETProject : public QObject QString besideContainerPath() const; QETResult writeCopyForOlderVersions(const QString &path); static bool writeContainer(const QDomDocument &project, const QString &path, - QString *error = nullptr, const QByteArray &index = QByteArray(), + QString *error = nullptr, const QHash &kept_from = QHash()); bool writeContainerFile(const QString &path, QString *error = nullptr); - QByteArray containerIndex(); - QDomElement readFolioPart(const QString &file); - /// How many folio parts of the .qetz were read so far - int folioPartsRead() const {return m_folio_parts_read;} - bool openedFromIndex() const {return m_opened_from_index;} private: static QUuid derivedUuid(const QByteArray &content); void refresh(); @@ -555,22 +550,14 @@ class QETProject : public QObject bool m_kept_folios_trusted = false; /// Read from a .qet whose path nobody chose since: saved beside it bool m_opened_from_qet = false; - /// Opened from a .qetz's index (stage D2): the - /// database filled from it, each folio's items in its part - bool m_opened_from_index = false; /// Saving a .qetz may copy the part of a folio not built /// (toXml(), stage D4) bool m_kept_parts_trusted = false; QSet m_waiting_link_targets; + /// The entries of the .qetz opened, and the part each folio + /// was read from, in order QHash m_container_files; - ///Reads the folio parts' engineering data, opened once - std::unique_ptr m_folio_reader; - int m_folio_parts_read = 0; - QStringList m_folio_files; - QList> m_folio_headers; - ///The drawn tables of the index opened from, if it has them - std::optional> m_index_tables; - void readFoliosFromIndex(); + QStringList m_folio_parts; bool m_kept_folios_stale = false; /// Something was edited since the project was opened: a table /// filled from the database may have changed diff --git a/tests/qttest/CMakeLists.txt b/tests/qttest/CMakeLists.txt index 18f285472..a71e43618 100644 --- a/tests/qttest/CMakeLists.txt +++ b/tests/qttest/CMakeLists.txt @@ -1060,10 +1060,13 @@ endif() add_executable( tst_resaveunchanged - tst_resaveunchanged.cpp) + tst_resaveunchanged.cpp + ${QET_DIR}/sources/container/qetzip.cpp + ${QET_DIR}/sources/container/miniz/miniz.c) add_test(NAME tst_resaveunchanged COMMAND tst_resaveunchanged) add_dependencies(tst_resaveunchanged qelectrotech) -target_link_libraries(tst_resaveunchanged PRIVATE Qt::Test) +target_include_directories(tst_resaveunchanged PRIVATE ${QET_DIR}/sources) +target_link_libraries(tst_resaveunchanged PRIVATE Qt::Test SQLite3::SQLite3) target_compile_definitions(tst_resaveunchanged PRIVATE "QET_TEST_BINARY_PATH=\"$\"" "QET_EXAMPLES_DIR=\"${QET_DIR}/examples\"") diff --git a/tests/qttest/lazy_folios_probe.cpp b/tests/qttest/lazy_folios_probe.cpp index 9e0d0e1cb..4e888e142 100644 --- a/tests/qttest/lazy_folios_probe.cpp +++ b/tests/qttest/lazy_folios_probe.cpp @@ -288,7 +288,7 @@ static QStringList savedLinks(const QString &qetz, const QString &path) db.setDatabaseName(db_path); check(db.open(), "a .qetz's database opened"); QSqlQuery query(db); - check(query.exec(QStringLiteral("SELECT element_uuid, linked_uuid FROM doc_link ORDER BY element_uuid, ord")), + check(query.exec(QStringLiteral("SELECT element_uuid, linked_uuid FROM link ORDER BY element_uuid, ord")), "saved links read"); while (query.next()) links[query.value(0).toString()] << query.value(1).toString(); @@ -750,8 +750,8 @@ int main(int argc, char **argv) check(opened.unloadedFolioCount() == 0, "a project with a shared uuid opens with every folio built"); check(built_late == 0, "a project with a shared uuid builds its folios as it reads them, not later"); } - //A .qetz opened from its index (stage D2): no - //folio read, each read from its part when shown, drawn and + //A .qetz opened lazily: no folio built but those + //the terminal strip needs, each built when shown, drawn and //linked as when built, saved as the .qet is { const QString zipped = output + "/lazy_folios.qetz"; @@ -759,12 +759,11 @@ int main(int argc, char **argv) check(eager.writeContainerFile(zipped, &error), "saved as a .qetz"); QETProject indexed(zipped); check(indexed.state() == QETProject::Ok, "the .qetz opens"); - check(indexed.openedFromIndex(), "the .qetz opens from its index"); - std::printf(".qetz opened from its index: %d of %d folios not built (strip folios %d)\n", + std::printf(".qetz opened: %d of %d folios not built (strip folios %d)\n", indexed.unloadedFolioCount(), int(folios.size()), int(strip_folios.size())); check(indexed.unloadedFolioCount() == folios.size() - strip_folios.size(), "the .qetz opens building only the folios of the terminal strip's terminals"); - check(numbering(indexed) == numbering(eager), "symbols' numbering from the index as when built"); + check(numbering(indexed) == numbering(eager), "symbols' numbering of a .qetz as when built"); //The drawn items, through the view made of their tables: the //folios built for it, as for the tables (D6: none were) { @@ -809,7 +808,7 @@ int main(int argc, char **argv) if (row.startsWith("diagram")) rows << row; return rows; }; - check(folioRows(indexed) == folioRows(eager), "the folios' rows from the index as when built"); + check(folioRows(indexed) == folioRows(eager), "the folios' rows of a .qetz as when built"); //Folio numbers that count the folios ("%id/%total") const QString totals = output + "/totals.qetz"; qputenv("QET_EAGER_FOLIOS", "1"); @@ -823,20 +822,19 @@ int main(int argc, char **argv) QETProject usual_totals(totals); qunsetenv("QET_EAGER_FOLIOS"); QETProject lazy_totals(totals); - check(lazy_totals.openedFromIndex(), "opened from its index"); + check(lazy_totals.unloadedFolioCount() > 0, "opened lazily"); const QStringList lazy_rows = folioRows(lazy_totals), usual_rows = folioRows(usual_totals); if (lazy_rows != usual_rows) for (int i = 0 ; i < lazy_rows.size() && i < 3 ; ++i) std::printf(" lazily %s\n built %s\n", qPrintable(lazy_rows.at(i).left(140)), qPrintable(usual_rows.value(i).left(140))); - check(lazy_rows == usual_rows, "folio numbers with totals from the index as when built"); + check(lazy_rows == usual_rows, "folio numbers with totals of a .qetz as when built"); } for (int i = 0; i < folios.size(); ++i) same(indexed, i, "shown from the .qetz"); check(indexed.toXml().toString() == eager.toXml().toString(), "the .qetz saves as when built"); //A folio whose table continues one on another folio, shown - //first: the index, not the folios' parts, says which folios - //its table is built with (D5) + //first: the folios its table is built with (D5) int continued = 0; for (int i = 0; i < folios.size(); ++i) { bool continues = false; @@ -852,8 +850,7 @@ int main(int argc, char **argv) check(continued > 0, "a folio whose table continues another"); //Numbering, the searches for symbols and the unused symbols - //are answered from the database: no folio file is read - //but those of the folios built (D5) + //are answered from the database (D5) { QETProject asked(zipped); check(numbering(asked) == numbering(eager), "numbering of a .qetz as when built"); @@ -874,10 +871,8 @@ int main(int argc, char **argv) check(found(search.freeElement(kinds)) == found(usual_search.freeElement(kinds)), "free symbols of a .qetz found as when built"); const int built = int(folios.size()) - asked.unloadedFolioCount(); - std::printf(".qetz asked numbering, searches, unused symbols: %d folio files read, %d folios built\n", - asked.folioPartsRead(), built); + std::printf(".qetz asked numbering, searches, unused symbols: %d folios built\n", built); check(asked.unloadedFolioCount() > 0, "asking a .qetz does not build every folio"); - check(asked.folioPartsRead() == built, "asking a .qetz reads no file of a folio not built"); } //A folio added, moved and removed: no other folio built, the diff --git a/tests/qttest/tst_qetcontainer.cpp b/tests/qttest/tst_qetcontainer.cpp index dd818596a..60986444e 100644 --- a/tests/qttest/tst_qetcontainer.cpp +++ b/tests/qttest/tst_qetcontainer.cpp @@ -70,14 +70,15 @@ class tst_qetcontainer : public QObject } //@p document without what QElectroTech computes, which a .qetz - //does not keep (format 3); tst_resaveunchanged checks that + //does not keep (format 4); tst_resaveunchanged checks that //QElectroTech gives it back static QDomDocument withoutComputed(const QDomDocument &document) { QDomDocument copy = document.cloneNode(true).toDocument(); + const bool own_uuids = QetContainer::ownUuids(copy.documentElement()); for (QDomElement diagram = copy.documentElement().firstChildElement(QStringLiteral("diagram")); !diagram.isNull() ; diagram = diagram.nextSiblingElement(QStringLiteral("diagram"))) - QetContainer::dropComputed(diagram, false); + QetContainer::dropComputed(diagram, own_uuids); return copy; } @@ -243,7 +244,7 @@ private slots: QCOMPARE(count("element_info"), int(original.elementsByTagName(QStringLiteral("elementInformation")).size())); QCOMPARE(count("link"), int(original.elementsByTagName(QStringLiteral("link_uuid")).size())); QCOMPARE(count("conductor"), int(original.elementsByTagName(QStringLiteral("conductor")).size())); - QVERIFY(count("folio_property") + count("doc_project_property") > 0); + QVERIFY(count("folio_property") + count("project_property") > 0); sqlite3_close(db); } diff --git a/tests/qttest/tst_resaveunchanged.cpp b/tests/qttest/tst_resaveunchanged.cpp index 9cceb2b50..72a8186a7 100644 --- a/tests/qttest/tst_resaveunchanged.cpp +++ b/tests/qttest/tst_resaveunchanged.cpp @@ -8,6 +8,10 @@ #include #include +#include "container/qetzip.h" + +#include + // Saving a project that was just saved must change nothing. Two things // made the second save differ from the first, both cleanup done on save // but not on load: @@ -306,6 +310,10 @@ private slots: QVERIFY2(!zipped.isEmpty(), "--resave as .qetz failed"); QVERIFY2(read(zipped).startsWith("PK\x03\x04"), "the .qetz is not a zip"); QVERIFY2(read(zipped).size() < read(straight).size() / 2, "the .qetz is not smaller"); + QCOMPARE(databaseTables(zipped), QStringList({ + "conductor", "element", "element_info", "folio", "folio_property", "link", "meta", + "project_property", "strip_bridge", "strip_bridge_terminal", "strip_terminal", + "terminal_strip"})); const QString back = resave(zipped); QVERIFY2(!back.isEmpty(), "opening the .qetz failed"); @@ -317,6 +325,31 @@ private slots: QCOMPARE(without_save_stamps(read(back)), without_save_stamps(read(straight))); } + // The tables of the database in the .qetz @p path: the document's + // engineering data only, each fact once -- not QElectroTech's own + // project database, built from it when the project opens + static QStringList databaseTables(const QString &path) + { + QList entries; + if (!QetZip::fromBytes(read(path), &entries)) return {QStringLiteral("not a zip")}; + QByteArray database; + for (const QetZip::Entry &entry : std::as_const(entries)) + if (entry.name == QLatin1String("project.sqlite")) database = entry.data; + sqlite3 *db = nullptr; + sqlite3_open(":memory:", &db); + unsigned char *copy = static_cast(sqlite3_malloc64(sqlite3_uint64(database.size()))); + memcpy(copy, database.constData(), size_t(database.size())); + sqlite3_deserialize(db, "main", copy, database.size(), database.size(), SQLITE_DESERIALIZE_FREEONCLOSE); + sqlite3_stmt *list = nullptr; + QStringList tables; + sqlite3_prepare_v2(db, "SELECT name FROM sqlite_master WHERE type = 'table' ORDER BY name", -1, &list, nullptr); + while (list && sqlite3_step(list) == SQLITE_ROW) + tables << QString::fromUtf8(reinterpret_cast(sqlite3_column_text(list, 0))); + sqlite3_finalize(list); + sqlite3_close(db); + return tables; + } + // QElectroTech run in a sandbox of its own with @p args, the store // check on (QET_CHECK_ELEMENT_INFO_STORE); its output QString runChecked(const QStringList &args, bool eager = false)