Speed up opening big projects with lazy folios

tests/qttest/CMakeLists.txt

@@ -87,6 +87,13 @@ if(Python3_Interpreter_FOUND AND CMAKE_GENERATOR STREQUAL "Ninja")
       --source ${CMAKE_CURRENT_SOURCE_DIR}/attached_plain_paste_probe.cpp
       --fixture ${CMAKE_CURRENT_SOURCE_DIR}/fixtures/unlinked_contact_label.qet)
   set_tests_properties(tst_attachedplainpaste PROPERTIES TIMEOUT 300 RUN_SERIAL TRUE)
+  add_test(NAME tst_lazyfolios
+    COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/run_lazy_folios_probe.py
+      --build ${CMAKE_BINARY_DIR} --binary $<TARGET_FILE:qelectrotech>
+      --source ${CMAKE_CURRENT_SOURCE_DIR}/lazy_folios_probe.cpp
+      --fixture ${CMAKE_CURRENT_SOURCE_DIR}/../../examples/industrial.qet)
+  set_tests_properties(tst_lazyfolios PROPERTIES TIMEOUT 600 RUN_SERIAL TRUE
+    ENVIRONMENT "QT_QPA_PLATFORM=offscreen")
   # Deleting a cable, undoing the delete twice and then editing anything
   # has both cable commands tearing down the same cable. Runs under the
   # same objects the application uses, so a second free is caught.

tests/qttest/lazy_folios_probe.cpp

@@ -0,0 +1,579 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+// Lazy folios (QET_LAZY_FOLIOS, LAZY-FOLIO-PLAN.md stages 5.2 and 5.3),
+// linked against the application's objects: a folio shown in a project
+// opened without building its folios draws and links as in the project
+// opened as usual, building only it and the folios its links and tables
+// reach, and the project saves the same.
+#include <QApplication>
+#include <QFile>
+#include <QFontDatabase>
+#include <QImage>
+#include <QSqlRecord>
+#include <QSqlQuery>
+#include <QRegularExpression>
+#include <QPainter>
+#include <QSettings>
+#include <QTemporaryDir>
+#include <cstdio>
+#include <stdexcept>
+#include "../../sources/diagram.h"
+#include "../../sources/qetproject.h"
+#include "../../sources/qetresult.h"
+#include "../../sources/qetmessagebox.h"
+#include "../../sources/qetgraphicsitem/element.h"
+#include "../../sources/autoNum/elementautonumschemecommand.h"
+#include "../../sources/elementprovider.h"
+#include "../../sources/TerminalStrip/terminalstrip.h"
+#include "../../sources/ui/elementpropertieswidget.h"
+#include "../../sources/conductorautonumerotation.h"
+#include "../../sources/diagramcontent.h"
+#include "../../sources/diagramview.h"
+#include "../../sources/diagramevent/diagrameventaddpaste.h"
+#include <QKeyEvent>
+#include "../../sources/qetgraphicsitem/conductor.h"
+#include "../../sources/qetgraphicsitem/terminal.h"
+#include "../../sources/undocommand/changeelementinformationcommand.h"
+#include "../../sources/undocommand/changetitleblockcommand.h"
+#include "../../sources/undocommand/deleteqgraphicsitemcommand.h"
+#include "../../sources/factory/elementfactory.h"
+#include "../../sources/undocommand/addgraphicsobjectcommand.h"
+
+static void check(bool value, const char *message) { if (!value) throw std::runtime_error(message); }
+
+	//Every symbol's numbering, in a stable order
+static QStringList numbering(const QETProject &project)
+{
+	QStringList list;
+	for (const auto &symbol : project.symbolNumbering())
+		list << QStringList{symbol.label, symbol.formula, symbol.scheme.toString(),
+							symbol.sequence.unit.join(';'), symbol.sequence.ten.join(';'),
+							symbol.sequence.hundred.join(';'),
+							QString::number(symbol.numbered)}.join('|');
+	list.sort();
+	return list;
+}
+
+	//What the scheme @p title tells of the numbers its symbols carry
+static QString gaps(const QETProject &project, const QString &title)
+{
+	QStringList list;
+	for (const auto &gap : ElementAutoNumSchemeCommand::gapRanges(&project, title))
+		list << QString("%1-%2").arg(gap.from).arg(gap.to);
+	return list.join(',');
+}
+
+	//A symbol of @p location placed on folio @p i as the folio's editor
+	//places one (DiagramEventAddElement::addElement()), numbered with the
+	//scheme @p title: its label, and the numbers then free for it
+static QString place(QETProject &project, int i, const QString &location, const QString &title)
+{
+	Diagram *folio = project.folios().at(i);
+	project.buildFolioToShow(folio);
+	project.setCurrrentElementAutonum(title);
+	int state = 0;
+	Element *element = ElementFactory::Instance()->createElement(
+				ElementsLocation(location, &project), nullptr, &state);
+	check(!state, "symbol made");
+	element->setPos(QPointF(-500, -500));
+	folio->addItem(element);
+	auto *undo = new QUndoCommand("insert");
+	new AddGraphicsObjectCommand(element, folio, element->pos(), undo);
+	element->setUpFormula(true, undo);
+	folio->undoStack().push(undo);
+	element->freezeNewAddedElement();
+	QStringList free;
+	for (int n : ElementAutoNumSchemeCommand::freeNumbers(&project, title, element))
+		free << QString::number(n);
+	return element->actualLabel() + " free " + free.join(',');
+}
+
+	//The symbols a search of a project finds, by uuid
+static QStringList found(const QVector<QPointer<Element>> &elements)
+{
+	QStringList list;
+	for (const auto &element : elements)
+		list << element->uuid().toString();
+	list.sort();
+	return list;
+}
+
+	//Everyday edits on folio @p i, as its editor makes them: a symbol's
+	//information, a wire, the folio's title block, a linked symbol
+	//deleted, then one undo and one redo
+static void edit(QETProject &project, int i)
+{
+	Diagram *folio = project.folios().at(i);
+	project.buildFolioToShow(folio);
+	QList<Element *> elements = folio->elements();
+	std::sort(elements.begin(), elements.end(), [](Element *a, Element *b) {
+		return a->uuid() < b->uuid(); });
+	check(elements.size() > 2, "a folio with symbols to edit");
+
+	const DiagramContext old_info = elements.first()->elementInformations();
+	DiagramContext new_info = old_info;
+	new_info.addValue(QStringLiteral("comment"), QStringLiteral("edited lazily"));
+	folio->undoStack().push(new ChangeElementInformationCommand(elements.first(), old_info, new_info));
+
+	Terminal *ends[2] = {nullptr, nullptr};
+	for (Element *element : elements) {
+		for (Terminal *terminal : element->terminals()) {
+			if (terminal->conductors().isEmpty()
+					&& (!ends[0] || ends[0]->parentElement() != element)) {
+				ends[ends[0] ? 1 : 0] = terminal;
+				break;
+			}
+		}
+		if (ends[1]) break;
+	}
+	if (ends[1]) {
+		auto *undo = new QUndoCommand(QStringLiteral("wire"));
+		auto *wire = new Conductor(ends[0], ends[1]);
+		new AddGraphicsObjectCommand(wire, folio, QPointF(), undo);
+		ConductorAutoNumerotation numbering(wire, folio, undo);
+		numbering.numerate();
+		folio->undoStack().push(undo);
+	}
+
+	const TitleBlockProperties old_block = folio->border_and_titleblock.exportTitleBlock();
+	TitleBlockProperties new_block = old_block;
+	new_block.title += QStringLiteral(" (edited)");
+	folio->undoStack().push(new ChangeTitleBlockCommand(folio, old_block, new_block));
+
+		//Copy three symbols and paste them, as Ctrl+C then Ctrl+Shift+V
+		//and Return do
+	{
+		DiagramView view(folio);
+		folio->clearSelection();
+		for (int n = 0 ; n < 3 ; ++n)
+			elements.at(n)->setSelected(true);
+		view.copy();
+		auto *paste = new DiagramEventAddPaste(folio, QPointF(), DiagramEventAddPaste::AtOrigin);
+		folio->setEventInterface(paste);
+		QKeyEvent drop(QEvent::KeyPress, Qt::Key_Return, Qt::NoModifier);
+		paste->keyPressEvent(&drop);
+		folio->clearEventInterface();
+	}
+
+	Element *linked = nullptr;
+	for (Element *element : elements)
+		if (!element->linkedElements().isEmpty()) { linked = element; break; }
+	folio->clearSelection();
+	(linked ? linked : elements.last())->setSelected(true);
+	folio->undoStack().push(new DeleteQGraphicsItemCommand(folio, DiagramContent(folio, true)));
+
+	project.undoStack()->undo();
+	project.undoStack()->redo();
+}
+
+	//The project as saving writes it: folios not built written as read
+	//when building them would write the same (QETProject::toXml())
+static QString saved(QETProject &project)
+{
+	QHash<int, QByteArray> kept;
+	const QDomDocument document = project.toXml(&kept);
+	return QETProject::spliceKeptFolios(document.toString(4), kept);
+}
+
+	//The rows of the project database's tables, in their order, read as
+	//every caller reads them (newQuery(): brought up to date first)
+static QStringList tables(QETProject &project)
+{
+	QStringList rows;
+	for (const char *table : {"diagram", "diagram_info", "element", "element_info",
+							  "terminal", "conductor", "link"}) {
+		QSqlQuery query = project.dataBase()->newQuery(
+					QStringLiteral("SELECT * FROM %1 ORDER BY rowid").arg(QLatin1String(table)));
+		check(query.exec(), "a table read");
+		QStringList table_rows;
+		while (query.next()) {
+			QStringList row;
+			for (int i = 0 ; i < query.record().count() ; ++i)
+				row << query.value(i).toString();
+			table_rows << QLatin1String(table) + ": " + row.join('|');
+		}
+			//Rows are written in the order the scene gives a folio's items,
+			//which is not the same from one opening to the next: two usual
+			//opens given the same edits differ in it (637 rows of 3 394 on
+			//industrial.qet). What a table holds is compared, not its order
+		table_rows.sort();
+		rows << table_rows;
+	}
+	return rows;
+}
+
+static QImage render(Diagram *folio)
+{
+	const QRectF rect = folio->border_and_titleblock.borderAndTitleBlockRect().adjusted(-5, -5, 5, 5);
+	QImage image(rect.size().toSize(), QImage::Format_RGB32);
+	image.fill(Qt::white);
+	QPainter painter(&image);
+	folio->render(&painter, QRectF(QPointF(0, 0), rect.size()), rect);
+	return image;
+}
+
+	//Each symbol's partners, in their order
+static QStringList links(Diagram *folio)
+{
+	QStringList list;
+	for (Element *element : folio->elements()) {
+		QStringList partners;
+		for (Element *partner : element->linkedElements())
+			partners << partner->uuid().toString();
+		list << element->uuid().toString() + ' ' + partners.join(',');
+	}
+	list.sort();
+	return list;
+}
+
+int main(int argc, char **argv)
+{
+	QApplication app(argc, argv); std::freopen(argv[2], "w", stdout);
+	QTemporaryDir settings; QSettings::setDefaultFormat(QSettings::IniFormat);
+	QSettings::setPath(QSettings::IniFormat, QSettings::UserScope, settings.path());
+	QCoreApplication::setOrganizationName("QETLazyFoliosRegression");
+	QETProject::setBackupEnabled(false); QET::QetMessageBox::setNonInteractive(true);
+	QFontDatabase::addApplicationFont(":/fonts/LiberationSans-Regular.ttf");
+		//Folios not built saved as read (QETProject::toXml()): this
+		//fixture is written by this version's own save
+	qputenv("QET_LAZY_SAVE_KEPT", "1");
+	try {
+		const QString output = QString::fromLocal8Bit(argv[3]);
+		qunsetenv("QET_LAZY_FOLIOS");
+			//A file saved by this version: its folios carry their uuids,
+			//which opening without building them needs
+		const QString path = output + "/lazy_folios.qet";
+		QSet<int> strip_folios;
+		{
+			QETProject source(QString::fromLocal8Bit(argv[1]));
+			check(source.state() == QETProject::Ok, "fixture opens");
+				//A terminal strip of terminals on several folios: it is
+				//read with the project, before any folio is built
+			TerminalStrip *strip = source.newTerminalStrip(QStringLiteral("=A1"), QStringLiteral("+B1"),
+														  QStringLiteral("X99"));
+			int added = 0;
+			for (Diagram *folio : source.folios())
+				for (Element *element : folio->elements())
+					if (element->elementData().m_type == ElementData::Terminal && added < 12
+							&& strip->addTerminal(element)) {
+						++added;
+						strip_folios.insert(source.folioIndex(folio));
+					}
+			check(added > 3, "a terminal strip with terminals");
+			source.setFilePath(path);
+			check(source.write().isOk(), "fixture saved by this version");
+		}
+		QETProject eager(path);
+		check(eager.state() == QETProject::Ok, "project opens");
+		{
+			qputenv("QET_LAZY_FOLIOS", "1");
+			QETProject lazy_strips(path);
+			qunsetenv("QET_LAZY_FOLIOS");
+			auto strip_of = [](const QETProject &project) {
+				for (TerminalStrip *strip : project.terminalStrip())
+					if (strip->name() == QStringLiteral("X99")) return strip->physicalTerminalCount();
+				return -1;
+			};
+			check(strip_of(lazy_strips) > 3 && strip_of(lazy_strips) == strip_of(eager),
+				  "a terminal strip keeps its terminals when the folios are not built");
+		}
+		const QList<Diagram *> folios = eager.folios();
+		check(folios.size() > 3, "several folios");
+		QList<QImage> images; QList<QStringList> partners;
+		int linked = 0;
+		for (Diagram *folio : folios) {
+			images << render(folio);
+			partners << links(folio);
+			for (Element *element : folio->elements())
+				linked += !element->linkedElements().isEmpty();
+		}
+		check(linked > 0, "the project has linked symbols");
+
+		qputenv("QET_LAZY_FOLIOS", "1");
+		auto same = [&](QETProject &lazy, int i, const char *what) {
+			Diagram *folio = lazy.folios().at(i);
+			lazy.buildFolioToShow(folio);
+			if (render(folio) != images.at(i) || links(folio) != partners.at(i)) {
+				std::printf("folio %d, %s: differs\n", i + 1, what);
+				throw std::runtime_error("a folio shown lazily draws or links as opened as usual");
+			}
+		};
+			//Each folio first after opening
+		for (int i = 0; i < folios.size(); ++i) {
+			QETProject lazy(path);
+			check(lazy.unloadedFolioCount() == folios.size() - strip_folios.size(),
+				  "opened building only the folios of the terminal strip's terminals");
+			same(lazy, i, "shown first");
+			check(lazy.unloadedFolioCount() > 0, "showing a folio does not build them all");
+		}
+			//Every folio, one after the other, then the save
+		QETProject lazy(path);
+		for (int i = 0; i < folios.size(); ++i)
+			same(lazy, i, "shown in turn");
+		check(lazy.toXml().toString() == eager.toXml().toString(), "saves as opened as usual");
+		check(lazy.undoStack()->isClean(), "showing folios changes nothing");
+
+			//Numbering asks every symbol of the project: it is answered for
+			//the folios not built without building them, as when built
+		QETProject numbered(path);
+		check(numbering(numbered) == numbering(eager), "symbols' numbering as when built");
+		const QStringList titles = eager.elementAutoNum().keys();
+		check(!titles.isEmpty(), "the project has element numbering schemes");
+		for (const QString &title : titles)
+			check(gaps(numbered, title) == gaps(eager, title), "a scheme's gaps as when built");
+		check(numbered.unloadedFolioCount() == folios.size() - strip_folios.size(), "numbering read without building");
+			//A symbol placed on a folio, numbered by a scheme others follow
+		QString location, title;
+		int folio = -1;
+		for (const auto &symbol : eager.symbolNumbering()) {
+			if (symbol.numbered && symbol.element && !eager.elementAutoNumTitle(symbol.scheme).isEmpty()) {
+				location = symbol.element->location().toString();
+				title = eager.elementAutoNumTitle(symbol.scheme);
+				folio = eager.folios().indexOf(symbol.element->diagram());
+				break;
+			}
+		}
+		check(folio >= 0, "a symbol following a scheme");
+		const QString placed = place(numbered, folio, location, title);
+		std::printf("placed on folio %d with %s: %s\n", folio + 1, qPrintable(title), qPrintable(placed));
+		check(numbered.unloadedFolioCount() > 0, "numbering a placed symbol does not build every folio");
+		QETProject usual(path);
+		check(placed == place(usual, folio, location, title), "a placed symbol numbered as when built");
+			//The candidates a link tab lists (ElementProvider): found as
+			//when built, building only the folios which hold some
+		const ElementProvider usual_search(&eager);
+		for (const ElementData::Types kinds : {ElementData::Types(ElementData::Slave),
+											   ElementData::Types(ElementData::Master),
+											   ElementData::Types(ElementData::AllReport),
+											   ElementData::Types(ElementData::Terminal)}) {
+			QETProject searched(path);
+			const ElementProvider search(&searched);
+			const QStringList lazy_free = found(search.freeElement(kinds)), usual_free = found(usual_search.freeElement(kinds));
+			if (lazy_free != usual_free) {
+				for (const QString &u : usual_free) if (!lazy_free.contains(u)) std::printf("  only when built: %s\n", qPrintable(u));
+				for (const QString &u : lazy_free) if (!usual_free.contains(u)) std::printf("  only lazily: %s\n", qPrintable(u));
+			}
+			check(lazy_free == usual_free, "free symbols found as when built");
+			std::printf("free symbols of kinds %d: %d folios built\n", int(kinds),
+						int(folios.size() - searched.unloadedFolioCount()));
+			check(found(search.find(kinds)) == found(usual_search.find(kinds)),
+				  "symbols found as when built");
+		}
+		{
+			QETProject searched(path);
+			ElementProvider(&searched).freeElement(ElementData::Slave);
+			check(searched.unloadedFolioCount() > 0, "a search for free contacts does not build every folio");
+		}
+			//Selecting a symbol shows its properties, a link tab for a
+			//coil, a contact or a report: the folio and few others built
+		for (const Element::kind kind : {Element::Master, Element::Slave, Element::NextReport,
+										 Element::PreviousReport}) {
+			QETProject selected(path);
+			Element *element = nullptr;
+			for (int i = 0 ; !element && i < folios.size() ; ++i) {
+				for (Element *candidate : eager.folios().at(i)->elements()) {
+					if (candidate->linkType() == kind) {
+						selected.buildFolioToShow(selected.folios().at(i));
+						for (Element *e : selected.folios().at(i)->elements())
+							if (e->uuid() == candidate->uuid()) element = e;
+						break;
+					}
+				}
+			}
+			check(element, "a symbol of each kind with a link tab");
+			const int before = selected.unloadedFolioCount();
+			delete new ElementPropertiesWidget(element);
+			std::printf("properties of a symbol of kind %d: %d more folios built, %d not built\n",
+						int(kind), before - selected.unloadedFolioCount(), selected.unloadedFolioCount());
+			check(selected.unloadedFolioCount() > 0, "selecting a symbol does not build every folio");
+		}
+			//Everyday edits on a folio linked to others: the others stay
+			//not built, and the project saves as when they are
+		int busiest = 0, most = -1;
+		for (int i = 0 ; i < folios.size() ; ++i) {
+			int partners = 0;
+			for (Element *element : folios.at(i)->elements())
+				for (Element *partner : element->linkedElements())
+					partners += partner->diagram() != folios.at(i);
+			if (partners > most) { most = partners; busiest = i; }
+		}
+		{
+			QETProject edited(path), usual(path);
+			qunsetenv("QET_LAZY_FOLIOS");
+			QETProject usual_edited(path);
+			qputenv("QET_LAZY_FOLIOS", "1");
+			edit(edited, busiest);
+			std::printf("edits on folio %d (%d links to other folios): %d folios not built\n",
+						busiest + 1, most, edited.unloadedFolioCount());
+			check(edited.unloadedFolioCount() > 0, "editing a folio does not build every folio");
+			edit(usual_edited, busiest);
+				//Made by the edits, a new wire's uuid is random: any uuid the
+				//file had not is "new" before comparing
+			QFile saved(path); saved.open(QIODevice::ReadOnly);
+			const QString before = QString::fromUtf8(saved.readAll());
+			static const QRegularExpression uuid_re(QStringLiteral("uuid=\"(\\{[0-9a-f-]{36}\\})\""));
+			auto known = [&before](QString xml) {
+				QString out;
+				qsizetype at = 0;
+				for (auto it = uuid_re.globalMatch(xml) ; it.hasNext() ; ) {
+					const auto match = it.next();
+					out += xml.mid(at, match.capturedStart() - at);
+					out += before.contains(match.captured(1)) ? match.captured(0)
+															  : QStringLiteral("uuid=\"new\"");
+					at = match.capturedEnd();
+				}
+				return out + xml.mid(at);
+			};
+			const QStringList lazy_rows = tables(edited), usual_rows = tables(usual_edited);
+			static const QRegularExpression bare_uuid_re(QStringLiteral("\\{[0-9a-f-]{36}\\}"));
+			auto known_row = [&before](QString row) {
+				for (auto it = bare_uuid_re.globalMatch(QString(row)) ; it.hasNext() ; ) {
+					const QString uuid = it.next().captured(0);
+					if (!before.contains(uuid)) row.replace(uuid, QStringLiteral("new"));
+				}
+				return row;
+			};
+			check(edited.unloadedFolioCount() > 0, "reading the database does not build every folio");
+			QStringList lazy_known, usual_known;
+			for (const QString &row : lazy_rows) lazy_known << known_row(row);
+			for (const QString &row : usual_rows) usual_known << known_row(row);
+			lazy_known.sort();
+			usual_known.sort();
+			int differing = 0;
+			for (int i = 0 ; i < std::max(lazy_known.size(), usual_known.size()) ; ++i)
+				if (lazy_known.value(i) != usual_known.value(i) && differing++ < 5)
+					std::printf("  row %d: lazy  %s\n         usual %s\n", i, qPrintable(lazy_known.value(i).left(160)),
+								qPrintable(usual_known.value(i).left(160)));
+			std::printf("database after the edits: %d rows, %d differ\n", int(usual_rows.size()), differing);
+			check(!differing, "the database after the edits as when every folio is built");
+				//A pasted copy has a random uuid, and a folio's symbols are
+				//saved in uuid order: the lines are compared, not their order
+			auto lines = [&known](const QString &xml) {
+				QStringList list = known(xml).split(QLatin1Char('\n'));
+				list.sort();
+				return list.join(QLatin1Char('\n'));
+			};
+			const QString a = lines(edited.toXml().toString()), b = lines(usual_edited.toXml().toString());
+			if (a != b) {
+				QFile fa(output + "/edited-lazy.xml"); fa.open(QIODevice::WriteOnly); fa.write(a.toUtf8());
+				QFile fb(output + "/edited-usual.xml"); fb.open(QIODevice::WriteOnly); fb.write(b.toUtf8());
+			}
+			check(a == b,
+				  "edits save as when every folio is built");
+		}
+			//Saving writes the folios not built as they were read, the same
+			//bytes as building them would write
+		const QString usual_text = saved(eager);
+		{
+			QETProject untouched(path);
+			const int unbuilt = untouched.unloadedFolioCount();
+			const QString untouched_text = saved(untouched);
+			if (untouched_text != usual_text) {
+				QFile fa(output + "/untouched-lazy.xml"); fa.open(QIODevice::WriteOnly); fa.write(untouched_text.toUtf8());
+				QFile fb(output + "/untouched-usual.xml"); fb.open(QIODevice::WriteOnly); fb.write(usual_text.toUtf8());
+			}
+			check(untouched_text == usual_text, "a project saved without building, as when built");
+				//Only the partners of the folios the terminal strip built
+			std::printf("saved untouched: %d of %d folios not built built to save\n",
+						unbuilt - untouched.unloadedFolioCount(), unbuilt);
+			check(untouched.unloadedFolioCount() > 0, "saving does not build every folio");
+		}
+			//After everyday edits on a folio: as when built, others not built
+		{
+			QETProject edited(path), usual(path);
+			qunsetenv("QET_LAZY_FOLIOS");
+			QETProject usual_edited(path);
+			qputenv("QET_LAZY_FOLIOS", "1");
+			edit(edited, busiest);
+			edit(usual_edited, busiest);
+			QFile file(path); file.open(QIODevice::ReadOnly);
+			const QString before = QString::fromUtf8(file.readAll());
+			static const QRegularExpression uuid_re(QStringLiteral("\\{[0-9a-f-]{36}\\}"));
+			auto lines = [&before](QString text) {
+				for (auto it = uuid_re.globalMatch(QString(text)) ; it.hasNext() ; ) {
+					const QString uuid = it.next().captured(0);
+					if (!before.contains(uuid)) text.replace(uuid, QStringLiteral("new"));
+				}
+				QStringList list = text.split(QLatin1Char('\n'));
+				list.sort();
+				return list;
+			};
+			const QStringList la = lines(saved(edited)), lb = lines(saved(usual_edited));
+			if (la != lb) {
+				int shown = 0;
+				for (const QString &l : lb) if (!la.contains(l) && shown++ < 6) std::printf("  only usual: %s\n", qPrintable(l.trimmed().left(150)));
+				shown = 0;
+				for (const QString &l : la) if (!lb.contains(l) && shown++ < 6) std::printf("  only lazy:  %s\n", qPrintable(l.trimmed().left(150)));
+			}
+			check(la == lb, "edits saved without building, as when built");
+			std::printf("saved after the edits: %d folios not built\n", edited.unloadedFolioCount());
+			check(edited.unloadedFolioCount() > 0, "saving after edits does not build every folio");
+		}
+			//A folio not built given a new title: it and the folios its
+			//links reach are built to be saved, the others not
+		{
+			QETProject renamed(path);
+			qunsetenv("QET_LAZY_FOLIOS");
+			QETProject usual_renamed(path);
+			qputenv("QET_LAZY_FOLIOS", "1");
+			for (QETProject *project : {&renamed, &usual_renamed}) {
+				Diagram *folio = project->folios().at(busiest);
+				TitleBlockProperties block = folio->border_and_titleblock.exportTitleBlock();
+				block.title += QStringLiteral(" (renamed)");
+				folio->border_and_titleblock.importTitleBlock(block);
+			}
+			check(saved(renamed) == saved(usual_renamed), "a renamed folio saved as when built");
+			std::printf("saved after a folio not built was renamed: %d folios not built\n", renamed.unloadedFolioCount());
+			check(renamed.unloadedFolioCount() > 0, "renaming a folio does not build every folio to save");
+		}
+			//Project properties, which any folio's texts can use: every
+			//folio is built to be saved
+		{
+			QETProject changed(path);
+			qunsetenv("QET_LAZY_FOLIOS");
+			QETProject usual_changed(path);
+			qputenv("QET_LAZY_FOLIOS", "1");
+			for (QETProject *project : {&changed, &usual_changed}) {
+				DiagramContext properties = project->projectProperties();
+				properties.addValue(QStringLiteral("lazy_test"), QStringLiteral("1"));
+				project->setProjectProperties(properties);
+			}
+			check(saved(changed) == saved(usual_changed), "changed project properties saved as when built");
+			check(changed.unloadedFolioCount() == 0, "changed project properties build every folio to save");
+		}
+			//A potential through folio reports is followed as far as when
+			//every folio is built
+		{
+			Conductor *usual_wire = nullptr;
+			for (Diagram *folio : eager.folios())
+				for (Element *element : folio->elements())
+					if ((element->linkType() & Element::AllReport) && !element->conductors().isEmpty()
+							&& !usual_wire)
+						usual_wire = element->conductors().first();
+			check(usual_wire, "a wire on a folio report");
+			auto potential = [](Conductor *wire) {
+				QStringList list;
+				for (Conductor *c : wire->relatedPotentialConductors(true))
+					list << c->uuid().toString();
+				list.sort();
+				return list;
+			};
+			QETProject followed(path);
+			Diagram *folio = followed.folios().at(eager.folioIndex(usual_wire->diagram()));
+			followed.buildFolioToShow(folio);
+			Conductor *wire = nullptr;
+			for (Conductor *c : folio->conductors())
+				if (c->uuid() == usual_wire->uuid()) wire = c;
+			check(wire, "the same wire");
+			check(potential(wire) == potential(usual_wire), "a potential followed as when built");
+			std::printf("potential through reports: %d wires, %d folios not built\n",
+						int(potential(wire).size()), followed.unloadedFolioCount());
+		}
+		std::printf("PASS: %d folios\n", int(folios.size()));
+	} catch (const std::exception &error) {
+		std::printf("FAIL: %s\n", error.what());
+		std::fflush(stdout);
+		return 1;
+	}
+	std::fflush(stdout);
+	return 0;
+}

tests/qttest/run_lazy_folios_probe.py

@@ -0,0 +1,60 @@
+#!/usr/bin/env python3
+"""Link the lazy folios integration probe using an existing Ninja app build.
+
+Reuses app objects without duplicating the app or introducing a core-library
+refactor. No application source or existing executable is replaced. The
+generated manifest, probe binary, reports and export fixtures stay in build/.
+"""
+import argparse
+import pathlib
+import re
+import shutil
+import subprocess
+import sys
+
+parser = argparse.ArgumentParser()
+parser.add_argument('--build', required=True)
+parser.add_argument('--binary', required=True)
+parser.add_argument('--source', required=True)
+parser.add_argument('--fixture', required=True)
+args = parser.parse_args()
+build = pathlib.Path(args.build).resolve()
+binary = pathlib.Path(args.binary).resolve()
+source = pathlib.Path(args.source).resolve()
+fixture = pathlib.Path(args.fixture).resolve()
+manifest = (build / 'build.ninja').read_text(encoding='utf-8')
+match = re.search(r'^build (\S*?/sources/main\.cpp\.(?:obj|o)):', manifest, re.MULTILINE)
+if not match:
+    sys.exit('Cannot find the application main object in the Ninja build.')
+main_object = match.group(1)
+probe_object = main_object.replace('main.cpp.', 'lazy_folios_probe.cpp.')
+probe_binary = binary.with_name('lazy_folios_probe' + binary.suffix)
+def escape(path):
+    return pathlib.Path(path).as_posix().replace('$', '$$').replace(':', '$:').replace(' ', '$ ')
+main_source = source.parents[2] / 'sources' / 'main.cpp'
+manifest = manifest.replace(main_object, probe_object)
+manifest = manifest.replace(escape(main_source), escape(source))
+if binary.suffix:
+    manifest = manifest.replace(binary.name, probe_binary.name)
+else:
+    old_target = escape(binary.name if binary.parent == build else binary)
+    new_target = escape(probe_binary.name if probe_binary.parent == build else probe_binary)
+    manifest = manifest.replace('build ' + old_target + ':', 'build ' + new_target + ':')
+    old_file = binary.name if binary.parent == build else binary.as_posix()
+    new_file = probe_binary.name if probe_binary.parent == build else probe_binary.as_posix()
+    manifest = re.sub(r'^  TARGET_FILE = ' + re.escape(old_file) + r'$',
+                      '  TARGET_FILE = ' + new_file, manifest, flags=re.MULTILINE)
+probe_manifest = build / 'lazy-folios-probe.ninja'
+probe_manifest.write_text(manifest, encoding='utf-8')
+ninja = shutil.which('ninja')
+if not ninja:
+    sys.exit('Ninja is required for the integration probe.')
+target = probe_binary.name if probe_binary.parent == build else str(probe_binary)
+subprocess.run([ninja, '-C', str(build), '-f', probe_manifest.name, target], check=True)
+output = build / 'lazy-folios'
+output.mkdir(exist_ok=True)
+report = output / 'results.txt'
+result = subprocess.run([str(probe_binary), str(fixture), str(report), str(output)], timeout=500)
+if report.exists():
+    print(report.read_text(encoding='utf-8', errors='replace'))
+sys.exit(result.returncode)

tests/qttest/tst_resaveunchanged.cpp

@@ -317,6 +317,169 @@ private slots:
 		QCOMPARE(without_save_stamps(read(back)), without_save_stamps(read(straight)));
 	}
 
+	// QElectroTech run in a sandbox of its own with @p args, the store
+	// check on (QET_CHECK_ELEMENT_INFO_STORE); its output
+	QString runChecked(const QStringList &args, bool eager = false)
+	{
+		const QString home = m_dir.filePath(QStringLiteral("home%1").arg(m_run++));
+		QDir().mkpath(home);
+		QProcessEnvironment env = QProcessEnvironment::systemEnvironment();
+		env.insert(QStringLiteral("QT_QPA_PLATFORM"), QStringLiteral("offscreen"));
+		env.insert(QStringLiteral("QET_ENABLE_SCRIPTING"), QStringLiteral("1"));
+		env.insert(QStringLiteral("QET_CHECK_ELEMENT_INFO_STORE"), QStringLiteral("1"));
+		if (eager) {
+			env.remove(QStringLiteral("QET_LAZY_FOLIOS")); //every folio built on opening
+		}
+		env.insert(QStringLiteral("HOME"), home);
+		env.insert(QStringLiteral("XDG_CONFIG_HOME"), home + QStringLiteral("/config"));
+		env.insert(QStringLiteral("XDG_DATA_HOME"), home + QStringLiteral("/data"));
+		QProcess proc;
+		proc.setProcessEnvironment(env);
+		proc.start(QStringLiteral(QET_TEST_BINARY_PATH), args);
+		if (!proc.waitForFinished(180000)) return {};
+		return QString::fromUtf8(proc.readAllStandardOutput() + proc.readAllStandardError());
+	}
+
+	// The stores of symbol information and folio title blocks
+	// (DB-ACCESSORS-PLAN.md stages 4.1, 4.4) agree with every placed symbol
+	// and folio once a project is open: every example.
+	void elementInfoStoreAgrees_data()
+	{
+		QTest::addColumn<QString>("project");
+		const QDir examples(QStringLiteral(QET_EXAMPLES_DIR));
+		for (const QString &f : examples.entryList({QStringLiteral("*.qet")}, QDir::Files, QDir::Name))
+			QTest::newRow(f.toUtf8().constData()) << examples.filePath(f);
+	}
+	void elementInfoStoreAgrees()
+	{
+		QFETCH(QString, project);
+		const QString out = runChecked({QStringLiteral("--resave"), project,
+										m_dir.filePath(QStringLiteral("store%1.qet").arg(m_run))});
+		QVERIFY2(out.contains(QStringLiteral("element information store: 0 differ")), qPrintable(out.right(600)));
+		QVERIFY2(out.contains(QStringLiteral("folio title block store: 0 differ")), qPrintable(out.right(600)));
+		QVERIFY2(out.contains(QStringLiteral("wire properties store: 0 differ")), qPrintable(out.right(600)));
+	}
+
+	// The stores filled from the document before the folios are built
+	// (DB-ACCESSORS-PLAN.md stage 4.6) hold what building them gives, for a
+	// project saved by this version (every symbol, folio and wire carries
+	// a uuid): nothing differs, nothing is left unplaced.
+	void storesFilledBeforeBuilding()
+	{
+		const QString saved = resave(QStringLiteral(QET_EXAMPLES_DIR "/industrial.qet"));
+		QVERIFY2(!saved.isEmpty(), "--resave failed");
+		const QString out = runChecked({QStringLiteral("--resave"), saved,
+										m_dir.filePath(QStringLiteral("prefilled.qet"))}, true);
+		const QRegularExpression line(QStringLiteral(
+				"filled before building: symbols (\\d+) agree, 0 differ, 0 not placed; "
+				"folios (\\d+) agree, 0 differ, 0 not placed; wires (\\d+) agree, 0 differ, 0 not placed"));
+		const QRegularExpressionMatch m = line.match(out);
+		QVERIFY2(m.hasMatch(), qPrintable(out.right(600)));
+		QVERIFY(m.captured(1).toInt() > 600 && m.captured(2).toInt() == 50 && m.captured(3).toInt() > 600);
+	}
+
+	// Title block fields stamped on every folio go through the store, and
+	// the saved folios carry them (a folio saves what the store holds).
+	void folioTitleBlockStoreAfterSetTitleBlock()
+	{
+		const QString saved = m_dir.filePath(QStringLiteral("stamped.qet"));
+		const QString out = runChecked({QStringLiteral("--set-titleblock"),
+										QStringLiteral(QET_EXAMPLES_DIR "/industrial.qet"), saved,
+										QStringLiteral("author=Ann"), QStringLiteral("extrafield=Hello")});
+		QVERIFY2(out.contains(QStringLiteral("folio title block store: 0 differ")), qPrintable(out.right(600)));
+		const QByteArray xml = read(saved);
+		QCOMPARE(xml.count("<diagram "), xml.count("author=\"Ann\"") - 1);
+		QCOMPARE(xml.count("<diagram "), xml.count("name=\"extrafield\"") - 1);
+	}
+
+	// ... and after a copy of a symbol, which briefly shares its original's
+	// uuid, and an undo: the original kept no row once (duplicateElements
+	// is the copy the paste and folio duplication make).
+	void elementInfoStoreAfterCopyAndUndo()
+	{
+#ifndef QET_HAS_SCRIPTING
+		QSKIP("needs --run: this QElectroTech is built without Qt Qml");
+#endif
+		const QString script = m_dir.filePath(QStringLiteral("copy.js"));
+		QFile js(script);
+		QVERIFY(js.open(QIODevice::WriteOnly));
+		js.write(QStringLiteral(
+				 "var f = 0; while (qet.elementUuids(f).length < 3) ++f;\n"
+				 "var e = qet.elementUuids(f);\n"
+				 "qet.log('COPY ' + qet.duplicateElements(f, [e[0], e[1]], f, 300, 300).length);\n"
+				 "qet.log('DELETE ' + qet.deleteElement(f, e[2]) + ' UNDO ' + qet.undo());\n"
+				 "qet.log('SAVED ' + qet.save('%1'));\n").arg(m_dir.filePath(QStringLiteral("copied.qet"))).toUtf8());
+		js.close();
+		const QString out = runChecked({QStringLiteral("--run"), script,
+										QStringLiteral(QET_EXAMPLES_DIR "/industrial.qet")});
+		QVERIFY2(out.contains(QStringLiteral("COPY 2")) && out.contains(QStringLiteral("UNDO true"))
+				 && out.contains(QStringLiteral("SAVED true")), qPrintable(out.right(600)));
+		QVERIFY2(out.contains(QStringLiteral("element information store: 0 differ")), qPrintable(out.right(600)));
+	}
+
+	// After copies of linked symbols, of wired symbols, and the undo of a
+	// folio's removal, the project database's symbol, link, wire and
+	// terminal rows are those a fresh open of the saved project builds
+	// (F106, F107): a copy is placed while it still carries its original's
+	// uuid, and an undone folio removal put back its symbols but not their
+	// rows.
+	void symbolRowsAfterCopyAndFolioUndo()
+	{
+#ifndef QET_HAS_SCRIPTING
+		QSKIP("needs --run: this QElectroTech is built without Qt Qml");
+#endif
+		const QString rows = QStringLiteral(
+				 "function rows() { return JSON.stringify(["
+				 "qet.query('SELECT uuid, diagram_uuid, type FROM element ORDER BY uuid'),"
+				 "qet.query('SELECT element_uuid, label, comment FROM element_info ORDER BY element_uuid'),"
+				 "qet.query('SELECT element_uuid, linked_uuid FROM link ORDER BY element_uuid, linked_uuid'),"
+				 "qet.query('SELECT uuid, terminal1_element_uuid, terminal2_element_uuid, text FROM conductor ORDER BY uuid'),"
+				 "qet.query('SELECT uuid, element_uuid FROM terminal ORDER BY uuid, element_uuid')]); }\n");
+		const QString edit = m_dir.filePath(QStringLiteral("rows_edit.js")),
+				dump = m_dir.filePath(QStringLiteral("rows_dump.js")),
+				saved = m_dir.filePath(QStringLiteral("rows.qet"));
+		QFile js(edit);
+		QVERIFY(js.open(QIODevice::WriteOnly));
+		js.write((rows + QStringLiteral(
+				 "var linked = -1;\n"
+				 "for (var f = 0; f < qet.folioCount() && linked < 0; ++f)\n"
+				 "  qet.elementUuids(f).forEach(function (u) { if (qet.linkedElements(f, u).length) linked = f; });\n"
+				 "qet.log('UNDONE ' + (qet.removeFolio(linked) && qet.undo()));\n"
+				 "var copied = 0;\n"
+				 "for (var f = 0; f < qet.folioCount() && copied < 3; ++f) {\n"
+				 "  var e = qet.elementUuids(f);\n"
+				 "  for (var i = 0; i < e.length && copied < 3; ++i)\n"
+				 "    if (qet.linkedElements(f, e[i]).length)\n"
+				 "      copied += qet.duplicateElements(f, [e[i]], f, 300, 300).length;\n"
+				 "}\n"
+				 "qet.log('COPIED ' + copied);\n"
+				 "var wf = 0; while (qet.elementUuids(wf).length < 6) ++wf;\n"
+				 "var wires = qet.conductorCount(wf), w = qet.elementUuids(wf);\n"
+				 "qet.log('WIRED ' + qet.duplicateElements(wf, w.slice(0, 6), wf, 300, 300).length"
+				 " + (qet.conductorCount(wf) > wires ? ' with wires' : ''));\n"
+				 "qet.log('ROWS ' + rows());\n"
+				 "qet.log('SAVED ' + qet.save('%1'));\n").arg(saved)).toUtf8());
+		js.close();
+		QFile js2(dump);
+		QVERIFY(js2.open(QIODevice::WriteOnly));
+		js2.write((rows + QStringLiteral("qet.log('ROWS ' + rows());\n")).toUtf8());
+		js2.close();
+
+		const QString edited = runChecked({QStringLiteral("--run"), edit,
+										   QStringLiteral(QET_EXAMPLES_DIR "/industrial.qet")});
+		QVERIFY2(edited.contains(QStringLiteral("COPIED 3")) && edited.contains(QStringLiteral("WIRED 6 with wires"))
+				 && edited.contains(QStringLiteral("UNDONE true"))
+				 && edited.contains(QStringLiteral("SAVED true")), qPrintable(edited.right(600)));
+		const QString reopened = runChecked({QStringLiteral("--run"), dump, saved});
+		auto rowsOf = [](const QString &log) {
+			const int at = log.indexOf(QStringLiteral("ROWS "));
+			return at < 0 ? QString() : log.mid(at + 5, log.indexOf(QLatin1Char('\n'), at) - at - 5);
+		};
+		QVERIFY(!rowsOf(reopened).isEmpty());
+		QCOMPARE(rowsOf(edited), rowsOf(reopened));
+	}
+
+
 	// A title-block value that is a single space is kept through two saves
 	// (#973), and a value with accents comes back as it went in.
 	void singleSpaceValueKept()