Open a .qetz from its database, each folio read when built (stage D2)

A .qetz now carries index.sqlite: the project database as QElectroTech
keeps it -- every table but the drawing items, which exist for built
folios only -- with each folio's own data (Diagram::headerXml()) and a
stamp of the schema it was written with (projectDataBase::indexFile(),
QETProject::writeContainerFile(): Save, --resave, --set-titleblock, a
script's and live mode's save). Written when every folio, symbol and wire
has a uuid of its own; a reader that does not know the entry skips it.

With QET_LAZY_FOLIOS=1, a .qetz with an index opens from it: the project
document is joined without its folios (QetContainer::join() with
folio_files), the database and the stores of symbol information and wire
properties are filled from the index (fillFromIndex()), each folio is made
from its own data and keeps the name of its part, read when the folio is
built (Diagram::deferredXml(), QETProject::readFolioPart(),
QetContainer::readFolio() -- pictures and engineering data back from
project.sqlite, QetContainerDb::restoreFolio()). A terminal strip finds its
terminals' folios in the database. Any other .qetz, or one whose index
cannot be used, is read whole as before.

The index is untrusted: refused unless written with this database's
schema, every table read a plain table (not a view), nothing in its schema
run (trusted_schema off); refused, the project is read whole. Checked: an
index of garbage, a wrong stamp, a table made a view -- each refused and
the project saved as a plain save.

The 136-folio f2099 as .qetz opens in 1.8 s (3.65 s read whole, 29 s
building every folio); what is left is mostly its 7 MB of symbol
definitions (0.8 s). The file grows from 2.6 to 5.2 MB, the index
overlapping project.sqlite until D3.

tst_lazyfolios: a .qetz opened from its index builds only the terminal
strip's folios, numbering read as when built, every folio shown in turn
draws and links as when built, the save as when built; fails when a folio
part's engineering data is not put back. Corpus, 342 projects: 340 .qetz
carry an index and open from it (2 have symbols sharing a uuid); saved
back as .qet = eager = master on all 342. Probe on industrial and f2099
.qetz: every folio as eager but the 3 render-noise folios of f2099. In a
real window the .qetz opens and pages building only shown folios and
partners. ctest 96/96 with and without QET_LAZY_FOLIOS=1.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

sources/container/qetcontainer.cpp

@@ -258,6 +258,43 @@ QList<QetZip::Entry> QetContainer::split(const QDomDocument &project, const QStr
 	return entries;
 }
 
+/**
+	@brief QetContainer::readFolio
+	The folio part @p file of the .qetz whose entries are @p files, as
+	join() puts it in the project: its pictures back as base64 and its
+	engineering data back from project.sqlite.
+*/
+bool QetContainer::readFolio(const QHash<QString, QByteArray> &files, const QString &file,
+							 QDomDocument *folio, QString *error)
+{
+	if (!files.contains(file)) {
+		setError(error, QStringLiteral("part \"%1\" is missing").arg(file));
+		return false;
+	}
+	QString part_error;
+	if (!parse(files.value(file), folio, &part_error)) {
+		setError(error, QStringLiteral("%1: %2").arg(file, part_error));
+		return false;
+	}
+	const QDomNodeList images = folio->elementsByTagName(QStringLiteral("image"));
+	for (int i = 0 ; i < images.size() ; ++i)
+	{
+		QDomElement image = images.at(i).toElement();
+		if (!image.hasAttribute(ImageFileAttribute)) continue;
+		const QString picture = image.attribute(ImageFileAttribute);
+		if (!files.contains(picture)) {
+			setError(error, QStringLiteral("picture \"%1\" is missing").arg(picture));
+			return false;
+		}
+		image.removeAttribute(ImageFileAttribute);
+		image.insertBefore(folio->createTextNode(QString::fromLatin1(files.value(picture).toBase64())),
+						   image.firstChild());
+	}
+	return !files.contains(QStringLiteral("project.sqlite"))
+			|| QetContainerDb::restoreFolio(folio->documentElement(),
+											files.value(QStringLiteral("project.sqlite")), error);
+}
+
 /**
 	@brief QetContainer::join
 	Rebuild in @p project the document split() made @p entries from.
@@ -265,7 +302,8 @@ QList<QetZip::Entry> QetContainer::split(const QDomDocument &project, const QStr
 	(the manifest's min-reader), or a part missing or unreadable. Entries
 	nothing names are ignored, so a later format may add some.
 */
-bool QetContainer::join(const QList<QetZip::Entry> &entries, QDomDocument *project, QString *error)
+bool QetContainer::join(const QList<QetZip::Entry> &entries, QDomDocument *project, QString *error,
+						QStringList *folio_files)
 {
 	QHash<QString, QByteArray> files;
 	for (const QetZip::Entry &entry : entries)
@@ -297,6 +335,16 @@ bool QetContainer::join(const QList<QetZip::Entry> &entries, QDomDocument *proje
 	for (QDomElement part : placeholders)
 	{
 		const QString file = part.attribute(FileAttribute);
+			//With @p folio_files, the folios stay parts, read when built
+		if (folio_files && part.parentNode() == project->documentElement()
+				&& file.startsWith(QLatin1String("folios/"))) {
+			if (!files.contains(file)) {
+				setError(error, QStringLiteral("part \"%1\" is missing").arg(file));
+				return false;
+			}
+			folio_files->append(file);
+			continue;
+		}
 		if (!files.contains(file)) {
 			setError(error, QStringLiteral("part \"%1\" is missing").arg(file));
 			return false;

sources/container/qetcontainer.h

@@ -53,7 +53,11 @@ namespace QetContainer
 							   const QString &writer = QString(),
 							   QString *error = nullptr);
 	bool join(const QList<QetZip::Entry> &entries, QDomDocument *project,
-			  QString *error = nullptr);
+			  QString *error = nullptr, QStringList *folio_files = nullptr);
+		/// A folio part of @p files, which join() left out, with its
+		/// pictures and its engineering data back (stage D2)
+	bool readFolio(const QHash<QString, QByteArray> &files, const QString &file,
+				   QDomDocument *folio, QString *error = nullptr);
 
 		/// Parse @p xml as a .qet is read (QETProject::openFile)
 	bool parse(const QByteArray &xml, QDomDocument *document, QString *error = nullptr);

sources/container/qetcontainerdb.cpp

@@ -434,6 +434,143 @@ QStringList keys(const QList<QDomElement> &items, const QString &prefix)
 }
 }
 
+namespace {
+	//@p database opened read only in @p db, its tables checked to be plain
+	//tables: a file is untrusted
+bool openDatabase(Database &db, const QByteArray &database, QString *error)
+{
+	if (sqlite3_open(":memory:", &db.m_db) != SQLITE_OK) {
+		setError(error, QStringLiteral("cannot open the project database"));
+		return false;
+	}
+	unsigned char *copy = static_cast<unsigned char *>(sqlite3_malloc64(sqlite3_uint64(database.size())));
+	if (!copy) { setError(error, QStringLiteral("out of memory")); return false; }
+	memcpy(copy, database.constData(), size_t(database.size()));
+	if (sqlite3_deserialize(db.m_db, "main", copy, database.size(), database.size(),
+							SQLITE_DESERIALIZE_FREEONCLOSE | SQLITE_DESERIALIZE_READONLY) != SQLITE_OK) {
+		setError(error, QStringLiteral("the project database cannot be read"));
+		return false;
+	}
+		//A file is untrusted: nothing in its schema may run, and every name
+		//read must be a plain table.
+	db.exec(QStringLiteral("PRAGMA trusted_schema = OFF"));
+	db.exec(QStringLiteral("PRAGMA query_only = ON"));
+	for (const char *table : {"meta", "project_property", "folio", "folio_property",
+							  "element", "element_info", "link", "conductor", "terminal_strip",
+							  "strip_terminal", "strip_bridge", "strip_bridge_terminal"}) {
+		Statement kind(db.m_db, QStringLiteral("SELECT type FROM sqlite_master WHERE name = ?"));
+		kind.bind({QString::fromLatin1(table)});
+		if (!kind.next() || kind.value(0).toString() != QLatin1String("table")) {
+			setError(error, QStringLiteral("the project database has no table %1").arg(QLatin1String(table)));
+			return false;
+		}
+	}
+	return true;
+}
+
+QDomElement entryElement(QDomDocument &document, const QVariant &name_attr, const QString &name,
+						 const QVariant &value, const QVariant &attrs, const QString &default_tag)
+{
+	const QJsonObject o = QJsonDocument::fromJson(attrs.toString().toUtf8()).object();
+	QDomElement e = document.createElement(o.value(QStringLiteral("tag")).toString(default_tag));
+	if (!name_attr.isNull()) e.setAttribute(name, name_attr.toString());
+	setAttributesJson(e, attrs);
+	if (!value.isNull()) e.appendChild(document.createTextNode(value.toString()));
+	return e;
+}
+
+	//The engineering data of the folio @p d (its qetz-folio id) back from @p db
+bool restoreDiagram(Database &db, QDomElement d, QString *error)
+{
+	QDomDocument document = d.ownerDocument();
+	auto entry = [&document](const QVariant &name_attr, const QString &name, const QVariant &value,
+							 const QVariant &attrs, const QString &default_tag) {
+		return entryElement(document, name_attr, name, value, attrs, default_tag);
+	};
+	const QString id = d.attribute(FolioAttribute);
+	if (id.isEmpty()) return true;
+	d.removeAttribute(FolioAttribute);
+	Statement f(db.m_db, QStringLiteral("SELECT %1, props_at, props_attrs FROM folio WHERE id = ?")
+				.arg(QStringLiteral("\"") + FolioData.join(QStringLiteral("\", \"")) + QStringLiteral("\"")));
+	f.bind({id});
+	if (!f.next()) { setError(error, QStringLiteral("folio %1 is not in the database").arg(id)); return false; }
+	QVariantList values;
+	for (int i = 0 ; i < FolioData.size() + 2 ; ++i) values << f.value(i);
+	put(d, FolioData, values, 0);
+	if (!values.at(FolioData.size()).isNull()) {
+		QDomElement props = document.createElement(QStringLiteral("properties"));
+		setAttributesJson(props, values.at(FolioData.size() + 1));
+		Statement p(db.m_db, QStringLiteral("SELECT name, value, attrs FROM folio_property WHERE folio_id = ? ORDER BY ord"));
+		p.bind({id});
+		while (p.next())
+			props.appendChild(entry(p.value(0), QStringLiteral("name"), p.value(1), p.value(2),
+									QStringLiteral("property")));
+		restoreWhitespace(props, values.at(FolioData.size() + 1));
+		if (!replaceMarker(d, props)) {
+			setError(error, QStringLiteral("folio %1's properties have no place").arg(id));
+			return false;
+		}
+	}
+
+	for (const QDomElement &list : children(d, QStringLiteral("elements"))) {
+		for (QDomElement e : children(list, QStringLiteral("element"))) {
+			if (!e.hasAttribute(KeyAttribute)) continue;
+			const QString key = e.attribute(KeyAttribute);
+			e.removeAttribute(KeyAttribute);
+			Statement s(db.m_db, QStringLiteral("SELECT %1, info_at, info_attrs, links_at, links_attrs FROM element WHERE key = ?")
+						.arg(QStringLiteral("\"") + ElementData.join(QStringLiteral("\", \"")) + QStringLiteral("\"")));
+			s.bind({key});
+			if (!s.next()) { setError(error, QStringLiteral("symbol %1 is not in the database").arg(key)); return false; }
+			QVariantList v;
+			for (int i = 0 ; i < ElementData.size() + 4 ; ++i) v << s.value(i);
+			put(e, ElementData, v, 0);
+			const int n = ElementData.size();
+			QDomElement info, links;
+			if (!v.at(n).isNull()) {
+				info = document.createElement(QStringLiteral("elementInformations"));
+				setAttributesJson(info, v.at(n + 1));
+				Statement i(db.m_db, QStringLiteral("SELECT name, value, attrs FROM element_info WHERE element_key = ? ORDER BY ord"));
+				i.bind({key});
+				while (i.next())
+					info.appendChild(entry(i.value(0), QStringLiteral("name"), i.value(1), i.value(2),
+										   QStringLiteral("elementInformation")));
+				restoreWhitespace(info, v.at(n + 1));
+			}
+			if (!v.at(n + 2).isNull()) {
+				links = document.createElement(QStringLiteral("links_uuids"));
+				setAttributesJson(links, v.at(n + 3));
+				Statement l(db.m_db, QStringLiteral("SELECT linked_uuid, attrs FROM link WHERE element_key = ? ORDER BY ord"));
+				l.bind({key});
+				while (l.next())
+					links.appendChild(entry(l.value(0), QStringLiteral("uuid"), QVariant(), l.value(1),
+											QStringLiteral("link_uuid")));
+				restoreWhitespace(links, v.at(n + 3));
+			}
+			if ((!info.isNull() && !replaceMarker(e, info))
+				|| (!links.isNull() && !replaceMarker(e, links))) {
+				setError(error, QStringLiteral("symbol %1's information has no place").arg(key));
+				return false;
+			}
+		}
+	}
+	for (const QDomElement &list : children(d, QStringLiteral("conductors"))) {
+		for (QDomElement c : children(list, QStringLiteral("conductor"))) {
+			if (!c.hasAttribute(KeyAttribute)) continue;
+			const QString key = c.attribute(KeyAttribute);
+			c.removeAttribute(KeyAttribute);
+			Statement s(db.m_db, QStringLiteral("SELECT %1 FROM conductor WHERE key = ?")
+						.arg(QStringLiteral("\"") + ConductorData.join(QStringLiteral("\", \"")) + QStringLiteral("\"")));
+			s.bind({key});
+			if (!s.next()) { setError(error, QStringLiteral("wire %1 is not in the database").arg(key)); return false; }
+			QVariantList v;
+			for (int i = 0 ; i < ConductorData.size() ; ++i) v << s.value(i);
+			put(c, ConductorData, v, 0);
+		}
+	}
+	return true;
+}
+}
+
 /**
 	@brief QetContainerDb::extract
 	Move the engineering data of @p project (the <project> element, its
@@ -609,32 +746,8 @@ bool QetContainerDb::restore(QDomElement project, const QByteArray &database, QS
 {
 	QDomDocument document = project.ownerDocument();
 	Database db;
-	if (sqlite3_open(":memory:", &db.m_db) != SQLITE_OK) {
-		setError(error, QStringLiteral("cannot open the project database"));
+	if (!openDatabase(db, database, error))
 		return false;
-	}
-	unsigned char *copy = static_cast<unsigned char *>(sqlite3_malloc64(sqlite3_uint64(database.size())));
-	if (!copy) { setError(error, QStringLiteral("out of memory")); return false; }
-	memcpy(copy, database.constData(), size_t(database.size()));
-	if (sqlite3_deserialize(db.m_db, "main", copy, database.size(), database.size(),
-							SQLITE_DESERIALIZE_FREEONCLOSE | SQLITE_DESERIALIZE_READONLY) != SQLITE_OK) {
-		setError(error, QStringLiteral("the project database cannot be read"));
-		return false;
-	}
-		//A file is untrusted: nothing in its schema may run, and every name
-		//read must be a plain table.
-	db.exec(QStringLiteral("PRAGMA trusted_schema = OFF"));
-	db.exec(QStringLiteral("PRAGMA query_only = ON"));
-	for (const char *table : {"meta", "project_property", "folio", "folio_property",
-							  "element", "element_info", "link", "conductor", "terminal_strip",
-							  "strip_terminal", "strip_bridge", "strip_bridge_terminal"}) {
-		Statement kind(db.m_db, QStringLiteral("SELECT type FROM sqlite_master WHERE name = ?"));
-		kind.bind({QString::fromLatin1(table)});
-		if (!kind.next() || kind.value(0).toString() != QLatin1String("table")) {
-			setError(error, QStringLiteral("the project database has no table %1").arg(QLatin1String(table)));
-			return false;
-		}
-	}
 
 	auto entry = [&document](const QVariant &name_attr, const QString &name, const QVariant &value,
 							 const QVariant &attrs, const QString &default_tag) {
@@ -687,87 +800,18 @@ bool QetContainerDb::restore(QDomElement project, const QByteArray &database, QS
 	}
 
 	for (QDomElement d : children(project, QStringLiteral("diagram")))
-	{
-		const QString id = d.attribute(FolioAttribute);
-		if (id.isEmpty()) continue;
-		d.removeAttribute(FolioAttribute);
-		Statement f(db.m_db, QStringLiteral("SELECT %1, props_at, props_attrs FROM folio WHERE id = ?")
-					.arg(QStringLiteral("\"") + FolioData.join(QStringLiteral("\", \"")) + QStringLiteral("\"")));
-		f.bind({id});
-		if (!f.next()) { setError(error, QStringLiteral("folio %1 is not in the database").arg(id)); return false; }
-		QVariantList values;
-		for (int i = 0 ; i < FolioData.size() + 2 ; ++i) values << f.value(i);
-		put(d, FolioData, values, 0);
-		if (!values.at(FolioData.size()).isNull()) {
-			QDomElement props = document.createElement(QStringLiteral("properties"));
-			setAttributesJson(props, values.at(FolioData.size() + 1));
-			Statement p(db.m_db, QStringLiteral("SELECT name, value, attrs FROM folio_property WHERE folio_id = ? ORDER BY ord"));
-			p.bind({id});
-			while (p.next())
-				props.appendChild(entry(p.value(0), QStringLiteral("name"), p.value(1), p.value(2),
-										QStringLiteral("property")));
-			restoreWhitespace(props, values.at(FolioData.size() + 1));
-			if (!replaceMarker(d, props)) {
-				setError(error, QStringLiteral("folio %1's properties have no place").arg(id));
-				return false;
-			}
-		}
-
-		for (const QDomElement &list : children(d, QStringLiteral("elements"))) {
-			for (QDomElement e : children(list, QStringLiteral("element"))) {
-				if (!e.hasAttribute(KeyAttribute)) continue;
-				const QString key = e.attribute(KeyAttribute);
-				e.removeAttribute(KeyAttribute);
-				Statement s(db.m_db, QStringLiteral("SELECT %1, info_at, info_attrs, links_at, links_attrs FROM element WHERE key = ?")
-							.arg(QStringLiteral("\"") + ElementData.join(QStringLiteral("\", \"")) + QStringLiteral("\"")));
-				s.bind({key});
-				if (!s.next()) { setError(error, QStringLiteral("symbol %1 is not in the database").arg(key)); return false; }
-				QVariantList v;
-				for (int i = 0 ; i < ElementData.size() + 4 ; ++i) v << s.value(i);
-				put(e, ElementData, v, 0);
-				const int n = ElementData.size();
-				QDomElement info, links;
-				if (!v.at(n).isNull()) {
-					info = document.createElement(QStringLiteral("elementInformations"));
-					setAttributesJson(info, v.at(n + 1));
-					Statement i(db.m_db, QStringLiteral("SELECT name, value, attrs FROM element_info WHERE element_key = ? ORDER BY ord"));
-					i.bind({key});
-					while (i.next())
-						info.appendChild(entry(i.value(0), QStringLiteral("name"), i.value(1), i.value(2),
-											   QStringLiteral("elementInformation")));
-					restoreWhitespace(info, v.at(n + 1));
-				}
-				if (!v.at(n + 2).isNull()) {
-					links = document.createElement(QStringLiteral("links_uuids"));
-					setAttributesJson(links, v.at(n + 3));
-					Statement l(db.m_db, QStringLiteral("SELECT linked_uuid, attrs FROM link WHERE element_key = ? ORDER BY ord"));
-					l.bind({key});
-					while (l.next())
-						links.appendChild(entry(l.value(0), QStringLiteral("uuid"), QVariant(), l.value(1),
-												QStringLiteral("link_uuid")));
-					restoreWhitespace(links, v.at(n + 3));
-				}
-				if ((!info.isNull() && !replaceMarker(e, info))
-					|| (!links.isNull() && !replaceMarker(e, links))) {
-					setError(error, QStringLiteral("symbol %1's information has no place").arg(key));
-					return false;
-				}
-			}
-		}
-		for (const QDomElement &list : children(d, QStringLiteral("conductors"))) {
-			for (QDomElement c : children(list, QStringLiteral("conductor"))) {
-				if (!c.hasAttribute(KeyAttribute)) continue;
-				const QString key = c.attribute(KeyAttribute);
-				c.removeAttribute(KeyAttribute);
-				Statement s(db.m_db, QStringLiteral("SELECT %1 FROM conductor WHERE key = ?")
-							.arg(QStringLiteral("\"") + ConductorData.join(QStringLiteral("\", \"")) + QStringLiteral("\"")));
-				s.bind({key});
-				if (!s.next()) { setError(error, QStringLiteral("wire %1 is not in the database").arg(key)); return false; }
-				QVariantList v;
-				for (int i = 0 ; i < ConductorData.size() ; ++i) v << s.value(i);
-				put(c, ConductorData, v, 0);
-			}
-		}
-	}
+		if (!restoreDiagram(db, d, error))
+			return false;
 	return true;
 }
+
+/**
+	@brief QetContainerDb::restoreFolio
+	restore() for the one folio @p diagram, read from its own part (a folio
+	built after its project was opened from a .qetz, stage D2)
+*/
+bool QetContainerDb::restoreFolio(QDomElement diagram, const QByteArray &database, QString *error)
+{
+	Database db;
+	return openDatabase(db, database, error) && restoreDiagram(db, diagram, error);
+}

sources/container/qetcontainerdb.h

@@ -39,6 +39,7 @@ namespace QetContainerDb
 {
 	QByteArray extract(QDomElement project, QString *error = nullptr);
 	bool restore(QDomElement project, const QByteArray &database, QString *error = nullptr);
+	bool restoreFolio(QDomElement diagram, const QByteArray &database, QString *error = nullptr);
 }
 
 #endif // QETCONTAINERDB_H