Keep each fact once in a .qetz: drop the stored project database

A .qetz held the engineering data twice: the document's tables, and
beside them QElectroTech's own project database (stage D2/D3), which
QElectroTech also builds from the document. Two copies of one fact can
disagree, and the copy cost a third of the file. Now project.sqlite holds
the document's tables only; QElectroTech builds its own project database
from them when the project opens, as it does for a .qet.

Measured (this build, lazy folios):
  industrial (50 folios): 620 -> 374 KB, open 0.37 -> 0.65 s
  f2099/nuri (136 folios): 3.27 -> 2.20 MB, open 2.2 -> 3.5 s
The same projects as .qet open in 3.3 s and 23 s.

Saving still copies the file of every folio not built, with its rows
(stage D4): the project keeps the part each folio was read from. nuri
opened and saved again: 6.3 s, no folio built, the same bytes.

A symbol's terminal list is left out of the folio file when every
folio, symbol and wire has a uuid of its own, now checked on the
document (QetContainer::ownUuids()) rather than by the stored database
being written.

Checked: 342 corpus projects open, .qetz -> .qet equals the plain
resave, a .qetz saved twice gives the same bytes; tst_lazyfolios,
tst_qetcontainer, tst_resaveunchanged (which now checks project.sqlite
holds only the document's tables).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

sources/container/qetcontainer.cpp

@@ -23,6 +23,7 @@
 #include <QMap>
 #include <QSet>
 #include <QRegularExpression>
+#include <QUuid>
 
 const char QetContainer::MimeType[] = "application/x-qelectrotech-project+zip";
 
@@ -107,6 +108,7 @@ void removeElement(QDomElement e)
 	}
 }
 
+
 	//The collection's symbol definitions, each to elements/<categories>/<name>
 void splitCollection(QDomDocument &document, const QDomElement &category, const QString &path,
 					 QList<QetZip::Entry> *parts, QSet<QString> *taken)
@@ -184,6 +186,33 @@ QString QetContainer::pictureFile(const QDomElement &image)
 	return image.attribute(ImageFileAttribute);
 }
 
+/**
+	@brief QetContainer::ownUuids
+	@return true if every folio, symbol and wire of @p project has a uuid
+	no other has: a wire then finds its ends without the symbols' terminal
+	lists, which split() leaves out
+*/
+bool QetContainer::ownUuids(const QDomElement &project)
+{
+	QSet<QString> seen;
+	auto own = [&seen](const QDomElement &item) {
+		const QString uuid = item.attribute(QStringLiteral("uuid"));
+		if (QUuid(uuid).isNull() || seen.contains(uuid)) return false;
+		seen.insert(uuid);
+		return true;
+	};
+	for (const QDomElement &diagram : childElements(project, QStringLiteral("diagram"))) {
+		if (!own(diagram)) return false;
+		for (const QDomElement &list : childElements(diagram, QStringLiteral("elements")))
+			for (const QDomElement &element : childElements(list, QStringLiteral("element")))
+				if (!own(element)) return false;
+		for (const QDomElement &list : childElements(diagram, QStringLiteral("conductors")))
+			for (const QDomElement &wire : childElements(list, QStringLiteral("conductor")))
+				if (!own(wire)) return false;
+	}
+	return true;
+}
+
 /**
 	@brief QetContainer::dropComputed
 	Take out of the folio @p diagram what QElectroTech computes when it
@@ -235,8 +264,7 @@ void QetContainer::dropComputed(QDomElement diagram, bool terminal_lists)
 	set if @p project is not a project
 */
 QList<QetZip::Entry> QetContainer::split(const QDomDocument &project, const QString &writer,
-										 QString *error, const QByteArray &index,
-										 const QHash<QString, QByteArray> &kept_from)
+										 QString *error, const QHash<QString, QByteArray> &kept_from)
 {
 	QDomDocument document = project.cloneNode(true).toDocument();
 	QDomElement root = document.documentElement();
@@ -250,7 +278,7 @@ QList<QetZip::Entry> QetContainer::split(const QDomDocument &project, const QStr
 		//stage D4) keeps its part and, through
 		//@p kept_from's project.sqlite, its rows
 	const QByteArray database = QetContainerDb::extract(
-				root, error, index, kept_from.value(QStringLiteral("project.sqlite")));
+				root, error, kept_from.value(QStringLiteral("project.sqlite")));
 	if (database.isEmpty())
 		return {};
 
@@ -259,9 +287,10 @@ QList<QetZip::Entry> QetContainer::split(const QDomDocument &project, const QStr
 
 		//Folios, each with its pictures taken out
 	int folio = 0;
-		//Nothing QElectroTech computes is kept (format 3)
+		//Nothing QElectroTech computes is kept (format 4)
+	const bool own_uuids = ownUuids(root);
 	for (QDomElement diagram : childElements(root, QStringLiteral("diagram")))
-		dropComputed(diagram, !index.isEmpty());
+		dropComputed(diagram, own_uuids);
 
 		//The parts of the folios kept as they were, and their pictures
 	QMap<QString, QByteArray> pictures;   //name -> png, once each
@@ -354,46 +383,6 @@ QList<QetZip::Entry> QetContainer::split(const QDomDocument &project, const QStr
 	return entries;
 }
 
-/**
-	@brief QetContainer::readFolio
-	The folio part @p file of the .qetz whose entries are @p files, as
-	join() puts it in the project: its pictures back as base64 and its
-	engineering data back from project.sqlite.
-*/
-bool QetContainer::readFolio(const QHash<QString, QByteArray> &files, const QString &file,
-							 QDomDocument *folio, QString *error,
-							 QetContainerDb::FolioReader *reader)
-{
-	if (!files.contains(file)) {
-		setError(error, QStringLiteral("part \"%1\" is missing").arg(file));
-		return false;
-	}
-	QString part_error;
-	if (!parse(files.value(file), folio, &part_error)) {
-		setError(error, QStringLiteral("%1: %2").arg(file, part_error));
-		return false;
-	}
-	const QDomNodeList images = folio->elementsByTagName(QStringLiteral("image"));
-	for (int i = 0 ; i < images.size() ; ++i)
-	{
-		QDomElement image = images.at(i).toElement();
-		if (!image.hasAttribute(ImageFileAttribute)) continue;
-		const QString picture = image.attribute(ImageFileAttribute);
-		if (!files.contains(picture)) {
-			setError(error, QStringLiteral("picture \"%1\" is missing").arg(picture));
-			return false;
-		}
-		image.removeAttribute(ImageFileAttribute);
-		image.insertBefore(folio->createTextNode(QString::fromLatin1(files.value(picture).toBase64())),
-						   image.firstChild());
-	}
-	if (!files.contains(QStringLiteral("project.sqlite")))
-		return true;
-	return reader ? reader->restore(folio->documentElement(), error)
-				  : QetContainerDb::restoreFolio(folio->documentElement(),
-												 files.value(QStringLiteral("project.sqlite")), error);
-}
-
 /**
 	@brief QetContainer::join
 	Rebuild in @p project the document split() made @p entries from.
@@ -401,7 +390,8 @@ bool QetContainer::readFolio(const QHash<QString, QByteArray> &files, const QStr
 	(the manifest's min-reader), or a part missing or unreadable. Entries
 	nothing names are ignored, so a later format may add some.
 	With @p pictures, the pictures stay out of the document: see the
-	namespace.
+	namespace. With @p folio_files, the part each folio was read from, in
+	order.
 */
 bool QetContainer::join(const QList<QetZip::Entry> &entries, QDomDocument *project, QString *error,
 						QStringList *folio_files, Pictures *pictures)
@@ -436,16 +426,9 @@ bool QetContainer::join(const QList<QetZip::Entry> &entries, QDomDocument *proje
 	for (QDomElement part : placeholders)
 	{
 		const QString file = part.attribute(FileAttribute);
-			//With @p folio_files, the folios stay parts, read when built
 		if (folio_files && part.parentNode() == project->documentElement()
-				&& file.startsWith(QLatin1String("folios/"))) {
-			if (!files.contains(file)) {
-				setError(error, QStringLiteral("part \"%1\" is missing").arg(file));
-				return false;
-			}
+				&& file.startsWith(QLatin1String("folios/")))
 			folio_files->append(file);
-			continue;
-		}
 		if (!files.contains(file)) {
 			setError(error, QStringLiteral("part \"%1\" is missing").arg(file));
 			return false;

sources/container/qetcontainer.h

@@ -18,7 +18,6 @@
 #ifndef QETCONTAINER_H
 #define QETCONTAINER_H
 
-#include "qetcontainerdb.h"
 #include "qetzip.h"
 
 #include <QDomDocument>
@@ -61,27 +60,24 @@ namespace QetContainer
 		/// 1/0, its PRAGMA user_version the format.
 		/// 4: what QElectroTech computes left out of the folios
 		/// (dropComputed()), the tables written without keys or indexes.
+		/// project.sqlite holds the engineering data once: QElectroTech's
+		/// own project database is built from it when the project opens.
 		/// Formats 2 and 3 are still read.
 
 	QList<QetZip::Entry> split(const QDomDocument &project,
 							   const QString &writer = QString(),
 							   QString *error = nullptr,
-							   const QByteArray &index = QByteArray(),
 							   const QHash<QString, QByteArray> &kept_from = QHash<QString, QByteArray>());
 	bool join(const QList<QetZip::Entry> &entries, QDomDocument *project,
 			  QString *error = nullptr, QStringList *folio_files = nullptr,
 			  Pictures *pictures = nullptr);
-		/// A folio part of @p files, which join() left out, with its
-		/// pictures and its engineering data back (stage D2)
-		/// @p reader, if any, reads the engineering data of @p files
-	bool readFolio(const QHash<QString, QByteArray> &files, const QString &file,
-				   QDomDocument *folio, QString *error = nullptr,
-				   QetContainerDb::FolioReader *reader = nullptr);
 		/// The file join() left the picture of @p image in, or an empty string
 	QString pictureFile(const QDomElement &image);
 
 		/// Take what QElectroTech computes out of a folio (format 4)
 	void dropComputed(QDomElement diagram, bool terminal_lists);
+		/// true if every folio, symbol and wire has a uuid of its own
+	bool ownUuids(const QDomElement &project);
 		/// Parse @p xml as a .qet is read (QETProject::openFile)
 	bool parse(const QByteArray &xml, QDomDocument *document, QString *error = nullptr);
 		/// true if @p content is a zip, so a .qetz rather than a .qet

sources/container/qetcontainerdb.cpp

@@ -19,7 +19,6 @@
 
 #include "qetcontainer.h"
 
-#include <QRegularExpression>
 #include <QHash>
 #include <QJsonArray>
 #include <QJsonDocument>
@@ -134,9 +133,7 @@ QString schema()
 			.arg(columns(FolioData), columns(ElementData), columns(ConductorData));
 }
 
-	//The tables of a document's engineering data. While the project
-	//database's own tables were in the same file they were named
-	//doc_<name> (stage D3)
+	//The tables of a document's engineering data
 const QStringList DocTables = {
 	QStringLiteral("strip_bridge_terminal"), QStringLiteral("strip_bridge"),
 	QStringLiteral("strip_terminal"), QStringLiteral("terminal_strip"),
@@ -144,26 +141,16 @@ const QStringList DocTables = {
 	QStringLiteral("element_info"), QStringLiteral("element"), QStringLiteral("link"),
 	QStringLiteral("conductor"), QStringLiteral("meta")};
 
-	//Owns an sqlite3 handle; its SQL names the document's tables by their
-	//plain names, given their prefix here, and its columns as the format of
-	//the database read names them
+	//Owns an sqlite3 handle; names its columns as the format of the
+	//database read names them
 class Database
 {
 	public:
 		~Database() { if (m_db) sqlite3_close(m_db); }
 		sqlite3 *m_db = nullptr;
-		QString m_prefix;
 		int m_format = QetContainer::Format;
-		QString sql(const QString &text) const {
-			if (m_prefix.isEmpty()) return text;
-			static const QRegularExpression names(QStringLiteral("(?<![\"\\w])(%1)(?![\\w\"])")
-												  .arg(DocTables.join(QLatin1Char('|'))));
-			QString out = text;
-			return out.replace(names, m_prefix + QStringLiteral("\\1"));
-		}
-		QString table(const QString &name) const { return m_prefix + name; }
-		bool exec(const QString &text) {
-			return sqlite3_exec(m_db, sql(text).toUtf8().constData(), nullptr, nullptr, nullptr) == SQLITE_OK;
+		bool exec(const QString &sql) {
+			return sqlite3_exec(m_db, sql.toUtf8().constData(), nullptr, nullptr, nullptr) == SQLITE_OK;
 		}
 		QString error() const { return QString::fromUtf8(sqlite3_errmsg(m_db)); }
 			//Format 2 (QElectroTech 0.200.1 development builds) keyed its
@@ -184,7 +171,7 @@ class Statement
 {
 	public:
 		Statement(const Database &db, const QString &sql) {
-			sqlite3_prepare_v2(db.m_db, db.sql(sql).toUtf8().constData(), -1, &m_stmt, nullptr);
+			sqlite3_prepare_v2(db.m_db, sql.toUtf8().constData(), -1, &m_stmt, nullptr);
 		}
 		~Statement() { sqlite3_finalize(m_stmt); }
 		bool ok() const { return m_stmt; }
@@ -583,19 +570,11 @@ bool openDatabase(Database &db, const QByteArray &database, QString *error)
 		//A file is untrusted: nothing in its schema may run, and every name
 		//read must be a plain table.
 	db.exec(QStringLiteral("PRAGMA trusted_schema = OFF"));
-		//Beside the project database's own tables the document's are
-		//named doc_<name>
-	{
-		Statement doc(db, QStringLiteral("SELECT count(*) FROM sqlite_master WHERE name = 'doc_meta'"));
-		if (doc.next() && doc.value(0).toInt() > 0) db.m_prefix = QStringLiteral("doc_");
-	}
-	for (const char *table : {"meta", "project_property", "folio", "folio_property",
-							  "element", "element_info", "link", "conductor", "terminal_strip",
-							  "strip_terminal", "strip_bridge", "strip_bridge_terminal"}) {
+	for (const QString &table : DocTables) {
 		Statement kind(db, QStringLiteral("SELECT type FROM sqlite_master WHERE name = ?"));
-		kind.bind({db.table(QString::fromLatin1(table))});
+		kind.bind({table});
 		if (!kind.next() || kind.value(0).toString() != QLatin1String("table")) {
-			setError(error, QStringLiteral("the project database has no table %1").arg(QLatin1String(table)));
+			setError(error, QStringLiteral("the project database has no table %1").arg(table));
 			return false;
 		}
 	}
@@ -755,28 +734,10 @@ bool restoreDiagram(Database &db, QDomElement d, QString *error)
 	folios still in it) into a new database.
 	@return the database file, or an empty array with @p error set
 */
-QByteArray QetContainerDb::extract(QDomElement project, QString *error, const QByteArray &index,
-								   const QByteArray &kept_from)
+QByteArray QetContainerDb::extract(QDomElement project, QString *error, const QByteArray &kept_from)
 {
 	Database db;
-	if (sqlite3_open(":memory:", &db.m_db) != SQLITE_OK) {
-		setError(error, QStringLiteral("cannot create the project database"));
-		return {};
-	}
-		//The project database's own tables first, when given: one file
-		//holds both (stage D3), the document's named doc_<name>
-	if (!index.isEmpty()) {
-		db.m_prefix = QStringLiteral("doc_");
-		unsigned char *copy = static_cast<unsigned char *>(sqlite3_malloc64(sqlite3_uint64(index.size())));
-		if (!copy) { setError(error, QStringLiteral("out of memory")); return {}; }
-		memcpy(copy, index.constData(), size_t(index.size()));
-		if (sqlite3_deserialize(db.m_db, "main", copy, index.size(), index.size(),
-								SQLITE_DESERIALIZE_FREEONCLOSE | SQLITE_DESERIALIZE_RESIZEABLE) != SQLITE_OK) {
-			setError(error, QStringLiteral("cannot read the project database's index"));
-			return {};
-		}
-	}
-	if (!db.exec(schema())) {
+	if (sqlite3_open(":memory:", &db.m_db) != SQLITE_OK || !db.exec(schema())) {
 		setError(error, QStringLiteral("cannot create the project database"));
 		return {};
 	}
@@ -795,7 +756,7 @@ QByteArray QetContainerDb::extract(QDomElement project, QString *error, const QB
 		db.exec(QStringLiteral("PRAGMA trusted_schema = OFF"));
 		for (const QString &table : DocTables) {
 			Statement kind(db, QStringLiteral("SELECT type FROM old.sqlite_master WHERE name = ?"));
-			kind.bind({db.table(table)});
+			kind.bind({table});
 			if (!kind.next() || kind.value(0).toString() != QLatin1String("table")) {
 				setError(error, QStringLiteral("the kept folios' database has no table %1").arg(table));
 				return {};
@@ -1105,58 +1066,6 @@ bool QetContainerDb::restore(QDomElement project, const QByteArray &database, QS
 	return true;
 }
 
-/**
-	@brief QetContainerDb::restoreFolio
-	restore() for the one folio @p diagram, read from its own part (a folio
-	built after its project was opened from a .qetz, stage D2)
-*/
-bool QetContainerDb::restoreFolio(QDomElement diagram, const QByteArray &database, QString *error)
-{
-	Database db;
-	return openDatabase(db, database, error) && restoreDiagram(db, diagram, error);
-}
-
-struct QetContainerDb::FolioReader::Private
-{
-	QByteArray database;
-	Database db;
-	bool opened = false;
-	QString open_error;
-};
-
-/**
-	@brief QetContainerDb::FolioReader::FolioReader
-	Reads folios' engineering data from @p database, opened the first time
-	one is read and kept open: a folio built after the project was opened
-	from a .qetz no longer copies the whole database (stage D5)
-*/
-QetContainerDb::FolioReader::FolioReader(const QByteArray &database) :
-	d(std::make_unique<Private>())
-{
-	d->database = database;
-}
-
-QetContainerDb::FolioReader::~FolioReader() = default;
-
-/**
-	@brief QetContainerDb::FolioReader::restore
-	restoreFolio() for @p diagram
-*/
-bool QetContainerDb::FolioReader::restore(QDomElement diagram, QString *error)
-{
-	if (!d->opened) {
-		d->opened = true;
-		if (!openDatabase(d->db, d->database, &d->open_error) && d->open_error.isEmpty())
-			d->open_error = QStringLiteral("the project database cannot be read");
-		d->database = QByteArray();
-	}
-	if (!d->open_error.isEmpty()) {
-		setError(error, d->open_error);
-		return false;
-	}
-	return restoreDiagram(d->db, diagram, error);
-}
-
 /**
 	@brief QetContainerDb::currentFormat
 	@return true if @p database, a project.sqlite, is of the format this

sources/container/qetcontainerdb.h

@@ -21,8 +21,6 @@
 #include <QByteArray>
 #include <QDomElement>
 
-#include <memory>
-
 /**
 	@brief The QetContainerDb namespace
 	The engineering data of a project, moved between its document and the
@@ -42,21 +40,8 @@
 namespace QetContainerDb
 {
 	QByteArray extract(QDomElement project, QString *error = nullptr,
-					   const QByteArray &index = QByteArray(),
 					   const QByteArray &kept_from = QByteArray());
 	bool restore(QDomElement project, const QByteArray &database, QString *error = nullptr);
-	bool restoreFolio(QDomElement diagram, const QByteArray &database, QString *error = nullptr);
-		/// restoreFolio() for the folios of one database, opened once
-	class FolioReader
-	{
-		public:
-			explicit FolioReader(const QByteArray &database);
-			~FolioReader();
-			bool restore(QDomElement diagram, QString *error = nullptr);
-		private:
-			struct Private;
-			std::unique_ptr<Private> d;
-	};
 		/// true if @p database is of the format this writes
 	bool currentFormat(const QByteArray &database);
 }