Keep each fact once in a .qetz: drop the stored project database

A .qetz held the engineering data twice: the document's tables, and
beside them QElectroTech's own project database (stage D2/D3), which
QElectroTech also builds from the document. Two copies of one fact can
disagree, and the copy cost a third of the file. Now project.sqlite holds
the document's tables only; QElectroTech builds its own project database
from them when the project opens, as it does for a .qet.

Measured (this build, lazy folios):
  industrial (50 folios): 620 -> 374 KB, open 0.37 -> 0.65 s
  f2099/nuri (136 folios): 3.27 -> 2.20 MB, open 2.2 -> 3.5 s
The same projects as .qet open in 3.3 s and 23 s.

Saving still copies the file of every folio not built, with its rows
(stage D4): the project keeps the part each folio was read from. nuri
opened and saved again: 6.3 s, no folio built, the same bytes.

A symbol's terminal list is left out of the folio file when every
folio, symbol and wire has a uuid of its own, now checked on the
document (QetContainer::ownUuids()) rather than by the stored database
being written.

Checked: 342 corpus projects open, .qetz -> .qet equals the plain
resave, a .qetz saved twice gives the same bytes; tst_lazyfolios,
tst_qetcontainer, tst_resaveunchanged (which now checks project.sqlite
holds only the document's tables).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

tests/qttest/CMakeLists.txt

@@ -1060,10 +1060,13 @@ endif()
 
 add_executable(
   tst_resaveunchanged
-  tst_resaveunchanged.cpp)
+  tst_resaveunchanged.cpp
+  ${QET_DIR}/sources/container/qetzip.cpp
+  ${QET_DIR}/sources/container/miniz/miniz.c)
 add_test(NAME tst_resaveunchanged COMMAND tst_resaveunchanged)
 add_dependencies(tst_resaveunchanged qelectrotech)
-target_link_libraries(tst_resaveunchanged PRIVATE Qt::Test)
+target_include_directories(tst_resaveunchanged PRIVATE ${QET_DIR}/sources)
+target_link_libraries(tst_resaveunchanged PRIVATE Qt::Test SQLite3::SQLite3)
 target_compile_definitions(tst_resaveunchanged PRIVATE
   "QET_TEST_BINARY_PATH=\"$<TARGET_FILE:qelectrotech>\""
   "QET_EXAMPLES_DIR=\"${QET_DIR}/examples\"")

tests/qttest/lazy_folios_probe.cpp

@@ -288,7 +288,7 @@ static QStringList savedLinks(const QString &qetz, const QString &path)
 		db.setDatabaseName(db_path);
 		check(db.open(), "a .qetz's database opened");
 		QSqlQuery query(db);
-		check(query.exec(QStringLiteral("SELECT element_uuid, linked_uuid FROM doc_link ORDER BY element_uuid, ord")),
+		check(query.exec(QStringLiteral("SELECT element_uuid, linked_uuid FROM link ORDER BY element_uuid, ord")),
 			  "saved links read");
 		while (query.next())
 			links[query.value(0).toString()] << query.value(1).toString();
@@ -750,8 +750,8 @@ int main(int argc, char **argv)
 			check(opened.unloadedFolioCount() == 0, "a project with a shared uuid opens with every folio built");
 			check(built_late == 0, "a project with a shared uuid builds its folios as it reads them, not later");
 		}
-			//A .qetz opened from its index (stage D2): no
-			//folio read, each read from its part when shown, drawn and
+			//A .qetz opened lazily: no folio built but those
+			//the terminal strip needs, each built when shown, drawn and
 			//linked as when built, saved as the .qet is
 		{
 			const QString zipped = output + "/lazy_folios.qetz";
@@ -759,12 +759,11 @@ int main(int argc, char **argv)
 			check(eager.writeContainerFile(zipped, &error), "saved as a .qetz");
 			QETProject indexed(zipped);
 			check(indexed.state() == QETProject::Ok, "the .qetz opens");
-			check(indexed.openedFromIndex(), "the .qetz opens from its index");
-			std::printf(".qetz opened from its index: %d of %d folios not built (strip folios %d)\n",
+			std::printf(".qetz opened: %d of %d folios not built (strip folios %d)\n",
 						indexed.unloadedFolioCount(), int(folios.size()), int(strip_folios.size()));
 			check(indexed.unloadedFolioCount() == folios.size() - strip_folios.size(),
 				  "the .qetz opens building only the folios of the terminal strip's terminals");
-			check(numbering(indexed) == numbering(eager), "symbols' numbering from the index as when built");
+			check(numbering(indexed) == numbering(eager), "symbols' numbering of a .qetz as when built");
 				//The drawn items, through the view made of their tables: the
 				//folios built for it, as for the tables (D6: none were)
 			{
@@ -809,7 +808,7 @@ int main(int argc, char **argv)
 						if (row.startsWith("diagram")) rows << row;
 					return rows;
 				};
-				check(folioRows(indexed) == folioRows(eager), "the folios' rows from the index as when built");
+				check(folioRows(indexed) == folioRows(eager), "the folios' rows of a .qetz as when built");
 					//Folio numbers that count the folios ("%id/%total")
 				const QString totals = output + "/totals.qetz";
 				qputenv("QET_EAGER_FOLIOS", "1");
@@ -823,20 +822,19 @@ int main(int argc, char **argv)
 				QETProject usual_totals(totals);
 				qunsetenv("QET_EAGER_FOLIOS");
 				QETProject lazy_totals(totals);
-				check(lazy_totals.openedFromIndex(), "opened from its index");
+				check(lazy_totals.unloadedFolioCount() > 0, "opened lazily");
 				const QStringList lazy_rows = folioRows(lazy_totals), usual_rows = folioRows(usual_totals);
 				if (lazy_rows != usual_rows)
 					for (int i = 0 ; i < lazy_rows.size() && i < 3 ; ++i)
 						std::printf("  lazily %s\n  built  %s\n", qPrintable(lazy_rows.at(i).left(140)),
 									qPrintable(usual_rows.value(i).left(140)));
-				check(lazy_rows == usual_rows, "folio numbers with totals from the index as when built");
+				check(lazy_rows == usual_rows, "folio numbers with totals of a .qetz as when built");
 			}
 			for (int i = 0; i < folios.size(); ++i)
 				same(indexed, i, "shown from the .qetz");
 			check(indexed.toXml().toString() == eager.toXml().toString(), "the .qetz saves as when built");
 				//A folio whose table continues one on another folio, shown
-				//first: the index, not the folios' parts, says which folios
-				//its table is built with (D5)
+				//first: the folios its table is built with (D5)
 			int continued = 0;
 			for (int i = 0; i < folios.size(); ++i) {
 				bool continues = false;
@@ -852,8 +850,7 @@ int main(int argc, char **argv)
 			check(continued > 0, "a folio whose table continues another");
 
 				//Numbering, the searches for symbols and the unused symbols
-				//are answered from the database: no folio file is read
-				//but those of the folios built (D5)
+				//are answered from the database (D5)
 			{
 				QETProject asked(zipped);
 				check(numbering(asked) == numbering(eager), "numbering of a .qetz as when built");
@@ -874,10 +871,8 @@ int main(int argc, char **argv)
 					check(found(search.freeElement(kinds)) == found(usual_search.freeElement(kinds)),
 						  "free symbols of a .qetz found as when built");
 				const int built = int(folios.size()) - asked.unloadedFolioCount();
-				std::printf(".qetz asked numbering, searches, unused symbols: %d folio files read, %d folios built\n",
-							asked.folioPartsRead(), built);
+				std::printf(".qetz asked numbering, searches, unused symbols: %d folios built\n", built);
 				check(asked.unloadedFolioCount() > 0, "asking a .qetz does not build every folio");
-				check(asked.folioPartsRead() == built, "asking a .qetz reads no file of a folio not built");
 			}
 
 				//A folio added, moved and removed: no other folio built, the

tests/qttest/tst_qetcontainer.cpp

@@ -70,14 +70,15 @@ class tst_qetcontainer : public QObject
 	}
 
 		//@p document without what QElectroTech computes, which a .qetz
-		//does not keep (format 3); tst_resaveunchanged checks that
+		//does not keep (format 4); tst_resaveunchanged checks that
 		//QElectroTech gives it back
 	static QDomDocument withoutComputed(const QDomDocument &document)
 	{
 		QDomDocument copy = document.cloneNode(true).toDocument();
+		const bool own_uuids = QetContainer::ownUuids(copy.documentElement());
 		for (QDomElement diagram = copy.documentElement().firstChildElement(QStringLiteral("diagram"));
 			 !diagram.isNull() ; diagram = diagram.nextSiblingElement(QStringLiteral("diagram")))
-			QetContainer::dropComputed(diagram, false);
+			QetContainer::dropComputed(diagram, own_uuids);
 		return copy;
 	}
 
@@ -243,7 +244,7 @@ private slots:
 		QCOMPARE(count("element_info"), int(original.elementsByTagName(QStringLiteral("elementInformation")).size()));
 		QCOMPARE(count("link"), int(original.elementsByTagName(QStringLiteral("link_uuid")).size()));
 		QCOMPARE(count("conductor"), int(original.elementsByTagName(QStringLiteral("conductor")).size()));
-		QVERIFY(count("folio_property") + count("doc_project_property") > 0);
+		QVERIFY(count("folio_property") + count("project_property") > 0);
 		sqlite3_close(db);
 	}
 

tests/qttest/tst_resaveunchanged.cpp

@@ -8,6 +8,10 @@
 #include <QRegularExpression>
 #include <QTemporaryDir>
 
+#include "container/qetzip.h"
+
+#include <sqlite3.h>
+
 // Saving a project that was just saved must change nothing. Two things
 // made the second save differ from the first, both cleanup done on save
 // but not on load:
@@ -306,6 +310,10 @@ private slots:
 		QVERIFY2(!zipped.isEmpty(), "--resave as .qetz failed");
 		QVERIFY2(read(zipped).startsWith("PK\x03\x04"), "the .qetz is not a zip");
 		QVERIFY2(read(zipped).size() < read(straight).size() / 2, "the .qetz is not smaller");
+		QCOMPARE(databaseTables(zipped), QStringList({
+			"conductor", "element", "element_info", "folio", "folio_property", "link", "meta",
+			"project_property", "strip_bridge", "strip_bridge_terminal", "strip_terminal",
+			"terminal_strip"}));
 		const QString back = resave(zipped);
 		QVERIFY2(!back.isEmpty(), "opening the .qetz failed");
 
@@ -317,6 +325,31 @@ private slots:
 		QCOMPARE(without_save_stamps(read(back)), without_save_stamps(read(straight)));
 	}
 
+	// The tables of the database in the .qetz @p path: the document's
+	// engineering data only, each fact once -- not QElectroTech's own
+	// project database, built from it when the project opens
+	static QStringList databaseTables(const QString &path)
+	{
+		QList<QetZip::Entry> entries;
+		if (!QetZip::fromBytes(read(path), &entries)) return {QStringLiteral("not a zip")};
+		QByteArray database;
+		for (const QetZip::Entry &entry : std::as_const(entries))
+			if (entry.name == QLatin1String("project.sqlite")) database = entry.data;
+		sqlite3 *db = nullptr;
+		sqlite3_open(":memory:", &db);
+		unsigned char *copy = static_cast<unsigned char *>(sqlite3_malloc64(sqlite3_uint64(database.size())));
+		memcpy(copy, database.constData(), size_t(database.size()));
+		sqlite3_deserialize(db, "main", copy, database.size(), database.size(), SQLITE_DESERIALIZE_FREEONCLOSE);
+		sqlite3_stmt *list = nullptr;
+		QStringList tables;
+		sqlite3_prepare_v2(db, "SELECT name FROM sqlite_master WHERE type = 'table' ORDER BY name", -1, &list, nullptr);
+		while (list && sqlite3_step(list) == SQLITE_ROW)
+			tables << QString::fromUtf8(reinterpret_cast<const char *>(sqlite3_column_text(list, 0)));
+		sqlite3_finalize(list);
+		sqlite3_close(db);
+		return tables;
+	}
+
 	// QElectroTech run in a sandbox of its own with @p args, the store
 	// check on (QET_CHECK_ELEMENT_INFO_STORE); its output
 	QString runChecked(const QStringList &args, bool eager = false)