Keep a .qetz project's engineering data in project.sqlite

Fourth stage of the zipped project file (#1440), format 2: what an
engineer would put in a list -- folio title-block fields and
properties, project properties, a symbol's uuid, type, prefix,
information and links, a wire's ends, number, function, colour,
section, cable and bus -- moves out of the XML into project.sqlite in
the .qetz, and lives only there. The folio files keep where things are
drawn. On opening it goes back into the document before the usual
reader runs, so nothing else in QElectroTech changes yet (the program
reading the database directly is the next step of the roadmap).

Each moved item keeps a key in the XML (qetz-key: its uuid where
unique); each moved block leaves a marker where it was, which keeps the
whitespace around it apart. A block of a shape this does not model
stays in the XML. A file is untrusted: the database is read with
trusted_schema off, query_only, and only from names that are plain
tables. Format 2 readers still open format 1; a format 1 reader
refuses format 2.

Tests (tst_qetcontainer): every example back exactly, whitespace
included; databaseHoldsTheData (industrial.qet: the folio files hold no
information, links, wire numbers or properties; the row counts match).
Outside ctest: 341 of 341 projects QET can open give the same .qet
through a format 2 .qetz.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_019Wga9B92BNurDEtxxtCDxf

cmake/qet_compilation_vars.cmake

@@ -168,6 +168,8 @@ set(QET_SRC_FILES
   ${QET_DIR}/sources/container/miniz/miniz.c
   ${QET_DIR}/sources/container/miniz/miniz.h
   ${QET_DIR}/sources/container/qetcontainer.cpp
+  ${QET_DIR}/sources/container/qetcontainerdb.cpp
+  ${QET_DIR}/sources/container/qetcontainerdb.h
   ${QET_DIR}/sources/container/qetcontainer.h
   ${QET_DIR}/sources/container/qetzip.cpp
   ${QET_DIR}/sources/container/qetzip.h

sources/container/qetcontainer.cpp

@@ -16,6 +16,7 @@
 	along with QElectroTech. If not, see <http://www.gnu.org/licenses/>.
 */
 #include "qetcontainer.h"
+#include "qetcontainerdb.h"
 
 #include <QCryptographicHash>
 #include <QHash>
@@ -180,6 +181,11 @@ QList<QetZip::Entry> QetContainer::split(const QDomDocument &project, const QStr
 		return {};
 	}
 
+		//The engineering data out of the document, into project.sqlite
+	const QByteArray database = QetContainerDb::extract(root, error);
+	if (database.isEmpty())
+		return {};
+
 	QList<QetZip::Entry> parts;
 	QSet<QString> taken;
 
@@ -245,6 +251,7 @@ QList<QetZip::Entry> QetContainer::split(const QDomDocument &project, const QStr
 	entries.append(QetZip::Entry{QStringLiteral("mimetype"), QByteArray(MimeType), false});
 	entries.append(QetZip::Entry{QStringLiteral("manifest.xml"), serialize(qetz), true});
 	entries.append(QetZip::Entry{QStringLiteral("project.xml"), serialize(root), true});
+	entries.append(QetZip::Entry{QStringLiteral("project.sqlite"), database, true});
 	entries += parts;
 	for (auto it = pictures.cbegin() ; it != pictures.cend() ; ++it)
 		entries.append(QetZip::Entry{it.key(), it.value(), false});   //PNG is compressed already
@@ -318,5 +325,11 @@ bool QetContainer::join(const QList<QetZip::Entry> &entries, QDomDocument *proje
 		image.insertBefore(project->createTextNode(QString::fromLatin1(files.value(file).toBase64())),
 						   image.firstChild());
 	}
+
+		//The engineering data back from project.sqlite (format 2 on)
+	if (files.contains(QStringLiteral("project.sqlite"))
+		&& !QetContainerDb::restore(project->documentElement(),
+									files.value(QStringLiteral("project.sqlite")), error))
+		return false;
 	return true;
 }

sources/container/qetcontainer.h

@@ -31,6 +31,9 @@
 	  manifest.xml             format and oldest format allowed to read it
 	  project.xml              the project, each part replaced by
 	                           <qetz-part file="..."/>
+	  project.sqlite           the engineering data (format 2,
+	                           QetContainerDb): title-block fields, symbol
+	                           information, links, wires
 	  folios/<uuid>.xml        one per folio
 	  elements/.../<name>      the project's symbol definitions
 	  titleblocks/<name>.titleblock
@@ -44,7 +47,7 @@ namespace QetContainer
 	extern const char MimeType[];
 		/// The format written; files of a later format are refused by join()
 		/// unless their min-reader allows this one.
-	constexpr int Format = 1;
+	constexpr int Format = 2;
 
 	QList<QetZip::Entry> split(const QDomDocument &project,
 							   const QString &writer = QString(),

sources/container/qetcontainerdb.cpp

@@ -0,0 +1,600 @@
+/*
+	Copyright 2006-2026 The QElectroTech Team
+	This file is part of QElectroTech.
+
+	QElectroTech is free software: you can redistribute it and/or modify
+	it under the terms of the GNU General Public License as published by
+	the Free Software Foundation, either version 2 of the License, or
+	(at your option) any later version.
+
+	QElectroTech is distributed in the hope that it will be useful,
+	but WITHOUT ANY WARRANTY; without even the implied warranty of
+	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+	GNU General Public License for more details.
+
+	You should have received a copy of the GNU General Public License
+	along with QElectroTech. If not, see <http://www.gnu.org/licenses/>.
+*/
+#include "qetcontainerdb.h"
+
+#include <QHash>
+#include <QJsonArray>
+#include <QJsonDocument>
+#include <QJsonObject>
+#include <QSet>
+#include <QVariant>
+
+#include <sqlite3.h>
+
+namespace {
+const QString KeyAttribute = QStringLiteral("qetz-key");
+const QString FolioAttribute = QStringLiteral("qetz-folio");
+	//What a moved block leaves where it was. Removing it instead would leave
+	//the whitespace before and after it side by side, read back as one text
+	//node: the parent would come back a child short.
+const QString MarkerTag = QStringLiteral("qetz-db");
+const QString MarkerBlock = QStringLiteral("block");
+
+void leaveMarker(QDomElement block)
+{
+	QDomElement marker = block.ownerDocument().createElement(MarkerTag);
+	marker.setAttribute(MarkerBlock, block.tagName());
+	block.parentNode().replaceChild(marker, block);
+}
+
+	//Put @p block where its marker is in @p parent
+bool replaceMarker(QDomElement parent, const QDomElement &block)
+{
+	for (QDomElement m = parent.firstChildElement(MarkerTag) ; !m.isNull() ; m = m.nextSiblingElement(MarkerTag)) {
+		if (m.attribute(MarkerBlock) == block.tagName()) {
+			parent.replaceChild(block, m);
+			return true;
+		}
+	}
+	return false;
+}
+
+const QStringList FolioData = {
+	QStringLiteral("title"), QStringLiteral("author"), QStringLiteral("date"),
+	QStringLiteral("folio"), QStringLiteral("indexrev"), QStringLiteral("filename"),
+	QStringLiteral("locmach"), QStringLiteral("plant"), QStringLiteral("auto_page_num")};
+const QStringList ElementData = {
+	QStringLiteral("uuid"), QStringLiteral("type"), QStringLiteral("prefix"),
+	QStringLiteral("freezeLabel")};
+const QStringList ConductorData = {
+	QStringLiteral("uuid"), QStringLiteral("element1"), QStringLiteral("terminal1"),
+	QStringLiteral("element2"), QStringLiteral("terminal2"), QStringLiteral("num"),
+	QStringLiteral("formula"), QStringLiteral("function"), QStringLiteral("tension_protocol"),
+	QStringLiteral("conductor_color"), QStringLiteral("conductor_section"),
+	QStringLiteral("cable"), QStringLiteral("bus"), QStringLiteral("freezeLabel")};
+
+QString columns(const QStringList &names)
+{
+	QStringList quoted;
+	for (const QString &n : names) quoted << QStringLiteral("\"%1\" TEXT").arg(n);
+	return quoted.join(QStringLiteral(", "));
+}
+
+QString schema()
+{
+	return QStringLiteral(
+		"CREATE TABLE meta (key TEXT PRIMARY KEY, value TEXT);"
+		"CREATE TABLE project_property (ord INTEGER PRIMARY KEY, name TEXT, value TEXT, attrs TEXT);"
+		"CREATE TABLE folio (id TEXT PRIMARY KEY, pos INTEGER NOT NULL UNIQUE, %1,"
+		" props_at INTEGER, props_attrs TEXT);"
+		"CREATE TABLE folio_property (folio_id TEXT NOT NULL REFERENCES folio, ord INTEGER NOT NULL,"
+		" name TEXT, value TEXT, attrs TEXT, PRIMARY KEY (folio_id, ord));"
+		"CREATE TABLE element (key TEXT PRIMARY KEY, folio_id TEXT NOT NULL REFERENCES folio, %2,"
+		" info_at INTEGER, info_attrs TEXT, links_at INTEGER, links_attrs TEXT);"
+		"CREATE TABLE element_info (element_key TEXT NOT NULL REFERENCES element, ord INTEGER NOT NULL,"
+		" name TEXT, value TEXT, attrs TEXT, PRIMARY KEY (element_key, ord));"
+		"CREATE TABLE link (element_key TEXT NOT NULL REFERENCES element, ord INTEGER NOT NULL,"
+		" linked_uuid TEXT, attrs TEXT, PRIMARY KEY (element_key, ord));"
+		"CREATE TABLE conductor (key TEXT PRIMARY KEY, folio_id TEXT NOT NULL REFERENCES folio, %3);")
+			.arg(columns(FolioData), columns(ElementData), columns(ConductorData));
+}
+
+	//Owns an sqlite3 handle
+class Database
+{
+	public:
+		~Database() { if (m_db) sqlite3_close(m_db); }
+		sqlite3 *m_db = nullptr;
+		bool exec(const QString &sql) {
+			return sqlite3_exec(m_db, sql.toUtf8().constData(), nullptr, nullptr, nullptr) == SQLITE_OK;
+		}
+		QString error() const { return QString::fromUtf8(sqlite3_errmsg(m_db)); }
+};
+
+	//Owns a prepared statement; binds and reads QVariants (null = SQL NULL)
+class Statement
+{
+	public:
+		Statement(sqlite3 *db, const QString &sql) {
+			sqlite3_prepare_v2(db, sql.toUtf8().constData(), -1, &m_stmt, nullptr);
+		}
+		~Statement() { sqlite3_finalize(m_stmt); }
+		bool ok() const { return m_stmt; }
+		void bind(const QVariantList &values) {
+			sqlite3_reset(m_stmt);
+			sqlite3_clear_bindings(m_stmt);
+			for (int i = 0 ; i < values.size() ; ++i) {
+				const QVariant &v = values.at(i);
+				if (v.isNull()) {
+					sqlite3_bind_null(m_stmt, i + 1);
+				} else if (v.typeId() == QMetaType::Int) {
+					sqlite3_bind_int(m_stmt, i + 1, v.toInt());
+				} else {
+					const QByteArray utf8 = v.toString().toUtf8();
+					sqlite3_bind_text(m_stmt, i + 1, utf8.constData(), utf8.size(), SQLITE_TRANSIENT);
+				}
+			}
+		}
+		bool run(const QVariantList &values) {
+			bind(values);
+			return sqlite3_step(m_stmt) == SQLITE_DONE;
+		}
+		bool next() { return sqlite3_step(m_stmt) == SQLITE_ROW; }
+		QVariant value(int column) const {
+			switch (sqlite3_column_type(m_stmt, column)) {
+				case SQLITE_NULL: return QVariant();
+				case SQLITE_INTEGER: return QVariant(sqlite3_column_int(m_stmt, column));
+				default: return QVariant(QString::fromUtf8(
+							reinterpret_cast<const char *>(sqlite3_column_text(m_stmt, column)),
+							sqlite3_column_bytes(m_stmt, column)));
+			}
+		}
+		sqlite3_stmt *m_stmt = nullptr;
+};
+
+void setError(QString *error, const QString &text) { if (error) *error = text; }
+
+QString placeholders(int n)
+{
+	QStringList q;
+	for (int i = 0 ; i < n ; ++i) q << QStringLiteral("?");
+	return q.join(QLatin1Char(','));
+}
+
+	//The attributes of @p e but @p skip, and its tag, as JSON
+QVariant attributesJson(const QDomElement &e, const QString &skip = QString(), bool with_tag = false)
+{
+	QJsonObject o;
+	if (with_tag) o.insert(QStringLiteral("tag"), e.tagName());
+	const QDomNamedNodeMap map = e.attributes();
+	for (int i = 0 ; i < map.count() ; ++i) {
+		const QDomAttr a = map.item(i).toAttr();
+		if (a.name() != skip) o.insert(a.name(), a.value());
+	}
+	if (o.isEmpty()) return QVariant();
+	return QString::fromUtf8(QJsonDocument(o).toJson(QJsonDocument::Compact));
+}
+
+void setAttributesJson(QDomElement e, const QVariant &json)
+{
+	if (json.isNull()) return;
+	const QJsonObject o = QJsonDocument::fromJson(json.toString().toUtf8()).object();
+	for (auto it = o.begin() ; it != o.end() ; ++it)
+		if (it.key() != QLatin1String("tag") && it.key() != QLatin1String("qetz-ws"))
+			e.setAttribute(it.key(), it.value().toString());
+}
+
+	//A list block's attributes, and the whitespace between its entries if
+	//it has any (a document read from a file does; a save has none):
+	//"qetz-ws" holds, for each entry and after the last, the text before it.
+QVariant blockJson(const QDomElement &block)
+{
+	QJsonObject o;
+	const QDomNamedNodeMap map = block.attributes();
+	for (int i = 0 ; i < map.count() ; ++i) {
+		const QDomAttr a = map.item(i).toAttr();
+		o.insert(a.name(), a.value());
+	}
+	QJsonArray ws;
+	QString pending;
+	bool any = false;
+	for (QDomNode n = block.firstChild() ; !n.isNull() ; n = n.nextSibling()) {
+		if (n.isText()) { pending += n.nodeValue(); any = true; }
+		else { ws.append(pending); pending.clear(); }
+	}
+	ws.append(pending);
+	if (any) o.insert(QStringLiteral("qetz-ws"), ws);
+	if (o.isEmpty()) return QVariant();
+	return QString::fromUtf8(QJsonDocument(o).toJson(QJsonDocument::Compact));
+}
+
+	//The whitespace blockJson() kept, back between the entries of @p block
+void restoreWhitespace(QDomElement block, const QVariant &json)
+{
+	if (json.isNull()) return;
+	const QJsonArray ws = QJsonDocument::fromJson(json.toString().toUtf8()).object()
+			.value(QStringLiteral("qetz-ws")).toArray();
+	if (ws.isEmpty()) return;
+	QDomDocument document = block.ownerDocument();
+	QList<QDomNode> entries;
+	for (QDomNode n = block.firstChild() ; !n.isNull() ; n = n.nextSibling()) entries << n;
+	for (int i = 0 ; i < entries.size() && i < ws.size() ; ++i)
+		if (!ws.at(i).toString().isEmpty())
+			block.insertBefore(document.createTextNode(ws.at(i).toString()), entries.at(i));
+	if (ws.size() > entries.size() && !ws.last().toString().isEmpty())
+		block.appendChild(document.createTextNode(ws.last().toString()));
+}
+
+	//The value of a list entry: its only child, a text node, or none
+bool simpleValue(const QDomElement &e, QVariant *value)
+{
+	const QDomNodeList kids = e.childNodes();
+	if (kids.size() == 0) { *value = QVariant(); return true; }
+	if (kids.size() == 1 && kids.at(0).isText() && !kids.at(0).isCDATASection()) {
+		*value = kids.at(0).nodeValue();
+		return true;
+	}
+	return false;
+}
+
+	//A list block -- <properties>, <elementInformations>, <links_uuids> --
+	//this can move: entries named @p entry with simple values, and
+	//whitespace between them that a save would write anyway.
+bool listBlock(const QDomElement &block, const QString &entry)
+{
+	for (QDomNode n = block.firstChild() ; !n.isNull() ; n = n.nextSibling()) {
+		if (n.isElement()) {
+			QVariant v;
+			if (n.toElement().tagName() != entry || !simpleValue(n.toElement(), &v)) return false;
+		} else if (!(n.isText() && !n.isCDATASection() && n.nodeValue().trimmed().isEmpty())) {
+			return false;
+		}
+	}
+	return true;
+}
+
+int childIndex(const QDomNode &child)
+{
+	int i = 0;
+	for (QDomNode n = child.parentNode().firstChild() ; !n.isNull() ; n = n.nextSibling(), ++i)
+		if (n == child) return i;
+	return -1;
+}
+
+
+QVariantList take(QDomElement e, const QStringList &names)
+{
+	QVariantList values;
+	for (const QString &n : names) {
+		values << (e.hasAttribute(n) ? QVariant(e.attribute(n)) : QVariant());
+		e.removeAttribute(n);
+	}
+	return values;
+}
+
+void put(QDomElement e, const QStringList &names, const QVariantList &values, int from)
+{
+	for (int i = 0 ; i < names.size() ; ++i)
+		if (!values.at(from + i).isNull()) e.setAttribute(names.at(i), values.at(from + i).toString());
+}
+
+QList<QDomElement> children(const QDomElement &parent, const QString &tag)
+{
+	QList<QDomElement> list;
+	for (QDomElement e = parent.firstChildElement(tag) ; !e.isNull() ; e = e.nextSiblingElement(tag))
+		list << e;
+	return list;
+}
+
+	//uuid when present and unique among @p items, else prefix + count
+QStringList keys(const QList<QDomElement> &items, const QString &prefix)
+{
+	QHash<QString, int> seen;
+	for (const QDomElement &e : items)
+		if (e.hasAttribute(QStringLiteral("uuid"))) ++seen[e.attribute(QStringLiteral("uuid"))];
+	QStringList out;
+	int made_up = 0;
+	for (const QDomElement &e : items) {
+		const QString uuid = e.attribute(QStringLiteral("uuid"));
+		out << ((!uuid.isEmpty() && seen.value(uuid) == 1)
+				? uuid : prefix + QString::number(++made_up));
+	}
+	return out;
+}
+}
+
+/**
+	@brief QetContainerDb::extract
+	Move the engineering data of @p project (the <project> element, its
+	folios still in it) into a new database.
+	@return the database file, or an empty array with @p error set
+*/
+QByteArray QetContainerDb::extract(QDomElement project, QString *error)
+{
+	Database db;
+	if (sqlite3_open(":memory:", &db.m_db) != SQLITE_OK || !db.exec(schema())) {
+		setError(error, QStringLiteral("cannot create the project database"));
+		return {};
+	}
+	db.exec(QStringLiteral("BEGIN"));
+
+	Statement meta(db.m_db, QStringLiteral("INSERT INTO meta VALUES (?,?)"));
+	Statement project_property(db.m_db, QStringLiteral("INSERT INTO project_property VALUES (?,?,?,?)"));
+	Statement folio(db.m_db, QStringLiteral("INSERT INTO folio VALUES (?,?,%1,?,?)").arg(placeholders(FolioData.size())));
+	Statement folio_property(db.m_db, QStringLiteral("INSERT INTO folio_property VALUES (?,?,?,?,?)"));
+	Statement element(db.m_db, QStringLiteral("INSERT INTO element VALUES (?,?,%1,?,?,?,?)").arg(placeholders(ElementData.size())));
+	Statement element_info(db.m_db, QStringLiteral("INSERT INTO element_info VALUES (?,?,?,?,?)"));
+	Statement link(db.m_db, QStringLiteral("INSERT INTO link VALUES (?,?,?,?)"));
+	Statement conductor(db.m_db, QStringLiteral("INSERT INTO conductor VALUES (?,?,%1)").arg(placeholders(ConductorData.size())));
+	bool ok = meta.ok() && project_property.ok() && folio.ok() && folio_property.ok()
+			&& element.ok() && element_info.ok() && link.ok() && conductor.ok();
+
+	ok = ok && meta.run({QStringLiteral("format"), QStringLiteral("2")});
+
+		//<properties> of a block, as rows of @p rows (with @p prefix values)
+	auto moveProperties = [&](QDomElement block, Statement &rows, const QVariantList &prefix) {
+		int ord = 0;
+		for (QDomElement p : children(block, QStringLiteral("property"))) {
+			QVariant value;
+			simpleValue(p, &value);
+			ok = ok && rows.run(QVariantList(prefix) << ord++ << (p.hasAttribute(QStringLiteral("name"))
+													   ? QVariant(p.attribute(QStringLiteral("name"))) : QVariant())
+							   << value << attributesJson(p, QStringLiteral("name"), true));
+		}
+	};
+
+		//Project properties
+	const QDomElement project_props = project.firstChildElement(QStringLiteral("properties"));
+	if (!project_props.isNull() && listBlock(project_props, QStringLiteral("property"))) {
+		ok = ok && meta.run({QStringLiteral("props_at"), QString::number(childIndex(project_props))});
+		ok = ok && meta.run({QStringLiteral("props_attrs"), blockJson(project_props)});
+		int ord = 0;
+		for (QDomElement p : children(project_props, QStringLiteral("property"))) {
+			QVariant value;
+			simpleValue(p, &value);
+			ok = ok && project_property.run({ord++, p.hasAttribute(QStringLiteral("name"))
+												? QVariant(p.attribute(QStringLiteral("name"))) : QVariant(),
+											 value, attributesJson(p, QStringLiteral("name"), true)});
+		}
+		leaveMarker(project_props);
+	}
+
+	const QList<QDomElement> diagrams = children(project, QStringLiteral("diagram"));
+	QList<QDomElement> all_elements, all_conductors;
+	for (const QDomElement &d : diagrams) {
+		for (const QDomElement &list : children(d, QStringLiteral("elements")))
+			all_elements += children(list, QStringLiteral("element"));
+		for (const QDomElement &list : children(d, QStringLiteral("conductors")))
+			all_conductors += children(list, QStringLiteral("conductor"));
+	}
+	const QStringList element_keys = keys(all_elements, QStringLiteral("e"));
+	const QStringList conductor_keys = keys(all_conductors, QStringLiteral("c"));
+	int e_index = 0, c_index = 0;
+
+	int pos = 0;
+	for (QDomElement d : diagrams)
+	{
+		const QString id = QStringLiteral("f%1").arg(++pos);
+		QVariantList row{id, pos};
+		row += take(d, FolioData);
+		QDomElement props = d.firstChildElement(QStringLiteral("properties"));
+		const bool move_props = !props.isNull() && listBlock(props, QStringLiteral("property"));
+		row << (move_props ? QVariant(childIndex(props)) : QVariant())
+			<< (move_props ? blockJson(props) : QVariant());
+		ok = ok && folio.run(row);
+		if (move_props) {
+			moveProperties(props, folio_property, {id});
+			leaveMarker(props);
+		}
+		d.setAttribute(FolioAttribute, id);
+
+		for (const QDomElement &list : children(d, QStringLiteral("elements"))) {
+			for (QDomElement e : children(list, QStringLiteral("element"))) {
+				const QString key = element_keys.at(e_index++);
+				QVariantList erow{key, id};
+				erow += take(e, ElementData);
+				QDomElement info = e.firstChildElement(QStringLiteral("elementInformations"));
+				QDomElement links = e.firstChildElement(QStringLiteral("links_uuids"));
+				if (!info.isNull() && !listBlock(info, QStringLiteral("elementInformation"))) info = QDomElement();
+				if (!links.isNull() && !listBlock(links, QStringLiteral("link_uuid"))) links = QDomElement();
+				const int info_at = info.isNull() ? -1 : childIndex(info);
+				const int links_at = links.isNull() ? -1 : childIndex(links);
+				erow << (info_at < 0 ? QVariant() : QVariant(info_at))
+					 << (info_at < 0 ? QVariant() : blockJson(info))
+					 << (links_at < 0 ? QVariant() : QVariant(links_at))
+					 << (links_at < 0 ? QVariant() : blockJson(links));
+				ok = ok && element.run(erow);
+				if (info_at >= 0) {
+					int ord = 0;
+					for (QDomElement i : children(info, QStringLiteral("elementInformation"))) {
+						QVariant value;
+						simpleValue(i, &value);
+						ok = ok && element_info.run({key, ord++, i.hasAttribute(QStringLiteral("name"))
+														? QVariant(i.attribute(QStringLiteral("name"))) : QVariant(),
+													 value, attributesJson(i, QStringLiteral("name"), true)});
+					}
+				}
+				if (links_at >= 0) {
+					int ord = 0;
+					for (QDomElement l : children(links, QStringLiteral("link_uuid")))
+						ok = ok && link.run({key, ord++, l.hasAttribute(QStringLiteral("uuid"))
+												? QVariant(l.attribute(QStringLiteral("uuid"))) : QVariant(),
+											 attributesJson(l, QStringLiteral("uuid"), true)});
+				}
+				if (info_at >= 0) leaveMarker(info);
+				if (links_at >= 0) leaveMarker(links);
+				e.setAttribute(KeyAttribute, key);
+			}
+		}
+		for (const QDomElement &list : children(d, QStringLiteral("conductors"))) {
+			for (QDomElement c : children(list, QStringLiteral("conductor"))) {
+				const QString key = conductor_keys.at(c_index++);
+				QVariantList crow{key, id};
+				crow += take(c, ConductorData);
+				ok = ok && conductor.run(crow);
+				c.setAttribute(KeyAttribute, key);
+			}
+		}
+	}
+
+	if (!ok || !db.exec(QStringLiteral("COMMIT"))) {
+		setError(error, QStringLiteral("cannot fill the project database: %1").arg(db.error()));
+		return {};
+	}
+
+	sqlite3_int64 size = 0;
+	unsigned char *bytes = sqlite3_serialize(db.m_db, "main", &size, 0);
+	if (!bytes) {
+		setError(error, QStringLiteral("cannot write the project database"));
+		return {};
+	}
+	const QByteArray file(reinterpret_cast<const char *>(bytes), qsizetype(size));
+	sqlite3_free(bytes);
+	return file;
+}
+
+/**
+	@brief QetContainerDb::restore
+	Put the engineering data in @p database back into @p project (the
+	<project> element, its folios joined back already).
+*/
+bool QetContainerDb::restore(QDomElement project, const QByteArray &database, QString *error)
+{
+	QDomDocument document = project.ownerDocument();
+	Database db;
+	if (sqlite3_open(":memory:", &db.m_db) != SQLITE_OK) {
+		setError(error, QStringLiteral("cannot open the project database"));
+		return false;
+	}
+	unsigned char *copy = static_cast<unsigned char *>(sqlite3_malloc64(sqlite3_uint64(database.size())));
+	if (!copy) { setError(error, QStringLiteral("out of memory")); return false; }
+	memcpy(copy, database.constData(), size_t(database.size()));
+	if (sqlite3_deserialize(db.m_db, "main", copy, database.size(), database.size(),
+							SQLITE_DESERIALIZE_FREEONCLOSE | SQLITE_DESERIALIZE_READONLY) != SQLITE_OK) {
+		setError(error, QStringLiteral("the project database cannot be read"));
+		return false;
+	}
+		//A file is untrusted: nothing in its schema may run, and every name
+		//read must be a plain table.
+	db.exec(QStringLiteral("PRAGMA trusted_schema = OFF"));
+	db.exec(QStringLiteral("PRAGMA query_only = ON"));
+	for (const char *table : {"meta", "project_property", "folio", "folio_property",
+							  "element", "element_info", "link", "conductor"}) {
+		Statement kind(db.m_db, QStringLiteral("SELECT type FROM sqlite_master WHERE name = ?"));
+		kind.bind({QString::fromLatin1(table)});
+		if (!kind.next() || kind.value(0).toString() != QLatin1String("table")) {
+			setError(error, QStringLiteral("the project database has no table %1").arg(QLatin1String(table)));
+			return false;
+		}
+	}
+
+	auto entry = [&document](const QVariant &name_attr, const QString &name, const QVariant &value,
+							 const QVariant &attrs, const QString &default_tag) {
+		const QJsonObject o = QJsonDocument::fromJson(attrs.toString().toUtf8()).object();
+		QDomElement e = document.createElement(o.value(QStringLiteral("tag")).toString(default_tag));
+		if (!name_attr.isNull()) e.setAttribute(name, name_attr.toString());
+		setAttributesJson(e, attrs);
+		if (!value.isNull()) e.appendChild(document.createTextNode(value.toString()));
+		return e;
+	};
+
+		//Project properties
+	QHash<QString, QVariant> meta;
+	{
+		Statement s(db.m_db, QStringLiteral("SELECT key, value FROM meta"));
+		while (s.next()) meta.insert(s.value(0).toString(), s.value(1));
+	}
+	if (meta.contains(QStringLiteral("props_at"))) {
+		QDomElement props = document.createElement(QStringLiteral("properties"));
+		setAttributesJson(props, meta.value(QStringLiteral("props_attrs")));
+		Statement s(db.m_db, QStringLiteral("SELECT name, value, attrs FROM project_property ORDER BY ord"));
+		while (s.next())
+			props.appendChild(entry(s.value(0), QStringLiteral("name"), s.value(1), s.value(2),
+									QStringLiteral("property")));
+		restoreWhitespace(props, meta.value(QStringLiteral("props_attrs")));
+		if (!replaceMarker(project, props)) {
+			setError(error, QStringLiteral("the project's properties have no place"));
+			return false;
+		}
+	}
+
+	for (QDomElement d : children(project, QStringLiteral("diagram")))
+	{
+		const QString id = d.attribute(FolioAttribute);
+		if (id.isEmpty()) continue;
+		d.removeAttribute(FolioAttribute);
+		Statement f(db.m_db, QStringLiteral("SELECT %1, props_at, props_attrs FROM folio WHERE id = ?")
+					.arg(QStringLiteral("\"") + FolioData.join(QStringLiteral("\", \"")) + QStringLiteral("\"")));
+		f.bind({id});
+		if (!f.next()) { setError(error, QStringLiteral("folio %1 is not in the database").arg(id)); return false; }
+		QVariantList values;
+		for (int i = 0 ; i < FolioData.size() + 2 ; ++i) values << f.value(i);
+		put(d, FolioData, values, 0);
+		if (!values.at(FolioData.size()).isNull()) {
+			QDomElement props = document.createElement(QStringLiteral("properties"));
+			setAttributesJson(props, values.at(FolioData.size() + 1));
+			Statement p(db.m_db, QStringLiteral("SELECT name, value, attrs FROM folio_property WHERE folio_id = ? ORDER BY ord"));
+			p.bind({id});
+			while (p.next())
+				props.appendChild(entry(p.value(0), QStringLiteral("name"), p.value(1), p.value(2),
+										QStringLiteral("property")));
+			restoreWhitespace(props, values.at(FolioData.size() + 1));
+			if (!replaceMarker(d, props)) {
+				setError(error, QStringLiteral("folio %1's properties have no place").arg(id));
+				return false;
+			}
+		}
+
+		for (const QDomElement &list : children(d, QStringLiteral("elements"))) {
+			for (QDomElement e : children(list, QStringLiteral("element"))) {
+				if (!e.hasAttribute(KeyAttribute)) continue;
+				const QString key = e.attribute(KeyAttribute);
+				e.removeAttribute(KeyAttribute);
+				Statement s(db.m_db, QStringLiteral("SELECT %1, info_at, info_attrs, links_at, links_attrs FROM element WHERE key = ?")
+							.arg(QStringLiteral("\"") + ElementData.join(QStringLiteral("\", \"")) + QStringLiteral("\"")));
+				s.bind({key});
+				if (!s.next()) { setError(error, QStringLiteral("symbol %1 is not in the database").arg(key)); return false; }
+				QVariantList v;
+				for (int i = 0 ; i < ElementData.size() + 4 ; ++i) v << s.value(i);
+				put(e, ElementData, v, 0);
+				const int n = ElementData.size();
+				QDomElement info, links;
+				if (!v.at(n).isNull()) {
+					info = document.createElement(QStringLiteral("elementInformations"));
+					setAttributesJson(info, v.at(n + 1));
+					Statement i(db.m_db, QStringLiteral("SELECT name, value, attrs FROM element_info WHERE element_key = ? ORDER BY ord"));
+					i.bind({key});
+					while (i.next())
+						info.appendChild(entry(i.value(0), QStringLiteral("name"), i.value(1), i.value(2),
+											   QStringLiteral("elementInformation")));
+					restoreWhitespace(info, v.at(n + 1));
+				}
+				if (!v.at(n + 2).isNull()) {
+					links = document.createElement(QStringLiteral("links_uuids"));
+					setAttributesJson(links, v.at(n + 3));
+					Statement l(db.m_db, QStringLiteral("SELECT linked_uuid, attrs FROM link WHERE element_key = ? ORDER BY ord"));
+					l.bind({key});
+					while (l.next())
+						links.appendChild(entry(l.value(0), QStringLiteral("uuid"), QVariant(), l.value(1),
+												QStringLiteral("link_uuid")));
+					restoreWhitespace(links, v.at(n + 3));
+				}
+				if ((!info.isNull() && !replaceMarker(e, info))
+					|| (!links.isNull() && !replaceMarker(e, links))) {
+					setError(error, QStringLiteral("symbol %1's information has no place").arg(key));
+					return false;
+				}
+			}
+		}
+		for (const QDomElement &list : children(d, QStringLiteral("conductors"))) {
+			for (QDomElement c : children(list, QStringLiteral("conductor"))) {
+				if (!c.hasAttribute(KeyAttribute)) continue;
+				const QString key = c.attribute(KeyAttribute);
+				c.removeAttribute(KeyAttribute);
+				Statement s(db.m_db, QStringLiteral("SELECT %1 FROM conductor WHERE key = ?")
+							.arg(QStringLiteral("\"") + ConductorData.join(QStringLiteral("\", \"")) + QStringLiteral("\"")));
+				s.bind({key});
+				if (!s.next()) { setError(error, QStringLiteral("wire %1 is not in the database").arg(key)); return false; }
+				QVariantList v;
+				for (int i = 0 ; i < ConductorData.size() ; ++i) v << s.value(i);
+				put(c, ConductorData, v, 0);
+			}
+		}
+	}
+	return true;
+}

sources/container/qetcontainerdb.h

@@ -0,0 +1,44 @@
+/*
+	Copyright 2006-2026 The QElectroTech Team
+	This file is part of QElectroTech.
+
+	QElectroTech is free software: you can redistribute it and/or modify
+	it under the terms of the GNU General Public License as published by
+	the Free Software Foundation, either version 2 of the License, or
+	(at your option) any later version.
+
+	QElectroTech is distributed in the hope that it will be useful,
+	but WITHOUT ANY WARRANTY; without even the implied warranty of
+	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+	GNU General Public License for more details.
+
+	You should have received a copy of the GNU General Public License
+	along with QElectroTech. If not, see <http://www.gnu.org/licenses/>.
+*/
+#ifndef QETCONTAINERDB_H
+#define QETCONTAINERDB_H
+
+#include <QByteArray>
+#include <QDomElement>
+
+/**
+	@brief The QetContainerDb namespace
+	The engineering data of a project, moved between its document and the
+	project.sqlite of a .qetz: what a folio's title block says, a symbol's
+	information, a coil's contacts, a wire's ends and number. In a .qetz
+	these live only in the database; the folio files keep where things are
+	drawn. extract() takes them out of the document into a database,
+	restore() puts them back, so that restore(extract(d)) is d.
+
+	Each moved item leaves its key in the document (qetz-key, qetz-folio):
+	its uuid where that is unique, else a made-up one. A block of a shape
+	this does not model -- a value holding elements, say -- stays in the
+	document, so nothing can be lost by moving.
+*/
+namespace QetContainerDb
+{
+	QByteArray extract(QDomElement project, QString *error = nullptr);
+	bool restore(QDomElement project, const QByteArray &database, QString *error = nullptr);
+}
+
+#endif // QETCONTAINERDB_H

tests/qttest/CMakeLists.txt

@@ -1180,10 +1180,11 @@ add_executable(
   tst_qetcontainer
   tst_qetcontainer.cpp
   ${QET_DIR}/sources/container/qetcontainer.cpp
+  ${QET_DIR}/sources/container/qetcontainerdb.cpp
   ${QET_DIR}/sources/container/qetzip.cpp
   ${QET_DIR}/sources/container/miniz/miniz.c)
 add_test(NAME tst_qetcontainer COMMAND tst_qetcontainer)
 target_include_directories(tst_qetcontainer PRIVATE ${QET_DIR}/sources)
-target_link_libraries(tst_qetcontainer PRIVATE Qt::Test Qt::Xml)
+target_link_libraries(tst_qetcontainer PRIVATE Qt::Test Qt::Xml SQLite3::SQLite3)
 target_compile_definitions(tst_qetcontainer PRIVATE
   "QET_EXAMPLES_DIR=\"${QET_DIR}/examples\"")

tests/qttest/tst_qetcontainer.cpp

@@ -8,6 +8,8 @@
 
 #include "container/qetcontainer.h"
 
+#include <sqlite3.h>
+
 // A project document split into the parts of a .qetz, zipped, unzipped and
 // joined back is the document it was: every example, and every .qet in the
 // folder QET_CONTAINER_CORPUS names, if set -- exactly, whitespace text
@@ -132,6 +134,53 @@ private slots:
 		QCOMPARE(whitespace_nodes(back), 0);
 	}
 
+	// The engineering data is in project.sqlite and nowhere else: the folio
+	// files hold no symbol information, links, wire numbers or title-block
+	// fields, and the database holds them all (industrial.qet: 50 folios).
+	void databaseHoldsTheData()
+	{
+		QDomDocument original;
+		const QList<QetZip::Entry> entries = splitFile(QStringLiteral(QET_EXAMPLES_DIR "/industrial.qet"), &original);
+		QVERIFY(!entries.isEmpty());
+		QByteArray database;
+		for (const QetZip::Entry &e : entries) {
+			if (e.name == QLatin1String("project.sqlite")) database = e.data;
+			if (e.name.startsWith(QStringLiteral("folios/")) || e.name == QLatin1String("project.xml")) {
+				QVERIFY2(!e.data.contains("<elementInformation "), qPrintable(e.name));
+				QVERIFY2(!e.data.contains("<link_uuid "), qPrintable(e.name));
+				QVERIFY2(!e.data.contains("<property "), qPrintable(e.name));
+				QVERIFY2(!QString::fromUtf8(e.data).contains(QRegularExpression(QStringLiteral("<conductor [^>]* num="))),
+						 qPrintable(e.name));
+			}
+		}
+		QVERIFY(database.startsWith("SQLite format 3"));
+
+		sqlite3 *db = nullptr;
+		QCOMPARE(sqlite3_open(":memory:", &db), SQLITE_OK);
+		unsigned char *copy = static_cast<unsigned char *>(sqlite3_malloc64(sqlite3_uint64(database.size())));
+		memcpy(copy, database.constData(), size_t(database.size()));
+		QCOMPARE(sqlite3_deserialize(db, "main", copy, database.size(), database.size(),
+									 SQLITE_DESERIALIZE_FREEONCLOSE), SQLITE_OK);
+		auto count = [db](const char *table) {
+			sqlite3_stmt *st = nullptr;
+			sqlite3_prepare_v2(db, (QByteArray("SELECT count(*) FROM ") + table).constData(), -1, &st, nullptr);
+			sqlite3_step(st);
+			const int n = sqlite3_column_int(st, 0);
+			sqlite3_finalize(st);
+			return n;
+		};
+		const QDomNodeList diagrams = original.elementsByTagName(QStringLiteral("diagram"));
+		QCOMPARE(count("folio"), int(diagrams.size()));
+		QCOMPARE(count("element"), int(original.elementsByTagName(QStringLiteral("element")).size())
+				 - int(original.elementsByTagName(QStringLiteral("collection")).at(0).toElement()
+					   .elementsByTagName(QStringLiteral("element")).size()));
+		QCOMPARE(count("element_info"), int(original.elementsByTagName(QStringLiteral("elementInformation")).size()));
+		QCOMPARE(count("link"), int(original.elementsByTagName(QStringLiteral("link_uuid")).size()));
+		QCOMPARE(count("conductor"), int(original.elementsByTagName(QStringLiteral("conductor")).size()));
+		QVERIFY(count("folio_property") + count("project_property") > 0);
+		sqlite3_close(db);
+	}
+
 	// The folio pictures leave the folios, once each
 	void picturesAreFiles()
 	{
@@ -161,7 +210,8 @@ private slots:
 		QVERIFY(!entries.isEmpty());
 		for (QetZip::Entry &e : entries)
 			if (e.name == QLatin1String("manifest.xml"))
-				e.data.replace("min-reader=\"1\"", "min-reader=\"9\"");
+				e.data = QString::fromUtf8(e.data).replace(QRegularExpression(QStringLiteral("min-reader=\"\\d+\"")),
+														   QStringLiteral("min-reader=\"9\"")).toUtf8();
 		QDomDocument back;
 		QString error;
 		QVERIFY(!QetContainer::join(entries, &back, &error));