Answer numbering, searches and unused symbols from the database (stage D5)

The project database's element table now holds each symbol's definition
(the path its file names it by) and its sequential numbers, filled from
the document and from built folios alike. Numbering, the searches for
symbols of a kind (a link tab's candidates) and the list of unused
symbols read them for the folios not built, instead of reading those
folios' files. Saving looks up a folio's tables in the index it was
opened from.

A .qetz asked its numbering, searches and unused symbols reads only the
files of the folios it builds (tst_lazyfolios, which fails when the
unused symbols are read from the folio files: 50 read, 22 built).
ctest 96/96 eager and lazy; 342 corpus projects save the same in all
four arms.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

sources/dataBase/projectdatabase.cpp

@@ -50,6 +50,8 @@
 #include <QTextDocument>
 #include <QFile>
 #include <QCryptographicHash>
+#include <QJsonDocument>
+#include <QJsonObject>
 #include <QTemporaryDir>
 #include <QRegularExpression>
 #include <QSqlDriver>
@@ -359,6 +361,8 @@ struct DocumentElement
 	QString type;
 	QString sub_type;
 	QVariant group;
+	QString location;
+	autonum::sequentialNumbers sequence;
 	DiagramContext informations;
 	QString label;
 	QHash<QUuid, DocumentTerminal> terminals;
@@ -379,6 +383,27 @@ bool readSequence(const QDomElement &item, autonum::sequentialNumbers *sequence)
 	return true;
 }
 
+	//The sequential values an element was saved with, as Element::fromXml()
+	//reads them, the older attributes included
+autonum::sequentialNumbers elementSequence(const QDomElement &item)
+{
+	autonum::sequentialNumbers sequence;
+	if (readSequence(item, &sequence)) {
+		return sequence;
+	}
+	auto load = [&item](const QString &prefix, QStringList *list) {
+		for (int i = 1 ; !item.attribute(prefix + QString::number(i)).isEmpty() ; ++i)
+			list->append(item.attribute(prefix + QString::number(i)));
+	};
+	load(QStringLiteral("sequ_"),  &sequence.unit);
+	load(QStringLiteral("sequf_"), &sequence.unit_folio);
+	load(QStringLiteral("seqt_"),  &sequence.ten);
+	load(QStringLiteral("seqtf_"), &sequence.ten_folio);
+	load(QStringLiteral("seqh_"),  &sequence.hundred);
+	load(QStringLiteral("seqhf_"), &sequence.hundred_folio);
+	return sequence;
+}
+
 struct DocumentConductor
 {
 	QString uuid;
@@ -609,6 +634,8 @@ bool projectDataBase::populateFromDocument(const QDomDocument &document, QString
 			element.type = known->type;
 			element.sub_type = known->sub_type;
 			element.terminals = known->terminals;
+			element.location = element_xml.attribute(QStringLiteral("type"));
+			element.sequence = elementSequence(element_xml);
 			const QUuid group = ItemGroups::read(element_xml);
 			element.group = group.isNull() ? QVariant() : QVariant(group.toString());
 			element.informations.fromXml(
@@ -810,6 +837,8 @@ bool projectDataBase::populateFromDocument(const QDomDocument &document, QString
 		m_insert_elements_query.bindValue(QStringLiteral(":type"), element.type);
 		m_insert_elements_query.bindValue(QStringLiteral(":sub_type"), element.sub_type);
 		m_insert_elements_query.bindValue(QStringLiteral(":group_uuid"), element.group);
+		m_insert_elements_query.bindValue(QStringLiteral(":definition"), element.location);
+		m_insert_elements_query.bindValue(QStringLiteral(":sequence"), sequenceValue(element.sequence));
 		if (!m_insert_elements_query.exec()) {
 			qDebug() << "projectDataBase::populateFromDocument element insert error : " << m_insert_elements_query.lastError();
 		}
@@ -1207,6 +1236,94 @@ QHash<QUuid, QString> projectDataBase::elementTypesOnFolios(const QSet<QUuid> &f
 	return types;
 }
 
+/**
+	@brief projectDataBase::symbolsOnFolios
+	@return the symbols on the folios @p folios, folio by folio in that
+	order and on each in the order of its rows -- the order the folio
+	places them -- read from the element table as it is (see
+	elementTypesOnFolios()). A symbol whose definition was not found has
+	no row.
+*/
+QList<projectDataBase::FolioSymbol> projectDataBase::symbolsOnFolios(const QList<QUuid> &folios) const
+{
+	QList<FolioSymbol> symbols;
+	QSqlQuery query(m_data_base);
+	query.prepare(QStringLiteral("SELECT uuid, type, definition, sequence FROM element "
+								 "WHERE diagram_uuid = :folio ORDER BY rowid"));
+	for (const QUuid &folio : folios) {
+		query.bindValue(QStringLiteral(":folio"), folio.toString());
+		if (!query.exec()) {
+			continue;
+		}
+		while (query.next()) {
+			symbols.append({QUuid(query.value(0).toString()), folio, query.value(1).toString(),
+							query.value(2).toString(), sequenceFromValue(query.value(3))});
+		}
+	}
+	return symbols;
+}
+
+/**
+	@brief projectDataBase::linkedSymbols
+	@return the symbols @p element is linked to, read from the link table
+	as it is (see elementTypesOnFolios())
+*/
+QList<QUuid> projectDataBase::linkedSymbols(const QUuid &element) const
+{
+	QList<QUuid> linked;
+	QSqlQuery query(m_data_base);
+	query.prepare(QStringLiteral("SELECT linked_uuid FROM link WHERE element_uuid = :uuid ORDER BY rowid"));
+	query.bindValue(QStringLiteral(":uuid"), element.toString());
+	if (query.exec()) {
+		while (query.next())
+			linked.append(QUuid(query.value(0).toString()));
+	}
+	return linked;
+}
+
+/**
+	@brief projectDataBase::sequenceValue
+	@return @p sequence as the element table's sequence column holds it:
+	each part the file would save (sequentialNumbers::toXml()), by its tag,
+	as JSON; null when the file would save none. sequenceFromValue() reads
+	it back as reading the file would (sequentialNumbers::fromXml()).
+*/
+QVariant projectDataBase::sequenceValue(const autonum::sequentialNumbers &sequence)
+{
+	QDomDocument document;
+	const QDomElement xml = sequence.toXml(document);
+	if (!xml.hasChildNodes()) {
+		return QVariant();
+	}
+	QJsonObject parts;
+	for (QDomElement part = xml.firstChildElement() ; !part.isNull() ;
+		 part = part.nextSiblingElement())
+		parts.insert(part.tagName(), part.text());
+	return QString::fromUtf8(QJsonDocument(parts).toJson(QJsonDocument::Compact));
+}
+
+/**
+	@brief projectDataBase::sequenceFromValue
+	@return the sequential numbers held by @p value (sequenceValue())
+*/
+autonum::sequentialNumbers projectDataBase::sequenceFromValue(const QVariant &value)
+{
+	autonum::sequentialNumbers sequence;
+	if (value.isNull()) {
+		return sequence;
+	}
+	const QJsonObject parts = QJsonDocument::fromJson(value.toString().toUtf8()).object();
+	QDomDocument document;
+	QDomElement xml = document.createElement(QStringLiteral("sequentialNumbers"));
+	for (auto it = parts.constBegin() ; it != parts.constEnd() ; ++it) {
+		QDomElement part = document.createElement(it.key());
+		part.appendChild(document.createTextNode(it.value().toString()));
+		xml.appendChild(part);
+	}
+	sequence.fromXml(xml);
+	return sequence;
+}
+
 /**
 	@brief projectDataBase::folioUuidsOfElements
 	@return the uuids of the folios the symbols @p elements are on, read
@@ -2322,6 +2439,8 @@ bool projectDataBase::createDataBase()
 						  "type VARCHAR(50),"
 						  "sub_type VARCHAR(50),"
 						  "group_uuid VARCHAR(50),"
+						  "definition TEXT,"
+						  "sequence TEXT,"
 						  "FOREIGN KEY (diagram_uuid) REFERENCES diagram (uuid)"
 						  ")");
 	if (!query_.exec(element_table)) {
@@ -3985,7 +4104,8 @@ void projectDataBase::prepareQuery()
 	m_diagram_info_order_changed.prepare("UPDATE diagram_info SET folio = :folio WHERE diagram_uuid = :uuid");
 
 		//INSERT ELEMENT
-	QString insert_element_query("INSERT INTO element (uuid, diagram_uuid, pos, type, sub_type, group_uuid) VALUES (:uuid, :diagram_uuid, :pos, :type, :sub_type, :group_uuid)");
+	QString insert_element_query("INSERT INTO element (uuid, diagram_uuid, pos, type, sub_type, group_uuid, definition, sequence) "
+								 "VALUES (:uuid, :diagram_uuid, :pos, :type, :sub_type, :group_uuid, :definition, :sequence)");
 	m_insert_elements_query = QSqlQuery(m_data_base);
 	m_insert_elements_query.prepare(insert_element_query);
 
@@ -4100,6 +4220,8 @@ void projectDataBase::bindElementValues(QSqlQuery &query, Element *element, Diag
 	query.bindValue(QStringLiteral(":type"), element_data.typeToString());
 	query.bindValue(QStringLiteral(":sub_type"), element_data.masterTypeToString());
 	query.bindValue(QStringLiteral(":group_uuid"), groupValue(element));
+	query.bindValue(QStringLiteral(":definition"), element->location().path());
+	query.bindValue(QStringLiteral(":sequence"), sequenceValue(element->sequenceStruct()));
 }
 
 /**

sources/dataBase/projectdatabase.h

@@ -34,6 +34,7 @@
 #include "../diagramcontext.h"
 #include "../titleblockproperties.h"
 #include "../conductorproperties.h"
+#include "../autoNum/assignvariables.h"
 
 class Element;
 class QETProject;
@@ -180,6 +181,18 @@ class projectDataBase : public QObject
 		QSet<QUuid> foliosWithUnjoinedWires() const;
 		QSet<QUuid> folioUuidsOfElements(const QSet<QUuid> &elements) const;
 		QHash<QUuid, QString> elementTypesOnFolios(const QSet<QUuid> &folios) const;
+			/// A symbol placed on a folio, as the element table has it
+		struct FolioSymbol {
+			QUuid uuid;
+			QUuid folio;
+			QString kind;       ///< ElementData::typeToString()
+			QString location;   ///< its definition, ElementsLocation::path()
+			autonum::sequentialNumbers sequence;
+		};
+		QList<FolioSymbol> symbolsOnFolios(const QList<QUuid> &folios) const;
+		QList<QUuid> linkedSymbols(const QUuid &element) const;
+		static QVariant sequenceValue(const autonum::sequentialNumbers &sequence);
+		static autonum::sequentialNumbers sequenceFromValue(const QVariant &value);
 
 		void addElement         (Element *element);
 		void removeElement      (Element *element);

sources/qetproject.cpp

@@ -796,29 +796,22 @@ void QETProject::buildFoliosHolding(ElementData::Types kinds, bool free_only)
 		for (auto it = types.cbegin() ; it != types.cend() ; ++it)
 			placed.insert(it.key());
 	}
+		//The symbols and their links as the database has them
 	QList<Diagram *> to_build;
 	for (Diagram *d : std::as_const(m_diagrams_list))
 	{
 		if (d->isLoaded()) {
 			continue;
 		}
-		for (const QDomElement &element_xml :
-			 QET::findInDomElement(d->deferredXml(),
-								   QStringLiteral("elements"),
-								   QStringLiteral("element")))
+		for (const projectDataBase::FolioSymbol &symbol : m_data_base.symbolsOnFolios({d->uuid()}))
 		{
-			const auto type = types.constFind(QUuid(element_xml.attribute(QStringLiteral("uuid"))));
-			if (type == types.constEnd()
-					|| !(kinds & ElementData::typeFromString(*type))) {
+			if (!(kinds & ElementData::typeFromString(symbol.kind))) {
 				continue;
 			}
 			bool linked = false;
 			if (free_only) {
-				for (const QDomElement &link :
-					 QET::findInDomElement(element_xml,
-										   QStringLiteral("links_uuids"),
-										   QStringLiteral("link_uuid"))) {
-					if (placed.contains(QUuid(link.attribute(QStringLiteral("uuid"))))) {
+				for (const QUuid &link : m_data_base.linkedSymbols(symbol.uuid)) {
+					if (placed.contains(link)) {
 						linked = true;
 						break;
 					}
@@ -844,7 +837,8 @@ void QETProject::buildFoliosHolding(ElementData::Types kinds, bool free_only)
 /**
 	@brief QETProject::buildFoliosHoldingSymbols
 	Build the folios not built yet (QET_LAZY_FOLIOS) holding one of the
-	symbols @p symbols, found in the XML kept for them: what needs those
+	symbols @p symbols, found in the project database once it holds the
+	folios not built, else in the XML kept for them: what needs those
 	symbols while the project is being read, before the database knows
 	where they are (a terminal strip's terminals)
 */
@@ -856,7 +850,7 @@ void QETProject::buildFoliosHoldingSymbols(const QSet<QUuid> &symbols)
 	QList<Diagram *> to_build;
 		//Opened from an index, the database knows where every symbol is
 		//without reading the folios' parts
-	if (m_opened_from_index) {
+	if (m_opened_from_index || m_data_base.rowsFromDocument()) {
 		const QSet<QUuid> folios = m_data_base.folioUuidsOfElements(symbols);
 		for (Diagram *d : std::as_const(m_diagrams_list))
 			if (!d->isLoaded() && folios.contains(d->uuid()))
@@ -1892,7 +1886,7 @@ std::optional<QUuid> QETProject::elementAutoNumIdToLink(
 	the symbol on a folio built and, on a folio not built yet
 	(QET_LAZY_FOLIOS), from what building it would give: its information
 	in the project database's store, the scheme it would be linked to,
-	its sequential numbers and its kind as the file has them.
+	its sequential numbers and its kind as the database has them.
 */
 QList<QETProject::SymbolNumbering> QETProject::symbolNumbering() const
 {
@@ -1923,38 +1917,27 @@ QList<QETProject::SymbolNumbering> QETProject::symbolNumbering() const
 		return list;
 	}
 
-		//A symbol the database has no row for is one building leaves out
-		//(its definition is missing)
-	const QHash<QUuid, QString> types = m_data_base.elementTypesOnFolios(unbuilt);
-	const QHash<QString, QStringList> titles_by_formula = elementAutoNumTitlesByFormula();
+		//The symbols of the folios not built, from the database. A symbol
+		//it has no row for is one building leaves out (its definition is
+		//missing)
+	QList<QUuid> unbuilt_in_order;
 	for (Diagram *d : m_diagrams_list)
+		if (!d->isLoaded()) unbuilt_in_order << d->uuid();
+	const QHash<QString, QStringList> titles_by_formula = elementAutoNumTitlesByFormula();
+	for (const projectDataBase::FolioSymbol &row : m_data_base.symbolsOnFolios(unbuilt_in_order))
 	{
-		if (d->isLoaded()) {
-			continue;
-		}
-		for (const QDomElement &element_xml :
-			 QET::findInDomElement(d->deferredXml(),
-								   QStringLiteral("elements"),
-								   QStringLiteral("element")))
-		{
-			const QUuid uuid(element_xml.attribute(QStringLiteral("uuid")));
-			const auto type = types.constFind(uuid);
-			if (type == types.constEnd()) {
-				continue;
-			}
-			const DiagramContext info = m_data_base.elementInformation(uuid);
-			SymbolNumbering symbol;
-			symbol.label = info.value(QETInformation::ELMT_LABEL).toString();
-			symbol.formula = info.value(QETInformation::ELMT_FORMULA).toString();
-			const auto linked = elementAutoNumIdToLink(info, titles_by_formula);
-			symbol.scheme = linked ? *linked
-								   : QUuid(info.value(QETInformation::ELMT_FORMULA_ID).toString());
-			symbol.sequence.fromXml(element_xml.firstChildElement(QStringLiteral("sequentialNumbers")));
-			const ElementData::Type kind = ElementData::typeFromString(*type);
-			symbol.numbered = kind != ElementData::Slave
-					&& !(kind & ElementData::AllReport);
-			list << symbol;
-		}
+		const DiagramContext info = m_data_base.elementInformation(row.uuid);
+		SymbolNumbering symbol;
+		symbol.label = info.value(QETInformation::ELMT_LABEL).toString();
+		symbol.formula = info.value(QETInformation::ELMT_FORMULA).toString();
+		const auto linked = elementAutoNumIdToLink(info, titles_by_formula);
+		symbol.scheme = linked ? *linked
+							   : QUuid(info.value(QETInformation::ELMT_FORMULA_ID).toString());
+		symbol.sequence = row.sequence;
+		const ElementData::Type kind = ElementData::typeFromString(row.kind);
+		symbol.numbered = kind != ElementData::Slave
+				&& !(kind & ElementData::AllReport);
+		list << symbol;
 	}
 	return list;
 }
@@ -2387,8 +2370,7 @@ QDomDocument QETProject::toXml(QHash<int, QByteArray> *kept_folios, QStringList
 			//give it, which any edit can change
 		if (m_edited_since_open)
 			for (Diagram *diagram : std::as_const(m_diagrams_list))
-				if (!diagram->isLoaded() && !diagram->deferredXml().elementsByTagName(
-						QetGraphicsTableItem::xmlTagName()).isEmpty())
+				if (!diagram->isLoaded() && unbuiltFolioHasTables(diagram))
 					buildFolioToShow(diagram);
 			//A built folio is written from its items, which need their
 			//partners: a table saves the table before it, and in a .qet a
@@ -2996,24 +2978,48 @@ bool QETProject::usesElement(const ElementsLocation &location) const
 	return(unbuiltFolioElements().contains(location));
 }
 
+/**
+	@brief QETProject::unbuiltFolioHasTables
+	@return true if @p folio, not built yet (QET_LAZY_FOLIOS), has a drawn
+	table: from the index it was opened from, else from its XML
+*/
+bool QETProject::unbuiltFolioHasTables(Diagram *folio) const
+{
+	if (m_index_tables) {
+		for (const projectDataBase::FolioTable &table : std::as_const(*m_index_tables))
+			if (table.folio == folio->uuid()) return true;
+		return false;
+	}
+	return !folio->deferredXml().elementsByTagName(QetGraphicsTableItem::xmlTagName()).isEmpty();
+}
+
 /**
 	@brief QETProject::unbuiltFolioElements
 	@return the locations of the symbols placed on the folios not built yet
-	(QET_LAZY_FOLIOS), each once, read from the XML kept for them as
-	building them reads it (Diagram::fromXml())
+	(QET_LAZY_FOLIOS), each once, read from the project database once it
+	holds those folios, else from the XML kept for them as building them
+	reads it (Diagram::fromXml())
 */
 QList<ElementsLocation> QETProject::unbuiltFolioElements() const
 {
 	QSet<QString> types;
-	for (Diagram *diagram : m_diagrams_list) {
-		if (diagram->isLoaded()) {
-			continue;
-		}
-		for (const QDomElement &element_xml :
-			 QET::findInDomElement(diagram->deferredXml(),
-								   QStringLiteral("elements"),
-								   QStringLiteral("element"))) {
-			types.insert(element_xml.attribute(QStringLiteral("type")));
+	if (m_data_base.rowsFromDocument()) {
+		QList<QUuid> unbuilt;
+		for (Diagram *diagram : m_diagrams_list)
+			if (!diagram->isLoaded()) unbuilt << diagram->uuid();
+		for (const projectDataBase::FolioSymbol &symbol : m_data_base.symbolsOnFolios(unbuilt))
+			types.insert(symbol.location);
+	} else {
+		for (Diagram *diagram : m_diagrams_list) {
+			if (diagram->isLoaded()) {
+				continue;
+			}
+			for (const QDomElement &element_xml :
+				 QET::findInDomElement(diagram->deferredXml(),
+									   QStringLiteral("elements"),
+									   QStringLiteral("element"))) {
+				types.insert(element_xml.attribute(QStringLiteral("type")));
+			}
 		}
 	}
 	QList<ElementsLocation> locations;
@@ -3413,6 +3419,7 @@ QDomElement QETProject::readFolioPart(const QString &file)
 	if (!m_folio_reader)
 		m_folio_reader = std::make_unique<QetContainerDb::FolioReader>(
 					m_container_files.value(QStringLiteral("project.sqlite")));
+	++m_folio_parts_read;
 	if (!QetContainer::readFolio(m_container_files, file, &folio, &error, m_folio_reader.get())) {
 		qWarning().noquote() << "Unable to read the folio" << file << ":" << error;
 		return QDomElement();

sources/qetproject.h

@@ -332,6 +332,7 @@ class QETProject : public QObject
 		bool usesElement(const ElementsLocation &) const;
 		QList <ElementsLocation> unusedElements() const;
 		QList<ElementsLocation> unbuiltFolioElements() const;
+		bool unbuiltFolioHasTables(Diagram *folio) const;
 		bool usesTitleBlockTemplate(const TitleBlockTemplateLocation &);
 		bool projectWasModified();
 		bool projectOptionsWereModified();
@@ -455,6 +456,8 @@ class QETProject : public QObject
 		bool writeContainerFile(const QString &path, QString *error = nullptr);
 		QByteArray containerIndex();
 		QDomElement readFolioPart(const QString &file);
+			/// How many folio parts of the .qetz were read so far
+		int folioPartsRead() const {return m_folio_parts_read;}
 		bool openedFromIndex() const {return m_opened_from_index;}
 	private:
 		static QUuid derivedUuid(const QByteArray &content);
@@ -550,6 +553,7 @@ class QETProject : public QObject
 		QHash<QString, QByteArray> m_container_files;
 			///Reads the folio parts' engineering data, opened once
 		std::unique_ptr<QetContainerDb::FolioReader> m_folio_reader;
+		int m_folio_parts_read = 0;
 		QStringList m_folio_files;
 		QList<std::pair<QUuid, QString>> m_folio_headers;
 			///The drawn tables of the index opened from, if it has them

tests/qttest/lazy_folios_probe.cpp

@@ -696,6 +696,35 @@ int main(int argc, char **argv)
 			}
 			check(continued > 0, "a folio whose table continues another");
 
+				//Numbering, the searches for symbols and the unused symbols
+				//are answered from the database: no folio file is read
+				//but those of the folios built (D5)
+			{
+				QETProject asked(zipped);
+				check(numbering(asked) == numbering(eager), "numbering of a .qetz as when built");
+				for (const QString &title : eager.elementAutoNum().keys())
+					check(gaps(asked, title) == gaps(eager, title), "a .qetz scheme's gaps as when built");
+				auto locations = [](const QList<ElementsLocation> &list) {
+					QStringList strings;
+					for (const ElementsLocation &location : list) strings << location.toString();
+					strings.sort();
+					return strings;
+				};
+				check(locations(asked.unusedElements()) == locations(eager.unusedElements()),
+					  "a .qetz's unused symbols as when built");
+				const ElementProvider usual_search(&eager), search(&asked);
+				for (const ElementData::Types kinds : {ElementData::Types(ElementData::Slave),
+													   ElementData::Types(ElementData::Master),
+													   ElementData::Types(ElementData::AllReport)})
+					check(found(search.freeElement(kinds)) == found(usual_search.freeElement(kinds)),
+						  "free symbols of a .qetz found as when built");
+				const int built = int(folios.size()) - asked.unloadedFolioCount();
+				std::printf(".qetz asked numbering, searches, unused symbols: %d folio files read, %d folios built\n",
+							asked.folioPartsRead(), built);
+				check(asked.unloadedFolioCount() > 0, "asking a .qetz does not build every folio");
+				check(asked.folioPartsRead() == built, "asking a .qetz reads no file of a folio not built");
+			}
+
 				//Saved as a .qetz without building the folios not touched
 				//(D4): their parts and rows copied as they were
 			QETProject untouched(zipped);