From 8ae937007c2ce88214ff989d33d4d9553ccf93c0 Mon Sep 17 00:00:00 2001 From: ispyisail Date: Mon, 12 Oct 2026 07:37:26 +1300 Subject: [PATCH] Answer numbering, searches and unused symbols from the database (stage D5) The project database's element table now holds each symbol's definition (the path its file names it by) and its sequential numbers, filled from the document and from built folios alike. Numbering, the searches for symbols of a kind (a link tab's candidates) and the list of unused symbols read them for the folios not built, instead of reading those folios' files. Saving looks up a folio's tables in the index it was opened from. A .qetz asked its numbering, searches and unused symbols reads only the files of the folios it builds (tst_lazyfolios, which fails when the unused symbols are read from the folio files: 50 read, 22 built). ctest 96/96 eager and lazy; 342 corpus projects save the same in all four arms. Co-Authored-By: Claude Opus 5.5 --- sources/dataBase/projectdatabase.cpp | 124 ++++++++++++++++++++++++++- sources/dataBase/projectdatabase.h | 13 +++ sources/qetproject.cpp | 123 +++++++++++++------------- sources/qetproject.h | 4 + tests/qttest/lazy_folios_probe.cpp | 29 +++++++ 5 files changed, 234 insertions(+), 59 deletions(-) diff --git a/sources/dataBase/projectdatabase.cpp b/sources/dataBase/projectdatabase.cpp index f43af8179..80c5d1ec5 100644 --- a/sources/dataBase/projectdatabase.cpp +++ b/sources/dataBase/projectdatabase.cpp @@ -50,6 +50,8 @@ #include #include #include +#include +#include #include #include #include @@ -359,6 +361,8 @@ struct DocumentElement QString type; QString sub_type; QVariant group; + QString location; + autonum::sequentialNumbers sequence; DiagramContext informations; QString label; QHash terminals; @@ -379,6 +383,27 @@ bool readSequence(const QDomElement &item, autonum::sequentialNumbers *sequence) return true; } + //The sequential values an element was saved with, as Element::fromXml() + //reads them, the older attributes included +autonum::sequentialNumbers elementSequence(const QDomElement &item) +{ + autonum::sequentialNumbers sequence; + if (readSequence(item, &sequence)) { + return sequence; + } + auto load = [&item](const QString &prefix, QStringList *list) { + for (int i = 1 ; !item.attribute(prefix + QString::number(i)).isEmpty() ; ++i) + list->append(item.attribute(prefix + QString::number(i))); + }; + load(QStringLiteral("sequ_"), &sequence.unit); + load(QStringLiteral("sequf_"), &sequence.unit_folio); + load(QStringLiteral("seqt_"), &sequence.ten); + load(QStringLiteral("seqtf_"), &sequence.ten_folio); + load(QStringLiteral("seqh_"), &sequence.hundred); + load(QStringLiteral("seqhf_"), &sequence.hundred_folio); + return sequence; +} + struct DocumentConductor { QString uuid; @@ -609,6 +634,8 @@ bool projectDataBase::populateFromDocument(const QDomDocument &document, QString element.type = known->type; element.sub_type = known->sub_type; element.terminals = known->terminals; + element.location = element_xml.attribute(QStringLiteral("type")); + element.sequence = elementSequence(element_xml); const QUuid group = ItemGroups::read(element_xml); element.group = group.isNull() ? QVariant() : QVariant(group.toString()); element.informations.fromXml( @@ -810,6 +837,8 @@ bool projectDataBase::populateFromDocument(const QDomDocument &document, QString m_insert_elements_query.bindValue(QStringLiteral(":type"), element.type); m_insert_elements_query.bindValue(QStringLiteral(":sub_type"), element.sub_type); m_insert_elements_query.bindValue(QStringLiteral(":group_uuid"), element.group); + m_insert_elements_query.bindValue(QStringLiteral(":definition"), element.location); + m_insert_elements_query.bindValue(QStringLiteral(":sequence"), sequenceValue(element.sequence)); if (!m_insert_elements_query.exec()) { qDebug() << "projectDataBase::populateFromDocument element insert error : " << m_insert_elements_query.lastError(); } @@ -1207,6 +1236,94 @@ QHash projectDataBase::elementTypesOnFolios(const QSet &f return types; } +/** + @brief projectDataBase::symbolsOnFolios + @return the symbols on the folios @p folios, folio by folio in that + order and on each in the order of its rows -- the order the folio + places them -- read from the element table as it is (see + elementTypesOnFolios()). A symbol whose definition was not found has + no row. +*/ +QList projectDataBase::symbolsOnFolios(const QList &folios) const +{ + QList symbols; + QSqlQuery query(m_data_base); + query.prepare(QStringLiteral("SELECT uuid, type, definition, sequence FROM element " + "WHERE diagram_uuid = :folio ORDER BY rowid")); + for (const QUuid &folio : folios) { + query.bindValue(QStringLiteral(":folio"), folio.toString()); + if (!query.exec()) { + continue; + } + while (query.next()) { + symbols.append({QUuid(query.value(0).toString()), folio, query.value(1).toString(), + query.value(2).toString(), sequenceFromValue(query.value(3))}); + } + } + return symbols; +} + +/** + @brief projectDataBase::linkedSymbols + @return the symbols @p element is linked to, read from the link table + as it is (see elementTypesOnFolios()) +*/ +QList projectDataBase::linkedSymbols(const QUuid &element) const +{ + QList linked; + QSqlQuery query(m_data_base); + query.prepare(QStringLiteral("SELECT linked_uuid FROM link WHERE element_uuid = :uuid ORDER BY rowid")); + query.bindValue(QStringLiteral(":uuid"), element.toString()); + if (query.exec()) { + while (query.next()) + linked.append(QUuid(query.value(0).toString())); + } + return linked; +} + +/** + @brief projectDataBase::sequenceValue + @return @p sequence as the element table's sequence column holds it: + each part the file would save (sequentialNumbers::toXml()), by its tag, + as JSON; null when the file would save none. sequenceFromValue() reads + it back as reading the file would (sequentialNumbers::fromXml()). +*/ +QVariant projectDataBase::sequenceValue(const autonum::sequentialNumbers &sequence) +{ + QDomDocument document; + const QDomElement xml = sequence.toXml(document); + if (!xml.hasChildNodes()) { + return QVariant(); + } + QJsonObject parts; + for (QDomElement part = xml.firstChildElement() ; !part.isNull() ; + part = part.nextSiblingElement()) + parts.insert(part.tagName(), part.text()); + return QString::fromUtf8(QJsonDocument(parts).toJson(QJsonDocument::Compact)); +} + +/** + @brief projectDataBase::sequenceFromValue + @return the sequential numbers held by @p value (sequenceValue()) +*/ +autonum::sequentialNumbers projectDataBase::sequenceFromValue(const QVariant &value) +{ + autonum::sequentialNumbers sequence; + if (value.isNull()) { + return sequence; + } + const QJsonObject parts = QJsonDocument::fromJson(value.toString().toUtf8()).object(); + QDomDocument document; + QDomElement xml = document.createElement(QStringLiteral("sequentialNumbers")); + for (auto it = parts.constBegin() ; it != parts.constEnd() ; ++it) { + QDomElement part = document.createElement(it.key()); + part.appendChild(document.createTextNode(it.value().toString())); + xml.appendChild(part); + } + sequence.fromXml(xml); + return sequence; +} + /** @brief projectDataBase::folioUuidsOfElements @return the uuids of the folios the symbols @p elements are on, read @@ -2322,6 +2439,8 @@ bool projectDataBase::createDataBase() "type VARCHAR(50)," "sub_type VARCHAR(50)," "group_uuid VARCHAR(50)," + "definition TEXT," + "sequence TEXT," "FOREIGN KEY (diagram_uuid) REFERENCES diagram (uuid)" ")"); if (!query_.exec(element_table)) { @@ -3985,7 +4104,8 @@ void projectDataBase::prepareQuery() m_diagram_info_order_changed.prepare("UPDATE diagram_info SET folio = :folio WHERE diagram_uuid = :uuid"); //INSERT ELEMENT - QString insert_element_query("INSERT INTO element (uuid, diagram_uuid, pos, type, sub_type, group_uuid) VALUES (:uuid, :diagram_uuid, :pos, :type, :sub_type, :group_uuid)"); + QString insert_element_query("INSERT INTO element (uuid, diagram_uuid, pos, type, sub_type, group_uuid, definition, sequence) " + "VALUES (:uuid, :diagram_uuid, :pos, :type, :sub_type, :group_uuid, :definition, :sequence)"); m_insert_elements_query = QSqlQuery(m_data_base); m_insert_elements_query.prepare(insert_element_query); @@ -4100,6 +4220,8 @@ void projectDataBase::bindElementValues(QSqlQuery &query, Element *element, Diag query.bindValue(QStringLiteral(":type"), element_data.typeToString()); query.bindValue(QStringLiteral(":sub_type"), element_data.masterTypeToString()); query.bindValue(QStringLiteral(":group_uuid"), groupValue(element)); + query.bindValue(QStringLiteral(":definition"), element->location().path()); + query.bindValue(QStringLiteral(":sequence"), sequenceValue(element->sequenceStruct())); } /** diff --git a/sources/dataBase/projectdatabase.h b/sources/dataBase/projectdatabase.h index ec0ac2e88..489880ecb 100644 --- a/sources/dataBase/projectdatabase.h +++ b/sources/dataBase/projectdatabase.h @@ -34,6 +34,7 @@ #include "../diagramcontext.h" #include "../titleblockproperties.h" #include "../conductorproperties.h" +#include "../autoNum/assignvariables.h" class Element; class QETProject; @@ -180,6 +181,18 @@ class projectDataBase : public QObject QSet foliosWithUnjoinedWires() const; QSet folioUuidsOfElements(const QSet &elements) const; QHash elementTypesOnFolios(const QSet &folios) const; + /// A symbol placed on a folio, as the element table has it + struct FolioSymbol { + QUuid uuid; + QUuid folio; + QString kind; ///< ElementData::typeToString() + QString location; ///< its definition, ElementsLocation::path() + autonum::sequentialNumbers sequence; + }; + QList symbolsOnFolios(const QList &folios) const; + QList linkedSymbols(const QUuid &element) const; + static QVariant sequenceValue(const autonum::sequentialNumbers &sequence); + static autonum::sequentialNumbers sequenceFromValue(const QVariant &value); void addElement (Element *element); void removeElement (Element *element); diff --git a/sources/qetproject.cpp b/sources/qetproject.cpp index 9937d7099..a63de72a7 100644 --- a/sources/qetproject.cpp +++ b/sources/qetproject.cpp @@ -796,29 +796,22 @@ void QETProject::buildFoliosHolding(ElementData::Types kinds, bool free_only) for (auto it = types.cbegin() ; it != types.cend() ; ++it) placed.insert(it.key()); } + //The symbols and their links as the database has them QList to_build; for (Diagram *d : std::as_const(m_diagrams_list)) { if (d->isLoaded()) { continue; } - for (const QDomElement &element_xml : - QET::findInDomElement(d->deferredXml(), - QStringLiteral("elements"), - QStringLiteral("element"))) + for (const projectDataBase::FolioSymbol &symbol : m_data_base.symbolsOnFolios({d->uuid()})) { - const auto type = types.constFind(QUuid(element_xml.attribute(QStringLiteral("uuid")))); - if (type == types.constEnd() - || !(kinds & ElementData::typeFromString(*type))) { + if (!(kinds & ElementData::typeFromString(symbol.kind))) { continue; } bool linked = false; if (free_only) { - for (const QDomElement &link : - QET::findInDomElement(element_xml, - QStringLiteral("links_uuids"), - QStringLiteral("link_uuid"))) { - if (placed.contains(QUuid(link.attribute(QStringLiteral("uuid"))))) { + for (const QUuid &link : m_data_base.linkedSymbols(symbol.uuid)) { + if (placed.contains(link)) { linked = true; break; } @@ -844,7 +837,8 @@ void QETProject::buildFoliosHolding(ElementData::Types kinds, bool free_only) /** @brief QETProject::buildFoliosHoldingSymbols Build the folios not built yet (QET_LAZY_FOLIOS) holding one of the - symbols @p symbols, found in the XML kept for them: what needs those + symbols @p symbols, found in the project database once it holds the + folios not built, else in the XML kept for them: what needs those symbols while the project is being read, before the database knows where they are (a terminal strip's terminals) */ @@ -856,7 +850,7 @@ void QETProject::buildFoliosHoldingSymbols(const QSet &symbols) QList to_build; //Opened from an index, the database knows where every symbol is //without reading the folios' parts - if (m_opened_from_index) { + if (m_opened_from_index || m_data_base.rowsFromDocument()) { const QSet folios = m_data_base.folioUuidsOfElements(symbols); for (Diagram *d : std::as_const(m_diagrams_list)) if (!d->isLoaded() && folios.contains(d->uuid())) @@ -1892,7 +1886,7 @@ std::optional QETProject::elementAutoNumIdToLink( the symbol on a folio built and, on a folio not built yet (QET_LAZY_FOLIOS), from what building it would give: its information in the project database's store, the scheme it would be linked to, - its sequential numbers and its kind as the file has them. + its sequential numbers and its kind as the database has them. */ QList QETProject::symbolNumbering() const { @@ -1923,38 +1917,27 @@ QList QETProject::symbolNumbering() const return list; } - //A symbol the database has no row for is one building leaves out - //(its definition is missing) - const QHash types = m_data_base.elementTypesOnFolios(unbuilt); - const QHash titles_by_formula = elementAutoNumTitlesByFormula(); + //The symbols of the folios not built, from the database. A symbol + //it has no row for is one building leaves out (its definition is + //missing) + QList unbuilt_in_order; for (Diagram *d : m_diagrams_list) + if (!d->isLoaded()) unbuilt_in_order << d->uuid(); + const QHash titles_by_formula = elementAutoNumTitlesByFormula(); + for (const projectDataBase::FolioSymbol &row : m_data_base.symbolsOnFolios(unbuilt_in_order)) { - if (d->isLoaded()) { - continue; - } - for (const QDomElement &element_xml : - QET::findInDomElement(d->deferredXml(), - QStringLiteral("elements"), - QStringLiteral("element"))) - { - const QUuid uuid(element_xml.attribute(QStringLiteral("uuid"))); - const auto type = types.constFind(uuid); - if (type == types.constEnd()) { - continue; - } - const DiagramContext info = m_data_base.elementInformation(uuid); - SymbolNumbering symbol; - symbol.label = info.value(QETInformation::ELMT_LABEL).toString(); - symbol.formula = info.value(QETInformation::ELMT_FORMULA).toString(); - const auto linked = elementAutoNumIdToLink(info, titles_by_formula); - symbol.scheme = linked ? *linked - : QUuid(info.value(QETInformation::ELMT_FORMULA_ID).toString()); - symbol.sequence.fromXml(element_xml.firstChildElement(QStringLiteral("sequentialNumbers"))); - const ElementData::Type kind = ElementData::typeFromString(*type); - symbol.numbered = kind != ElementData::Slave - && !(kind & ElementData::AllReport); - list << symbol; - } + const DiagramContext info = m_data_base.elementInformation(row.uuid); + SymbolNumbering symbol; + symbol.label = info.value(QETInformation::ELMT_LABEL).toString(); + symbol.formula = info.value(QETInformation::ELMT_FORMULA).toString(); + const auto linked = elementAutoNumIdToLink(info, titles_by_formula); + symbol.scheme = linked ? *linked + : QUuid(info.value(QETInformation::ELMT_FORMULA_ID).toString()); + symbol.sequence = row.sequence; + const ElementData::Type kind = ElementData::typeFromString(row.kind); + symbol.numbered = kind != ElementData::Slave + && !(kind & ElementData::AllReport); + list << symbol; } return list; } @@ -2387,8 +2370,7 @@ QDomDocument QETProject::toXml(QHash *kept_folios, QStringList //give it, which any edit can change if (m_edited_since_open) for (Diagram *diagram : std::as_const(m_diagrams_list)) - if (!diagram->isLoaded() && !diagram->deferredXml().elementsByTagName( - QetGraphicsTableItem::xmlTagName()).isEmpty()) + if (!diagram->isLoaded() && unbuiltFolioHasTables(diagram)) buildFolioToShow(diagram); //A built folio is written from its items, which need their //partners: a table saves the table before it, and in a .qet a @@ -2996,24 +2978,48 @@ bool QETProject::usesElement(const ElementsLocation &location) const return(unbuiltFolioElements().contains(location)); } +/** + @brief QETProject::unbuiltFolioHasTables + @return true if @p folio, not built yet (QET_LAZY_FOLIOS), has a drawn + table: from the index it was opened from, else from its XML +*/ +bool QETProject::unbuiltFolioHasTables(Diagram *folio) const +{ + if (m_index_tables) { + for (const projectDataBase::FolioTable &table : std::as_const(*m_index_tables)) + if (table.folio == folio->uuid()) return true; + return false; + } + return !folio->deferredXml().elementsByTagName(QetGraphicsTableItem::xmlTagName()).isEmpty(); +} + /** @brief QETProject::unbuiltFolioElements @return the locations of the symbols placed on the folios not built yet - (QET_LAZY_FOLIOS), each once, read from the XML kept for them as - building them reads it (Diagram::fromXml()) + (QET_LAZY_FOLIOS), each once, read from the project database once it + holds those folios, else from the XML kept for them as building them + reads it (Diagram::fromXml()) */ QList QETProject::unbuiltFolioElements() const { QSet types; - for (Diagram *diagram : m_diagrams_list) { - if (diagram->isLoaded()) { - continue; - } - for (const QDomElement &element_xml : - QET::findInDomElement(diagram->deferredXml(), - QStringLiteral("elements"), - QStringLiteral("element"))) { - types.insert(element_xml.attribute(QStringLiteral("type"))); + if (m_data_base.rowsFromDocument()) { + QList unbuilt; + for (Diagram *diagram : m_diagrams_list) + if (!diagram->isLoaded()) unbuilt << diagram->uuid(); + for (const projectDataBase::FolioSymbol &symbol : m_data_base.symbolsOnFolios(unbuilt)) + types.insert(symbol.location); + } else { + for (Diagram *diagram : m_diagrams_list) { + if (diagram->isLoaded()) { + continue; + } + for (const QDomElement &element_xml : + QET::findInDomElement(diagram->deferredXml(), + QStringLiteral("elements"), + QStringLiteral("element"))) { + types.insert(element_xml.attribute(QStringLiteral("type"))); + } } } QList locations; @@ -3413,6 +3419,7 @@ QDomElement QETProject::readFolioPart(const QString &file) if (!m_folio_reader) m_folio_reader = std::make_unique( m_container_files.value(QStringLiteral("project.sqlite"))); + ++m_folio_parts_read; if (!QetContainer::readFolio(m_container_files, file, &folio, &error, m_folio_reader.get())) { qWarning().noquote() << "Unable to read the folio" << file << ":" << error; return QDomElement(); diff --git a/sources/qetproject.h b/sources/qetproject.h index aad50ad7e..61f9729d7 100644 --- a/sources/qetproject.h +++ b/sources/qetproject.h @@ -332,6 +332,7 @@ class QETProject : public QObject bool usesElement(const ElementsLocation &) const; QList unusedElements() const; QList unbuiltFolioElements() const; + bool unbuiltFolioHasTables(Diagram *folio) const; bool usesTitleBlockTemplate(const TitleBlockTemplateLocation &); bool projectWasModified(); bool projectOptionsWereModified(); @@ -455,6 +456,8 @@ class QETProject : public QObject bool writeContainerFile(const QString &path, QString *error = nullptr); QByteArray containerIndex(); QDomElement readFolioPart(const QString &file); + /// How many folio parts of the .qetz were read so far + int folioPartsRead() const {return m_folio_parts_read;} bool openedFromIndex() const {return m_opened_from_index;} private: static QUuid derivedUuid(const QByteArray &content); @@ -550,6 +553,7 @@ class QETProject : public QObject QHash m_container_files; ///Reads the folio parts' engineering data, opened once std::unique_ptr m_folio_reader; + int m_folio_parts_read = 0; QStringList m_folio_files; QList> m_folio_headers; ///The drawn tables of the index opened from, if it has them diff --git a/tests/qttest/lazy_folios_probe.cpp b/tests/qttest/lazy_folios_probe.cpp index e0bcf3d4c..24d71f743 100644 --- a/tests/qttest/lazy_folios_probe.cpp +++ b/tests/qttest/lazy_folios_probe.cpp @@ -696,6 +696,35 @@ int main(int argc, char **argv) } check(continued > 0, "a folio whose table continues another"); + //Numbering, the searches for symbols and the unused symbols + //are answered from the database: no folio file is read + //but those of the folios built (D5) + { + QETProject asked(zipped); + check(numbering(asked) == numbering(eager), "numbering of a .qetz as when built"); + for (const QString &title : eager.elementAutoNum().keys()) + check(gaps(asked, title) == gaps(eager, title), "a .qetz scheme's gaps as when built"); + auto locations = [](const QList &list) { + QStringList strings; + for (const ElementsLocation &location : list) strings << location.toString(); + strings.sort(); + return strings; + }; + check(locations(asked.unusedElements()) == locations(eager.unusedElements()), + "a .qetz's unused symbols as when built"); + const ElementProvider usual_search(&eager), search(&asked); + for (const ElementData::Types kinds : {ElementData::Types(ElementData::Slave), + ElementData::Types(ElementData::Master), + ElementData::Types(ElementData::AllReport)}) + check(found(search.freeElement(kinds)) == found(usual_search.freeElement(kinds)), + "free symbols of a .qetz found as when built"); + const int built = int(folios.size()) - asked.unloadedFolioCount(); + std::printf(".qetz asked numbering, searches, unused symbols: %d folio files read, %d folios built\n", + asked.folioPartsRead(), built); + check(asked.unloadedFolioCount() > 0, "asking a .qetz does not build every folio"); + check(asked.folioPartsRead() == built, "asking a .qetz reads no file of a folio not built"); + } + //Saved as a .qetz without building the folios not touched //(D4): their parts and rows copied as they were QETProject untouched(zipped);