From 2f98872d7a992aa444feeb85eb70938e145e3825 Mon Sep 17 00:00:00 2001 From: ispyisail Date: Sun, 11 Oct 2026 19:32:27 +1300 Subject: [PATCH] .qetz format 3: one project database (stage D3a) project.sqlite now holds both what QElectroTech keeps in its project database (the index D2 wrote as index.sqlite) and the document's engineering data, the latter's tables named doc_ beside the former's -- element, element_info, link and conductor exist in both (QetContainerDb's SQL names them as before; Database::sql() puts the prefix on). The manifest says format 3, min-reader 3: a format 2 reader would find no tables of its names. Format 2 files, with or without an index.sqlite entry, are read as before: the prefix is found from the file (doc_meta). A .qetz with an index opens from project.sqlite. Measured: the 136-folio f2099 1.8 s, 5.2 MB (the two table sets still overlap: D3b). Checked: tst_qetcontainer with the format 3 names, 34/34; a format 2 file of the last commit opens from its index.sqlite and saves the same; corpus 342: 340 .qetz carry the index in project.sqlite and open from it, saved back = eager = master on all 342. ctest 96/96 with and without QET_LAZY_FOLIOS=1. Co-Authored-By: Claude Opus 5.5 --- sources/container/qetcontainer.cpp | 4 +- sources/container/qetcontainer.h | 5 +- sources/container/qetcontainerdb.cpp | 106 +++++++++++++++++++-------- sources/container/qetcontainerdb.h | 3 +- sources/qetproject.cpp | 19 +++-- tests/qttest/tst_qetcontainer.cpp | 16 ++-- 6 files changed, 101 insertions(+), 52 deletions(-) diff --git a/sources/container/qetcontainer.cpp b/sources/container/qetcontainer.cpp index 3cb20fd84..84d0a293c 100644 --- a/sources/container/qetcontainer.cpp +++ b/sources/container/qetcontainer.cpp @@ -172,7 +172,7 @@ bool QetContainer::readDocument(const QByteArray &content, QDomDocument *documen set if @p project is not a project */ QList QetContainer::split(const QDomDocument &project, const QString &writer, - QString *error) + QString *error, const QByteArray &index) { QDomDocument document = project.cloneNode(true).toDocument(); QDomElement root = document.documentElement(); @@ -182,7 +182,7 @@ QList QetContainer::split(const QDomDocument &project, const QStr } //The engineering data out of the document, into project.sqlite - const QByteArray database = QetContainerDb::extract(root, error); + const QByteArray database = QetContainerDb::extract(root, error, index); if (database.isEmpty()) return {}; diff --git a/sources/container/qetcontainer.h b/sources/container/qetcontainer.h index ad050efb4..9a7d1692b 100644 --- a/sources/container/qetcontainer.h +++ b/sources/container/qetcontainer.h @@ -47,11 +47,12 @@ namespace QetContainer extern const char MimeType[]; /// The format written; files of a later format are refused by join() /// unless their min-reader allows this one. - constexpr int Format = 2; + constexpr int Format = 3; QList split(const QDomDocument &project, const QString &writer = QString(), - QString *error = nullptr); + QString *error = nullptr, + const QByteArray &index = QByteArray()); bool join(const QList &entries, QDomDocument *project, QString *error = nullptr, QStringList *folio_files = nullptr); /// A folio part of @p files, which join() left out, with its diff --git a/sources/container/qetcontainerdb.cpp b/sources/container/qetcontainerdb.cpp index e5c48c121..49beba7b9 100644 --- a/sources/container/qetcontainerdb.cpp +++ b/sources/container/qetcontainerdb.cpp @@ -17,6 +17,8 @@ */ #include "qetcontainerdb.h" +#include + #include #include #include @@ -103,14 +105,34 @@ QString schema() .arg(columns(FolioData), columns(ElementData), columns(ConductorData)); } - //Owns an sqlite3 handle + //The tables of a document's engineering data. From format 3 on they are + //named doc_, beside the project database's own tables in the same + //file (stage D3) +const QStringList DocTables = { + QStringLiteral("strip_bridge_terminal"), QStringLiteral("strip_bridge"), + QStringLiteral("strip_terminal"), QStringLiteral("terminal_strip"), + QStringLiteral("project_property"), QStringLiteral("folio_property"), QStringLiteral("folio"), + QStringLiteral("element_info"), QStringLiteral("element"), QStringLiteral("link"), + QStringLiteral("conductor"), QStringLiteral("meta")}; + + //Owns an sqlite3 handle; its SQL names the document's tables by their + //format 2 names, given their prefix here class Database { public: ~Database() { if (m_db) sqlite3_close(m_db); } sqlite3 *m_db = nullptr; - bool exec(const QString &sql) { - return sqlite3_exec(m_db, sql.toUtf8().constData(), nullptr, nullptr, nullptr) == SQLITE_OK; + QString m_prefix; + QString sql(const QString &text) const { + if (m_prefix.isEmpty()) return text; + static const QRegularExpression names(QStringLiteral("(?, format 2 not + { + Statement doc(db, QStringLiteral("SELECT count(*) FROM sqlite_master WHERE name = 'doc_meta'")); + if (doc.next() && doc.value(0).toInt() > 0) db.m_prefix = QStringLiteral("doc_"); + } for (const char *table : {"meta", "project_property", "folio", "folio_property", "element", "element_info", "link", "conductor", "terminal_strip", "strip_terminal", "strip_bridge", "strip_bridge_terminal"}) { - Statement kind(db.m_db, QStringLiteral("SELECT type FROM sqlite_master WHERE name = ?")); - kind.bind({QString::fromLatin1(table)}); + Statement kind(db, QStringLiteral("SELECT type FROM sqlite_master WHERE name = ?")); + kind.bind({db.table(QString::fromLatin1(table))}); if (!kind.next() || kind.value(0).toString() != QLatin1String("table")) { setError(error, QStringLiteral("the project database has no table %1").arg(QLatin1String(table))); return false; @@ -490,7 +517,7 @@ bool restoreDiagram(Database &db, QDomElement d, QString *error) const QString id = d.attribute(FolioAttribute); if (id.isEmpty()) return true; d.removeAttribute(FolioAttribute); - Statement f(db.m_db, QStringLiteral("SELECT %1, props_at, props_attrs FROM folio WHERE id = ?") + Statement f(db, QStringLiteral("SELECT %1, props_at, props_attrs FROM folio WHERE id = ?") .arg(QStringLiteral("\"") + FolioData.join(QStringLiteral("\", \"")) + QStringLiteral("\""))); f.bind({id}); if (!f.next()) { setError(error, QStringLiteral("folio %1 is not in the database").arg(id)); return false; } @@ -500,7 +527,7 @@ bool restoreDiagram(Database &db, QDomElement d, QString *error) if (!values.at(FolioData.size()).isNull()) { QDomElement props = document.createElement(QStringLiteral("properties")); setAttributesJson(props, values.at(FolioData.size() + 1)); - Statement p(db.m_db, QStringLiteral("SELECT name, value, attrs FROM folio_property WHERE folio_id = ? ORDER BY ord")); + Statement p(db, QStringLiteral("SELECT name, value, attrs FROM folio_property WHERE folio_id = ? ORDER BY ord")); p.bind({id}); while (p.next()) props.appendChild(entry(p.value(0), QStringLiteral("name"), p.value(1), p.value(2), @@ -517,7 +544,7 @@ bool restoreDiagram(Database &db, QDomElement d, QString *error) if (!e.hasAttribute(KeyAttribute)) continue; const QString key = e.attribute(KeyAttribute); e.removeAttribute(KeyAttribute); - Statement s(db.m_db, QStringLiteral("SELECT %1, info_at, info_attrs, links_at, links_attrs FROM element WHERE key = ?") + Statement s(db, QStringLiteral("SELECT %1, info_at, info_attrs, links_at, links_attrs FROM element WHERE key = ?") .arg(QStringLiteral("\"") + ElementData.join(QStringLiteral("\", \"")) + QStringLiteral("\""))); s.bind({key}); if (!s.next()) { setError(error, QStringLiteral("symbol %1 is not in the database").arg(key)); return false; } @@ -529,7 +556,7 @@ bool restoreDiagram(Database &db, QDomElement d, QString *error) if (!v.at(n).isNull()) { info = document.createElement(QStringLiteral("elementInformations")); setAttributesJson(info, v.at(n + 1)); - Statement i(db.m_db, QStringLiteral("SELECT name, value, attrs FROM element_info WHERE element_key = ? ORDER BY ord")); + Statement i(db, QStringLiteral("SELECT name, value, attrs FROM element_info WHERE element_key = ? ORDER BY ord")); i.bind({key}); while (i.next()) info.appendChild(entry(i.value(0), QStringLiteral("name"), i.value(1), i.value(2), @@ -539,7 +566,7 @@ bool restoreDiagram(Database &db, QDomElement d, QString *error) if (!v.at(n + 2).isNull()) { links = document.createElement(QStringLiteral("links_uuids")); setAttributesJson(links, v.at(n + 3)); - Statement l(db.m_db, QStringLiteral("SELECT linked_uuid, attrs FROM link WHERE element_key = ? ORDER BY ord")); + Statement l(db, QStringLiteral("SELECT linked_uuid, attrs FROM link WHERE element_key = ? ORDER BY ord")); l.bind({key}); while (l.next()) links.appendChild(entry(l.value(0), QStringLiteral("uuid"), QVariant(), l.value(1), @@ -558,7 +585,7 @@ bool restoreDiagram(Database &db, QDomElement d, QString *error) if (!c.hasAttribute(KeyAttribute)) continue; const QString key = c.attribute(KeyAttribute); c.removeAttribute(KeyAttribute); - Statement s(db.m_db, QStringLiteral("SELECT %1 FROM conductor WHERE key = ?") + Statement s(db, QStringLiteral("SELECT %1 FROM conductor WHERE key = ?") .arg(QStringLiteral("\"") + ConductorData.join(QStringLiteral("\", \"")) + QStringLiteral("\""))); s.bind({key}); if (!s.next()) { setError(error, QStringLiteral("wire %1 is not in the database").arg(key)); return false; } @@ -577,27 +604,44 @@ bool restoreDiagram(Database &db, QDomElement d, QString *error) folios still in it) into a new database. @return the database file, or an empty array with @p error set */ -QByteArray QetContainerDb::extract(QDomElement project, QString *error) +QByteArray QetContainerDb::extract(QDomElement project, QString *error, const QByteArray &index) { Database db; - if (sqlite3_open(":memory:", &db.m_db) != SQLITE_OK || !db.exec(schema())) { + db.m_prefix = QStringLiteral("doc_"); + if (sqlite3_open(":memory:", &db.m_db) != SQLITE_OK) { + setError(error, QStringLiteral("cannot create the project database")); + return {}; + } + //The project database's own tables first, when given: one file + //holds both (stage D3) + if (!index.isEmpty()) { + unsigned char *copy = static_cast(sqlite3_malloc64(sqlite3_uint64(index.size()))); + if (!copy) { setError(error, QStringLiteral("out of memory")); return {}; } + memcpy(copy, index.constData(), size_t(index.size())); + if (sqlite3_deserialize(db.m_db, "main", copy, index.size(), index.size(), + SQLITE_DESERIALIZE_FREEONCLOSE | SQLITE_DESERIALIZE_RESIZEABLE) != SQLITE_OK) { + setError(error, QStringLiteral("cannot read the project database's index")); + return {}; + } + } + if (!db.exec(schema())) { setError(error, QStringLiteral("cannot create the project database")); return {}; } db.exec(QStringLiteral("BEGIN")); - Statement meta(db.m_db, QStringLiteral("INSERT INTO meta VALUES (?,?)")); - Statement project_property(db.m_db, QStringLiteral("INSERT INTO project_property VALUES (?,?,?,?)")); - Statement folio(db.m_db, QStringLiteral("INSERT INTO folio VALUES (?,?,%1,?,?)").arg(placeholders(FolioData.size()))); - Statement folio_property(db.m_db, QStringLiteral("INSERT INTO folio_property VALUES (?,?,?,?,?)")); - Statement element(db.m_db, QStringLiteral("INSERT INTO element VALUES (?,?,%1,?,?,?,?)").arg(placeholders(ElementData.size()))); - Statement element_info(db.m_db, QStringLiteral("INSERT INTO element_info VALUES (?,?,?,?,?)")); - Statement link(db.m_db, QStringLiteral("INSERT INTO link VALUES (?,?,?,?)")); - Statement conductor(db.m_db, QStringLiteral("INSERT INTO conductor VALUES (?,?,%1)").arg(placeholders(ConductorData.size()))); + Statement meta(db, QStringLiteral("INSERT INTO meta VALUES (?,?)")); + Statement project_property(db, QStringLiteral("INSERT INTO project_property VALUES (?,?,?,?)")); + Statement folio(db, QStringLiteral("INSERT INTO folio VALUES (?,?,%1,?,?)").arg(placeholders(FolioData.size()))); + Statement folio_property(db, QStringLiteral("INSERT INTO folio_property VALUES (?,?,?,?,?)")); + Statement element(db, QStringLiteral("INSERT INTO element VALUES (?,?,%1,?,?,?,?)").arg(placeholders(ElementData.size()))); + Statement element_info(db, QStringLiteral("INSERT INTO element_info VALUES (?,?,?,?,?)")); + Statement link(db, QStringLiteral("INSERT INTO link VALUES (?,?,?,?)")); + Statement conductor(db, QStringLiteral("INSERT INTO conductor VALUES (?,?,%1)").arg(placeholders(ConductorData.size()))); bool ok = meta.ok() && project_property.ok() && folio.ok() && folio_property.ok() && element.ok() && element_info.ok() && link.ok() && conductor.ok(); - ok = ok && meta.run({QStringLiteral("format"), QStringLiteral("2")}); + ok = ok && meta.run({QStringLiteral("format"), QStringLiteral("3")}); // of a block, as rows of @p rows (with @p prefix values) auto moveProperties = [&](QDomElement block, Statement &rows, const QVariantList &prefix) { @@ -709,10 +753,10 @@ QByteArray QetContainerDb::extract(QDomElement project, QString *error) const QDomElement strips = project.firstChildElement(QStringLiteral("terminal_strips")); StripRows strip_rows; if (!strips.isNull() && stripRows(strips, &strip_rows)) { - Statement st(db.m_db, QStringLiteral("INSERT INTO terminal_strip VALUES (?,?,?,?,?,?,?)")); - Statement tt(db.m_db, QStringLiteral("INSERT INTO strip_terminal VALUES (?,?,?,?)")); - Statement bt(db.m_db, QStringLiteral("INSERT INTO strip_bridge VALUES (?,?,?,?)")); - Statement btt(db.m_db, QStringLiteral("INSERT INTO strip_bridge_terminal VALUES (?,?,?)")); + Statement st(db, QStringLiteral("INSERT INTO terminal_strip VALUES (?,?,?,?,?,?,?)")); + Statement tt(db, QStringLiteral("INSERT INTO strip_terminal VALUES (?,?,?,?)")); + Statement bt(db, QStringLiteral("INSERT INTO strip_bridge VALUES (?,?,?,?)")); + Statement btt(db, QStringLiteral("INSERT INTO strip_bridge_terminal VALUES (?,?,?)")); for (const QVariantList &r : strip_rows.strips) ok = ok && st.run(r); for (const QVariantList &r : strip_rows.terminals) ok = ok && tt.run(r); for (const QVariantList &r : strip_rows.bridges) ok = ok && bt.run(r); @@ -762,13 +806,13 @@ bool QetContainerDb::restore(QDomElement project, const QByteArray &database, QS //Project properties QHash meta; { - Statement s(db.m_db, QStringLiteral("SELECT key, value FROM meta")); + Statement s(db, QStringLiteral("SELECT key, value FROM meta")); while (s.next()) meta.insert(s.value(0).toString(), s.value(1)); } if (meta.contains(QStringLiteral("props_at"))) { QDomElement props = document.createElement(QStringLiteral("properties")); setAttributesJson(props, meta.value(QStringLiteral("props_attrs"))); - Statement s(db.m_db, QStringLiteral("SELECT name, value, attrs FROM project_property ORDER BY ord")); + Statement s(db, QStringLiteral("SELECT name, value, attrs FROM project_property ORDER BY ord")); while (s.next()) props.appendChild(entry(s.value(0), QStringLiteral("name"), s.value(1), s.value(2), QStringLiteral("property"))); @@ -782,7 +826,7 @@ bool QetContainerDb::restore(QDomElement project, const QByteArray &database, QS if (meta.contains(QStringLiteral("terminal_strips"))) { StripRows rows; auto read = [&db](const QString &sql, int columns, QList *out) { - Statement s(db.m_db, sql); + Statement s(db, sql); while (s.next()) { QVariantList r; for (int i = 0 ; i < columns ; ++i) r << s.value(i); diff --git a/sources/container/qetcontainerdb.h b/sources/container/qetcontainerdb.h index e43c87946..5e1ec63ca 100644 --- a/sources/container/qetcontainerdb.h +++ b/sources/container/qetcontainerdb.h @@ -37,7 +37,8 @@ */ namespace QetContainerDb { - QByteArray extract(QDomElement project, QString *error = nullptr); + QByteArray extract(QDomElement project, QString *error = nullptr, + const QByteArray &index = QByteArray()); bool restore(QDomElement project, const QByteArray &database, QString *error = nullptr); bool restoreFolio(QDomElement diagram, const QByteArray &database, QString *error = nullptr); } diff --git a/sources/qetproject.cpp b/sources/qetproject.cpp index f6c02dfc9..11b8c3f22 100644 --- a/sources/qetproject.cpp +++ b/sources/qetproject.cpp @@ -478,11 +478,15 @@ QETProject::ProjectState QETProject::openFile(QFile *file) files.insert(entry.name, entry.data); QStringList folio_files; QDomDocument document; - if (files.contains(QStringLiteral("index.sqlite")) + //The index: in project.sqlite from format 3, its own entry + //before + const QByteArray index = files.contains(QStringLiteral("index.sqlite")) + ? files.value(QStringLiteral("index.sqlite")) + : files.value(QStringLiteral("project.sqlite")); + if (!index.isEmpty() && index.contains("folio_header") && QetContainer::join(entries, &document, &why, &folio_files)) { - if (m_data_base.fillFromIndex(files.value(QStringLiteral("index.sqlite")), - &m_folio_headers, &why) + if (m_data_base.fillFromIndex(index, &m_folio_headers, &why) && m_folio_headers.size() == folio_files.size()) { xml_project = document; m_container_files = files; @@ -2518,14 +2522,13 @@ bool QETProject::isContainerPath(const QString &path) bool QETProject::writeContainer(const QDomDocument &project, const QString &path, QString *error, const QByteArray &index) { + //The project database, for opening without reading the folios + //(stage D2), is in project.sqlite with the + //document's engineering data (format 3, D3) QList entries = QetContainer::split( project, QStringLiteral("QElectroTech %1").arg(QetVersion::currentVersion().toString()), - error); - //The project database, for opening without reading the folios - //(stage D2); a reader that does not know it skips it - if (!entries.isEmpty() && !index.isEmpty()) - entries.append(QetZip::Entry{QStringLiteral("index.sqlite"), index, true}); + error, index); return !entries.isEmpty() && QetZip::write(path, entries, error); } diff --git a/tests/qttest/tst_qetcontainer.cpp b/tests/qttest/tst_qetcontainer.cpp index ea2a9525c..be09a3593 100644 --- a/tests/qttest/tst_qetcontainer.cpp +++ b/tests/qttest/tst_qetcontainer.cpp @@ -170,14 +170,14 @@ private slots: return n; }; const QDomNodeList diagrams = original.elementsByTagName(QStringLiteral("diagram")); - QCOMPARE(count("folio"), int(diagrams.size())); - QCOMPARE(count("element"), int(original.elementsByTagName(QStringLiteral("element")).size()) + QCOMPARE(count("doc_folio"), int(diagrams.size())); + QCOMPARE(count("doc_element"), int(original.elementsByTagName(QStringLiteral("element")).size()) - int(original.elementsByTagName(QStringLiteral("collection")).at(0).toElement() .elementsByTagName(QStringLiteral("element")).size())); - QCOMPARE(count("element_info"), int(original.elementsByTagName(QStringLiteral("elementInformation")).size())); - QCOMPARE(count("link"), int(original.elementsByTagName(QStringLiteral("link_uuid")).size())); - QCOMPARE(count("conductor"), int(original.elementsByTagName(QStringLiteral("conductor")).size())); - QVERIFY(count("folio_property") + count("project_property") > 0); + QCOMPARE(count("doc_element_info"), int(original.elementsByTagName(QStringLiteral("elementInformation")).size())); + QCOMPARE(count("doc_link"), int(original.elementsByTagName(QStringLiteral("link_uuid")).size())); + QCOMPARE(count("doc_conductor"), int(original.elementsByTagName(QStringLiteral("conductor")).size())); + QVERIFY(count("doc_folio_property") + count("doc_project_property") > 0); sqlite3_close(db); } @@ -285,8 +285,8 @@ private slots: memcpy(copy, e.data.constData(), size_t(e.data.size())); sqlite3_deserialize(db, "main", copy, e.data.size(), e.data.size(), SQLITE_DESERIALIZE_FREEONCLOSE | SQLITE_DESERIALIZE_RESIZEABLE); - QCOMPARE(sqlite3_exec(db, "ALTER TABLE element RENAME TO element_data;" - "CREATE VIEW element AS SELECT * FROM element_data;", + QCOMPARE(sqlite3_exec(db, "ALTER TABLE doc_element RENAME TO element_data;" + "CREATE VIEW doc_element AS SELECT * FROM element_data;", nullptr, nullptr, nullptr), SQLITE_OK); sqlite3_int64 size = 0; unsigned char *bytes = sqlite3_serialize(db, "main", &size, 0);