kotschak/qelectrotech-source-mirror
0
0
Fork 0
mirror of https://github.com/qelectrotech/qelectrotech-source-mirror.git synced 2026-10-15 18:14:14 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity

Keep each fact once in a .qetz: drop the stored project database

Browse Source 
A .qetz held the engineering data twice: the document's tables, and
beside them QElectroTech's own project database (stage D2/D3), which
QElectroTech also builds from the document. Two copies of one fact can
disagree, and the copy cost a third of the file. Now project.sqlite holds
the document's tables only; QElectroTech builds its own project database
from them when the project opens, as it does for a .qet.

Measured (this build, lazy folios):
  industrial (50 folios): 620 -> 374 KB, open 0.37 -> 0.65 s
  f2099/nuri (136 folios): 3.27 -> 2.20 MB, open 2.2 -> 3.5 s
The same projects as .qet open in 3.3 s and 23 s.

Saving still copies the file of every folio not built, with its rows
(stage D4): the project keeps the part each folio was read from. nuri
opened and saved again: 6.3 s, no folio built, the same bytes.

A symbol's terminal list is left out of the folio file when every
folio, symbol and wire has a uuid of its own, now checked on the
document (QetContainer::ownUuids()) rather than by the stored database
being written.

Checked: 342 corpus projects open, .qetz -> .qet equals the plain
resave, a .qetz saved twice gives the same bytes; tst_lazyfolios,
tst_qetcontainer, tst_resaveunchanged (which now checks project.sqlite
holds only the document's tables).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
ispyisail
2026-10-14 07:35:34 +13:00
parent 39855e1bb5
commit c259cae429
14 changed files with 165 additions and 842 deletions
Download Patch File Download Diff File Expand all files Collapse all files
sources/dataBase/projectdatabase.cpp
-371
View File
@@ -49,10 +49,8 @@
#include <QMetaEnum>
#include <QTextDocument>
#include <QFile>
#include <QCryptographicHash>
#include <QJsonDocument>
#include <QJsonObject>
#include <QTemporaryDir>
#include <QRegularExpression>
#include <QSqlDriver>
#include <QSqlError>
@@ -896,375 +894,6 @@ bool projectDataBase::populateFromDocument(const QDomDocument &document, QString
return true;
}
/**
@brief projectDataBase::schemaStamp
@return a stamp of the tables of this database as created: an index
file (indexFile()) is read back only into a database with the same
*/
QString projectDataBase::schemaStamp() const
{
QSqlQuery query(m_data_base);
query.exec(QStringLiteral("SELECT sql FROM sqlite_master WHERE type IN ('table', 'index') "
"AND name NOT LIKE 'sqlite_%' ORDER BY name"));
QCryptographicHash hash(QCryptographicHash::Sha256);
while (query.next())
hash.addData(query.value(0).toString().toUtf8() + ';');
return QString::fromLatin1(hash.result().toHex());
}
namespace {
//The tables an index file carries, all of this database's but the
//drawing items, which exist for built folios only
QStringList indexTables(const QSqlDatabase &db)
{
QStringList tables;
QSqlQuery query(db);
query.exec(QStringLiteral("SELECT name FROM sqlite_master WHERE type = 'table' "
"AND name NOT LIKE 'sqlite_%' ORDER BY name"));
while (query.next()) {
const QString name = query.value(0).toString();
if (name != QLatin1String("shape") && name != QLatin1String("independent_text")
&& name != QLatin1String("image"))
tables << name;
}
return tables;
}
}
namespace {
//A wire's properties read from @p attributes, as a saved wire's are
ConductorProperties propertiesFrom(const QHash<QString, QString> &attributes)
{
QDomDocument scratch;
QDomElement e = scratch.createElement(QStringLiteral("conductor"));
for (auto it = attributes.cbegin() ; it != attributes.cend() ; ++it)
e.setAttribute(it.key(), it.value());
ConductorProperties properties;
properties.fromXml(e);
return properties;
}
//@p attributes without those reading them would give anyway: each
//left out in turn when the properties read without it are the same
//(stage D3b)
QHash<QString, QString> fewestAttributes(QHash<QString, QString> attributes)
{
const ConductorProperties full = propertiesFrom(attributes);
QStringList names = attributes.keys();
names.sort();
for (const QString &name : std::as_const(names)) {
QHash<QString, QString> without = attributes;
without.remove(name);
const ConductorProperties read = propertiesFrom(without);
if (read == full && read.attributes() == full.attributes())
attributes = without;
}
return attributes;
}
}
/**
@brief projectDataBase::indexFile
This database written to an SQLite file, for a .qetz to be opened from
(stage D2): every table but the drawing items, brought
up to date first, with the folios' own data @p folio_headers (in folio
order, each as Diagram::headerXml() writes it), the drawn tables
@p tables and how they continue one another (which folios a folio's
tables must be built with), and the stamp of the schema it was written
with.
@return the file, or nothing if it could not be written
*/
QByteArray projectDataBase::indexFile(const QList<std::pair<QUuid, QString>> &folio_headers,
const QList<FolioTable> &tables)
{
if (m_content_changed) {
updateDB();
}
flushLinks();
flushElementPositions();
flushConductorProperties();
QTemporaryDir dir;
if (!dir.isValid()) {
return QByteArray();
}
const QString path = dir.filePath(QStringLiteral("index.sqlite"));
QString escaped = path;
escaped.replace(QLatin1Char('\''), QStringLiteral("''"));
QSqlQuery vacuum(m_data_base);
if (!vacuum.exec(QStringLiteral("VACUUM INTO '%1'").arg(escaped))) {
qWarning() << "projectDataBase::indexFile:" << vacuum.lastError().text();
return QByteArray();
}
const QString stamp = schemaStamp();
const QString connection = QStringLiteral("qet_index_") + QUuid::createUuid().toString();
bool ok = false;
{
QSqlDatabase file = QSqlDatabase::addDatabase(QStringLiteral("QSQLITE"), connection);
file.setDatabaseName(path);
if (file.open())
{
QSqlQuery query(file);
//Its views read the drawing items too: an index holds tables
QStringList views;
query.exec(QStringLiteral("SELECT name FROM sqlite_master WHERE type = 'view'"));
while (query.next()) views << query.value(0).toString();
for (const QString &view : std::as_const(views))
query.exec(QStringLiteral("DROP VIEW \"%1\"").arg(view));
ok = query.exec(QStringLiteral("DROP TABLE shape"))
&& query.exec(QStringLiteral("DROP TABLE independent_text"))
&& query.exec(QStringLiteral("DROP TABLE image"))
&& query.exec(QStringLiteral("CREATE TABLE index_meta (key TEXT PRIMARY KEY, value TEXT)"))
&& query.exec(QStringLiteral("CREATE TABLE folio_header (pos INTEGER PRIMARY KEY, "
"uuid TEXT NOT NULL, xml TEXT NOT NULL)"))
&& query.exec(QStringLiteral("CREATE TABLE folio_table (uuid TEXT NOT NULL, "
"folio_uuid TEXT NOT NULL, previous_uuid TEXT)"));
file.transaction();
query.prepare(QStringLiteral("INSERT INTO index_meta (key, value) VALUES (?, ?)"));
for (const auto &meta : {std::pair<QString, QString>{QStringLiteral("schema"), stamp},
std::pair<QString, QString>{QStringLiteral("writer"),
QStringLiteral("QElectroTech ") + QetVersion::currentVersion().toString()}}) {
query.addBindValue(meta.first);
query.addBindValue(meta.second);
ok = ok && query.exec();
}
query.prepare(QStringLiteral("INSERT INTO folio_header (pos, uuid, xml) VALUES (?, ?, ?)"));
int pos = 0;
for (const auto &header : folio_headers) {
query.addBindValue(pos++);
query.addBindValue(header.first.toString());
query.addBindValue(header.second);
ok = ok && query.exec();
}
query.prepare(QStringLiteral("INSERT INTO folio_table (uuid, folio_uuid, previous_uuid) "
"VALUES (?, ?, ?)"));
for (const FolioTable &table : tables) {
query.addBindValue(table.table.toString());
query.addBindValue(table.folio.toString());
query.addBindValue(table.previous.isNull() ? QVariant() : QVariant(table.previous.toString()));
ok = ok && query.exec();
}
//A wire's properties: only the attributes reading them needs,
//by wire and name (fillFromIndex() writes the others back)
{
QMap<QString, QHash<QString, QString>> wires;
QSqlQuery rows(file);
rows.exec(QStringLiteral("SELECT conductor_uuid, name, value FROM conductor_properties"));
while (rows.next())
wires[rows.value(0).toString()].insert(rows.value(1).toString(), rows.value(2).toString());
ok = ok && query.exec(QStringLiteral("DELETE FROM conductor_properties"));
query.prepare(QStringLiteral("INSERT INTO conductor_properties (conductor_uuid, name, value) "
"VALUES (?, ?, ?)"));
for (auto wire = wires.cbegin() ; wire != wires.cend() ; ++wire) {
const QHash<QString, QString> fewest = fewestAttributes(wire.value());
QStringList names = fewest.keys();
names.sort();
for (const QString &name : std::as_const(names)) {
query.addBindValue(wire.key());
query.addBindValue(name);
query.addBindValue(fewest.value(name));
ok = ok && query.exec();
}
}
}
ok = file.commit() && ok;
ok = query.exec(QStringLiteral("VACUUM")) && ok;
file.close();
}
}
QSqlDatabase::removeDatabase(connection);
if (!ok) {
return QByteArray();
}
QFile file(path);
if (!file.open(QIODevice::ReadOnly)) {
return QByteArray();
}
return file.readAll();
}
/**
@brief projectDataBase::discardFill
Undo fillFromIndex(): the project is read from its document instead
*/
void projectDataBase::discardFill()
{
QSqlQuery query(m_data_base);
for (const QString &table : indexTables(m_data_base))
query.exec(QStringLiteral("DELETE FROM \"%1\"").arg(table));
m_element_information.clear();
m_folio_titleblocks.clear();
m_conductor_properties.clear();
m_conductor_attributes.clear();
m_prefilled_elements.clear();
m_prefilled_folios.clear();
m_prefilled_conductors.clear();
m_rows_from_document = false;
m_content_changed = true;
}
/**
@brief projectDataBase::fillFromIndex
Fill this database from @p index, a file indexFile() wrote, instead of
from the project's document: its tables, and the stores of symbol
information and wire properties (the folios' title blocks are filled
from @p folio_headers by the folios). The file is untrusted: read only
when written with this database's schema, its tables plain tables, and
nothing in its schema run.
@param folio_headers : the folios' own data, in folio order
@param folio_tables : if not null, the drawn tables indexFile() wrote, or
nothing if the index predates them
@return false, with @p why set and nothing changed in the tables, if
the file cannot be read
*/
bool projectDataBase::fillFromIndex(const QByteArray &index,
QList<std::pair<QUuid, QString>> *folio_headers,
QString *why,
std::optional<QList<FolioTable>> *folio_tables)
{
auto refuse = [why](const QString &reason) {
if (why) *why = reason;
return false;
};
QTemporaryDir dir;
if (!dir.isValid()) {
return refuse(QStringLiteral("no temporary folder"));
}
const QString path = dir.filePath(QStringLiteral("index.sqlite"));
{
QFile file(path);
if (!file.open(QIODevice::WriteOnly) || file.write(index) != index.size()) {
return refuse(QStringLiteral("cannot write the index to a temporary file"));
}
}
QString escaped = path;
escaped.replace(QLatin1Char('\''), QStringLiteral("''"));
QSqlQuery query(m_data_base);
query.exec(QStringLiteral("PRAGMA trusted_schema = OFF"));
if (!query.exec(QStringLiteral("ATTACH DATABASE '%1' AS idx").arg(escaped))) {
return refuse(QStringLiteral("the index cannot be read"));
}
auto detach = [this]() {
QSqlQuery q(m_data_base);
q.exec(QStringLiteral("DETACH DATABASE idx"));
};
const QStringList tables = indexTables(m_data_base);
auto isTable = [this](const QString &name) {
QSqlQuery kind(m_data_base);
kind.prepare(QStringLiteral("SELECT type FROM idx.sqlite_master WHERE name = ?"));
kind.addBindValue(name);
return kind.exec() && kind.next() && kind.value(0).toString() == QLatin1String("table");
};
for (const QString &table : tables + QStringList{QStringLiteral("index_meta"),
QStringLiteral("folio_header")}) {
if (!isTable(table)) {
detach();
return refuse(QStringLiteral("the index has no table %1").arg(table));
}
}
query.exec(QStringLiteral("SELECT value FROM idx.index_meta WHERE key = 'schema'"));
if (!query.next() || query.value(0).toString() != schemaStamp()) {
detach();
return refuse(QStringLiteral("the index was written with another database schema"));
}
m_data_base.transaction();
bool ok = true;
for (const QString &table : tables) {
QSqlQuery columns(m_data_base);
columns.exec(QStringLiteral("PRAGMA main.table_info(\"%1\")").arg(table));
QStringList names;
while (columns.next())
names << QStringLiteral("\"%1\"").arg(columns.value(1).toString());
const QString list = names.join(QLatin1Char(','));
ok = ok && query.exec(QStringLiteral("DELETE FROM main.\"%1\"").arg(table))
&& query.exec(QStringLiteral("INSERT INTO main.\"%1\" (%2) SELECT %2 FROM idx.\"%1\"")
.arg(table, list));
}
if (!ok) {
m_data_base.rollback();
detach();
return refuse(QStringLiteral("the index's rows cannot be read"));
}
m_data_base.commit();
folio_headers->clear();
query.exec(QStringLiteral("SELECT uuid, xml FROM idx.folio_header ORDER BY pos"));
while (query.next())
folio_headers->append({QUuid(query.value(0).toString()), query.value(1).toString()});
if (folio_tables) {
folio_tables->reset();
if (isTable(QStringLiteral("folio_table"))) {
QList<FolioTable> read;
query.exec(QStringLiteral("SELECT uuid, folio_uuid, previous_uuid FROM idx.folio_table"));
while (query.next())
read.append({QUuid(query.value(0).toString()), QUuid(query.value(1).toString()),
QUuid(query.value(2).toString())});
*folio_tables = read;
}
}
detach();
//The stores, from their tables, as the document would fill them
m_prefilled_elements.clear();
m_prefilled_folios.clear();
m_prefilled_conductors.clear();
for (PrefillTally &tally : m_prefill_tally) tally = PrefillTally();
query.exec(QStringLiteral("SELECT element_uuid, name, value, show FROM element_information "
"ORDER BY element_uuid, ord"));
while (query.next()) {
const QUuid uuid(query.value(0).toString());
m_prefilled_elements[uuid].addValue(query.value(1).toString(), query.value(2).toString(),
query.value(3).toInt() != 0);
}
QHash<QUuid, QHash<QString, QString>> attributes;
QList<QUuid> wire_order;
query.exec(QStringLiteral("SELECT conductor_uuid, name, value FROM conductor_properties ORDER BY rowid"));
while (query.next()) {
const QUuid wire(query.value(0).toString());
if (!attributes.contains(wire)) wire_order << wire;
attributes[wire].insert(query.value(1).toString(), query.value(2).toString());
}
//A file may hold only the attributes reading needs (indexFile()):
//the table gets every attribute a saved wire carries, as before
query.exec(QStringLiteral("DELETE FROM conductor_properties"));
query.prepare(QStringLiteral("INSERT INTO conductor_properties (conductor_uuid, name, value) VALUES (?, ?, ?)"));
for (const QUuid &wire : std::as_const(wire_order)) {
const ConductorProperties properties = propertiesFrom(attributes.value(wire));
m_prefilled_conductors.insert(wire, properties);
QHash<QString, QString> &full = attributes[wire];
full.clear();
for (const auto &attribute : properties.attributes()) {
full.insert(attribute.first, attribute.second);
query.addBindValue(wire.toString());
query.addBindValue(attribute.first);
query.addBindValue(attribute.second);
query.exec();
}
}
for (auto it = m_prefilled_elements.cbegin() ; it != m_prefilled_elements.cend() ; ++it)
m_element_information.insert(it.key(), it.value());
for (auto it = m_prefilled_conductors.cbegin() ; it != m_prefilled_conductors.cend() ; ++it)
m_conductor_properties.insert(it.key(), it.value());
m_conductor_attributes = attributes;
for (const auto &header : std::as_const(*folio_headers)) {
QDomDocument document;
if (!document.setContent(header.second)) continue;
TitleBlockProperties properties;
properties.fromXml(document.documentElement());
properties.version = QetVersion::displayedVersion();
properties.collection = QET::QetCollection::Embedded;
m_prefilled_folios.insert(header.first, properties);
m_folio_titleblocks.insert(header.first, properties);
}
m_rows_from_document = true;
m_content_changed = false;
emit dataBaseUpdated();
return true;
}
/**
@brief projectDataBase::foliosWithUnjoinedWires
@return the uuids of the folios whose document has a wire that building
sources/dataBase/projectdatabase.h
-11
View File
@@ -166,17 +166,6 @@ class projectDataBase : public QObject
void setBuildingFolio(bool building);
bool isBuildingFolio() const {return m_building_folios > 0;}
bool rowsFromDocument() const {return m_rows_from_document;}
QString schemaStamp() const;
/// A drawn table of a folio and the table it continues
struct FolioTable { QUuid table; QUuid folio; QUuid previous; };
QByteArray indexFile(const QList<std::pair<QUuid, QString>> &folio_headers,
const QList<FolioTable> &tables = QList<FolioTable>());
void discardFill();
/// The tables are right as they are (filled from an index)
void markCurrent() {m_content_changed = false;}
bool fillFromIndex(const QByteArray &index, QList<std::pair<QUuid, QString>> *folio_headers,
QString *why = nullptr,
std::optional<QList<FolioTable>> *folio_tables = nullptr);
QString prefillReport() const;
QSet<QUuid> foliosWithUnjoinedWires() const;
QSet<QUuid> folioUuidsOfElements(const QSet<QUuid> &elements) const;