diff --git a/sources/container/QETZ-FORMAT.md b/sources/container/QETZ-FORMAT.md new file mode 100644 index 000000000..d7b2f0234 --- /dev/null +++ b/sources/container/QETZ-FORMAT.md @@ -0,0 +1,182 @@ +# The `.qetz` project file, format 3 + +A `.qetz` is a QElectroTech project saved as a zip archive: the same project +a `.qet` holds, split into files, with the engineering data in an SQLite +database. This document describes format 3. `tst_qetcontainer` checks that +the schema below is the one QElectroTech writes. + +The code is `qetcontainer.cpp` (the zip and its files), `qetcontainerdb.cpp` +(the database) and `qetzip.cpp` (reading and writing zip archives). + +## 1. Principles + +- **One writer.** A `.qet` and a `.qetz` are written from the same project + document. Saving a `.qetz` splits that document into files and a database; + opening one puts it back together, and the program reads it exactly as a + `.qet`. Opening a `.qetz` and saving it as `.qet` gives the file a plain + `.qet` save gives. +- **Each fact once.** A value moved into the database is taken out of the + folio file. Nothing is stored in both. +- **What a thing is, against where it is drawn.** The database holds the + engineering data; the folio files hold the drawing: + + | | database | folio file | + |---|---|---| + | Symbol | its information (label, comment, …), prefix, links | which symbol it is (`type`), position, rotation | + | Label shown on a folio | the value | the text item: position, font, rotation | + | Wire | its ends, number, function, voltage, colour, section | its path, the number's text position and font | + | Folio | title-block fields and custom variables | everything drawn on it | + | Terminal strip | its terminals, their order and bridges | the strip drawn on a folio | + + Cables are kept in `project.xml` in this format. +- **Nothing is lost.** Anything the database does not model stays in the + XML as it was. A block is moved only if it can be rebuilt exactly. + +## 2. The archive + +| Entry | Content | +|---|---| +| `mimetype` | `application/x-qelectrotech-project+zip`, the first entry, stored uncompressed | +| `manifest.xml` | `` | +| `project.xml` | the project document, each part replaced by `` | +| `project.sqlite` | the engineering data (section 4) | +| `folios/.xml` | one `` per folio, named by the folio's uuid (or its position when it has none) | +| `elements//` | the project's symbol definitions, as in the symbol collection | +| `titleblocks/.titleblock` | the project's title-block templates | +| `images/.png` | each folio picture once, as a file instead of base64 text (`.bin` if it is not a PNG) | + +Names that would collide get `~2`, `~3`… before the extension. A picture +in a folio file keeps the name of its file in the attribute `qetz-file`. + +A reader refuses the whole archive if an entry is encrypted, uses a +compression method it cannot read, has a name that is absolute, holds `..`, +`\` or `:`, or repeats another entry's name, or if the archive holds more +than 100 000 entries or more than 2 GiB once unpacked. Entries the format +does not name are ignored. + +## 3. Versions + +`manifest.xml` gives two numbers: `format`, the format written, and +`min-reader`, the oldest format allowed to read the file. + +- A reader opens a file whose `min-reader` is not above its own format. +- If `format` is above its own, the project opens **read-only**, with a + message: saving it could drop what that reader does not know. *Save As* + writes a copy in the reader's own format. +- If `min-reader` is above its own format, the file is refused with a + message. + +The database repeats the format in `meta` (key `format`) and in +`PRAGMA user_version`. + +**Rules for a new format:** + +1. New data goes in a new column or table. An older reader skips both. +2. A change an older reader would misread raises `min-reader`. +3. Every format keeps a saved file in `tests/qttest/fixtures` that must + keep opening. + +| Format | Written by | Reading | +|---|---|---| +| 2 | QElectroTech 0.200.1 development builds (#1462) | read: rows keyed by position (`f1`, `e1`, `c1`) where an item had no uuid of its own, every value as text | +| 3 | this document | read and written | + +## 4. The database + +### 4.1 Schema + + +```sql +CREATE TABLE meta (key TEXT PRIMARY KEY, value TEXT); +CREATE TABLE project_property (ord INTEGER PRIMARY KEY, name TEXT, value TEXT, show INTEGER, attrs TEXT); +CREATE TABLE folio (uuid TEXT PRIMARY KEY, pos INTEGER NOT NULL UNIQUE, + "title" TEXT, "author" TEXT, "date" TEXT, "folio" TEXT, "indexrev" TEXT, "filename" TEXT, + "locmach" TEXT, "plant" TEXT, "auto_page_num" TEXT, props_attrs TEXT); +CREATE TABLE folio_property (folio_uuid TEXT NOT NULL REFERENCES folio, ord INTEGER NOT NULL, + name TEXT, value TEXT, show INTEGER, attrs TEXT, PRIMARY KEY (folio_uuid, ord)); +CREATE TABLE element (uuid TEXT PRIMARY KEY, folio_uuid TEXT NOT NULL REFERENCES folio, + "prefix" TEXT, "freezeLabel" INTEGER, info_attrs TEXT, links_attrs TEXT); +CREATE TABLE element_info (element_uuid TEXT NOT NULL REFERENCES element, ord INTEGER NOT NULL, + name TEXT, value TEXT, show INTEGER, attrs TEXT, PRIMARY KEY (element_uuid, ord)); +CREATE TABLE link (element_uuid TEXT NOT NULL REFERENCES element, ord INTEGER NOT NULL, + linked_uuid TEXT, attrs TEXT, PRIMARY KEY (element_uuid, ord)); +CREATE TABLE conductor (uuid TEXT PRIMARY KEY, folio_uuid TEXT NOT NULL REFERENCES folio, + "element1" TEXT, "terminal1" TEXT, "element2" TEXT, "terminal2" TEXT, "num" TEXT, + "formula" TEXT, "function" TEXT, "tension_protocol" TEXT, "conductor_color" TEXT, + "conductor_section" TEXT, "cable" TEXT, "bus" TEXT, "freezeLabel" INTEGER); +CREATE TABLE terminal_strip (uuid TEXT PRIMARY KEY, ord INTEGER NOT NULL UNIQUE, + installation TEXT, location TEXT, name TEXT, comment TEXT, description TEXT); +CREATE TABLE strip_terminal (strip_uuid TEXT NOT NULL REFERENCES terminal_strip, + pos INTEGER NOT NULL, level INTEGER NOT NULL, element_uuid TEXT, + PRIMARY KEY (strip_uuid, pos, level)); +CREATE TABLE strip_bridge (uuid TEXT PRIMARY KEY, strip_uuid TEXT NOT NULL REFERENCES terminal_strip, + ord INTEGER NOT NULL, color TEXT); +CREATE TABLE strip_bridge_terminal (bridge_uuid TEXT NOT NULL REFERENCES strip_bridge, + ord INTEGER NOT NULL, element_uuid TEXT, PRIMARY KEY (bridge_uuid, ord)); +``` + +### 4.2 What each table holds + +| Table | One row per | Comes from | +|---|---|---| +| `meta` | setting of the file | `format`; `props_attrs` (attributes of the project's ``); `terminal_strips` = `1` when the strips are in the database | +| `project_property` | project property (project-wide variable) | `` | +| `folio` | folio, in folio order (`pos` from 1) | the title-block attributes of `` | +| `folio_property` | custom title-block variable of a folio | `` | +| `element` | symbol placed on a folio | ``: its label prefix and frozen-label flag | +| `element_info` | piece of symbol information, in order | `` | +| `link` | cross-reference link of a symbol, in order | `` | +| `conductor` | wire | ``: its two ends (symbol uuid and terminal), number, formula, function, voltage/protocol, colour, section, cable, bus, frozen-label flag | +| `terminal_strip`, `strip_terminal`, `strip_bridge`, `strip_bridge_terminal` | terminal strip, terminal, bridge, bridged terminal | `` | + +A value is NULL when the XML has no such attribute. + +- `freezeLabel` is 1 for `true`, 0 for `false`. Any other text stays as an + attribute in the folio file. +- `show` is 0 or 1. Any other text goes in `attrs`. +- `attrs`, `props_attrs`, `info_attrs` and `links_attrs` are JSON objects of + the attributes that have no column, and the entry's tag when it is not the + usual one. In a project QElectroTech saves they are all NULL. They hold + something only for older or hand-edited files. Whitespace between + entries, which only a hand-edited file has, is kept in them as + `qetz-ws`. + +### 4.3 Keys + +Every row is keyed by its item's **uuid**, which stays in the folio file: +that is how a folio file's item finds its row. Folios, symbols and wires +are keyed this way. + +An item with no uuid of its own (none, or one another item of the same kind +shares) gets a key made from the item itself: a version 5 uuid in the +namespace `{3f0c6c1e-5b8e-4b8a-9d55-6b2f5f0f9a41}` from its XML text, how +many identical items came before it, and a counter that avoids taken keys. +The key is the same on every save and never depends on where the item is. +It is written on the item as `qetz-key` (a symbol or wire) or `qetz-folio` +(a folio). + +### 4.4 Markers + +A block moved into the database leaves `` where it was: +`properties`, `elementInformations`, `links_uuids` or `terminal_strips`. +Reading puts the block back where its marker is; a block without a marker +was never moved. + +Only a block that can be rebuilt exactly is moved: entries of the expected +tag with plain text values. Terminal strips are moved only if their rows +rebuild the same XML. Anything else stays in the folio file or +`project.xml` unchanged. + +## 5. Reading a file safely + +A `.qetz` is untrusted. A reader: + +- runs nothing from the database's schema (`PRAGMA trusted_schema = OFF`, + read-only) and refuses a database where any table of section 4.1 is not a + plain table (a view, say); +- refuses the file, with a message saying what is wrong, if the database + and the folio files disagree: an item whose row is missing or on another + folio, an item twice, a row no item has, or information, links or + properties of a symbol or folio that does not exist. + +A damaged archive is caught by the zip format's own checksums. diff --git a/sources/container/qetcontainer.cpp b/sources/container/qetcontainer.cpp index 40b8c0db4..73a5f65d2 100644 --- a/sources/container/qetcontainer.cpp +++ b/sources/container/qetcontainer.cpp @@ -153,16 +153,18 @@ bool QetContainer::isZip(const QByteArray &content) /** @brief QetContainer::readDocument The project document held in @p content: unzipped and joined for a - .qetz, parsed for a .qet. + .qetz, parsed for a .qet. @p format, if any: the .qetz's format, 0 + for a .qet. */ bool QetContainer::readDocument(const QByteArray &content, QDomDocument *document, - QString *error, Pictures *pictures) + QString *error, Pictures *pictures, int *format) { + if (format) *format = 0; if (!isZip(content)) return parse(content, document, error); QList entries; return QetZip::fromBytes(content, &entries, error) - && join(entries, document, error, pictures); + && join(entries, document, error, pictures, format); } QString QetContainer::pictureFile(const QDomElement &image) @@ -272,9 +274,11 @@ QList QetContainer::split(const QDomDocument &project, const QStr nothing names are ignored, so a later format may add some. With @p pictures, the pictures stay out of the document: see the namespace. + @p format, if any: the format the file says it is, above Format for a + later one this may read (its min-reader allows it) */ bool QetContainer::join(const QList &entries, QDomDocument *project, QString *error, - Pictures *pictures) + Pictures *pictures, int *format) { QHash files; for (const QetZip::Entry &entry : entries) @@ -295,6 +299,8 @@ bool QetContainer::join(const QList &entries, QDomDocument *proje .arg(Format)); return false; } + if (format) + *format = manifest.documentElement().attribute(QStringLiteral("format")).toInt(); if (!parse(files.value(QStringLiteral("project.xml")), project, error)) return false; diff --git a/sources/container/qetcontainer.h b/sources/container/qetcontainer.h index 7d0b4152a..f53e9ab79 100644 --- a/sources/container/qetcontainer.h +++ b/sources/container/qetcontainer.h @@ -32,7 +32,7 @@ manifest.xml format and oldest format allowed to read it project.xml the project, each part replaced by - project.sqlite the engineering data (format 2, + project.sqlite the engineering data (format 2 on, QetContainerDb): title-block fields, symbol information, links, wires folios/.xml one per folio @@ -47,6 +47,8 @@ each keeps the name of its file, pictureFile(), and the bytes go in the map. Turning them back into base64 text costs seconds on a project with large pictures, and the folio would only decode it again. + + The format is written down in QETZ-FORMAT.md, beside this file. */ namespace QetContainer { @@ -55,13 +57,19 @@ namespace QetContainer using Pictures = QHash; /// The format written; files of a later format are refused by join() /// unless their min-reader allows this one. - constexpr int Format = 2; + constexpr int Format = 3; + /// 3: the database's rows keyed by the items' uuids, true/false and + /// show flags as 1/0, its PRAGMA user_version the format, the symbols' type left + /// in the folio files, no column for where a block sat. Format 2 is + /// still read; a later format whose min-reader allows this one is + /// read too, and said so through join()'s @p format. QList split(const QDomDocument &project, const QString &writer = QString(), QString *error = nullptr); bool join(const QList &entries, QDomDocument *project, - QString *error = nullptr, Pictures *pictures = nullptr); + QString *error = nullptr, Pictures *pictures = nullptr, + int *format = nullptr); /// The file join() left the picture of @p image in, or an empty string QString pictureFile(const QDomElement &image); @@ -71,7 +79,8 @@ namespace QetContainer bool isZip(const QByteArray &content); /// The project document in @p content, a .qet or a .qetz bool readDocument(const QByteArray &content, QDomDocument *document, - QString *error = nullptr, Pictures *pictures = nullptr); + QString *error = nullptr, Pictures *pictures = nullptr, + int *format = nullptr); } #endif // QETCONTAINER_H diff --git a/sources/container/qetcontainerdb.cpp b/sources/container/qetcontainerdb.cpp index 321fddd49..66cf9cbf8 100644 --- a/sources/container/qetcontainerdb.cpp +++ b/sources/container/qetcontainerdb.cpp @@ -17,11 +17,15 @@ */ #include "qetcontainerdb.h" +#include "qetcontainer.h" + #include #include #include #include #include +#include +#include #include #include @@ -54,43 +58,81 @@ bool replaceMarker(QDomElement parent, const QDomElement &block) return false; } + //true if @p parent holds the marker of a block @p tag moved out: where + //the block goes back, and that it was there at all +bool hasMarker(const QDomElement &parent, const QString &tag) +{ + for (QDomElement m = parent.firstChildElement(MarkerTag) ; !m.isNull() ; m = m.nextSiblingElement(MarkerTag)) + if (m.attribute(MarkerBlock) == tag) return true; + return false; +} + const QStringList FolioData = { QStringLiteral("title"), QStringLiteral("author"), QStringLiteral("date"), QStringLiteral("folio"), QStringLiteral("indexrev"), QStringLiteral("filename"), QStringLiteral("locmach"), QStringLiteral("plant"), QStringLiteral("auto_page_num")}; + //From format 3 on an item's uuid is its row's key, not a column of its + //own: it stays in the folio file, where it says which row is the item's. + //So does a symbol's type, the symbol it is: a folio file read alone + //still says what is drawn where const QStringList ElementData = { - QStringLiteral("uuid"), QStringLiteral("type"), QStringLiteral("prefix"), - QStringLiteral("freezeLabel")}; + QStringLiteral("prefix"), QStringLiteral("freezeLabel")}; + //Format 2 also moved the type +const QStringList ElementDataV2 = { + QStringLiteral("uuid"), QStringLiteral("type"), QStringLiteral("prefix"), QStringLiteral("freezeLabel")}; const QStringList ConductorData = { - QStringLiteral("uuid"), QStringLiteral("element1"), QStringLiteral("terminal1"), + QStringLiteral("element1"), QStringLiteral("terminal1"), QStringLiteral("element2"), QStringLiteral("terminal2"), QStringLiteral("num"), QStringLiteral("formula"), QStringLiteral("function"), QStringLiteral("tension_protocol"), QStringLiteral("conductor_color"), QStringLiteral("conductor_section"), QStringLiteral("cable"), QStringLiteral("bus"), QStringLiteral("freezeLabel")}; + //Attributes held as 0 or 1 (format 3 on). A value QElectroTech does not + //write, "yes" say, stays in the folio file as it was. +const QStringList BooleanData = {QStringLiteral("freezeLabel")}; + QString columns(const QStringList &names) { QStringList quoted; - for (const QString &n : names) quoted << QStringLiteral("\"%1\" TEXT").arg(n); + for (const QString &n : names) + quoted << QStringLiteral("\"%1\" %2").arg(n, BooleanData.contains(n) ? QStringLiteral("INTEGER") + : QStringLiteral("TEXT")); return quoted.join(QStringLiteral(", ")); } + //@p text as attribute @p name is stored, or invalid if it cannot be +QVariant stored(const QString &name, const QString &text) +{ + if (!BooleanData.contains(name)) return text; + if (text == QLatin1String("true")) return 1; + if (text == QLatin1String("false")) return 0; + return {}; +} + + //A stored value as attribute @p name writes it +QString attributeText(const QString &name, const QVariant &value) +{ + if (BooleanData.contains(name) && value.typeId() == QMetaType::Int) + return value.toInt() ? QStringLiteral("true") : QStringLiteral("false"); + return value.toString(); +} + QString schema() { return QStringLiteral( "CREATE TABLE meta (key TEXT PRIMARY KEY, value TEXT);" - "CREATE TABLE project_property (ord INTEGER PRIMARY KEY, name TEXT, value TEXT, attrs TEXT);" - "CREATE TABLE folio (id TEXT PRIMARY KEY, pos INTEGER NOT NULL UNIQUE, %1," - " props_at INTEGER, props_attrs TEXT);" - "CREATE TABLE folio_property (folio_id TEXT NOT NULL REFERENCES folio, ord INTEGER NOT NULL," - " name TEXT, value TEXT, attrs TEXT, PRIMARY KEY (folio_id, ord));" - "CREATE TABLE element (key TEXT PRIMARY KEY, folio_id TEXT NOT NULL REFERENCES folio, %2," - " info_at INTEGER, info_attrs TEXT, links_at INTEGER, links_attrs TEXT);" - "CREATE TABLE element_info (element_key TEXT NOT NULL REFERENCES element, ord INTEGER NOT NULL," - " name TEXT, value TEXT, attrs TEXT, PRIMARY KEY (element_key, ord));" - "CREATE TABLE link (element_key TEXT NOT NULL REFERENCES element, ord INTEGER NOT NULL," - " linked_uuid TEXT, attrs TEXT, PRIMARY KEY (element_key, ord));" - "CREATE TABLE conductor (key TEXT PRIMARY KEY, folio_id TEXT NOT NULL REFERENCES folio, %3);" + "CREATE TABLE project_property (ord INTEGER PRIMARY KEY, name TEXT, value TEXT, show INTEGER, attrs TEXT);" + "CREATE TABLE folio (uuid TEXT PRIMARY KEY, pos INTEGER NOT NULL UNIQUE, %1," + " props_attrs TEXT);" + "CREATE TABLE folio_property (folio_uuid TEXT NOT NULL REFERENCES folio, ord INTEGER NOT NULL," + " name TEXT, value TEXT, show INTEGER, attrs TEXT, PRIMARY KEY (folio_uuid, ord));" + "CREATE TABLE element (uuid TEXT PRIMARY KEY, folio_uuid TEXT NOT NULL REFERENCES folio, %2," + " info_attrs TEXT, links_attrs TEXT);" + "CREATE TABLE element_info (element_uuid TEXT NOT NULL REFERENCES element, ord INTEGER NOT NULL," + " name TEXT, value TEXT, show INTEGER, attrs TEXT, PRIMARY KEY (element_uuid, ord));" + "CREATE TABLE link (element_uuid TEXT NOT NULL REFERENCES element, ord INTEGER NOT NULL," + " linked_uuid TEXT, attrs TEXT, PRIMARY KEY (element_uuid, ord));" + "CREATE TABLE conductor (uuid TEXT PRIMARY KEY, folio_uuid TEXT NOT NULL REFERENCES folio, %3);" "CREATE TABLE terminal_strip (uuid TEXT PRIMARY KEY, ord INTEGER NOT NULL UNIQUE," " installation TEXT, location TEXT, name TEXT, comment TEXT, description TEXT);" "CREATE TABLE strip_terminal (strip_uuid TEXT NOT NULL REFERENCES terminal_strip," @@ -165,15 +207,16 @@ QString placeholders(int n) return q.join(QLatin1Char(',')); } - //The attributes of @p e but @p skip, and its tag, as JSON -QVariant attributesJson(const QDomElement &e, const QString &skip = QString(), bool with_tag = false) + //The attributes of @p e but @p skip as JSON, and its tag if it is not + //@p usual_tag +QVariant attributesJson(const QDomElement &e, const QStringList &skip, const QString &usual_tag) { QJsonObject o; - if (with_tag) o.insert(QStringLiteral("tag"), e.tagName()); + if (e.tagName() != usual_tag) o.insert(QStringLiteral("tag"), e.tagName()); const QDomNamedNodeMap map = e.attributes(); for (int i = 0 ; i < map.count() ; ++i) { const QDomAttr a = map.item(i).toAttr(); - if (a.name() != skip) o.insert(a.name(), a.value()); + if (!skip.contains(a.name())) o.insert(a.name(), a.value()); } if (o.isEmpty()) return QVariant(); return QString::fromUtf8(QJsonDocument(o).toJson(QJsonDocument::Compact)); @@ -257,21 +300,15 @@ bool listBlock(const QDomElement &block, const QString &entry) return true; } -int childIndex(const QDomNode &child) -{ - int i = 0; - for (QDomNode n = child.parentNode().firstChild() ; !n.isNull() ; n = n.nextSibling(), ++i) - if (n == child) return i; - return -1; -} - - + //The attributes @p names of @p e, taken out of it; one that cannot be + //stored stays, its value null QVariantList take(QDomElement e, const QStringList &names) { QVariantList values; for (const QString &n : names) { - values << (e.hasAttribute(n) ? QVariant(e.attribute(n)) : QVariant()); - e.removeAttribute(n); + const QVariant v = e.hasAttribute(n) ? stored(n, e.attribute(n)) : QVariant(); + values << (v.isValid() ? v : QVariant()); + if (v.isValid()) e.removeAttribute(n); } return values; } @@ -279,7 +316,8 @@ QVariantList take(QDomElement e, const QStringList &names) void put(QDomElement e, const QStringList &names, const QVariantList &values, int from) { for (int i = 0 ; i < names.size() ; ++i) - if (!values.at(from + i).isNull()) e.setAttribute(names.at(i), values.at(from + i).toString()); + if (!values.at(from + i).isNull()) + e.setAttribute(names.at(i), attributeText(names.at(i), values.at(from + i))); } //Same tags, attributes, text and child order @@ -417,18 +455,55 @@ QList children(const QDomElement &parent, const QString &tag) return list; } - //uuid when present and unique among @p items, else prefix + count -QStringList keys(const QList &items, const QString &prefix) + //A list entry -- a , an -- as its row: + //its name, value, show flag (0 or 1, else left in the JSON) and any + //other attribute as JSON, its tag too if it is not @p usual_tag +QVariantList entryRow(const QDomElement &e, const QString &usual_tag) { + QVariant value; + simpleValue(e, &value); + const QString show = e.attribute(QStringLiteral("show")); + const bool show_stored = show == QLatin1String("0") || show == QLatin1String("1"); + QStringList kept{QStringLiteral("name")}; + if (show_stored) kept << QStringLiteral("show"); + return {e.hasAttribute(QStringLiteral("name")) ? QVariant(e.attribute(QStringLiteral("name"))) : QVariant(), + value, show_stored ? QVariant(show.toInt()) : QVariant(), attributesJson(e, kept, usual_tag)}; +} + + //The key of each of @p items: its uuid, when it has one no other of + //@p items has, so the uuid the folio file already holds says which row + //is the item's (@p own true). Else (F100: symbols copied with their + //uuid) a uuid made from the item itself -- the same on every save, + //never from where the item is -- kept in the folio file as qetz-key. +QStringList keys(const QList &items, QList *own) +{ + static const QUuid key_namespace(QStringLiteral("{3f0c6c1e-5b8e-4b8a-9d55-6b2f5f0f9a41}")); QHash seen; for (const QDomElement &e : items) - if (e.hasAttribute(QStringLiteral("uuid"))) ++seen[e.attribute(QStringLiteral("uuid"))]; + if (!QUuid(e.attribute(QStringLiteral("uuid"))).isNull()) ++seen[e.attribute(QStringLiteral("uuid"))]; + QSet taken(seen.keyBegin(), seen.keyEnd()); + QHash twins; QStringList out; - int made_up = 0; for (const QDomElement &e : items) { const QString uuid = e.attribute(QStringLiteral("uuid")); - out << ((!uuid.isEmpty() && seen.value(uuid) == 1) - ? uuid : prefix + QString::number(++made_up)); + if (seen.value(uuid) == 1) { + out << uuid; + *own << true; + continue; + } + QString text; + QTextStream stream(&text); + e.save(stream, -1); + stream.flush(); + //Items alike to the byte: how many came before tells them apart + const int twin = twins[text]++; + QString key; + for (int salt = 0 ; key.isEmpty() || taken.contains(key) ; ++salt) + key = QUuid::createUuidV5(key_namespace, QStringLiteral("%1\n%2\n%3").arg(text).arg(twin).arg(salt)) + .toString(); + taken.insert(key); + out << key; + *own << false; } return out; } @@ -450,43 +525,31 @@ QByteArray QetContainerDb::extract(QDomElement project, QString *error) db.exec(QStringLiteral("BEGIN")); Statement meta(db.m_db, QStringLiteral("INSERT INTO meta VALUES (?,?)")); - Statement project_property(db.m_db, QStringLiteral("INSERT INTO project_property VALUES (?,?,?,?)")); - Statement folio(db.m_db, QStringLiteral("INSERT INTO folio VALUES (?,?,%1,?,?)").arg(placeholders(FolioData.size()))); - Statement folio_property(db.m_db, QStringLiteral("INSERT INTO folio_property VALUES (?,?,?,?,?)")); - Statement element(db.m_db, QStringLiteral("INSERT INTO element VALUES (?,?,%1,?,?,?,?)").arg(placeholders(ElementData.size()))); - Statement element_info(db.m_db, QStringLiteral("INSERT INTO element_info VALUES (?,?,?,?,?)")); + Statement project_property(db.m_db, QStringLiteral("INSERT INTO project_property VALUES (?,?,?,?,?)")); + Statement folio(db.m_db, QStringLiteral("INSERT INTO folio VALUES (?,?,%1,?)").arg(placeholders(FolioData.size()))); + Statement folio_property(db.m_db, QStringLiteral("INSERT INTO folio_property VALUES (?,?,?,?,?,?)")); + Statement element(db.m_db, QStringLiteral("INSERT INTO element VALUES (?,?,%1,?,?)").arg(placeholders(ElementData.size()))); + Statement element_info(db.m_db, QStringLiteral("INSERT INTO element_info VALUES (?,?,?,?,?,?)")); Statement link(db.m_db, QStringLiteral("INSERT INTO link VALUES (?,?,?,?)")); Statement conductor(db.m_db, QStringLiteral("INSERT INTO conductor VALUES (?,?,%1)").arg(placeholders(ConductorData.size()))); bool ok = meta.ok() && project_property.ok() && folio.ok() && folio_property.ok() && element.ok() && element_info.ok() && link.ok() && conductor.ok(); - ok = ok && meta.run({QStringLiteral("format"), QStringLiteral("2")}); + ok = ok && meta.run({QStringLiteral("format"), QString::number(QetContainer::Format)}) + && db.exec(QStringLiteral("PRAGMA user_version = %1").arg(QetContainer::Format)); // of a block, as rows of @p rows (with @p prefix values) auto moveProperties = [&](QDomElement block, Statement &rows, const QVariantList &prefix) { int ord = 0; - for (QDomElement p : children(block, QStringLiteral("property"))) { - QVariant value; - simpleValue(p, &value); - ok = ok && rows.run(QVariantList(prefix) << ord++ << (p.hasAttribute(QStringLiteral("name")) - ? QVariant(p.attribute(QStringLiteral("name"))) : QVariant()) - << value << attributesJson(p, QStringLiteral("name"), true)); - } + for (QDomElement p : children(block, QStringLiteral("property"))) + ok = ok && rows.run(QVariantList(prefix) << ord++ << entryRow(p, QStringLiteral("property"))); }; //Project properties const QDomElement project_props = project.firstChildElement(QStringLiteral("properties")); if (!project_props.isNull() && listBlock(project_props, QStringLiteral("property"))) { - ok = ok && meta.run({QStringLiteral("props_at"), QString::number(childIndex(project_props))}); ok = ok && meta.run({QStringLiteral("props_attrs"), blockJson(project_props)}); - int ord = 0; - for (QDomElement p : children(project_props, QStringLiteral("property"))) { - QVariant value; - simpleValue(p, &value); - ok = ok && project_property.run({ord++, p.hasAttribute(QStringLiteral("name")) - ? QVariant(p.attribute(QStringLiteral("name"))) : QVariant(), - value, attributesJson(p, QStringLiteral("name"), true)}); - } + moveProperties(project_props, project_property, {}); leaveMarker(project_props); } @@ -498,72 +561,69 @@ QByteArray QetContainerDb::extract(QDomElement project, QString *error) for (const QDomElement &list : children(d, QStringLiteral("conductors"))) all_conductors += children(list, QStringLiteral("conductor")); } - const QStringList element_keys = keys(all_elements, QStringLiteral("e")); - const QStringList conductor_keys = keys(all_conductors, QStringLiteral("c")); + QList folio_own, element_own, conductor_own; + const QStringList folio_keys = keys(diagrams, &folio_own); + const QStringList element_keys = keys(all_elements, &element_own); + const QStringList conductor_keys = keys(all_conductors, &conductor_own); int e_index = 0, c_index = 0; int pos = 0; for (QDomElement d : diagrams) { - const QString id = QStringLiteral("f%1").arg(++pos); + const QString id = folio_keys.at(pos); + const bool own_id = folio_own.at(pos); + ++pos; QVariantList row{id, pos}; row += take(d, FolioData); QDomElement props = d.firstChildElement(QStringLiteral("properties")); const bool move_props = !props.isNull() && listBlock(props, QStringLiteral("property")); - row << (move_props ? QVariant(childIndex(props)) : QVariant()) - << (move_props ? blockJson(props) : QVariant()); + row << (move_props ? blockJson(props) : QVariant()); ok = ok && folio.run(row); if (move_props) { moveProperties(props, folio_property, {id}); leaveMarker(props); } - d.setAttribute(FolioAttribute, id); + if (!own_id) d.setAttribute(FolioAttribute, id); for (const QDomElement &list : children(d, QStringLiteral("elements"))) { for (QDomElement e : children(list, QStringLiteral("element"))) { - const QString key = element_keys.at(e_index++); + const QString key = element_keys.at(e_index); + const bool own_key = element_own.at(e_index++); QVariantList erow{key, id}; erow += take(e, ElementData); QDomElement info = e.firstChildElement(QStringLiteral("elementInformations")); QDomElement links = e.firstChildElement(QStringLiteral("links_uuids")); if (!info.isNull() && !listBlock(info, QStringLiteral("elementInformation"))) info = QDomElement(); if (!links.isNull() && !listBlock(links, QStringLiteral("link_uuid"))) links = QDomElement(); - const int info_at = info.isNull() ? -1 : childIndex(info); - const int links_at = links.isNull() ? -1 : childIndex(links); - erow << (info_at < 0 ? QVariant() : QVariant(info_at)) - << (info_at < 0 ? QVariant() : blockJson(info)) - << (links_at < 0 ? QVariant() : QVariant(links_at)) - << (links_at < 0 ? QVariant() : blockJson(links)); + erow << (info.isNull() ? QVariant() : blockJson(info)) + << (links.isNull() ? QVariant() : blockJson(links)); ok = ok && element.run(erow); - if (info_at >= 0) { + if (!info.isNull()) { int ord = 0; - for (QDomElement i : children(info, QStringLiteral("elementInformation"))) { - QVariant value; - simpleValue(i, &value); - ok = ok && element_info.run({key, ord++, i.hasAttribute(QStringLiteral("name")) - ? QVariant(i.attribute(QStringLiteral("name"))) : QVariant(), - value, attributesJson(i, QStringLiteral("name"), true)}); - } + for (QDomElement i : children(info, QStringLiteral("elementInformation"))) + ok = ok && element_info.run(QVariantList{key, ord++} + << entryRow(i, QStringLiteral("elementInformation"))); } - if (links_at >= 0) { + if (!links.isNull()) { int ord = 0; for (QDomElement l : children(links, QStringLiteral("link_uuid"))) ok = ok && link.run({key, ord++, l.hasAttribute(QStringLiteral("uuid")) ? QVariant(l.attribute(QStringLiteral("uuid"))) : QVariant(), - attributesJson(l, QStringLiteral("uuid"), true)}); + attributesJson(l, {QStringLiteral("uuid")}, QStringLiteral("link_uuid"))}); } - if (info_at >= 0) leaveMarker(info); - if (links_at >= 0) leaveMarker(links); - e.setAttribute(KeyAttribute, key); + if (!info.isNull()) leaveMarker(info); + if (!links.isNull()) leaveMarker(links); + if (!own_key) e.setAttribute(KeyAttribute, key); } } for (const QDomElement &list : children(d, QStringLiteral("conductors"))) { for (QDomElement c : children(list, QStringLiteral("conductor"))) { - const QString key = conductor_keys.at(c_index++); + const QString key = conductor_keys.at(c_index); + const bool own_key = conductor_own.at(c_index++); QVariantList crow{key, id}; crow += take(c, ConductorData); ok = ok && conductor.run(crow); - c.setAttribute(KeyAttribute, key); + if (!own_key) c.setAttribute(KeyAttribute, key); } } } @@ -637,10 +697,11 @@ bool QetContainerDb::restore(QDomElement project, const QByteArray &database, QS } auto entry = [&document](const QVariant &name_attr, const QString &name, const QVariant &value, - const QVariant &attrs, const QString &default_tag) { + const QVariant &attrs, const QString &default_tag, const QVariant &show = QVariant()) { const QJsonObject o = QJsonDocument::fromJson(attrs.toString().toUtf8()).object(); QDomElement e = document.createElement(o.value(QStringLiteral("tag")).toString(default_tag)); if (!name_attr.isNull()) e.setAttribute(name, name_attr.toString()); + if (!show.isNull()) e.setAttribute(QStringLiteral("show"), show.toString()); setAttributesJson(e, attrs); if (!value.isNull()) e.appendChild(document.createTextNode(value.toString())); return e; @@ -652,13 +713,47 @@ bool QetContainerDb::restore(QDomElement project, const QByteArray &database, QS Statement s(db.m_db, QStringLiteral("SELECT key, value FROM meta")); while (s.next()) meta.insert(s.value(0).toString(), s.value(1)); } - if (meta.contains(QStringLiteral("props_at"))) { + //Format 2 (QElectroTech 0.200.1 development builds) keyed its rows + //apart from the items' uuids: "f1", "e1", "c1" where an item had + //none of its own, and every item's key in the folio file + const int format = meta.value(QStringLiteral("format")).toString().toInt(); + //A later format is read when the manifest's min-reader allows it + //(QetContainer::join()): its writer kept what this reads + if (format < 2) { + setError(error, QStringLiteral("the project database is of format %1, this reads format 2 on") + .arg(meta.value(QStringLiteral("format")).toString())); + return false; + } + const bool v2 = format == 2; + const QString folio_key = v2 ? QStringLiteral("id") : QStringLiteral("uuid"); + const QString folio_ref = v2 ? QStringLiteral("folio_id") : QStringLiteral("folio_uuid"); + const QString item_key = v2 ? QStringLiteral("key") : QStringLiteral("uuid"); + const QString element_ref = v2 ? QStringLiteral("element_key") : QStringLiteral("element_uuid"); + const QString show_column = v2 ? QStringLiteral("NULL") : QStringLiteral("show"); + const QStringList element_data = v2 ? ElementDataV2 : ElementData; + const QStringList conductor_data = v2 ? QStringList{QStringLiteral("uuid")} + ConductorData : ConductorData; + auto quoted = [](const QStringList &names) { + return QStringLiteral("\"") + names.join(QStringLiteral("\", \"")) + QStringLiteral("\""); + }; + //An item's key: the one it was given, else (format 3) its uuid + auto keyOf = [v2](QDomElement e, const QString &attribute) { + if (e.hasAttribute(attribute)) { + const QString key = e.attribute(attribute); + e.removeAttribute(attribute); + return key; + } + return v2 ? QString() : e.attribute(QStringLiteral("uuid")); + }; + //A moved block goes back where its marker is: the marker says it + //was there + if (hasMarker(project, QStringLiteral("properties"))) { QDomElement props = document.createElement(QStringLiteral("properties")); setAttributesJson(props, meta.value(QStringLiteral("props_attrs"))); - Statement s(db.m_db, QStringLiteral("SELECT name, value, attrs FROM project_property ORDER BY ord")); + Statement s(db.m_db, QStringLiteral("SELECT name, value, attrs, %1 FROM project_property ORDER BY ord") + .arg(show_column)); while (s.next()) props.appendChild(entry(s.value(0), QStringLiteral("name"), s.value(1), s.value(2), - QStringLiteral("property"))); + QStringLiteral("property"), s.value(3))); restoreWhitespace(props, meta.value(QStringLiteral("props_attrs"))); if (!replaceMarker(project, props)) { setError(error, QStringLiteral("the project's properties have no place")); @@ -686,27 +781,39 @@ bool QetContainerDb::restore(QDomElement project, const QByteArray &database, QS } } + //Every row must be an item's, once, on the folio the item is on: + //nothing enforces it in the file, which is untrusted + QSet folios_used, elements_used, conductors_used; + auto use = [error](QSet &used, const QString &key, const QString &what) { + if (used.contains(key)) { + setError(error, QStringLiteral("%1 %2 is in the folios twice").arg(what, key)); + return false; + } + used.insert(key); + return true; + }; for (QDomElement d : children(project, QStringLiteral("diagram"))) { - const QString id = d.attribute(FolioAttribute); + const QString id = keyOf(d, FolioAttribute); if (id.isEmpty()) continue; - d.removeAttribute(FolioAttribute); - Statement f(db.m_db, QStringLiteral("SELECT %1, props_at, props_attrs FROM folio WHERE id = ?") - .arg(QStringLiteral("\"") + FolioData.join(QStringLiteral("\", \"")) + QStringLiteral("\""))); + if (!use(folios_used, id, QStringLiteral("folio"))) return false; + Statement f(db.m_db, QStringLiteral("SELECT %1, props_attrs FROM folio WHERE %2 = ?") + .arg(quoted(FolioData), folio_key)); f.bind({id}); if (!f.next()) { setError(error, QStringLiteral("folio %1 is not in the database").arg(id)); return false; } QVariantList values; - for (int i = 0 ; i < FolioData.size() + 2 ; ++i) values << f.value(i); + for (int i = 0 ; i < FolioData.size() + 1 ; ++i) values << f.value(i); put(d, FolioData, values, 0); - if (!values.at(FolioData.size()).isNull()) { + if (hasMarker(d, QStringLiteral("properties"))) { QDomElement props = document.createElement(QStringLiteral("properties")); - setAttributesJson(props, values.at(FolioData.size() + 1)); - Statement p(db.m_db, QStringLiteral("SELECT name, value, attrs FROM folio_property WHERE folio_id = ? ORDER BY ord")); + setAttributesJson(props, values.at(FolioData.size())); + Statement p(db.m_db, QStringLiteral("SELECT name, value, attrs, %1 FROM folio_property WHERE %2 = ? ORDER BY ord") + .arg(show_column, folio_ref)); p.bind({id}); while (p.next()) props.appendChild(entry(p.value(0), QStringLiteral("name"), p.value(1), p.value(2), - QStringLiteral("property"))); - restoreWhitespace(props, values.at(FolioData.size() + 1)); + QStringLiteral("property"), p.value(3))); + restoreWhitespace(props, values.at(FolioData.size())); if (!replaceMarker(d, props)) { setError(error, QStringLiteral("folio %1's properties have no place").arg(id)); return false; @@ -715,37 +822,42 @@ bool QetContainerDb::restore(QDomElement project, const QByteArray &database, QS for (const QDomElement &list : children(d, QStringLiteral("elements"))) { for (QDomElement e : children(list, QStringLiteral("element"))) { - if (!e.hasAttribute(KeyAttribute)) continue; - const QString key = e.attribute(KeyAttribute); - e.removeAttribute(KeyAttribute); - Statement s(db.m_db, QStringLiteral("SELECT %1, info_at, info_attrs, links_at, links_attrs FROM element WHERE key = ?") - .arg(QStringLiteral("\"") + ElementData.join(QStringLiteral("\", \"")) + QStringLiteral("\""))); + const QString key = keyOf(e, KeyAttribute); + if (key.isEmpty()) continue; + if (!use(elements_used, key, QStringLiteral("symbol"))) return false; + Statement s(db.m_db, QStringLiteral("SELECT %1, info_attrs, links_attrs, %2 FROM element WHERE %3 = ?") + .arg(quoted(element_data), folio_ref, item_key)); s.bind({key}); if (!s.next()) { setError(error, QStringLiteral("symbol %1 is not in the database").arg(key)); return false; } + if (s.value(element_data.size() + 2).toString() != id) { + setError(error, QStringLiteral("symbol %1 is on folio %2, the database says another").arg(key, id)); + return false; + } QVariantList v; - for (int i = 0 ; i < ElementData.size() + 4 ; ++i) v << s.value(i); - put(e, ElementData, v, 0); - const int n = ElementData.size(); + for (int i = 0 ; i < element_data.size() + 2 ; ++i) v << s.value(i); + put(e, element_data, v, 0); + const int n = element_data.size(); QDomElement info, links; - if (!v.at(n).isNull()) { + if (hasMarker(e, QStringLiteral("elementInformations"))) { info = document.createElement(QStringLiteral("elementInformations")); - setAttributesJson(info, v.at(n + 1)); - Statement i(db.m_db, QStringLiteral("SELECT name, value, attrs FROM element_info WHERE element_key = ? ORDER BY ord")); + setAttributesJson(info, v.at(n)); + Statement i(db.m_db, QStringLiteral("SELECT name, value, attrs, %1 FROM element_info WHERE %2 = ? ORDER BY ord") + .arg(show_column, element_ref)); i.bind({key}); while (i.next()) info.appendChild(entry(i.value(0), QStringLiteral("name"), i.value(1), i.value(2), - QStringLiteral("elementInformation"))); - restoreWhitespace(info, v.at(n + 1)); + QStringLiteral("elementInformation"), i.value(3))); + restoreWhitespace(info, v.at(n)); } - if (!v.at(n + 2).isNull()) { + if (hasMarker(e, QStringLiteral("links_uuids"))) { links = document.createElement(QStringLiteral("links_uuids")); - setAttributesJson(links, v.at(n + 3)); - Statement l(db.m_db, QStringLiteral("SELECT linked_uuid, attrs FROM link WHERE element_key = ? ORDER BY ord")); + setAttributesJson(links, v.at(n + 1)); + Statement l(db.m_db, QStringLiteral("SELECT linked_uuid, attrs FROM link WHERE %1 = ? ORDER BY ord").arg(element_ref)); l.bind({key}); while (l.next()) links.appendChild(entry(l.value(0), QStringLiteral("uuid"), QVariant(), l.value(1), QStringLiteral("link_uuid"))); - restoreWhitespace(links, v.at(n + 3)); + restoreWhitespace(links, v.at(n + 1)); } if ((!info.isNull() && !replaceMarker(e, info)) || (!links.isNull() && !replaceMarker(e, links))) { @@ -756,18 +868,47 @@ bool QetContainerDb::restore(QDomElement project, const QByteArray &database, QS } for (const QDomElement &list : children(d, QStringLiteral("conductors"))) { for (QDomElement c : children(list, QStringLiteral("conductor"))) { - if (!c.hasAttribute(KeyAttribute)) continue; - const QString key = c.attribute(KeyAttribute); - c.removeAttribute(KeyAttribute); - Statement s(db.m_db, QStringLiteral("SELECT %1 FROM conductor WHERE key = ?") - .arg(QStringLiteral("\"") + ConductorData.join(QStringLiteral("\", \"")) + QStringLiteral("\""))); + const QString key = keyOf(c, KeyAttribute); + if (key.isEmpty()) continue; + if (!use(conductors_used, key, QStringLiteral("wire"))) return false; + Statement s(db.m_db, QStringLiteral("SELECT %1, %2 FROM conductor WHERE %3 = ?") + .arg(quoted(conductor_data), folio_ref, item_key)); s.bind({key}); if (!s.next()) { setError(error, QStringLiteral("wire %1 is not in the database").arg(key)); return false; } + if (s.value(conductor_data.size()).toString() != id) { + setError(error, QStringLiteral("wire %1 is on folio %2, the database says another").arg(key, id)); + return false; + } QVariantList v; - for (int i = 0 ; i < ConductorData.size() ; ++i) v << s.value(i); - put(c, ConductorData, v, 0); + for (int i = 0 ; i < conductor_data.size() ; ++i) v << s.value(i); + put(c, conductor_data, v, 0); } } } + + //No row left over: each table's rows, counted, are the items read, + //and every entry belongs to a row + auto count = [&db](const QString &sql) { + Statement s(db.m_db, sql); + return s.next() ? s.value(0).toInt() : -1; + }; + const struct { QString sql; int expected; QString what; } checks[] = { + {QStringLiteral("SELECT count(*) FROM folio"), int(folios_used.size()), QStringLiteral("folio rows")}, + {QStringLiteral("SELECT count(*) FROM element"), int(elements_used.size()), QStringLiteral("symbol rows")}, + {QStringLiteral("SELECT count(*) FROM conductor"), int(conductors_used.size()), QStringLiteral("wire rows")}, + {QStringLiteral("SELECT count(*) FROM folio_property WHERE %1 NOT IN (SELECT %2 FROM folio)") + .arg(folio_ref, folio_key), 0, QStringLiteral("folio property rows of no folio")}, + {QStringLiteral("SELECT count(*) FROM element_info WHERE %1 NOT IN (SELECT %2 FROM element)") + .arg(element_ref, item_key), 0, QStringLiteral("information rows of no symbol")}, + {QStringLiteral("SELECT count(*) FROM link WHERE %1 NOT IN (SELECT %2 FROM element)") + .arg(element_ref, item_key), 0, QStringLiteral("link rows of no symbol")}}; + for (const auto &check : checks) { + const int found = count(check.sql); + if (found != check.expected) { + setError(error, QStringLiteral("the project database does not match its folios: %1 %2, %3 expected") + .arg(found).arg(check.what).arg(check.expected)); + return false; + } + } return true; } diff --git a/sources/container/qetcontainerdb.h b/sources/container/qetcontainerdb.h index a70dfb1fe..a60c293b3 100644 --- a/sources/container/qetcontainerdb.h +++ b/sources/container/qetcontainerdb.h @@ -30,8 +30,10 @@ drawn. extract() takes them out of the document into a database, restore() puts them back, so that restore(extract(d)) is d. - Each moved item leaves its key in the document (qetz-key, qetz-folio): - its uuid where that is unique, else a made-up one. A block of a shape + A row's key is its item's uuid, which stays in the folio file. An item + without a uuid of its own -- none, or one another item shares -- gets a + uuid made from what it holds, left in the document as qetz-key or + qetz-folio. A block of a shape this does not model -- a value holding elements, say -- stays in the document, so nothing can be lost by moving. */ diff --git a/sources/qetdiagrameditor.cpp b/sources/qetdiagrameditor.cpp index 8c9924606..798f29e23 100644 --- a/sources/qetdiagrameditor.cpp +++ b/sources/qetdiagrameditor.cpp @@ -1969,6 +1969,16 @@ bool QETDiagramEditor::openAndAddProject( addProject(project); DialogWaiting::dropInstance(); + if (interactive && project->isFromLaterVersion()) { + QET::QetMessageBox::warning( + this, + tr("Opening the file read-only", "message box title"), + tr("This project was saved by a later version of QElectroTech. " + "It is opened read-only, so that saving it cannot lose what " + "this version does not know. Use Save As to keep a copy " + "in this version's format.")); + } + //Report font descriptions which could not be read as-is (written by //an incompatible Qt version or corrupted), so the user learns about //it from somewhere else than the console. See issue #553. diff --git a/sources/qetproject.cpp b/sources/qetproject.cpp index 805ffae00..a96d3d0de 100644 --- a/sources/qetproject.cpp +++ b/sources/qetproject.cpp @@ -485,7 +485,9 @@ QETProject::ProjectState QETProject::openFile(QFile *file) // QetContainer::parse() does exactly that, for a .qet and for each part // of a .qetz alike. QString read_error; - if (!QetContainer::readDocument(content, &xml_project, &read_error, &m_container_pictures)) + int container_format = 0; + if (!QetContainer::readDocument(content, &xml_project, &read_error, &m_container_pictures, + &container_format)) { qWarning().noquote() << "Unable to read" << fi.fileName() << ":" << read_error; m_container_pictures.clear(); @@ -519,9 +521,16 @@ QETProject::ProjectState QETProject::openFile(QFile *file) << xml_parse_ms / 1000.0 << ", content " << (total_ms - xml_parse_ms) / 1000.0 << ")"; - if (!fi.isWritable()) { + //Written by a later QElectroTech that lets this one read it: saving + //here would drop what this version does not know + m_from_later_version = container_format > QetContainer::Format; + if (!fi.isWritable() || m_from_later_version) { setReadOnly(true); } + if (m_from_later_version) { + qInfo().noquote() << QStringLiteral("Project %1 is of .qetz format %2 (this version writes %3): opened read-only") + .arg(fi.fileName()).arg(container_format).arg(QetContainer::Format); + } if(opened_here) { file->close(); } diff --git a/sources/qetproject.h b/sources/qetproject.h index 4ae7281ab..e5d0c2220 100644 --- a/sources/qetproject.h +++ b/sources/qetproject.h @@ -319,6 +319,8 @@ class QETProject : public QObject bool close(); QETResult write(); bool isReadOnly() const; + /// Opened from a .qetz of a later format (opened read-only) + bool isFromLaterVersion() const {return m_from_later_version;} void setReadOnly(bool); bool isEmpty() const; ElementsLocation importElement(ElementsLocation &location); @@ -466,6 +468,7 @@ class QETProject : public QObject bool m_modified = false; /// Whether the project is read only bool m_read_only = false; + bool m_from_later_version = false; /// Filepath for which this project is considered read only QString read_only_file_path_; /// Default dimensions and properties for new diagrams created within the project diff --git a/tests/qttest/CMakeLists.txt b/tests/qttest/CMakeLists.txt index 3a7c68b0f..d0cb7e286 100644 --- a/tests/qttest/CMakeLists.txt +++ b/tests/qttest/CMakeLists.txt @@ -1262,4 +1262,6 @@ add_test(NAME tst_qetcontainer COMMAND tst_qetcontainer) target_include_directories(tst_qetcontainer PRIVATE ${QET_DIR}/sources) target_link_libraries(tst_qetcontainer PRIVATE Qt::Test Qt::Xml SQLite3::SQLite3) target_compile_definitions(tst_qetcontainer PRIVATE - "QET_EXAMPLES_DIR=\"${QET_DIR}/examples\"") + "QET_EXAMPLES_DIR=\"${QET_DIR}/examples\"" + "QET_FIXTURES_DIR=\"${CMAKE_CURRENT_SOURCE_DIR}/fixtures\"" + "QET_FORMAT_SPEC=\"${QET_DIR}/sources/container/QETZ-FORMAT.md\"") diff --git a/tests/qttest/fixtures/qetz_format2.qetz b/tests/qttest/fixtures/qetz_format2.qetz new file mode 100644 index 000000000..fdf8d066b Binary files /dev/null and b/tests/qttest/fixtures/qetz_format2.qetz differ diff --git a/tests/qttest/tst_qetcontainer.cpp b/tests/qttest/tst_qetcontainer.cpp index 42e7b0c8f..fa981f537 100644 --- a/tests/qttest/tst_qetcontainer.cpp +++ b/tests/qttest/tst_qetcontainer.cpp @@ -69,6 +69,57 @@ class tst_qetcontainer : public QObject return QetContainer::split(*original, QStringLiteral("tst_qetcontainer"), &error); } + static QByteArray entry(const QList &entries, const QString &name) + { + for (const QetZip::Entry &e : entries) + if (e.name == name) return e.data; + return {}; + } + + static QByteArray folioFiles(const QList &entries) + { + QByteArray all; + for (const QetZip::Entry &e : entries) + if (e.name.startsWith(QStringLiteral("folios/"))) all += e.data; + return all; + } + + // The first column of the first row @p sql gives, from @p database + static QString query(const QByteArray &database, const char *sql) + { + sqlite3 *db = nullptr; + sqlite3_open(":memory:", &db); + unsigned char *copy = static_cast(sqlite3_malloc64(sqlite3_uint64(database.size()))); + memcpy(copy, database.constData(), size_t(database.size())); + sqlite3_deserialize(db, "main", copy, database.size(), database.size(), SQLITE_DESERIALIZE_FREEONCLOSE); + sqlite3_stmt *st = nullptr; + QString out; + if (sqlite3_prepare_v2(db, sql, -1, &st, nullptr) == SQLITE_OK && sqlite3_step(st) == SQLITE_ROW) + out = QString::fromUtf8(reinterpret_cast(sqlite3_column_text(st, 0))); + sqlite3_finalize(st); + sqlite3_close(db); + return out; + } + + // grafcet.qet as QElectroTech saves it: every folio, symbol and wire + // with a uuid of its own, which the examples' wires and folios lack + static QDomDocument savedProject() + { + QList entries; + QDomDocument project; + QetZip::fromBytes(read(QStringLiteral(QET_FIXTURES_DIR "/qetz_format2.qetz")), &entries); + QetContainer::join(entries, &project); + return project; + } + + static QString backDifference(const QDomDocument &original, const QList &entries) + { + QDomDocument back; + QString error; + if (!QetContainer::join(entries, &back, &error)) return error; + return difference(original.documentElement(), back.documentElement(), QString()); + } + private slots: void sameDocumentBack_data() { @@ -280,6 +331,107 @@ private slots: } // A file whose manifest needs a later reader is refused, not half read + // A later format whose min-reader allows this one opens, and join() says + // it is later: QElectroTech opens it read-only + void laterFormatAllowedRead() + { + QDomDocument original; + QList entries = splitFile(QStringLiteral(QET_EXAMPLES_DIR "/industrial.qet"), &original); + QVERIFY(!entries.isEmpty()); + const int later = QetContainer::Format + 1; + for (QetZip::Entry &e : entries) + if (e.name == QLatin1String("manifest.xml")) + e.data = QString::fromUtf8(e.data).replace(QRegularExpression(QStringLiteral(" format=\"\\d+\"")), + QStringLiteral(" format=\"%1\"").arg(later)).toUtf8(); + alterDatabase(entries, QStringLiteral("UPDATE meta SET value = '%1' WHERE key = 'format'") + .arg(later).toUtf8().constData()); + QDomDocument back; + QString error; + int format = 0; + QVERIFY2(QetContainer::join(entries, &back, &error, nullptr, &format), qPrintable(error)); + QCOMPARE(format, later); + const QString d = difference(original.documentElement(), back.documentElement(), QString()); + QVERIFY2(d.isEmpty(), qPrintable(d)); + } + + // The database holds data, not where a block sat in the XML: the folio + // file's marker says where it goes back + void noPositionColumns() + { + QDomDocument original; + const QList entries = splitFile(QStringLiteral(QET_EXAMPLES_DIR "/industrial.qet"), &original); + const QByteArray database = entry(entries, QStringLiteral("project.sqlite")); + QCOMPARE(query(database, "SELECT group_concat(name) FROM pragma_table_info('element')"), + QStringLiteral("uuid,folio_uuid,prefix,freezeLabel,info_attrs,links_attrs")); + QCOMPARE(query(database, "SELECT count(*) FROM pragma_table_info('folio') WHERE name LIKE '%_at'"), + QStringLiteral("0")); + QCOMPARE(query(database, "SELECT count(*) FROM meta WHERE key = 'props_at'"), QStringLiteral("0")); + } + + // A project as QElectroTech saves it needs no JSON: every attribute it + // writes has a column, the show flag of a property too + void savedProjectNeedsNoJson() + { + QString error; + const QList entries = QetContainer::split(savedProject(), QString(), &error); + QVERIFY2(!entries.isEmpty(), qPrintable(error)); + const QByteArray database = entry(entries, QStringLiteral("project.sqlite")); + QVERIFY(query(database, "SELECT count(*) FROM project_property").toInt() > 0); + QCOMPARE(query(database, "SELECT count(*) FROM project_property WHERE show IS NOT NULL"), + query(database, "SELECT count(*) FROM project_property")); + for (const char *sql : {"SELECT count(*) FROM project_property WHERE attrs IS NOT NULL", + "SELECT count(*) FROM folio_property WHERE attrs IS NOT NULL", + "SELECT count(*) FROM element_info WHERE attrs IS NOT NULL", + "SELECT count(*) FROM link WHERE attrs IS NOT NULL", + "SELECT count(*) FROM folio WHERE props_attrs IS NOT NULL", + "SELECT count(*) FROM element WHERE info_attrs IS NOT NULL OR links_attrs IS NOT NULL"}) + QVERIFY2(query(database, sql) == QLatin1String("0"), sql); + } + + // The schema in QETZ-FORMAT.md is the one written: same tables, same + // columns in the same order, same types and keys + void specMatchesSchema() + { + const QString spec = QString::fromUtf8(read(QStringLiteral(QET_FORMAT_SPEC))); + const int from = spec.indexOf(QStringLiteral("```sql\n")); + const int to = spec.indexOf(QStringLiteral("```"), from + 7); + QVERIFY2(from >= 0 && to > from, "no schema in QETZ-FORMAT.md"); + const QByteArray sql = spec.mid(from + 7, to - from - 7).toUtf8(); + + QDomDocument original; + const QList entries = splitFile(QStringLiteral(QET_EXAMPLES_DIR "/industrial.qet"), &original); + const QByteArray written = entry(entries, QStringLiteral("project.sqlite")); + + auto describe = [](sqlite3 *db) { + QStringList out; + sqlite3_stmt *st = nullptr; + sqlite3_prepare_v2(db, "SELECT m.name, p.name, p.type, p.pk, p.\"notnull\" FROM sqlite_master m, " + "pragma_table_info(m.name) p WHERE m.type = 'table' ORDER BY m.name, p.cid", + -1, &st, nullptr); + while (st && sqlite3_step(st) == SQLITE_ROW) { + QStringList row; + for (int i = 0 ; i < 5 ; ++i) + row << QString::fromUtf8(reinterpret_cast(sqlite3_column_text(st, i))); + out << row.join(QLatin1Char(' ')); + } + sqlite3_finalize(st); + return out; + }; + sqlite3 *from_spec = nullptr; + sqlite3_open(":memory:", &from_spec); + QCOMPARE(sqlite3_exec(from_spec, sql.constData(), nullptr, nullptr, nullptr), SQLITE_OK); + sqlite3 *from_file = nullptr; + sqlite3_open(":memory:", &from_file); + unsigned char *copy = static_cast(sqlite3_malloc64(sqlite3_uint64(written.size()))); + memcpy(copy, written.constData(), size_t(written.size())); + sqlite3_deserialize(from_file, "main", copy, written.size(), written.size(), SQLITE_DESERIALIZE_FREEONCLOSE); + const QStringList spec_tables = describe(from_spec), file_tables = describe(from_file); + sqlite3_close(from_spec); + sqlite3_close(from_file); + QVERIFY(!spec_tables.isEmpty()); + QCOMPARE(spec_tables, file_tables); + } + void laterFormatRefused() { QDomDocument original; @@ -309,6 +461,86 @@ private slots: // A crafted project.sqlite whose "element" is a view, not a table, is // refused: nothing in a file's schema is trusted. + // project.sqlite changed by @p sql in @p entries + static void alterDatabase(QList &entries, const char *sql) + { + for (QetZip::Entry &e : entries) { + if (e.name != QLatin1String("project.sqlite")) continue; + sqlite3 *db = nullptr; + sqlite3_open(":memory:", &db); + unsigned char *copy = static_cast(sqlite3_malloc64(sqlite3_uint64(e.data.size()))); + memcpy(copy, e.data.constData(), size_t(e.data.size())); + sqlite3_deserialize(db, "main", copy, e.data.size(), e.data.size(), + SQLITE_DESERIALIZE_FREEONCLOSE | SQLITE_DESERIALIZE_RESIZEABLE); + QCOMPARE(sqlite3_exec(db, sql, nullptr, nullptr, nullptr), SQLITE_OK); + sqlite3_int64 size = 0; + unsigned char *bytes = sqlite3_serialize(db, "main", &size, 0); + e.data = QByteArray(reinterpret_cast(bytes), qsizetype(size)); + sqlite3_free(bytes); + sqlite3_close(db); + } + } + + // A database that does not match its folio files -- a row no item has, + // an item's row on another folio, entries of no symbol -- is refused + // with what is wrong, not read as far as it goes + void mismatchRefused_data() + { + QTest::addColumn("sql"); + QTest::addColumn("said"); + QTest::newRow("symbol row of no symbol") + << QByteArray("INSERT INTO element SELECT 'x', folio_uuid, prefix, freezeLabel, info_attrs," + " links_attrs FROM element LIMIT 1") + << QStringLiteral("symbol rows"); + QTest::newRow("wire row of no wire") + << QByteArray("INSERT INTO conductor SELECT 'x', folio_uuid, element1, terminal1, element2, terminal2," + " num, formula, function, tension_protocol, conductor_color, conductor_section, cable," + " bus, freezeLabel FROM conductor LIMIT 1") + << QStringLiteral("wire rows"); + QTest::newRow("symbol on another folio") + << QByteArray("UPDATE element SET folio_uuid = (SELECT uuid FROM folio WHERE pos = 2)" + " WHERE folio_uuid = (SELECT uuid FROM folio WHERE pos = 1)") + << QStringLiteral("the database says another"); + QTest::newRow("information of no symbol") + << QByteArray("UPDATE element_info SET element_uuid = 'x' WHERE rowid = 1") + << QStringLiteral("information rows of no symbol"); + } + void mismatchRefused() + { + QFETCH(QByteArray, sql); + QFETCH(QString, said); + QDomDocument original; + QList entries = splitFile(QStringLiteral(QET_EXAMPLES_DIR "/industrial.qet"), &original); + QVERIFY(!entries.isEmpty()); + { + QDomDocument back; + QString error; + QVERIFY2(QetContainer::join(entries, &back, &error), qPrintable(error)); + } + alterDatabase(entries, sql.constData()); + QDomDocument back; + QString error; + QVERIFY(!QetContainer::join(entries, &back, &error)); + QVERIFY2(error.contains(said), qPrintable(error)); + } + + // A folio file says which symbol each item is (its type), read alone + void folioSaysWhichSymbol() + { + QDomDocument original; + const QList entries = splitFile(QStringLiteral(QET_EXAMPLES_DIR "/industrial.qet"), &original); + QVERIFY(!entries.isEmpty()); + int typed = 0, symbols = 0; + for (const QetZip::Entry &e : entries) { + if (!e.name.startsWith(QStringLiteral("folios/"))) continue; + const QString text = QString::fromUtf8(e.data); + symbols += int(text.count(QStringLiteral("]*type=\"embed://")))); + } + QVERIFY(symbols > 0); + QCOMPARE(typed, symbols); + } + void viewInsteadOfTableRefused() { QDomDocument original; @@ -337,6 +569,104 @@ private slots: QVERIFY2(error.contains(QStringLiteral("element")), qPrintable(error)); } + // A row's key is its item's uuid, which the folio file keeps: no key + // made up from where an item is (format 2 had f1, f2...), no key + // attribute in the folio files of a project as QElectroTech saves it, + // true/false held as 1/0, and the format in the database's + // user_version. + void keysAreTheUuids() + { + const QDomDocument original = savedProject(); + original.elementsByTagName(QStringLiteral("conductor")).at(0).toElement() + .setAttribute(QStringLiteral("freezeLabel"), QStringLiteral("true")); + QString error; + const QList entries = QetContainer::split(original, QString(), &error); + QVERIFY2(!entries.isEmpty(), qPrintable(error)); + const QByteArray database = entry(entries, QStringLiteral("project.sqlite")); + const QByteArray folios = folioFiles(entries); + QVERIFY(!folios.contains("qetz-key")); + QVERIFY(!folios.contains("qetz-folio")); + QCOMPARE(query(database, "PRAGMA user_version"), QString::number(QetContainer::Format)); + const QString first = original.elementsByTagName(QStringLiteral("diagram")).at(0).toElement() + .attribute(QStringLiteral("uuid")); + QCOMPARE(query(database, "SELECT uuid FROM folio WHERE pos = 1"), first); + QCOMPARE(query(database, "SELECT count(*) FROM folio WHERE uuid NOT LIKE '{%}'"), QStringLiteral("0")); + QCOMPARE(query(database, "SELECT count(*) FROM element WHERE uuid NOT LIKE '{%}'"), QStringLiteral("0")); + QCOMPARE(query(database, "SELECT count(*) FROM conductor WHERE uuid NOT LIKE '{%}'"), QStringLiteral("0")); + QCOMPARE(query(database, "SELECT count(*) FROM element WHERE typeof(freezeLabel) <> 'integer'"), + QStringLiteral("0")); + QCOMPARE(query(database, "SELECT count(*) FROM conductor WHERE typeof(freezeLabel) <> 'integer'"), + QStringLiteral("0")); + QCOMPARE(query(database, "SELECT sum(freezeLabel) FROM conductor"), QStringLiteral("1")); + const QString d = backDifference(original, entries); + QVERIFY2(d.isEmpty(), qPrintable(d)); + QCOMPARE(query(database, "SELECT count(*) FROM element_info WHERE typeof(show) <> 'integer'"), + QStringLiteral("0")); + QCOMPARE(query(database, "SELECT count(*) FROM element_info WHERE attrs LIKE '%\"tag\"%'"), + QStringLiteral("0")); + } + + // Two symbols sharing a uuid (F100) each get a key made from what they + // hold, the same on every save, and come back as they were. + void sharedUuidKeyed() + { + QDomDocument original = savedProject(); + const QDomNodeList elements = original.elementsByTagName(QStringLiteral("element")); + QDomElement a, b; + for (int i = 0 ; i < elements.size() && b.isNull() ; ++i) { + const QDomElement e = elements.at(i).toElement(); + if (e.parentNode().nodeName() != QLatin1String("elements")) continue; + if (a.isNull()) a = e; else b = e; + } + QVERIFY(!b.isNull()); + b.setAttribute(QStringLiteral("uuid"), a.attribute(QStringLiteral("uuid"))); + QString error; + const QList once = QetContainer::split(original, QString(), &error); + const QList twice = QetContainer::split(original, QString(), &error); + QVERIFY2(!once.isEmpty(), qPrintable(error)); + QCOMPARE(folioFiles(once).count("qetz-key"), 2); + QCOMPARE(entry(once, QStringLiteral("project.sqlite")), entry(twice, QStringLiteral("project.sqlite"))); + const QString d = backDifference(original, once); + QVERIFY2(d.isEmpty(), qPrintable(d)); + } + + // A true/false attribute QElectroTech would not write stays in the + // folio file as it was. + void oddBooleanKept() + { + QDomDocument original; + QVERIFY(QetContainer::parse(read(QStringLiteral(QET_EXAMPLES_DIR "/perceuse.qet")), &original)); + const QDomNodeList conductors = original.elementsByTagName(QStringLiteral("conductor")); + QVERIFY(conductors.size() > 0); + conductors.at(0).toElement().setAttribute(QStringLiteral("freezeLabel"), QStringLiteral("yes")); + QString error; + const QList entries = QetContainer::split(original, QString(), &error); + QVERIFY2(!entries.isEmpty(), qPrintable(error)); + QVERIFY(folioFiles(entries).contains("freezeLabel=\"yes\"")); + const QString d = backDifference(original, entries); + QVERIFY2(d.isEmpty(), qPrintable(d)); + } + + // A format 2 file, as QElectroTech 0.200.1 development builds wrote it + // (grafcet.qet saved as .qetz), still opens, and saved again as the + // current format comes back the same. + void format2Read() + { + QList entries; + QString error; + QVERIFY2(QetZip::fromBytes(read(QStringLiteral(QET_FIXTURES_DIR "/qetz_format2.qetz")), &entries, &error), + qPrintable(error)); + QVERIFY(entry(entries, QStringLiteral("manifest.xml")).contains("format=\"2\"")); + QDomDocument project; + QVERIFY2(QetContainer::join(entries, &project, &error), qPrintable(error)); + QVERIFY(project.toString().contains(QStringLiteral(" resaved = QetContainer::split(project, QString(), &error); + QVERIFY2(!resaved.isEmpty(), qPrintable(error)); + const QString d = backDifference(project, resaved); + QVERIFY2(d.isEmpty(), qPrintable(d)); + } + void notAProjectRefused() { QDomDocument back;