diff --git a/sources/container/QETZ-FORMAT.md b/sources/container/QETZ-FORMAT.md new file mode 100644 index 000000000..d7b2f0234 --- /dev/null +++ b/sources/container/QETZ-FORMAT.md @@ -0,0 +1,182 @@ +# The `.qetz` project file, format 3 + +A `.qetz` is a QElectroTech project saved as a zip archive: the same project +a `.qet` holds, split into files, with the engineering data in an SQLite +database. This document describes format 3. `tst_qetcontainer` checks that +the schema below is the one QElectroTech writes. + +The code is `qetcontainer.cpp` (the zip and its files), `qetcontainerdb.cpp` +(the database) and `qetzip.cpp` (reading and writing zip archives). + +## 1. Principles + +- **One writer.** A `.qet` and a `.qetz` are written from the same project + document. Saving a `.qetz` splits that document into files and a database; + opening one puts it back together, and the program reads it exactly as a + `.qet`. Opening a `.qetz` and saving it as `.qet` gives the file a plain + `.qet` save gives. +- **Each fact once.** A value moved into the database is taken out of the + folio file. Nothing is stored in both. +- **What a thing is, against where it is drawn.** The database holds the + engineering data; the folio files hold the drawing: + + | | database | folio file | + |---|---|---| + | Symbol | its information (label, comment, …), prefix, links | which symbol it is (`type`), position, rotation | + | Label shown on a folio | the value | the text item: position, font, rotation | + | Wire | its ends, number, function, voltage, colour, section | its path, the number's text position and font | + | Folio | title-block fields and custom variables | everything drawn on it | + | Terminal strip | its terminals, their order and bridges | the strip drawn on a folio | + + Cables are kept in `project.xml` in this format. +- **Nothing is lost.** Anything the database does not model stays in the + XML as it was. A block is moved only if it can be rebuilt exactly. + +## 2. The archive + +| Entry | Content | +|---|---| +| `mimetype` | `application/x-qelectrotech-project+zip`, the first entry, stored uncompressed | +| `manifest.xml` | `` | +| `project.xml` | the project document, each part replaced by `` | +| `project.sqlite` | the engineering data (section 4) | +| `folios/.xml` | one `` per folio, named by the folio's uuid (or its position when it has none) | +| `elements//` | the project's symbol definitions, as in the symbol collection | +| `titleblocks/.titleblock` | the project's title-block templates | +| `images/.png` | each folio picture once, as a file instead of base64 text (`.bin` if it is not a PNG) | + +Names that would collide get `~2`, `~3`… before the extension. A picture +in a folio file keeps the name of its file in the attribute `qetz-file`. + +A reader refuses the whole archive if an entry is encrypted, uses a +compression method it cannot read, has a name that is absolute, holds `..`, +`\` or `:`, or repeats another entry's name, or if the archive holds more +than 100 000 entries or more than 2 GiB once unpacked. Entries the format +does not name are ignored. + +## 3. Versions + +`manifest.xml` gives two numbers: `format`, the format written, and +`min-reader`, the oldest format allowed to read the file. + +- A reader opens a file whose `min-reader` is not above its own format. +- If `format` is above its own, the project opens **read-only**, with a + message: saving it could drop what that reader does not know. *Save As* + writes a copy in the reader's own format. +- If `min-reader` is above its own format, the file is refused with a + message. + +The database repeats the format in `meta` (key `format`) and in +`PRAGMA user_version`. + +**Rules for a new format:** + +1. New data goes in a new column or table. An older reader skips both. +2. A change an older reader would misread raises `min-reader`. +3. Every format keeps a saved file in `tests/qttest/fixtures` that must + keep opening. + +| Format | Written by | Reading | +|---|---|---| +| 2 | QElectroTech 0.200.1 development builds (#1462) | read: rows keyed by position (`f1`, `e1`, `c1`) where an item had no uuid of its own, every value as text | +| 3 | this document | read and written | + +## 4. The database + +### 4.1 Schema + + +```sql +CREATE TABLE meta (key TEXT PRIMARY KEY, value TEXT); +CREATE TABLE project_property (ord INTEGER PRIMARY KEY, name TEXT, value TEXT, show INTEGER, attrs TEXT); +CREATE TABLE folio (uuid TEXT PRIMARY KEY, pos INTEGER NOT NULL UNIQUE, + "title" TEXT, "author" TEXT, "date" TEXT, "folio" TEXT, "indexrev" TEXT, "filename" TEXT, + "locmach" TEXT, "plant" TEXT, "auto_page_num" TEXT, props_attrs TEXT); +CREATE TABLE folio_property (folio_uuid TEXT NOT NULL REFERENCES folio, ord INTEGER NOT NULL, + name TEXT, value TEXT, show INTEGER, attrs TEXT, PRIMARY KEY (folio_uuid, ord)); +CREATE TABLE element (uuid TEXT PRIMARY KEY, folio_uuid TEXT NOT NULL REFERENCES folio, + "prefix" TEXT, "freezeLabel" INTEGER, info_attrs TEXT, links_attrs TEXT); +CREATE TABLE element_info (element_uuid TEXT NOT NULL REFERENCES element, ord INTEGER NOT NULL, + name TEXT, value TEXT, show INTEGER, attrs TEXT, PRIMARY KEY (element_uuid, ord)); +CREATE TABLE link (element_uuid TEXT NOT NULL REFERENCES element, ord INTEGER NOT NULL, + linked_uuid TEXT, attrs TEXT, PRIMARY KEY (element_uuid, ord)); +CREATE TABLE conductor (uuid TEXT PRIMARY KEY, folio_uuid TEXT NOT NULL REFERENCES folio, + "element1" TEXT, "terminal1" TEXT, "element2" TEXT, "terminal2" TEXT, "num" TEXT, + "formula" TEXT, "function" TEXT, "tension_protocol" TEXT, "conductor_color" TEXT, + "conductor_section" TEXT, "cable" TEXT, "bus" TEXT, "freezeLabel" INTEGER); +CREATE TABLE terminal_strip (uuid TEXT PRIMARY KEY, ord INTEGER NOT NULL UNIQUE, + installation TEXT, location TEXT, name TEXT, comment TEXT, description TEXT); +CREATE TABLE strip_terminal (strip_uuid TEXT NOT NULL REFERENCES terminal_strip, + pos INTEGER NOT NULL, level INTEGER NOT NULL, element_uuid TEXT, + PRIMARY KEY (strip_uuid, pos, level)); +CREATE TABLE strip_bridge (uuid TEXT PRIMARY KEY, strip_uuid TEXT NOT NULL REFERENCES terminal_strip, + ord INTEGER NOT NULL, color TEXT); +CREATE TABLE strip_bridge_terminal (bridge_uuid TEXT NOT NULL REFERENCES strip_bridge, + ord INTEGER NOT NULL, element_uuid TEXT, PRIMARY KEY (bridge_uuid, ord)); +``` + +### 4.2 What each table holds + +| Table | One row per | Comes from | +|---|---|---| +| `meta` | setting of the file | `format`; `props_attrs` (attributes of the project's ``); `terminal_strips` = `1` when the strips are in the database | +| `project_property` | project property (project-wide variable) | `` | +| `folio` | folio, in folio order (`pos` from 1) | the title-block attributes of `` | +| `folio_property` | custom title-block variable of a folio | `` | +| `element` | symbol placed on a folio | ``: its label prefix and frozen-label flag | +| `element_info` | piece of symbol information, in order | `` | +| `link` | cross-reference link of a symbol, in order | `` | +| `conductor` | wire | ``: its two ends (symbol uuid and terminal), number, formula, function, voltage/protocol, colour, section, cable, bus, frozen-label flag | +| `terminal_strip`, `strip_terminal`, `strip_bridge`, `strip_bridge_terminal` | terminal strip, terminal, bridge, bridged terminal | `` | + +A value is NULL when the XML has no such attribute. + +- `freezeLabel` is 1 for `true`, 0 for `false`. Any other text stays as an + attribute in the folio file. +- `show` is 0 or 1. Any other text goes in `attrs`. +- `attrs`, `props_attrs`, `info_attrs` and `links_attrs` are JSON objects of + the attributes that have no column, and the entry's tag when it is not the + usual one. In a project QElectroTech saves they are all NULL. They hold + something only for older or hand-edited files. Whitespace between + entries, which only a hand-edited file has, is kept in them as + `qetz-ws`. + +### 4.3 Keys + +Every row is keyed by its item's **uuid**, which stays in the folio file: +that is how a folio file's item finds its row. Folios, symbols and wires +are keyed this way. + +An item with no uuid of its own (none, or one another item of the same kind +shares) gets a key made from the item itself: a version 5 uuid in the +namespace `{3f0c6c1e-5b8e-4b8a-9d55-6b2f5f0f9a41}` from its XML text, how +many identical items came before it, and a counter that avoids taken keys. +The key is the same on every save and never depends on where the item is. +It is written on the item as `qetz-key` (a symbol or wire) or `qetz-folio` +(a folio). + +### 4.4 Markers + +A block moved into the database leaves `` where it was: +`properties`, `elementInformations`, `links_uuids` or `terminal_strips`. +Reading puts the block back where its marker is; a block without a marker +was never moved. + +Only a block that can be rebuilt exactly is moved: entries of the expected +tag with plain text values. Terminal strips are moved only if their rows +rebuild the same XML. Anything else stays in the folio file or +`project.xml` unchanged. + +## 5. Reading a file safely + +A `.qetz` is untrusted. A reader: + +- runs nothing from the database's schema (`PRAGMA trusted_schema = OFF`, + read-only) and refuses a database where any table of section 4.1 is not a + plain table (a view, say); +- refuses the file, with a message saying what is wrong, if the database + and the folio files disagree: an item whose row is missing or on another + folio, an item twice, a row no item has, or information, links or + properties of a symbol or folio that does not exist. + +A damaged archive is caught by the zip format's own checksums. diff --git a/sources/container/qetcontainer.h b/sources/container/qetcontainer.h index 72184045d..1fc642f3c 100644 --- a/sources/container/qetcontainer.h +++ b/sources/container/qetcontainer.h @@ -41,6 +41,8 @@ join(split(d)) is d. Nothing here knows what the parts mean: the project is read and written by the same code as a .qet file. + + The format is written down in QETZ-FORMAT.md, beside this file. */ namespace QetContainer { diff --git a/tests/qttest/CMakeLists.txt b/tests/qttest/CMakeLists.txt index 4c3bda623..7f42b2b45 100644 --- a/tests/qttest/CMakeLists.txt +++ b/tests/qttest/CMakeLists.txt @@ -1254,4 +1254,5 @@ target_include_directories(tst_qetcontainer PRIVATE ${QET_DIR}/sources) target_link_libraries(tst_qetcontainer PRIVATE Qt::Test Qt::Xml SQLite3::SQLite3) target_compile_definitions(tst_qetcontainer PRIVATE "QET_EXAMPLES_DIR=\"${QET_DIR}/examples\"" - "QET_FIXTURES_DIR=\"${CMAKE_CURRENT_SOURCE_DIR}/fixtures\"") + "QET_FIXTURES_DIR=\"${CMAKE_CURRENT_SOURCE_DIR}/fixtures\"" + "QET_FORMAT_SPEC=\"${QET_DIR}/sources/container/QETZ-FORMAT.md\"") diff --git a/tests/qttest/tst_qetcontainer.cpp b/tests/qttest/tst_qetcontainer.cpp index 82f0e8020..7fdf4af9b 100644 --- a/tests/qttest/tst_qetcontainer.cpp +++ b/tests/qttest/tst_qetcontainer.cpp @@ -351,6 +351,50 @@ private slots: QVERIFY2(query(database, sql) == QLatin1String("0"), sql); } + // The schema in QETZ-FORMAT.md is the one written: same tables, same + // columns in the same order, same types and keys + void specMatchesSchema() + { + const QString spec = QString::fromUtf8(read(QStringLiteral(QET_FORMAT_SPEC))); + const int from = spec.indexOf(QStringLiteral("```sql\n")); + const int to = spec.indexOf(QStringLiteral("```"), from + 7); + QVERIFY2(from >= 0 && to > from, "no schema in QETZ-FORMAT.md"); + const QByteArray sql = spec.mid(from + 7, to - from - 7).toUtf8(); + + QDomDocument original; + const QList entries = splitFile(QStringLiteral(QET_EXAMPLES_DIR "/industrial.qet"), &original); + const QByteArray written = entry(entries, QStringLiteral("project.sqlite")); + + auto describe = [](sqlite3 *db) { + QStringList out; + sqlite3_stmt *st = nullptr; + sqlite3_prepare_v2(db, "SELECT m.name, p.name, p.type, p.pk, p.\"notnull\" FROM sqlite_master m, " + "pragma_table_info(m.name) p WHERE m.type = 'table' ORDER BY m.name, p.cid", + -1, &st, nullptr); + while (st && sqlite3_step(st) == SQLITE_ROW) { + QStringList row; + for (int i = 0 ; i < 5 ; ++i) + row << QString::fromUtf8(reinterpret_cast(sqlite3_column_text(st, i))); + out << row.join(QLatin1Char(' ')); + } + sqlite3_finalize(st); + return out; + }; + sqlite3 *from_spec = nullptr; + sqlite3_open(":memory:", &from_spec); + QCOMPARE(sqlite3_exec(from_spec, sql.constData(), nullptr, nullptr, nullptr), SQLITE_OK); + sqlite3 *from_file = nullptr; + sqlite3_open(":memory:", &from_file); + unsigned char *copy = static_cast(sqlite3_malloc64(sqlite3_uint64(written.size()))); + memcpy(copy, written.constData(), size_t(written.size())); + sqlite3_deserialize(from_file, "main", copy, written.size(), written.size(), SQLITE_DESERIALIZE_FREEONCLOSE); + const QStringList spec_tables = describe(from_spec), file_tables = describe(from_file); + sqlite3_close(from_spec); + sqlite3_close(from_file); + QVERIFY(!spec_tables.isEmpty()); + QCOMPARE(spec_tables, file_tables); + } + void laterFormatRefused() { QDomDocument original;