// SPDX-License-Identifier: GPL-2.0-or-later #include #include #include #include #include "container/qetcontainer.h" #include // A project document split into the parts of a .qetz, zipped, unzipped and // joined back is the document it was: every example, and every .qet in the // folder QET_CONTAINER_CORPUS names, if set -- exactly, whitespace text // included: QETProject writes the symbol definitions back as it read them, // spacing and all, so a part that came back re-indented would change the // next save. class tst_qetcontainer : public QObject { Q_OBJECT static QList children(const QDomNode &node) { QList all; for (QDomNode c = node.firstChild() ; !c.isNull() ; c = c.nextSibling()) all << c; return all; } // empty when equal, else where they differ static QString difference(const QDomNode &a, const QDomNode &b, const QString &path) { const QString here = path + QLatin1Char('/') + a.nodeName(); if (a.nodeType() != b.nodeType() || a.nodeName() != b.nodeName()) return here + QStringLiteral(": %1 vs %2").arg(a.nodeName(), b.nodeName()); if (!a.isElement() && !a.isDocument() && a.nodeValue() != b.nodeValue()) return here + QStringLiteral(": value differs"); if (a.isElement()) { const QDomNamedNodeMap aa = a.attributes(), ba = b.attributes(); if (aa.count() != ba.count()) return here + QStringLiteral(": %1 vs %2 attributes").arg(aa.count()).arg(ba.count()); for (int i = 0 ; i < aa.count() ; ++i) { const QDomAttr attr = aa.item(i).toAttr(); if (!ba.contains(attr.name()) || ba.namedItem(attr.name()).nodeValue() != attr.value()) return here + QStringLiteral(": attribute %1 differs").arg(attr.name()); } } const QList ac = children(a), bc = children(b); if (ac.size() != bc.size()) return here + QStringLiteral(": %1 vs %2 children").arg(ac.size()).arg(bc.size()); for (int i = 0 ; i < ac.size() ; ++i) { const QString d = difference(ac.at(i), bc.at(i), here + QStringLiteral("[%1]").arg(i)); if (!d.isEmpty()) return d; } return {}; } static QByteArray read(const QString &path) { QFile f(path); return f.open(QIODevice::ReadOnly) ? f.readAll() : QByteArray(); } static QList splitFile(const QString &path, QDomDocument *original) { QString error; if (!QetContainer::parse(read(path), original, &error)) return {}; return QetContainer::split(*original, QStringLiteral("tst_qetcontainer"), &error); } //@p document without what QElectroTech computes, which a .qetz //does not keep (format 3); tst_resaveunchanged checks that //QElectroTech gives it back static QDomDocument withoutComputed(const QDomDocument &document) { QDomDocument copy = document.cloneNode(true).toDocument(); for (QDomElement diagram = copy.documentElement().firstChildElement(QStringLiteral("diagram")); !diagram.isNull() ; diagram = diagram.nextSiblingElement(QStringLiteral("diagram"))) QetContainer::dropComputed(diagram, false); return copy; } static QByteArray entry(const QList &entries, const QString &name) { for (const QetZip::Entry &e : entries) if (e.name == name) return e.data; return {}; } static QByteArray folioFiles(const QList &entries) { QByteArray all; for (const QetZip::Entry &e : entries) if (e.name.startsWith(QStringLiteral("folios/"))) all += e.data; return all; } // The first column of the first row @p sql gives, from @p database static QString query(const QByteArray &database, const char *sql) { sqlite3 *db = nullptr; sqlite3_open(":memory:", &db); unsigned char *copy = static_cast(sqlite3_malloc64(sqlite3_uint64(database.size()))); memcpy(copy, database.constData(), size_t(database.size())); sqlite3_deserialize(db, "main", copy, database.size(), database.size(), SQLITE_DESERIALIZE_FREEONCLOSE); sqlite3_stmt *st = nullptr; QString out; if (sqlite3_prepare_v2(db, sql, -1, &st, nullptr) == SQLITE_OK && sqlite3_step(st) == SQLITE_ROW) out = QString::fromUtf8(reinterpret_cast(sqlite3_column_text(st, 0))); sqlite3_finalize(st); sqlite3_close(db); return out; } // grafcet.qet as QElectroTech saves it: every folio, symbol and wire // with a uuid of its own, which the examples' wires and folios lack static QDomDocument savedProject() { QList entries; QDomDocument project; QetZip::fromBytes(read(QStringLiteral(QET_FIXTURES_DIR "/qetz_format2.qetz")), &entries); QetContainer::join(entries, &project); return project; } static QString backDifference(const QDomDocument &original, const QList &entries) { QDomDocument back; QString error; if (!QetContainer::join(entries, &back, &error)) return error; return difference(withoutComputed(original).documentElement(), back.documentElement(), QString()); } private slots: void sameDocumentBack_data() { QTest::addColumn("file"); QStringList dirs{QStringLiteral(QET_EXAMPLES_DIR)}; const QString corpus = qEnvironmentVariable("QET_CONTAINER_CORPUS"); if (!corpus.isEmpty()) dirs << corpus; for (const QString &dir : dirs) { QDirIterator it(dir, {QStringLiteral("*.qet")}, QDir::Files, QDirIterator::Subdirectories); QStringList files; while (it.hasNext()) files << it.next(); files.sort(); for (const QString &f : files) QTest::newRow(qPrintable(QDir(dir).relativeFilePath(f))) << f; } } void sameDocumentBack() { QFETCH(QString, file); QDomDocument original; QString error; if (!QetContainer::parse(read(file), &original, &error)) QSKIP(qPrintable(QStringLiteral("not readable as XML, as QElectroTech finds too: ") + error)); const QList entries = QetContainer::split(original, QString(), &error); QVERIFY2(!entries.isEmpty(), qPrintable(error)); QCOMPARE(entries.first().name, QStringLiteral("mimetype")); QList unzipped; QVERIFY2(QetZip::fromBytes(QetZip::toBytes(entries, &error), &unzipped, &error), qPrintable(error)); QDomDocument back; QVERIFY2(QetContainer::join(unzipped, &back, &error), qPrintable(error)); const QString d = difference(withoutComputed(original).documentElement(), back.documentElement(), QString()); QVERIFY2(d.isEmpty(), qPrintable(d)); } // Splitting and joining adds no whitespace text. QETProject keeps the // symbol definitions as it read them and writes them back as they // are, so indentation a part gained would change the next save // (forum attachments 1328, 1978, 2057, 3003, 3004 did, with indented // parts). Here: a document without any whitespace-only text, as // QETProject::toXml() builds one, must come back without any. void noWhitespaceAdded() { QDomDocument original; QVERIFY(original.setContent(read(QStringLiteral(QET_EXAMPLES_DIR "/industrial.qet")))); auto whitespace_nodes = [](const QDomDocument &d) { int n = 0; std::function walk = [&](const QDomNode &node) { for (QDomNode c = node.firstChild() ; !c.isNull() ; c = c.nextSibling()) { if (c.isText() && c.nodeValue().trimmed().isEmpty()) ++n; walk(c); } }; walk(d); return n; }; QCOMPARE(whitespace_nodes(original), 0); QString error; const QList entries = QetContainer::split(original, QString(), &error); QVERIFY2(!entries.isEmpty(), qPrintable(error)); QDomDocument back; QVERIFY2(QetContainer::join(entries, &back, &error), qPrintable(error)); QCOMPARE(whitespace_nodes(back), 0); } // The engineering data is in project.sqlite and nowhere else: the folio // files hold no symbol information, links, wire numbers or title-block // fields, nor what QElectroTech computes from them (a wire's // description of its ends, a symbol text showing its information), and // the database holds them all (industrial.qet: 50 folios). void databaseHoldsTheData() { QDomDocument original; const QList entries = splitFile(QStringLiteral(QET_EXAMPLES_DIR "/industrial.qet"), &original); QVERIFY(!entries.isEmpty()); QByteArray database; for (const QetZip::Entry &e : entries) { if (e.name == QLatin1String("project.sqlite")) database = e.data; if (e.name.startsWith(QStringLiteral("folios/")) || e.name == QLatin1String("project.xml")) { QVERIFY2(!e.data.contains("]* num="))), qPrintable(e.name)); } } QVERIFY(database.startsWith("SQLite format 3")); sqlite3 *db = nullptr; QCOMPARE(sqlite3_open(":memory:", &db), SQLITE_OK); unsigned char *copy = static_cast(sqlite3_malloc64(sqlite3_uint64(database.size()))); memcpy(copy, database.constData(), size_t(database.size())); QCOMPARE(sqlite3_deserialize(db, "main", copy, database.size(), database.size(), SQLITE_DESERIALIZE_FREEONCLOSE), SQLITE_OK); auto count = [db](const char *table) { sqlite3_stmt *st = nullptr; sqlite3_prepare_v2(db, (QByteArray("SELECT count(*) FROM ") + table).constData(), -1, &st, nullptr); sqlite3_step(st); const int n = sqlite3_column_int(st, 0); sqlite3_finalize(st); return n; }; const QDomNodeList diagrams = original.elementsByTagName(QStringLiteral("diagram")); QCOMPARE(count("folio"), int(diagrams.size())); QCOMPARE(count("element"), int(original.elementsByTagName(QStringLiteral("element")).size()) - int(original.elementsByTagName(QStringLiteral("collection")).at(0).toElement() .elementsByTagName(QStringLiteral("element")).size())); QCOMPARE(count("element_info"), int(original.elementsByTagName(QStringLiteral("elementInformation")).size())); QCOMPARE(count("link"), int(original.elementsByTagName(QStringLiteral("link_uuid")).size())); QCOMPARE(count("conductor"), int(original.elementsByTagName(QStringLiteral("conductor")).size())); QVERIFY(count("folio_property") + count("doc_project_property") > 0); sqlite3_close(db); } // Nor does a folio file hold what QElectroTech computes (format 3): a // wire's description of its ends, a symbol text showing its information. // A file holding both, so that the check can fail. void computedNotKept() { QDomDocument original; QString error; QVERIFY(QetContainer::parse(read(QStringLiteral(QET_EXAMPLES_DIR "/schema_unifilaire_voltaique2.qet")), &original, &error)); const QDomNodeList texts = original.elementsByTagName(QStringLiteral("dynamic_elmt_text")); bool filled = false; for (int i = 0 ; i < texts.size() && !filled ; ++i) { QDomElement text = texts.at(i).toElement(); if (text.attribute(QStringLiteral("text_from")) != QLatin1String("ElementInfo")) continue; QDomElement content = text.firstChildElement(QStringLiteral("text")); while (content.hasChildNodes()) content.removeChild(content.firstChild()); content.appendChild(original.createTextNode(QStringLiteral("COMPUTED-LABEL"))); filled = true; } QVERIFY(filled); QVERIFY(original.toString().contains(QStringLiteral("element1_label="))); const QList entries = QetContainer::split(original, QString(), &error); QVERIFY2(!entries.isEmpty(), qPrintable(error)); for (const QetZip::Entry &e : entries) { if (!e.name.startsWith(QStringLiteral("folios/"))) continue; QVERIFY2(!e.data.contains("element1_label=") && !e.data.contains("terminalname2="), qPrintable(e.name)); QVERIFY2(!e.data.contains("COMPUTED-LABEL"), qPrintable(e.name)); } } // Terminal strips move into the database, and come back exactly: // industrial.qet with two strips on its own terminal symbols, parsed as // a save leaves it (no whitespace text). void terminalStripsInDatabase() { QByteArray xml = read(QStringLiteral(QET_EXAMPLES_DIR "/industrial.qet")); const QByteArray strips = "" "" "=A1+CAB1" "X1main & aux" "" "" "" "" "" "" "" "" "" "X2" "" ""; const int at = xml.indexOf(" 0); xml.insert(at, strips); QDomDocument original; QVERIFY(original.setContent(xml)); // as a save leaves it: no whitespace text QString error; const QList entries = QetContainer::split(original, QString(), &error); QVERIFY2(!entries.isEmpty(), qPrintable(error)); for (const QetZip::Entry &e : entries) if (e.name == QLatin1String("project.xml")) QVERIFY2(!e.data.contains(""), "the strips stayed in project.xml"); QDomDocument back; QVERIFY2(QetContainer::join(entries, &back, &error), qPrintable(error)); const QString d = difference(withoutComputed(original).documentElement(), back.documentElement(), QString()); QVERIFY2(d.isEmpty(), qPrintable(d)); } // The folio pictures leave the folios, once each void picturesAreFiles() { QDomDocument original; const QList entries = splitFile( QStringLiteral(QET_EXAMPLES_DIR "/weneedpolonez-Polonez_MR89_wiring_diagram.qet"), &original); QVERIFY(!entries.isEmpty()); int pictures = 0; for (const QetZip::Entry &e : entries) { if (e.name.startsWith(QStringLiteral("images/"))) { ++pictures; QVERIFY(e.data.startsWith("\x89PNG")); QVERIFY(!e.compress); } if (e.name.startsWith(QStringLiteral("folios/"))) QVERIFY2(!e.data.contains("iVBORw0KGgo"), "a folio still holds a PNG in base64"); } QVERIFY(pictures > 0); QVERIFY(pictures < 121); // the 121 placed pictures repeat a few files } // Given a map, join() hands the pictures over as bytes instead of // putting them back in the document as base64 void picturesKeptAside() { QDomDocument original; QList entries = splitFile( QStringLiteral(QET_EXAMPLES_DIR "/weneedpolonez-Polonez_MR89_wiring_diagram.qet"), &original); QVERIFY(!entries.isEmpty()); QHash files; for (const QetZip::Entry &e : entries) files.insert(e.name, e.data); QDomDocument back; QetContainer::Pictures pictures; QString error; QVERIFY2(QetContainer::join(entries, &back, &error, nullptr, &pictures), qPrintable(error)); const QDomNodeList images = back.elementsByTagName(QStringLiteral("image")); const QDomNodeList original_images = original.elementsByTagName(QStringLiteral("image")); QCOMPARE(images.size(), 121); QCOMPARE(images.size(), original_images.size()); for (int i = 0 ; i < images.size() ; ++i) { const QDomElement image = images.at(i).toElement(); const QString file = QetContainer::pictureFile(image); QVERIFY(!file.isEmpty()); QVERIFY(!image.firstChild().isText()); QCOMPARE(pictures.value(file), files.value(file)); QCOMPARE(pictures.value(file), QByteArray::fromBase64(original_images.at(i).firstChild().toText().data().toLatin1())); } // still refused when a picture is missing for (int i = 0 ; i < entries.size() ; ++i) if (entries.at(i).name.startsWith(QStringLiteral("images/"))) { entries.removeAt(i); break; } QDomDocument broken; QVERIFY(!QetContainer::join(entries, &broken, &error, nullptr, &pictures)); QVERIFY2(error.contains(QStringLiteral("missing")), qPrintable(error)); } // A file whose manifest needs a later reader is refused, not half read void laterFormatRefused() { QDomDocument original; QList entries = splitFile(QStringLiteral(QET_EXAMPLES_DIR "/perceuse.qet"), &original); QVERIFY(!entries.isEmpty()); for (QetZip::Entry &e : entries) if (e.name == QLatin1String("manifest.xml")) e.data = QString::fromUtf8(e.data).replace(QRegularExpression(QStringLiteral("min-reader=\"\\d+\"")), QStringLiteral("min-reader=\"9\"")).toUtf8(); QDomDocument back; QString error; QVERIFY(!QetContainer::join(entries, &back, &error)); QVERIFY2(error.contains(QStringLiteral("later")), qPrintable(error)); } void missingPartRefused() { QDomDocument original; QList entries = splitFile(QStringLiteral(QET_EXAMPLES_DIR "/perceuse.qet"), &original); for (int i = 0 ; i < entries.size() ; ++i) if (entries.at(i).name.startsWith(QStringLiteral("folios/"))) { entries.removeAt(i); break; } QDomDocument back; QString error; QVERIFY(!QetContainer::join(entries, &back, &error)); QVERIFY2(error.contains(QStringLiteral("missing")), qPrintable(error)); } // A crafted project.sqlite whose "element" is a view, not a table, is // refused: nothing in a file's schema is trusted. void garbageDatabaseRefused() { //Bytes that are no database: refused, not read (crashed reading //the first statement until stage D7) QDomDocument original; QList entries = splitFile(QStringLiteral(QET_EXAMPLES_DIR "/perceuse.qet"), &original); bool replaced = false; for (QetZip::Entry &entry : entries) if (entry.name == QLatin1String("project.sqlite")) { entry.data = QByteArray("not a database").repeated(100); replaced = true; } QVERIFY(replaced); QDomDocument back; QString error; QVERIFY(!QetContainer::join(entries, &back, &error)); QVERIFY(!error.isEmpty()); } void viewInsteadOfTableRefused() { QDomDocument original; QList entries = splitFile(QStringLiteral(QET_EXAMPLES_DIR "/perceuse.qet"), &original); QVERIFY(!entries.isEmpty()); for (QetZip::Entry &e : entries) { if (e.name != QLatin1String("project.sqlite")) continue; sqlite3 *db = nullptr; sqlite3_open(":memory:", &db); unsigned char *copy = static_cast(sqlite3_malloc64(sqlite3_uint64(e.data.size()))); memcpy(copy, e.data.constData(), size_t(e.data.size())); sqlite3_deserialize(db, "main", copy, e.data.size(), e.data.size(), SQLITE_DESERIALIZE_FREEONCLOSE | SQLITE_DESERIALIZE_RESIZEABLE); QCOMPARE(sqlite3_exec(db, "ALTER TABLE element RENAME TO element_data;" "CREATE VIEW element AS SELECT * FROM element_data;", nullptr, nullptr, nullptr), SQLITE_OK); sqlite3_int64 size = 0; unsigned char *bytes = sqlite3_serialize(db, "main", &size, 0); e.data = QByteArray(reinterpret_cast(bytes), qsizetype(size)); sqlite3_free(bytes); sqlite3_close(db); } QDomDocument back; QString error; QVERIFY(!QetContainer::join(entries, &back, &error)); QVERIFY2(error.contains(QStringLiteral("element")), qPrintable(error)); } // A row's key is its item's uuid, which the folio file keeps: no key // made up from where an item is (format 2 had f1, f2...), no key // attribute in the folio files of a project as QElectroTech saves it, // true/false held as 1/0, and the format in the database's // user_version. void keysAreTheUuids() { const QDomDocument original = savedProject(); original.elementsByTagName(QStringLiteral("conductor")).at(0).toElement() .setAttribute(QStringLiteral("freezeLabel"), QStringLiteral("true")); QString error; const QList entries = QetContainer::split(original, QString(), &error); QVERIFY2(!entries.isEmpty(), qPrintable(error)); const QByteArray database = entry(entries, QStringLiteral("project.sqlite")); const QByteArray folios = folioFiles(entries); QVERIFY(!folios.contains("qetz-key")); QVERIFY(!folios.contains("qetz-folio")); QCOMPARE(query(database, "PRAGMA user_version"), QString::number(QetContainer::Format)); const QString first = original.elementsByTagName(QStringLiteral("diagram")).at(0).toElement() .attribute(QStringLiteral("uuid")); QCOMPARE(query(database, "SELECT uuid FROM folio WHERE pos = 1"), first); QCOMPARE(query(database, "SELECT count(*) FROM folio WHERE uuid NOT LIKE '{%}'"), QStringLiteral("0")); QCOMPARE(query(database, "SELECT count(*) FROM element WHERE uuid NOT LIKE '{%}'"), QStringLiteral("0")); QCOMPARE(query(database, "SELECT count(*) FROM conductor WHERE uuid NOT LIKE '{%}'"), QStringLiteral("0")); QCOMPARE(query(database, "SELECT count(*) FROM element WHERE typeof(freezeLabel) <> 'integer'"), QStringLiteral("0")); QCOMPARE(query(database, "SELECT count(*) FROM conductor WHERE typeof(freezeLabel) <> 'integer'"), QStringLiteral("0")); QCOMPARE(query(database, "SELECT sum(freezeLabel) FROM conductor"), QStringLiteral("1")); const QString d = backDifference(original, entries); QVERIFY2(d.isEmpty(), qPrintable(d)); QCOMPARE(query(database, "SELECT count(*) FROM element_info WHERE typeof(show) <> 'integer'"), QStringLiteral("0")); QCOMPARE(query(database, "SELECT count(*) FROM element_info WHERE attrs LIKE '%\"tag\"%'"), QStringLiteral("0")); } // Two symbols sharing a uuid (F100) each get a key made from what they // hold, the same on every save, and come back as they were. void sharedUuidKeyed() { QDomDocument original = savedProject(); const QDomNodeList elements = original.elementsByTagName(QStringLiteral("element")); QDomElement a, b; for (int i = 0 ; i < elements.size() && b.isNull() ; ++i) { const QDomElement e = elements.at(i).toElement(); if (e.parentNode().nodeName() != QLatin1String("elements")) continue; if (a.isNull()) a = e; else b = e; } QVERIFY(!b.isNull()); b.setAttribute(QStringLiteral("uuid"), a.attribute(QStringLiteral("uuid"))); QString error; const QList once = QetContainer::split(original, QString(), &error); const QList twice = QetContainer::split(original, QString(), &error); QVERIFY2(!once.isEmpty(), qPrintable(error)); QCOMPARE(folioFiles(once).count("qetz-key"), 2); QCOMPARE(entry(once, QStringLiteral("project.sqlite")), entry(twice, QStringLiteral("project.sqlite"))); const QString d = backDifference(original, once); QVERIFY2(d.isEmpty(), qPrintable(d)); } // A true/false attribute QElectroTech would not write stays in the // folio file as it was. void oddBooleanKept() { QDomDocument original; QVERIFY(QetContainer::parse(read(QStringLiteral(QET_EXAMPLES_DIR "/perceuse.qet")), &original)); const QDomNodeList conductors = original.elementsByTagName(QStringLiteral("conductor")); QVERIFY(conductors.size() > 0); conductors.at(0).toElement().setAttribute(QStringLiteral("freezeLabel"), QStringLiteral("yes")); QString error; const QList entries = QetContainer::split(original, QString(), &error); QVERIFY2(!entries.isEmpty(), qPrintable(error)); QVERIFY(folioFiles(entries).contains("freezeLabel=\"yes\"")); const QString d = backDifference(original, entries); QVERIFY2(d.isEmpty(), qPrintable(d)); } // A format 2 file, as QElectroTech 0.200.1 development builds wrote it // (grafcet.qet saved as .qetz), still opens, and saved again as the // current format comes back the same. void format2Read() { QList entries; QString error; QVERIFY2(QetZip::fromBytes(read(QStringLiteral(QET_FIXTURES_DIR "/qetz_format2.qetz")), &entries, &error), qPrintable(error)); QVERIFY(entry(entries, QStringLiteral("manifest.xml")).contains("format=\"2\"")); QDomDocument project; QVERIFY2(QetContainer::join(entries, &project, &error), qPrintable(error)); QVERIFY(project.toString().contains(QStringLiteral(" resaved = QetContainer::split(project, QString(), &error); QVERIFY2(!resaved.isEmpty(), qPrintable(error)); const QString d = backDifference(project, resaved); QVERIFY2(d.isEmpty(), qPrintable(d)); } void notAProjectRefused() { QDomDocument back; QString error; QVERIFY(!QetContainer::join({{QStringLiteral("mimetype"), "application/zip", false}}, &back, &error)); } }; QTEST_APPLESS_MAIN(tst_qetcontainer) #include "tst_qetcontainer.moc"