From a475383915bb3303e245d5e2475aa5d1a8106f63 Mon Sep 17 00:00:00 2001 From: ispyisail Date: Sat, 10 Oct 2026 22:32:10 +1300 Subject: [PATCH] Split a project into the parts of a .qetz file and join it back Second stage of the zipped project file (#1440): QetContainer::split() turns a project document into the entries of a .qetz -- mimetype first and stored, a manifest with the format and the oldest format allowed to read it, project.xml with each part replaced by , one file per folio, per symbol definition and per title block template, and folio pictures as PNG files named by their SHA-256, stored once each. join() puts it back and refuses a file of a later format, a missing part or anything that is not a .qetz. A picture leaves its folio only if its base64 comes back exactly, so nothing here can change a project. Parts are written as QET writes a .qet and read as it reads one. Still nothing opens or saves a .qetz. Test: tst_qetcontainer, every example split, zipped, unzipped and joined back to the same document (whitespace between elements aside); pictures out of the folios once each; later format and missing part refused. With QET_CONTAINER_CORPUS it also runs on a folder of projects: 341 of 341 readable projects collected from the bugtracker, GitHub and the forum come back the same. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_019Wga9B92BNurDEtxxtCDxf --- cmake/qet_compilation_vars.cmake | 2 + sources/container/qetcontainer.cpp | 295 +++++++++++++++++++++++++++++ sources/container/qetcontainer.h | 59 ++++++ tests/qttest/CMakeLists.txt | 14 ++ tests/qttest/tst_qetcontainer.cpp | 167 ++++++++++++++++ 5 files changed, 537 insertions(+) create mode 100644 sources/container/qetcontainer.cpp create mode 100644 sources/container/qetcontainer.h create mode 100644 tests/qttest/tst_qetcontainer.cpp diff --git a/cmake/qet_compilation_vars.cmake b/cmake/qet_compilation_vars.cmake index 076c6fc5d..8993a81fa 100644 --- a/cmake/qet_compilation_vars.cmake +++ b/cmake/qet_compilation_vars.cmake @@ -167,6 +167,8 @@ set(QET_SRC_FILES ${QET_DIR}/sources/import/edz/lzma/Ppmd7Dec.c ${QET_DIR}/sources/container/miniz/miniz.c ${QET_DIR}/sources/container/miniz/miniz.h + ${QET_DIR}/sources/container/qetcontainer.cpp + ${QET_DIR}/sources/container/qetcontainer.h ${QET_DIR}/sources/container/qetzip.cpp ${QET_DIR}/sources/container/qetzip.h ${QET_DIR}/sources/borderproperties.cpp diff --git a/sources/container/qetcontainer.cpp b/sources/container/qetcontainer.cpp new file mode 100644 index 000000000..0b8d430d2 --- /dev/null +++ b/sources/container/qetcontainer.cpp @@ -0,0 +1,295 @@ +/* + Copyright 2006-2026 The QElectroTech Team + This file is part of QElectroTech. + + QElectroTech is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + QElectroTech is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with QElectroTech. If not, see . +*/ +#include "qetcontainer.h" + +#include +#include +#include +#include + +const char QetContainer::MimeType[] = "application/x-qelectrotech-project+zip"; + +namespace { +const QString PartTag = QStringLiteral("qetz-part"); +const QString FileAttribute = QStringLiteral("file"); +const QString ImageFileAttribute = QStringLiteral("qetz-file"); + +void setError(QString *error, const QString &text) +{ + if (error) *error = text; +} + + //An element on its own, as a .qet is written (QET::writeXmlFile()) +QByteArray serialize(const QDomNode &node) +{ + QDomDocument document; + document.appendChild(document.createProcessingInstruction( + QStringLiteral("xml"), + QStringLiteral("version=\"1.0\" encoding=\"UTF-8\""))); + document.appendChild(document.importNode(node, true)); + return document.toString(4).toUtf8(); +} + + //A file name from a name chosen by the user: no folders, nothing a zip + //tool or a file system would read another way. +QString safeSegment(const QString &name, const QString &fallback) +{ + QString out; + for (const QChar c : name) { + out += (c.isLetterOrNumber() || c == QLatin1Char('-') || c == QLatin1Char('_') + || c == QLatin1Char('.') || c == QLatin1Char(' ')) + ? c : QLatin1Char('_'); + } + out = out.trimmed(); + while (out.startsWith(QLatin1Char('.'))) out.remove(0, 1); + return out.isEmpty() ? fallback : out; +} + + //@p name, or name~2, name~3... if already taken +QString unique(const QString &name, QSet *taken) +{ + QString candidate = name; + for (int n = 2 ; taken->contains(candidate) ; ++n) { + const int dot = name.lastIndexOf(QLatin1Char('.')); + candidate = dot > name.lastIndexOf(QLatin1Char('/')) + ? name.left(dot) + QStringLiteral("~%1").arg(n) + name.mid(dot) + : name + QStringLiteral("~%1").arg(n); + } + taken->insert(candidate); + return candidate; +} + +QDomElement placeholder(QDomDocument &document, const QString &file) +{ + QDomElement part = document.createElement(PartTag); + part.setAttribute(FileAttribute, file); + return part; +} + +QList childElements(const QDomElement &parent, const QString &tag) +{ + QList list; + for (QDomElement e = parent.firstChildElement(tag) ; !e.isNull() ; e = e.nextSiblingElement(tag)) + list << e; + return list; +} + + //The collection's symbol definitions, each to elements// +void splitCollection(QDomDocument &document, const QDomElement &category, const QString &path, + QList *parts, QSet *taken) +{ + for (QDomElement child = category.firstChildElement() ; !child.isNull() ; + child = child.nextSiblingElement()) + { + const QString name = safeSegment(child.attribute(QStringLiteral("name")), + QStringLiteral("unnamed")); + if (child.tagName() == QLatin1String("category")) { + splitCollection(document, child, path + name + QLatin1Char('/'), parts, taken); + } else if (child.tagName() == QLatin1String("element")) { + QDomElement definition = child.firstChildElement(QStringLiteral("definition")); + if (definition.isNull()) continue; + const QString file = unique(path + name, taken); + parts->append({file, serialize(definition), true}); + child.replaceChild(placeholder(document, file), definition); + } + } +} +} + +/** + @brief QetContainer::parse + Parse @p xml as QETProject::openFile() reads a .qet: a text node made + only of spaces is kept (a title-block value of " ", #973). +*/ +bool QetContainer::parse(const QByteArray &xml, QDomDocument *document, QString *error) +{ +#if QT_VERSION >= QT_VERSION_CHECK(6, 5, 0) + const auto result = document->setContent(xml, QDomDocument::ParseOption::PreserveSpacingOnlyNodes); + if (!result) { + setError(error, QStringLiteral("line %1: %2").arg(result.errorLine).arg(result.errorMessage)); + return false; + } + return true; +#else + QString message; + int line = 0; + if (!document->setContent(xml, &message, &line)) { + setError(error, QStringLiteral("line %1: %2").arg(line).arg(message)); + return false; + } + return true; +#endif +} + +/** + @brief QetContainer::split + @param project : the project document, as QETProject::toXml() makes it + @param writer : the program writing it, kept in the manifest + @return the entries of the .qetz, mimetype first; empty with @p error + set if @p project is not a project +*/ +QList QetContainer::split(const QDomDocument &project, const QString &writer, + QString *error) +{ + QDomDocument document = project.cloneNode(true).toDocument(); + QDomElement root = document.documentElement(); + if (root.tagName() != QLatin1String("project")) { + setError(error, QStringLiteral("not a project document")); + return {}; + } + + QList parts; + QSet taken; + + //Folios, each with its pictures taken out + int folio = 0; + QMap pictures; //name -> png, once each + for (QDomElement diagram : childElements(root, QStringLiteral("diagram"))) + { + ++folio; + for (QDomElement images : childElements(diagram, QStringLiteral("images"))) { + for (QDomElement image : childElements(images, QStringLiteral("image"))) { + const QDomText text = image.firstChild().toText(); + if (text.isNull() || image.hasAttribute(ImageFileAttribute)) continue; + const QByteArray base64 = text.data().toLatin1(); + const auto decoded = QByteArray::fromBase64Encoding( + base64, QByteArray::AbortOnBase64DecodingErrors); + //Only what comes back exactly leaves the folio + if (!decoded || decoded.decoded.toBase64() != base64 || decoded.decoded.isEmpty()) + continue; + const QString name = QStringLiteral("images/%1.%2") + .arg(QString::fromLatin1(QCryptographicHash::hash( + decoded.decoded, QCryptographicHash::Sha256).toHex()), + decoded.decoded.startsWith("\x89PNG") ? QStringLiteral("png") + : QStringLiteral("bin")); + pictures.insert(name, decoded.decoded); + image.removeChild(text); + image.setAttribute(ImageFileAttribute, name); + } + } + const QString uuid = diagram.attribute(QStringLiteral("uuid")); + const QString file = unique(QStringLiteral("folios/%1.xml").arg( + safeSegment(QString(uuid).remove(QLatin1Char('{')).remove(QLatin1Char('}')), + QString::number(folio))), + &taken); + parts.append(QetZip::Entry{file, serialize(diagram), true}); + root.replaceChild(placeholder(document, file), diagram); + } + + //The project's symbols + for (QDomElement collection : childElements(root, QStringLiteral("collection"))) + splitCollection(document, collection, QStringLiteral("elements/"), &parts, &taken); + + //Title block templates + for (QDomElement templates : childElements(root, QStringLiteral("titleblocktemplates"))) { + for (QDomElement tbt : childElements(templates, QStringLiteral("titleblocktemplate"))) { + const QString file = unique(QStringLiteral("titleblocks/%1.titleblock").arg( + safeSegment(tbt.attribute(QStringLiteral("name")), + QStringLiteral("unnamed"))), + &taken); + parts.append(QetZip::Entry{file, serialize(tbt), true}); + templates.replaceChild(placeholder(document, file), tbt); + } + } + + QDomDocument manifest; + QDomElement qetz = manifest.createElement(QStringLiteral("qetz")); + qetz.setAttribute(QStringLiteral("format"), Format); + qetz.setAttribute(QStringLiteral("min-reader"), Format); + if (!writer.isEmpty()) qetz.setAttribute(QStringLiteral("writer"), writer); + manifest.appendChild(qetz); + + QList entries; + entries.append(QetZip::Entry{QStringLiteral("mimetype"), QByteArray(MimeType), false}); + entries.append(QetZip::Entry{QStringLiteral("manifest.xml"), serialize(qetz), true}); + entries.append(QetZip::Entry{QStringLiteral("project.xml"), serialize(root), true}); + entries += parts; + for (auto it = pictures.cbegin() ; it != pictures.cend() ; ++it) + entries.append(QetZip::Entry{it.key(), it.value(), false}); //PNG is compressed already + return entries; +} + +/** + @brief QetContainer::join + Rebuild in @p project the document split() made @p entries from. + Refused, with @p error set: not a .qetz, made for a later QElectroTech + (the manifest's min-reader), or a part missing or unreadable. Entries + nothing names are ignored, so a later format may add some. +*/ +bool QetContainer::join(const QList &entries, QDomDocument *project, QString *error) +{ + QHash files; + for (const QetZip::Entry &entry : entries) + files.insert(entry.name, entry.data); + + if (files.value(QStringLiteral("mimetype")) != QByteArray(MimeType)) { + setError(error, QStringLiteral("not a QElectroTech project (.qetz)")); + return false; + } + QDomDocument manifest; + if (!parse(files.value(QStringLiteral("manifest.xml")), &manifest, error)) + return false; + bool ok = false; + const int min_reader = manifest.documentElement().attribute(QStringLiteral("min-reader")).toInt(&ok); + if (!ok || min_reader > Format) { + setError(error, QStringLiteral("made by a later QElectroTech (format %1, this one reads up to %2)") + .arg(manifest.documentElement().attribute(QStringLiteral("format"))) + .arg(Format)); + return false; + } + + if (!parse(files.value(QStringLiteral("project.xml")), project, error)) + return false; + + //Parts, a snapshot first: the list is live and shrinks as they go + const QDomNodeList list = project->elementsByTagName(PartTag); + QList placeholders; + for (int i = 0 ; i < list.size() ; ++i) placeholders << list.at(i).toElement(); + for (QDomElement part : placeholders) + { + const QString file = part.attribute(FileAttribute); + if (!files.contains(file)) { + setError(error, QStringLiteral("part \"%1\" is missing").arg(file)); + return false; + } + QDomDocument document; + QString part_error; + if (!parse(files.value(file), &document, &part_error)) { + setError(error, QStringLiteral("%1: %2").arg(file, part_error)); + return false; + } + part.parentNode().replaceChild(project->importNode(document.documentElement(), true), part); + } + + //Pictures back to base64, as the image's first child + const QDomNodeList images = project->elementsByTagName(QStringLiteral("image")); + for (int i = 0 ; i < images.size() ; ++i) + { + QDomElement image = images.at(i).toElement(); + if (!image.hasAttribute(ImageFileAttribute)) continue; + const QString file = image.attribute(ImageFileAttribute); + if (!files.contains(file)) { + setError(error, QStringLiteral("picture \"%1\" is missing").arg(file)); + return false; + } + image.removeAttribute(ImageFileAttribute); + image.insertBefore(project->createTextNode(QString::fromLatin1(files.value(file).toBase64())), + image.firstChild()); + } + return true; +} diff --git a/sources/container/qetcontainer.h b/sources/container/qetcontainer.h new file mode 100644 index 000000000..5143ab220 --- /dev/null +++ b/sources/container/qetcontainer.h @@ -0,0 +1,59 @@ +/* + Copyright 2006-2026 The QElectroTech Team + This file is part of QElectroTech. + + QElectroTech is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + QElectroTech is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with QElectroTech. If not, see . +*/ +#ifndef QETCONTAINER_H +#define QETCONTAINER_H + +#include "qetzip.h" + +#include + +/** + @brief The QetContainer namespace + Splits a project document into the parts of a .qetz file and joins + them back: + + mimetype first, stored + manifest.xml format and oldest format allowed to read it + project.xml the project, each part replaced by + + folios/.xml one per folio + elements/.../ the project's symbol definitions + titleblocks/.titleblock + images/.png folio pictures, once each, not base64 + + join(split(d)) is d. Nothing here knows what the parts mean: the + project is read and written by the same code as a .qet file. +*/ +namespace QetContainer +{ + extern const char MimeType[]; + /// The format written; files of a later format are refused by join() + /// unless their min-reader allows this one. + constexpr int Format = 1; + + QList split(const QDomDocument &project, + const QString &writer = QString(), + QString *error = nullptr); + bool join(const QList &entries, QDomDocument *project, + QString *error = nullptr); + + /// Parse @p xml as a .qet is read (QETProject::openFile) + bool parse(const QByteArray &xml, QDomDocument *document, QString *error = nullptr); +} + +#endif // QETCONTAINER_H diff --git a/tests/qttest/CMakeLists.txt b/tests/qttest/CMakeLists.txt index d79cbac4e..569b8c195 100644 --- a/tests/qttest/CMakeLists.txt +++ b/tests/qttest/CMakeLists.txt @@ -1173,3 +1173,17 @@ add_executable( add_test(NAME tst_qetzip COMMAND tst_qetzip) target_include_directories(tst_qetzip PRIVATE ${QET_DIR}/sources) target_link_libraries(tst_qetzip PRIVATE Qt::Test) + +# A project split into the parts of a .qetz and joined back is the same +# document, on every example (and QET_CONTAINER_CORPUS if set). +add_executable( + tst_qetcontainer + tst_qetcontainer.cpp + ${QET_DIR}/sources/container/qetcontainer.cpp + ${QET_DIR}/sources/container/qetzip.cpp + ${QET_DIR}/sources/container/miniz/miniz.c) +add_test(NAME tst_qetcontainer COMMAND tst_qetcontainer) +target_include_directories(tst_qetcontainer PRIVATE ${QET_DIR}/sources) +target_link_libraries(tst_qetcontainer PRIVATE Qt::Test Qt::Xml) +target_compile_definitions(tst_qetcontainer PRIVATE + "QET_EXAMPLES_DIR=\"${QET_DIR}/examples\"") diff --git a/tests/qttest/tst_qetcontainer.cpp b/tests/qttest/tst_qetcontainer.cpp new file mode 100644 index 000000000..4cfe857cd --- /dev/null +++ b/tests/qttest/tst_qetcontainer.cpp @@ -0,0 +1,167 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +#include + +#include +#include + +#include "container/qetcontainer.h" + +// A project document split into the parts of a .qetz, zipped, unzipped and +// joined back is the document it was: every example, and every .qet in the +// folder QET_CONTAINER_CORPUS names, if set. Whitespace-only text between +// elements is indentation and is not compared; elsewhere it is (a +// title-block value of " ", #973). +class tst_qetcontainer : public QObject +{ + Q_OBJECT + + static QList children(const QDomNode &node) + { + QList all; + bool has_element = false; + for (QDomNode c = node.firstChild() ; !c.isNull() ; c = c.nextSibling()) { + all << c; + has_element |= c.isElement(); + } + if (!has_element) return all; + QList kept; + for (const QDomNode &c : all) + if (!(c.isText() && c.nodeValue().trimmed().isEmpty())) kept << c; + return kept; + } + + // empty when equal, else where they differ + static QString difference(const QDomNode &a, const QDomNode &b, const QString &path) + { + const QString here = path + QLatin1Char('/') + a.nodeName(); + if (a.nodeType() != b.nodeType() || a.nodeName() != b.nodeName()) + return here + QStringLiteral(": %1 vs %2").arg(a.nodeName(), b.nodeName()); + if (!a.isElement() && !a.isDocument() && a.nodeValue() != b.nodeValue()) + return here + QStringLiteral(": value differs"); + if (a.isElement()) { + const QDomNamedNodeMap aa = a.attributes(), ba = b.attributes(); + if (aa.count() != ba.count()) + return here + QStringLiteral(": %1 vs %2 attributes").arg(aa.count()).arg(ba.count()); + for (int i = 0 ; i < aa.count() ; ++i) { + const QDomAttr attr = aa.item(i).toAttr(); + if (!ba.contains(attr.name()) || ba.namedItem(attr.name()).nodeValue() != attr.value()) + return here + QStringLiteral(": attribute %1 differs").arg(attr.name()); + } + } + const QList ac = children(a), bc = children(b); + if (ac.size() != bc.size()) + return here + QStringLiteral(": %1 vs %2 children").arg(ac.size()).arg(bc.size()); + for (int i = 0 ; i < ac.size() ; ++i) { + const QString d = difference(ac.at(i), bc.at(i), here + QStringLiteral("[%1]").arg(i)); + if (!d.isEmpty()) return d; + } + return {}; + } + + static QByteArray read(const QString &path) + { + QFile f(path); + return f.open(QIODevice::ReadOnly) ? f.readAll() : QByteArray(); + } + + static QList splitFile(const QString &path, QDomDocument *original) + { + QString error; + if (!QetContainer::parse(read(path), original, &error)) return {}; + return QetContainer::split(*original, QStringLiteral("tst_qetcontainer"), &error); + } + +private slots: + void sameDocumentBack_data() + { + QTest::addColumn("file"); + QStringList dirs{QStringLiteral(QET_EXAMPLES_DIR)}; + const QString corpus = qEnvironmentVariable("QET_CONTAINER_CORPUS"); + if (!corpus.isEmpty()) dirs << corpus; + for (const QString &dir : dirs) { + QDirIterator it(dir, {QStringLiteral("*.qet")}, QDir::Files, QDirIterator::Subdirectories); + QStringList files; + while (it.hasNext()) files << it.next(); + files.sort(); + for (const QString &f : files) + QTest::newRow(qPrintable(QDir(dir).relativeFilePath(f))) << f; + } + } + void sameDocumentBack() + { + QFETCH(QString, file); + QDomDocument original; + QString error; + if (!QetContainer::parse(read(file), &original, &error)) + QSKIP(qPrintable(QStringLiteral("not readable as XML, as QElectroTech finds too: ") + error)); + const QList entries = QetContainer::split(original, QString(), &error); + QVERIFY2(!entries.isEmpty(), qPrintable(error)); + QCOMPARE(entries.first().name, QStringLiteral("mimetype")); + + QList unzipped; + QVERIFY2(QetZip::fromBytes(QetZip::toBytes(entries, &error), &unzipped, &error), qPrintable(error)); + QDomDocument back; + QVERIFY2(QetContainer::join(unzipped, &back, &error), qPrintable(error)); + const QString d = difference(original.documentElement(), back.documentElement(), QString()); + QVERIFY2(d.isEmpty(), qPrintable(d)); + } + + // The folio pictures leave the folios, once each + void picturesAreFiles() + { + QDomDocument original; + const QList entries = splitFile( + QStringLiteral(QET_EXAMPLES_DIR "/weneedpolonez-Polonez_MR89_wiring_diagram.qet"), &original); + QVERIFY(!entries.isEmpty()); + int pictures = 0; + for (const QetZip::Entry &e : entries) { + if (e.name.startsWith(QStringLiteral("images/"))) { + ++pictures; + QVERIFY(e.data.startsWith("\x89PNG")); + QVERIFY(!e.compress); + } + if (e.name.startsWith(QStringLiteral("folios/"))) + QVERIFY2(!e.data.contains("iVBORw0KGgo"), "a folio still holds a PNG in base64"); + } + QVERIFY(pictures > 0); + QVERIFY(pictures < 121); // the 121 placed pictures repeat a few files + } + + // A file whose manifest needs a later reader is refused, not half read + void laterFormatRefused() + { + QDomDocument original; + QList entries = splitFile(QStringLiteral(QET_EXAMPLES_DIR "/perceuse.qet"), &original); + QVERIFY(!entries.isEmpty()); + for (QetZip::Entry &e : entries) + if (e.name == QLatin1String("manifest.xml")) + e.data.replace("min-reader=\"1\"", "min-reader=\"9\""); + QDomDocument back; + QString error; + QVERIFY(!QetContainer::join(entries, &back, &error)); + QVERIFY2(error.contains(QStringLiteral("later")), qPrintable(error)); + } + + void missingPartRefused() + { + QDomDocument original; + QList entries = splitFile(QStringLiteral(QET_EXAMPLES_DIR "/perceuse.qet"), &original); + for (int i = 0 ; i < entries.size() ; ++i) + if (entries.at(i).name.startsWith(QStringLiteral("folios/"))) { entries.removeAt(i); break; } + QDomDocument back; + QString error; + QVERIFY(!QetContainer::join(entries, &back, &error)); + QVERIFY2(error.contains(QStringLiteral("missing")), qPrintable(error)); + } + + void notAProjectRefused() + { + QDomDocument back; + QString error; + QVERIFY(!QetContainer::join({{QStringLiteral("mimetype"), "application/zip", false}}, &back, &error)); + } +}; + +QTEST_APPLESS_MAIN(tst_qetcontainer) + +#include "tst_qetcontainer.moc"