Split a project into the parts of a .qetz file and join it back

Second stage of the zipped project file (#1440): QetContainer::split()
turns a project document into the entries of a .qetz -- mimetype first
and stored, a manifest with the format and the oldest format allowed to
read it, project.xml with each part replaced by <qetz-part file=.../>,
one file per folio, per symbol definition and per title block template,
and folio pictures as PNG files named by their SHA-256, stored once
each. join() puts it back and refuses a file of a later format, a
missing part or anything that is not a .qetz.

A picture leaves its folio only if its base64 comes back exactly, so
nothing here can change a project. Parts are written as QET writes a
.qet and read as it reads one. Still nothing opens or saves a .qetz.

Test: tst_qetcontainer, every example split, zipped, unzipped and
joined back to the same document (whitespace between elements aside);
pictures out of the folios once each; later format and missing part
refused. With QET_CONTAINER_CORPUS it also runs on a folder of projects:
341 of 341 readable projects collected from the bugtracker, GitHub and
the forum come back the same.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_019Wga9B92BNurDEtxxtCDxf
This commit is contained in:
ispyisail
2026-10-10 22:32:10 +13:00
parent d7ac633043
commit a475383915
5 changed files with 537 additions and 0 deletions
+2
View File
@@ -167,6 +167,8 @@ set(QET_SRC_FILES
${QET_DIR}/sources/import/edz/lzma/Ppmd7Dec.c
${QET_DIR}/sources/container/miniz/miniz.c
${QET_DIR}/sources/container/miniz/miniz.h
${QET_DIR}/sources/container/qetcontainer.cpp
${QET_DIR}/sources/container/qetcontainer.h
${QET_DIR}/sources/container/qetzip.cpp
${QET_DIR}/sources/container/qetzip.h
${QET_DIR}/sources/borderproperties.cpp
+295
View File
@@ -0,0 +1,295 @@
/*
Copyright 2006-2026 The QElectroTech Team
This file is part of QElectroTech.
QElectroTech is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
QElectroTech is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with QElectroTech. If not, see <http://www.gnu.org/licenses/>.
*/
#include "qetcontainer.h"
#include <QCryptographicHash>
#include <QHash>
#include <QMap>
#include <QSet>
const char QetContainer::MimeType[] = "application/x-qelectrotech-project+zip";
namespace {
const QString PartTag = QStringLiteral("qetz-part");
const QString FileAttribute = QStringLiteral("file");
const QString ImageFileAttribute = QStringLiteral("qetz-file");
void setError(QString *error, const QString &text)
{
if (error) *error = text;
}
//An element on its own, as a .qet is written (QET::writeXmlFile())
QByteArray serialize(const QDomNode &node)
{
QDomDocument document;
document.appendChild(document.createProcessingInstruction(
QStringLiteral("xml"),
QStringLiteral("version=\"1.0\" encoding=\"UTF-8\"")));
document.appendChild(document.importNode(node, true));
return document.toString(4).toUtf8();
}
//A file name from a name chosen by the user: no folders, nothing a zip
//tool or a file system would read another way.
QString safeSegment(const QString &name, const QString &fallback)
{
QString out;
for (const QChar c : name) {
out += (c.isLetterOrNumber() || c == QLatin1Char('-') || c == QLatin1Char('_')
|| c == QLatin1Char('.') || c == QLatin1Char(' '))
? c : QLatin1Char('_');
}
out = out.trimmed();
while (out.startsWith(QLatin1Char('.'))) out.remove(0, 1);
return out.isEmpty() ? fallback : out;
}
//@p name, or name~2, name~3... if already taken
QString unique(const QString &name, QSet<QString> *taken)
{
QString candidate = name;
for (int n = 2 ; taken->contains(candidate) ; ++n) {
const int dot = name.lastIndexOf(QLatin1Char('.'));
candidate = dot > name.lastIndexOf(QLatin1Char('/'))
? name.left(dot) + QStringLiteral("~%1").arg(n) + name.mid(dot)
: name + QStringLiteral("~%1").arg(n);
}
taken->insert(candidate);
return candidate;
}
QDomElement placeholder(QDomDocument &document, const QString &file)
{
QDomElement part = document.createElement(PartTag);
part.setAttribute(FileAttribute, file);
return part;
}
QList<QDomElement> childElements(const QDomElement &parent, const QString &tag)
{
QList<QDomElement> list;
for (QDomElement e = parent.firstChildElement(tag) ; !e.isNull() ; e = e.nextSiblingElement(tag))
list << e;
return list;
}
//The collection's symbol definitions, each to elements/<categories>/<name>
void splitCollection(QDomDocument &document, const QDomElement &category, const QString &path,
QList<QetZip::Entry> *parts, QSet<QString> *taken)
{
for (QDomElement child = category.firstChildElement() ; !child.isNull() ;
child = child.nextSiblingElement())
{
const QString name = safeSegment(child.attribute(QStringLiteral("name")),
QStringLiteral("unnamed"));
if (child.tagName() == QLatin1String("category")) {
splitCollection(document, child, path + name + QLatin1Char('/'), parts, taken);
} else if (child.tagName() == QLatin1String("element")) {
QDomElement definition = child.firstChildElement(QStringLiteral("definition"));
if (definition.isNull()) continue;
const QString file = unique(path + name, taken);
parts->append({file, serialize(definition), true});
child.replaceChild(placeholder(document, file), definition);
}
}
}
}
/**
@brief QetContainer::parse
Parse @p xml as QETProject::openFile() reads a .qet: a text node made
only of spaces is kept (a title-block value of " ", #973).
*/
bool QetContainer::parse(const QByteArray &xml, QDomDocument *document, QString *error)
{
#if QT_VERSION >= QT_VERSION_CHECK(6, 5, 0)
const auto result = document->setContent(xml, QDomDocument::ParseOption::PreserveSpacingOnlyNodes);
if (!result) {
setError(error, QStringLiteral("line %1: %2").arg(result.errorLine).arg(result.errorMessage));
return false;
}
return true;
#else
QString message;
int line = 0;
if (!document->setContent(xml, &message, &line)) {
setError(error, QStringLiteral("line %1: %2").arg(line).arg(message));
return false;
}
return true;
#endif
}
/**
@brief QetContainer::split
@param project : the project document, as QETProject::toXml() makes it
@param writer : the program writing it, kept in the manifest
@return the entries of the .qetz, mimetype first; empty with @p error
set if @p project is not a project
*/
QList<QetZip::Entry> QetContainer::split(const QDomDocument &project, const QString &writer,
QString *error)
{
QDomDocument document = project.cloneNode(true).toDocument();
QDomElement root = document.documentElement();
if (root.tagName() != QLatin1String("project")) {
setError(error, QStringLiteral("not a project document"));
return {};
}
QList<QetZip::Entry> parts;
QSet<QString> taken;
//Folios, each with its pictures taken out
int folio = 0;
QMap<QString, QByteArray> pictures; //name -> png, once each
for (QDomElement diagram : childElements(root, QStringLiteral("diagram")))
{
++folio;
for (QDomElement images : childElements(diagram, QStringLiteral("images"))) {
for (QDomElement image : childElements(images, QStringLiteral("image"))) {
const QDomText text = image.firstChild().toText();
if (text.isNull() || image.hasAttribute(ImageFileAttribute)) continue;
const QByteArray base64 = text.data().toLatin1();
const auto decoded = QByteArray::fromBase64Encoding(
base64, QByteArray::AbortOnBase64DecodingErrors);
//Only what comes back exactly leaves the folio
if (!decoded || decoded.decoded.toBase64() != base64 || decoded.decoded.isEmpty())
continue;
const QString name = QStringLiteral("images/%1.%2")
.arg(QString::fromLatin1(QCryptographicHash::hash(
decoded.decoded, QCryptographicHash::Sha256).toHex()),
decoded.decoded.startsWith("\x89PNG") ? QStringLiteral("png")
: QStringLiteral("bin"));
pictures.insert(name, decoded.decoded);
image.removeChild(text);
image.setAttribute(ImageFileAttribute, name);
}
}
const QString uuid = diagram.attribute(QStringLiteral("uuid"));
const QString file = unique(QStringLiteral("folios/%1.xml").arg(
safeSegment(QString(uuid).remove(QLatin1Char('{')).remove(QLatin1Char('}')),
QString::number(folio))),
&taken);
parts.append(QetZip::Entry{file, serialize(diagram), true});
root.replaceChild(placeholder(document, file), diagram);
}
//The project's symbols
for (QDomElement collection : childElements(root, QStringLiteral("collection")))
splitCollection(document, collection, QStringLiteral("elements/"), &parts, &taken);
//Title block templates
for (QDomElement templates : childElements(root, QStringLiteral("titleblocktemplates"))) {
for (QDomElement tbt : childElements(templates, QStringLiteral("titleblocktemplate"))) {
const QString file = unique(QStringLiteral("titleblocks/%1.titleblock").arg(
safeSegment(tbt.attribute(QStringLiteral("name")),
QStringLiteral("unnamed"))),
&taken);
parts.append(QetZip::Entry{file, serialize(tbt), true});
templates.replaceChild(placeholder(document, file), tbt);
}
}
QDomDocument manifest;
QDomElement qetz = manifest.createElement(QStringLiteral("qetz"));
qetz.setAttribute(QStringLiteral("format"), Format);
qetz.setAttribute(QStringLiteral("min-reader"), Format);
if (!writer.isEmpty()) qetz.setAttribute(QStringLiteral("writer"), writer);
manifest.appendChild(qetz);
QList<QetZip::Entry> entries;
entries.append(QetZip::Entry{QStringLiteral("mimetype"), QByteArray(MimeType), false});
entries.append(QetZip::Entry{QStringLiteral("manifest.xml"), serialize(qetz), true});
entries.append(QetZip::Entry{QStringLiteral("project.xml"), serialize(root), true});
entries += parts;
for (auto it = pictures.cbegin() ; it != pictures.cend() ; ++it)
entries.append(QetZip::Entry{it.key(), it.value(), false}); //PNG is compressed already
return entries;
}
/**
@brief QetContainer::join
Rebuild in @p project the document split() made @p entries from.
Refused, with @p error set: not a .qetz, made for a later QElectroTech
(the manifest's min-reader), or a part missing or unreadable. Entries
nothing names are ignored, so a later format may add some.
*/
bool QetContainer::join(const QList<QetZip::Entry> &entries, QDomDocument *project, QString *error)
{
QHash<QString, QByteArray> files;
for (const QetZip::Entry &entry : entries)
files.insert(entry.name, entry.data);
if (files.value(QStringLiteral("mimetype")) != QByteArray(MimeType)) {
setError(error, QStringLiteral("not a QElectroTech project (.qetz)"));
return false;
}
QDomDocument manifest;
if (!parse(files.value(QStringLiteral("manifest.xml")), &manifest, error))
return false;
bool ok = false;
const int min_reader = manifest.documentElement().attribute(QStringLiteral("min-reader")).toInt(&ok);
if (!ok || min_reader > Format) {
setError(error, QStringLiteral("made by a later QElectroTech (format %1, this one reads up to %2)")
.arg(manifest.documentElement().attribute(QStringLiteral("format")))
.arg(Format));
return false;
}
if (!parse(files.value(QStringLiteral("project.xml")), project, error))
return false;
//Parts, a snapshot first: the list is live and shrinks as they go
const QDomNodeList list = project->elementsByTagName(PartTag);
QList<QDomElement> placeholders;
for (int i = 0 ; i < list.size() ; ++i) placeholders << list.at(i).toElement();
for (QDomElement part : placeholders)
{
const QString file = part.attribute(FileAttribute);
if (!files.contains(file)) {
setError(error, QStringLiteral("part \"%1\" is missing").arg(file));
return false;
}
QDomDocument document;
QString part_error;
if (!parse(files.value(file), &document, &part_error)) {
setError(error, QStringLiteral("%1: %2").arg(file, part_error));
return false;
}
part.parentNode().replaceChild(project->importNode(document.documentElement(), true), part);
}
//Pictures back to base64, as the image's first child
const QDomNodeList images = project->elementsByTagName(QStringLiteral("image"));
for (int i = 0 ; i < images.size() ; ++i)
{
QDomElement image = images.at(i).toElement();
if (!image.hasAttribute(ImageFileAttribute)) continue;
const QString file = image.attribute(ImageFileAttribute);
if (!files.contains(file)) {
setError(error, QStringLiteral("picture \"%1\" is missing").arg(file));
return false;
}
image.removeAttribute(ImageFileAttribute);
image.insertBefore(project->createTextNode(QString::fromLatin1(files.value(file).toBase64())),
image.firstChild());
}
return true;
}
+59
View File
@@ -0,0 +1,59 @@
/*
Copyright 2006-2026 The QElectroTech Team
This file is part of QElectroTech.
QElectroTech is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
QElectroTech is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with QElectroTech. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef QETCONTAINER_H
#define QETCONTAINER_H
#include "qetzip.h"
#include <QDomDocument>
/**
@brief The QetContainer namespace
Splits a project document into the parts of a .qetz file and joins
them back:
mimetype first, stored
manifest.xml format and oldest format allowed to read it
project.xml the project, each part replaced by
<qetz-part file="..."/>
folios/<uuid>.xml one per folio
elements/.../<name> the project's symbol definitions
titleblocks/<name>.titleblock
images/<sha256>.png folio pictures, once each, not base64
join(split(d)) is d. Nothing here knows what the parts mean: the
project is read and written by the same code as a .qet file.
*/
namespace QetContainer
{
extern const char MimeType[];
/// The format written; files of a later format are refused by join()
/// unless their min-reader allows this one.
constexpr int Format = 1;
QList<QetZip::Entry> split(const QDomDocument &project,
const QString &writer = QString(),
QString *error = nullptr);
bool join(const QList<QetZip::Entry> &entries, QDomDocument *project,
QString *error = nullptr);
/// Parse @p xml as a .qet is read (QETProject::openFile)
bool parse(const QByteArray &xml, QDomDocument *document, QString *error = nullptr);
}
#endif // QETCONTAINER_H
+14
View File
@@ -1173,3 +1173,17 @@ add_executable(
add_test(NAME tst_qetzip COMMAND tst_qetzip)
target_include_directories(tst_qetzip PRIVATE ${QET_DIR}/sources)
target_link_libraries(tst_qetzip PRIVATE Qt::Test)
# A project split into the parts of a .qetz and joined back is the same
# document, on every example (and QET_CONTAINER_CORPUS if set).
add_executable(
tst_qetcontainer
tst_qetcontainer.cpp
${QET_DIR}/sources/container/qetcontainer.cpp
${QET_DIR}/sources/container/qetzip.cpp
${QET_DIR}/sources/container/miniz/miniz.c)
add_test(NAME tst_qetcontainer COMMAND tst_qetcontainer)
target_include_directories(tst_qetcontainer PRIVATE ${QET_DIR}/sources)
target_link_libraries(tst_qetcontainer PRIVATE Qt::Test Qt::Xml)
target_compile_definitions(tst_qetcontainer PRIVATE
"QET_EXAMPLES_DIR=\"${QET_DIR}/examples\"")
+167
View File
@@ -0,0 +1,167 @@
// SPDX-License-Identifier: GPL-2.0-or-later
#include <QtTest>
#include <QDir>
#include <QFile>
#include "container/qetcontainer.h"
// A project document split into the parts of a .qetz, zipped, unzipped and
// joined back is the document it was: every example, and every .qet in the
// folder QET_CONTAINER_CORPUS names, if set. Whitespace-only text between
// elements is indentation and is not compared; elsewhere it is (a
// title-block value of " ", #973).
class tst_qetcontainer : public QObject
{
Q_OBJECT
static QList<QDomNode> children(const QDomNode &node)
{
QList<QDomNode> all;
bool has_element = false;
for (QDomNode c = node.firstChild() ; !c.isNull() ; c = c.nextSibling()) {
all << c;
has_element |= c.isElement();
}
if (!has_element) return all;
QList<QDomNode> kept;
for (const QDomNode &c : all)
if (!(c.isText() && c.nodeValue().trimmed().isEmpty())) kept << c;
return kept;
}
// empty when equal, else where they differ
static QString difference(const QDomNode &a, const QDomNode &b, const QString &path)
{
const QString here = path + QLatin1Char('/') + a.nodeName();
if (a.nodeType() != b.nodeType() || a.nodeName() != b.nodeName())
return here + QStringLiteral(": %1 vs %2").arg(a.nodeName(), b.nodeName());
if (!a.isElement() && !a.isDocument() && a.nodeValue() != b.nodeValue())
return here + QStringLiteral(": value differs");
if (a.isElement()) {
const QDomNamedNodeMap aa = a.attributes(), ba = b.attributes();
if (aa.count() != ba.count())
return here + QStringLiteral(": %1 vs %2 attributes").arg(aa.count()).arg(ba.count());
for (int i = 0 ; i < aa.count() ; ++i) {
const QDomAttr attr = aa.item(i).toAttr();
if (!ba.contains(attr.name()) || ba.namedItem(attr.name()).nodeValue() != attr.value())
return here + QStringLiteral(": attribute %1 differs").arg(attr.name());
}
}
const QList<QDomNode> ac = children(a), bc = children(b);
if (ac.size() != bc.size())
return here + QStringLiteral(": %1 vs %2 children").arg(ac.size()).arg(bc.size());
for (int i = 0 ; i < ac.size() ; ++i) {
const QString d = difference(ac.at(i), bc.at(i), here + QStringLiteral("[%1]").arg(i));
if (!d.isEmpty()) return d;
}
return {};
}
static QByteArray read(const QString &path)
{
QFile f(path);
return f.open(QIODevice::ReadOnly) ? f.readAll() : QByteArray();
}
static QList<QetZip::Entry> splitFile(const QString &path, QDomDocument *original)
{
QString error;
if (!QetContainer::parse(read(path), original, &error)) return {};
return QetContainer::split(*original, QStringLiteral("tst_qetcontainer"), &error);
}
private slots:
void sameDocumentBack_data()
{
QTest::addColumn<QString>("file");
QStringList dirs{QStringLiteral(QET_EXAMPLES_DIR)};
const QString corpus = qEnvironmentVariable("QET_CONTAINER_CORPUS");
if (!corpus.isEmpty()) dirs << corpus;
for (const QString &dir : dirs) {
QDirIterator it(dir, {QStringLiteral("*.qet")}, QDir::Files, QDirIterator::Subdirectories);
QStringList files;
while (it.hasNext()) files << it.next();
files.sort();
for (const QString &f : files)
QTest::newRow(qPrintable(QDir(dir).relativeFilePath(f))) << f;
}
}
void sameDocumentBack()
{
QFETCH(QString, file);
QDomDocument original;
QString error;
if (!QetContainer::parse(read(file), &original, &error))
QSKIP(qPrintable(QStringLiteral("not readable as XML, as QElectroTech finds too: ") + error));
const QList<QetZip::Entry> entries = QetContainer::split(original, QString(), &error);
QVERIFY2(!entries.isEmpty(), qPrintable(error));
QCOMPARE(entries.first().name, QStringLiteral("mimetype"));
QList<QetZip::Entry> unzipped;
QVERIFY2(QetZip::fromBytes(QetZip::toBytes(entries, &error), &unzipped, &error), qPrintable(error));
QDomDocument back;
QVERIFY2(QetContainer::join(unzipped, &back, &error), qPrintable(error));
const QString d = difference(original.documentElement(), back.documentElement(), QString());
QVERIFY2(d.isEmpty(), qPrintable(d));
}
// The folio pictures leave the folios, once each
void picturesAreFiles()
{
QDomDocument original;
const QList<QetZip::Entry> entries = splitFile(
QStringLiteral(QET_EXAMPLES_DIR "/weneedpolonez-Polonez_MR89_wiring_diagram.qet"), &original);
QVERIFY(!entries.isEmpty());
int pictures = 0;
for (const QetZip::Entry &e : entries) {
if (e.name.startsWith(QStringLiteral("images/"))) {
++pictures;
QVERIFY(e.data.startsWith("\x89PNG"));
QVERIFY(!e.compress);
}
if (e.name.startsWith(QStringLiteral("folios/")))
QVERIFY2(!e.data.contains("iVBORw0KGgo"), "a folio still holds a PNG in base64");
}
QVERIFY(pictures > 0);
QVERIFY(pictures < 121); // the 121 placed pictures repeat a few files
}
// A file whose manifest needs a later reader is refused, not half read
void laterFormatRefused()
{
QDomDocument original;
QList<QetZip::Entry> entries = splitFile(QStringLiteral(QET_EXAMPLES_DIR "/perceuse.qet"), &original);
QVERIFY(!entries.isEmpty());
for (QetZip::Entry &e : entries)
if (e.name == QLatin1String("manifest.xml"))
e.data.replace("min-reader=\"1\"", "min-reader=\"9\"");
QDomDocument back;
QString error;
QVERIFY(!QetContainer::join(entries, &back, &error));
QVERIFY2(error.contains(QStringLiteral("later")), qPrintable(error));
}
void missingPartRefused()
{
QDomDocument original;
QList<QetZip::Entry> entries = splitFile(QStringLiteral(QET_EXAMPLES_DIR "/perceuse.qet"), &original);
for (int i = 0 ; i < entries.size() ; ++i)
if (entries.at(i).name.startsWith(QStringLiteral("folios/"))) { entries.removeAt(i); break; }
QDomDocument back;
QString error;
QVERIFY(!QetContainer::join(entries, &back, &error));
QVERIFY2(error.contains(QStringLiteral("missing")), qPrintable(error));
}
void notAProjectRefused()
{
QDomDocument back;
QString error;
QVERIFY(!QetContainer::join({{QStringLiteral("mimetype"), "application/zip", false}}, &back, &error));
}
};
QTEST_APPLESS_MAIN(tst_qetcontainer)
#include "tst_qetcontainer.moc"