diff --git a/sources/container/qetcontainer.cpp b/sources/container/qetcontainer.cpp index df69c9e54..4b8f18a14 100644 --- a/sources/container/qetcontainer.cpp +++ b/sources/container/qetcontainer.cpp @@ -169,13 +169,19 @@ bool QetContainer::isZip(const QByteArray &content) The project document held in @p content: unzipped and joined for a .qetz, parsed for a .qet. */ -bool QetContainer::readDocument(const QByteArray &content, QDomDocument *document, QString *error) +bool QetContainer::readDocument(const QByteArray &content, QDomDocument *document, + QString *error, Pictures *pictures) { if (!isZip(content)) return parse(content, document, error); QList entries; return QetZip::fromBytes(content, &entries, error) - && join(entries, document, error); + && join(entries, document, error, nullptr, pictures); +} + +QString QetContainer::pictureFile(const QDomElement &image) +{ + return image.attribute(ImageFileAttribute); } /** @@ -394,9 +400,11 @@ bool QetContainer::readFolio(const QHash &files, const QStr Refused, with @p error set: not a .qetz, made for a later QElectroTech (the manifest's min-reader), or a part missing or unreadable. Entries nothing names are ignored, so a later format may add some. + With @p pictures, the pictures stay out of the document: see the + namespace. */ bool QetContainer::join(const QList &entries, QDomDocument *project, QString *error, - QStringList *folio_files) + QStringList *folio_files, Pictures *pictures) { QHash files; for (const QetZip::Entry &entry : entries) @@ -451,7 +459,8 @@ bool QetContainer::join(const QList &entries, QDomDocument *proje part.parentNode().replaceChild(project->importNode(document.documentElement(), true), part); } - //Pictures back to base64, as the image's first child + //Pictures back to base64, as the image's first child, or into + //pictures const QDomNodeList images = project->elementsByTagName(QStringLiteral("image")); for (int i = 0 ; i < images.size() ; ++i) { @@ -462,6 +471,10 @@ bool QetContainer::join(const QList &entries, QDomDocument *proje setError(error, QStringLiteral("picture \"%1\" is missing").arg(file)); return false; } + if (pictures) { + pictures->insert(file, files.value(file)); + continue; + } image.removeAttribute(ImageFileAttribute); image.insertBefore(project->createTextNode(QString::fromLatin1(files.value(file).toBase64())), image.firstChild()); diff --git a/sources/container/qetcontainer.h b/sources/container/qetcontainer.h index 814ca2595..8daa78424 100644 --- a/sources/container/qetcontainer.h +++ b/sources/container/qetcontainer.h @@ -43,10 +43,17 @@ join(split(d)) is d. Nothing here knows what the parts mean: the project is read and written by the same code as a .qet file. + + Given a Pictures map, join() leaves the pictures out of the document: + each keeps the name of its file, pictureFile(), and the bytes + go in the map. Turning them back into base64 text costs seconds on a + project with large pictures, and the folio would only decode it again. */ namespace QetContainer { extern const char MimeType[]; + /// Picture bytes by file name, see join() + using Pictures = QHash; /// The format written; files of a later format are refused by join() /// unless their min-reader allows this one. constexpr int Format = 3; @@ -57,13 +64,16 @@ namespace QetContainer const QByteArray &index = QByteArray(), const QHash &kept_from = QHash()); bool join(const QList &entries, QDomDocument *project, - QString *error = nullptr, QStringList *folio_files = nullptr); + QString *error = nullptr, QStringList *folio_files = nullptr, + Pictures *pictures = nullptr); /// A folio part of @p files, which join() left out, with its /// pictures and its engineering data back (stage D2) /// @p reader, if any, reads the engineering data of @p files bool readFolio(const QHash &files, const QString &file, QDomDocument *folio, QString *error = nullptr, QetContainerDb::FolioReader *reader = nullptr); + /// The file join() left the picture of @p image in, or an empty string + QString pictureFile(const QDomElement &image); /// Take what QElectroTech computes out of a folio (format 3) void dropComputed(QDomElement diagram, bool terminal_lists); @@ -72,7 +82,8 @@ namespace QetContainer /// true if @p content is a zip, so a .qetz rather than a .qet bool isZip(const QByteArray &content); /// The project document in @p content, a .qet or a .qetz - bool readDocument(const QByteArray &content, QDomDocument *document, QString *error = nullptr); + bool readDocument(const QByteArray &content, QDomDocument *document, + QString *error = nullptr, Pictures *pictures = nullptr); } #endif // QETCONTAINER_H diff --git a/sources/container/qetzip.cpp b/sources/container/qetzip.cpp index 7a0be3c26..dc9e2c9f7 100644 --- a/sources/container/qetzip.cpp +++ b/sources/container/qetzip.cpp @@ -22,6 +22,7 @@ #include #include #include +#include namespace { void setError(QString *error, const QString &text) @@ -44,6 +45,42 @@ bool safeName(const QString &name) } return true; } + + //Built with MINIZ_NO_TIME, miniz leaves each entry's DOS date at 0, + //day 0 of month 0, which is no date: unzip shows 30 Nov 1979, other + //tools 13 Dec 1901. Give every entry 1 Jan 1980 00:00 instead, the + //first valid DOS date, so the bytes still do not depend on the time. +constexpr quint16 EntryDosDate = (1 << 5) | 1; + +bool setEntryDates(QByteArray *bytes) +{ + char *data = bytes->data(); + mz_zip_archive zip; + mz_zip_zero_struct(&zip); + if (!mz_zip_reader_init_mem(&zip, data, size_t(bytes->size()), 0)) return false; + + bool ok = true; + const mz_uint count = mz_zip_reader_get_num_files(&zip); + for (mz_uint i = 0 ; i < count && ok ; ++i) + { + mz_zip_archive_file_stat stat; + if (!mz_zip_reader_file_stat(&zip, i, &stat)) { + ok = false; + break; + } + //date at offset 12 of the local header, 14 of the central one + const quint64 local = stat.m_local_header_ofs + 12; + const quint64 central = zip.m_central_directory_file_ofs + stat.m_central_dir_ofs + 14; + if (local + 2 > quint64(bytes->size()) || central + 2 > quint64(bytes->size())) { + ok = false; + break; + } + qToLittleEndian(EntryDosDate, data + local); + qToLittleEndian(EntryDosDate, data + central); + } + mz_zip_reader_end(&zip); + return ok; +} } /** @@ -87,8 +124,12 @@ QByteArray QetZip::toBytes(const QList &entries, QString *error) mz_zip_writer_end(&zip); return {}; } - const QByteArray bytes(static_cast(buffer), qsizetype(size)); + QByteArray bytes(static_cast(buffer), qsizetype(size)); mz_zip_writer_end(&zip); //frees buffer + if (!setEntryDates(&bytes)) { + setError(error, QStringLiteral("cannot date the zip entries")); + return {}; + } return bytes; } diff --git a/sources/container/qetzip.h b/sources/container/qetzip.h index 9238f6ebf..808a861ce 100644 --- a/sources/container/qetzip.h +++ b/sources/container/qetzip.h @@ -30,8 +30,8 @@ megabytes, not gigabytes -- and written through QSaveFile, so a failed save leaves the previous file as it was. - Entries carry no timestamp: saving the same project twice gives the - same bytes. + Every entry is dated 1 Jan 1980 00:00, not the time of saving: saving + the same project twice gives the same bytes. */ namespace QetZip { diff --git a/sources/diagram.cpp b/sources/diagram.cpp index c47406143..483f2dd62 100644 --- a/sources/diagram.cpp +++ b/sources/diagram.cpp @@ -49,6 +49,7 @@ #include "diagramsortkeys.h" #include "itemgroups.h" #include "textgrid.h" +#include "container/qetcontainer.h" #include "foliogrid.h" #include #include @@ -2149,7 +2150,11 @@ bool Diagram::fromXml(QDomElement &document, QStringLiteral("images"), QStringLiteral("image"))) { DiagramImageItem *dii = new DiagramImageItem (); - dii -> fromXml(image_xml); + const QString picture_file = QetContainer::pictureFile(image_xml); + if (!picture_file.isEmpty() && project()) + dii -> fromXml(image_xml, project() -> containerPicture(picture_file)); + else + dii -> fromXml(image_xml); settle_uuid(dii, image_xml, QStringLiteral("image"), added_images.size()); ItemGroups::setGroup(dii, ItemGroups::read(image_xml)); addItem(dii); diff --git a/sources/qetgraphicsitem/diagramimageitem.cpp b/sources/qetgraphicsitem/diagramimageitem.cpp index a952c4851..4c5f05145 100644 --- a/sources/qetgraphicsitem/diagramimageitem.cpp +++ b/sources/qetgraphicsitem/diagramimageitem.cpp @@ -1787,6 +1787,17 @@ const QByteArray &DiagramImageItem::encodedPng(const QPixmap &pixmap, QByteArray @return true if successfully loaded. */ bool DiagramImageItem::fromXml(const QDomElement &e) +{ + return fromXml(e, QByteArray()); +} + +/** + @brief DiagramImageItem::fromXml + Load this image from xml element e, its picture being @p picture, not + the base64 text of @p e, unless @p picture is empty. A .qetz keeps its + pictures as files (QetContainer::join()). +*/ +bool DiagramImageItem::fromXml(const QDomElement &e, const QByteArray &picture) { if (e.tagName() != "image") { return (false); @@ -1796,28 +1807,27 @@ bool DiagramImageItem::fromXml(const QDomElement &e) const QUuid uuid(e.attribute(QStringLiteral("uuid"))); if (!uuid.isNull() && uuid != m_uuid) setUuid(uuid); - QDomNode image_node = e.firstChild(); - if (!image_node.isText()) { - return (false); + // From the first text node only, not e.text(): that also collects the + // text of every child element, so on a cropped or colour-keyed picture + // it carries 's bytes too. Decoded once: on a project with + // large pictures each decoding costs seconds. + QByteArray array = picture; + if (array.isEmpty()) + { + QDomNode image_node = e.firstChild(); + if (!image_node.isText()) { + return (false); + } + array = QByteArray::fromBase64(image_node.toText().data().toLatin1()); } - //load xml image to QByteArray - QByteArray array; - array = QByteArray::fromBase64(e.text().toLatin1()); - //Set QPixmap from the array QPixmap pixmap; pixmap.loadFromData(array); setPixmap(pixmap); - // From the first text node only, not e.text(): that also collects the - // text of every child element, so on a cropped or colour-keyed picture - // it carries 's bytes too. loadFromData() stops at the end - // of the first PNG and never notices; a cache written back verbatim - // would. - const QByteArray ownArray = QByteArray::fromBase64(image_node.toText().data().toLatin1()); - if (isPng(ownArray) && !pixmap_.isNull()) + if (isPng(array) && !pixmap_.isNull()) { - m_png_cache = ownArray; + m_png_cache = array; m_png_cache_key = pixmap_.cacheKey(); } diff --git a/sources/qetgraphicsitem/diagramimageitem.h b/sources/qetgraphicsitem/diagramimageitem.h index 6e0f308f8..df8f77b1a 100644 --- a/sources/qetgraphicsitem/diagramimageitem.h +++ b/sources/qetgraphicsitem/diagramimageitem.h @@ -120,6 +120,7 @@ class DiagramImageItem : public QetGraphicsItem { int type() const override { return Type; } virtual bool fromXml(const QDomElement &); + bool fromXml(const QDomElement &, const QByteArray &picture); virtual QDomElement toXml(QDomDocument &) const; QUuid uuid() const {return m_uuid;} void setUuid(const QUuid &uuid) {m_uuid = uuid; emit uuidChanged();} diff --git a/sources/qetproject.cpp b/sources/qetproject.cpp index bd75b4eeb..31c603181 100644 --- a/sources/qetproject.cpp +++ b/sources/qetproject.cpp @@ -283,6 +283,20 @@ QUuid QETProject::uuid() const return m_uuid; } +/** + @brief QETProject::containerPicture + The folio pictures of a .qetz are kept as bytes beside the document, + not as base64 text in it (QetContainer::join()), for as long as the + project is open: a folio built after opening still finds its pictures. + They cost almost no memory, sharing their bytes with the pictures + built from them. + @return the picture in @p file, or an empty array when there is none +*/ +QByteArray QETProject::containerPicture(const QString &file) const +{ + return m_container_pictures.value(file); +} + /** @brief QETProject::derivedItemUuid A uuid for an item of this project that was saved without one, the same @@ -503,9 +517,10 @@ QETProject::ProjectState QETProject::openFile(QFile *file) } } } - if (!m_opened_from_index && !QetContainer::readDocument(content, &xml_project, &read_error)) + if (!m_opened_from_index && !QetContainer::readDocument(content, &xml_project, &read_error, &m_container_pictures)) { qWarning().noquote() << "Unable to read" << fi.fileName() << ":" << read_error; + m_container_pictures.clear(); if(opened_here) { file->close(); } diff --git a/sources/qetproject.h b/sources/qetproject.h index cd8f12335..a76a5d60f 100644 --- a/sources/qetproject.h +++ b/sources/qetproject.h @@ -121,6 +121,7 @@ class QETProject : public QObject projectDataBase *dataBase(); QUuid uuid() const; QUuid derivedItemUuid(const QString &kind, const QString &key); + QByteArray containerPicture(const QString &file) const; ProjectState state() const; QList diagrams() const; //The folios whether their items are built or not @@ -477,6 +478,9 @@ class QETProject : public QObject static bool m_backup_enabled; /// Something changed since the last backup, see writeBackup() bool m_backup_needed = true; + /// The pictures of the .qetz this project was opened from, see + /// containerPicture() + QHash m_container_pictures; /// File path this project is saved to QString m_file_path; /// Current state of the project diff --git a/tests/qttest/tst_qetcontainer.cpp b/tests/qttest/tst_qetcontainer.cpp index 592d011e6..93676ead3 100644 --- a/tests/qttest/tst_qetcontainer.cpp +++ b/tests/qttest/tst_qetcontainer.cpp @@ -289,6 +289,43 @@ private slots: QVERIFY(pictures < 121); // the 121 placed pictures repeat a few files } + // Given a map, join() hands the pictures over as bytes instead of + // putting them back in the document as base64 + void picturesKeptAside() + { + QDomDocument original; + QList entries = splitFile( + QStringLiteral(QET_EXAMPLES_DIR "/weneedpolonez-Polonez_MR89_wiring_diagram.qet"), &original); + QVERIFY(!entries.isEmpty()); + QHash files; + for (const QetZip::Entry &e : entries) files.insert(e.name, e.data); + + QDomDocument back; + QetContainer::Pictures pictures; + QString error; + QVERIFY2(QetContainer::join(entries, &back, &error, nullptr, &pictures), qPrintable(error)); + const QDomNodeList images = back.elementsByTagName(QStringLiteral("image")); + const QDomNodeList original_images = original.elementsByTagName(QStringLiteral("image")); + QCOMPARE(images.size(), 121); + QCOMPARE(images.size(), original_images.size()); + for (int i = 0 ; i < images.size() ; ++i) { + const QDomElement image = images.at(i).toElement(); + const QString file = QetContainer::pictureFile(image); + QVERIFY(!file.isEmpty()); + QVERIFY(!image.firstChild().isText()); + QCOMPARE(pictures.value(file), files.value(file)); + QCOMPARE(pictures.value(file), + QByteArray::fromBase64(original_images.at(i).firstChild().toText().data().toLatin1())); + } + + // still refused when a picture is missing + for (int i = 0 ; i < entries.size() ; ++i) + if (entries.at(i).name.startsWith(QStringLiteral("images/"))) { entries.removeAt(i); break; } + QDomDocument broken; + QVERIFY(!QetContainer::join(entries, &broken, &error, nullptr, &pictures)); + QVERIFY2(error.contains(QStringLiteral("missing")), qPrintable(error)); + } + // A file whose manifest needs a later reader is refused, not half read void laterFormatRefused() { diff --git a/tests/qttest/tst_qetzip.cpp b/tests/qttest/tst_qetzip.cpp index 4c69afef3..156cb8fb6 100644 --- a/tests/qttest/tst_qetzip.cpp +++ b/tests/qttest/tst_qetzip.cpp @@ -2,6 +2,7 @@ #include #include +#include #include "container/qetzip.h" #include "container/miniz/miniz.h" @@ -72,6 +73,33 @@ private slots: QCOMPARE(QetZip::toBytes(sample()), QetZip::toBytes(sample())); } + // every entry carries a valid date in both of its headers, not the + // 0 miniz writes without a clock (shown as 1979 or 1901 by unzip tools) + void entriesHaveValidDate() + { + const QByteArray zip = QetZip::toBytes(sample()); + QVERIFY(!zip.isEmpty()); + mz_zip_archive archive; + mz_zip_zero_struct(&archive); + QVERIFY(mz_zip_reader_init_mem(&archive, zip.constData(), size_t(zip.size()), 0)); + const mz_uint count = mz_zip_reader_get_num_files(&archive); + QCOMPARE(int(count), sample().size()); + for (mz_uint i = 0 ; i < count ; ++i) { + mz_zip_archive_file_stat stat; + QVERIFY(mz_zip_reader_file_stat(&archive, i, &stat)); + const char *local = zip.constData() + stat.m_local_header_ofs; + const char *central = zip.constData() + archive.m_central_directory_file_ofs + stat.m_central_dir_ofs; + QCOMPARE(qFromLittleEndian(local), quint16(0x50 | 0x4b << 8)); // "PK" + QCOMPARE(qFromLittleEndian(central), quint16(0x50 | 0x4b << 8)); + // 1 Jan 1980: day 1, month 1, year 0 since 1980; time 00:00:00 + QCOMPARE(qFromLittleEndian(local + 12), quint16(0x21)); + QCOMPARE(qFromLittleEndian(central + 14), quint16(0x21)); + QCOMPARE(qFromLittleEndian(local + 10), quint16(0)); + QCOMPARE(qFromLittleEndian(central + 12), quint16(0)); + } + mz_zip_reader_end(&archive); + } + void refusesUnsafeNamesOnWrite_data() { QTest::addColumn("name");