Merge master into feat/qetz-db-truth-v2

#1470 keeps a .qetz's pictures out of the document; the index path still
reads them in, as before.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

sources/container/qetcontainer.cpp

@@ -169,13 +169,19 @@ bool QetContainer::isZip(const QByteArray &content)
 	The project document held in @p content: unzipped and joined for a
 	.qetz, parsed for a .qet.
 */
-bool QetContainer::readDocument(const QByteArray &content, QDomDocument *document, QString *error)
+bool QetContainer::readDocument(const QByteArray &content, QDomDocument *document,
+								QString *error, Pictures *pictures)
 {
 	if (!isZip(content))
 		return parse(content, document, error);
 	QList<QetZip::Entry> entries;
 	return QetZip::fromBytes(content, &entries, error)
-			&& join(entries, document, error);
+			&& join(entries, document, error, nullptr, pictures);
+}
+
+QString QetContainer::pictureFile(const QDomElement &image)
+{
+	return image.attribute(ImageFileAttribute);
 }
 
 /**
@@ -394,9 +400,11 @@ bool QetContainer::readFolio(const QHash<QString, QByteArray> &files, const QStr
 	Refused, with @p error set: not a .qetz, made for a later QElectroTech
 	(the manifest's min-reader), or a part missing or unreadable. Entries
 	nothing names are ignored, so a later format may add some.
+	With @p pictures, the pictures stay out of the document: see the
+	namespace.
 */
 bool QetContainer::join(const QList<QetZip::Entry> &entries, QDomDocument *project, QString *error,
-						QStringList *folio_files)
+						QStringList *folio_files, Pictures *pictures)
 {
 	QHash<QString, QByteArray> files;
 	for (const QetZip::Entry &entry : entries)
@@ -451,7 +459,8 @@ bool QetContainer::join(const QList<QetZip::Entry> &entries, QDomDocument *proje
 		part.parentNode().replaceChild(project->importNode(document.documentElement(), true), part);
 	}
 
-		//Pictures back to base64, as the image's first child
+		//Pictures back to base64, as the image's first child, or into
+		//pictures
 	const QDomNodeList images = project->elementsByTagName(QStringLiteral("image"));
 	for (int i = 0 ; i < images.size() ; ++i)
 	{
@@ -462,6 +471,10 @@ bool QetContainer::join(const QList<QetZip::Entry> &entries, QDomDocument *proje
 			setError(error, QStringLiteral("picture \"%1\" is missing").arg(file));
 			return false;
 		}
+		if (pictures) {
+			pictures->insert(file, files.value(file));
+			continue;
+		}
 		image.removeAttribute(ImageFileAttribute);
 		image.insertBefore(project->createTextNode(QString::fromLatin1(files.value(file).toBase64())),
 						   image.firstChild());

sources/container/qetcontainer.h

@@ -43,10 +43,17 @@
 
 	join(split(d)) is d. Nothing here knows what the parts mean: the
 	project is read and written by the same code as a .qet file.
+
+	Given a Pictures map, join() leaves the pictures out of the document:
+	each <image> keeps the name of its file, pictureFile(), and the bytes
+	go in the map. Turning them back into base64 text costs seconds on a
+	project with large pictures, and the folio would only decode it again.
 */
 namespace QetContainer
 {
 	extern const char MimeType[];
+		/// Picture bytes by file name, see join()
+	using Pictures = QHash<QString, QByteArray>;
 		/// The format written; files of a later format are refused by join()
 		/// unless their min-reader allows this one.
 	constexpr int Format = 3;
@@ -57,13 +64,16 @@ namespace QetContainer
 							   const QByteArray &index = QByteArray(),
 							   const QHash<QString, QByteArray> &kept_from = QHash<QString, QByteArray>());
 	bool join(const QList<QetZip::Entry> &entries, QDomDocument *project,
-			  QString *error = nullptr, QStringList *folio_files = nullptr);
+			  QString *error = nullptr, QStringList *folio_files = nullptr,
+			  Pictures *pictures = nullptr);
 		/// A folio part of @p files, which join() left out, with its
 		/// pictures and its engineering data back (stage D2)
 		/// @p reader, if any, reads the engineering data of @p files
 	bool readFolio(const QHash<QString, QByteArray> &files, const QString &file,
 				   QDomDocument *folio, QString *error = nullptr,
 				   QetContainerDb::FolioReader *reader = nullptr);
+		/// The file join() left the picture of @p image in, or an empty string
+	QString pictureFile(const QDomElement &image);
 
 		/// Take what QElectroTech computes out of a folio (format 3)
 	void dropComputed(QDomElement diagram, bool terminal_lists);
@@ -72,7 +82,8 @@ namespace QetContainer
 		/// true if @p content is a zip, so a .qetz rather than a .qet
 	bool isZip(const QByteArray &content);
 		/// The project document in @p content, a .qet or a .qetz
-	bool readDocument(const QByteArray &content, QDomDocument *document, QString *error = nullptr);
+	bool readDocument(const QByteArray &content, QDomDocument *document,
+					  QString *error = nullptr, Pictures *pictures = nullptr);
 }
 
 #endif // QETCONTAINER_H

sources/container/qetzip.cpp

@@ -22,6 +22,7 @@
 #include <QFile>
 #include <QSaveFile>
 #include <QSet>
+#include <QtEndian>
 
 namespace {
 void setError(QString *error, const QString &text)
@@ -44,6 +45,42 @@ bool safeName(const QString &name)
 	}
 	return true;
 }
+
+	//Built with MINIZ_NO_TIME, miniz leaves each entry's DOS date at 0,
+	//day 0 of month 0, which is no date: unzip shows 30 Nov 1979, other
+	//tools 13 Dec 1901. Give every entry 1 Jan 1980 00:00 instead, the
+	//first valid DOS date, so the bytes still do not depend on the time.
+constexpr quint16 EntryDosDate = (1 << 5) | 1;
+
+bool setEntryDates(QByteArray *bytes)
+{
+	char *data = bytes->data();
+	mz_zip_archive zip;
+	mz_zip_zero_struct(&zip);
+	if (!mz_zip_reader_init_mem(&zip, data, size_t(bytes->size()), 0)) return false;
+
+	bool ok = true;
+	const mz_uint count = mz_zip_reader_get_num_files(&zip);
+	for (mz_uint i = 0 ; i < count && ok ; ++i)
+	{
+		mz_zip_archive_file_stat stat;
+		if (!mz_zip_reader_file_stat(&zip, i, &stat)) {
+			ok = false;
+			break;
+		}
+			//date at offset 12 of the local header, 14 of the central one
+		const quint64 local = stat.m_local_header_ofs + 12;
+		const quint64 central = zip.m_central_directory_file_ofs + stat.m_central_dir_ofs + 14;
+		if (local + 2 > quint64(bytes->size()) || central + 2 > quint64(bytes->size())) {
+			ok = false;
+			break;
+		}
+		qToLittleEndian<quint16>(EntryDosDate, data + local);
+		qToLittleEndian<quint16>(EntryDosDate, data + central);
+	}
+	mz_zip_reader_end(&zip);
+	return ok;
+}
 }
 
 /**
@@ -87,8 +124,12 @@ QByteArray QetZip::toBytes(const QList<Entry> &entries, QString *error)
 		mz_zip_writer_end(&zip);
 		return {};
 	}
-	const QByteArray bytes(static_cast<const char *>(buffer), qsizetype(size));
+	QByteArray bytes(static_cast<const char *>(buffer), qsizetype(size));
 	mz_zip_writer_end(&zip);   //frees buffer
+	if (!setEntryDates(&bytes)) {
+		setError(error, QStringLiteral("cannot date the zip entries"));
+		return {};
+	}
 	return bytes;
 }
 

sources/container/qetzip.h

@@ -30,8 +30,8 @@
 	megabytes, not gigabytes -- and written through QSaveFile, so a failed
 	save leaves the previous file as it was.
 
-	Entries carry no timestamp: saving the same project twice gives the
-	same bytes.
+	Every entry is dated 1 Jan 1980 00:00, not the time of saving: saving
+	the same project twice gives the same bytes.
 */
 namespace QetZip
 {

sources/diagram.cpp

@@ -49,6 +49,7 @@
 #include "diagramsortkeys.h"
 #include "itemgroups.h"
 #include "textgrid.h"
+#include "container/qetcontainer.h"
 #include "foliogrid.h"
 #include <QGraphicsView>
 #include <QTextStream>
@@ -2149,7 +2150,11 @@ bool Diagram::fromXml(QDomElement &document,
 												QStringLiteral("images"),
 												QStringLiteral("image"))) {
 		DiagramImageItem *dii = new DiagramImageItem ();
-		dii -> fromXml(image_xml);
+		const QString picture_file = QetContainer::pictureFile(image_xml);
+		if (!picture_file.isEmpty() && project())
+			dii -> fromXml(image_xml, project() -> containerPicture(picture_file));
+		else
+			dii -> fromXml(image_xml);
 		settle_uuid(dii, image_xml, QStringLiteral("image"), added_images.size());
 		ItemGroups::setGroup(dii, ItemGroups::read(image_xml));
 		addItem(dii);

sources/qetgraphicsitem/diagramimageitem.cpp

@@ -1787,6 +1787,17 @@ const QByteArray &DiagramImageItem::encodedPng(const QPixmap &pixmap, QByteArray
 	@return true if successfully loaded.
 */
 bool DiagramImageItem::fromXml(const QDomElement &e)
+{
+	return fromXml(e, QByteArray());
+}
+
+/**
+	@brief DiagramImageItem::fromXml
+	Load this image from xml element e, its picture being @p picture, not
+	the base64 text of @p e, unless @p picture is empty. A .qetz keeps its
+	pictures as files (QetContainer::join()).
+*/
+bool DiagramImageItem::fromXml(const QDomElement &e, const QByteArray &picture)
 {
 	if (e.tagName() != "image") {
 		return (false);
@@ -1796,28 +1807,27 @@ bool DiagramImageItem::fromXml(const QDomElement &e)
 	const QUuid uuid(e.attribute(QStringLiteral("uuid")));
 	if (!uuid.isNull() && uuid != m_uuid) setUuid(uuid);
 
-	QDomNode image_node = e.firstChild();
-	if (!image_node.isText()) {
-		return (false);
+	// From the first text node only, not e.text(): that also collects the
+	// text of every child element, so on a cropped or colour-keyed picture
+	// it carries <image_base>'s bytes too. Decoded once: on a project with
+	// large pictures each decoding costs seconds.
+	QByteArray array = picture;
+	if (array.isEmpty())
+	{
+		QDomNode image_node = e.firstChild();
+		if (!image_node.isText()) {
+			return (false);
+		}
+		array = QByteArray::fromBase64(image_node.toText().data().toLatin1());
 	}
 
-	//load xml image to QByteArray
-	QByteArray array;
-	array = QByteArray::fromBase64(e.text().toLatin1());
-
 	//Set QPixmap from the array
 	QPixmap pixmap;
 	pixmap.loadFromData(array);
 	setPixmap(pixmap);
-	// From the first text node only, not e.text(): that also collects the
-	// text of every child element, so on a cropped or colour-keyed picture
-	// it carries <image_base>'s bytes too. loadFromData() stops at the end
-	// of the first PNG and never notices; a cache written back verbatim
-	// would.
-	const QByteArray ownArray = QByteArray::fromBase64(image_node.toText().data().toLatin1());
-	if (isPng(ownArray) && !pixmap_.isNull())
+	if (isPng(array) && !pixmap_.isNull())
 	{
-		m_png_cache = ownArray;
+		m_png_cache = array;
 		m_png_cache_key = pixmap_.cacheKey();
 	}
 

sources/qetgraphicsitem/diagramimageitem.h

@@ -120,6 +120,7 @@ class DiagramImageItem : public QetGraphicsItem {
 	int type() const override { return Type; }
 	
 	virtual bool fromXml(const QDomElement &);
+	bool fromXml(const QDomElement &, const QByteArray &picture);
 	virtual QDomElement toXml(QDomDocument &) const;
 	QUuid uuid() const {return m_uuid;}
 	void setUuid(const QUuid &uuid) {m_uuid = uuid; emit uuidChanged();}

sources/qetproject.cpp

@@ -283,6 +283,20 @@ QUuid QETProject::uuid() const
 	return m_uuid;
 }
 
+/**
+	@brief QETProject::containerPicture
+	The folio pictures of a .qetz are kept as bytes beside the document,
+	not as base64 text in it (QetContainer::join()), for as long as the
+	project is open: a folio built after opening still finds its pictures.
+	They cost almost no memory, sharing their bytes with the pictures
+	built from them.
+	@return the picture in @p file, or an empty array when there is none
+*/
+QByteArray QETProject::containerPicture(const QString &file) const
+{
+	return m_container_pictures.value(file);
+}
+
 /**
 	@brief QETProject::derivedItemUuid
 	A uuid for an item of this project that was saved without one, the same
@@ -503,9 +517,10 @@ QETProject::ProjectState QETProject::openFile(QFile *file)
 			}
 		}
 	}
-	if (!m_opened_from_index && !QetContainer::readDocument(content, &xml_project, &read_error))
+	if (!m_opened_from_index && !QetContainer::readDocument(content, &xml_project, &read_error, &m_container_pictures))
 	{
 		qWarning().noquote() << "Unable to read" << fi.fileName() << ":" << read_error;
+		m_container_pictures.clear();
 		if(opened_here) {
 			file->close();
 		}

sources/qetproject.h

@@ -121,6 +121,7 @@ class QETProject : public QObject
 		projectDataBase *dataBase();
 		QUuid uuid() const;
 		QUuid derivedItemUuid(const QString &kind, const QString &key);
+		QByteArray containerPicture(const QString &file) const;
 		ProjectState state() const;
 		QList<Diagram *> diagrams() const;
 			//The folios whether their items are built or not
@@ -477,6 +478,9 @@ class QETProject : public QObject
 		static bool m_backup_enabled;
 			/// Something changed since the last backup, see writeBackup()
 		bool m_backup_needed = true;
+			/// The pictures of the .qetz this project was opened from, see
+			/// containerPicture()
+		QHash<QString, QByteArray> m_container_pictures;
 			/// File path this project is saved to
 		QString m_file_path;
 			/// Current state of the project

tests/qttest/tst_qetcontainer.cpp

@@ -289,6 +289,43 @@ private slots:
 		QVERIFY(pictures < 121);   // the 121 placed pictures repeat a few files
 	}
 
+	// Given a map, join() hands the pictures over as bytes instead of
+	// putting them back in the document as base64
+	void picturesKeptAside()
+	{
+		QDomDocument original;
+		QList<QetZip::Entry> entries = splitFile(
+					QStringLiteral(QET_EXAMPLES_DIR "/weneedpolonez-Polonez_MR89_wiring_diagram.qet"), &original);
+		QVERIFY(!entries.isEmpty());
+		QHash<QString, QByteArray> files;
+		for (const QetZip::Entry &e : entries) files.insert(e.name, e.data);
+
+		QDomDocument back;
+		QetContainer::Pictures pictures;
+		QString error;
+		QVERIFY2(QetContainer::join(entries, &back, &error, nullptr, &pictures), qPrintable(error));
+		const QDomNodeList images = back.elementsByTagName(QStringLiteral("image"));
+		const QDomNodeList original_images = original.elementsByTagName(QStringLiteral("image"));
+		QCOMPARE(images.size(), 121);
+		QCOMPARE(images.size(), original_images.size());
+		for (int i = 0 ; i < images.size() ; ++i) {
+			const QDomElement image = images.at(i).toElement();
+			const QString file = QetContainer::pictureFile(image);
+			QVERIFY(!file.isEmpty());
+			QVERIFY(!image.firstChild().isText());
+			QCOMPARE(pictures.value(file), files.value(file));
+			QCOMPARE(pictures.value(file),
+					 QByteArray::fromBase64(original_images.at(i).firstChild().toText().data().toLatin1()));
+		}
+
+		// still refused when a picture is missing
+		for (int i = 0 ; i < entries.size() ; ++i)
+			if (entries.at(i).name.startsWith(QStringLiteral("images/"))) { entries.removeAt(i); break; }
+		QDomDocument broken;
+		QVERIFY(!QetContainer::join(entries, &broken, &error, nullptr, &pictures));
+		QVERIFY2(error.contains(QStringLiteral("missing")), qPrintable(error));
+	}
+
 	// A file whose manifest needs a later reader is refused, not half read
 	void laterFormatRefused()
 	{

tests/qttest/tst_qetzip.cpp

@@ -2,6 +2,7 @@
 #include <QtTest>
 
 #include <QTemporaryDir>
+#include <QtEndian>
 
 #include "container/qetzip.h"
 #include "container/miniz/miniz.h"
@@ -72,6 +73,33 @@ private slots:
 		QCOMPARE(QetZip::toBytes(sample()), QetZip::toBytes(sample()));
 	}
 
+	// every entry carries a valid date in both of its headers, not the
+	// 0 miniz writes without a clock (shown as 1979 or 1901 by unzip tools)
+	void entriesHaveValidDate()
+	{
+		const QByteArray zip = QetZip::toBytes(sample());
+		QVERIFY(!zip.isEmpty());
+		mz_zip_archive archive;
+		mz_zip_zero_struct(&archive);
+		QVERIFY(mz_zip_reader_init_mem(&archive, zip.constData(), size_t(zip.size()), 0));
+		const mz_uint count = mz_zip_reader_get_num_files(&archive);
+		QCOMPARE(int(count), sample().size());
+		for (mz_uint i = 0 ; i < count ; ++i) {
+			mz_zip_archive_file_stat stat;
+			QVERIFY(mz_zip_reader_file_stat(&archive, i, &stat));
+			const char *local = zip.constData() + stat.m_local_header_ofs;
+			const char *central = zip.constData() + archive.m_central_directory_file_ofs + stat.m_central_dir_ofs;
+			QCOMPARE(qFromLittleEndian<quint16>(local), quint16(0x50 | 0x4b << 8));   // "PK"
+			QCOMPARE(qFromLittleEndian<quint16>(central), quint16(0x50 | 0x4b << 8));
+			// 1 Jan 1980: day 1, month 1, year 0 since 1980; time 00:00:00
+			QCOMPARE(qFromLittleEndian<quint16>(local + 12), quint16(0x21));
+			QCOMPARE(qFromLittleEndian<quint16>(central + 14), quint16(0x21));
+			QCOMPARE(qFromLittleEndian<quint16>(local + 10), quint16(0));
+			QCOMPARE(qFromLittleEndian<quint16>(central + 12), quint16(0));
+		}
+		mz_zip_reader_end(&archive);
+	}
+
 	void refusesUnsafeNamesOnWrite_data()
 	{
 		QTest::addColumn<QString>("name");