Merge pull request #1471 from ispyisail/fix/lazy-refused-opens-eager

Fix lazy folios opening older projects slower than without them

sources/bordertitleblock.cpp

@@ -321,8 +321,8 @@ TitleBlockProperties BorderTitleBlock::localTitleBlock() const
 /**
 	@brief BorderTitleBlock::current
 	@return the properties of the titleblock: for a folio placed in a
-	project the project's store holds them (DB-ACCESSORS-PLAN.md stage
-	4.4), for any other this object's own copy.
+	project the project's store holds them, for any other this object's
+	own copy.
 */
 TitleBlockProperties BorderTitleBlock::current() const
 {

sources/bordertitleblock.h

@@ -110,8 +110,8 @@ class BorderTitleBlock : public QObject
 		QRectF insideBorderRect() const;
 	
 		// methods to get title block basic data. A folio placed in a
-		// project reads them from the project's store
-		// (DB-ACCESSORS-PLAN.md stage 4.4), any other from its own copy.
+		// project reads them from the project's store, any other from its
+		// own copy.
 		/// @return the value of the title block "Author" field
 		QString author() const { const TitleBlockProperties *s = stored(); return(s ? s->author : btb_author_); }
 		/// @return the value of the title block "Date" field

sources/container/qetcontainer.cpp

@@ -155,13 +155,19 @@ bool QetContainer::isZip(const QByteArray &content)
 	The project document held in @p content: unzipped and joined for a
 	.qetz, parsed for a .qet.
 */
-bool QetContainer::readDocument(const QByteArray &content, QDomDocument *document, QString *error)
+bool QetContainer::readDocument(const QByteArray &content, QDomDocument *document,
+								QString *error, Pictures *pictures)
 {
 	if (!isZip(content))
 		return parse(content, document, error);
 	QList<QetZip::Entry> entries;
 	return QetZip::fromBytes(content, &entries, error)
-			&& join(entries, document, error);
+			&& join(entries, document, error, pictures);
+}
+
+QString QetContainer::pictureFile(const QDomElement &image)
+{
+	return image.attribute(ImageFileAttribute);
 }
 
 /**
@@ -264,8 +270,11 @@ QList<QetZip::Entry> QetContainer::split(const QDomDocument &project, const QStr
 	Refused, with @p error set: not a .qetz, made for a later QElectroTech
 	(the manifest's min-reader), or a part missing or unreadable. Entries
 	nothing names are ignored, so a later format may add some.
+	With @p pictures, the pictures stay out of the document: see the
+	namespace.
 */
-bool QetContainer::join(const QList<QetZip::Entry> &entries, QDomDocument *project, QString *error)
+bool QetContainer::join(const QList<QetZip::Entry> &entries, QDomDocument *project, QString *error,
+						Pictures *pictures)
 {
 	QHash<QString, QByteArray> files;
 	for (const QetZip::Entry &entry : entries)
@@ -310,7 +319,8 @@ bool QetContainer::join(const QList<QetZip::Entry> &entries, QDomDocument *proje
 		part.parentNode().replaceChild(project->importNode(document.documentElement(), true), part);
 	}
 
-		//Pictures back to base64, as the image's first child
+		//Pictures back to base64, as the image's first child, or into
+		//pictures
 	const QDomNodeList images = project->elementsByTagName(QStringLiteral("image"));
 	for (int i = 0 ; i < images.size() ; ++i)
 	{
@@ -321,6 +331,10 @@ bool QetContainer::join(const QList<QetZip::Entry> &entries, QDomDocument *proje
 			setError(error, QStringLiteral("picture \"%1\" is missing").arg(file));
 			return false;
 		}
+		if (pictures) {
+			pictures->insert(file, files.value(file));
+			continue;
+		}
 		image.removeAttribute(ImageFileAttribute);
 		image.insertBefore(project->createTextNode(QString::fromLatin1(files.value(file).toBase64())),
 						   image.firstChild());

sources/container/qetcontainer.h

@@ -21,6 +21,7 @@
 #include "qetzip.h"
 
 #include <QDomDocument>
+#include <QHash>
 
 /**
 	@brief The QetContainer namespace
@@ -41,10 +42,17 @@
 
 	join(split(d)) is d. Nothing here knows what the parts mean: the
 	project is read and written by the same code as a .qet file.
+
+	Given a Pictures map, join() leaves the pictures out of the document:
+	each <image> keeps the name of its file, pictureFile(), and the bytes
+	go in the map. Turning them back into base64 text costs seconds on a
+	project with large pictures, and the folio would only decode it again.
 */
 namespace QetContainer
 {
 	extern const char MimeType[];
+		/// Picture bytes by file name, see join()
+	using Pictures = QHash<QString, QByteArray>;
 		/// The format written; files of a later format are refused by join()
 		/// unless their min-reader allows this one.
 	constexpr int Format = 2;
@@ -53,14 +61,17 @@ namespace QetContainer
 							   const QString &writer = QString(),
 							   QString *error = nullptr);
 	bool join(const QList<QetZip::Entry> &entries, QDomDocument *project,
-			  QString *error = nullptr);
+			  QString *error = nullptr, Pictures *pictures = nullptr);
+		/// The file join() left the picture of @p image in, or an empty string
+	QString pictureFile(const QDomElement &image);
 
 		/// Parse @p xml as a .qet is read (QETProject::openFile)
 	bool parse(const QByteArray &xml, QDomDocument *document, QString *error = nullptr);
 		/// true if @p content is a zip, so a .qetz rather than a .qet
 	bool isZip(const QByteArray &content);
 		/// The project document in @p content, a .qet or a .qetz
-	bool readDocument(const QByteArray &content, QDomDocument *document, QString *error = nullptr);
+	bool readDocument(const QByteArray &content, QDomDocument *document,
+					  QString *error = nullptr, Pictures *pictures = nullptr);
 }
 
 #endif // QETCONTAINER_H

sources/container/qetzip.cpp

@@ -22,6 +22,7 @@
 #include <QFile>
 #include <QSaveFile>
 #include <QSet>
+#include <QtEndian>
 
 namespace {
 void setError(QString *error, const QString &text)
@@ -44,6 +45,42 @@ bool safeName(const QString &name)
 	}
 	return true;
 }
+
+	//Built with MINIZ_NO_TIME, miniz leaves each entry's DOS date at 0,
+	//day 0 of month 0, which is no date: unzip shows 30 Nov 1979, other
+	//tools 13 Dec 1901. Give every entry 1 Jan 1980 00:00 instead, the
+	//first valid DOS date, so the bytes still do not depend on the time.
+constexpr quint16 EntryDosDate = (1 << 5) | 1;
+
+bool setEntryDates(QByteArray *bytes)
+{
+	char *data = bytes->data();
+	mz_zip_archive zip;
+	mz_zip_zero_struct(&zip);
+	if (!mz_zip_reader_init_mem(&zip, data, size_t(bytes->size()), 0)) return false;
+
+	bool ok = true;
+	const mz_uint count = mz_zip_reader_get_num_files(&zip);
+	for (mz_uint i = 0 ; i < count && ok ; ++i)
+	{
+		mz_zip_archive_file_stat stat;
+		if (!mz_zip_reader_file_stat(&zip, i, &stat)) {
+			ok = false;
+			break;
+		}
+			//date at offset 12 of the local header, 14 of the central one
+		const quint64 local = stat.m_local_header_ofs + 12;
+		const quint64 central = zip.m_central_directory_file_ofs + stat.m_central_dir_ofs + 14;
+		if (local + 2 > quint64(bytes->size()) || central + 2 > quint64(bytes->size())) {
+			ok = false;
+			break;
+		}
+		qToLittleEndian<quint16>(EntryDosDate, data + local);
+		qToLittleEndian<quint16>(EntryDosDate, data + central);
+	}
+	mz_zip_reader_end(&zip);
+	return ok;
+}
 }
 
 /**
@@ -87,8 +124,12 @@ QByteArray QetZip::toBytes(const QList<Entry> &entries, QString *error)
 		mz_zip_writer_end(&zip);
 		return {};
 	}
-	const QByteArray bytes(static_cast<const char *>(buffer), qsizetype(size));
+	QByteArray bytes(static_cast<const char *>(buffer), qsizetype(size));
 	mz_zip_writer_end(&zip);   //frees buffer
+	if (!setEntryDates(&bytes)) {
+		setError(error, QStringLiteral("cannot date the zip entries"));
+		return {};
+	}
 	return bytes;
 }
 

sources/container/qetzip.h

@@ -30,8 +30,8 @@
 	megabytes, not gigabytes -- and written through QSaveFile, so a failed
 	save leaves the previous file as it was.
 
-	Entries carry no timestamp: saving the same project twice gives the
-	same bytes.
+	Every entry is dated 1 Jan 1980 00:00, not the time of saving: saving
+	the same project twice gives the same bytes.
 */
 namespace QetZip
 {

sources/dataBase/projectdatabase.cpp

@@ -253,6 +253,58 @@ void projectDataBase::updateDB(const QDomDocument &document)
 	updateDB();
 }
 
+/**
+	@brief projectDataBase::documentRefusal
+	@return why populateFromDocument() will refuse @p document, for the
+	reasons it gives that the document alone can tell -- a folio, symbol
+	or wire with no saved uuid, two symbols sharing one, a folio number
+	using %autonum -- or an empty string. The other reasons need the
+	project's definitions or every folio's links; a document refused for
+	one of those is still filled from the folios, only later.
+*/
+QString projectDataBase::documentRefusal(const QDomDocument &document)
+{
+	if (qEnvironmentVariableIntValue("QET_DATABASE_FROM_FOLIOS") == 1) {
+		return QStringLiteral("QET_DATABASE_FROM_FOLIOS is set");
+	}
+	QSet<QUuid> element_uuids;
+	const QDomNodeList diagram_nodes = document.elementsByTagName(QStringLiteral("diagram"));
+	for (int i = 0 ; i < diagram_nodes.size() ; ++i)
+	{
+		QDomElement diagram_xml = diagram_nodes.at(i).toElement();
+		if (QUuid(diagram_xml.attribute(QStringLiteral("uuid"))).isNull()) {
+			return QStringLiteral("a folio has no saved uuid");
+		}
+		if (diagram_xml.attribute(QStringLiteral("folio")).contains(QStringLiteral("%autonum"))) {
+			return QStringLiteral("a folio number uses %autonum");
+		}
+		for (QDomElement element_xml : QET::findInDomElement(
+				 diagram_xml, QStringLiteral("elements"), QStringLiteral("element")))
+		{
+				//Skipped by Diagram::fromXml() as well
+			if (!Element::valideXml(element_xml)) {
+				continue;
+			}
+			const QUuid uuid(element_xml.attribute(QStringLiteral("uuid")));
+			if (uuid.isNull() || element_uuids.contains(uuid)) {
+				return QStringLiteral("an element has no saved uuid, or shares one");
+			}
+			element_uuids.insert(uuid);
+		}
+		for (QDomElement conductor_xml : QET::findInDomElement(
+				 diagram_xml, QStringLiteral("conductors"), QStringLiteral("conductor")))
+		{
+			if (Conductor::valideXml(conductor_xml)
+				&& (QUuid(conductor_xml.attribute(QStringLiteral("uuid"))).isNull()
+					|| !conductor_xml.hasAttribute(QStringLiteral("element1"))
+					|| !conductor_xml.hasAttribute(QStringLiteral("element2")))) {
+				return QStringLiteral("a conductor has no saved uuid, or names its ends the older way");
+			}
+		}
+	}
+	return QString();
+}
+
 namespace {
 struct DocumentTerminal
 {
@@ -2445,7 +2497,7 @@ void projectDataBase::populateElementTable()
 /**
 	@brief projectDataBase::populateElementInfoTable
 	Populate the element info table from the symbol information store
-	(DB-ACCESSORS-PLAN.md stage 4.3), in the order of the element table.
+	in the order of the element table.
 	The label column is the label a symbol shows, which its folio's
 	numbering decides: the symbol holding the uuid gives it.
 

sources/dataBase/projectdatabase.h

@@ -67,6 +67,14 @@ class projectDataBase : public QObject
 				populateFromDocument()), from the built folios otherwise.
 			*/
 		void updateDB(const QDomDocument &document);
+			/**
+				Why updateDB(document) will fill the tables from the folios
+				rather than from @p document, for the reasons the document
+				alone can tell; empty when it knows of none. Read before the
+				folios are: a project whose tables will come from the folios
+				builds every folio as it opens, even with QET_LAZY_FOLIOS.
+			*/
+		static QString documentRefusal(const QDomDocument &document);
 			/**
 				Suppress the full rebuild performed by updateDB().
 
@@ -106,7 +114,7 @@ class projectDataBase : public QObject
 
 			//The information of every symbol placed in the project, by
 			//symbol uuid: the store each symbol's information is written
-			//through (DB-ACCESSORS-PLAN.md stage 4.1). A symbol not placed
+			//through. A symbol not placed
 			//(a preview, the symbol editor) keeps its own.
 		DiagramContext elementInformation(const QUuid &element) const;
 		bool hasElementInformation(const QUuid &element) const;
@@ -124,7 +132,7 @@ class projectDataBase : public QObject
 
 			//The title block properties of every folio of the project, by
 			//folio uuid: the store each folio's title block is written
-			//through (DB-ACCESSORS-PLAN.md stage 4.4). A folio not in a
+			//through. A folio not in a
 			//project keeps its own.
 		TitleBlockProperties folioTitleBlock(const QUuid &folio) const;
 		const TitleBlockProperties *storedFolioTitleBlock(const QUuid &folio) const;
@@ -136,7 +144,7 @@ class projectDataBase : public QObject
 
 			//The properties of every wire placed in the project, by wire
 			//uuid: the store each wire's properties are written through
-			//(DB-ACCESSORS-PLAN.md stage 4.5). A wire not placed keeps its own.
+			//A wire not placed keeps its own.
 		ConductorProperties conductorProperties(const QUuid &conductor) const;
 		bool hasConductorProperties(const QUuid &conductor) const;
 		int placedConductorCount(const QUuid &conductor) const;
@@ -145,7 +153,7 @@ class projectDataBase : public QObject
 		QStringList conductorPropertiesMismatches() const;
 
 			//Fill the three stores from a project's document before its
-			//folios are built (DB-ACCESSORS-PLAN.md stage 4.6), then, once
+			//folios are built, then, once
 			//they are, say how many of those values building agreed with.
 		void prefillFromDocument(const QDomDocument &document);
 		void endPrefill(bool keep_unplaced = false);

sources/diagram.cpp

@@ -49,6 +49,7 @@
 #include "diagramsortkeys.h"
 #include "itemgroups.h"
 #include "textgrid.h"
+#include "container/qetcontainer.h"
 #include "foliogrid.h"
 #include <QGraphicsView>
 #include <QTextStream>
@@ -2010,7 +2011,11 @@ bool Diagram::fromXml(QDomElement &document,
 												QStringLiteral("images"),
 												QStringLiteral("image"))) {
 		DiagramImageItem *dii = new DiagramImageItem ();
-		dii -> fromXml(image_xml);
+		const QString picture_file = QetContainer::pictureFile(image_xml);
+		if (!picture_file.isEmpty() && project())
+			dii -> fromXml(image_xml, project() -> containerPicture(picture_file));
+		else
+			dii -> fromXml(image_xml);
 		settle_uuid(dii, image_xml, QStringLiteral("image"), added_images.size());
 		ItemGroups::setGroup(dii, ItemGroups::read(image_xml));
 		addItem(dii);

sources/diagram.h

@@ -211,7 +211,7 @@ class Diagram : public QGraphicsScene
 				 bool = true,
 				 DiagramContent * = nullptr);
 			/// false while this folio's items wait in its XML to be
-			/// built (QET_LAZY_FOLIOS, LAZY-FOLIO-PLAN.md)
+			/// built (QET_LAZY_FOLIOS)
 		bool isLoaded() const {return m_deferred.isNull();}
 		QDomElement deferredXml() const {return m_deferred;}
 		QString headerXml();

sources/qetgraphicsitem/conductor.cpp

@@ -1857,7 +1857,7 @@ ConductorProperties Conductor::properties() const
 /**
 	@brief Conductor::placedStore
 	@return the project database holding this wire's properties
-	(DB-ACCESSORS-PLAN.md stage 4.5), or nullptr if it is not placed in a
+	or nullptr if it is not placed in a
 	project, or shares its uuid with another wire for the moment a copy
 	takes to get its own
 */

sources/qetgraphicsitem/diagramimageitem.cpp

@@ -1787,6 +1787,17 @@ const QByteArray &DiagramImageItem::encodedPng(const QPixmap &pixmap, QByteArray
 	@return true if successfully loaded.
 */
 bool DiagramImageItem::fromXml(const QDomElement &e)
+{
+	return fromXml(e, QByteArray());
+}
+
+/**
+	@brief DiagramImageItem::fromXml
+	Load this image from xml element e, its picture being @p picture, not
+	the base64 text of @p e, unless @p picture is empty. A .qetz keeps its
+	pictures as files (QetContainer::join()).
+*/
+bool DiagramImageItem::fromXml(const QDomElement &e, const QByteArray &picture)
 {
 	if (e.tagName() != "image") {
 		return (false);
@@ -1796,28 +1807,27 @@ bool DiagramImageItem::fromXml(const QDomElement &e)
 	const QUuid uuid(e.attribute(QStringLiteral("uuid")));
 	if (!uuid.isNull() && uuid != m_uuid) setUuid(uuid);
 
-	QDomNode image_node = e.firstChild();
-	if (!image_node.isText()) {
-		return (false);
+	// From the first text node only, not e.text(): that also collects the
+	// text of every child element, so on a cropped or colour-keyed picture
+	// it carries <image_base>'s bytes too. Decoded once: on a project with
+	// large pictures each decoding costs seconds.
+	QByteArray array = picture;
+	if (array.isEmpty())
+	{
+		QDomNode image_node = e.firstChild();
+		if (!image_node.isText()) {
+			return (false);
+		}
+		array = QByteArray::fromBase64(image_node.toText().data().toLatin1());
 	}
 
-	//load xml image to QByteArray
-	QByteArray array;
-	array = QByteArray::fromBase64(e.text().toLatin1());
-
 	//Set QPixmap from the array
 	QPixmap pixmap;
 	pixmap.loadFromData(array);
 	setPixmap(pixmap);
-	// From the first text node only, not e.text(): that also collects the
-	// text of every child element, so on a cropped or colour-keyed picture
-	// it carries <image_base>'s bytes too. loadFromData() stops at the end
-	// of the first PNG and never notices; a cache written back verbatim
-	// would.
-	const QByteArray ownArray = QByteArray::fromBase64(image_node.toText().data().toLatin1());
-	if (isPng(ownArray) && !pixmap_.isNull())
+	if (isPng(array) && !pixmap_.isNull())
 	{
-		m_png_cache = ownArray;
+		m_png_cache = array;
 		m_png_cache_key = pixmap_.cacheKey();
 	}
 

sources/qetgraphicsitem/diagramimageitem.h

@@ -120,6 +120,7 @@ class DiagramImageItem : public QetGraphicsItem {
 	int type() const override { return Type; }
 	
 	virtual bool fromXml(const QDomElement &);
+	bool fromXml(const QDomElement &, const QByteArray &picture);
 	virtual QDomElement toXml(QDomDocument &) const;
 	QUuid uuid() const {return m_uuid;}
 	void setUuid(const QUuid &uuid) {m_uuid = uuid; emit uuidChanged();}

sources/qetgraphicsitem/element.cpp

@@ -1859,7 +1859,7 @@ void Element::setUuid(const QUuid &uuid)
 /**
 	@brief Element::storeInformation
 	Write this symbol's information to its project's store, once it is
-	placed (DB-ACCESSORS-PLAN.md stage 4.1): every change of a placed
+	placed: every change of a placed
 	symbol's information comes through here.
 */
 void Element::storeInformation()
@@ -1925,7 +1925,7 @@ ElementData Element::elementData() const
 /**
 	@brief Element::elementInformations
 	@return this symbol's information: for a placed symbol the project's
-	store holds it (DB-ACCESSORS-PLAN.md stage 4.2), for one not placed
+	store holds it, for one not placed
 	(a preview, the symbol editor) its own copy.
 */
 DiagramContext Element::elementInformations() const

sources/qetproject.cpp

@@ -278,6 +278,20 @@ QUuid QETProject::uuid() const
 	return m_uuid;
 }
 
+/**
+	@brief QETProject::containerPicture
+	The folio pictures of a .qetz are kept as bytes beside the document,
+	not as base64 text in it (QetContainer::join()), for as long as the
+	project is open: a folio built after opening still finds its pictures.
+	They cost almost no memory, sharing their bytes with the pictures
+	built from them.
+	@return the picture in @p file, or an empty array when there is none
+*/
+QByteArray QETProject::containerPicture(const QString &file) const
+{
+	return m_container_pictures.value(file);
+}
+
 /**
 	@brief QETProject::derivedItemUuid
 	A uuid for an item of this project that was saved without one, the same
@@ -456,9 +470,10 @@ QETProject::ProjectState QETProject::openFile(QFile *file)
 	// QetContainer::parse() does exactly that, for a .qet and for each part
 	// of a .qetz alike.
 	QString read_error;
-	if (!QetContainer::readDocument(content, &xml_project, &read_error))
+	if (!QetContainer::readDocument(content, &xml_project, &read_error, &m_container_pictures))
 	{
 		qWarning().noquote() << "Unable to read" << fi.fileName() << ":" << read_error;
+		m_container_pictures.clear();
 		if(opened_here) {
 			file->close();
 		}
@@ -544,7 +559,7 @@ QETProject::ProjectState QETProject::state() const
 QList<Diagram *> QETProject::diagrams() const
 {
 		//Every caller of diagrams() expects every folio built: those that
-		//do not are moved to folios() one by one (LAZY-FOLIO-PLAN.md)
+		//do not are moved to folios() one by one
 	loadFolios();
 	return(m_diagrams_list);
 }
@@ -2291,8 +2306,7 @@ QDomDocument QETProject::toXml(QHash<int, QByteArray> *kept_folios)
 		loadFolios();
 	}
 		//QET_CHECK_ELEMENT_INFO_STORE=1: say whether the stores of symbol
-		//information, folio title blocks and wire properties
-		//(DB-ACCESSORS-PLAN.md) agree with every placed symbol, folio and
+		//information, folio title blocks and wire properties agree with every placed symbol, folio and
 		//wire, at each save. A test and debugging aid; it changes nothing.
 	if (qEnvironmentVariableIntValue("QET_CHECK_ELEMENT_INFO_STORE") == 1) {
 		const QStringList mismatches = m_data_base.elementInformationMismatches();
@@ -3015,9 +3029,22 @@ void QETProject::readProjectXml(QDomDocument &xml_project)
 
 		//The stores of symbol information, folio title blocks and wire
 		//properties are filled from the document before the folios are
-		//built (DB-ACCESSORS-PLAN.md stage 4.6)
+		//built
 	m_data_base.prefillFromDocument(xml_project);
 
+		//QET_LAZY_FOLIOS leaves the folios to be built when needed only
+		//when the database can be filled from the document: otherwise
+		//filling it from the folios builds every one of them anyway, late
+		//and more slowly than reading them now
+	m_lazy_open = qEnvironmentVariableIntValue("QET_LAZY_FOLIOS") == 1;
+	if (m_lazy_open) {
+		const QString why = projectDataBase::documentRefusal(xml_project);
+		if (!why.isEmpty()) {
+			qInfo().noquote() << "Project opened with every folio built:" << why;
+			m_lazy_open = false;
+		}
+	}
+
 		//Load the diagrams
 	readDiagramsXml(xml_project);
 	const qint64 diagrams_ms = phase_timer.restart();
@@ -3057,7 +3084,6 @@ void QETProject::readProjectXml(QDomDocument &xml_project)
 		//number can hold what a save rewrites (tst_freetextnonfinite,
 		//tst_foliofreezeflags): only with QET_LAZY_SAVE_KEPT=1, until a
 		//.qetz, whose manifest names its writer, does it
-		//(QETZ-DB-TRUTH-PLAN.md D4)
 	m_kept_folios_trusted = qEnvironmentVariableIntValue("QET_LAZY_SAVE_KEPT") == 1
 			&& unloadedFolioCount() > 0
 			&& m_project_qet_version == QetVersion::currentVersion()
@@ -3134,7 +3160,7 @@ void QETProject::readDiagramsXml(QDomDocument &xml_project)
 		//Where each line of the file starts, for a folio's text: lines
 		//[its start, the next element's start) (QETProject::toXml())
 	QList<qsizetype> line_starts;
-	if (qEnvironmentVariableIntValue("QET_LAZY_FOLIOS") == 1 && !m_opened_content.isEmpty()) {
+	if (m_lazy_open && !m_opened_content.isEmpty()) {
 		line_starts << 0;
 		for (qsizetype i = 0 ; i < m_opened_content.size() ; ++i)
 			if (m_opened_content.at(i) == '\n') line_starts << i + 1;
@@ -3181,7 +3207,7 @@ void QETProject::readDiagramsXml(QDomDocument &xml_project)
 			connect(diagram, &Diagram::usedTitleBlockTemplateChanged,
 					this, &QETProject::usedTitleBlockTemplateChanged);
 
-			m_deferring_folios = qEnvironmentVariableIntValue("QET_LAZY_FOLIOS") == 1;
+			m_deferring_folios = m_lazy_open;
 			diagram->initFromXml(diagram_xml_element);
 			m_deferring_folios = false;
 			if (!diagram->isLoaded() && !line_starts.isEmpty())

sources/qetproject.h

@@ -119,10 +119,11 @@ class QETProject : public QObject
 		projectDataBase *dataBase();
 		QUuid uuid() const;
 		QUuid derivedItemUuid(const QString &kind, const QString &key);
+		QByteArray containerPicture(const QString &file) const;
 		ProjectState state() const;
 		QList<Diagram *> diagrams() const;
 			//The folios whether their items are built or not
-			//(QET_LAZY_FOLIOS, LAZY-FOLIO-PLAN.md); diagrams() builds them
+			//(QET_LAZY_FOLIOS); diagrams() builds them
 		QList<Diagram *> folios() const;
 		bool deferringFolios() const {return m_deferring_folios;}
 		int unloadedFolioCount() const;
@@ -448,6 +449,9 @@ class QETProject : public QObject
 		static bool m_backup_enabled;
 			/// Something changed since the last backup, see writeBackup()
 		bool m_backup_needed = true;
+			/// The pictures of the .qetz this project was opened from, see
+			/// containerPicture()
+		QHash<QString, QByteArray> m_container_pictures;
 			/// File path this project is saved to
 		QString m_file_path;
 			/// Current state of the project
@@ -512,6 +516,9 @@ class QETProject : public QObject
 				const DiagramContext &information,
 				const QHash<QString, QStringList> &titles_by_formula) const;
 		bool m_deferring_folios = false;
+			/// Opening with folios built only when needed: QET_LAZY_FOLIOS
+			/// is set and the database can be filled from the document
+		bool m_lazy_open = false;
 			/// Saving may write a folio not built as it was read
 			/// (writesKeptFolios()): the file is this version's, its
 			/// database rows came from it, and nothing project-wide that

tests/qttest/lazy_folios_probe.cpp

@@ -1,5 +1,5 @@
 // SPDX-License-Identifier: GPL-2.0-or-later
-// Lazy folios (QET_LAZY_FOLIOS, LAZY-FOLIO-PLAN.md stages 5.2 and 5.3),
+// Lazy folios (QET_LAZY_FOLIOS),
 // linked against the application's objects: a folio shown in a project
 // opened without building its folios draws and links as in the project
 // opened as usual, building only it and the folios its links and tables
@@ -37,6 +37,8 @@
 #include "../../sources/undocommand/deleteqgraphicsitemcommand.h"
 #include "../../sources/factory/elementfactory.h"
 #include "../../sources/undocommand/addgraphicsobjectcommand.h"
+#include "../../sources/dataBase/projectdatabase.h"
+#include <QDomDocument>
 
 static void check(bool value, const char *message) { if (!value) throw std::runtime_error(message); }
 
@@ -567,6 +569,44 @@ int main(int argc, char **argv)
 			check(potential(wire) == potential(usual_wire), "a potential followed as when built");
 			std::printf("potential through reports: %d wires, %d folios not built\n",
 						int(potential(wire).size()), followed.unloadedFolioCount());
+		}
+			//Two symbols sharing a uuid: the database is filled from the
+			//folios, which builds every one of them, so the project opens
+			//with every folio built rather than building them all later
+		{
+			QFile file(path);
+			check(file.open(QIODevice::ReadOnly), "fixture read");
+			QDomDocument document;
+			check(bool(document.setContent(&file)), "fixture parsed");
+			file.close();
+			check(projectDataBase::documentRefusal(document).isEmpty(), "the fixture fills the database itself");
+			QDomNodeList elements = document.elementsByTagName(QStringLiteral("element"));
+			QList<QDomElement> placed;
+			for (int i = 0; i < elements.size(); ++i) {
+				QDomElement e = elements.at(i).toElement();
+				if (Element::valideXml(e)) placed << e;
+			}
+			check(placed.size() > 1, "two placed symbols");
+			placed.last().setAttribute(QStringLiteral("uuid"), placed.first().attribute(QStringLiteral("uuid")));
+			check(!projectDataBase::documentRefusal(document).isEmpty(), "a shared uuid is seen before the folios are read");
+			const QString shared = output + "/lazy_folios_shared_uuid.qet";
+			QFile out(shared);
+			check(out.open(QIODevice::WriteOnly), "copy written");
+			out.write(document.toByteArray(4));
+			out.close();
+			qputenv("QET_LAZY_FOLIOS", "1");
+			qputenv("QET_LAZY_FOLIOS_TRACE", "1");
+			static int built_late; built_late = 0;
+			const QtMessageHandler previous = qInstallMessageHandler(
+					[](QtMsgType, const QMessageLogContext &, const QString &message) {
+				if (message.contains(QStringLiteral("built for a caller of diagrams()"))) ++built_late;
+			});
+			QETProject opened(shared);
+			qInstallMessageHandler(previous);
+			qunsetenv("QET_LAZY_FOLIOS_TRACE");
+			check(opened.state() == QETProject::Ok, "a project with a shared uuid opens");
+			check(opened.unloadedFolioCount() == 0, "a project with a shared uuid opens with every folio built");
+			check(built_late == 0, "a project with a shared uuid builds its folios as it reads them, not later");
 		}
 		std::printf("PASS: %d folios\n", int(folios.size()));
 	} catch (const std::exception &error) {

tests/qttest/tst_qetcontainer.cpp

@@ -242,6 +242,43 @@ private slots:
 		QVERIFY(pictures < 121);   // the 121 placed pictures repeat a few files
 	}
 
+	// Given a map, join() hands the pictures over as bytes instead of
+	// putting them back in the document as base64
+	void picturesKeptAside()
+	{
+		QDomDocument original;
+		QList<QetZip::Entry> entries = splitFile(
+					QStringLiteral(QET_EXAMPLES_DIR "/weneedpolonez-Polonez_MR89_wiring_diagram.qet"), &original);
+		QVERIFY(!entries.isEmpty());
+		QHash<QString, QByteArray> files;
+		for (const QetZip::Entry &e : entries) files.insert(e.name, e.data);
+
+		QDomDocument back;
+		QetContainer::Pictures pictures;
+		QString error;
+		QVERIFY2(QetContainer::join(entries, &back, &error, &pictures), qPrintable(error));
+		const QDomNodeList images = back.elementsByTagName(QStringLiteral("image"));
+		const QDomNodeList original_images = original.elementsByTagName(QStringLiteral("image"));
+		QCOMPARE(images.size(), 121);
+		QCOMPARE(images.size(), original_images.size());
+		for (int i = 0 ; i < images.size() ; ++i) {
+			const QDomElement image = images.at(i).toElement();
+			const QString file = QetContainer::pictureFile(image);
+			QVERIFY(!file.isEmpty());
+			QVERIFY(!image.firstChild().isText());
+			QCOMPARE(pictures.value(file), files.value(file));
+			QCOMPARE(pictures.value(file),
+					 QByteArray::fromBase64(original_images.at(i).firstChild().toText().data().toLatin1()));
+		}
+
+		// still refused when a picture is missing
+		for (int i = 0 ; i < entries.size() ; ++i)
+			if (entries.at(i).name.startsWith(QStringLiteral("images/"))) { entries.removeAt(i); break; }
+		QDomDocument broken;
+		QVERIFY(!QetContainer::join(entries, &broken, &error, &pictures));
+		QVERIFY2(error.contains(QStringLiteral("missing")), qPrintable(error));
+	}
+
 	// A file whose manifest needs a later reader is refused, not half read
 	void laterFormatRefused()
 	{

tests/qttest/tst_qetzip.cpp

@@ -2,6 +2,7 @@
 #include <QtTest>
 
 #include <QTemporaryDir>
+#include <QtEndian>
 
 #include "container/qetzip.h"
 #include "container/miniz/miniz.h"
@@ -72,6 +73,33 @@ private slots:
 		QCOMPARE(QetZip::toBytes(sample()), QetZip::toBytes(sample()));
 	}
 
+	// every entry carries a valid date in both of its headers, not the
+	// 0 miniz writes without a clock (shown as 1979 or 1901 by unzip tools)
+	void entriesHaveValidDate()
+	{
+		const QByteArray zip = QetZip::toBytes(sample());
+		QVERIFY(!zip.isEmpty());
+		mz_zip_archive archive;
+		mz_zip_zero_struct(&archive);
+		QVERIFY(mz_zip_reader_init_mem(&archive, zip.constData(), size_t(zip.size()), 0));
+		const mz_uint count = mz_zip_reader_get_num_files(&archive);
+		QCOMPARE(int(count), sample().size());
+		for (mz_uint i = 0 ; i < count ; ++i) {
+			mz_zip_archive_file_stat stat;
+			QVERIFY(mz_zip_reader_file_stat(&archive, i, &stat));
+			const char *local = zip.constData() + stat.m_local_header_ofs;
+			const char *central = zip.constData() + archive.m_central_directory_file_ofs + stat.m_central_dir_ofs;
+			QCOMPARE(qFromLittleEndian<quint16>(local), quint16(0x50 | 0x4b << 8));   // "PK"
+			QCOMPARE(qFromLittleEndian<quint16>(central), quint16(0x50 | 0x4b << 8));
+			// 1 Jan 1980: day 1, month 1, year 0 since 1980; time 00:00:00
+			QCOMPARE(qFromLittleEndian<quint16>(local + 12), quint16(0x21));
+			QCOMPARE(qFromLittleEndian<quint16>(central + 14), quint16(0x21));
+			QCOMPARE(qFromLittleEndian<quint16>(local + 10), quint16(0));
+			QCOMPARE(qFromLittleEndian<quint16>(central + 12), quint16(0));
+		}
+		mz_zip_reader_end(&archive);
+	}
+
 	void refusesUnsafeNamesOnWrite_data()
 	{
 		QTest::addColumn<QString>("name");

tests/qttest/tst_resaveunchanged.cpp

@@ -340,8 +340,8 @@ private slots:
 		return QString::fromUtf8(proc.readAllStandardOutput() + proc.readAllStandardError());
 	}
 
-	// The stores of symbol information and folio title blocks
-	// (DB-ACCESSORS-PLAN.md stages 4.1, 4.4) agree with every placed symbol
+	// The stores of symbol information and folio title blocks agree with
+	// every placed symbol
 	// and folio once a project is open: every example.
 	void elementInfoStoreAgrees_data()
 	{
@@ -361,7 +361,7 @@ private slots:
 	}
 
 	// The stores filled from the document before the folios are built
-	// (DB-ACCESSORS-PLAN.md stage 4.6) hold what building them gives, for a
+	// hold what building them gives, for a
 	// project saved by this version (every symbol, folio and wire carries
 	// a uuid): nothing differs, nothing is left unplaced.
 	void storesFilledBeforeBuilding()