diff --git a/sources/bordertitleblock.cpp b/sources/bordertitleblock.cpp index f0e6f65c1..8747aa674 100644 --- a/sources/bordertitleblock.cpp +++ b/sources/bordertitleblock.cpp @@ -321,8 +321,8 @@ TitleBlockProperties BorderTitleBlock::localTitleBlock() const /** @brief BorderTitleBlock::current @return the properties of the titleblock: for a folio placed in a - project the project's store holds them (DB-ACCESSORS-PLAN.md stage - 4.4), for any other this object's own copy. + project the project's store holds them, for any other this object's + own copy. */ TitleBlockProperties BorderTitleBlock::current() const { diff --git a/sources/bordertitleblock.h b/sources/bordertitleblock.h index adc422752..009089c75 100644 --- a/sources/bordertitleblock.h +++ b/sources/bordertitleblock.h @@ -110,8 +110,8 @@ class BorderTitleBlock : public QObject QRectF insideBorderRect() const; // methods to get title block basic data. A folio placed in a - // project reads them from the project's store - // (DB-ACCESSORS-PLAN.md stage 4.4), any other from its own copy. + // project reads them from the project's store, any other from its + // own copy. /// @return the value of the title block "Author" field QString author() const { const TitleBlockProperties *s = stored(); return(s ? s->author : btb_author_); } /// @return the value of the title block "Date" field diff --git a/sources/container/qetcontainer.cpp b/sources/container/qetcontainer.cpp index 89c464e22..73a5f65d2 100644 --- a/sources/container/qetcontainer.cpp +++ b/sources/container/qetcontainer.cpp @@ -156,15 +156,20 @@ bool QetContainer::isZip(const QByteArray &content) .qetz, parsed for a .qet. @p format, if any: the .qetz's format, 0 for a .qet. */ -bool QetContainer::readDocument(const QByteArray &content, QDomDocument *document, QString *error, - int *format) +bool QetContainer::readDocument(const QByteArray &content, QDomDocument *document, + QString *error, Pictures *pictures, int *format) { if (format) *format = 0; if (!isZip(content)) return parse(content, document, error); QList entries; return QetZip::fromBytes(content, &entries, error) - && join(entries, document, error, format); + && join(entries, document, error, pictures, format); +} + +QString QetContainer::pictureFile(const QDomElement &image) +{ + return image.attribute(ImageFileAttribute); } /** @@ -267,11 +272,13 @@ QList QetContainer::split(const QDomDocument &project, const QStr Refused, with @p error set: not a .qetz, made for a later QElectroTech (the manifest's min-reader), or a part missing or unreadable. Entries nothing names are ignored, so a later format may add some. + With @p pictures, the pictures stay out of the document: see the + namespace. @p format, if any: the format the file says it is, above Format for a later one this may read (its min-reader allows it) */ bool QetContainer::join(const QList &entries, QDomDocument *project, QString *error, - int *format) + Pictures *pictures, int *format) { QHash files; for (const QetZip::Entry &entry : entries) @@ -318,7 +325,8 @@ bool QetContainer::join(const QList &entries, QDomDocument *proje part.parentNode().replaceChild(project->importNode(document.documentElement(), true), part); } - //Pictures back to base64, as the image's first child + //Pictures back to base64, as the image's first child, or into + //pictures const QDomNodeList images = project->elementsByTagName(QStringLiteral("image")); for (int i = 0 ; i < images.size() ; ++i) { @@ -329,6 +337,10 @@ bool QetContainer::join(const QList &entries, QDomDocument *proje setError(error, QStringLiteral("picture \"%1\" is missing").arg(file)); return false; } + if (pictures) { + pictures->insert(file, files.value(file)); + continue; + } image.removeAttribute(ImageFileAttribute); image.insertBefore(project->createTextNode(QString::fromLatin1(files.value(file).toBase64())), image.firstChild()); diff --git a/sources/container/qetcontainer.h b/sources/container/qetcontainer.h index 1fc642f3c..f53e9ab79 100644 --- a/sources/container/qetcontainer.h +++ b/sources/container/qetcontainer.h @@ -21,6 +21,7 @@ #include "qetzip.h" #include +#include /** @brief The QetContainer namespace @@ -42,11 +43,18 @@ join(split(d)) is d. Nothing here knows what the parts mean: the project is read and written by the same code as a .qet file. + Given a Pictures map, join() leaves the pictures out of the document: + each keeps the name of its file, pictureFile(), and the bytes + go in the map. Turning them back into base64 text costs seconds on a + project with large pictures, and the folio would only decode it again. + The format is written down in QETZ-FORMAT.md, beside this file. */ namespace QetContainer { extern const char MimeType[]; + /// Picture bytes by file name, see join() + using Pictures = QHash; /// The format written; files of a later format are refused by join() /// unless their min-reader allows this one. constexpr int Format = 3; @@ -60,14 +68,18 @@ namespace QetContainer const QString &writer = QString(), QString *error = nullptr); bool join(const QList &entries, QDomDocument *project, - QString *error = nullptr, int *format = nullptr); + QString *error = nullptr, Pictures *pictures = nullptr, + int *format = nullptr); + /// The file join() left the picture of @p image in, or an empty string + QString pictureFile(const QDomElement &image); /// Parse @p xml as a .qet is read (QETProject::openFile) bool parse(const QByteArray &xml, QDomDocument *document, QString *error = nullptr); /// true if @p content is a zip, so a .qetz rather than a .qet bool isZip(const QByteArray &content); /// The project document in @p content, a .qet or a .qetz - bool readDocument(const QByteArray &content, QDomDocument *document, QString *error = nullptr, + bool readDocument(const QByteArray &content, QDomDocument *document, + QString *error = nullptr, Pictures *pictures = nullptr, int *format = nullptr); } diff --git a/sources/container/qetzip.cpp b/sources/container/qetzip.cpp index 7a0be3c26..dc9e2c9f7 100644 --- a/sources/container/qetzip.cpp +++ b/sources/container/qetzip.cpp @@ -22,6 +22,7 @@ #include #include #include +#include namespace { void setError(QString *error, const QString &text) @@ -44,6 +45,42 @@ bool safeName(const QString &name) } return true; } + + //Built with MINIZ_NO_TIME, miniz leaves each entry's DOS date at 0, + //day 0 of month 0, which is no date: unzip shows 30 Nov 1979, other + //tools 13 Dec 1901. Give every entry 1 Jan 1980 00:00 instead, the + //first valid DOS date, so the bytes still do not depend on the time. +constexpr quint16 EntryDosDate = (1 << 5) | 1; + +bool setEntryDates(QByteArray *bytes) +{ + char *data = bytes->data(); + mz_zip_archive zip; + mz_zip_zero_struct(&zip); + if (!mz_zip_reader_init_mem(&zip, data, size_t(bytes->size()), 0)) return false; + + bool ok = true; + const mz_uint count = mz_zip_reader_get_num_files(&zip); + for (mz_uint i = 0 ; i < count && ok ; ++i) + { + mz_zip_archive_file_stat stat; + if (!mz_zip_reader_file_stat(&zip, i, &stat)) { + ok = false; + break; + } + //date at offset 12 of the local header, 14 of the central one + const quint64 local = stat.m_local_header_ofs + 12; + const quint64 central = zip.m_central_directory_file_ofs + stat.m_central_dir_ofs + 14; + if (local + 2 > quint64(bytes->size()) || central + 2 > quint64(bytes->size())) { + ok = false; + break; + } + qToLittleEndian(EntryDosDate, data + local); + qToLittleEndian(EntryDosDate, data + central); + } + mz_zip_reader_end(&zip); + return ok; +} } /** @@ -87,8 +124,12 @@ QByteArray QetZip::toBytes(const QList &entries, QString *error) mz_zip_writer_end(&zip); return {}; } - const QByteArray bytes(static_cast(buffer), qsizetype(size)); + QByteArray bytes(static_cast(buffer), qsizetype(size)); mz_zip_writer_end(&zip); //frees buffer + if (!setEntryDates(&bytes)) { + setError(error, QStringLiteral("cannot date the zip entries")); + return {}; + } return bytes; } diff --git a/sources/container/qetzip.h b/sources/container/qetzip.h index 9238f6ebf..808a861ce 100644 --- a/sources/container/qetzip.h +++ b/sources/container/qetzip.h @@ -30,8 +30,8 @@ megabytes, not gigabytes -- and written through QSaveFile, so a failed save leaves the previous file as it was. - Entries carry no timestamp: saving the same project twice gives the - same bytes. + Every entry is dated 1 Jan 1980 00:00, not the time of saving: saving + the same project twice gives the same bytes. */ namespace QetZip { diff --git a/sources/dataBase/projectdatabase.cpp b/sources/dataBase/projectdatabase.cpp index d71bedfe4..68a98486e 100644 --- a/sources/dataBase/projectdatabase.cpp +++ b/sources/dataBase/projectdatabase.cpp @@ -253,6 +253,58 @@ void projectDataBase::updateDB(const QDomDocument &document) updateDB(); } +/** + @brief projectDataBase::documentRefusal + @return why populateFromDocument() will refuse @p document, for the + reasons it gives that the document alone can tell -- a folio, symbol + or wire with no saved uuid, two symbols sharing one, a folio number + using %autonum -- or an empty string. The other reasons need the + project's definitions or every folio's links; a document refused for + one of those is still filled from the folios, only later. +*/ +QString projectDataBase::documentRefusal(const QDomDocument &document) +{ + if (qEnvironmentVariableIntValue("QET_DATABASE_FROM_FOLIOS") == 1) { + return QStringLiteral("QET_DATABASE_FROM_FOLIOS is set"); + } + QSet element_uuids; + const QDomNodeList diagram_nodes = document.elementsByTagName(QStringLiteral("diagram")); + for (int i = 0 ; i < diagram_nodes.size() ; ++i) + { + QDomElement diagram_xml = diagram_nodes.at(i).toElement(); + if (QUuid(diagram_xml.attribute(QStringLiteral("uuid"))).isNull()) { + return QStringLiteral("a folio has no saved uuid"); + } + if (diagram_xml.attribute(QStringLiteral("folio")).contains(QStringLiteral("%autonum"))) { + return QStringLiteral("a folio number uses %autonum"); + } + for (QDomElement element_xml : QET::findInDomElement( + diagram_xml, QStringLiteral("elements"), QStringLiteral("element"))) + { + //Skipped by Diagram::fromXml() as well + if (!Element::valideXml(element_xml)) { + continue; + } + const QUuid uuid(element_xml.attribute(QStringLiteral("uuid"))); + if (uuid.isNull() || element_uuids.contains(uuid)) { + return QStringLiteral("an element has no saved uuid, or shares one"); + } + element_uuids.insert(uuid); + } + for (QDomElement conductor_xml : QET::findInDomElement( + diagram_xml, QStringLiteral("conductors"), QStringLiteral("conductor"))) + { + if (Conductor::valideXml(conductor_xml) + && (QUuid(conductor_xml.attribute(QStringLiteral("uuid"))).isNull() + || !conductor_xml.hasAttribute(QStringLiteral("element1")) + || !conductor_xml.hasAttribute(QStringLiteral("element2")))) { + return QStringLiteral("a conductor has no saved uuid, or names its ends the older way"); + } + } + } + return QString(); +} + namespace { struct DocumentTerminal { @@ -2445,7 +2497,7 @@ void projectDataBase::populateElementTable() /** @brief projectDataBase::populateElementInfoTable Populate the element info table from the symbol information store - (DB-ACCESSORS-PLAN.md stage 4.3), in the order of the element table. + in the order of the element table. The label column is the label a symbol shows, which its folio's numbering decides: the symbol holding the uuid gives it. diff --git a/sources/dataBase/projectdatabase.h b/sources/dataBase/projectdatabase.h index 0dd92aaa1..be3322c7d 100644 --- a/sources/dataBase/projectdatabase.h +++ b/sources/dataBase/projectdatabase.h @@ -67,6 +67,14 @@ class projectDataBase : public QObject populateFromDocument()), from the built folios otherwise. */ void updateDB(const QDomDocument &document); + /** + Why updateDB(document) will fill the tables from the folios + rather than from @p document, for the reasons the document + alone can tell; empty when it knows of none. Read before the + folios are: a project whose tables will come from the folios + builds every folio as it opens, even with QET_LAZY_FOLIOS. + */ + static QString documentRefusal(const QDomDocument &document); /** Suppress the full rebuild performed by updateDB(). @@ -106,7 +114,7 @@ class projectDataBase : public QObject //The information of every symbol placed in the project, by //symbol uuid: the store each symbol's information is written - //through (DB-ACCESSORS-PLAN.md stage 4.1). A symbol not placed + //through. A symbol not placed //(a preview, the symbol editor) keeps its own. DiagramContext elementInformation(const QUuid &element) const; bool hasElementInformation(const QUuid &element) const; @@ -124,7 +132,7 @@ class projectDataBase : public QObject //The title block properties of every folio of the project, by //folio uuid: the store each folio's title block is written - //through (DB-ACCESSORS-PLAN.md stage 4.4). A folio not in a + //through. A folio not in a //project keeps its own. TitleBlockProperties folioTitleBlock(const QUuid &folio) const; const TitleBlockProperties *storedFolioTitleBlock(const QUuid &folio) const; @@ -136,7 +144,7 @@ class projectDataBase : public QObject //The properties of every wire placed in the project, by wire //uuid: the store each wire's properties are written through - //(DB-ACCESSORS-PLAN.md stage 4.5). A wire not placed keeps its own. + //A wire not placed keeps its own. ConductorProperties conductorProperties(const QUuid &conductor) const; bool hasConductorProperties(const QUuid &conductor) const; int placedConductorCount(const QUuid &conductor) const; @@ -145,7 +153,7 @@ class projectDataBase : public QObject QStringList conductorPropertiesMismatches() const; //Fill the three stores from a project's document before its - //folios are built (DB-ACCESSORS-PLAN.md stage 4.6), then, once + //folios are built, then, once //they are, say how many of those values building agreed with. void prefillFromDocument(const QDomDocument &document); void endPrefill(bool keep_unplaced = false); diff --git a/sources/diagram.cpp b/sources/diagram.cpp index 9586b9555..13a62e7af 100644 --- a/sources/diagram.cpp +++ b/sources/diagram.cpp @@ -49,6 +49,7 @@ #include "diagramsortkeys.h" #include "itemgroups.h" #include "textgrid.h" +#include "container/qetcontainer.h" #include "foliogrid.h" #include #include @@ -2010,7 +2011,11 @@ bool Diagram::fromXml(QDomElement &document, QStringLiteral("images"), QStringLiteral("image"))) { DiagramImageItem *dii = new DiagramImageItem (); - dii -> fromXml(image_xml); + const QString picture_file = QetContainer::pictureFile(image_xml); + if (!picture_file.isEmpty() && project()) + dii -> fromXml(image_xml, project() -> containerPicture(picture_file)); + else + dii -> fromXml(image_xml); settle_uuid(dii, image_xml, QStringLiteral("image"), added_images.size()); ItemGroups::setGroup(dii, ItemGroups::read(image_xml)); addItem(dii); diff --git a/sources/diagram.h b/sources/diagram.h index 6b2e6c604..44748c181 100644 --- a/sources/diagram.h +++ b/sources/diagram.h @@ -211,7 +211,7 @@ class Diagram : public QGraphicsScene bool = true, DiagramContent * = nullptr); /// false while this folio's items wait in its XML to be - /// built (QET_LAZY_FOLIOS, LAZY-FOLIO-PLAN.md) + /// built (QET_LAZY_FOLIOS) bool isLoaded() const {return m_deferred.isNull();} QDomElement deferredXml() const {return m_deferred;} QString headerXml(); diff --git a/sources/qetgraphicsitem/conductor.cpp b/sources/qetgraphicsitem/conductor.cpp index 720db396c..339c4a243 100644 --- a/sources/qetgraphicsitem/conductor.cpp +++ b/sources/qetgraphicsitem/conductor.cpp @@ -1857,7 +1857,7 @@ ConductorProperties Conductor::properties() const /** @brief Conductor::placedStore @return the project database holding this wire's properties - (DB-ACCESSORS-PLAN.md stage 4.5), or nullptr if it is not placed in a + or nullptr if it is not placed in a project, or shares its uuid with another wire for the moment a copy takes to get its own */ diff --git a/sources/qetgraphicsitem/diagramimageitem.cpp b/sources/qetgraphicsitem/diagramimageitem.cpp index a952c4851..4c5f05145 100644 --- a/sources/qetgraphicsitem/diagramimageitem.cpp +++ b/sources/qetgraphicsitem/diagramimageitem.cpp @@ -1787,6 +1787,17 @@ const QByteArray &DiagramImageItem::encodedPng(const QPixmap &pixmap, QByteArray @return true if successfully loaded. */ bool DiagramImageItem::fromXml(const QDomElement &e) +{ + return fromXml(e, QByteArray()); +} + +/** + @brief DiagramImageItem::fromXml + Load this image from xml element e, its picture being @p picture, not + the base64 text of @p e, unless @p picture is empty. A .qetz keeps its + pictures as files (QetContainer::join()). +*/ +bool DiagramImageItem::fromXml(const QDomElement &e, const QByteArray &picture) { if (e.tagName() != "image") { return (false); @@ -1796,28 +1807,27 @@ bool DiagramImageItem::fromXml(const QDomElement &e) const QUuid uuid(e.attribute(QStringLiteral("uuid"))); if (!uuid.isNull() && uuid != m_uuid) setUuid(uuid); - QDomNode image_node = e.firstChild(); - if (!image_node.isText()) { - return (false); + // From the first text node only, not e.text(): that also collects the + // text of every child element, so on a cropped or colour-keyed picture + // it carries 's bytes too. Decoded once: on a project with + // large pictures each decoding costs seconds. + QByteArray array = picture; + if (array.isEmpty()) + { + QDomNode image_node = e.firstChild(); + if (!image_node.isText()) { + return (false); + } + array = QByteArray::fromBase64(image_node.toText().data().toLatin1()); } - //load xml image to QByteArray - QByteArray array; - array = QByteArray::fromBase64(e.text().toLatin1()); - //Set QPixmap from the array QPixmap pixmap; pixmap.loadFromData(array); setPixmap(pixmap); - // From the first text node only, not e.text(): that also collects the - // text of every child element, so on a cropped or colour-keyed picture - // it carries 's bytes too. loadFromData() stops at the end - // of the first PNG and never notices; a cache written back verbatim - // would. - const QByteArray ownArray = QByteArray::fromBase64(image_node.toText().data().toLatin1()); - if (isPng(ownArray) && !pixmap_.isNull()) + if (isPng(array) && !pixmap_.isNull()) { - m_png_cache = ownArray; + m_png_cache = array; m_png_cache_key = pixmap_.cacheKey(); } diff --git a/sources/qetgraphicsitem/diagramimageitem.h b/sources/qetgraphicsitem/diagramimageitem.h index 6e0f308f8..df8f77b1a 100644 --- a/sources/qetgraphicsitem/diagramimageitem.h +++ b/sources/qetgraphicsitem/diagramimageitem.h @@ -120,6 +120,7 @@ class DiagramImageItem : public QetGraphicsItem { int type() const override { return Type; } virtual bool fromXml(const QDomElement &); + bool fromXml(const QDomElement &, const QByteArray &picture); virtual QDomElement toXml(QDomDocument &) const; QUuid uuid() const {return m_uuid;} void setUuid(const QUuid &uuid) {m_uuid = uuid; emit uuidChanged();} diff --git a/sources/qetgraphicsitem/element.cpp b/sources/qetgraphicsitem/element.cpp index 779c6f7c9..6a59bbcbc 100644 --- a/sources/qetgraphicsitem/element.cpp +++ b/sources/qetgraphicsitem/element.cpp @@ -1859,7 +1859,7 @@ void Element::setUuid(const QUuid &uuid) /** @brief Element::storeInformation Write this symbol's information to its project's store, once it is - placed (DB-ACCESSORS-PLAN.md stage 4.1): every change of a placed + placed: every change of a placed symbol's information comes through here. */ void Element::storeInformation() @@ -1925,7 +1925,7 @@ ElementData Element::elementData() const /** @brief Element::elementInformations @return this symbol's information: for a placed symbol the project's - store holds it (DB-ACCESSORS-PLAN.md stage 4.2), for one not placed + store holds it, for one not placed (a preview, the symbol editor) its own copy. */ DiagramContext Element::elementInformations() const diff --git a/sources/qetproject.cpp b/sources/qetproject.cpp index fb96ab0e6..c907d196f 100644 --- a/sources/qetproject.cpp +++ b/sources/qetproject.cpp @@ -278,6 +278,20 @@ QUuid QETProject::uuid() const return m_uuid; } +/** + @brief QETProject::containerPicture + The folio pictures of a .qetz are kept as bytes beside the document, + not as base64 text in it (QetContainer::join()), for as long as the + project is open: a folio built after opening still finds its pictures. + They cost almost no memory, sharing their bytes with the pictures + built from them. + @return the picture in @p file, or an empty array when there is none +*/ +QByteArray QETProject::containerPicture(const QString &file) const +{ + return m_container_pictures.value(file); +} + /** @brief QETProject::derivedItemUuid A uuid for an item of this project that was saved without one, the same @@ -457,9 +471,11 @@ QETProject::ProjectState QETProject::openFile(QFile *file) // of a .qetz alike. QString read_error; int container_format = 0; - if (!QetContainer::readDocument(content, &xml_project, &read_error, &container_format)) + if (!QetContainer::readDocument(content, &xml_project, &read_error, &m_container_pictures, + &container_format)) { qWarning().noquote() << "Unable to read" << fi.fileName() << ":" << read_error; + m_container_pictures.clear(); if(opened_here) { file->close(); } @@ -552,7 +568,7 @@ QETProject::ProjectState QETProject::state() const QList QETProject::diagrams() const { //Every caller of diagrams() expects every folio built: those that - //do not are moved to folios() one by one (LAZY-FOLIO-PLAN.md) + //do not are moved to folios() one by one loadFolios(); return(m_diagrams_list); } @@ -2299,8 +2315,7 @@ QDomDocument QETProject::toXml(QHash *kept_folios) loadFolios(); } //QET_CHECK_ELEMENT_INFO_STORE=1: say whether the stores of symbol - //information, folio title blocks and wire properties - //(DB-ACCESSORS-PLAN.md) agree with every placed symbol, folio and + //information, folio title blocks and wire properties agree with every placed symbol, folio and //wire, at each save. A test and debugging aid; it changes nothing. if (qEnvironmentVariableIntValue("QET_CHECK_ELEMENT_INFO_STORE") == 1) { const QStringList mismatches = m_data_base.elementInformationMismatches(); @@ -3023,9 +3038,22 @@ void QETProject::readProjectXml(QDomDocument &xml_project) //The stores of symbol information, folio title blocks and wire //properties are filled from the document before the folios are - //built (DB-ACCESSORS-PLAN.md stage 4.6) + //built m_data_base.prefillFromDocument(xml_project); + //QET_LAZY_FOLIOS leaves the folios to be built when needed only + //when the database can be filled from the document: otherwise + //filling it from the folios builds every one of them anyway, late + //and more slowly than reading them now + m_lazy_open = qEnvironmentVariableIntValue("QET_LAZY_FOLIOS") == 1; + if (m_lazy_open) { + const QString why = projectDataBase::documentRefusal(xml_project); + if (!why.isEmpty()) { + qInfo().noquote() << "Project opened with every folio built:" << why; + m_lazy_open = false; + } + } + //Load the diagrams readDiagramsXml(xml_project); const qint64 diagrams_ms = phase_timer.restart(); @@ -3065,7 +3093,6 @@ void QETProject::readProjectXml(QDomDocument &xml_project) //number can hold what a save rewrites (tst_freetextnonfinite, //tst_foliofreezeflags): only with QET_LAZY_SAVE_KEPT=1, until a //.qetz, whose manifest names its writer, does it - //(QETZ-DB-TRUTH-PLAN.md D4) m_kept_folios_trusted = qEnvironmentVariableIntValue("QET_LAZY_SAVE_KEPT") == 1 && unloadedFolioCount() > 0 && m_project_qet_version == QetVersion::currentVersion() @@ -3142,7 +3169,7 @@ void QETProject::readDiagramsXml(QDomDocument &xml_project) //Where each line of the file starts, for a folio's text: lines //[its start, the next element's start) (QETProject::toXml()) QList line_starts; - if (qEnvironmentVariableIntValue("QET_LAZY_FOLIOS") == 1 && !m_opened_content.isEmpty()) { + if (m_lazy_open && !m_opened_content.isEmpty()) { line_starts << 0; for (qsizetype i = 0 ; i < m_opened_content.size() ; ++i) if (m_opened_content.at(i) == '\n') line_starts << i + 1; @@ -3189,7 +3216,7 @@ void QETProject::readDiagramsXml(QDomDocument &xml_project) connect(diagram, &Diagram::usedTitleBlockTemplateChanged, this, &QETProject::usedTitleBlockTemplateChanged); - m_deferring_folios = qEnvironmentVariableIntValue("QET_LAZY_FOLIOS") == 1; + m_deferring_folios = m_lazy_open; diagram->initFromXml(diagram_xml_element); m_deferring_folios = false; if (!diagram->isLoaded() && !line_starts.isEmpty()) diff --git a/sources/qetproject.h b/sources/qetproject.h index edf054a80..e5d0c2220 100644 --- a/sources/qetproject.h +++ b/sources/qetproject.h @@ -119,10 +119,11 @@ class QETProject : public QObject projectDataBase *dataBase(); QUuid uuid() const; QUuid derivedItemUuid(const QString &kind, const QString &key); + QByteArray containerPicture(const QString &file) const; ProjectState state() const; QList diagrams() const; //The folios whether their items are built or not - //(QET_LAZY_FOLIOS, LAZY-FOLIO-PLAN.md); diagrams() builds them + //(QET_LAZY_FOLIOS); diagrams() builds them QList folios() const; bool deferringFolios() const {return m_deferring_folios;} int unloadedFolioCount() const; @@ -450,6 +451,9 @@ class QETProject : public QObject static bool m_backup_enabled; /// Something changed since the last backup, see writeBackup() bool m_backup_needed = true; + /// The pictures of the .qetz this project was opened from, see + /// containerPicture() + QHash m_container_pictures; /// File path this project is saved to QString m_file_path; /// Current state of the project @@ -515,6 +519,9 @@ class QETProject : public QObject const DiagramContext &information, const QHash &titles_by_formula) const; bool m_deferring_folios = false; + /// Opening with folios built only when needed: QET_LAZY_FOLIOS + /// is set and the database can be filled from the document + bool m_lazy_open = false; /// Saving may write a folio not built as it was read /// (writesKeptFolios()): the file is this version's, its /// database rows came from it, and nothing project-wide that diff --git a/tests/qttest/lazy_folios_probe.cpp b/tests/qttest/lazy_folios_probe.cpp index daebbba50..2ed2d085e 100644 --- a/tests/qttest/lazy_folios_probe.cpp +++ b/tests/qttest/lazy_folios_probe.cpp @@ -1,5 +1,5 @@ // SPDX-License-Identifier: GPL-2.0-or-later -// Lazy folios (QET_LAZY_FOLIOS, LAZY-FOLIO-PLAN.md stages 5.2 and 5.3), +// Lazy folios (QET_LAZY_FOLIOS), // linked against the application's objects: a folio shown in a project // opened without building its folios draws and links as in the project // opened as usual, building only it and the folios its links and tables @@ -37,6 +37,8 @@ #include "../../sources/undocommand/deleteqgraphicsitemcommand.h" #include "../../sources/factory/elementfactory.h" #include "../../sources/undocommand/addgraphicsobjectcommand.h" +#include "../../sources/dataBase/projectdatabase.h" +#include static void check(bool value, const char *message) { if (!value) throw std::runtime_error(message); } @@ -567,6 +569,44 @@ int main(int argc, char **argv) check(potential(wire) == potential(usual_wire), "a potential followed as when built"); std::printf("potential through reports: %d wires, %d folios not built\n", int(potential(wire).size()), followed.unloadedFolioCount()); + } + //Two symbols sharing a uuid: the database is filled from the + //folios, which builds every one of them, so the project opens + //with every folio built rather than building them all later + { + QFile file(path); + check(file.open(QIODevice::ReadOnly), "fixture read"); + QDomDocument document; + check(bool(document.setContent(&file)), "fixture parsed"); + file.close(); + check(projectDataBase::documentRefusal(document).isEmpty(), "the fixture fills the database itself"); + QDomNodeList elements = document.elementsByTagName(QStringLiteral("element")); + QList placed; + for (int i = 0; i < elements.size(); ++i) { + QDomElement e = elements.at(i).toElement(); + if (Element::valideXml(e)) placed << e; + } + check(placed.size() > 1, "two placed symbols"); + placed.last().setAttribute(QStringLiteral("uuid"), placed.first().attribute(QStringLiteral("uuid"))); + check(!projectDataBase::documentRefusal(document).isEmpty(), "a shared uuid is seen before the folios are read"); + const QString shared = output + "/lazy_folios_shared_uuid.qet"; + QFile out(shared); + check(out.open(QIODevice::WriteOnly), "copy written"); + out.write(document.toByteArray(4)); + out.close(); + qputenv("QET_LAZY_FOLIOS", "1"); + qputenv("QET_LAZY_FOLIOS_TRACE", "1"); + static int built_late; built_late = 0; + const QtMessageHandler previous = qInstallMessageHandler( + [](QtMsgType, const QMessageLogContext &, const QString &message) { + if (message.contains(QStringLiteral("built for a caller of diagrams()"))) ++built_late; + }); + QETProject opened(shared); + qInstallMessageHandler(previous); + qunsetenv("QET_LAZY_FOLIOS_TRACE"); + check(opened.state() == QETProject::Ok, "a project with a shared uuid opens"); + check(opened.unloadedFolioCount() == 0, "a project with a shared uuid opens with every folio built"); + check(built_late == 0, "a project with a shared uuid builds its folios as it reads them, not later"); } std::printf("PASS: %d folios\n", int(folios.size())); } catch (const std::exception &error) { diff --git a/tests/qttest/tst_qetcontainer.cpp b/tests/qttest/tst_qetcontainer.cpp index 7fdf4af9b..fa981f537 100644 --- a/tests/qttest/tst_qetcontainer.cpp +++ b/tests/qttest/tst_qetcontainer.cpp @@ -293,6 +293,43 @@ private slots: QVERIFY(pictures < 121); // the 121 placed pictures repeat a few files } + // Given a map, join() hands the pictures over as bytes instead of + // putting them back in the document as base64 + void picturesKeptAside() + { + QDomDocument original; + QList entries = splitFile( + QStringLiteral(QET_EXAMPLES_DIR "/weneedpolonez-Polonez_MR89_wiring_diagram.qet"), &original); + QVERIFY(!entries.isEmpty()); + QHash files; + for (const QetZip::Entry &e : entries) files.insert(e.name, e.data); + + QDomDocument back; + QetContainer::Pictures pictures; + QString error; + QVERIFY2(QetContainer::join(entries, &back, &error, &pictures), qPrintable(error)); + const QDomNodeList images = back.elementsByTagName(QStringLiteral("image")); + const QDomNodeList original_images = original.elementsByTagName(QStringLiteral("image")); + QCOMPARE(images.size(), 121); + QCOMPARE(images.size(), original_images.size()); + for (int i = 0 ; i < images.size() ; ++i) { + const QDomElement image = images.at(i).toElement(); + const QString file = QetContainer::pictureFile(image); + QVERIFY(!file.isEmpty()); + QVERIFY(!image.firstChild().isText()); + QCOMPARE(pictures.value(file), files.value(file)); + QCOMPARE(pictures.value(file), + QByteArray::fromBase64(original_images.at(i).firstChild().toText().data().toLatin1())); + } + + // still refused when a picture is missing + for (int i = 0 ; i < entries.size() ; ++i) + if (entries.at(i).name.startsWith(QStringLiteral("images/"))) { entries.removeAt(i); break; } + QDomDocument broken; + QVERIFY(!QetContainer::join(entries, &broken, &error, &pictures)); + QVERIFY2(error.contains(QStringLiteral("missing")), qPrintable(error)); + } + // A file whose manifest needs a later reader is refused, not half read // A later format whose min-reader allows this one opens, and join() says // it is later: QElectroTech opens it read-only @@ -311,7 +348,7 @@ private slots: QDomDocument back; QString error; int format = 0; - QVERIFY2(QetContainer::join(entries, &back, &error, &format), qPrintable(error)); + QVERIFY2(QetContainer::join(entries, &back, &error, nullptr, &format), qPrintable(error)); QCOMPARE(format, later); const QString d = difference(original.documentElement(), back.documentElement(), QString()); QVERIFY2(d.isEmpty(), qPrintable(d)); diff --git a/tests/qttest/tst_qetzip.cpp b/tests/qttest/tst_qetzip.cpp index 4c69afef3..156cb8fb6 100644 --- a/tests/qttest/tst_qetzip.cpp +++ b/tests/qttest/tst_qetzip.cpp @@ -2,6 +2,7 @@ #include #include +#include #include "container/qetzip.h" #include "container/miniz/miniz.h" @@ -72,6 +73,33 @@ private slots: QCOMPARE(QetZip::toBytes(sample()), QetZip::toBytes(sample())); } + // every entry carries a valid date in both of its headers, not the + // 0 miniz writes without a clock (shown as 1979 or 1901 by unzip tools) + void entriesHaveValidDate() + { + const QByteArray zip = QetZip::toBytes(sample()); + QVERIFY(!zip.isEmpty()); + mz_zip_archive archive; + mz_zip_zero_struct(&archive); + QVERIFY(mz_zip_reader_init_mem(&archive, zip.constData(), size_t(zip.size()), 0)); + const mz_uint count = mz_zip_reader_get_num_files(&archive); + QCOMPARE(int(count), sample().size()); + for (mz_uint i = 0 ; i < count ; ++i) { + mz_zip_archive_file_stat stat; + QVERIFY(mz_zip_reader_file_stat(&archive, i, &stat)); + const char *local = zip.constData() + stat.m_local_header_ofs; + const char *central = zip.constData() + archive.m_central_directory_file_ofs + stat.m_central_dir_ofs; + QCOMPARE(qFromLittleEndian(local), quint16(0x50 | 0x4b << 8)); // "PK" + QCOMPARE(qFromLittleEndian(central), quint16(0x50 | 0x4b << 8)); + // 1 Jan 1980: day 1, month 1, year 0 since 1980; time 00:00:00 + QCOMPARE(qFromLittleEndian(local + 12), quint16(0x21)); + QCOMPARE(qFromLittleEndian(central + 14), quint16(0x21)); + QCOMPARE(qFromLittleEndian(local + 10), quint16(0)); + QCOMPARE(qFromLittleEndian(central + 12), quint16(0)); + } + mz_zip_reader_end(&archive); + } + void refusesUnsafeNamesOnWrite_data() { QTest::addColumn("name"); diff --git a/tests/qttest/tst_resaveunchanged.cpp b/tests/qttest/tst_resaveunchanged.cpp index 1babd2c55..c36ac8127 100644 --- a/tests/qttest/tst_resaveunchanged.cpp +++ b/tests/qttest/tst_resaveunchanged.cpp @@ -340,8 +340,8 @@ private slots: return QString::fromUtf8(proc.readAllStandardOutput() + proc.readAllStandardError()); } - // The stores of symbol information and folio title blocks - // (DB-ACCESSORS-PLAN.md stages 4.1, 4.4) agree with every placed symbol + // The stores of symbol information and folio title blocks agree with + // every placed symbol // and folio once a project is open: every example. void elementInfoStoreAgrees_data() { @@ -361,7 +361,7 @@ private slots: } // The stores filled from the document before the folios are built - // (DB-ACCESSORS-PLAN.md stage 4.6) hold what building them gives, for a + // hold what building them gives, for a // project saved by this version (every symbol, folio and wire carries // a uuid): nothing differs, nothing is left unplaced. void storesFilledBeforeBuilding()