diff --git a/misc/qet-mcp/README.md b/misc/qet-mcp/README.md index d6b391456..53e55d833 100644 --- a/misc/qet-mcp/README.md +++ b/misc/qet-mcp/README.md @@ -570,6 +570,15 @@ Python, plus the hang guard on `addConductor` and the database refresh in a terminal sharing its point with another, where the order is undefined; the uuid tells those apart. `wiring_list_view` carries the same four columns for `qet_query`. +- **`"reproducible": true` makes a PDF comparable byte for byte.** Two + exports of the same project otherwise differ in their dates and document + id. The option sets `SOURCE_DATE_EPOCH` for the run + ([reproducible-builds.org](https://reproducible-builds.org/specs/source-date-epoch/)): + the PDF then carries that date in UTC, a document id derived from the + project file, and its fonts in a fixed order. The date is + `source_date_epoch` if given, else the server's own `SOURCE_DATE_EPOCH`, + else 0 (1 January 1970). The result's `"reproducible"` is false, with a + hint, when the QElectroTech build is too old to honour it. - **`qet_export` isolates its launch.** SingleApplication keys its socket on `applicationFilePath()`, so a second launch of the same binary path forwards its request to an already-running instance and returns *that* diff --git a/misc/qet-mcp/qet_mcp.py b/misc/qet-mcp/qet_mcp.py index ca9b597ba..ef07c4428 100755 --- a/misc/qet-mcp/qet_mcp.py +++ b/misc/qet-mcp/qet_mcp.py @@ -56,6 +56,7 @@ workspace policy applies exactly as it does over stdio. from __future__ import annotations +import datetime import json import re import os @@ -942,7 +943,8 @@ def _collection_setting(collection: PurePath) -> str: def _run_qet(binary: str, args: list[str], timeout: int = 180, elements_dir: str | None = None, - script: str | None = None, tail: int = 4000) -> dict: + script: str | None = None, tail: int = 4000, + extra_env: dict | None = None) -> dict: """Launch QElectroTech headlessly, carrying the known launch traps. SingleApplication keys its socket on applicationFilePath(), so a second @@ -1000,6 +1002,7 @@ def _run_qet(binary: str, args: list[str], timeout: int = 180, script_path.write_text(script, encoding="utf-8") args = ["--run", str(script_path), *args] env = _launch_env(dict(os.environ), home, os.name == "nt") + env.update(extra_env or {}) try: p = subprocess.run([str(exe), *args], env=env, timeout=timeout, capture_output=True, text=True) @@ -1032,8 +1035,30 @@ def _run_qet(binary: str, args: list[str], timeout: int = 180, return result +def _source_date_epoch(value) -> int: + """The SOURCE_DATE_EPOCH a reproducible export uses: @p value when the + caller gives one, else the one this server was started with, else 0. + + 0 (1 January 1970) rather than a date read from the project: the file's + modification time changes with every copy and checkout, and a date the + drawing carries is a title block field, not when the PDF was made. A + caller who wants a meaningful date passes it. + """ + if value is None: + value = os.environ.get("SOURCE_DATE_EPOCH", "0") + try: + seconds = None if isinstance(value, (bool, float)) else int(value) + except (TypeError, ValueError): + seconds = None + if seconds is None or seconds < 0: + raise ValueError(f"source_date_epoch must be a whole number of seconds " + f"since 1970, got {value!r}") + return seconds + + def tool_export(binary: str, project: str, format: str, output: str, - timeout: int = 180) -> dict: + timeout: int = 180, reproducible: bool = False, + source_date_epoch: int | None = None) -> dict: if format not in EXPORT_FORMATS: raise ValueError(f"unknown format {format!r}; " f"expected one of {', '.join(sorted(EXPORT_FORMATS))}") @@ -1047,12 +1072,31 @@ def tool_export(binary: str, project: str, format: str, output: str, # application starts its GUI instead, which then hangs on an offscreen # platform. Order matters here. flag = EXPORT_FORMATS[format] - result = _run_qet(binary, [flag, str(proj), output], timeout) + # SOURCE_DATE_EPOCH (reproducible-builds.org) makes --export-pdf write + # the same bytes for the same project: the dates it names instead of + # now, a document id from the project file, fonts in a fixed order. + extra_env = None + if reproducible or source_date_epoch is not None: + epoch = _source_date_epoch(source_date_epoch) + extra_env = {"SOURCE_DATE_EPOCH": str(epoch)} + result = _run_qet(binary, [flag, str(proj), output], timeout, + extra_env=extra_env) out = Path(output).expanduser() result["output"] = str(out) result["output_exists"] = out.exists() if out.exists() and out.is_file(): result["output_bytes"] = out.stat().st_size + if extra_env and format == "pdf": + # A QElectroTech older than this option ignores the variable and + # writes the time of the export, so say whether this one did. + stamp = datetime.datetime.fromtimestamp(epoch, datetime.timezone.utc) + honoured = (b"/CreationDate (D:" + stamp.strftime("%Y%m%d%H%M%S").encode() + + b"Z)") in out.read_bytes() + result["reproducible"] = honoured + if not honoured: + result["hint"] = ("this QElectroTech ignores SOURCE_DATE_EPOCH, so the " + "PDF carries the time of the export; it needs a build " + "with repeatable PDF export") return result @@ -3485,11 +3529,25 @@ TOOLS = [ "description": "replace \"output\" if it already exists; " "without this an existing file is never clobbered"}, "timeout": {"type": "integer", "default": 180}, + "reproducible": {"type": "boolean", "default": False, + "description": "pdf: write the same bytes for the same " + "project in every run, so two exports can be " + "compared byte for byte. Sets " + "SOURCE_DATE_EPOCH; the result's " + "\"reproducible\" says whether this " + "QElectroTech honoured it"}, + "source_date_epoch": {"type": "integer", "minimum": 0, + "description": "with reproducible: the date the PDF " + "carries, in seconds since 1970 UTC. " + "Default: this server's own " + "SOURCE_DATE_EPOCH, else 0"}, }, "required": ["project", "format", "output"], }, "handler": lambda a: tool_export(a["binary"], a["project"], a["format"], - a["output"], a.get("timeout", 180)), + a["output"], a.get("timeout", 180), + a.get("reproducible", False), + a.get("source_date_epoch")), }, { "name": "qet_edit", diff --git a/misc/qet-mcp/test_qet_mcp.py b/misc/qet-mcp/test_qet_mcp.py index 3d933df2d..7fc26eb75 100644 --- a/misc/qet-mcp/test_qet_mcp.py +++ b/misc/qet-mcp/test_qet_mcp.py @@ -31,6 +31,7 @@ import subprocess import sys import tempfile import unittest +import unittest.mock from unittest import mock import xml.etree.ElementTree as ET from pathlib import Path @@ -3453,6 +3454,43 @@ class ProjectNewValidation(unittest.TestCase): self.assertFalse(Path(new).exists()) +class ReproducibleExport(unittest.TestCase): + """qet_export's "reproducible" sets SOURCE_DATE_EPOCH for the run.""" + + def run_export(self, **kw): + seen = {} + def fake(binary, args, timeout=180, **rest): + seen.update(rest) + return {"ok": True} + with tempfile.TemporaryDirectory() as tmp: + proj = Path(tmp) / "a.qet" + proj.write_text("") + with unittest.mock.patch.object(m, "_run_qet", fake): + m.tool_export("qet", str(proj), "pdf", str(Path(tmp) / "o.pdf"), **kw) + return seen.get("extra_env") + + def test_off_by_default(self): + self.assertIsNone(self.run_export()) + + def test_zero_unless_told_otherwise(self): + with unittest.mock.patch.dict(os.environ): + os.environ.pop("SOURCE_DATE_EPOCH", None) + self.assertEqual(self.run_export(reproducible=True), + {"SOURCE_DATE_EPOCH": "0"}) + + def test_the_servers_own_variable_then_the_callers_value(self): + with unittest.mock.patch.dict(os.environ, {"SOURCE_DATE_EPOCH": "1600000000"}): + self.assertEqual(self.run_export(reproducible=True), + {"SOURCE_DATE_EPOCH": "1600000000"}) + self.assertEqual(self.run_export(reproducible=True, source_date_epoch=5), + {"SOURCE_DATE_EPOCH": "5"}) + + def test_a_bad_date_is_refused_before_launching(self): + for bad in (-1, "soon", 1.5, True): + with self.subTest(value=bad), self.assertRaises(ValueError): + self.run_export(reproducible=True, source_date_epoch=bad) + + class ReadTools(unittest.TestCase): def test_project_info_and_scan_on_a_fixture(self): with tempfile.TemporaryDirectory() as tmp: @@ -5415,6 +5453,27 @@ class UuidIndexLookups(unittest.TestCase): @needs_examples class CorpusIntegration(unittest.TestCase): + def test_a_reproducible_pdf_is_the_same_bytes_every_run(self): + """Two reproducible exports of one project, in separate runs, are + byte for byte the same; a plain one carries the time of the export. + 741.qet has no cross-reference links, whose order is fixed apart.""" + project = str(Path(EXAMPLES) / "741.qet") + with tempfile.TemporaryDirectory() as tmp: + outs = [] + for i in range(2): + out = str(Path(tmp) / f"r{i}.pdf") + r = m.tool_export(BINARY, project, "pdf", out, reproducible=True, + source_date_epoch=1700000000) + self.assertTrue(r["ok"], r) + self.assertTrue(r["reproducible"], r) + outs.append(Path(out).read_bytes()) + self.assertEqual(outs[0], outs[1]) + plain = str(Path(tmp) / "plain.pdf") + r = m.tool_export(BINARY, project, "pdf", plain) + self.assertNotIn("reproducible", r) + self.assertNotIn(b"/CreationDate (D:20231114221320Z)", + Path(plain).read_bytes()) + def test_folio_counts_match_what_qelectrotech_itself_holds(self): """Element and conductor counts per folio, from the file, against QElectroTech's own counts after loading it -- over every example. diff --git a/sources/cli_export.cpp b/sources/cli_export.cpp index d99481345..b26a3c27e 100644 --- a/sources/cli_export.cpp +++ b/sources/cli_export.cpp @@ -42,7 +42,9 @@ #include #include #include +#include #include +#include #include #include #include @@ -59,7 +61,9 @@ #include #include #include +#include #include +#include namespace { @@ -147,6 +151,23 @@ void renderDiagram(Diagram *diagram, QPainter &painter, const QRectF &target, diagram->setDrawTerminalNames(was_drawing_terminal_names); } +/// The time SOURCE_DATE_EPOCH names, in seconds since 1970 UTC, or an +/// invalid QDateTime when it is unset. A value that is not a whole number of +/// seconds is reported and ignored. +QDateTime sourceDateEpoch() +{ + const QByteArray value = qgetenv("SOURCE_DATE_EPOCH"); + if (value.isEmpty()) + return {}; + bool ok = false; + const qlonglong seconds = value.toLongLong(&ok); + if (!ok || seconds < 0) { + err << "SOURCE_DATE_EPOCH '" << value << "' is not a number of seconds; ignored.\n"; + return {}; + } + return QDateTime::fromSecsSinceEpoch(seconds, QTimeZone::UTC); +} + int exportPdf(QETProject &project, const QString &output, bool showTerminals = false) { @@ -166,6 +187,19 @@ int exportPdf(QETProject &project, const QString &output, writer.setCreator("QElectroTech"); writer.setResolution(96); + // SOURCE_DATE_EPOCH (reproducible-builds.org) asks for the same file + // from the same input: the time it names instead of now, and a document + // id from the project file instead of a random one. Qt has no setter for + // the dates, so they are rewritten once the file is written. + const QDateTime sourceDate = sourceDateEpoch(); + if (sourceDate.isValid()) { + QCryptographicHash hash(QCryptographicHash::Sha256); + QFile file(project.filePath()); + if (file.open(QIODevice::ReadOnly)) + hash.addData(&file); + writer.setDocumentId(QUuid::createUuidV5(project.uuid(), hash.result())); + } + QPainter painter; bool first = true; for (Diagram *diagram : diagrams) { @@ -240,6 +274,8 @@ int exportPdf(QETProject &project, const QString &output, // the cross-references jump inside the document in any PDF viewer. PdfLinks::convertUriToGoTo(output); PdfLinks::removeUnusedPdfxNamespace(output); + if (sourceDate.isValid()) + PdfLinks::setDocumentDate(output, sourceDate); out << "Exported " << diagrams.size() << " page(s) -> " << output << "\n"; return 0; diff --git a/sources/pdf_links.cpp b/sources/pdf_links.cpp index 6bdf563c1..1a6adeceb 100644 --- a/sources/pdf_links.cpp +++ b/sources/pdf_links.cpp @@ -35,6 +35,8 @@ #include #include +#include + namespace PdfLinks { void injectCrossRefLinks(QPdfEngine *engine, Diagram *diagram, @@ -411,6 +413,136 @@ void removeUnusedPdfxNamespace(const QString &pdfPath) f.close(); } +void setDocumentDate(const QString &pdfPath, const QDateTime &when) +{ + QFile f(pdfPath); + if (!f.open(QIODevice::ReadOnly)) return; + const QByteArray data = f.readAll(); + f.close(); + + const QDateTime utc = when.toUTC(); + const QByteArray pdfDate = + "(D:" + utc.toString(QStringLiteral("yyyyMMddHHmmss")).toLatin1() + "Z)"; + const QByteArray xmpDate = + utc.toString(QStringLiteral("yyyy-MM-ddTHH:mm:ss")).toLatin1() + "Z"; + + // Each edit replaces [start, end) of the original file. + struct Edit { int start; int end; QByteArray text; }; + QList edits; + + // The document information: "/CreationDate (D:...)" and "/ModDate (D:...)". + // Its other strings are UTF-16, so they cannot hold these keys. + for (const QByteArray &key : {QByteArray("/CreationDate "), QByteArray("/ModDate ")}) { + const int k = data.indexOf(key); + if (k == -1) return; + const int start = k + key.size(); + const int end = data.indexOf(')', start); + if (end == -1 || data.at(start) != '(') return; + edits.append({start, end + 1, pdfDate}); + } + + // The XMP metadata stream, which Qt writes uncompressed: its three + // dates, and its /Length, which changes with them. + const QByteArray metadata("/Type /Metadata /Subtype /XML"); + const int m = data.indexOf(metadata); + if (m == -1) return; + const QByteArray lengthKey("/Length "); + const int l = data.indexOf(lengthKey, m); + const int streamStart = data.indexOf("stream\n", m); + if (l == -1 || streamStart == -1 || l > streamStart) return; + const int lengthStart = l + lengthKey.size(); + int lengthEnd = lengthStart; + while (lengthEnd < data.size() && QChar(data.at(lengthEnd)).isDigit()) + ++lengthEnd; + const int streamEnd = data.indexOf("endstream", streamStart); + if (lengthEnd == lengthStart || streamEnd == -1) return; + + // The last of each in the stream: the title and author come before + // the dates, and are text a project could fill with anything. + int xmpDelta = 0; + QList xmpEdits; + for (const QByteArray &attr : {QByteArray("xmp:CreateDate=\""), + QByteArray("xmp:ModifyDate=\""), + QByteArray("xmp:MetadataDate=\"")}) { + const int a = data.lastIndexOf(attr, streamEnd); + if (a < streamStart) return; + const int start = a + attr.size(); + const int end = data.indexOf('"', start); + if (end == -1 || end > streamEnd) return; + xmpEdits.append({start, end, xmpDate}); + xmpDelta += xmpDate.size() - (end - start); + } + const int length = data.mid(lengthStart, lengthEnd - lengthStart).toInt(); + edits.append({lengthStart, lengthEnd, QByteArray::number(length + xmpDelta)}); + edits.append(xmpEdits); + + std::sort(edits.begin(), edits.end(), + [](const Edit &a, const Edit &b) { return a.start < b.start; }); + + // Where an offset in the original file is in the new one. + auto moved = [&edits](int offset) { + int shift = 0; + for (const Edit &e : std::as_const(edits)) + if (e.end <= offset) + shift += e.text.size() - (e.end - e.start); + return offset + shift; + }; + + // The xref table the last startxref points at, which is the one a + // reader uses. + const int sx = data.lastIndexOf("startxref"); + if (sx == -1) return; + int numStart = sx + 9; + while (numStart < data.size() && QChar(data.at(numStart)).isSpace()) + ++numStart; + int numEnd = numStart; + while (numEnd < data.size() && QChar(data.at(numEnd)).isDigit()) + ++numEnd; + const int xref = data.mid(numStart, numEnd - numStart).toInt(); + if (numEnd == numStart || !data.mid(xref).startsWith("xref")) return; + for (const Edit &e : std::as_const(edits)) + if (e.end > xref) return; // every edit is in the objects before it + + QByteArray out; + out.reserve(data.size() + 64); + int pos = 0; + for (const Edit &e : std::as_const(edits)) { + if (e.start < pos) return; // overlapping: not a file Qt wrote + out += data.mid(pos, e.start - pos); + out += e.text; + pos = e.end; + } + + // The xref entries: "0000012345 00000 n \n", 20 bytes each, after the + // "xref" line and one "first count" line. + int line = data.indexOf('\n', xref) + 1; + line = data.indexOf('\n', line) + 1; + if (line <= 0) return; + out += data.mid(pos, line - pos); + pos = line; + static const QRegularExpression entry(QStringLiteral(R"(^(\d{10}) (\d{5}) ([nf])\s*$)")); + while (pos + 18 <= data.size()) { + const QByteArray row = data.mid(pos, 20); + const auto match = entry.match(QString::fromLatin1(row)); + if (!match.hasMatch()) break; + if (match.captured(3) == QLatin1String("n")) { + const int offset = match.captured(1).toInt(); + out += QByteArray::number(moved(offset)).rightJustified(10, '0'); + out += row.mid(10); + } else { + out += row; + } + pos += 20; + } + out += data.mid(pos, numStart - pos); + out += QByteArray::number(moved(xref)); + out += data.mid(numEnd); + + if (!f.open(QIODevice::WriteOnly | QIODevice::Truncate)) return; + f.write(out); + f.close(); +} + void convertComponentInfoAnnotations(const QString &pdfPath, const QList &annotations) { diff --git a/sources/pdf_links.h b/sources/pdf_links.h index 325367c53..f404352da 100644 --- a/sources/pdf_links.h +++ b/sources/pdf_links.h @@ -18,6 +18,7 @@ #ifndef PDF_LINKS_H #define PDF_LINKS_H +#include #include #include #include @@ -83,6 +84,16 @@ namespace PdfLinks { */ void removeUnusedPdfxNamespace(const QString &pdfPath); + /** + Post-process a Qt-generated PDF file: replace the creation and + modification dates Qt wrote (the time of the export, in local time) + with @p when, in UTC, in the document information and the XMP + metadata, and shift the xref table to match. Used for a repeatable + export (SOURCE_DATE_EPOCH). Leaves the file alone if it is not laid + out as Qt writes it. + */ + void setDocumentDate(const QString &pdfPath, const QDateTime &when); + struct ComponentInfo { QString contents; }; diff --git a/tests/qttest/CMakeLists.txt b/tests/qttest/CMakeLists.txt index 41951877e..1aea301e6 100644 --- a/tests/qttest/CMakeLists.txt +++ b/tests/qttest/CMakeLists.txt @@ -407,6 +407,19 @@ target_link_libraries(tst_pdfpagesize PRIVATE Qt::Test Qt::Gui) target_compile_definitions(tst_pdfpagesize PRIVATE "QET_TEST_BINARY_PATH=\"$\"") +# With SOURCE_DATE_EPOCH set, --export-pdf writes the same bytes for the +# same project in every run. Exports examples/741.qet several times, in +# separate processes. +add_executable( + tst_pdfreproducible + tst_pdfreproducible.cpp) +add_test(NAME tst_pdfreproducible COMMAND tst_pdfreproducible) +add_dependencies(tst_pdfreproducible qelectrotech) +target_link_libraries(tst_pdfreproducible PRIVATE Qt::Test) +target_compile_definitions(tst_pdfreproducible PRIVATE + "QET_TEST_BINARY_PATH=\"$\"" + "QET_EXAMPLES_DIR=\"${QET_DIR}/examples\"") + # One text per potential: the number goes on the same conductor in every # run when two conductors tie for longest. Exports # fixtures/one_text_per_potential_tie.qet through the real binary, several diff --git a/tests/qttest/tst_pdfreproducible.cpp b/tests/qttest/tst_pdfreproducible.cpp new file mode 100644 index 000000000..ff60fac44 --- /dev/null +++ b/tests/qttest/tst_pdfreproducible.cpp @@ -0,0 +1,91 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + With SOURCE_DATE_EPOCH set, --export-pdf writes the same bytes for the + same project in every run: the dates are the ones it names, the document + id comes from the project file, and the fonts come in a fixed order. + Without it the dates are the time of the export, as before. Exported in + separate processes, since what used to differ changed between runs. +*/ +#include +#include +#include +#include +#include +#include + +class tst_pdfreproducible : public QObject +{ + Q_OBJECT + + QTemporaryDir m_dir; + int m_run = 0; + + QByteArray exportPdf(const QString &project, const QByteArray &epoch) + { + const int run = m_run++; + const QString home = m_dir.filePath(QStringLiteral("home%1").arg(run)); + const QString tmp = m_dir.filePath(QStringLiteral("tmp%1").arg(run)); + const QString out = m_dir.filePath(QStringLiteral("out%1.pdf").arg(run)); + QDir().mkpath(home); + QDir().mkpath(tmp); + QProcessEnvironment env = QProcessEnvironment::systemEnvironment(); + env.insert(QStringLiteral("QT_QPA_PLATFORM"), QStringLiteral("offscreen")); + env.insert(QStringLiteral("HOME"), home); + env.insert(QStringLiteral("XDG_CONFIG_HOME"), home + QStringLiteral("/.config")); + env.insert(QStringLiteral("XDG_DATA_HOME"), home + QStringLiteral("/.local/share")); + env.insert(QStringLiteral("TMPDIR"), tmp); + env.remove(QStringLiteral("QT_HASH_SEED")); + env.remove(QStringLiteral("SOURCE_DATE_EPOCH")); + if (!epoch.isEmpty()) + env.insert(QStringLiteral("SOURCE_DATE_EPOCH"), QString::fromLatin1(epoch)); + + QProcess proc; + proc.setProcessEnvironment(env); + proc.start(QStringLiteral(QET_TEST_BINARY_PATH), + {QStringLiteral("--export-pdf"), project, out}); + if (!proc.waitForFinished(120000) || proc.exitCode() != 0) + return {}; + QFile file(out); + if (!file.open(QIODevice::ReadOnly)) + return {}; + return file.readAll(); + } + +private slots: + void initTestCase() + { + QVERIFY(m_dir.isValid()); + QVERIFY(QFile::exists(QStringLiteral(QET_TEST_BINARY_PATH))); + } + + void sameBytesEveryRun() + { + // A project with no cross-reference links: their order is fixed + // separately. + const QString project = + QStringLiteral(QET_EXAMPLES_DIR) + QStringLiteral("/741.qet"); + QVERIFY2(QFile::exists(project), "examples/741.qet not found"); + + const QByteArray first = exportPdf(project, "1700000000"); + QVERIFY(!first.isEmpty()); + QVERIFY(first.contains("/CreationDate (D:20231114221320Z)")); + QVERIFY(first.contains("xmp:CreateDate=\"2023-11-14T22:13:20Z\"")); + for (int i = 0; i < 3; ++i) + QVERIFY2(exportPdf(project, "1700000000") == first, + "two exports of the same project differ"); + } + + void nowWithoutTheVariable() + { + const QString project = + QStringLiteral(QET_EXAMPLES_DIR) + QStringLiteral("/741.qet"); + const QByteArray pdf = exportPdf(project, QByteArray()); + QVERIFY(!pdf.isEmpty()); + const QByteArray year = + "/CreationDate (D:" + QByteArray::number(QDate::currentDate().year()); + QVERIFY(pdf.contains(year)); + } +}; + +QTEST_MAIN(tst_pdfreproducible) +#include "tst_pdfreproducible.moc"