Write the same PDF for the same project when SOURCE_DATE_EPOCH is set

Two exports of one project differed in the creation and modification
dates (the time of the export, in local time) and in the document id
(random). With SOURCE_DATE_EPOCH set (reproducible-builds.org),
--export-pdf now uses the time it names, in UTC, and a document id
derived from the project: version 5 of the project uuid with the
SHA-256 of the project file, so it is the same for the same file and
changes when the file does. Qt has no setter for the dates, so
PdfLinks::setDocumentDate() rewrites them in the document information
and the XMP metadata after the file is written, fixing the XMP /Length
and shifting the xref table. Without the variable nothing changes.

Only --export-pdf reads it; the print window's PDF export does not, as
a person exporting by hand wants the real date.

qet_export (misc/qet-mcp) gains "reproducible" and "source_date_epoch".
It reports whether the build honoured the variable, since an older one
ignores it.

tst_pdfreproducible exports examples/741.qet four times and requires
identical bytes and the epoch's date; it fails with either this change
or the hash seed fix reverted. Projects with cross-reference links also
need their links in a fixed order, which is a separate fix.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
ispyisail
2026-10-03 07:40:32 +13:00
parent 6e478e0e3f
commit 51d20abca4
8 changed files with 413 additions and 4 deletions
+9
View File
@@ -570,6 +570,15 @@ Python, plus the hang guard on `addConductor` and the database refresh in
a terminal sharing its point with another, where the order is undefined;
the uuid tells those apart. `wiring_list_view` carries the same four
columns for `qet_query`.
- **`"reproducible": true` makes a PDF comparable byte for byte.** Two
exports of the same project otherwise differ in their dates and document
id. The option sets `SOURCE_DATE_EPOCH` for the run
([reproducible-builds.org](https://reproducible-builds.org/specs/source-date-epoch/)):
the PDF then carries that date in UTC, a document id derived from the
project file, and its fonts in a fixed order. The date is
`source_date_epoch` if given, else the server's own `SOURCE_DATE_EPOCH`,
else 0 (1 January 1970). The result's `"reproducible"` is false, with a
hint, when the QElectroTech build is too old to honour it.
- **`qet_export` isolates its launch.** SingleApplication keys its socket
on `applicationFilePath()`, so a second launch of the same binary path
forwards its request to an already-running instance and returns *that*
+62 -4
View File
@@ -56,6 +56,7 @@ workspace policy applies exactly as it does over stdio.
from __future__ import annotations
import datetime
import json
import re
import os
@@ -942,7 +943,8 @@ def _collection_setting(collection: PurePath) -> str:
def _run_qet(binary: str, args: list[str], timeout: int = 180,
elements_dir: str | None = None,
script: str | None = None, tail: int = 4000) -> dict:
script: str | None = None, tail: int = 4000,
extra_env: dict | None = None) -> dict:
"""Launch QElectroTech headlessly, carrying the known launch traps.
SingleApplication keys its socket on applicationFilePath(), so a second
@@ -1000,6 +1002,7 @@ def _run_qet(binary: str, args: list[str], timeout: int = 180,
script_path.write_text(script, encoding="utf-8")
args = ["--run", str(script_path), *args]
env = _launch_env(dict(os.environ), home, os.name == "nt")
env.update(extra_env or {})
try:
p = subprocess.run([str(exe), *args], env=env, timeout=timeout,
capture_output=True, text=True)
@@ -1032,8 +1035,30 @@ def _run_qet(binary: str, args: list[str], timeout: int = 180,
return result
def _source_date_epoch(value) -> int:
"""The SOURCE_DATE_EPOCH a reproducible export uses: @p value when the
caller gives one, else the one this server was started with, else 0.
0 (1 January 1970) rather than a date read from the project: the file's
modification time changes with every copy and checkout, and a date the
drawing carries is a title block field, not when the PDF was made. A
caller who wants a meaningful date passes it.
"""
if value is None:
value = os.environ.get("SOURCE_DATE_EPOCH", "0")
try:
seconds = None if isinstance(value, (bool, float)) else int(value)
except (TypeError, ValueError):
seconds = None
if seconds is None or seconds < 0:
raise ValueError(f"source_date_epoch must be a whole number of seconds "
f"since 1970, got {value!r}")
return seconds
def tool_export(binary: str, project: str, format: str, output: str,
timeout: int = 180) -> dict:
timeout: int = 180, reproducible: bool = False,
source_date_epoch: int | None = None) -> dict:
if format not in EXPORT_FORMATS:
raise ValueError(f"unknown format {format!r}; "
f"expected one of {', '.join(sorted(EXPORT_FORMATS))}")
@@ -1047,12 +1072,31 @@ def tool_export(binary: str, project: str, format: str, output: str,
# application starts its GUI instead, which then hangs on an offscreen
# platform. Order matters here.
flag = EXPORT_FORMATS[format]
result = _run_qet(binary, [flag, str(proj), output], timeout)
# SOURCE_DATE_EPOCH (reproducible-builds.org) makes --export-pdf write
# the same bytes for the same project: the dates it names instead of
# now, a document id from the project file, fonts in a fixed order.
extra_env = None
if reproducible or source_date_epoch is not None:
epoch = _source_date_epoch(source_date_epoch)
extra_env = {"SOURCE_DATE_EPOCH": str(epoch)}
result = _run_qet(binary, [flag, str(proj), output], timeout,
extra_env=extra_env)
out = Path(output).expanduser()
result["output"] = str(out)
result["output_exists"] = out.exists()
if out.exists() and out.is_file():
result["output_bytes"] = out.stat().st_size
if extra_env and format == "pdf":
# A QElectroTech older than this option ignores the variable and
# writes the time of the export, so say whether this one did.
stamp = datetime.datetime.fromtimestamp(epoch, datetime.timezone.utc)
honoured = (b"/CreationDate (D:" + stamp.strftime("%Y%m%d%H%M%S").encode()
+ b"Z)") in out.read_bytes()
result["reproducible"] = honoured
if not honoured:
result["hint"] = ("this QElectroTech ignores SOURCE_DATE_EPOCH, so the "
"PDF carries the time of the export; it needs a build "
"with repeatable PDF export")
return result
@@ -3485,11 +3529,25 @@ TOOLS = [
"description": "replace \"output\" if it already exists; "
"without this an existing file is never clobbered"},
"timeout": {"type": "integer", "default": 180},
"reproducible": {"type": "boolean", "default": False,
"description": "pdf: write the same bytes for the same "
"project in every run, so two exports can be "
"compared byte for byte. Sets "
"SOURCE_DATE_EPOCH; the result's "
"\"reproducible\" says whether this "
"QElectroTech honoured it"},
"source_date_epoch": {"type": "integer", "minimum": 0,
"description": "with reproducible: the date the PDF "
"carries, in seconds since 1970 UTC. "
"Default: this server's own "
"SOURCE_DATE_EPOCH, else 0"},
},
"required": ["project", "format", "output"],
},
"handler": lambda a: tool_export(a["binary"], a["project"], a["format"],
a["output"], a.get("timeout", 180)),
a["output"], a.get("timeout", 180),
a.get("reproducible", False),
a.get("source_date_epoch")),
},
{
"name": "qet_edit",
+59
View File
@@ -31,6 +31,7 @@ import subprocess
import sys
import tempfile
import unittest
import unittest.mock
from unittest import mock
import xml.etree.ElementTree as ET
from pathlib import Path
@@ -3453,6 +3454,43 @@ class ProjectNewValidation(unittest.TestCase):
self.assertFalse(Path(new).exists())
class ReproducibleExport(unittest.TestCase):
"""qet_export's "reproducible" sets SOURCE_DATE_EPOCH for the run."""
def run_export(self, **kw):
seen = {}
def fake(binary, args, timeout=180, **rest):
seen.update(rest)
return {"ok": True}
with tempfile.TemporaryDirectory() as tmp:
proj = Path(tmp) / "a.qet"
proj.write_text("<project/>")
with unittest.mock.patch.object(m, "_run_qet", fake):
m.tool_export("qet", str(proj), "pdf", str(Path(tmp) / "o.pdf"), **kw)
return seen.get("extra_env")
def test_off_by_default(self):
self.assertIsNone(self.run_export())
def test_zero_unless_told_otherwise(self):
with unittest.mock.patch.dict(os.environ):
os.environ.pop("SOURCE_DATE_EPOCH", None)
self.assertEqual(self.run_export(reproducible=True),
{"SOURCE_DATE_EPOCH": "0"})
def test_the_servers_own_variable_then_the_callers_value(self):
with unittest.mock.patch.dict(os.environ, {"SOURCE_DATE_EPOCH": "1600000000"}):
self.assertEqual(self.run_export(reproducible=True),
{"SOURCE_DATE_EPOCH": "1600000000"})
self.assertEqual(self.run_export(reproducible=True, source_date_epoch=5),
{"SOURCE_DATE_EPOCH": "5"})
def test_a_bad_date_is_refused_before_launching(self):
for bad in (-1, "soon", 1.5, True):
with self.subTest(value=bad), self.assertRaises(ValueError):
self.run_export(reproducible=True, source_date_epoch=bad)
class ReadTools(unittest.TestCase):
def test_project_info_and_scan_on_a_fixture(self):
with tempfile.TemporaryDirectory() as tmp:
@@ -5415,6 +5453,27 @@ class UuidIndexLookups(unittest.TestCase):
@needs_examples
class CorpusIntegration(unittest.TestCase):
def test_a_reproducible_pdf_is_the_same_bytes_every_run(self):
"""Two reproducible exports of one project, in separate runs, are
byte for byte the same; a plain one carries the time of the export.
741.qet has no cross-reference links, whose order is fixed apart."""
project = str(Path(EXAMPLES) / "741.qet")
with tempfile.TemporaryDirectory() as tmp:
outs = []
for i in range(2):
out = str(Path(tmp) / f"r{i}.pdf")
r = m.tool_export(BINARY, project, "pdf", out, reproducible=True,
source_date_epoch=1700000000)
self.assertTrue(r["ok"], r)
self.assertTrue(r["reproducible"], r)
outs.append(Path(out).read_bytes())
self.assertEqual(outs[0], outs[1])
plain = str(Path(tmp) / "plain.pdf")
r = m.tool_export(BINARY, project, "pdf", plain)
self.assertNotIn("reproducible", r)
self.assertNotIn(b"/CreationDate (D:20231114221320Z)",
Path(plain).read_bytes())
def test_folio_counts_match_what_qelectrotech_itself_holds(self):
"""Element and conductor counts per folio, from the file, against
QElectroTech's own counts after loading it -- over every example.
+36
View File
@@ -42,7 +42,9 @@
#include <QDir>
#include <QDirIterator>
#include <QDomDocument>
#include <QCryptographicHash>
#include <QDate>
#include <QDateTime>
#include <QFile>
#include <QSaveFile>
#include <QFileInfo>
@@ -59,7 +61,9 @@
#include <QSqlQuery>
#include <QSvgGenerator>
#include <QTextStream>
#include <QTimeZone>
#include <QTransform>
#include <QUuid>
namespace {
@@ -147,6 +151,23 @@ void renderDiagram(Diagram *diagram, QPainter &painter, const QRectF &target,
diagram->setDrawTerminalNames(was_drawing_terminal_names);
}
/// The time SOURCE_DATE_EPOCH names, in seconds since 1970 UTC, or an
/// invalid QDateTime when it is unset. A value that is not a whole number of
/// seconds is reported and ignored.
QDateTime sourceDateEpoch()
{
const QByteArray value = qgetenv("SOURCE_DATE_EPOCH");
if (value.isEmpty())
return {};
bool ok = false;
const qlonglong seconds = value.toLongLong(&ok);
if (!ok || seconds < 0) {
err << "SOURCE_DATE_EPOCH '" << value << "' is not a number of seconds; ignored.\n";
return {};
}
return QDateTime::fromSecsSinceEpoch(seconds, QTimeZone::UTC);
}
int exportPdf(QETProject &project, const QString &output,
bool showTerminals = false)
{
@@ -166,6 +187,19 @@ int exportPdf(QETProject &project, const QString &output,
writer.setCreator("QElectroTech");
writer.setResolution(96);
// SOURCE_DATE_EPOCH (reproducible-builds.org) asks for the same file
// from the same input: the time it names instead of now, and a document
// id from the project file instead of a random one. Qt has no setter for
// the dates, so they are rewritten once the file is written.
const QDateTime sourceDate = sourceDateEpoch();
if (sourceDate.isValid()) {
QCryptographicHash hash(QCryptographicHash::Sha256);
QFile file(project.filePath());
if (file.open(QIODevice::ReadOnly))
hash.addData(&file);
writer.setDocumentId(QUuid::createUuidV5(project.uuid(), hash.result()));
}
QPainter painter;
bool first = true;
for (Diagram *diagram : diagrams) {
@@ -240,6 +274,8 @@ int exportPdf(QETProject &project, const QString &output,
// the cross-references jump inside the document in any PDF viewer.
PdfLinks::convertUriToGoTo(output);
PdfLinks::removeUnusedPdfxNamespace(output);
if (sourceDate.isValid())
PdfLinks::setDocumentDate(output, sourceDate);
out << "Exported " << diagrams.size() << " page(s) -> " << output << "\n";
return 0;
+132
View File
@@ -35,6 +35,8 @@
#include <QUrl>
#include <QVector>
#include <algorithm>
namespace PdfLinks {
void injectCrossRefLinks(QPdfEngine *engine, Diagram *diagram,
@@ -411,6 +413,136 @@ void removeUnusedPdfxNamespace(const QString &pdfPath)
f.close();
}
void setDocumentDate(const QString &pdfPath, const QDateTime &when)
{
QFile f(pdfPath);
if (!f.open(QIODevice::ReadOnly)) return;
const QByteArray data = f.readAll();
f.close();
const QDateTime utc = when.toUTC();
const QByteArray pdfDate =
"(D:" + utc.toString(QStringLiteral("yyyyMMddHHmmss")).toLatin1() + "Z)";
const QByteArray xmpDate =
utc.toString(QStringLiteral("yyyy-MM-ddTHH:mm:ss")).toLatin1() + "Z";
// Each edit replaces [start, end) of the original file.
struct Edit { int start; int end; QByteArray text; };
QList<Edit> edits;
// The document information: "/CreationDate (D:...)" and "/ModDate (D:...)".
// Its other strings are UTF-16, so they cannot hold these keys.
for (const QByteArray &key : {QByteArray("/CreationDate "), QByteArray("/ModDate ")}) {
const int k = data.indexOf(key);
if (k == -1) return;
const int start = k + key.size();
const int end = data.indexOf(')', start);
if (end == -1 || data.at(start) != '(') return;
edits.append({start, end + 1, pdfDate});
}
// The XMP metadata stream, which Qt writes uncompressed: its three
// dates, and its /Length, which changes with them.
const QByteArray metadata("/Type /Metadata /Subtype /XML");
const int m = data.indexOf(metadata);
if (m == -1) return;
const QByteArray lengthKey("/Length ");
const int l = data.indexOf(lengthKey, m);
const int streamStart = data.indexOf("stream\n", m);
if (l == -1 || streamStart == -1 || l > streamStart) return;
const int lengthStart = l + lengthKey.size();
int lengthEnd = lengthStart;
while (lengthEnd < data.size() && QChar(data.at(lengthEnd)).isDigit())
++lengthEnd;
const int streamEnd = data.indexOf("endstream", streamStart);
if (lengthEnd == lengthStart || streamEnd == -1) return;
// The last of each in the stream: the title and author come before
// the dates, and are text a project could fill with anything.
int xmpDelta = 0;
QList<Edit> xmpEdits;
for (const QByteArray &attr : {QByteArray("xmp:CreateDate=\""),
QByteArray("xmp:ModifyDate=\""),
QByteArray("xmp:MetadataDate=\"")}) {
const int a = data.lastIndexOf(attr, streamEnd);
if (a < streamStart) return;
const int start = a + attr.size();
const int end = data.indexOf('"', start);
if (end == -1 || end > streamEnd) return;
xmpEdits.append({start, end, xmpDate});
xmpDelta += xmpDate.size() - (end - start);
}
const int length = data.mid(lengthStart, lengthEnd - lengthStart).toInt();
edits.append({lengthStart, lengthEnd, QByteArray::number(length + xmpDelta)});
edits.append(xmpEdits);
std::sort(edits.begin(), edits.end(),
[](const Edit &a, const Edit &b) { return a.start < b.start; });
// Where an offset in the original file is in the new one.
auto moved = [&edits](int offset) {
int shift = 0;
for (const Edit &e : std::as_const(edits))
if (e.end <= offset)
shift += e.text.size() - (e.end - e.start);
return offset + shift;
};
// The xref table the last startxref points at, which is the one a
// reader uses.
const int sx = data.lastIndexOf("startxref");
if (sx == -1) return;
int numStart = sx + 9;
while (numStart < data.size() && QChar(data.at(numStart)).isSpace())
++numStart;
int numEnd = numStart;
while (numEnd < data.size() && QChar(data.at(numEnd)).isDigit())
++numEnd;
const int xref = data.mid(numStart, numEnd - numStart).toInt();
if (numEnd == numStart || !data.mid(xref).startsWith("xref")) return;
for (const Edit &e : std::as_const(edits))
if (e.end > xref) return; // every edit is in the objects before it
QByteArray out;
out.reserve(data.size() + 64);
int pos = 0;
for (const Edit &e : std::as_const(edits)) {
if (e.start < pos) return; // overlapping: not a file Qt wrote
out += data.mid(pos, e.start - pos);
out += e.text;
pos = e.end;
}
// The xref entries: "0000012345 00000 n \n", 20 bytes each, after the
// "xref" line and one "first count" line.
int line = data.indexOf('\n', xref) + 1;
line = data.indexOf('\n', line) + 1;
if (line <= 0) return;
out += data.mid(pos, line - pos);
pos = line;
static const QRegularExpression entry(QStringLiteral(R"(^(\d{10}) (\d{5}) ([nf])\s*$)"));
while (pos + 18 <= data.size()) {
const QByteArray row = data.mid(pos, 20);
const auto match = entry.match(QString::fromLatin1(row));
if (!match.hasMatch()) break;
if (match.captured(3) == QLatin1String("n")) {
const int offset = match.captured(1).toInt();
out += QByteArray::number(moved(offset)).rightJustified(10, '0');
out += row.mid(10);
} else {
out += row;
}
pos += 20;
}
out += data.mid(pos, numStart - pos);
out += QByteArray::number(moved(xref));
out += data.mid(numEnd);
if (!f.open(QIODevice::WriteOnly | QIODevice::Truncate)) return;
f.write(out);
f.close();
}
void convertComponentInfoAnnotations(const QString &pdfPath,
const QList<ComponentInfo> &annotations)
{
+11
View File
@@ -18,6 +18,7 @@
#ifndef PDF_LINKS_H
#define PDF_LINKS_H
#include <QDateTime>
#include <QMap>
#include <QPointF>
#include <QRectF>
@@ -83,6 +84,16 @@ namespace PdfLinks {
*/
void removeUnusedPdfxNamespace(const QString &pdfPath);
/**
Post-process a Qt-generated PDF file: replace the creation and
modification dates Qt wrote (the time of the export, in local time)
with @p when, in UTC, in the document information and the XMP
metadata, and shift the xref table to match. Used for a repeatable
export (SOURCE_DATE_EPOCH). Leaves the file alone if it is not laid
out as Qt writes it.
*/
void setDocumentDate(const QString &pdfPath, const QDateTime &when);
struct ComponentInfo {
QString contents;
};
+13
View File
@@ -407,6 +407,19 @@ target_link_libraries(tst_pdfpagesize PRIVATE Qt::Test Qt::Gui)
target_compile_definitions(tst_pdfpagesize PRIVATE
"QET_TEST_BINARY_PATH=\"$<TARGET_FILE:qelectrotech>\"")
# With SOURCE_DATE_EPOCH set, --export-pdf writes the same bytes for the
# same project in every run. Exports examples/741.qet several times, in
# separate processes.
add_executable(
tst_pdfreproducible
tst_pdfreproducible.cpp)
add_test(NAME tst_pdfreproducible COMMAND tst_pdfreproducible)
add_dependencies(tst_pdfreproducible qelectrotech)
target_link_libraries(tst_pdfreproducible PRIVATE Qt::Test)
target_compile_definitions(tst_pdfreproducible PRIVATE
"QET_TEST_BINARY_PATH=\"$<TARGET_FILE:qelectrotech>\""
"QET_EXAMPLES_DIR=\"${QET_DIR}/examples\"")
# One text per potential: the number goes on the same conductor in every
# run when two conductors tie for longest. Exports
# fixtures/one_text_per_potential_tie.qet through the real binary, several
+91
View File
@@ -0,0 +1,91 @@
// SPDX-License-Identifier: GPL-2.0-or-later
/*
With SOURCE_DATE_EPOCH set, --export-pdf writes the same bytes for the
same project in every run: the dates are the ones it names, the document
id comes from the project file, and the fonts come in a fixed order.
Without it the dates are the time of the export, as before. Exported in
separate processes, since what used to differ changed between runs.
*/
#include <QDateTime>
#include <QDir>
#include <QFile>
#include <QProcess>
#include <QTemporaryDir>
#include <QtTest>
class tst_pdfreproducible : public QObject
{
Q_OBJECT
QTemporaryDir m_dir;
int m_run = 0;
QByteArray exportPdf(const QString &project, const QByteArray &epoch)
{
const int run = m_run++;
const QString home = m_dir.filePath(QStringLiteral("home%1").arg(run));
const QString tmp = m_dir.filePath(QStringLiteral("tmp%1").arg(run));
const QString out = m_dir.filePath(QStringLiteral("out%1.pdf").arg(run));
QDir().mkpath(home);
QDir().mkpath(tmp);
QProcessEnvironment env = QProcessEnvironment::systemEnvironment();
env.insert(QStringLiteral("QT_QPA_PLATFORM"), QStringLiteral("offscreen"));
env.insert(QStringLiteral("HOME"), home);
env.insert(QStringLiteral("XDG_CONFIG_HOME"), home + QStringLiteral("/.config"));
env.insert(QStringLiteral("XDG_DATA_HOME"), home + QStringLiteral("/.local/share"));
env.insert(QStringLiteral("TMPDIR"), tmp);
env.remove(QStringLiteral("QT_HASH_SEED"));
env.remove(QStringLiteral("SOURCE_DATE_EPOCH"));
if (!epoch.isEmpty())
env.insert(QStringLiteral("SOURCE_DATE_EPOCH"), QString::fromLatin1(epoch));
QProcess proc;
proc.setProcessEnvironment(env);
proc.start(QStringLiteral(QET_TEST_BINARY_PATH),
{QStringLiteral("--export-pdf"), project, out});
if (!proc.waitForFinished(120000) || proc.exitCode() != 0)
return {};
QFile file(out);
if (!file.open(QIODevice::ReadOnly))
return {};
return file.readAll();
}
private slots:
void initTestCase()
{
QVERIFY(m_dir.isValid());
QVERIFY(QFile::exists(QStringLiteral(QET_TEST_BINARY_PATH)));
}
void sameBytesEveryRun()
{
// A project with no cross-reference links: their order is fixed
// separately.
const QString project =
QStringLiteral(QET_EXAMPLES_DIR) + QStringLiteral("/741.qet");
QVERIFY2(QFile::exists(project), "examples/741.qet not found");
const QByteArray first = exportPdf(project, "1700000000");
QVERIFY(!first.isEmpty());
QVERIFY(first.contains("/CreationDate (D:20231114221320Z)"));
QVERIFY(first.contains("xmp:CreateDate=\"2023-11-14T22:13:20Z\""));
for (int i = 0; i < 3; ++i)
QVERIFY2(exportPdf(project, "1700000000") == first,
"two exports of the same project differ");
}
void nowWithoutTheVariable()
{
const QString project =
QStringLiteral(QET_EXAMPLES_DIR) + QStringLiteral("/741.qet");
const QByteArray pdf = exportPdf(project, QByteArray());
QVERIFY(!pdf.isEmpty());
const QByteArray year =
"/CreationDate (D:" + QByteArray::number(QDate::currentDate().year());
QVERIFY(pdf.contains(year));
}
};
QTEST_MAIN(tst_pdfreproducible)
#include "tst_pdfreproducible.moc"