Skip to content

Commit

Permalink
Fixed cleanup of temporary files from tasks
Browse files Browse the repository at this point in the history 
closes #1141
  • Loading branch information
@hstct
hstct committed Sep 19, 2024
1 parent 3600541 commit 2123727
Show file tree
Hide file tree
Showing 7 changed files with 159 additions and 117 deletions.
1 change: 1 addition & 0 deletions CHANGES/1141.bugfix
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Temporary files are now cleaned after a task is done.
197 changes: 103 additions & 94 deletions pulp_deb/app/models/signing_service.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,106 +34,115 @@ def validate(self):
"""
with tempfile.TemporaryDirectory() as temp_directory_name:
test_release_path = os.path.join(temp_directory_name, "Release")
with open(test_release_path, "wb") as test_file:
test_data = b"arbitrary data"
test_file.write(test_data)
test_file.flush()
return_value = self.sign(test_release_path)

signatures = return_value.get("signatures")

if not signatures:
message = "The signing service script must report a 'signatures' field!"
raise RuntimeError(message)

if not isinstance(signatures, dict):
message = (
"The 'signatures' field reported by the signing service script must "
"contain a dict!"
)
raise RuntimeError(message)

if "inline" not in signatures and "detached" not in signatures:
message = (
"The dict contained in the 'signatures' field of the singing service "
"script must include an 'inline' field, a 'detached' field, or both!"
)
raise RuntimeError(message)

for signature_type, signature_file in signatures.items():
if not os.path.exists(signature_file):
try:
with open(test_release_path, "wb") as test_file:
test_data = b"arbitrary data"
test_file.write(test_data)
test_file.flush()
return_value = self.sign(test_release_path)

signatures = return_value.get("signatures")

if not signatures:
message = "The signing service script must report a 'signatures' field!"
raise RuntimeError(message)

if not isinstance(signatures, dict):
message = (
"The '{}' file, as reported in the 'signatures.{}' field of the "
"signing service script, doesn't appear to exist!"
"The 'signatures' field reported by the signing service script must "
"contain a dict!"
)
raise RuntimeError(message.format(signature_file, signature_type))

# Prepare GPG:
gpg = gnupg.GPG(gnupghome=temp_directory_name)
gpg.import_keys(self.public_key)
imported_keys = gpg.list_keys()

if len(imported_keys) != 1:
message = "We have imported more than one key! Aborting validation!"
raise RuntimeError(message)

if imported_keys[0]["fingerprint"] != self.pubkey_fingerprint:
message = (
"The signing service fingerprint does not appear to match its public key!"
)
raise RuntimeError(message)

# Verify InRelease file
inline_path = signatures.get("inline")
if inline_path:
if os.path.basename(inline_path) != "InRelease":
raise RuntimeError(message)

if "inline" not in signatures and "detached" not in signatures:
message = (
"The path returned via the 'signatures.inline' field of the signing "
"service script, must end with the 'InRelease' file name!"
"The dict contained in the 'signatures' field of the singing service "
"script must include an 'inline' field, a 'detached' field, or both!"
)
raise RuntimeError(message)
with open(inline_path, "rb") as inline:
verified = gpg.verify_file(inline)
if not verified.valid:
message = "GPG Verification of the inline file '{}' failed!"
raise RuntimeError(message.format(inline_path))

if verified.pubkey_fingerprint != self.pubkey_fingerprint:
message = "'{}' appears to have been signed using the wrong key!"
raise RuntimeError(message.format(inline_path))

# Also check that the non-signature part of the InRelease file is the same as
# the original Release file!
with open(inline_path, "rb") as inline:
inline_data = inline.read()
if b"-----BEGIN PGP SIGNED MESSAGE-----\n" not in inline_data:
message = "PGP message header is missing in the inline file '{}'."
raise RuntimeError(message.format(inline_path))
if b"-----BEGIN PGP SIGNATURE-----\n" not in inline_data:
message = "PGP signature header is missing in inline file '{}'."
raise RuntimeError(message.format(inline_path))
if test_data not in inline_data:

for signature_type, signature_file in signatures.items():
if not os.path.exists(signature_file):
message = (
"The inline file '{}' contains different data from the original "
"file."
"The '{}' file, as reported in the 'signatures.{}' field of the "
"signing service script, doesn't appear to exist!"
)
raise RuntimeError(message.format(inline_path))
raise RuntimeError(message.format(signature_file, signature_type))

# Verify Release.gpg file
detached_path = signatures.get("detached")
if detached_path:
if os.path.basename(detached_path) != "Release.gpg":
message = (
"The path returned via the 'signatures.detached' field of the signing "
"service script, must end with the 'Release.gpg' file name!"
)
# Prepare GPG:
gpg = gnupg.GPG(gnupghome=temp_directory_name)
gpg.import_keys(self.public_key)
imported_keys = gpg.list_keys()

if len(imported_keys) != 1:
message = "We have imported more than one key! Aborting validation!"
raise RuntimeError(message)

if imported_keys[0]["fingerprint"] != self.pubkey_fingerprint:
message = "The signing service fingerprint does not appear to match its public key!"
raise RuntimeError(message)
with open(signatures.get("detached"), "rb") as detached:
verified = gpg.verify_file(detached, test_release_path)
if not verified.valid:
message = "GPG Verification of the detached file '{}' failed!"
raise RuntimeError(message.format(detached_path))

if verified.pubkey_fingerprint != self.pubkey_fingerprint:
message = "'{}' appears to have been signed using the wrong key!"
raise RuntimeError(message.format(detached_path))

# Verify InRelease file
inline_path = signatures.get("inline")
if inline_path:
if os.path.basename(inline_path) != "InRelease":
message = (
"The path returned via the 'signatures.inline' field of the signing "
"service script, must end with the 'InRelease' file name!"
)
raise RuntimeError(message)
with open(inline_path, "rb") as inline:
verified = gpg.verify_file(inline)
if not verified.valid:
message = "GPG Verification of the inline file '{}' failed!"
raise RuntimeError(message.format(inline_path))

if verified.pubkey_fingerprint != self.pubkey_fingerprint:
message = "'{}' appears to have been signed using the wrong key!"
raise RuntimeError(message.format(inline_path))

# Also check that the non-signature part of the InRelease file is the same as
# the original Release file!
with open(inline_path, "rb") as inline:
inline_data = inline.read()
if b"-----BEGIN PGP SIGNED MESSAGE-----\n" not in inline_data:
message = "PGP message header is missing in the inline file '{}'."
raise RuntimeError(message.format(inline_path))
if b"-----BEGIN PGP SIGNATURE-----\n" not in inline_data:
message = "PGP signature header is missing in inline file '{}'."
raise RuntimeError(message.format(inline_path))
if test_data not in inline_data:
message = (
"The inline file '{}' contains different data from the original "
"file."
)
raise RuntimeError(message.format(inline_path))

# Verify Release.gpg file
detached_path = signatures.get("detached")
if detached_path:
if os.path.basename(detached_path) != "Release.gpg":
message = (
"The path returned via the 'signatures.detached' field of the signing "
"service script, must end with the 'Release.gpg' file name!"
)
raise RuntimeError(message)
with open(signatures.get("detached"), "rb") as detached:
verified = gpg.verify_file(detached, test_release_path)
if not verified.valid:
message = "GPG Verification of the detached file '{}' failed!"
raise RuntimeError(message.format(detached_path))

if verified.pubkey_fingerprint != self.pubkey_fingerprint:
message = "'{}' appears to have been signed using the wrong key!"
raise RuntimeError(message.format(detached_path))
finally:
cleanup_paths = [inline_path, detached_path]
for path in cleanup_paths:
if path and os.path.exists(path):
# Remove the file
os.remove(path)
# Remove the directory if it's empty
dir_path = os.path.dirname(path)
if os.path.exists(dir_path) and not os.listdir(dir_path):
os.rmdir(dir_path)
27 changes: 18 additions & 9 deletions pulp_deb/app/tasks/publishing.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -503,15 +503,24 @@ async def sign_metadata(self):
self.signed = await self.signing_service.asign(self.release_path)

def save_signed_metadata(self):
for signature_file in self.signed["signatures"].values():
file_name = os.path.basename(signature_file)
relative_path = os.path.join(self.release_dir, file_name)
metadata = PublishedMetadata.create_from_file(
publication=self.publication,
file=File(open(signature_file, "rb")),
relative_path=relative_path,
)
metadata.save()
try:
for signature_file in self.signed["signatures"].values():
with open(signature_file, "rb") as sig_file:
file_name = os.path.basename(signature_file)
relative_path = os.path.join(self.release_dir, file_name)
metadata = PublishedMetadata.create_from_file(
publication=self.publication,
file=File(sig_file),
relative_path=relative_path,
)
metadata.save()
finally:
for signature_file in self.signed["signatures"].values():
if os.path.exists(signature_file):
os.remove(signature_file)
signature_dir = os.path.dirname(signature_file)
if os.path.exists(signature_dir) and not os.listdir(signature_dir):
os.rmdir(signature_dir)


def _zip_file(file_path):
Expand Down
26 changes: 16 additions & 10 deletions pulp_deb/app/tasks/synchronizing.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -502,16 +502,22 @@ async def run(self):
# No main_artifact found, uncompress one
relative_dir = os.path.dirname(d_content.content.relative_path)
filename = _uncompress_artifact(d_content.d_artifacts, relative_dir)
da = DeclarativeArtifact(
artifact=Artifact.init_and_validate(
filename, expected_digests={"sha256": content.sha256}
),
url=filename,
relative_path=content.relative_path,
remote=d_content.d_artifacts[0].remote,
)
d_content.d_artifacts.append(da)
await _save_artifact_blocking(da)

try:
da = DeclarativeArtifact(
artifact=Artifact.init_and_validate(
filename, expected_digests={"sha256": content.sha256}
),
url=filename,
relative_path=content.relative_path,
remote=d_content.d_artifacts[0].remote,
)
d_content.d_artifacts.append(da)
await _save_artifact_blocking(da)
finally:
# Ensure the uncompressed file is deleted after usage
if os.path.exists(filename):
os.remove(filename)
content.artifact_set_sha256 = _get_artifact_set_sha256(
d_content, PackageIndex.SUPPORTED_ARTIFACTS
)
Expand Down
21 changes: 17 additions & 4 deletions pulp_deb/tests/functional/conftest.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
import pytest
import os
import re
import shutil
import stat
import subprocess

Expand Down Expand Up @@ -506,7 +507,7 @@ def deb_signing_service_factory(
"--gnupghome",
str(gpg.gnupghome),
)
process = subprocess.run(cmd, capture_output=True)
process = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)

assert process.returncode == 0

Expand All @@ -517,11 +518,23 @@ def deb_signing_service_factory(

yield signing_service

temp_dir_log = "/tmp/signing_temp_dir.log"
if os.path.exists(temp_dir_log):
with open(temp_dir_log, "r") as f:
temp_dir = f.read().strip()

if os.path.exists(temp_dir):
shutil.rmtree(temp_dir)
os.remove(temp_dir_log)

cmd = (
"from pulpcore.app.models import SigningService;"
f"SigningService.objects.filter(name='{service_name}').delete()"
"pulpcore-manager",
"remove-signing-service",
service_name,
"--class",
"deb:AptReleaseSigningService",
)
process = subprocess.run(["pulpcore-manager", "shell", "-c", cmd], capture_output=True)
process = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
assert process.returncode == 0


Expand Down
2 changes: 2 additions & 0 deletions pulp_deb/tests/functional/constants.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -380,6 +380,8 @@ def _clean_dict(d):
GPG_KEY_ID="GPGKEYIDHERE"
COMMON_GPG_OPTS="--batch --armor --digest-algo SHA256"
echo "${OUTPUT_DIR}" > /tmp/signing_temp_dir.log
# Create a detached signature
/usr/bin/gpg ${COMMON_GPG_OPTS} \
--detach-sign \
Expand Down
2 changes: 2 additions & 0 deletions pulp_deb/tests/functional/sign_deb_release.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,8 @@ INLINE_SIGNATURE_PATH="${OUTPUT_DIR}/InRelease"
GPG_KEY_ID="Pulp QE"
COMMON_GPG_OPTS="--batch --armor --digest-algo SHA256"

echo "${OUTPUT_DIR}" > /tmp/signing_temp_dir.log

# Create a detached signature
/usr/bin/gpg ${COMMON_GPG_OPTS} \
--detach-sign \
Expand Down

0 comments on commit 2123727

Please sign in to comment.