From 54e96e9e674787efbe681b28c164c2453b8a023d Mon Sep 17 00:00:00 2001 From: Deep Mistry Date: Fri, 5 Dec 2025 12:57:08 -0500 Subject: [PATCH] Updates garbage collection for orphaned ReleasePayloads to use coordinated deletion via remove__ prefixed tags in quay.io --- cmd/release-controller/sync.go | 10 ++- cmd/release-controller/sync_gc.go | 18 ++++- cmd/release-controller/sync_release.go | 94 ++++++++++++++++++++++++++ pkg/release-controller/types.go | 2 +- 4 files changed, 120 insertions(+), 4 deletions(-) diff --git a/cmd/release-controller/sync.go b/cmd/release-controller/sync.go index 544f923d7..08e9de08f 100644 --- a/cmd/release-controller/sync.go +++ b/cmd/release-controller/sync.go @@ -490,8 +490,14 @@ func (c *Controller) syncReady(release *releasecontroller.Release) error { mirror, err := releasecontroller.GetMirror(release, releaseTag.Name, c.releaseLister) if err != nil { klog.Errorf("Failed to identify `from` mirror for creation of release mirror job: %v", err) - } else if _, err := c.ensureReleaseMirrorJob(release, releaseTag.Name, mirror); err != nil { - klog.Errorf("Failed to create release mirror job: %v", err) + } else { + if _, err := c.ensureReleaseMirrorJob(release, releaseTag.Name, mirror); err != nil { + klog.Errorf("Failed to create release mirror job: %v", err) + } + // For CI payloads, also create rc_payload__ tag for pruner to preserve component images + if _, err := c.ensureRCPayloadTagJob(release, releaseTag.Name, mirror); err != nil { + klog.Errorf("Failed to create rc_payload__ tag job: %v", err) + } } if err := c.ensureReleaseUpgradeJobs(release, releaseTag); err != nil { diff --git a/cmd/release-controller/sync_gc.go b/cmd/release-controller/sync_gc.go index 0aa50e3a9..11ce6fd96 100644 --- a/cmd/release-controller/sync_gc.go +++ b/cmd/release-controller/sync_gc.go @@ -122,11 +122,27 @@ func (c *Controller) garbageCollectSync() error { } } - // all releasepayloads created for releases that no longer exist should be deleted + // all releasepayloads created for releases that no longer exist should be garbage collected for _, payload := range payloads { if active.Has(payload.Name) { continue } + + // Get the release config from the ImageStream to check if alternate repository is configured + imageStream, err := c.releaseLister.ImageStreams(payload.Spec.PayloadCoordinates.Namespace).Get(payload.Spec.PayloadCoordinates.ImagestreamName) + if err == nil { + release, ok, err := releasecontroller.ReleaseDefinition(imageStream, c.parsedReleaseConfigCache, c.eventRecorder, *c.releaseLister) + if err == nil && ok && len(release.Config.AlternateImageRepository) > 0 && len(release.Config.AlternateImageRepositorySecretName) > 0 { + _, err := c.ensureRemoveTagJob(payload, release) + if err != nil { + klog.V(2).Infof("Failed to create remove tag job for releasepayload %s/%s: %v, proceeding with direct deletion", payload.Namespace, payload.Name, err) + } else { + klog.V(2).Infof("Created remove tag job for orphaned releasepayload %s/%s, pruner will handle quay.io tag deletion", payload.Namespace, payload.Name) + } + } + } + + // Delete the ReleasePayload klog.V(2).Infof("Removing orphaned releasepayload %s/%s", payload.Namespace, payload.Name) if err := c.releasePayloadClient.ReleasePayloads(payload.Namespace).Delete(context.TODO(), payload.Name, metav1.DeleteOptions{}); err != nil && !errors.IsNotFound(err) { utilruntime.HandleError(fmt.Errorf("can't delete orphaned releasepayload %s/%s: %v", payload.Namespace, payload.Name, err)) diff --git a/cmd/release-controller/sync_release.go b/cmd/release-controller/sync_release.go index 6d654603c..fe2b3be24 100644 --- a/cmd/release-controller/sync_release.go +++ b/cmd/release-controller/sync_release.go @@ -4,8 +4,10 @@ import ( "context" "fmt" "strconv" + "strings" "time" + "github.com/openshift/release-controller/pkg/apis/release/v1alpha1" releasecontroller "github.com/openshift/release-controller/pkg/release-controller" batchv1 "k8s.io/api/batch/v1" @@ -417,3 +419,95 @@ func (c *Controller) ensureReleaseMirrorJob(release *releasecontroller.Release, func releaseMirrorJobName(tagName string) string { return fmt.Sprintf("%s-alternate-mirror", tagName) } + +// ensureRCPayloadTagJob creates a job to mirror the release to rc_payload__{version} in quay.io for CI payloads +// This is required for the pruner to preserve component images referenced by CI payloads +func (c *Controller) ensureRCPayloadTagJob(release *releasecontroller.Release, name string, mirror *imagev1.ImageStream) (*batchv1.Job, error) { + if len(release.Config.AlternateImageRepository) == 0 || len(release.Config.AlternateImageRepositorySecretName) == 0 { + return nil, fmt.Errorf("alternate repository or secret not configured") + } + + // Only create rc_payload__ tags for CI payloads (tags containing .ci-) + // Nightly payloads reference ART images that are permanently stored + if !strings.Contains(name, ".ci-") { + return nil, nil + } + + jobName := fmt.Sprintf("%s-rc-payload-tag", name) + return c.ensureJob(jobName, nil, func() (*batchv1.Job, error) { + fromImage := fmt.Sprintf("%s:%s", release.Target.Status.PublicDockerImageRepository, name) + rcPayloadTag := fmt.Sprintf("rc_payload__%s", name) + toImage := fmt.Sprintf("%s:%s", release.Config.AlternateImageRepository, rcPayloadTag) + + cliImage := fmt.Sprintf("%s:cli", mirror.Status.DockerImageRepository) + if len(release.Config.OverrideCLIImage) > 0 { + cliImage = release.Config.OverrideCLIImage + } + + job, prefix := newReleaseJobBase(jobName, cliImage, release.Config.AlternateImageRepositorySecretName) + + manifestListMode := "false" + if c.manifestListMode && !release.Config.DisableManifestListMode { + manifestListMode = "true" + } + + job.Spec.Template.Spec.Containers[0].Command = []string{ + "/bin/bash", "-c", + prefix + ` + oc image mirror --keep-manifest-list=$1 $2 $3 + `, + "", + manifestListMode, fromImage, toImage, + } + + job.Annotations[releasecontroller.ReleaseAnnotationSource] = mirror.Annotations[releasecontroller.ReleaseAnnotationSource] + job.Annotations[releasecontroller.ReleaseAnnotationTarget] = mirror.Annotations[releasecontroller.ReleaseAnnotationTarget] + job.Annotations[releasecontroller.ReleaseAnnotationGeneration] = strconv.FormatInt(release.Target.Generation, 10) + job.Annotations[releasecontroller.ReleaseAnnotationReleaseTag] = mirror.Annotations[releasecontroller.ReleaseAnnotationReleaseTag] + + klog.V(2).Infof("Creating rc_payload__ tag job %s/%s to mirror %s to %s", c.jobNamespace, job.Name, fromImage, toImage) + return job, nil + }) +} + +// ensureRemoveTagJob creates a job to copy the release tag to remove__rc_payload__{version} in quay.io +func (c *Controller) ensureRemoveTagJob(payload *v1alpha1.ReleasePayload, release *releasecontroller.Release) (*batchv1.Job, error) { + if len(release.Config.AlternateImageRepository) == 0 || len(release.Config.AlternateImageRepositorySecretName) == 0 { + return nil, fmt.Errorf("alternate repository or secret not configured") + } + + jobName := fmt.Sprintf("%s-remove-tag", payload.Name) + return c.ensureJob(jobName, nil, func() (*batchv1.Job, error) { + // Get cli image from mirror or config + cliImage := "registry.ci.openshift.org/ocp/4.21:cli" + if mirror, err := c.releaseLister.ImageStreams(release.Target.Namespace).Get(release.Target.Name); err == nil { + cliImage = fmt.Sprintf("%s:cli", mirror.Status.DockerImageRepository) + } + if len(release.Config.OverrideCLIImage) > 0 { + cliImage = release.Config.OverrideCLIImage + } + + job, prefix := newReleaseJobBase(jobName, cliImage, release.Config.AlternateImageRepositorySecretName) + + // Mirror from the actual release tag (which exists in quay.io) to the removal request tag + // The pruner only cares about the tag name, not the content + removeTag := fmt.Sprintf("remove__rc_payload__%s", payload.Name) + fromImage := fmt.Sprintf("%s:%s", release.Config.AlternateImageRepository, payload.Name) + toImage := fmt.Sprintf("%s:%s", release.Config.AlternateImageRepository, removeTag) + + job.Spec.Template.Spec.Containers[0].Command = []string{ + "/bin/bash", "-c", + prefix + ` + oc image mirror --keep-manifest-list=true $1 $2 + `, + "", + fromImage, toImage, + } + + job.Annotations[releasecontroller.ReleaseAnnotationReleaseTag] = payload.Name + job.Annotations[releasecontroller.ReleaseAnnotationTarget] = fmt.Sprintf("%s/%s", payload.Spec.PayloadCoordinates.Namespace, payload.Spec.PayloadCoordinates.ImagestreamName) + + klog.V(2).Infof("Creating remove tag job %s/%s to copy %s to %s", c.jobNamespace, job.Name, fromImage, toImage) + return job, nil + }) +} diff --git a/pkg/release-controller/types.go b/pkg/release-controller/types.go index 287421a7a..eae6969a9 100644 --- a/pkg/release-controller/types.go +++ b/pkg/release-controller/types.go @@ -152,7 +152,7 @@ type ReleaseConfig struct { // AlternateImageRepository is the full path to an external Image Repository where we // will mirror Accepted releases to. // For example: - // "alternateImageRepository": "quay.io/openshift-release-dev/dev-release" + // "alternateImageRepository": "quay.io/openshift/ci" AlternateImageRepository string `json:"alternateImageRepository"` // AlternateImageRepositorySecret is the name of the secret containing credentials to the