Wait for SQLDatabase to go away during rollback

cmd/rollback/main.go

@@ -107,22 +107,28 @@ func main() {
 		os.Exit(13)
 	}
 
+	err = instance.WaitForSQLDatabaseResourceToGoAway(ctx, cfg.ApplicationName, mgr)
+	if err != nil {
+		mgr.Logger.Error("sqldatabase is stuck", "error", err)
+		os.Exit(14)
+	}
+
 	app, err = application.UpdateApplicationInstance(ctx, &cfg.Config, &cfg.SourceInstance, mgr)
 	if err != nil {
 		mgr.Logger.Error("failed to update application", "error", err)
-		os.Exit(14)
+		os.Exit(15)
 	}
 
 	source, err := resolved.ResolveInstance(ctx, app, mgr)
 	if err != nil {
 		mgr.Logger.Error("failed to resolve updated target", "error", err)
-		os.Exit(15)
+		os.Exit(16)
 	}
 
 	err = application.UpdateApplicationUser(ctx, source, gcpProject, mgr)
 	if err != nil {
 		mgr.Logger.Error("failed to update application user", "error", err)
-		os.Exit(16)
+		os.Exit(17)
 	}
 
 	mgr.Logger.Info("deleting SQL SSL Certificates used during migration")
@@ -131,7 +137,7 @@ func main() {
 	})
 	if err != nil {
 		mgr.Logger.Error("failed to delete SQL SSL Certificates", "error", err)
-		os.Exit(17)
+		os.Exit(18)
 	}
 
 	mgr.Logger.Info("deleting Network Policy used during migration")
@@ -140,6 +146,6 @@ func main() {
 	})
 	if err != nil {
 		mgr.Logger.Error("failed to delete Network Policy", "error", err)
-		os.Exit(18)
+		os.Exit(19)
 	}
 }

internal/pkg/instance/instance.go

@@ -191,6 +191,25 @@ func PrepareSourceInstance(ctx context.Context, source *resolved.Instance, targe
 	return nil
 }
 
+func WaitForSQLDatabaseResourceToGoAway(ctx context.Context, appName string, mgr *common_main.Manager) error {
+	mgr.Logger.Info("waiting for SQLDatabase resource to go away...")
+
+	b := retry.NewConstant(5 * time.Second)
+	b = retry.WithMaxDuration(5*time.Minute, b)
+
+	err := retry.Do(ctx, b, func(ctx context.Context) error {
+		exists, err := mgr.SqlDatabaseClient.ExistsByLabel(ctx, fmt.Sprintf("app=%s", appName))
+		if exists {
+			return retry.RetryableError(fmt.Errorf("resource still exists"))
+		}
+		return retry.RetryableError(err)
+	})
+	if err == nil {
+		mgr.Logger.Info("resource has been deleted")
+	}
+	return err
+}
+
 func WaitForCnrmResourcesToGoAway(ctx context.Context, name string, mgr *common_main.Manager) error {
 	logger := mgr.Logger.With("instance_name", name)
 	logger.Info("waiting for relevant CNRM resources to go away...")