[Mold] 백업 관련 오류 수정

api/src/main/java/com/cloud/event/EventTypes.java

@@ -882,6 +882,7 @@ public class EventTypes {
 
     // Backup
     public static final String EVENT_HOST_AGENT_INSTALL = "HOST.AGENT.INSTALL";
+    public static final String EVENT_BACKUP_AGENT_INSTALL = "BACKUP.AGENT.INSTALL";
 
     static {
 
@@ -1433,6 +1434,7 @@ public class EventTypes {
         entityEventDetails.put(EVENT_BACKUP_REPOSITORY_UPDATE, BackupRepositoryService.class);
         // Backup
         entityEventDetails.put(EVENT_HOST_AGENT_INSTALL, Backup.class);
+        entityEventDetails.put(EVENT_BACKUP_AGENT_INSTALL, Host.class);
     }
 
     public static boolean isNetworkEvent(String eventType) {

api/src/main/java/org/apache/cloudstack/api/response/UserVmResponse.java

@@ -360,6 +360,10 @@ public class UserVmResponse extends BaseResponseWithTagInformation implements Co
     @Param(description = "device ID of the volume currently being flattened by SharedMountPoint fast clone.")
     private Long cloneFastFlattenDeviceId;
 
+    @SerializedName("activebackupstatus")
+    @Param(description = "Active backup status of the virtual machine.")
+    private String activeBackupStatus;
+
     @SerializedName("readonlydetails")
     @Param(description = "List of read-only Instance details as comma separated string.", since = "4.16.0")
     private String readOnlyDetails;
@@ -1175,6 +1179,10 @@ public void setCloneFastFlattenDeviceId(Long cloneFastFlattenDeviceId) {
         this.cloneFastFlattenDeviceId = cloneFastFlattenDeviceId;
     }
 
+    public void setActiveBackupStatus(String activeBackupStatus) {
+        this.activeBackupStatus = activeBackupStatus;
+    }
+
     public void setReadOnlyDetails(String readOnlyDetails) {
         this.readOnlyDetails = readOnlyDetails;
     }
@@ -1223,6 +1231,10 @@ public Long getCloneFastFlattenDeviceId() {
         return cloneFastFlattenDeviceId;
     }
 
+    public String getActiveBackupStatus() {
+        return activeBackupStatus;
+    }
+
     public String getReadOnlyDetails() {
         return readOnlyDetails;
     }

engine/schema/src/main/java/com/cloud/event/dao/EventDao.java

@@ -33,6 +33,8 @@ public interface EventDao extends GenericDao<EventVO, Long> {
 
     public List<EventVO> listToArchiveOrDeleteEvents(List<Long> ids, String type, Date startDate, Date endDate, List<Long> accountIds);
 
+    boolean existsByTypeAndResource(String type, long resourceId, String resourceType);
+
     public void archiveEvents(List<EventVO> events);
 
 }

engine/schema/src/main/java/com/cloud/event/dao/EventDaoImpl.java

@@ -98,6 +98,16 @@ public List<EventVO> listToArchiveOrDeleteEvents(List<Long> ids, String type, Da
         return search(sc, null);
     }
 
+    @Override
+    public boolean existsByTypeAndResource(String type, long resourceId, String resourceType) {
+        SearchCriteria<EventVO> sc = createSearchCriteria();
+        sc.addAnd("type", Op.EQ, type);
+        sc.addAnd("resourceId", Op.EQ, resourceId);
+        sc.addAnd("resourceType", Op.EQ, resourceType);
+        sc.addAnd("archived", Op.EQ, false);
+        return findOneIncludingRemovedBy(sc) != null;
+    }
+
     @Override
     public void archiveEvents(List<EventVO> events) {
         if (events != null && !events.isEmpty()) {

plugins/backup/ablestack-commvault/src/main/java/org/apache/cloudstack/backup/AblestackCommvaultBackupProvider.java

@@ -51,6 +51,7 @@
 import com.cloud.utils.exception.CloudRuntimeException;
 import com.cloud.event.ActionEventUtils;
 import com.cloud.event.EventTypes;
+import com.cloud.event.dao.EventDao;
 import com.cloud.vm.VMInstanceVO;
 import com.cloud.vm.VirtualMachine;
 import com.cloud.vm.dao.VMInstanceDao;
@@ -90,7 +91,6 @@
 import java.util.ArrayList;
 import java.util.HashSet;
 import java.util.Date;
-import java.util.HashMap;
 import java.util.LinkedHashMap;
 import java.util.LinkedHashSet;
 import java.util.List;
@@ -105,6 +105,7 @@
 import java.util.Comparator;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
+import java.util.concurrent.TimeUnit;
 import javax.inject.Inject;
 
 import static org.apache.cloudstack.backup.BackupManager.BackupChainSize;
@@ -119,6 +120,8 @@ public class AblestackCommvaultBackupProvider extends AdapterBase implements Bac
     private static final String BACKUP_TYPE_INCREMENTAL = "INCREMENTAL";
     private static final String BACKUP_ENGINE_QCOW2 = "QCOW2";
     private static final String BACKUP_ENGINE_RBD_DIFF = "RBD_DIFF";
+    private static final long COMMVAULT_INSTALL_JOB_POLL_INTERVAL_MS = TimeUnit.SECONDS.toMillis(30);
+    private static final long COMMVAULT_INSTALL_JOB_WAIT_TIMEOUT_MS = TimeUnit.HOURS.toMillis(12);
     private static final String DETAIL_CHECKPOINT_NAME = "commvault.checkpoint.name";
     private static final String DETAIL_CHECKPOINT_PATH = "commvault.checkpoint.path";
     private static final String DETAIL_CHECKPOINT_XML = "commvault.checkpoint.xml";
@@ -138,6 +141,7 @@ public class AblestackCommvaultBackupProvider extends AdapterBase implements Bac
     private static final String DETAIL_FAILURE_PHASE = "commvault.failure.phase";
     private static final String DETAIL_FAILURE_REASON = "commvault.failure.reason";
     private static final String ERROR_REASON_METADATA_FINALIZE = "metadata-finalize";
+    private static final String COMMVAULT_PERMANENT_INSTALL_FAILURE_MESSAGE = "Commvault backup agent automatic installation cannot continue because required install media is missing in the Commvault Software Cache.";
     private static final int BASE_MAJOR = 11;
     private static final int BASE_FR = 32;
     private static final int BASE_MT = 89;
@@ -226,6 +230,9 @@ public class AblestackCommvaultBackupProvider extends AdapterBase implements Bac
     @Inject
     private DiskOfferingDao diskOfferingDao;
 
+    @Inject
+    private EventDao eventDao;
+
 
     private Long getClusterIdFromRootVolume(VirtualMachine vm) {
         VolumeVO rootVolume = volumeDao.getInstanceRootVolume(vm.getId());
@@ -2259,73 +2266,97 @@ private boolean reconcileErrorBackupWithCompletedJob(VirtualMachine vm, Backup b
 
     @Override
     public boolean checkBackupAgent(final Long zoneId) {
-        Map<String, String> checkResult = new HashMap<>();
         final AblestackCommvaultClient client = getClient(zoneId);
         String csVersionInfo = client.getCvtVersion();
         boolean version = versionCheck(csVersionInfo);
         if (version) {
             List<HostVO> Hosts = hostDao.findByDataCenterId(zoneId);
+            if (CollectionUtils.isEmpty(Hosts)) {
+                LOG.warn("No hosts found in zone [{}] for Commvault backup agent readiness check.", zoneId);
+                return false;
+            }
+            int targetHostCount = 0;
             for (final HostVO host : Hosts) {
                 if (host.getStatus() == Status.Up && host.getHypervisorType() == Hypervisor.HypervisorType.KVM) {
+                    targetHostCount++;
                     String checkHost = client.getClientId(host.getName());
                     if (checkHost == null) {
+                        LOG.info("Commvault client is not registered for host [{}] using host name [{}].", host.getPrivateIpAddress(), host.getName());
                         return false;
                     } else {
                         boolean installJob = client.getInstallActiveJob(host.getPrivateIpAddress());
                         boolean checkInstall = client.getClientProps(checkHost);
                         if (installJob || !checkInstall) {
                             if (!checkInstall) {
-                                LOG.error("The host is registered with the client, but the readiness status is not normal and you must manually check the client status.");
+                                LOG.error("The host is registered with the client, but the readiness status is not normal and you must manually check the client status. host=[{}], clientId=[{}]",
+                                        host.getPrivateIpAddress(), checkHost);
                             }
                             return false;
                         }
                     }
                 }
             }
+            if (targetHostCount == 0) {
+                LOG.warn("No Up KVM hosts found in zone [{}] for Commvault backup agent readiness check. The check will be retried.", zoneId);
+                return false;
+            }
+            LOG.info("Commvault backup agent readiness check passed for zone [{}].", zoneId);
             return true;
         }
+        LOG.warn("Commvault version check failed for zone [{}]. version=[{}]", zoneId, csVersionInfo);
         return false;
     }
 
     @Override
     public boolean installBackupAgent(final Long zoneId) {
-        Map<String, String> failResult = new HashMap<>();
         final AblestackCommvaultClient client = getClient(zoneId);
         List<HostVO> Hosts = hostDao.findByDataCenterId(zoneId);
+        if (CollectionUtils.isEmpty(Hosts)) {
+            LOG.warn("No hosts found in zone [{}] for Commvault backup agent automatic installation.", zoneId);
+            return false;
+        }
+        int targetHostCount = 0;
         for (final HostVO host : Hosts) {
             if (host.getStatus() == Status.Up && host.getHypervisorType() == Hypervisor.HypervisorType.KVM) {
+                targetHostCount++;
                 String commCell = client.getCommcell();
                 JSONObject jsonObject = new JSONObject(commCell);
                 String commCellId = String.valueOf(jsonObject.get("commCellId"));
                 String commServeHostName = String.valueOf(jsonObject.get("commCellName"));
                 Ternary<String, String, String> credentials = getKVMHyperisorCredentials(host);
-                boolean installJob = true;
-                LOG.info("checking for install agent on the Commvault Backup Provider in host " + host.getPrivateIpAddress());
                 // 설치가 진행중인 호스트가 있는지 확인
-                while (installJob) {
-                    installJob = client.getInstallActiveJob(host.getName());
-                    try {
-                        Thread.sleep(30000);
-                    } catch (InterruptedException e) {
-                        LOG.error("checkBackupAgent get install active job result sleep interrupted error");
-                    }
+                if (!waitForInstallActiveJobToFinish(client, host)) {
+                    publishBackupAgentInstallFailureEventIfNeeded(host);
+                    return false;
                 }
                 String checkHost = client.getClientId(host.getName());
                 // 호스트가 클라이언트에 등록되지 않은 경우
                 if (checkHost == null) {
+                    LOG.info("Commvault client is not registered for host [{}] using host name [{}]. Creating install task with client name [{}].",
+                            host.getPrivateIpAddress(), host.getName(), host.getPrivateIpAddress());
                     String jobId = client.installAgent(host.getPrivateIpAddress(), commCellId, commServeHostName, credentials.first(), credentials.second());
                     if (jobId != null) {
-                        String jobStatus = client.getJobStatus(jobId);
-                        if (!jobStatus.equalsIgnoreCase("Completed")) {
-                            LOG.error("installing agent on the Commvault Backup Provider failed jogId : " + jobId + " , jobStatus : " + jobStatus);
-                            ActionEventUtils.onActionEvent(User.UID_SYSTEM, Account.ACCOUNT_ID_SYSTEM, Domain.ROOT_DOMAIN, EventTypes.EVENT_HOST_AGENT_INSTALL,
-                                "Failed install the commvault client agent on the host : " + host.getPrivateIpAddress(), User.UID_SYSTEM, ApiCommandResourceType.Host.toString());
-                            failResult.put(host.getPrivateIpAddress(), jobId);
+                        LOG.info("Created Commvault backup agent install job [{}] for host [{}]. Waiting for completion.", jobId, host.getPrivateIpAddress());
+                        String jobStatus = client.getJobStatus(jobId, COMMVAULT_INSTALL_JOB_WAIT_TIMEOUT_MS);
+                        if (!"Completed".equalsIgnoreCase(jobStatus)) {
+                            String failureReason = client.getLastJobFailureReason();
+                            LOG.error("installing agent on the Commvault Backup Provider failed jogId : {} , jobStatus : {}, reason=[{}]",
+                                    jobId, jobStatus, failureReason);
+                            publishBackupAgentInstallFailureEventIfNeeded(host);
+                            if (isPermanentCommvaultInstallFailure(failureReason)) {
+                                throw new CloudRuntimeException(String.format("%s host=[%s], jobId=[%s], reason=[%s]",
+                                        COMMVAULT_PERMANENT_INSTALL_FAILURE_MESSAGE, host.getPrivateIpAddress(), jobId, failureReason));
+                            }
+                            return false;
                         }
+                        LOG.info("Completed Commvault backup agent install job [{}] for host [{}].", jobId, host.getPrivateIpAddress());
                     } else {
+                        LOG.error("installing agent on the Commvault Backup Provider failed to create install job on host [{}]", host.getPrivateIpAddress());
+                        publishBackupAgentInstallFailureEventIfNeeded(host);
                         return false;
                     }
                 } else {
+                    LOG.info("Commvault client [{}] already exists for host [{}]. Checking readiness.", checkHost, host.getPrivateIpAddress());
                     // 호스트가 클라이언트에는 등록되었지만 구성이 정상적으로 되지 않은 경우 준비 상태 체크
                     boolean checkInstall = client.getClientCheckReadiness(checkHost);
                     if (!checkInstall) {
@@ -2337,12 +2368,64 @@ public boolean installBackupAgent(final Long zoneId) {
                 }
             }
         }
-        if (!failResult.isEmpty()) {
+        if (targetHostCount == 0) {
+            LOG.warn("No Up KVM hosts found in zone [{}] for Commvault backup agent automatic installation. The installation will be retried.", zoneId);
             return false;
         }
         return true;
     }
 
+    private boolean waitForInstallActiveJobToFinish(AblestackCommvaultClient client, HostVO host) {
+        final long deadline = System.currentTimeMillis() + COMMVAULT_INSTALL_JOB_WAIT_TIMEOUT_MS;
+        boolean loggedWaiting = false;
+        while (hasInstallActiveJob(client, host)) {
+            if (!loggedWaiting) {
+                LOG.info("Waiting for existing Commvault backup agent install job to finish before creating a new install job. host=[{}], timeoutMillis=[{}]",
+                        host.getPrivateIpAddress(), COMMVAULT_INSTALL_JOB_WAIT_TIMEOUT_MS);
+                loggedWaiting = true;
+            }
+            if (System.currentTimeMillis() >= deadline) {
+                LOG.warn("Timed out waiting for existing Commvault client agent install job to finish. host=[{}], timeoutMillis=[{}]",
+                        host.getPrivateIpAddress(), COMMVAULT_INSTALL_JOB_WAIT_TIMEOUT_MS);
+                return false;
+            }
+            try {
+                Thread.sleep(COMMVAULT_INSTALL_JOB_POLL_INTERVAL_MS);
+            } catch (InterruptedException e) {
+                Thread.currentThread().interrupt();
+                LOG.error("Interrupted while waiting for Commvault client agent install job to finish. host=[{}]",
+                        host.getPrivateIpAddress(), e);
+                return false;
+            }
+        }
+        return true;
+    }
+
+    private boolean hasInstallActiveJob(AblestackCommvaultClient client, HostVO host) {
+        return client.getInstallActiveJob(host.getName()) || client.getInstallActiveJob(host.getPrivateIpAddress());
+    }
+
+    private boolean isPermanentCommvaultInstallFailure(String failureReason) {
+        if (StringUtils.isBlank(failureReason)) {
+            return false;
+        }
+        String normalizedReason = failureReason.toLowerCase(Locale.ROOT);
+        return normalizedReason.contains("software cache") &&
+                (normalizedReason.contains("required media version") || normalizedReason.contains("missing"));
+    }
+
+    private void publishBackupAgentInstallFailureEventIfNeeded(HostVO host) {
+        if (hasBackupAgentInstallFailureEvent(host.getId())) {
+            return;
+        }
+        ActionEventUtils.onActionEvent(User.UID_SYSTEM, Account.ACCOUNT_ID_SYSTEM, Domain.ROOT_DOMAIN, EventTypes.EVENT_BACKUP_AGENT_INSTALL,
+                "Failed to install the Commvault backup agent on host: " + host.getPrivateIpAddress(), host.getId(), ApiCommandResourceType.Host.toString());
+    }
+
+    private boolean hasBackupAgentInstallFailureEvent(long hostId) {
+        return eventDao.existsByTypeAndResource(EventTypes.EVENT_BACKUP_AGENT_INSTALL, hostId, ApiCommandResourceType.Host.toString());
+    }
+
     @Override
     public boolean importBackupPlan(final Long zoneId, final String retentionPeriod, final String externalId) {
         final AblestackCommvaultClient client = getClient(zoneId);