diff --git a/core/src/main/java/com/cloud/agent/api/MigrateBetweenSecondaryStoragesCommandAnswer.java b/core/src/main/java/com/cloud/agent/api/MigrateBetweenSecondaryStoragesCommandAnswer.java new file mode 100644 index 000000000000..293907a78bab --- /dev/null +++ b/core/src/main/java/com/cloud/agent/api/MigrateBetweenSecondaryStoragesCommandAnswer.java @@ -0,0 +1,41 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// + +package com.cloud.agent.api; + +import com.cloud.utils.Pair; + +import java.util.List; + +public class MigrateBetweenSecondaryStoragesCommandAnswer extends Answer { + + List> migratedResourcesIdAndCheckpointPath; + + public MigrateBetweenSecondaryStoragesCommandAnswer() { + } + + public MigrateBetweenSecondaryStoragesCommandAnswer(MigrateSnapshotsBetweenSecondaryStoragesCommand cmd, boolean success, String result, List> migratedResourcesIdAndCheckpointPath) { + super(cmd, success, result); + this.migratedResourcesIdAndCheckpointPath = migratedResourcesIdAndCheckpointPath; + } + + public List> getMigratedResources() { + return migratedResourcesIdAndCheckpointPath; + } +} diff --git a/core/src/main/java/com/cloud/agent/api/MigrateSnapshotsBetweenSecondaryStoragesCommand.java b/core/src/main/java/com/cloud/agent/api/MigrateSnapshotsBetweenSecondaryStoragesCommand.java new file mode 100644 index 000000000000..bb049bf0e5ef --- /dev/null +++ b/core/src/main/java/com/cloud/agent/api/MigrateSnapshotsBetweenSecondaryStoragesCommand.java @@ -0,0 +1,64 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// + +package com.cloud.agent.api; + +import com.cloud.agent.api.to.DataStoreTO; +import com.cloud.agent.api.to.DataTO; + +import java.util.List; +import java.util.Set; + +public class MigrateSnapshotsBetweenSecondaryStoragesCommand extends Command { + + DataStoreTO srcDataStore; + DataStoreTO destDataStore; + List snapshotChain; + Set snapshotsIdToMigrate; + + public MigrateSnapshotsBetweenSecondaryStoragesCommand() { + } + + public MigrateSnapshotsBetweenSecondaryStoragesCommand(List snapshotChain, DataStoreTO srcDataStore, DataStoreTO destDataStore, Set snapshotsIdToMigrate) { + this.srcDataStore = srcDataStore; + this.destDataStore = destDataStore; + this.snapshotChain = snapshotChain; + this.snapshotsIdToMigrate = snapshotsIdToMigrate; + } + + @Override + public boolean executeInSequence() { + return false; + } + + public List getSnapshotChain() { + return snapshotChain; + } + + public Set getSnapshotsIdToMigrate() { + return snapshotsIdToMigrate; + } + + public DataStoreTO getSrcDataStore() { + return srcDataStore; + } + + public DataStoreTO getDestDataStore() { + return destDataStore; + } +} diff --git a/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/SnapshotInfo.java b/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/SnapshotInfo.java index d47aa5865a0f..ec726291d76b 100644 --- a/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/SnapshotInfo.java +++ b/engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/SnapshotInfo.java @@ -30,6 +30,8 @@ public interface SnapshotInfo extends DataObject, Snapshot { SnapshotInfo getParent(); + List getParents(); + String getPath(); DataStore getImageStore(); @@ -40,6 +42,8 @@ public interface SnapshotInfo extends DataObject, Snapshot { List getChildren(); + List getChildAndGrandchildren(); + VolumeInfo getBaseVolume(); void addPayload(Object data); diff --git a/engine/components-api/src/main/java/com/cloud/storage/StorageManager.java b/engine/components-api/src/main/java/com/cloud/storage/StorageManager.java index 77e97b00c1e9..de25befd9e4e 100644 --- a/engine/components-api/src/main/java/com/cloud/storage/StorageManager.java +++ b/engine/components-api/src/main/java/com/cloud/storage/StorageManager.java @@ -228,6 +228,9 @@ public interface StorageManager extends StorageService { ConfigKey.Scope.Global, null); + ConfigKey AgentMaxDataMigrationWaitTime = new ConfigKey<>("Advanced", Integer.class, "agent.max.data.migration.wait.time", "3600", + "The maximum time (in seconds) that the secondary storage data migration command sent to the KVM Agent will be executed before a timeout occurs.", true, ConfigKey.Scope.Cluster); + /** * should we execute in sequence not involving any storages? * @return true if commands should execute in sequence diff --git a/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/DataMigrationUtility.java b/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/DataMigrationUtility.java index 9609ba7751d9..e44065ce17c5 100644 --- a/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/DataMigrationUtility.java +++ b/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/DataMigrationUtility.java @@ -22,10 +22,13 @@ import java.util.Collections; import java.util.Comparator; import java.util.HashMap; +import java.util.HashSet; import java.util.LinkedHashMap; import java.util.LinkedList; import java.util.List; import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; import javax.inject.Inject; @@ -92,7 +95,7 @@ public class DataMigrationUtility { * "Ready" "Allocated", "Destroying", "Destroyed", "Failed". If this is the case, and if the migration policy is complete, * the migration is terminated. */ - public boolean filesReadyToMigrate(Long srcDataStoreId, List templates, List snapshots, List volumes) { + public boolean filesReadyToMigrate(List templates, List snapshots, List volumes) { State[] validStates = {State.Ready, State.Allocated, State.Destroying, State.Destroyed, State.Failed}; boolean isReady = true; for (TemplateDataStoreVO template : templates) { @@ -114,7 +117,8 @@ private boolean filesReadyToMigrate(Long srcDataStoreId) { List templates = templateDataStoreDao.listByStoreId(srcDataStoreId); List snapshots = snapshotDataStoreDao.listByStoreId(srcDataStoreId, DataStoreRole.Image); List volumes = volumeDataStoreDao.listByStoreId(srcDataStoreId); - return filesReadyToMigrate(srcDataStoreId, templates, snapshots, volumes); + + return filesReadyToMigrate(templates, snapshots, volumes); } protected void checkIfCompleteMigrationPossible(ImageStoreService.MigrationPolicy policy, Long srcDataStoreId) { @@ -163,11 +167,11 @@ public int compare(Map.Entry> e1, Map.Entry getSortedValidSourcesList(DataStore srcDataStore, Map, Long>> snapshotChains, - Map, Long>> childTemplates, List templates, List snapshots) { + Map, Long>> childTemplates, List templates, List snapshots, Set snapshotIdsToMigrate) { List files = new ArrayList<>(); files.addAll(getAllReadyTemplates(srcDataStore, childTemplates, templates)); - files.addAll(getAllReadySnapshotsAndChains(srcDataStore, snapshotChains, snapshots)); + files.addAll(getAllReadySnapshotsAndChains(srcDataStore, snapshotChains, snapshots, snapshotIdsToMigrate)); files = sortFilesOnSize(files, snapshotChains); @@ -175,10 +179,10 @@ protected List getSortedValidSourcesList(DataStore srcDataStore, Map } protected List getSortedValidSourcesList(DataStore srcDataStore, Map, Long>> snapshotChains, - Map, Long>> childTemplates) { + Map, Long>> childTemplates, Set snapshotIdsToMigrate) { List files = new ArrayList<>(); files.addAll(getAllReadyTemplates(srcDataStore, childTemplates)); - files.addAll(getAllReadySnapshotsAndChains(srcDataStore, snapshotChains)); + files.addAll(getAllReadySnapshotsAndChains(srcDataStore, snapshotChains, snapshotIdsToMigrate)); files.addAll(getAllReadyVolumes(srcDataStore)); files = sortFilesOnSize(files, snapshotChains); @@ -186,6 +190,17 @@ protected List getSortedValidSourcesList(DataStore srcDataStore, Map return files; } + private List createKvmIncrementalSnapshotChain(SnapshotDataStoreVO snapshot) { + List chain = new LinkedList<>(); + SnapshotInfo snapshotInfo = snapshotFactory.getSnapshot(snapshot.getSnapshotId(), snapshot.getDataStoreId(), snapshot.getRole()); + + chain.addAll(snapshotInfo.getParents()); + chain.add(snapshotInfo); + chain.addAll(snapshotInfo.getChildAndGrandchildren()); + + return chain; + } + protected List sortFilesOnSize(List files, Map, Long>> snapshotChains) { Collections.sort(files, new Comparator() { @Override @@ -261,16 +276,24 @@ protected boolean shouldMigrateTemplate(TemplateDataStoreVO template, VMTemplate * for each parent snapshot and the cumulative size of the chain - this is done to ensure that all the snapshots in a chain * are migrated to the same datastore */ - protected List getAllReadySnapshotsAndChains(DataStore srcDataStore, Map, Long>> snapshotChains, List snapshots) { + protected List getAllReadySnapshotsAndChains(DataStore srcDataStore, Map, Long>> snapshotChains, List snapshots, Set snapshotIdsToMigrate) { List files = new LinkedList<>(); + Set snapshotIdsAlreadyInChain = new HashSet<>(); for (SnapshotDataStoreVO snapshot : snapshots) { SnapshotVO snapshotVO = snapshotDao.findById(snapshot.getSnapshotId()); if (snapshot.getState() == ObjectInDataStoreStateMachine.State.Ready && - snapshotVO != null && snapshotVO.getHypervisorType() != Hypervisor.HypervisorType.Simulator - && snapshot.getParentSnapshotId() == 0 ) { - SnapshotInfo snap = snapshotFactory.getSnapshot(snapshotVO.getSnapshotId(), snapshot.getDataStoreId(), snapshot.getRole()); - if (snap != null) { - files.add(snap); + snapshotVO != null && snapshotVO.getHypervisorType() != Hypervisor.HypervisorType.Simulator) { + if (snapshotVO.getHypervisorType() == Hypervisor.HypervisorType.KVM && snapshot.getKvmCheckpointPath() != null && !snapshotIdsAlreadyInChain.contains(snapshotVO.getId())) { + List kvmIncrementalSnapshotChain = createKvmIncrementalSnapshotChain(snapshot); + SnapshotInfo parent = kvmIncrementalSnapshotChain.get(0); + snapshotIdsAlreadyInChain.addAll(kvmIncrementalSnapshotChain.stream().map(DataObject::getId).collect(Collectors.toSet())); + snapshotChains.put(parent, new Pair<>(kvmIncrementalSnapshotChain, getTotalChainSize(kvmIncrementalSnapshotChain.stream().filter(snap -> snapshotIdsToMigrate.contains(snap.getId())).collect(Collectors.toList())))); + files.add(parent); + } else if (snapshot.getParentSnapshotId() == 0) { + SnapshotInfo snap = snapshotFactory.getSnapshot(snapshotVO.getSnapshotId(), snapshot.getDataStoreId(), snapshot.getRole()); + if (snap != null) { + files.add(snap); + } } } } @@ -285,15 +308,16 @@ protected List getAllReadySnapshotsAndChains(DataStore srcDataStore, chain.addAll(children); } } - snapshotChains.put(parent, new Pair, Long>(chain, getTotalChainSize(chain))); + snapshotChains.putIfAbsent(parent, new Pair<>(chain, getTotalChainSize(chain))); } return (List) (List) files; } - protected List getAllReadySnapshotsAndChains(DataStore srcDataStore, Map, Long>> snapshotChains) { + protected List getAllReadySnapshotsAndChains(DataStore srcDataStore, Map, Long>> snapshotChains, Set snapshotIdsToMigrate) { List snapshots = snapshotDataStoreDao.listByStoreId(srcDataStore.getId(), DataStoreRole.Image); - return getAllReadySnapshotsAndChains(srcDataStore, snapshotChains, snapshots); + snapshotIdsToMigrate.addAll(snapshots.stream().map(SnapshotDataStoreVO::getSnapshotId).collect(Collectors.toSet())); + return getAllReadySnapshotsAndChains(srcDataStore, snapshotChains, snapshots, snapshotIdsToMigrate); } protected Long getTotalChainSize(List chain) { diff --git a/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/StorageOrchestrator.java b/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/StorageOrchestrator.java index 0773c20b6b98..4cdfdd1908d8 100644 --- a/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/StorageOrchestrator.java +++ b/engine/orchestration/src/main/java/org/apache/cloudstack/engine/orchestration/StorageOrchestrator.java @@ -21,13 +21,19 @@ import java.util.Collections; import java.util.Date; import java.util.HashMap; +import java.util.HashSet; import java.util.Hashtable; import java.util.LinkedList; import java.util.List; import java.util.Map; +import java.util.Random; +import java.util.Set; import java.util.concurrent.ArrayBlockingQueue; import java.util.concurrent.Callable; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; import java.util.concurrent.Future; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; @@ -60,7 +66,15 @@ import org.apache.commons.math3.stat.descriptive.moment.Mean; import org.apache.commons.math3.stat.descriptive.moment.StandardDeviation; +import com.cloud.agent.AgentManager; +import com.cloud.agent.api.MigrateBetweenSecondaryStoragesCommandAnswer; +import com.cloud.agent.api.MigrateSnapshotsBetweenSecondaryStoragesCommand; import com.cloud.capacity.CapacityManager; +import com.cloud.exception.AgentUnavailableException; +import com.cloud.exception.OperationTimedoutException; +import com.cloud.host.HostVO; +import com.cloud.host.dao.HostDao; +import com.cloud.hypervisor.Hypervisor; import com.cloud.server.StatsCollector; import com.cloud.storage.DataStoreRole; import com.cloud.storage.SnapshotVO; @@ -70,6 +84,8 @@ import com.cloud.storage.dao.SnapshotDao; import com.cloud.utils.Pair; import com.cloud.utils.component.ManagerBase; +import com.cloud.utils.db.Transaction; +import com.cloud.utils.db.TransactionCallback; import com.cloud.utils.exception.CloudRuntimeException; public class StorageOrchestrator extends ManagerBase implements StorageOrchestrationService, Configurable { @@ -96,6 +112,10 @@ public class StorageOrchestrator extends ManagerBase implements StorageOrchestra VolumeDataStoreDao volumeDataStoreDao; @Inject DataMigrationUtility migrationHelper; + @Inject + AgentManager agentManager; + @Inject + HostDao hostDao; ConfigKey ImageStoreImbalanceThreshold = new ConfigKey<>("Advanced", Double.class, "image.store.imbalance.threshold", @@ -106,6 +126,8 @@ public class StorageOrchestrator extends ManagerBase implements StorageOrchestra Integer numConcurrentCopyTasksPerSSVM = 2; + private final Map zoneKvmIncrementalExecutorMap = new ConcurrentHashMap<>(); + @Override public String getConfigComponentName() { return StorageOrchestrationService.class.getName(); @@ -146,9 +168,10 @@ public MigrationResponse migrateData(Long srcDataStoreId, List destDatasto migrationHelper.checkIfCompleteMigrationPossible(migrationPolicy, srcDataStoreId); DataStore srcDatastore = dataStoreManager.getDataStore(srcDataStoreId, DataStoreRole.Image); + Set snapshotIdsToMigrate = new HashSet<>(); Map, Long>> snapshotChains = new HashMap<>(); Map, Long>> childTemplates = new HashMap<>(); - files = migrationHelper.getSortedValidSourcesList(srcDatastore, snapshotChains, childTemplates); + files = migrationHelper.getSortedValidSourcesList(srcDatastore, snapshotChains, childTemplates, snapshotIdsToMigrate); if (files.isEmpty()) { return new MigrationResponse(String.format("No files in Image store: %s to migrate", srcDatastore), migrationPolicy.toString(), true); @@ -206,7 +229,7 @@ public MigrationResponse migrateData(Long srcDataStoreId, List destDatasto } if (shouldMigrate(chosenFileForMigration, srcDatastore.getId(), destDatastoreId, storageCapacities, snapshotChains, childTemplates, migrationPolicy)) { - storageCapacities = migrateAway(chosenFileForMigration, storageCapacities, snapshotChains, childTemplates, srcDatastore, destDatastoreId, executor, futures); + storageCapacities = migrateAway(chosenFileForMigration, storageCapacities, snapshotChains, childTemplates, snapshotIdsToMigrate, srcDatastore, destDatastoreId, executor, futures); } else { if (migrationPolicy == MigrationPolicy.BALANCE) { continue; @@ -234,11 +257,12 @@ public MigrationResponse migrateResources(Long srcImgStoreId, Long destImgStoreI List templates = templateDataStoreDao.listByStoreIdAndTemplateIds(srcImgStoreId, templateIdList); List snapshots = snapshotDataStoreDao.listByStoreAndSnapshotIds(srcImgStoreId, DataStoreRole.Image, snapshotIdList); + Set snapshotIdsToMigrate = snapshots.stream().map(SnapshotDataStoreVO::getSnapshotId).collect(Collectors.toSet()); - if (!migrationHelper.filesReadyToMigrate(srcImgStoreId, templates, snapshots, Collections.emptyList())) { + if (!migrationHelper.filesReadyToMigrate(templates, snapshots, Collections.emptyList())) { throw new CloudRuntimeException("Migration failed as there are data objects which are not Ready - i.e, they may be in Migrating, creating, copying, etc. states"); } - files = migrationHelper.getSortedValidSourcesList(srcDatastore, snapshotChains, childTemplates, templates, snapshots); + files = migrationHelper.getSortedValidSourcesList(srcDatastore, snapshotChains, childTemplates, templates, snapshots, snapshotIdsToMigrate); if (files.isEmpty()) { return new MigrationResponse(String.format("No files in Image store: %s to migrate", srcDatastore.getUuid()), null, true); @@ -272,7 +296,7 @@ public MigrationResponse migrateResources(Long srcImgStoreId, Long destImgStoreI } if (storageCapacityBelowThreshold(storageCapacities, destImgStoreId)) { - storageCapacities = migrateAway(chosenFileForMigration, storageCapacities, snapshotChains, childTemplates, srcDatastore, destImgStoreId, executor, futures); + storageCapacities = migrateAway(chosenFileForMigration, storageCapacities, snapshotChains, childTemplates, snapshotIdsToMigrate, srcDatastore, destImgStoreId, executor, futures); } else { message = "Migration failed. Destination store doesn't have enough capacity for migration"; success = false; @@ -327,9 +351,9 @@ protected Pair migrateCompleted(Long destDatastoreId, DataStore protected Map> migrateAway( DataObject chosenFileForMigration, Map> storageCapacities, - Map, Long>> snapshotChains, + Map, Long>> snapshotChains, Map, Long>> templateChains, + Set snapshotIdsToMigrate, DataStore srcDatastore, Long destDatastoreId, ThreadPoolExecutor executor, @@ -346,8 +370,92 @@ protected Map> migrateAway( executor.setCorePoolSize((int) (totalJobs)); } - MigrateDataTask task = new MigrateDataTask(chosenFileForMigration, srcDatastore, dataStoreManager.getDataStore(destDatastoreId, DataStoreRole.Image)); - if (chosenFileForMigration instanceof SnapshotInfo ) { + DataStore destDataStore = dataStoreManager.getDataStore(destDatastoreId, DataStoreRole.Image); + + boolean isKvmIncrementalSnapshot = chosenFileForMigration instanceof SnapshotInfo && ((SnapshotInfo) chosenFileForMigration).isKvmIncrementalSnapshot() && snapshotChains.containsKey(chosenFileForMigration); + + if (isKvmIncrementalSnapshot) { + MigrateKvmIncrementalSnapshotTask task = new MigrateKvmIncrementalSnapshotTask(chosenFileForMigration, snapshotChains, srcDatastore, destDataStore, snapshotIdsToMigrate); + futures.add(submitKvmIncrementalMigration(srcDatastore.getScope().getScopeId(), task)); + logger.debug("Incremental snapshot migration {} submitted to incremental pool.", chosenFileForMigration.getUuid()); + } else { + createMigrateDataTask(chosenFileForMigration, snapshotChains, templateChains, srcDatastore, destDataStore, executor, futures); + } + + return storageCapacities; + } + + private AsyncCallFuture migrateKvmIncrementalSnapshotChain(DataObject chosenFileForMigration, Map, Long>> snapshotChains, DataStore srcDatastore, DataStore destDataStore, Set snapshotIdsToMigrate) { + return Transaction.execute((TransactionCallback>) status -> { + MigrateBetweenSecondaryStoragesCommandAnswer answer = null; + AsyncCallFuture future = new AsyncCallFuture<>(); + DataObjectResult result = new DataObjectResult(chosenFileForMigration); + + try { + List snapshotChain = snapshotChains.get(chosenFileForMigration).first(); + MigrateSnapshotsBetweenSecondaryStoragesCommand migrateBetweenSecondaryStoragesCmd = new MigrateSnapshotsBetweenSecondaryStoragesCommand(snapshotChain.stream().map(DataObject::getTO).collect(Collectors.toList()), srcDatastore.getTO(), destDataStore.getTO(), snapshotIdsToMigrate); + + HostVO host = getAvailableHost(((SnapshotInfo) chosenFileForMigration).getDataCenterId()); + if (host == null) { + throw new CloudRuntimeException("No suitable hosts found to send migrate command."); + } + + migrateBetweenSecondaryStoragesCmd.setWait(StorageManager.AgentMaxDataMigrationWaitTime.valueIn(host.getClusterId())); + answer = (MigrateBetweenSecondaryStoragesCommandAnswer) agentManager.send(host.getId(), migrateBetweenSecondaryStoragesCmd); + if (answer == null || !answer.getResult()) { + logger.warn("Unable to migrate snapshots [{}].", snapshotChain); + throw new CloudRuntimeException("Unable to migrate KVM incremental snapshots to another secondary storage"); + } + } catch (final OperationTimedoutException | AgentUnavailableException e) { + throw new CloudRuntimeException("Error while migrating KVM incremental snapshot chain. Check the logs for more information.", e); + } finally { + if (answer != null) { + updateSnapshotsReference(destDataStore, answer); + } + } + result.setSuccess(true); + future.complete(result); + return future; + }); + } + + private void updateSnapshotsReference(DataStore destDataStore, MigrateBetweenSecondaryStoragesCommandAnswer answer) { + for (Pair snapshotIdAndUpdatedCheckpointPath : answer.getMigratedResources()) { + Long snapshotId = snapshotIdAndUpdatedCheckpointPath.first(); + String newCheckpointPath = snapshotIdAndUpdatedCheckpointPath.second(); + + SnapshotDataStoreVO snapshotDataStore = snapshotDataStoreDao.findOneBySnapshotAndDatastoreRole(snapshotId, DataStoreRole.Image); + + if (snapshotDataStore == null) { + logger.warn("Snapshot [{}] not found.", snapshotId); + continue; + } + + snapshotDataStore.setDataStoreId(destDataStore.getId()); + snapshotDataStore.setKvmCheckpointPath(newCheckpointPath); + snapshotDataStoreDao.update(snapshotDataStore.getId(), snapshotDataStore); + } + } + + protected Future submitKvmIncrementalMigration(Long zoneId, Callable task) { + if (!zoneKvmIncrementalExecutorMap.containsKey(zoneId)) { + zoneKvmIncrementalExecutorMap.put(zoneId, Executors.newSingleThreadExecutor()); + } + return zoneKvmIncrementalExecutorMap.get(zoneId).submit(task); + } + + private HostVO getAvailableHost(long zoneId) throws AgentUnavailableException, OperationTimedoutException { + List hosts = hostDao.listByDataCenterIdAndHypervisorType(zoneId, Hypervisor.HypervisorType.KVM); + if (CollectionUtils.isNotEmpty(hosts)) { + return hosts.get(new Random().nextInt(hosts.size())); + } + + return null; + } + + private void createMigrateDataTask(DataObject chosenFileForMigration, Map, Long>> snapshotChains, Map, Long>> templateChains, DataStore srcDatastore, DataStore destDataStore, ThreadPoolExecutor executor, List>> futures) { + MigrateDataTask task = new MigrateDataTask(chosenFileForMigration, srcDatastore, destDataStore); + if (chosenFileForMigration instanceof SnapshotInfo) { task.setSnapshotChains(snapshotChains); } if (chosenFileForMigration instanceof TemplateInfo) { @@ -355,11 +463,9 @@ protected Map> migrateAway( } futures.add((executor.submit(task))); logger.debug("Migration of {}: {} is initiated.", chosenFileForMigration.getType().name(), chosenFileForMigration.getUuid()); - return storageCapacities; } - private MigrationResponse handleResponse(List>> futures, MigrationPolicy migrationPolicy, String message, boolean success) { int successCount = 0; for (Future> future : futures) { @@ -561,4 +667,34 @@ public AsyncCallFuture call() throws Exception { return secStgSrv.migrateData(file, srcDataStore, destDataStore, snapshotChain, templateChain); } } + + private class MigrateKvmIncrementalSnapshotTask implements Callable> { + private final DataObject chosenFile; + private final Map, Long>> snapshotChains; + private final DataStore srcDataStore; + private final DataStore destDataStore; + private final Set snapshotIdsToMigrate; + + public MigrateKvmIncrementalSnapshotTask(DataObject chosenFile, Map, Long>> snapshotChains, DataStore srcDataStore, DataStore destDataStore, Set snapshotIdsToMigrate) { + this.chosenFile = chosenFile; + this.snapshotChains = snapshotChains; + this.srcDataStore = srcDataStore; + this.destDataStore = destDataStore; + this.snapshotIdsToMigrate = snapshotIdsToMigrate; + } + + @Override + public AsyncCallFuture call() { + try { + return migrateKvmIncrementalSnapshotChain(chosenFile, snapshotChains, srcDataStore, destDataStore, snapshotIdsToMigrate); + } catch (Exception e) { + logger.warn("Failed migrating incremental snapshot {} due to {}.", chosenFile.getUuid(), e); + AsyncCallFuture future = new AsyncCallFuture<>(); + DataObjectResult result = new DataObjectResult(chosenFile); + result.setResult(e.toString()); + future.complete(result); + return future; + } + } + } } diff --git a/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/snapshot/SnapshotDataFactoryImpl.java b/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/snapshot/SnapshotDataFactoryImpl.java index cd314b0be638..b1119bf12d66 100644 --- a/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/snapshot/SnapshotDataFactoryImpl.java +++ b/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/snapshot/SnapshotDataFactoryImpl.java @@ -123,7 +123,9 @@ public SnapshotInfo getSnapshot(long snapshotId, long storeId, DataStoreRole rol return null; } DataStore store = storeMgr.getDataStore(snapshotStore.getDataStoreId(), role); - return SnapshotObject.getSnapshotObject(snapshot, store); + SnapshotObject snapshotObject = SnapshotObject.getSnapshotObject(snapshot, store); + snapshotObject.setKvmIncrementalSnapshot(snapshotStore.getKvmCheckpointPath() != null); + return snapshotObject; } @Override diff --git a/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/snapshot/SnapshotObject.java b/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/snapshot/SnapshotObject.java index 8264fcd42869..17535c04abed 100644 --- a/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/snapshot/SnapshotObject.java +++ b/engine/storage/snapshot/src/main/java/org/apache/cloudstack/storage/snapshot/SnapshotObject.java @@ -20,6 +20,7 @@ import java.util.ArrayList; import java.util.Date; +import java.util.LinkedList; import java.util.List; import javax.inject.Inject; @@ -128,6 +129,19 @@ public SnapshotInfo getParent() { return null; } + @Override + public List getParents() { + LinkedList parents = new LinkedList<>(); + SnapshotInfo parent = getParent(); + + while (parent != null) { + parents.addFirst(parent); + parent = parent.getParent(); + } + + return parents; + } + /** * Returns the snapshotInfo of the passed snapshot parentId. Will search for the snapshot reference which has a checkpoint path. If none is found, throws an exception. * */ @@ -165,7 +179,7 @@ public SnapshotInfo getChild() { if (vo == null) { return null; } - return snapshotFactory.getSnapshot(vo.getSnapshotId(), store); + return snapshotFactory.getSnapshot(vo.getSnapshotId(), vo.getDataStoreId(), DataStoreRole.Image); } @Override @@ -189,6 +203,19 @@ public List getChildren() { return children; } + @Override + public List getChildAndGrandchildren() { + LinkedList snapshots = new LinkedList<>(); + SnapshotInfo child = getChild(); + + while (child != null) { + snapshots.add(child); + child = child.getChild(); + } + + return snapshots; + } + @Override public boolean isRevertable() { SnapshotStrategy snapshotStrategy = storageStrategyFactory.getSnapshotStrategy(snapshot, SnapshotOperation.REVERT); diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateSnapshotsBetweenSecondaryStoragesCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateSnapshotsBetweenSecondaryStoragesCommandWrapper.java new file mode 100644 index 000000000000..42f777402a59 --- /dev/null +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateSnapshotsBetweenSecondaryStoragesCommandWrapper.java @@ -0,0 +1,182 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// + +package com.cloud.hypervisor.kvm.resource.wrapper; + +import com.cloud.agent.api.Answer; +import com.cloud.agent.api.MigrateSnapshotsBetweenSecondaryStoragesCommand; +import com.cloud.agent.api.MigrateBetweenSecondaryStoragesCommandAnswer; +import com.cloud.agent.api.to.DataStoreTO; +import com.cloud.agent.api.to.DataTO; +import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource; +import com.cloud.hypervisor.kvm.storage.KVMStoragePool; +import com.cloud.hypervisor.kvm.storage.KVMStoragePoolManager; +import com.cloud.resource.CommandWrapper; +import com.cloud.resource.ResourceWrapper; +import com.cloud.utils.Pair; +import com.cloud.utils.exception.CloudRuntimeException; +import com.cloud.utils.script.Script; +import org.apache.cloudstack.utils.qemu.QemuImageOptions; +import org.apache.cloudstack.utils.qemu.QemuImg; +import org.apache.cloudstack.utils.qemu.QemuImgException; +import org.apache.cloudstack.utils.qemu.QemuImgFile; +import org.libvirt.LibvirtException; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +@ResourceWrapper(handles = MigrateSnapshotsBetweenSecondaryStoragesCommand.class) +public class LibvirtMigrateSnapshotsBetweenSecondaryStoragesCommandWrapper extends CommandWrapper { + + protected Set filesToRemove; + protected List> resourcesToUpdate; + protected String resourceType; + protected int wait; + + @Override + public Answer execute(MigrateSnapshotsBetweenSecondaryStoragesCommand command, LibvirtComputingResource serverResource) { + filesToRemove = new HashSet<>(); + resourcesToUpdate = new ArrayList<>(); + wait = command.getWait() * 1000; + + DataStoreTO srcDataStore = command.getSrcDataStore(); + DataStoreTO destDataStore = command.getDestDataStore(); + KVMStoragePoolManager storagePoolManager = serverResource.getStoragePoolMgr(); + + Set imagePools = new HashSet<>(); + KVMStoragePool destImagePool = storagePoolManager.getStoragePoolByURI(destDataStore.getUrl()); + imagePools.add(destImagePool); + + String imagePoolUrl; + KVMStoragePool imagePool = null; + + String parentSnapshotPath = null; + boolean parentSnapshotWasMigrated = false; + Set snapshotsIdToMigrate = command.getSnapshotsIdToMigrate(); + + try { + for (DataTO snapshot : command.getSnapshotChain()) { + imagePoolUrl = snapshot.getDataStore().getUrl(); + imagePool = storagePoolManager.getStoragePoolByURI(imagePoolUrl); + imagePools.add(imagePool); + + String resourceCurrentPath = imagePool.getLocalPathFor(snapshot.getPath()); + + if (imagePoolUrl.equals(srcDataStore.getUrl()) && snapshotsIdToMigrate.contains(snapshot.getId())) { + parentSnapshotPath = copyResourceToDestDataStore(snapshot, resourceCurrentPath, destImagePool, parentSnapshotPath); + parentSnapshotWasMigrated = true; + } else { + if (parentSnapshotWasMigrated) { + parentSnapshotPath = rebaseResourceToNewParentPath(resourceCurrentPath, parentSnapshotPath); + } else { + parentSnapshotPath = resourceCurrentPath; + } + parentSnapshotWasMigrated = false; + } + } + } catch (LibvirtException | QemuImgException e) { + logger.error("Exception while migrating snapshots [{}] to secondary storage [{}] due to: [{}].", command.getSnapshotChain(), imagePool, e.getMessage(), e); + return new MigrateBetweenSecondaryStoragesCommandAnswer(command, false, "Migration of snapshots between secondary storages failed", resourcesToUpdate); + } finally { + for (String file : filesToRemove) { + removeResourceFromSourceDataStore(file); + } + + for (KVMStoragePool storagePool : imagePools) { + storagePoolManager.deleteStoragePool(storagePool.getType(), storagePool.getUuid()); + } + } + + return new MigrateBetweenSecondaryStoragesCommandAnswer(command, true, "success", resourcesToUpdate); + } + + public String copyResourceToDestDataStore(DataTO resource, String resourceCurrentPath, KVMStoragePool destImagePool, String resourceParentPath) throws QemuImgException, LibvirtException { + String resourceDestDataStoreFullPath = destImagePool.getLocalPathFor(resource.getPath()); + String resourceDestCheckpointPath = resourceDestDataStoreFullPath.replace("snapshots", "checkpoints"); + + QemuImgFile resourceOrigin = new QemuImgFile(resourceCurrentPath, QemuImg.PhysicalDiskFormat.QCOW2); + QemuImgFile resourceDestination = new QemuImgFile(resourceDestDataStoreFullPath, QemuImg.PhysicalDiskFormat.QCOW2); + QemuImgFile parentResource = null; + + if (resourceParentPath != null) { + parentResource = new QemuImgFile(resourceParentPath, QemuImg.PhysicalDiskFormat.QCOW2); + } + + logger.debug("Migrating {} [{}] to [{}] with {}", resourceType, resourceOrigin, resourceDestination, parentResource == null ? "no parent." : String.format("parent [%s].", parentResource)); + + long resourceId = resource.getId(); + + createDirsIfNeeded(resourceDestDataStoreFullPath, resourceId); + + QemuImg qemuImg = new QemuImg(wait); + qemuImg.convert(resourceOrigin, resourceDestination, parentResource, null, null, new QemuImageOptions(resourceOrigin.getFormat(), resourceOrigin.getFileName(), null), null, true, false); + + filesToRemove.add(resourceCurrentPath); + + String resourceCurrentCheckpointPath = resourceCurrentPath.replace("snapshots", "checkpoints"); + createDirsIfNeeded(resourceDestCheckpointPath, resourceId); + migrateCheckpointFile(resourceCurrentPath, resourceDestDataStoreFullPath); + filesToRemove.add(resourceCurrentCheckpointPath); + resourcesToUpdate.add(new Pair<>(resourceId, resourceDestCheckpointPath)); + + return resourceDestDataStoreFullPath; + } + + private void migrateCheckpointFile(String resourceCurrentPath, String resourceDestDataStoreFullPath) { + resourceCurrentPath = resourceCurrentPath.replace("snapshots", "checkpoints"); + resourceDestDataStoreFullPath = resourceDestDataStoreFullPath.replace("snapshots", "checkpoints"); + + String copyCommand = String.format("cp %s %s", resourceCurrentPath, resourceDestDataStoreFullPath); + Script.runSimpleBashScript(copyCommand); + } + + public void removeResourceFromSourceDataStore(String resourcePath) { + logger.debug("Removing file [{}].", resourcePath); + try { + Files.deleteIfExists(Path.of(resourcePath)); + } catch (IOException ex) { + logger.error("Failed to remove {} [{}].", resourceType, resourcePath, ex); + } + } + + public String rebaseResourceToNewParentPath(String resourcePath, String parentResourcePath) throws LibvirtException, QemuImgException { + QemuImgFile resource = new QemuImgFile(resourcePath, QemuImg.PhysicalDiskFormat.QCOW2); + QemuImgFile parentResource = new QemuImgFile(parentResourcePath, QemuImg.PhysicalDiskFormat.QCOW2); + + QemuImg qemuImg = new QemuImg(wait); + qemuImg.rebase(resource, parentResource, parentResource.getFormat().toString(), false); + + return resourcePath; + } + + private void createDirsIfNeeded(String resourceFullPath, Long resourceId) { + String dirs = resourceFullPath.substring(0, resourceFullPath.lastIndexOf(File.separator)); + try { + Files.createDirectories(Path.of(dirs)); + } catch (IOException e) { + throw new CloudRuntimeException(String.format("Error while creating directories for migration of %s [%s].", resourceType, resourceId), e); + } + } +} diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptor.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptor.java index e8924ecf5ebc..1c384fff2bf7 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptor.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptor.java @@ -1619,7 +1619,7 @@ to support snapshots(backuped) as qcow2 files. */ destFile = new QemuImgFile(destPath, destFormat); try { boolean isQCOW2 = PhysicalDiskFormat.QCOW2.equals(sourceFormat); - qemu.convert(srcFile, destFile, null, null, new QemuImageOptions(srcFile.getFormat(), srcFile.getFileName(), null), + qemu.convert(srcFile, destFile, null, null, null, new QemuImageOptions(srcFile.getFormat(), srcFile.getFileName(), null), null, false, isQCOW2); Map destInfo = qemu.info(destFile); Long virtualSize = Long.parseLong(destInfo.get(QemuImg.VIRTUAL_SIZE)); diff --git a/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/utils/qemu/QemuImg.java b/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/utils/qemu/QemuImg.java index 0a8ea27cd490..412d2c9a5b42 100644 --- a/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/utils/qemu/QemuImg.java +++ b/plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/utils/qemu/QemuImg.java @@ -390,7 +390,7 @@ public void convert(final QemuImgFile srcFile, final QemuImgFile destFile, */ public void convert(final QemuImgFile srcFile, final QemuImgFile destFile, final Map options, final List qemuObjects, final QemuImageOptions srcImageOpts, final String snapshotName, final boolean forceSourceFormat) throws QemuImgException { - convert(srcFile, destFile, options, qemuObjects, srcImageOpts, snapshotName, forceSourceFormat, false); + convert(srcFile, destFile, null, options, qemuObjects, srcImageOpts, snapshotName, forceSourceFormat, false); } /** @@ -402,6 +402,8 @@ public void convert(final QemuImgFile srcFile, final QemuImgFile destFile, * The source file. * @param destFile * The destination file. + * @param backingFile + * The destination's backing file. * @param options * Options for the conversion. Takes a Map with key value * pairs which are passed on to qemu-img without validation. @@ -417,7 +419,7 @@ public void convert(final QemuImgFile srcFile, final QemuImgFile destFile, * If true, copies the bitmaps to the destination image. * @return void */ - public void convert(final QemuImgFile srcFile, final QemuImgFile destFile, + public void convert(final QemuImgFile srcFile, final QemuImgFile destFile, QemuImgFile backingFile, final Map options, final List qemuObjects, final QemuImageOptions srcImageOpts, final String snapshotName, final boolean forceSourceFormat, boolean keepBitmaps) throws QemuImgException { @@ -443,6 +445,7 @@ public void convert(final QemuImgFile srcFile, final QemuImgFile destFile, script.add("-O"); script.add(destFile.getFormat().toString()); + addBackingFileToConvertCommand(script, backingFile); addScriptOptionsFromMap(options, script); addSnapshotToConvertCommand(srcFile.getFormat().toString(), snapshotName, forceSourceFormat, script, version); @@ -488,6 +491,22 @@ public void convert(final QemuImgFile srcFile, final QemuImgFile destFile, } } + private void addBackingFileToConvertCommand(Script script, QemuImgFile backingFile) { + if (backingFile == null) { + return; + } + + script.add("-o"); + + String opts; + if (backingFile.getFormat() == null) { + opts = String.format("backing_file=%s", backingFile.getFileName()); + } else { + opts = String.format("backing_file=%s,backing_fmt=%s", backingFile.getFileName(), backingFile.getFormat().toString()); + } + script.add(opts); + } + /** * Qemu version 2.0.0 added (via commit ef80654d0dc1edf2dd2a51feff8cc3e1102a6583) the * flag "-l" to inform the snapshot name or ID diff --git a/server/src/main/java/com/cloud/storage/StorageManagerImpl.java b/server/src/main/java/com/cloud/storage/StorageManagerImpl.java index df0283ae2d61..5d97fbb027c0 100644 --- a/server/src/main/java/com/cloud/storage/StorageManagerImpl.java +++ b/server/src/main/java/com/cloud/storage/StorageManagerImpl.java @@ -4602,7 +4602,8 @@ public ConfigKey[] getConfigKeys() { DataStoreDownloadFollowRedirects, AllowVolumeReSizeBeyondAllocation, StoragePoolHostConnectWorkers, - ObjectStorageCapacityThreshold + ObjectStorageCapacityThreshold, + AgentMaxDataMigrationWaitTime }; }