From 1093a9626e1d401ba3501882797c2ea2b0fd54c7 Mon Sep 17 00:00:00 2001 From: Likitha Shetty Date: Thu, 30 Apr 2015 10:35:57 +0530 Subject: [PATCH 1/2] CLOUDSTACK-8612. [VMware] Make vCenter session timeout configurable for volume snapshot. Separate out vmware vcenter session timeout config for volume snapshot backup operation. --- .../com/cloud/hypervisor/guru/VMwareGuru.java | 14 +++++++- .../vmware/VmwareServerDiscoverer.java | 3 +- .../vmware/manager/VmwareManager.java | 3 ++ .../vmware/manager/VmwareManagerImpl.java | 15 ++++++-- .../vmware/resource/VmwareContextFactory.java | 16 ++++----- .../vmware/resource/VmwareResource.java | 19 +++++++--- .../VmwareSecondaryStorageContextFactory.java | 18 +++++----- ...VmwareSecondaryStorageResourceHandler.java | 35 +++++++++++++------ .../src/com/cloud/configuration/Config.java | 1 + .../vmware/util/VmwareContextPool.java | 9 ++--- .../vmware/mo/TestVmwareContextFactory.java | 12 +++---- 11 files changed, 99 insertions(+), 46 deletions(-) diff --git a/plugins/hypervisors/vmware/src/com/cloud/hypervisor/guru/VMwareGuru.java b/plugins/hypervisors/vmware/src/com/cloud/hypervisor/guru/VMwareGuru.java index 9ac32d4757eb..630f529a8dbe 100644 --- a/plugins/hypervisors/vmware/src/com/cloud/hypervisor/guru/VMwareGuru.java +++ b/plugins/hypervisors/vmware/src/com/cloud/hypervisor/guru/VMwareGuru.java @@ -447,7 +447,19 @@ public Pair getCommandHostDelegation(long hostId, Command cmd) { _cmdExecLogDao.persist(execLog); cmd.setContextParam("execid", String.valueOf(execLog.getId())); cmd.setContextParam("noderuninfo", String.format("%d-%d", _clusterMgr.getManagementNodeId(), _clusterMgr.getCurrentRunId())); - cmd.setContextParam("vCenterSessionTimeout", String.valueOf(_vmwareMgr.getVcenterSessionTimeout())); + if (cmd instanceof CopyCommand) { // Snapshot backup + CopyCommand cpyCommand = (CopyCommand)cmd; + DataTO srcData = cpyCommand.getSrcTO(); + DataTO destData = cpyCommand.getDestTO(); + if (srcData.getObjectType() == DataObjectType.SNAPSHOT && destData.getObjectType() == DataObjectType.SNAPSHOT && + srcData.getDataStore().getRole() == DataStoreRole.Primary) { + cmd.setContextParam("vCenterSessionTimeout", String.valueOf(_vmwareMgr.getSnapshotBackupSessionTimeout())); + } else { + cmd.setContextParam("vCenterSessionTimeout", String.valueOf(_vmwareMgr.getVcenterSessionTimeout())); + } + } else { + cmd.setContextParam("vCenterSessionTimeout", String.valueOf(_vmwareMgr.getVcenterSessionTimeout())); + } if (cmd instanceof BackupSnapshotCommand || cmd instanceof CreatePrivateTemplateFromVolumeCommand || cmd instanceof CreatePrivateTemplateFromSnapshotCommand || cmd instanceof CopyVolumeCommand || cmd instanceof CopyCommand || diff --git a/plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/VmwareServerDiscoverer.java b/plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/VmwareServerDiscoverer.java index 20eb6595f83a..5f7179a2b017 100644 --- a/plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/VmwareServerDiscoverer.java +++ b/plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/VmwareServerDiscoverer.java @@ -314,7 +314,8 @@ public VmwareServerDiscoverer() { VmwareContext context = null; try { - context = VmwareContextFactory.create(url.getHost(), username, password); + int sessionTimeout = _vmwareMgr.getVcenterSessionTimeout(); + context = VmwareContextFactory.create(url.getHost(), username, password, sessionTimeout); if (privateTrafficLabel != null) context.registerStockObject("privateTrafficLabel", privateTrafficLabel); diff --git a/plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/manager/VmwareManager.java b/plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/manager/VmwareManager.java index 35e275e004e7..f952bea4eb1c 100644 --- a/plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/manager/VmwareManager.java +++ b/plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/manager/VmwareManager.java @@ -79,4 +79,7 @@ public interface VmwareManager { boolean isLegacyZone(long dcId); boolean hasNexusVSM(Long clusterId); + + public int getSnapshotBackupSessionTimeout(); + } diff --git a/plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/manager/VmwareManagerImpl.java b/plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/manager/VmwareManagerImpl.java index 5b85c29d331b..fe05f77f714a 100644 --- a/plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/manager/VmwareManagerImpl.java +++ b/plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/manager/VmwareManagerImpl.java @@ -185,6 +185,7 @@ public class VmwareManagerImpl extends ManagerBase implements VmwareManager, Vmw private int _additionalPortRangeSize; private int _routerExtraPublicNics = 2; private int _vCenterSessionTimeout = 1200000; // Timeout in milliseconds + private int _snapshotBackupSessionTimeout = 1200000; // Timeout in milliseconds private String _rootDiskController = DiskControllerType.ide.toString(); @@ -283,6 +284,9 @@ public boolean configure(String name, Map params) throws Configu _vCenterSessionTimeout = NumbersUtil.parseInt(_configDao.getValue(Config.VmwareVcenterSessionTimeout.key()), 1200) * 1000; s_logger.info("VmwareManagerImpl config - vmware.vcenter.session.timeout: " + _vCenterSessionTimeout); + _snapshotBackupSessionTimeout = NumbersUtil.parseInt(_configDao.getValue(Config.VmwareSnapshotBackupSessionTimeout.key()), 1200) * 1000; + s_logger.info("VmwareManagerImpl config - vmware.snapshot.backup.session.timeout: " + _snapshotBackupSessionTimeout); + _recycleHungWorker = _configDao.getValue(Config.VmwareRecycleHungWorker.key()); if (_recycleHungWorker == null || _recycleHungWorker.isEmpty()) { _recycleHungWorker = "false"; @@ -487,6 +491,8 @@ public void setupResourceStartupParams(Map params) { params.put("vmware.root.disk.controller", _rootDiskController); params.put("vmware.recycle.hung.wokervm", _recycleHungWorker); params.put("ports.per.dvportgroup", _portsPerDvPortGroup); + params.put("vmware.vcenter.session.timeout", _vCenterSessionTimeout); + params.put("vmware.snapshot.backup.session.timeout", _snapshotBackupSessionTimeout); } @Override @@ -942,6 +948,11 @@ public int getVcenterSessionTimeout() { return _vCenterSessionTimeout; } + @Override + public int getSnapshotBackupSessionTimeout(){ + return _snapshotBackupSessionTimeout; + } + @Override public List> getCommands() { List> cmdList = new ArrayList>(); @@ -1022,7 +1033,7 @@ public VmwareDatacenterVO addVmwareDatacenter(AddVmwareDcCmd cmd) throws Resourc String guid; ManagedObjectReference dcMor; try { - context = VmwareContextFactory.create(vCenterHost, userName, password); + context = VmwareContextFactory.create(vCenterHost, userName, password, _vCenterSessionTimeout); // Check if DC exists on vCenter dcMo = new DatacenterMO(context, vmwareDcName); @@ -1119,7 +1130,7 @@ public void doInTransactionWithoutResult(TransactionStatus status) { // Construct context VmwareContext context = null; try { - context = VmwareContextFactory.create(vCenterHost, userName, password); + context = VmwareContextFactory.create(vCenterHost, userName, password, _vCenterSessionTimeout); // Check if DC exists on vCenter try { diff --git a/plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/resource/VmwareContextFactory.java b/plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/resource/VmwareContextFactory.java index 11163e90d6f9..9a1e7092e6f1 100644 --- a/plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/resource/VmwareContextFactory.java +++ b/plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/resource/VmwareContextFactory.java @@ -55,7 +55,7 @@ void init() { s_clusterMgr = _clusterMgr; } - public static VmwareContext create(String vCenterAddress, String vCenterUserName, String vCenterPassword) throws Exception { + public static VmwareContext create(String vCenterAddress, String vCenterUserName, String vCenterPassword, int sessionTimeout) throws Exception { assert (vCenterAddress != null); assert (vCenterUserName != null); assert (vCenterPassword != null); @@ -66,7 +66,7 @@ public static VmwareContext create(String vCenterAddress, String vCenterUserName StringUtils.getMaskedPasswordForDisplay(vCenterPassword)); VmwareClient vimClient = new VmwareClient(vCenterAddress + "-" + s_seq++); - vimClient.setVcenterSessionTimeout(s_vmwareMgr.getVcenterSessionTimeout()); + vimClient.setVcenterSessionTimeout(sessionTimeout); vimClient.connect(serviceUrl, vCenterUserName, vCenterPassword); VmwareContext context = new VmwareContext(vimClient, vCenterAddress); @@ -76,22 +76,22 @@ public static VmwareContext create(String vCenterAddress, String vCenterUserName context.registerStockObject("manageportgroup", s_vmwareMgr.getManagementPortGroupName()); context.registerStockObject("noderuninfo", String.format("%d-%d", s_clusterMgr.getManagementNodeId(), s_clusterMgr.getCurrentRunId())); - context.setPoolInfo(s_pool, VmwareContextPool.composePoolKey(vCenterAddress, vCenterUserName)); + context.setPoolInfo(s_pool, VmwareContextPool.composePoolKey(vCenterAddress, vCenterUserName, sessionTimeout)); s_pool.registerOutstandingContext(context); return context; } - public static VmwareContext getContext(String vCenterAddress, String vCenterUserName, String vCenterPassword) throws Exception { - VmwareContext context = s_pool.getContext(vCenterAddress, vCenterUserName); + public static VmwareContext getContext(String vCenterAddress, String vCenterUserName, String vCenterPassword, int sessionTimeout) throws Exception { + VmwareContext context = s_pool.getContext(vCenterAddress, vCenterUserName, sessionTimeout); if (context == null) { - context = create(vCenterAddress, vCenterUserName, vCenterPassword); + context = create(vCenterAddress, vCenterUserName, vCenterPassword, sessionTimeout); } else { // Validate current context and verify if vCenter session timeout value of the context matches the timeout value set by Admin - if (!context.validate() || (context.getVimClient().getVcenterSessionTimeout() != s_vmwareMgr.getVcenterSessionTimeout())) { + if (!context.validate()) { s_logger.info("Validation of the context failed, dispose and create a new one"); context.close(); - context = create(vCenterAddress, vCenterUserName, vCenterPassword); + context = create(vCenterAddress, vCenterUserName, vCenterPassword, sessionTimeout); } } diff --git a/plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/resource/VmwareResource.java b/plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/resource/VmwareResource.java index 6284f86e64cd..6e9e6b41a054 100644 --- a/plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/resource/VmwareResource.java +++ b/plugins/hypervisors/vmware/src/com/cloud/hypervisor/vmware/resource/VmwareResource.java @@ -308,7 +308,7 @@ public class VmwareResource implements StoragePoolResource, ServerResource, Vmwa protected int _portsPerDvPortGroup; protected boolean _fullCloneFlag = false; protected boolean _instanceNameFlag = false; - + protected int _vCenterSessionTimeout = 1200000; // Timeout in milliseconds protected boolean _recycleHungWorker = false; protected DiskControllerType _rootDiskController = DiskControllerType.ide; @@ -4983,6 +4983,10 @@ public boolean configure(String name, Map params) throws Configu _instanceNameFlag = false; } + Integer sessionTimeout = (Integer) params.get("vmware.vcenter.session.timeout"); + if (sessionTimeout != null) + _vCenterSessionTimeout = sessionTimeout.intValue(); + value = (String)params.get("scripts.timeout"); int timeout = NumbersUtil.parseInt(value, 1440) * 1000; _storageProcessor = new VmwareStorageProcessor((VmwareHostService)this, _fullCloneFlag, (VmwareStorageMount)mgr, timeout, this, _shutdownWaitMs, null); @@ -5035,9 +5039,16 @@ public VmwareHypervisorHost getHyperHost(VmwareContext context) { @Override public VmwareContext getServiceContext(Command cmd) { VmwareContext context = null; + int vCenterSessionTimeout; + if (cmd != null && cmd.getContextParam("vCenterSessionTimeout") != null) { + vCenterSessionTimeout = NumbersUtil.parseInt(cmd.getContextParam("vCenterSessionTimeout"), 1200000); + } else { + vCenterSessionTimeout = _vCenterSessionTimeout; + } + if(s_serviceContext.get() != null) { context = s_serviceContext.get(); - String poolKey = VmwareContextPool.composePoolKey(_vCenterAddress, _username); + String poolKey = VmwareContextPool.composePoolKey(_vCenterAddress, _username, vCenterSessionTimeout); // Before re-using the thread local context, ensure it corresponds to the right vCenter API session and that it is valid to make calls. if(context.getPoolKey().equals(poolKey)) { if (context.validate()) { @@ -5051,11 +5062,11 @@ public VmwareContext getServiceContext(Command cmd) { } } else { // Exisitng ThreadLocal context corresponds to a different vCenter API session. Why has it not been recycled? - s_logger.warn("ThreadLocal VMware context: " + poolKey + " doesn't correspond to the right vCenter. Expected VMware context: " + context.getPoolKey()); + s_logger.warn("ThreadLocal VMware context: " + poolKey + ". Expected VMware context: " + context.getPoolKey()); } } try { - context = VmwareContextFactory.getContext(_vCenterAddress, _username, _password); + context = VmwareContextFactory.getContext(_vCenterAddress, _username, _password, vCenterSessionTimeout); s_serviceContext.set(context); } catch (Exception e) { s_logger.error("Unable to connect to vSphere server: " + _vCenterAddress, e); diff --git a/plugins/hypervisors/vmware/src/com/cloud/storage/resource/VmwareSecondaryStorageContextFactory.java b/plugins/hypervisors/vmware/src/com/cloud/storage/resource/VmwareSecondaryStorageContextFactory.java index ee0273ca1baf..fb5f14c0d48c 100644 --- a/plugins/hypervisors/vmware/src/com/cloud/storage/resource/VmwareSecondaryStorageContextFactory.java +++ b/plugins/hypervisors/vmware/src/com/cloud/storage/resource/VmwareSecondaryStorageContextFactory.java @@ -36,34 +36,34 @@ public static void initFactoryEnvironment() { s_pool = new VmwareContextPool(); } - public static VmwareContext create(String vCenterAddress, String vCenterUserName, String vCenterPassword) throws Exception { + public static VmwareContext create(String vCenterAddress, String vCenterUserName, String vCenterPassword, int sessionTimeout) throws Exception { assert (vCenterAddress != null); assert (vCenterUserName != null); assert (vCenterPassword != null); String serviceUrl = "https://" + vCenterAddress + "/sdk/vimService"; VmwareClient vimClient = new VmwareClient(vCenterAddress + "-" + s_seq++); - vimClient.setVcenterSessionTimeout(s_vCenterSessionTimeout); + vimClient.setVcenterSessionTimeout(sessionTimeout); vimClient.connect(serviceUrl, vCenterUserName, vCenterPassword); VmwareContext context = new VmwareContext(vimClient, vCenterAddress); assert (context != null); - context.setPoolInfo(s_pool, VmwareContextPool.composePoolKey(vCenterAddress, vCenterUserName)); + context.setPoolInfo(s_pool, VmwareContextPool.composePoolKey(vCenterAddress, vCenterUserName, sessionTimeout)); s_pool.registerOutstandingContext(context); return context; } - public static VmwareContext getContext(String vCenterAddress, String vCenterUserName, String vCenterPassword) throws Exception { - VmwareContext context = s_pool.getContext(vCenterAddress, vCenterUserName); + public static VmwareContext getContext(String vCenterAddress, String vCenterUserName, String vCenterPassword, int sessionTimeout) throws Exception { + VmwareContext context = s_pool.getContext(vCenterAddress, vCenterUserName, sessionTimeout); if (context == null) { - context = create(vCenterAddress, vCenterUserName, vCenterPassword); + context = create(vCenterAddress, vCenterUserName, vCenterPassword, sessionTimeout); } else { - // Validate current context and verify if vCenter session timeout value of the context matches the timeout value set by Admin - if (!context.validate() || (context.getVimClient().getVcenterSessionTimeout() != s_vCenterSessionTimeout)) { + // Validate current context + if (!context.validate()) { s_logger.info("Validation of the context faild. dispose and create a new one"); context.close(); - context = create(vCenterAddress, vCenterUserName, vCenterPassword); + context = create(vCenterAddress, vCenterUserName, vCenterPassword, sessionTimeout); } } diff --git a/plugins/hypervisors/vmware/src/com/cloud/storage/resource/VmwareSecondaryStorageResourceHandler.java b/plugins/hypervisors/vmware/src/com/cloud/storage/resource/VmwareSecondaryStorageResourceHandler.java index 8a277991aac6..5e5524dad46a 100644 --- a/plugins/hypervisors/vmware/src/com/cloud/storage/resource/VmwareSecondaryStorageResourceHandler.java +++ b/plugins/hypervisors/vmware/src/com/cloud/storage/resource/VmwareSecondaryStorageResourceHandler.java @@ -46,6 +46,7 @@ import com.cloud.hypervisor.vmware.mo.VmwareHypervisorHost; import com.cloud.hypervisor.vmware.mo.VmwareHypervisorHostNetworkSummary; import com.cloud.hypervisor.vmware.util.VmwareContext; +import com.cloud.hypervisor.vmware.util.VmwareContextPool; import com.cloud.serializer.GsonHelper; import com.cloud.utils.NumbersUtil; import com.cloud.utils.Pair; @@ -209,20 +210,32 @@ public VmwareContext getServiceContext(Command cmd) { _resource.ensureOutgoingRuleForAddress(vCenterAddress); VmwareContext context = currentContext.get(); - if (context != null && !context.validate()) { - invalidateServiceContext(context); - context = null; + + if (context != null) { + String poolKey = VmwareContextPool.composePoolKey(vCenterAddress, username, vCenterSessionTimeout); + // Before re-using the thread local context, ensure it corresponds to the right vCenter API session with the expected timeout and that it is valid to make calls. + if(context.getPoolKey().equals(poolKey)) { + if (context.validate()) { + if (s_logger.isTraceEnabled()) { + s_logger.trace("ThreadLocal context is still valid, just reuse"); + } + } else { + s_logger.info("Validation of the context failed, dispose and use a new one"); + invalidateServiceContext(context); + context = null; + } + } else { + s_logger.warn("ThreadLocal VMware context: " + poolKey + ". Expected VMware context: " + context.getPoolKey()); + context = null; + } } + if (context == null) { - s_logger.info("Open new VmwareContext. vCenter: " + vCenterAddress + ", user: " + username + ", password: " + StringUtils.getMaskedPasswordForDisplay(password)); - VmwareSecondaryStorageContextFactory.setVcenterSessionTimeout(vCenterSessionTimeout); - context = VmwareSecondaryStorageContextFactory.getContext(vCenterAddress, username, password); - } - if (context != null) { - context.registerStockObject("serviceconsole", cmd.getContextParam("serviceconsole")); - context.registerStockObject("manageportgroup", cmd.getContextParam("manageportgroup")); - context.registerStockObject("noderuninfo", cmd.getContextParam("noderuninfo")); + context = VmwareSecondaryStorageContextFactory.getContext(vCenterAddress, username, password, vCenterSessionTimeout); } + context.registerStockObject("serviceconsole", cmd.getContextParam("serviceconsole")); + context.registerStockObject("manageportgroup", cmd.getContextParam("manageportgroup")); + context.registerStockObject("noderuninfo", cmd.getContextParam("noderuninfo")); currentContext.set(context); return context; } catch (Exception e) { diff --git a/server/src/com/cloud/configuration/Config.java b/server/src/com/cloud/configuration/Config.java index ca898819f6e9..8afdf98ac600 100644 --- a/server/src/com/cloud/configuration/Config.java +++ b/server/src/com/cloud/configuration/Config.java @@ -1247,6 +1247,7 @@ public enum Config { "When set to true this will enable nested virtualization when this is supported by the hypervisor", null), VmwareVcenterSessionTimeout("Advanced", ManagementServer.class, Long.class, "vmware.vcenter.session.timeout", "1200", "VMware client timeout in seconds", null), + VmwareSnapshotBackupSessionTimeout("Advanced", ManagementServer.class, Long.class, "vmware.snapshot.backup.session.timeout", "1200", "VMware client timeout in seconds for snapshot backup", null), // Midonet MidoNetAPIServerAddress( diff --git a/vmware-base/src/com/cloud/hypervisor/vmware/util/VmwareContextPool.java b/vmware-base/src/com/cloud/hypervisor/vmware/util/VmwareContextPool.java index c97c01f32c33..92729f5e0379 100644 --- a/vmware-base/src/com/cloud/hypervisor/vmware/util/VmwareContextPool.java +++ b/vmware-base/src/com/cloud/hypervisor/vmware/util/VmwareContextPool.java @@ -72,8 +72,8 @@ public void unregisterOutstandingContext(VmwareContext context) { } } - public VmwareContext getContext(String vCenterAddress, String vCenterUserName) { - String poolKey = composePoolKey(vCenterAddress, vCenterUserName); + public VmwareContext getContext(String vCenterAddress, String vCenterUserName, int vCenterSessionTimeout) { + String poolKey = composePoolKey(vCenterAddress, vCenterUserName, vCenterSessionTimeout); synchronized (this) { List l = _pool.get(poolKey); if (l == null) @@ -162,9 +162,10 @@ private void doKeepAlive() { } } - public static String composePoolKey(String vCenterAddress, String vCenterUserName) { + public static String composePoolKey(String vCenterAddress, String vCenterUserName, int vCenterSessionTimeout) { assert (vCenterUserName != null); assert (vCenterAddress != null); - return vCenterUserName + "@" + vCenterAddress; + int sessionTimeoutInSeconds = vCenterSessionTimeout/1000; + return vCenterUserName + "@" + vCenterAddress + ";" + String.valueOf(sessionTimeoutInSeconds); } } diff --git a/vmware-base/test/com/cloud/hypervisor/vmware/mo/TestVmwareContextFactory.java b/vmware-base/test/com/cloud/hypervisor/vmware/mo/TestVmwareContextFactory.java index 3d0e73690172..bd3849914c68 100644 --- a/vmware-base/test/com/cloud/hypervisor/vmware/mo/TestVmwareContextFactory.java +++ b/vmware-base/test/com/cloud/hypervisor/vmware/mo/TestVmwareContextFactory.java @@ -37,7 +37,7 @@ public class TestVmwareContextFactory { s_pool = new VmwareContextPool(); } - public static VmwareContext create(String vCenterAddress, String vCenterUserName, String vCenterPassword) throws Exception { + public static VmwareContext create(String vCenterAddress, String vCenterUserName, String vCenterPassword, int sessionTimeout) throws Exception { assert (vCenterAddress != null); assert (vCenterUserName != null); assert (vCenterPassword != null); @@ -49,20 +49,20 @@ public static VmwareContext create(String vCenterAddress, String vCenterUserName StringUtils.getMaskedPasswordForDisplay(vCenterPassword)); VmwareClient vimClient = new VmwareClient(vCenterAddress + "-" + s_seq++); - vimClient.setVcenterSessionTimeout(1200000); + vimClient.setVcenterSessionTimeout(sessionTimeout); vimClient.connect(serviceUrl, vCenterUserName, vCenterPassword); VmwareContext context = new VmwareContext(vimClient, vCenterAddress); return context; } - public static VmwareContext getContext(String vCenterAddress, String vCenterUserName, String vCenterPassword) throws Exception { - VmwareContext context = s_pool.getContext(vCenterAddress, vCenterUserName); + public static VmwareContext getContext(String vCenterAddress, String vCenterUserName, String vCenterPassword, int sessionTimeout) throws Exception { + VmwareContext context = s_pool.getContext(vCenterAddress, vCenterUserName, sessionTimeout); if (context == null) - context = create(vCenterAddress, vCenterUserName, vCenterPassword); + context = create(vCenterAddress, vCenterUserName, vCenterPassword, sessionTimeout); if (context != null) { - context.setPoolInfo(s_pool, VmwareContextPool.composePoolKey(vCenterAddress, vCenterUserName)); + context.setPoolInfo(s_pool, VmwareContextPool.composePoolKey(vCenterAddress, vCenterUserName, sessionTimeout)); } return context; From c8c9d81b0c2f08f506e828453efbed7911d9a517 Mon Sep 17 00:00:00 2001 From: Likitha Shetty Date: Tue, 5 May 2015 13:59:49 +0530 Subject: [PATCH 2/2] CLOUDSTACK-8612. [VMware] Make vCenter session timeout configurable for volume snapshot. Separate out async job timeout for volume snapshot job from other vm work jobs. --- .../framework/jobs/dao/SyncQueueItemDao.java | 2 +- .../jobs/dao/SyncQueueItemDaoImpl.java | 49 ++++++++++++++----- .../jobs/impl/AsyncJobManagerImpl.java | 7 ++- .../framework/jobs/impl/SyncQueueManager.java | 2 +- .../jobs/impl/SyncQueueManagerImpl.java | 4 +- 5 files changed, 45 insertions(+), 19 deletions(-) diff --git a/framework/jobs/src/org/apache/cloudstack/framework/jobs/dao/SyncQueueItemDao.java b/framework/jobs/src/org/apache/cloudstack/framework/jobs/dao/SyncQueueItemDao.java index 7b6eed75a008..798de4da4402 100644 --- a/framework/jobs/src/org/apache/cloudstack/framework/jobs/dao/SyncQueueItemDao.java +++ b/framework/jobs/src/org/apache/cloudstack/framework/jobs/dao/SyncQueueItemDao.java @@ -30,7 +30,7 @@ public interface SyncQueueItemDao extends GenericDao { public List getActiveQueueItems(Long msid, boolean exclusive); - public List getBlockedQueueItems(long thresholdMs, boolean exclusive); + public List getBlockedQueueItems(long thresholdMs, long snapshotThresholdMs, String jobCmd, boolean exclusive); public Long getQueueItemIdByContentIdAndType(long contentId, String contentType); } diff --git a/framework/jobs/src/org/apache/cloudstack/framework/jobs/dao/SyncQueueItemDaoImpl.java b/framework/jobs/src/org/apache/cloudstack/framework/jobs/dao/SyncQueueItemDaoImpl.java index 29c3f1b289fb..1b064180d747 100644 --- a/framework/jobs/src/org/apache/cloudstack/framework/jobs/dao/SyncQueueItemDaoImpl.java +++ b/framework/jobs/src/org/apache/cloudstack/framework/jobs/dao/SyncQueueItemDaoImpl.java @@ -141,23 +141,46 @@ public List getActiveQueueItems(Long msid, boolean exclusive) { } @Override - public List getBlockedQueueItems(long thresholdMs, boolean exclusive) { + public List getBlockedQueueItems(long thresholdMs, long snapshotThresholdMs, String jobCmd, boolean exclusive) { Date cutTime = DateUtil.currentGMTTime(); + List l = new ArrayList(); - SearchBuilder sbItem = createSearchBuilder(); - sbItem.and("lastProcessMsid", sbItem.entity().getLastProcessMsid(), SearchCriteria.Op.NNULL); - sbItem.and("lastProcessNumber", sbItem.entity().getLastProcessNumber(), SearchCriteria.Op.NNULL); - sbItem.and("lastProcessTime", sbItem.entity().getLastProcessTime(), SearchCriteria.Op.NNULL); - sbItem.and("lastProcessTime2", sbItem.entity().getLastProcessTime(), SearchCriteria.Op.LT); - - sbItem.done(); + Date date1 = new Date(cutTime.getTime() - thresholdMs); + String dateString1 = DateUtil.getDateDisplayString(TimeZone.getTimeZone("GMT"), date1); - SearchCriteria sc = sbItem.create(); - sc.setParameters("lastProcessTime2", new Date(cutTime.getTime() - thresholdMs)); + Date date2 = new Date(cutTime.getTime() - snapshotThresholdMs); + String dateString2 = DateUtil.getDateDisplayString(TimeZone.getTimeZone("GMT"), date2); - if(exclusive) - return lockRows(sc, null, true); - return listBy(sc, null); + String sql = "SELECT s.id, s.queue_id, s.content_type, s.content_id, s.queue_proc_msid, s.queue_proc_number, s.queue_proc_time, s.created FROM sync_queue_item AS s " + + "JOIN async_job as a ON s.content_id = a.id WHERE s.queue_proc_msid IS NOT NULL AND s.queue_proc_number IS NOT NULL AND s.queue_proc_time IS NOT NULL" + + " AND ((a.job_cmd NOT LIKE ? AND s.queue_proc_time < ? ) OR (a.job_cmd LIKE ? AND s.queue_proc_time < ?))"; + TransactionLegacy txn = TransactionLegacy.currentTxn(); + PreparedStatement pstmt = null; + try { + pstmt = txn.prepareAutoCloseStatement(sql); + pstmt.setString(1, "%" + jobCmd + "%"); + pstmt.setString(2, dateString1); + pstmt.setString(3, "%" + jobCmd + "%"); + pstmt.setString(4, dateString2); + ResultSet rs = pstmt.executeQuery(); + while(rs.next()) { + SyncQueueItemVO item = new SyncQueueItemVO(); + item.setId(rs.getLong(1)); + item.setQueueId(rs.getLong(2)); + item.setContentType(rs.getString(3)); + item.setContentId(rs.getLong(4)); + item.setLastProcessMsid(rs.getLong(5)); + item.setLastProcessNumber(rs.getLong(6)); + item.setLastProcessTime(rs.getDate(7)); + item.setCreated(rs.getDate(8)); + l.add(item); + } + } catch (SQLException e) { + s_logger.error("Unexpected sql excetpion, ", e); + } catch (Throwable e) { + s_logger.error("Unexpected excetpion, ", e); + } + return l; } @Override diff --git a/framework/jobs/src/org/apache/cloudstack/framework/jobs/impl/AsyncJobManagerImpl.java b/framework/jobs/src/org/apache/cloudstack/framework/jobs/impl/AsyncJobManagerImpl.java index 1ea3c7810167..3078eb9f6178 100644 --- a/framework/jobs/src/org/apache/cloudstack/framework/jobs/impl/AsyncJobManagerImpl.java +++ b/framework/jobs/src/org/apache/cloudstack/framework/jobs/impl/AsyncJobManagerImpl.java @@ -92,6 +92,8 @@ public class AsyncJobManagerImpl extends ManagerBase implements AsyncJobManager, private static final ConfigKey VmJobLockTimeout = new ConfigKey("Advanced", Integer.class, "vm.job.lock.timeout", "1800", "Time in seconds to wait in acquiring lock to submit a vm worker job", false); + private static final ConfigKey VolumeSnapshotJobCancelThreshold = new ConfigKey("Advanced", Long.class, "volume.snapshot.job.cancel.threshold", "60", + "Time (in minutes) for volume snapshot async-job to be forcefully cancelled if it has been in process for long", true, ConfigKey.Scope.Global); private static final Logger s_logger = Logger.getLogger(AsyncJobManagerImpl.class); @@ -133,7 +135,7 @@ public String getConfigComponentName() { @Override public ConfigKey[] getConfigKeys() { - return new ConfigKey[] {JobExpireMinutes, JobCancelThresholdMinutes, VmJobLockTimeout}; + return new ConfigKey[] {JobExpireMinutes, JobCancelThresholdMinutes, VmJobLockTimeout, VolumeSnapshotJobCancelThreshold}; } @Override @@ -792,7 +794,8 @@ public void reallyRun() { s_logger.info("Begin cleanup expired async-jobs"); // forcefully cancel blocking queue items if they've been staying there for too long - List blockItems = _queueMgr.getBlockedQueueItems(JobCancelThresholdMinutes.value() * 60000, false); + List blockItems = _queueMgr.getBlockedQueueItems(JobCancelThresholdMinutes.value() * 60000, VolumeSnapshotJobCancelThreshold.value() * 60000, + "VmWorkTakeVolumeSnapshot", false); if (blockItems != null && blockItems.size() > 0) { for (SyncQueueItemVO item : blockItems) { try { diff --git a/framework/jobs/src/org/apache/cloudstack/framework/jobs/impl/SyncQueueManager.java b/framework/jobs/src/org/apache/cloudstack/framework/jobs/impl/SyncQueueManager.java index 32d84647a2d4..f5894696b842 100644 --- a/framework/jobs/src/org/apache/cloudstack/framework/jobs/impl/SyncQueueManager.java +++ b/framework/jobs/src/org/apache/cloudstack/framework/jobs/impl/SyncQueueManager.java @@ -33,7 +33,7 @@ public interface SyncQueueManager extends Manager { public List getActiveQueueItems(Long msid, boolean exclusive); - public List getBlockedQueueItems(long thresholdMs, boolean exclusive); + public List getBlockedQueueItems(long thresholdMs, long snapshotThresholdMs, String jobCmd, boolean exclusive); void purgeAsyncJobQueueItemId(long asyncJobId); diff --git a/framework/jobs/src/org/apache/cloudstack/framework/jobs/impl/SyncQueueManagerImpl.java b/framework/jobs/src/org/apache/cloudstack/framework/jobs/impl/SyncQueueManagerImpl.java index c17c5812f03f..5a56f2874260 100644 --- a/framework/jobs/src/org/apache/cloudstack/framework/jobs/impl/SyncQueueManagerImpl.java +++ b/framework/jobs/src/org/apache/cloudstack/framework/jobs/impl/SyncQueueManagerImpl.java @@ -237,8 +237,8 @@ public List getActiveQueueItems(Long msid, boolean exclusive) { } @Override - public List getBlockedQueueItems(long thresholdMs, boolean exclusive) { - return _syncQueueItemDao.getBlockedQueueItems(thresholdMs, exclusive); + public List getBlockedQueueItems(long thresholdMs, long snapshotThresholdMs, String jobCmd, boolean exclusive) { + return _syncQueueItemDao.getBlockedQueueItems(thresholdMs, snapshotThresholdMs, jobCmd, exclusive); } private boolean queueReadyToProcess(SyncQueueVO queueVO) {