From fc59449d2f3eded8ee6392a52bf5228295589bec Mon Sep 17 00:00:00 2001 From: Collin Brown Date: Fri, 25 Sep 2015 08:14:36 -0700 Subject: [PATCH 001/105] versioning --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 38e1f2a..7076ea0 100644 --- a/pom.xml +++ b/pom.xml @@ -7,7 +7,7 @@ org.activestack syncengine - 1.1.18-httpconnector-SNAPSHOT + 1.1.18-SNAPSHOT 3.2.4.RELEASE From c2c14255af33a072a6825b46f1199da2b8653942 Mon Sep 17 00:00:00 2001 From: Collin Brown Date: Fri, 25 Sep 2015 08:36:49 -0700 Subject: [PATCH 002/105] Gently handle invalid Update Table row types --- .../percero/agents/sync/jobs/UpdateTableProcessor.java | 7 ++++++- .../percero/agents/sync/jobs/UpdateTableRegistry.java | 10 +++++++--- .../percero/agents/sync/jobs/UpdateTableRowType.java | 2 +- 3 files changed, 14 insertions(+), 5 deletions(-) diff --git a/src/main/java/com/percero/agents/sync/jobs/UpdateTableProcessor.java b/src/main/java/com/percero/agents/sync/jobs/UpdateTableProcessor.java index 2be31fb..9d54613 100644 --- a/src/main/java/com/percero/agents/sync/jobs/UpdateTableProcessor.java +++ b/src/main/java/com/percero/agents/sync/jobs/UpdateTableProcessor.java @@ -646,7 +646,12 @@ protected UpdateTableRow fromResultSet(ResultSet resultSet) throws SQLException{ row.rowId = resultSet.getString("row_id"); row.lockId = resultSet.getInt("lock_id"); row.lockDate = resultSet.getDate("lock_date"); - row.type = UpdateTableRowType.valueOf(resultSet.getString("type")); + try { + row.type = UpdateTableRowType.valueOf(resultSet.getString("type")); + } catch(IllegalArgumentException iae) { + logger.warn("Invalid UpdateTableRow TYPE, ignoring"); + row.type = UpdateTableRowType.NONE; + } row.timestamp = resultSet.getDate("time_stamp"); return row; diff --git a/src/main/java/com/percero/agents/sync/jobs/UpdateTableRegistry.java b/src/main/java/com/percero/agents/sync/jobs/UpdateTableRegistry.java index b2ca188..3c90751 100644 --- a/src/main/java/com/percero/agents/sync/jobs/UpdateTableRegistry.java +++ b/src/main/java/com/percero/agents/sync/jobs/UpdateTableRegistry.java @@ -2,12 +2,11 @@ import java.beans.PropertyVetoException; import java.io.File; -import java.io.FileInputStream; import java.io.FileNotFoundException; import java.io.FileReader; +import java.net.URL; import java.util.ArrayList; import java.util.List; -import java.util.Map; import javax.annotation.PostConstruct; @@ -37,7 +36,12 @@ public List getConnectionFactories() { @PostConstruct public void init() throws YamlException { try { - File configFile = new File(UpdateTableRegistry.class.getClassLoader().getResource("updateTables.yml").getFile()); + URL ymlUrl = UpdateTableRegistry.class.getClassLoader().getResource("updateTables.yml"); + if (ymlUrl == null) { + logger.warn("No configuration found for UpdateTables (updateTables.yml), skipping UpdateTables"); + return; + } + File configFile = new File(ymlUrl.getFile()); YamlReader reader = new YamlReader(new FileReader(configFile)); while (true) { UpdateTableConnectionFactory updateTableConnectionFactory = reader.read(UpdateTableConnectionFactory.class); diff --git a/src/main/java/com/percero/agents/sync/jobs/UpdateTableRowType.java b/src/main/java/com/percero/agents/sync/jobs/UpdateTableRowType.java index e2318a8..257b54f 100644 --- a/src/main/java/com/percero/agents/sync/jobs/UpdateTableRowType.java +++ b/src/main/java/com/percero/agents/sync/jobs/UpdateTableRowType.java @@ -4,5 +4,5 @@ * Created by jonnysamps on 9/3/15. */ public enum UpdateTableRowType { - UPDATE, INSERT, DELETE + UPDATE, INSERT, DELETE, NONE } From f9682873da9938421a5a2746e1be4e46547cff71 Mon Sep 17 00:00:00 2001 From: Collin Brown Date: Fri, 25 Sep 2015 10:01:33 -0700 Subject: [PATCH 003/105] Handle no defaultDataProvider --- .../sync/services/DataProviderManager.java | 18 ++++++++++++++++-- .../sync/services/SyncAgentDataProvider.java | 4 ++++ 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/percero/agents/sync/services/DataProviderManager.java b/src/main/java/com/percero/agents/sync/services/DataProviderManager.java index 612d74a..d6877f2 100644 --- a/src/main/java/com/percero/agents/sync/services/DataProviderManager.java +++ b/src/main/java/com/percero/agents/sync/services/DataProviderManager.java @@ -1,6 +1,7 @@ package com.percero.agents.sync.services; import java.util.HashMap; +import java.util.Iterator; import java.util.Map; import org.springframework.beans.factory.annotation.Autowired; @@ -33,6 +34,19 @@ public void setDefaultDataProvider(IDataProvider value) { defaultDataProvider.initialize(); addDataProvider(defaultDataProvider); } + public IDataProvider getDefaultDataProvider() { + if (defaultDataProvider == null) { + // Check to see if there are ANY data providers. + if (dataProvidersByName.size() > 0) { + Iterator itrDataProviders = dataProvidersByName.values().iterator(); + if (itrDataProviders.hasNext()) { + return itrDataProviders.next(); + } + } + } + + return defaultDataProvider; + } private Map dataProvidersByName = new HashMap(); @@ -43,7 +57,7 @@ public void addDataProvider(IDataProvider theDataProvider) { public IDataProvider getDataProviderByName(String aName) { if (!StringUtils.hasText(aName)) - return defaultDataProvider; + return getDefaultDataProvider(); if (!dataProvidersByName.containsKey(aName)) { // Attempt to get the bean from the ApplicationContext. @@ -54,7 +68,7 @@ public IDataProvider getDataProviderByName(String aName) { return dataProvider; } catch(Exception e) { // If no data provider is found, then assume the default. - return defaultDataProvider; + return getDefaultDataProvider(); } } else return dataProvidersByName.get(aName); diff --git a/src/main/java/com/percero/agents/sync/services/SyncAgentDataProvider.java b/src/main/java/com/percero/agents/sync/services/SyncAgentDataProvider.java index 316d351..adf8efa 100644 --- a/src/main/java/com/percero/agents/sync/services/SyncAgentDataProvider.java +++ b/src/main/java/com/percero/agents/sync/services/SyncAgentDataProvider.java @@ -13,6 +13,8 @@ import java.util.UUID; import java.util.concurrent.TimeUnit; +import javax.annotation.PostConstruct; + import javassist.ClassPool; import javassist.CtClass; import javassist.CtField; @@ -68,9 +70,11 @@ public class SyncAgentDataProvider implements IDataProvider { private static final Logger log = Logger.getLogger(SyncAgentDataProvider.class); + @PostConstruct public void initialize() { // Do nothing. + DataProviderManager.getInstance().addDataProvider(this); } public String getName() { From 36e3bd096aa6461a705c15278077c49019540013 Mon Sep 17 00:00:00 2001 From: Collin Brown Date: Tue, 29 Sep 2015 15:06:35 -0700 Subject: [PATCH 004/105] Handle array of objects for run process request when issued to built-in logic connector --- pom.xml | 2 +- .../percero/agents/auth/services/AuthService.java | 12 ++++++++++++ .../agents/sync/services/SyncAgentService.java | 7 ++++++- 3 files changed, 19 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index 7076ea0..b10b5fd 100644 --- a/pom.xml +++ b/pom.xml @@ -7,7 +7,7 @@ org.activestack syncengine - 1.1.18-SNAPSHOT + 1.1.19-SNAPSHOT 3.2.4.RELEASE diff --git a/src/main/java/com/percero/agents/auth/services/AuthService.java b/src/main/java/com/percero/agents/auth/services/AuthService.java index f39caaf..c8bea97 100644 --- a/src/main/java/com/percero/agents/auth/services/AuthService.java +++ b/src/main/java/com/percero/agents/auth/services/AuthService.java @@ -327,6 +327,18 @@ public OAuthResponse setupServiceUser(String authProviderID, ServiceUser service UserAccount theFoundUserAccount = updateUserAccountToken(queryUserAccount, true, serviceUser); if (theFoundUserAccount != null) { + try { + // Validate the user in the project database (by + // checking the IUserAnchor class) + // In the case that the UserAnchor object already exists + // in the project database but is NOT linked to the + // User, we need to link it here. + Object validateUserResult = accountHelper.validateUser(null, theFoundUserAccount.getUser().getID(), this); + System.out.println(validateUserResult); + } catch (Exception e) { + log.warn("Error validating user", e); + } + // Now check Service Application Roles. Boolean foundMatchingRole = validateUserRoles(serviceUser, theFoundUserAccount.getUser().getID()); diff --git a/src/main/java/com/percero/agents/sync/services/SyncAgentService.java b/src/main/java/com/percero/agents/sync/services/SyncAgentService.java index c587a21..91dce80 100644 --- a/src/main/java/com/percero/agents/sync/services/SyncAgentService.java +++ b/src/main/java/com/percero/agents/sync/services/SyncAgentService.java @@ -368,7 +368,12 @@ public Object runProcess(String processName, Object queryArguments, String clien ILogicConnector nextConnectorFactory = itrConnectorFactories.next(); if (nextConnectorFactory.getConnectorPrefix().equalsIgnoreCase(prefix)) { // We have found our Connector Factory. - return nextConnectorFactory.runOperation(operation, clientId, queryArguments); + if (queryArguments instanceof Object[] && ((Object[]) queryArguments).length > 0) { + return nextConnectorFactory.runOperation(operation, clientId, ((Object[]) queryArguments)[0]); + } + else { + return nextConnectorFactory.runOperation(operation, clientId, queryArguments); + } } } } From 415dcf7a3e6c3833b317b8edccc587bd409ad20d Mon Sep 17 00:00:00 2001 From: Jonathan Samples Date: Fri, 2 Oct 2015 07:17:57 -0700 Subject: [PATCH 005/105] Started getting hibernate out of AuthService2 --- .../agents/auth/services/AuthService2.java | 25 +++++++++++++------ .../spring/percero-spring-config.xml | 4 +-- 2 files changed, 20 insertions(+), 9 deletions(-) diff --git a/src/main/java/com/percero/agents/auth/services/AuthService2.java b/src/main/java/com/percero/agents/auth/services/AuthService2.java index 8c99819..09a62a1 100644 --- a/src/main/java/com/percero/agents/auth/services/AuthService2.java +++ b/src/main/java/com/percero/agents/auth/services/AuthService2.java @@ -4,6 +4,8 @@ import com.percero.agents.auth.hibernate.AuthHibernateUtils; import com.percero.agents.auth.hibernate.BaseDataObjectPropertySelector; import com.percero.agents.auth.vo.*; +import com.percero.agents.sync.dao.DAORegistry; +import com.percero.agents.sync.dao.IDataAccessObject; import com.percero.agents.sync.hibernate.SyncHibernateUtils; import com.percero.agents.sync.metadata.*; import com.percero.agents.sync.services.ISyncAgentService; @@ -339,19 +341,28 @@ public void ensureAnchorUserExists(ServiceUser serviceUser, User user){ EntityImplementation userAnchorEI = getUserAnchorEntityImplementation(); MappedClass mc = userAnchorEI.mappedClass; - String userAnchorQueryString = "SELECT ua FROM " + mc.tableName + " ua WHERE ua.userId=:userId"; - Query userAnchorQuery = s.createQuery(userAnchorQueryString); - userAnchorQuery.setString("userId", user.getID()); - - List foundUserAnchors = (List) userAnchorQuery.list(); + DAORegistry daoRegistry = DAORegistry.getInstance(); + IDataAccessObject dao = (IDataAccessObject) daoRegistry.getDataAccessObject(mc.className); +// String userAnchorQueryString = "SELECT ua FROM " + mc.tableName + " ua WHERE ua.userId=:userId"; +// Query userAnchorQuery = s.createQuery(userAnchorQueryString); +// userAnchorQuery.setString("userId", user.getID()); + +// List foundUserAnchors = (List) userAnchorQuery.list(); + List foundUserAnchors = null; + try { + Class anchorUserClass = Class.forName(mc.className); + IUserAnchor example = (IUserAnchor) anchorUserClass.newInstance(); + example.setUserId(user.getID()); + foundUserAnchors = dao.findByExample((IPerceroObject) example, null, null, false); + }catch(Exception e){} - if (foundUserAnchors.size() <= 0) + if (foundUserAnchors != null && foundUserAnchors.size() <= 0) addOrUpdateUserAnchorFromServiceUserList(serviceUser, user, null); else { if(foundUserAnchors.size() > 1) logger.warn("Found more than one IUserAnchor for userId: "+ user.getID()); - result = foundUserAnchors.get(0); // Just take the first one + result = (IUserAnchor) foundUserAnchors.get(0); // Just take the first one handleUserAnchorFound(serviceUser, user, result); } diff --git a/src/main/resources/spring/percero-spring-config.xml b/src/main/resources/spring/percero-spring-config.xml index b75fd88..4c4ebc1 100644 --- a/src/main/resources/spring/percero-spring-config.xml +++ b/src/main/resources/spring/percero-spring-config.xml @@ -220,7 +220,7 @@ hibernate.dialect=org.hibernate.dialect.MySQLDialect - hibernate.hbm2ddl.auto=update + hibernate.show_sql=false hibernate.format_sql=false @@ -299,7 +299,7 @@ hibernate.dialect=org.hibernate.dialect.MySQLDialect - hibernate.hbm2ddl.auto=update + hibernate.show_sql=false hibernate.format_sql=false From 3965be8211e6697d0265f4302b68f67e52e9b39f Mon Sep 17 00:00:00 2001 From: Jonathan Samples Date: Sun, 4 Oct 2015 06:26:24 -0700 Subject: [PATCH 006/105] Changed quite a few of the auth queries to us the DAOs --- .../agents/auth/helpers/AccountHelper.java | 1298 ++++++++--------- .../agents/auth/helpers/RoleHelper.java | 76 + .../agents/auth/helpers/UserAnchorHelper.java | 43 + .../agents/auth/services/AuthService2.java | 45 +- .../agents/sync/metadata/MappedField.java | 15 +- .../spring/percero-spring-config.xml | 7 +- 6 files changed, 755 insertions(+), 729 deletions(-) create mode 100644 src/main/java/com/percero/agents/auth/helpers/RoleHelper.java create mode 100644 src/main/java/com/percero/agents/auth/helpers/UserAnchorHelper.java diff --git a/src/main/java/com/percero/agents/auth/helpers/AccountHelper.java b/src/main/java/com/percero/agents/auth/helpers/AccountHelper.java index bf1cfb7..81984fe 100644 --- a/src/main/java/com/percero/agents/auth/helpers/AccountHelper.java +++ b/src/main/java/com/percero/agents/auth/helpers/AccountHelper.java @@ -1,291 +1,281 @@ package com.percero.agents.auth.helpers; -import java.lang.reflect.InvocationTargetException; -import java.security.Principal; -import java.util.ArrayList; -import java.util.Date; -import java.util.HashSet; -import java.util.Iterator; -import java.util.List; -import java.util.Set; -import java.util.UUID; - -import org.apache.log4j.Logger; -import org.hibernate.HibernateException; -import org.hibernate.NonUniqueResultException; -import org.hibernate.Query; -import org.hibernate.Session; -import org.hibernate.SessionFactory; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Component; -import org.springframework.util.StringUtils; - import com.percero.agents.auth.principal.PrincipalUser; import com.percero.agents.auth.services.IAuthService; -import com.percero.agents.auth.vo.IUserAnchor; -import com.percero.agents.auth.vo.IUserIdentifier; -import com.percero.agents.auth.vo.IUserRole; -import com.percero.agents.auth.vo.ServiceIdentifier; -import com.percero.agents.auth.vo.ServiceUser; +import com.percero.agents.auth.vo.*; import com.percero.agents.sync.access.IAccessManager; +import com.percero.agents.sync.dao.DAORegistry; +import com.percero.agents.sync.dao.IDataAccessObject; import com.percero.agents.sync.helpers.PostCreateHelper; import com.percero.agents.sync.helpers.PostDeleteHelper; import com.percero.agents.sync.helpers.PostPutHelper; import com.percero.agents.sync.hibernate.SyncHibernateUtils; -import com.percero.agents.sync.metadata.EntityImplementation; -import com.percero.agents.sync.metadata.IMappedClassManager; -import com.percero.agents.sync.metadata.MappedClass; -import com.percero.agents.sync.metadata.MappedClassManagerFactory; -import com.percero.agents.sync.metadata.PropertyImplementation; -import com.percero.agents.sync.metadata.PropertyImplementationParam; -import com.percero.agents.sync.metadata.RelationshipImplementation; +import com.percero.agents.sync.metadata.*; import com.percero.agents.sync.services.IDataProviderManager; import com.percero.agents.sync.services.ISyncAgentService; import com.percero.agents.sync.vo.Client; import com.percero.framework.bl.IManifest; import com.percero.framework.bl.ManifestHelper; import com.percero.framework.vo.IPerceroObject; +import org.apache.log4j.Logger; +import org.hibernate.*; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; +import org.springframework.util.StringUtils; + +import java.lang.reflect.InvocationTargetException; +import java.security.Principal; +import java.util.*; @Component public class AccountHelper implements IAccountHelper { - private static final Logger log = Logger.getLogger(AccountHelper.class); - - public AccountHelper() { - } - - @Autowired - protected IManifest manifest = null; - public void setManifest(IManifest value) { - manifest = value; - } - - @Autowired - protected IAuthService authService; - public void setAuthService(IAuthService value) { - authService = value; - } - - @Autowired - protected IAccessManager accessManager; - public void setAccessManager(IAccessManager value) { - accessManager = value; - } - - @Autowired - protected SessionFactory appSessionFactory; - public void setAppSessionFactory(SessionFactory value) { - appSessionFactory = value; - } - - @Autowired - protected ISyncAgentService syncAgentService; - public void setSyncAgentService(ISyncAgentService value) { - syncAgentService = value; - } - - @Autowired - protected PostPutHelper postPutHelper; - public void setPostPutHelper(PostPutHelper value) { - postPutHelper = value; - } - - @Autowired - protected IDataProviderManager dataProviderManager; - public void setDataProviderManager(IDataProviderManager value) { - dataProviderManager = value; - } - - @Autowired - protected PostCreateHelper postCreateHelper; - public void setPostCreateHelper(PostCreateHelper value) { - postCreateHelper = value; - } - @Autowired - protected PostDeleteHelper postDeleteHelper; - public void setPostDeleteHelper(PostDeleteHelper value) { - postDeleteHelper = value; - } - - public Principal authenticateOAuth(String regAppKey, String svcOAuthKey, String userId, String userToken, String clientId, String clientType, String deviceId) { - return authenticateOAuth(regAppKey, svcOAuthKey, userId, userToken, clientId, clientType, deviceId, null); - } - - public Principal authenticateOAuth(String regAppKey, String svcOAuthKey, String userId, String userToken, String clientId, String clientType, String deviceId, Set existingClientIds) { - if (!StringUtils.hasText(clientType)) { - clientType = Client.NON_PERSISTENT_TYPE; - } - - // Only worry about existing client if it is set and NOT equal to the "new" client Id. - Boolean isExistingClient = ( existingClientIds != null && !existingClientIds.isEmpty() ); - boolean validated = false; - try { - if (isExistingClient) { - validated = authService.validateUserByToken(regAppKey, userId, userToken, clientId, existingClientIds); - } else { - validated = authService.validateUserByToken(regAppKey, userId, userToken, clientId); - } - log.debug("ValidateUserByToken Result: " + userId + " / " + userToken + " / " + clientId + " = " + (validated ? "true" : "false")); - } catch (Exception e) { - log.error("Error validating User by Token", e); - e.printStackTrace(); - validated = false; - } - - if (validated) { - try { - // Make sure a valid Person exists for this User. - IUserAnchor userAnchor = validateUser(regAppKey, userId, authService); - - if (userAnchor == null) { - // Something went wrong here. - throw new Error("Invalid UserAnchor object."); - } - - List roleList = getUserRoles(userId); - - String[] groups = (String[]) roleList.toArray(new String[0]); - PrincipalUser theUser = new PrincipalUser(userId, userId, userToken, clientType, clientId, groups); - - Boolean foundValidClient = false; - if (StringUtils.hasText(deviceId)) - { - if (isExistingClient) { - Iterator itrExistingClientIds = existingClientIds.iterator(); - while (itrExistingClientIds.hasNext()) { - String existingClientId = itrExistingClientIds.next(); - if (!existingClientId.equals(clientId)) { - // Need to move the existing client to the new client. - log.debug("Renaming client " + existingClientId + " to " + clientId); - accessManager.renameClient(existingClientId, clientId); - } - } - } - } - - if (!foundValidClient) - foundValidClient = accessManager.findClientByClientIdUserId(clientId, userId); - - if (!foundValidClient) { - // Unable to find a valid client, so need to create one. - accessManager.createClient(clientId, userId, clientType, deviceId); - } - - return theUser; - } catch(Exception e) { - log.error("Error getting Roles", e); - e.printStackTrace(); - } - } - - log.debug("authenticateOAuth: Returning null result"); - return null; - } - - public IUserAnchor validateUser(String regAppKey, String userId, IAuthService authService) throws HibernateException { - IUserAnchor result = null; - Session s = appSessionFactory.openSession(); - try { - ManifestHelper.setManifest(manifest); - //Class userAnchorClass = ManifestHelper.findImplementingClass(IUserAnchor.class); - EntityImplementation userAnchorEI = null; - List userAnchorMappedClasses = MappedClass.findEntityImplementation(IUserAnchor.class); - if (userAnchorMappedClasses.size() > 0) { - userAnchorEI = userAnchorMappedClasses.get(0); - } - - if (userAnchorEI != null) { - //IMappedClassManager mcm = MappedClassManagerFactory.getMappedClassManager(); - MappedClass mc = userAnchorEI.mappedClass; - if (mc != null) { - String userAnchorQueryString = "SELECT ua FROM " + mc.tableName + " ua WHERE ua.userId=:userId"; - Query userAnchorQuery = s.createQuery(userAnchorQueryString); - userAnchorQuery.setString("userId", userId); - - IUserAnchor foundUserAnchor = (IUserAnchor) userAnchorQuery.uniqueResult(); - - if (foundUserAnchor == null) { - result = handleUserAnchorNotFound(regAppKey, userId, authService); - } else { - handleUserAnchorFound(regAppKey, userId, authService, foundUserAnchor); - result = foundUserAnchor; - } - } - } - } catch (NonUniqueResultException nre) { - log.error("More than one UserAnchor objects found for userId " + userId, nre); - } finally { - if (s != null && s.isOpen()) - s.close(); - } - return result; - } - - protected IUserAnchor handleUserAnchorNotFound(String regAppKey, String userId, IAuthService authService) { - // Get this person's email addresses from the AuthManager. - List serviceUserList = authService.getServiceUsers(userId); - IUserAnchor result = addOrUpdateUserAnchorFromServiceUserList(userId, serviceUserList, null); - //setupUserRoles(userId, serviceUserList); - - return result; - } - - protected void handleUserAnchorFound(String regAppKey, String userId, IAuthService authService, IUserAnchor userAnchor) { - // Get this person's email addresses from the AuthManager. - List serviceUserList = authService.getServiceUsers(userId); - - if (userAnchor != null && (userAnchor instanceof IPerceroObject)) { - try { - // Attempt to get updated information from ServiceProvider. - EntityImplementation userAnchorEI = null; - List userAnchorMappedClasses = MappedClass.findEntityImplementation(IUserAnchor.class); - if (userAnchorMappedClasses.size() > 0) { - userAnchorEI = userAnchorMappedClasses.get(0); - } - - PropertyImplementation firstNamePropImpl = userAnchorEI.findPropertyImplementationByName(IUserAnchor.FIRST_NAME_FIELD); - PropertyImplementation lastNamePropImpl = userAnchorEI.findPropertyImplementationByName(IUserAnchor.LAST_NAME_FIELD); - //MappedField firstNameField = userAnchorEI.mappedClass.getExternalizeFieldByName("firstName"); - //MappedField lastNameField = userAnchorappedClass.getExternalizeFieldByName("lastName"); - String firstName = ""; - String lastName = ""; - - if (firstNamePropImpl != null || lastNamePropImpl != null) { - if (firstNamePropImpl != null) - firstName = (String) firstNamePropImpl.mappedField.getGetter().invoke(userAnchor); - if (lastNamePropImpl != null) - lastName = (String) lastNamePropImpl.mappedField.getGetter().invoke(userAnchor); - - boolean userAnchorUpdated = false; - for (ServiceUser nextServiceUser : serviceUserList) { - if (StringUtils.hasText(nextServiceUser.getFirstName()) && firstNamePropImpl != null) { - if (firstName == null || !firstName.equals(nextServiceUser.getFirstName())) { - firstName = nextServiceUser.getFirstName(); - firstNamePropImpl.mappedField.getSetter().invoke(userAnchor, firstName); - userAnchorUpdated = true; - } - } - if (StringUtils.hasText(nextServiceUser.getLastName()) && lastNamePropImpl != null) { - if (lastName == null || !lastName.equals(nextServiceUser.getLastName())) { - lastName = nextServiceUser.getLastName(); - lastNamePropImpl.mappedField.getSetter().invoke(userAnchor, lastName); - userAnchorUpdated = true; - } - } - } - - if (userAnchorUpdated) { - syncAgentService.systemPutObject((IPerceroObject) userAnchor, null, new Date(), null, true); - } - } - } catch(Exception e) { - log.warn("Unable to get ServiceUser information for First/Last Name", e); - } - } - - //setupUserRoles(userId, serviceUserList); - addOrUpdateUserAnchorFromServiceUserList(userId, serviceUserList, userAnchor); - } + private static final Logger log = Logger.getLogger(AccountHelper.class); + + public AccountHelper() { + } + + @Autowired + protected IManifest manifest = null; + public void setManifest(IManifest value) { + manifest = value; + } + + @Autowired + protected IAuthService authService; + public void setAuthService(IAuthService value) { + authService = value; + } + + @Autowired + protected IAccessManager accessManager; + public void setAccessManager(IAccessManager value) { + accessManager = value; + } + + @Autowired + protected SessionFactory appSessionFactory; + public void setAppSessionFactory(SessionFactory value) { + appSessionFactory = value; + } + + @Autowired + protected ISyncAgentService syncAgentService; + public void setSyncAgentService(ISyncAgentService value) { + syncAgentService = value; + } + + @Autowired + protected PostPutHelper postPutHelper; + public void setPostPutHelper(PostPutHelper value) { + postPutHelper = value; + } + + @Autowired + protected IDataProviderManager dataProviderManager; + public void setDataProviderManager(IDataProviderManager value) { + dataProviderManager = value; + } + + @Autowired + protected PostCreateHelper postCreateHelper; + public void setPostCreateHelper(PostCreateHelper value) { + postCreateHelper = value; + } + @Autowired + protected PostDeleteHelper postDeleteHelper; + public void setPostDeleteHelper(PostDeleteHelper value) { + postDeleteHelper = value; + } + + public Principal authenticateOAuth(String regAppKey, String svcOAuthKey, String userId, String userToken, String clientId, String clientType, String deviceId) { + return authenticateOAuth(regAppKey, svcOAuthKey, userId, userToken, clientId, clientType, deviceId, null); + } + + public Principal authenticateOAuth(String regAppKey, String svcOAuthKey, String userId, String userToken, String clientId, String clientType, String deviceId, Set existingClientIds) { + if (!StringUtils.hasText(clientType)) { + clientType = Client.NON_PERSISTENT_TYPE; + } + + // Only worry about existing client if it is set and NOT equal to the "new" client Id. + Boolean isExistingClient = ( existingClientIds != null && !existingClientIds.isEmpty() ); + boolean validated = false; + try { + if (isExistingClient) { + validated = authService.validateUserByToken(regAppKey, userId, userToken, clientId, existingClientIds); + } else { + validated = authService.validateUserByToken(regAppKey, userId, userToken, clientId); + } + log.debug("ValidateUserByToken Result: " + userId + " / " + userToken + " / " + clientId + " = " + (validated ? "true" : "false")); + } catch (Exception e) { + log.error("Error validating User by Token", e); + e.printStackTrace(); + validated = false; + } + + if (validated) { + try { + // Make sure a valid Person exists for this User. + IUserAnchor userAnchor = validateUser(regAppKey, userId, authService); + + if (userAnchor == null) { + // Something went wrong here. + throw new Error("Invalid UserAnchor object."); + } + + List roleList = getUserRoles(userId); + + String[] groups = (String[]) roleList.toArray(new String[0]); + PrincipalUser theUser = new PrincipalUser(userId, userId, userToken, clientType, clientId, groups); + + Boolean foundValidClient = false; + if (StringUtils.hasText(deviceId)) + { + if (isExistingClient) { + Iterator itrExistingClientIds = existingClientIds.iterator(); + while (itrExistingClientIds.hasNext()) { + String existingClientId = itrExistingClientIds.next(); + if (!existingClientId.equals(clientId)) { + // Need to move the existing client to the new client. + log.debug("Renaming client " + existingClientId + " to " + clientId); + accessManager.renameClient(existingClientId, clientId); + } + } + } + } + + if (!foundValidClient) + foundValidClient = accessManager.findClientByClientIdUserId(clientId, userId); + + if (!foundValidClient) { + // Unable to find a valid client, so need to create one. + accessManager.createClient(clientId, userId, clientType, deviceId); + } + + return theUser; + } catch(Exception e) { + log.error("Error getting Roles", e); + e.printStackTrace(); + } + } + + log.debug("authenticateOAuth: Returning null result"); + return null; + } + + public IUserAnchor validateUser(String regAppKey, String userId, IAuthService authService) throws HibernateException { + IUserAnchor result = null; + Session s = appSessionFactory.openSession(); + try { + ManifestHelper.setManifest(manifest); + //Class userAnchorClass = ManifestHelper.findImplementingClass(IUserAnchor.class); + EntityImplementation userAnchorEI = null; + List userAnchorMappedClasses = MappedClass.findEntityImplementation(IUserAnchor.class); + if (userAnchorMappedClasses.size() > 0) { + userAnchorEI = userAnchorMappedClasses.get(0); + } + + if (userAnchorEI != null) { + //IMappedClassManager mcm = MappedClassManagerFactory.getMappedClassManager(); + MappedClass mc = userAnchorEI.mappedClass; + if (mc != null) { + IUserAnchor foundUserAnchor = null; + try { + IDataAccessObject dao = (IDataAccessObject) DAORegistry.getInstance().getDataAccessObject(mc.className); + Class userAnchorClass = Class.forName(mc.className); + IUserAnchor example = (IUserAnchor) userAnchorClass.newInstance(); + example.setUserId(userId); + List list = dao.findByExample((IPerceroObject)example, null, null, false); + if(list.size() > 0) + foundUserAnchor = (IUserAnchor) list.get(0); +// String userAnchorQueryString = "SELECT ua FROM " + mc.tableName + " ua WHERE ua.userId=:userId"; +// Query userAnchorQuery = s.createQuery(userAnchorQueryString); +// userAnchorQuery.setString("userId", userId); +// IUserAnchor foundUserAnchor = (IUserAnchor) userAnchorQuery.uniqueResult(); + }catch(Exception e){} + + if (foundUserAnchor == null) { + result = handleUserAnchorNotFound(regAppKey, userId, authService); + } else { + handleUserAnchorFound(regAppKey, userId, authService, foundUserAnchor); + result = foundUserAnchor; + } + } + } + } catch (NonUniqueResultException nre) { + log.error("More than one UserAnchor objects found for userId " + userId, nre); + } finally { + if (s != null && s.isOpen()) + s.close(); + } + return result; + } + + protected IUserAnchor handleUserAnchorNotFound(String regAppKey, String userId, IAuthService authService) { + // Get this person's email addresses from the AuthManager. + List serviceUserList = authService.getServiceUsers(userId); + IUserAnchor result = addOrUpdateUserAnchorFromServiceUserList(userId, serviceUserList, null); + //setupUserRoles(userId, serviceUserList); + + return result; + } + + protected void handleUserAnchorFound(String regAppKey, String userId, IAuthService authService, IUserAnchor userAnchor) { + // Get this person's email addresses from the AuthManager. + List serviceUserList = authService.getServiceUsers(userId); + + if (userAnchor != null && (userAnchor instanceof IPerceroObject)) { + try { + // Attempt to get updated information from ServiceProvider. + EntityImplementation userAnchorEI = null; + List userAnchorMappedClasses = MappedClass.findEntityImplementation(IUserAnchor.class); + if (userAnchorMappedClasses.size() > 0) { + userAnchorEI = userAnchorMappedClasses.get(0); + } + + PropertyImplementation firstNamePropImpl = userAnchorEI.findPropertyImplementationByName(IUserAnchor.FIRST_NAME_FIELD); + PropertyImplementation lastNamePropImpl = userAnchorEI.findPropertyImplementationByName(IUserAnchor.LAST_NAME_FIELD); + //MappedField firstNameField = userAnchorEI.mappedClass.getExternalizeFieldByName("firstName"); + //MappedField lastNameField = userAnchorappedClass.getExternalizeFieldByName("lastName"); + String firstName = ""; + String lastName = ""; + + if (firstNamePropImpl != null || lastNamePropImpl != null) { + if (firstNamePropImpl != null) + firstName = (String) firstNamePropImpl.mappedField.getGetter().invoke(userAnchor); + if (lastNamePropImpl != null) + lastName = (String) lastNamePropImpl.mappedField.getGetter().invoke(userAnchor); + + boolean userAnchorUpdated = false; + for (ServiceUser nextServiceUser : serviceUserList) { + if (StringUtils.hasText(nextServiceUser.getFirstName()) && firstNamePropImpl != null) { + if (firstName == null || !firstName.equals(nextServiceUser.getFirstName())) { + firstName = nextServiceUser.getFirstName(); + firstNamePropImpl.mappedField.getSetter().invoke(userAnchor, firstName); + userAnchorUpdated = true; + } + } + if (StringUtils.hasText(nextServiceUser.getLastName()) && lastNamePropImpl != null) { + if (lastName == null || !lastName.equals(nextServiceUser.getLastName())) { + lastName = nextServiceUser.getLastName(); + lastNamePropImpl.mappedField.getSetter().invoke(userAnchor, lastName); + userAnchorUpdated = true; + } + } + } + + if (userAnchorUpdated) { + syncAgentService.systemPutObject((IPerceroObject) userAnchor, null, new Date(), null, true); + } + } + } catch(Exception e) { + log.warn("Unable to get ServiceUser information for First/Last Name", e); + } + } + + //setupUserRoles(userId, serviceUserList); + addOrUpdateUserAnchorFromServiceUserList(userId, serviceUserList, userAnchor); + } // public Object validateServiceUser(String regAppKey, String svcOauthKey, ServiceUser theServiceUser, IAuthService theAuthService) throws Exception { // try { @@ -311,224 +301,184 @@ protected void handleUserAnchorFound(String regAppKey, String userId, IAuthServi // return Boolean.FALSE; // } - /* (non-Javadoc) - * @see com.com.percero.agents.auth.helpers.IAccountHelper#getUserRoles(java.lang.String) - */ - @SuppressWarnings({ "rawtypes", "unchecked" }) - public List getUserRoles(String userId) throws Exception { - Session s = appSessionFactory.openSession(); - - try { - ManifestHelper.setManifest(manifest); - Class userAnchorClass = ManifestHelper.findImplementingClass(IUserAnchor.class); - Class userRoleClass = ManifestHelper.findImplementingClass(IUserRole.class); - - if (userAnchorClass != null && userRoleClass != null) { - EntityImplementation userAnchorEI = null; - List userAnchorMappedClasses = MappedClass.findEntityImplementation(IUserAnchor.class); - if (userAnchorMappedClasses.size() > 0) { - userAnchorEI = userAnchorMappedClasses.get(0); - } - EntityImplementation userRoleEI = null; - List userRoleMappedClasses = MappedClass.findEntityImplementation(IUserRole.class); - if (userRoleMappedClasses.size() > 0) { - userRoleEI = userRoleMappedClasses.get(0); - } - - RelationshipImplementation userAnchorRelImpl = userRoleEI.findRelationshipImplementationBySourceVarName(IUserRole.USER_ANCHOR_FIELD_NAME); - - if (userAnchorEI != null && userRoleEI != null) { - String userRoleEiQueryString = "SELECT ur.roleName FROM " + userRoleEI.mappedClass.tableName - + " ur WHERE ur." + userAnchorRelImpl.sourceMappedField.getField().getName() + " IN (SELECT ua FROM " + userAnchorEI.mappedClass.tableName + " ua WHERE ua.userId=:userId)"; - Query userRoleEiQuery = s.createQuery(userRoleEiQueryString); - userRoleEiQuery.setString("userId", userId); - - List result = (List) userRoleEiQuery.list(); - return result; - } - else { - if (userAnchorEI == null) { - log.debug("No User Anchor Entity Implementation Class found!"); - } - if (userRoleEI == null) { - log.debug("No User Role Entity Implementation Class found!"); - } - } - } - } catch (Exception e) { - log.error("Unable to Get Person Roles", e); - } finally { - s.close(); - } - - return new ArrayList(); - } - - /** - @SuppressWarnings({ "rawtypes", "unchecked" }) - protected IUserRoleA getUserRoleAnnotation(Class userRoleClass) { - IUserRoleA userRoleAnnotation = null; - Class nextClazz = userRoleClass; - while(userRoleAnnotation == null && nextClazz != null) { - userRoleAnnotation = (IUserRoleA) nextClazz.getAnnotation(IUserRoleA.class); - if (userRoleAnnotation == null) - nextClazz = nextClazz.getSuperclass(); - } - return userRoleAnnotation; - } - - @SuppressWarnings({ "rawtypes", "unchecked" }) - protected IUserIdentifierA getUserIdentifierAnnotation(Class userIdentifierClass) { - IUserIdentifierA userIdentifierAnnotation = null; - Class nextClazz = userIdentifierClass; - while(userIdentifierAnnotation == null && nextClazz != null) { - userIdentifierAnnotation = (IUserIdentifierA) nextClazz.getAnnotation(IUserIdentifierA.class); - if (userIdentifierAnnotation == null) - nextClazz = nextClazz.getSuperclass(); - } - return userIdentifierAnnotation; - } - - @SuppressWarnings({ "rawtypes", "unchecked" }) - protected IUserAnchorA getUserAnchorAnnotation(Class userAnchorClass) { - IUserAnchorA userAnchorAnnotation = null; - Class nextClazz = userAnchorClass; - while(userAnchorAnnotation == null && nextClazz != null) { - userAnchorAnnotation = (IUserAnchorA) nextClazz.getAnnotation(IUserAnchorA.class); - if (userAnchorAnnotation == null) - nextClazz = nextClazz.getSuperclass(); - } - return userAnchorAnnotation; - } - - @SuppressWarnings({ "rawtypes" }) - protected Field getUserAnchorField(Class entityInterfaceClass, String fieldName) { - Class nextClazz = entityInterfaceClass; - while(nextClazz != null) { - List fields = SyncHibernateUtils.getClassFields(nextClazz); - Iterator itrFields = fields.iterator(); - while (itrFields.hasNext()) { - Field nextField = itrFields.next(); - PropertyInterfaces nextEntityInterfacePropertiesA = nextField.getAnnotation(PropertyInterfaces.class); - - if (nextEntityInterfacePropertiesA != null && nextEntityInterfacePropertiesA.entityInterfaceClass() == entityInterfaceClass) { - PropertyInterface[] propInterfaces = nextEntityInterfacePropertiesA.propertyInterfaces(); - for(PropertyInterface nextPropInterface : propInterfaces) { - if (nextPropInterface.propertyName() != null && nextPropInterface.propertyName().equals(fieldName)) { - // We have found the appropriate field. - return nextField; - } - } - } - } - nextClazz = nextClazz.getSuperclass(); - } - - return null; - } - - @SuppressWarnings({ "rawtypes" }) - protected PropertyInterface getEntityPropertyInterfaceAnnotation(Class entityInterfaceClass, String fieldName) { - Class nextClazz = entityInterfaceClass; - while(nextClazz != null) { - List fields = SyncHibernateUtils.getClassFields(nextClazz); - Iterator itrFields = fields.iterator(); - while (itrFields.hasNext()) { - Field nextField = itrFields.next(); - PropertyInterfaces nextEntityInterfacePropertiesA = nextField.getAnnotation(PropertyInterfaces.class); - - if (nextEntityInterfacePropertiesA != null && nextEntityInterfacePropertiesA.entityInterfaceClass() == entityInterfaceClass) { - PropertyInterface[] propInterfaces = nextEntityInterfacePropertiesA.propertyInterfaces(); - for(PropertyInterface nextPropInterface : propInterfaces) { - if (nextPropInterface.propertyName() != null && nextPropInterface.propertyName().equals(fieldName)) { - // We have found the appropriate field. - return nextPropInterface; - } - } - } - } - nextClazz = nextClazz.getSuperclass(); - } - - return null; - }*/ - - protected IUserAnchor addOrUpdateUserAnchorFromServiceUserList(String userId, List serviceUserList, IUserAnchor result) { - Session s = null; - try { - ManifestHelper.setManifest(manifest); - //Set userIdentifierClasses = ManifestHelper.findImplementingClasses(IUserIdentifier.class); - //Class userAnchorClass = ManifestHelper.findImplementingClass(IUserAnchor.class); - EntityImplementation eiUserAnchor = null; - List userAnchorEntityImplementations = MappedClass.findEntityImplementation(IUserAnchor.class); - if (userAnchorEntityImplementations.size() > 0) { - eiUserAnchor = userAnchorEntityImplementations.get(0); - } - List userIdentifierEntityImplementations = MappedClass.findEntityImplementation(IUserIdentifier.class); - - if (userIdentifierEntityImplementations.size() > 0) { - List identifiersToSave = new ArrayList(); - - Iterator itrUserIdentifierEntityImplementations = userIdentifierEntityImplementations.iterator(); - while (itrUserIdentifierEntityImplementations.hasNext()) { - EntityImplementation userIdentifierEntityImplementation = itrUserIdentifierEntityImplementations.next(); - - //MappedField userAnchorMappedField = userIdentifierEntityImplementation.getMappedFieldByName(userIdentifierAnnotation.userAnchorFieldName()); - PropertyImplementation userIdentifierPropImpl = userIdentifierEntityImplementation.findPropertyImplementationByName(IUserIdentifier.USER_IDENTIFIER_FIELD_NAME); - RelationshipImplementation userAnchorRelImpl = userIdentifierEntityImplementation.findRelationshipImplementationBySourceVarName(IUserIdentifier.USER_ANCHOR_FIELD_NAME); - //IUserIdentifierA userIdentifierAnnotation = getUserIdentifierAnnotation(userIdentifierEntityImplementation); - if (userAnchorRelImpl != null) { - - if (s == null) { - s = appSessionFactory.openSession(); - } - - // Get this userAnchor's identifier(s). - String userIdentifierQueryString = "SELECT ui FROM " + userIdentifierEntityImplementation.mappedClass.tableName - + " ui WHERE ui." + userIdentifierPropImpl.mappedField.getField().getName() + "=:value AND (ui." + - userAnchorRelImpl.sourceMappedField.getField().getName() + " IS NULL OR ui." + userAnchorRelImpl.sourceMappedField.getField().getName() + - " IN (SELECT ua FROM " + eiUserAnchor.mappedClass.tableName + " ua WHERE (ua.userId=null OR ua.userId='' OR ua.userId=:userId)))"; - for (ServiceUser nextServiceUser : serviceUserList) { - for (ServiceIdentifier nextIdentifier : nextServiceUser.getIdentifiers()) { - try { - // Make sure this Identifier is in the same paradigm. - String paradigm = null; - Iterator itrParams = userIdentifierPropImpl.params.iterator(); - while (itrParams.hasNext()) { - PropertyImplementationParam nextParam = itrParams.next(); - if (nextParam.name.equalsIgnoreCase(IUserIdentifier.PARADIGM_PARAM_NAME)) { - paradigm = nextParam.value; - break; - } - } - if (nextIdentifier.getParadigm() == null || !nextIdentifier.getParadigm().equalsIgnoreCase(paradigm)) { - continue; - } - - // Look for this existing identifier. - Query userIdentifierQuery = s.createQuery(userIdentifierQueryString); - userIdentifierQuery.setString("userId", userId); - userIdentifierQuery.setString("value", nextIdentifier.getValue()); - IUserIdentifier foundIdentifier = (IUserIdentifier) userIdentifierQuery.uniqueResult(); - - if (foundIdentifier != null) { - // If the email does not have a Person, then associate email with this Person. - foundIdentifier = (IUserIdentifier) SyncHibernateUtils.cleanObject(foundIdentifier, s); - IUserAnchor existingUserAnchor = (IUserAnchor) SyncHibernateUtils.cleanObject(userAnchorRelImpl.sourceMappedField.getGetter().invoke(foundIdentifier), s); - if (existingUserAnchor == null) { - if (result != null) { - userAnchorRelImpl.sourceMappedField.getSetter().invoke(foundIdentifier, result); - syncAgentService.systemPutObject((IPerceroObject) foundIdentifier, null, null, null, true); - } - else { - identifiersToSave.add(foundIdentifier); - } - } else if (result == null) { - result = existingUserAnchor; - } - } else { // Identifier object NOT found, need to add. - // Identifier.identifierValue will be set later, once that is determined. - IUserIdentifier newIdentifier = (IUserIdentifier) userIdentifierEntityImplementation.mappedClass.clazz.newInstance(); + /* (non-Javadoc) + * @see com.com.percero.agents.auth.helpers.IAccountHelper#getUserRoles(java.lang.String) + */ + @SuppressWarnings({ "rawtypes", "unchecked" }) + public List getUserRoles(String userId) throws Exception { + + ManifestHelper.setManifest(manifest); + IUserAnchor userAnchor = UserAnchorHelper.getUserAnchor(userId); + List result = RoleHelper.getUserRoleNames(userAnchor); + + return result; + } + + /** + @SuppressWarnings({ "rawtypes", "unchecked" }) + protected IUserRoleA getUserRoleAnnotation(Class userRoleClass) { + IUserRoleA userRoleAnnotation = null; + Class nextClazz = userRoleClass; + while(userRoleAnnotation == null && nextClazz != null) { + userRoleAnnotation = (IUserRoleA) nextClazz.getAnnotation(IUserRoleA.class); + if (userRoleAnnotation == null) + nextClazz = nextClazz.getSuperclass(); + } + return userRoleAnnotation; + } + + @SuppressWarnings({ "rawtypes", "unchecked" }) + protected IUserIdentifierA getUserIdentifierAnnotation(Class userIdentifierClass) { + IUserIdentifierA userIdentifierAnnotation = null; + Class nextClazz = userIdentifierClass; + while(userIdentifierAnnotation == null && nextClazz != null) { + userIdentifierAnnotation = (IUserIdentifierA) nextClazz.getAnnotation(IUserIdentifierA.class); + if (userIdentifierAnnotation == null) + nextClazz = nextClazz.getSuperclass(); + } + return userIdentifierAnnotation; + } + + @SuppressWarnings({ "rawtypes", "unchecked" }) + protected IUserAnchorA getUserAnchorAnnotation(Class userAnchorClass) { + IUserAnchorA userAnchorAnnotation = null; + Class nextClazz = userAnchorClass; + while(userAnchorAnnotation == null && nextClazz != null) { + userAnchorAnnotation = (IUserAnchorA) nextClazz.getAnnotation(IUserAnchorA.class); + if (userAnchorAnnotation == null) + nextClazz = nextClazz.getSuperclass(); + } + return userAnchorAnnotation; + } + + @SuppressWarnings({ "rawtypes" }) + protected Field getUserAnchorField(Class entityInterfaceClass, String fieldName) { + Class nextClazz = entityInterfaceClass; + while(nextClazz != null) { + List fields = SyncHibernateUtils.getClassFields(nextClazz); + Iterator itrFields = fields.iterator(); + while (itrFields.hasNext()) { + Field nextField = itrFields.next(); + PropertyInterfaces nextEntityInterfacePropertiesA = nextField.getAnnotation(PropertyInterfaces.class); + + if (nextEntityInterfacePropertiesA != null && nextEntityInterfacePropertiesA.entityInterfaceClass() == entityInterfaceClass) { + PropertyInterface[] propInterfaces = nextEntityInterfacePropertiesA.propertyInterfaces(); + for(PropertyInterface nextPropInterface : propInterfaces) { + if (nextPropInterface.propertyName() != null && nextPropInterface.propertyName().equals(fieldName)) { + // We have found the appropriate field. + return nextField; + } + } + } + } + nextClazz = nextClazz.getSuperclass(); + } + + return null; + } + + @SuppressWarnings({ "rawtypes" }) + protected PropertyInterface getEntityPropertyInterfaceAnnotation(Class entityInterfaceClass, String fieldName) { + Class nextClazz = entityInterfaceClass; + while(nextClazz != null) { + List fields = SyncHibernateUtils.getClassFields(nextClazz); + Iterator itrFields = fields.iterator(); + while (itrFields.hasNext()) { + Field nextField = itrFields.next(); + PropertyInterfaces nextEntityInterfacePropertiesA = nextField.getAnnotation(PropertyInterfaces.class); + + if (nextEntityInterfacePropertiesA != null && nextEntityInterfacePropertiesA.entityInterfaceClass() == entityInterfaceClass) { + PropertyInterface[] propInterfaces = nextEntityInterfacePropertiesA.propertyInterfaces(); + for(PropertyInterface nextPropInterface : propInterfaces) { + if (nextPropInterface.propertyName() != null && nextPropInterface.propertyName().equals(fieldName)) { + // We have found the appropriate field. + return nextPropInterface; + } + } + } + } + nextClazz = nextClazz.getSuperclass(); + } + + return null; + }*/ + + protected IUserAnchor addOrUpdateUserAnchorFromServiceUserList(String userId, List serviceUserList, IUserAnchor result) { + Session s = null; + try { + ManifestHelper.setManifest(manifest); + //Set userIdentifierClasses = ManifestHelper.findImplementingClasses(IUserIdentifier.class); + //Class userAnchorClass = ManifestHelper.findImplementingClass(IUserAnchor.class); + EntityImplementation eiUserAnchor = null; + List userAnchorEntityImplementations = MappedClass.findEntityImplementation(IUserAnchor.class); + if (userAnchorEntityImplementations.size() > 0) { + eiUserAnchor = userAnchorEntityImplementations.get(0); + } + List userIdentifierEntityImplementations = MappedClass.findEntityImplementation(IUserIdentifier.class); + + if (userIdentifierEntityImplementations.size() > 0) { + List identifiersToSave = new ArrayList(); + + Iterator itrUserIdentifierEntityImplementations = userIdentifierEntityImplementations.iterator(); + while (itrUserIdentifierEntityImplementations.hasNext()) { + EntityImplementation userIdentifierEntityImplementation = itrUserIdentifierEntityImplementations.next(); + + //MappedField userAnchorMappedField = userIdentifierEntityImplementation.getMappedFieldByName(userIdentifierAnnotation.userAnchorFieldName()); + PropertyImplementation userIdentifierPropImpl = userIdentifierEntityImplementation.findPropertyImplementationByName(IUserIdentifier.USER_IDENTIFIER_FIELD_NAME); + RelationshipImplementation userAnchorRelImpl = userIdentifierEntityImplementation.findRelationshipImplementationBySourceVarName(IUserIdentifier.USER_ANCHOR_FIELD_NAME); + //IUserIdentifierA userIdentifierAnnotation = getUserIdentifierAnnotation(userIdentifierEntityImplementation); + if (userAnchorRelImpl != null) { + + if (s == null) { + s = appSessionFactory.openSession(); + } + + // Get this userAnchor's identifier(s). + String userIdentifierQueryString = "SELECT ui FROM " + userIdentifierEntityImplementation.mappedClass.tableName + + " ui WHERE ui." + userIdentifierPropImpl.mappedField.getField().getName() + "=:value AND (ui." + + userAnchorRelImpl.sourceMappedField.getField().getName() + " IS NULL OR ui." + userAnchorRelImpl.sourceMappedField.getField().getName() + + " IN (SELECT ua FROM " + eiUserAnchor.mappedClass.tableName + " ua WHERE (ua.userId=null OR ua.userId='' OR ua.userId=:userId)))"; + for (ServiceUser nextServiceUser : serviceUserList) { + for (ServiceIdentifier nextIdentifier : nextServiceUser.getIdentifiers()) { + try { + // Make sure this Identifier is in the same paradigm. + String paradigm = null; + Iterator itrParams = userIdentifierPropImpl.params.iterator(); + while (itrParams.hasNext()) { + PropertyImplementationParam nextParam = itrParams.next(); + if (nextParam.name.equalsIgnoreCase(IUserIdentifier.PARADIGM_PARAM_NAME)) { + paradigm = nextParam.value; + break; + } + } + if (nextIdentifier.getParadigm() == null || !nextIdentifier.getParadigm().equalsIgnoreCase(paradigm)) { + continue; + } + + // Look for this existing identifier. + Query userIdentifierQuery = s.createQuery(userIdentifierQueryString); + userIdentifierQuery.setString("userId", userId); + userIdentifierQuery.setString("value", nextIdentifier.getValue()); + IUserIdentifier foundIdentifier = (IUserIdentifier) userIdentifierQuery.uniqueResult(); + + if (foundIdentifier != null) { + // If the email does not have a Person, then associate email with this Person. + foundIdentifier = (IUserIdentifier) SyncHibernateUtils.cleanObject(foundIdentifier, s); + IUserAnchor existingUserAnchor = (IUserAnchor) SyncHibernateUtils.cleanObject(userAnchorRelImpl.sourceMappedField.getGetter().invoke(foundIdentifier), s); + if (existingUserAnchor == null) { + if (result != null) { + userAnchorRelImpl.sourceMappedField.getSetter().invoke(foundIdentifier, result); + syncAgentService.systemPutObject((IPerceroObject) foundIdentifier, null, null, null, true); + } + else { + identifiersToSave.add(foundIdentifier); + } + } else if (result == null) { + result = existingUserAnchor; + } + } else { // Identifier object NOT found, need to add. + // Identifier.identifierValue will be set later, once that is determined. + IUserIdentifier newIdentifier = (IUserIdentifier) userIdentifierEntityImplementation.mappedClass.clazz.newInstance(); /* IUserIdentifierA userIdentifierA = null; Class nextClass = newIdentifier.getClass(); @@ -544,65 +494,65 @@ protected IUserAnchor addOrUpdateUserAnchorFromServiceUserList(String userId, Li MappedField mappedField = mappedClass.getExternalizeFieldByName(userIdentifierA.userIdentifierFieldName()); mappedField.getSetter().invoke(newIdentifier, nextIdentifier.getValue()); */ - userIdentifierPropImpl.mappedField.getSetter().invoke(newIdentifier, nextIdentifier.getValue()); - identifiersToSave.add(newIdentifier); - } - } catch(NonUniqueResultException nure) { - log.warn("Non-unique User Identifier: " + nextIdentifier, nure); - } - } - } - } - } - - if (s != null && s.isOpen()) { - s.close(); - } - - // Need to create a new IUserAnchor - if (result == null) { - result = (IUserAnchor) eiUserAnchor.mappedClass.clazz.newInstance(); - ((IPerceroObject) result).setID(UUID.randomUUID().toString()); - result.setUserId(userId); - if (serviceUserList.size() > 0) { - /** - IUserAnchorA userAnchorAnnotation = getUserAnchorAnnotation(userAnchorClass); - - if (userAnchorAnnotation.firstNameFieldName() != null && !userAnchorAnnotation.firstNameFieldName().isEmpty()) - firstNameMappedField = mcUserAnchor.getMappedFieldByName(userAnchorAnnotation.firstNameFieldName()); - MappedField lastNameMappedField = null; - if (userAnchorAnnotation.lastNameFieldName() != null && !userAnchorAnnotation.lastNameFieldName().isEmpty()) - lastNameMappedField = mcUserAnchor.getMappedFieldByName(userAnchorAnnotation.lastNameFieldName()); - */ - - // Find field that has firstName and lastName PropertyImplementations. - PropertyImplementation firstNamePropImpl = eiUserAnchor.findPropertyImplementationByName(IUserAnchor.FIRST_NAME_FIELD); - PropertyImplementation lastNamePropImpl = eiUserAnchor.findPropertyImplementationByName(IUserAnchor.LAST_NAME_FIELD); - - ServiceUser firstServiceUser = serviceUserList.get(0); - - if (firstNamePropImpl != null) - firstNamePropImpl.mappedField.getSetter().invoke(result, firstServiceUser.getFirstName()); - if (lastNamePropImpl != null) - lastNamePropImpl.mappedField.getSetter().invoke(result, firstServiceUser.getLastName()); - } - - syncAgentService.systemCreateObject((IPerceroObject) result, null); - - } - else if (result.getUserId() == null || result.getUserId().isEmpty()) { - result.setUserId(userId); - - syncAgentService.systemPutObject((IPerceroObject) result, null, null, null, true); - } - - for (IUserIdentifier nextUserIdentifier : identifiersToSave) { - // Set Email.person now that we have a firm handle on that Person object. - Boolean isNewObject = false; - if (((IPerceroObject)nextUserIdentifier).getID() == null || ((IPerceroObject)nextUserIdentifier).getID().isEmpty()) { - isNewObject = true; - ((IPerceroObject)nextUserIdentifier).setID(UUID.randomUUID().toString()); - } + userIdentifierPropImpl.mappedField.getSetter().invoke(newIdentifier, nextIdentifier.getValue()); + identifiersToSave.add(newIdentifier); + } + } catch(NonUniqueResultException nure) { + log.warn("Non-unique User Identifier: " + nextIdentifier, nure); + } + } + } + } + } + + if (s != null && s.isOpen()) { + s.close(); + } + + // Need to create a new IUserAnchor + if (result == null) { + result = (IUserAnchor) eiUserAnchor.mappedClass.clazz.newInstance(); + ((IPerceroObject) result).setID(UUID.randomUUID().toString()); + result.setUserId(userId); + if (serviceUserList.size() > 0) { + /** + IUserAnchorA userAnchorAnnotation = getUserAnchorAnnotation(userAnchorClass); + + if (userAnchorAnnotation.firstNameFieldName() != null && !userAnchorAnnotation.firstNameFieldName().isEmpty()) + firstNameMappedField = mcUserAnchor.getMappedFieldByName(userAnchorAnnotation.firstNameFieldName()); + MappedField lastNameMappedField = null; + if (userAnchorAnnotation.lastNameFieldName() != null && !userAnchorAnnotation.lastNameFieldName().isEmpty()) + lastNameMappedField = mcUserAnchor.getMappedFieldByName(userAnchorAnnotation.lastNameFieldName()); + */ + + // Find field that has firstName and lastName PropertyImplementations. + PropertyImplementation firstNamePropImpl = eiUserAnchor.findPropertyImplementationByName(IUserAnchor.FIRST_NAME_FIELD); + PropertyImplementation lastNamePropImpl = eiUserAnchor.findPropertyImplementationByName(IUserAnchor.LAST_NAME_FIELD); + + ServiceUser firstServiceUser = serviceUserList.get(0); + + if (firstNamePropImpl != null) + firstNamePropImpl.mappedField.getSetter().invoke(result, firstServiceUser.getFirstName()); + if (lastNamePropImpl != null) + lastNamePropImpl.mappedField.getSetter().invoke(result, firstServiceUser.getLastName()); + } + + syncAgentService.systemCreateObject((IPerceroObject) result, null); + + } + else if (result.getUserId() == null || result.getUserId().isEmpty()) { + result.setUserId(userId); + + syncAgentService.systemPutObject((IPerceroObject) result, null, null, null, true); + } + + for (IUserIdentifier nextUserIdentifier : identifiersToSave) { + // Set Email.person now that we have a firm handle on that Person object. + Boolean isNewObject = false; + if (((IPerceroObject)nextUserIdentifier).getID() == null || ((IPerceroObject)nextUserIdentifier).getID().isEmpty()) { + isNewObject = true; + ((IPerceroObject)nextUserIdentifier).setID(UUID.randomUUID().toString()); + } /* IUserIdentifierA userIdentifierAnnotation = getUserIdentifierAnnotation(nextUserIdentifier.getClass()); @@ -618,147 +568,117 @@ else if (result.getUserId() == null || result.getUserId().isEmpty()) { } */ - IMappedClassManager mcm = MappedClassManagerFactory.getMappedClassManager(); - MappedClass mc = mcm.getMappedClassByClassName(nextUserIdentifier.getClass().getName()); - EntityImplementation entityImpl = mc.entityImplementations.get(IUserIdentifier.class); - if (entityImpl != null) { - RelationshipImplementation relImpl = entityImpl.findRelationshipImplementationBySourceVarName(IUserIdentifier.USER_ANCHOR_FIELD_NAME); - if (relImpl != null) { - relImpl.sourceMappedField.getSetter().invoke(nextUserIdentifier, result); - if (isNewObject) { - syncAgentService.systemCreateObject((IPerceroObject) nextUserIdentifier, null); - } - else { - syncAgentService.systemPutObject((IPerceroObject) nextUserIdentifier, null, null, null, true); - } - } - else { - log.warn("No UserAnchor found for IUserIdentifier class " + nextUserIdentifier.getClass().getCanonicalName()); - } - } - } - - if (result != null) { - setupUserRoles(userId, serviceUserList); - return result; - } - } - - } catch (IllegalArgumentException e) { - log.error(e.getMessage(), e); - } catch (IllegalAccessException e) { - log.error(e.getMessage(), e); - } catch (InvocationTargetException e) { - log.error(e.getMessage(), e); - } catch (Exception e) { - log.error(e.getMessage(), e); - } finally { - if (s != null && s.isOpen()) - s.close(); - } - - return null; - } - - @SuppressWarnings({ "unchecked" }) - public void setupUserRoles(String userId, List serviceUserList) throws Exception { - Session s = appSessionFactory.openSession(); - - try { - ManifestHelper.setManifest(manifest); - EntityImplementation userAnchorEI = null; - List userAnchorMappedClasses = MappedClass.findEntityImplementation(IUserAnchor.class); - if (userAnchorMappedClasses.size() > 0) { - userAnchorEI = userAnchorMappedClasses.get(0); - } - EntityImplementation userRoleEI = null; - List userRoleMappedClasses = MappedClass.findEntityImplementation(IUserRole.class); - if (userRoleMappedClasses.size() > 0) { - userRoleEI = userRoleMappedClasses.get(0); - } - - if (userAnchorEI != null && userRoleEI != null) { - RelationshipImplementation userAnchorRelImpl = userRoleEI.findRelationshipImplementationBySourceVarName(IUserRole.USER_ANCHOR_FIELD_NAME); - - String userAnchorQueryString = "SELECT ua FROM " + userAnchorEI.mappedClass.tableName + " ua WHERE ua.userId=:userId"; - Query userAnchorQuery = s.createQuery(userAnchorQueryString); - userAnchorQuery.setString("userId", userId); - - IUserAnchor userAnchor = (IUserAnchor) userAnchorQuery.uniqueResult(); - - //IUserRoleA userRoleAnnotation = getUserRoleAnnotation(userRoleClass); - String personRoleQueryString = "SELECT ur FROM " + userRoleEI.mappedClass.tableName - + " ur WHERE ur." + userAnchorRelImpl.sourceMappedField.getField().getName() + " IN (SELECT ua FROM " + userAnchorEI.mappedClass.tableName + " ua WHERE ua.userId=:userId)"; - Query personRoleQuery = s.createQuery(personRoleQueryString); - personRoleQuery.setString("userId", userId); - - List userRoles = (List) personRoleQuery.list(); - List updatedUserRoles = new ArrayList(); - - // First, remove all roles that a Person has that are not in the serviceUserList. - for(IUserRole nextUserRole : userRoles) { - Boolean serviceUserRoleExists = false; - Boolean isInaccurateList = false; - for (ServiceUser nextServiceUser : serviceUserList) { - if (!nextServiceUser.getAreRoleNamesAccurate()) { - log.debug("Ignoring role names from " + nextServiceUser.getAuthProviderID().toString()); - isInaccurateList = true; - break; - } - else { - if (nextServiceUser.getRoleNames().contains(nextUserRole.getRoleName())) { - serviceUserRoleExists = true; - break; - } - } - } - - if (!isInaccurateList && !serviceUserRoleExists) { - log.warn("Deleting role " + nextUserRole.getRoleName() + " for " + userId); - syncAgentService.systemDeleteObject(nextUserRole, null, true, new HashSet()); - } else - updatedUserRoles.add(nextUserRole); - } - - // Second, add all roles that are not on the Person. - // Get MappedFields to set. - //MappedField userAnchorMappedField = userRoleMappedClass.getMappedFieldByName(userRoleAnnotation.userAnchorFieldName()); - //MappedField userRoleDateCreatedField = userRoleMappedClass.getMappedFieldByName("dateCreated"); - //MappedField userRoleDateModifiedField = userRoleMappedClass.getMappedFieldByName("dateModified"); - - for (ServiceUser nextServiceUser : serviceUserList) { - if (nextServiceUser.getAreRoleNamesAccurate()) { - // Role names from the service are accurate, so we can match them up here. - for(String nextServiceRoleName : nextServiceUser.getRoleNames()) { - Boolean personRoleExists = false; - for(IUserRole nextUserRole : updatedUserRoles) { - if (nextUserRole.getRoleName().equalsIgnoreCase(nextServiceRoleName)) { - personRoleExists = true; - break; - } - } - - if (!personRoleExists) { - IUserRole nextPersonRole = (IUserRole) userRoleEI.mappedClass.clazz.newInstance(); - /*if (userRoleDateCreatedField != null) - userRoleDateCreatedField.getSetter().invoke(nextPersonRole, currentDate); - if (userRoleDateModifiedField != null) - userRoleDateModifiedField.getSetter().invoke(nextPersonRole, currentDate);*/ - userAnchorRelImpl.sourceMappedField.getSetter().invoke(nextPersonRole, userAnchor); - - nextPersonRole.setRoleName(nextServiceRoleName); - nextPersonRole.setID(UUID.randomUUID().toString()); - - syncAgentService.systemCreateObject(nextPersonRole, null); - } - } - } - } - } - } catch (Exception e) { - log.error("Unable to Get Person Roles", e); - } finally { - s.close(); - } - } + IMappedClassManager mcm = MappedClassManagerFactory.getMappedClassManager(); + MappedClass mc = mcm.getMappedClassByClassName(nextUserIdentifier.getClass().getName()); + EntityImplementation entityImpl = mc.entityImplementations.get(IUserIdentifier.class); + if (entityImpl != null) { + RelationshipImplementation relImpl = entityImpl.findRelationshipImplementationBySourceVarName(IUserIdentifier.USER_ANCHOR_FIELD_NAME); + if (relImpl != null) { + relImpl.sourceMappedField.getSetter().invoke(nextUserIdentifier, result); + if (isNewObject) { + syncAgentService.systemCreateObject((IPerceroObject) nextUserIdentifier, null); + } + else { + syncAgentService.systemPutObject((IPerceroObject) nextUserIdentifier, null, null, null, true); + } + } + else { + log.warn("No UserAnchor found for IUserIdentifier class " + nextUserIdentifier.getClass().getCanonicalName()); + } + } + } + + if (result != null) { + setupUserRoles(userId, serviceUserList); + return result; + } + } + + } catch (IllegalArgumentException e) { + log.error(e.getMessage(), e); + } catch (IllegalAccessException e) { + log.error(e.getMessage(), e); + } catch (InvocationTargetException e) { + log.error(e.getMessage(), e); + } catch (Exception e) { + log.error(e.getMessage(), e); + } finally { + if (s != null && s.isOpen()) + s.close(); + } + + return null; + } + + @SuppressWarnings({ "unchecked" }) + public void setupUserRoles(String userId, List serviceUserList) throws Exception { + + try { + ManifestHelper.setManifest(manifest); + EntityImplementation userRoleEI = RoleHelper.getUserRoleEntityImplementation(); + RelationshipImplementation userAnchorRelImpl = userRoleEI.findRelationshipImplementationBySourceVarName(IUserRole.USER_ANCHOR_FIELD_NAME); + + IUserAnchor userAnchor = UserAnchorHelper.getUserAnchor(userId); + List userRoles = RoleHelper.getUserRoles(userAnchor); + List updatedUserRoles = new ArrayList(); + + // First, remove all roles that a Person has that are not in the serviceUserList. + for(IUserRole nextUserRole : userRoles) { + Boolean serviceUserRoleExists = false; + Boolean isInaccurateList = false; + for (ServiceUser nextServiceUser : serviceUserList) { + if (!nextServiceUser.getAreRoleNamesAccurate()) { + log.debug("Ignoring role names from " + nextServiceUser.getAuthProviderID().toString()); + isInaccurateList = true; + break; + } + else { + if (nextServiceUser.getRoleNames().contains(nextUserRole.getRoleName())) { + serviceUserRoleExists = true; + break; + } + } + } + + if (!isInaccurateList && !serviceUserRoleExists) { + log.warn("Deleting role " + nextUserRole.getRoleName() + " for " + userId); + syncAgentService.systemDeleteObject(nextUserRole, null, true, new HashSet()); + } else + updatedUserRoles.add(nextUserRole); + } + + // Second, add all roles that are not on the Person. + // Get MappedFields to set. + //MappedField userAnchorMappedField = userRoleMappedClass.getMappedFieldByName(userRoleAnnotation.userAnchorFieldName()); + //MappedField userRoleDateCreatedField = userRoleMappedClass.getMappedFieldByName("dateCreated"); + //MappedField userRoleDateModifiedField = userRoleMappedClass.getMappedFieldByName("dateModified"); + + for (ServiceUser nextServiceUser : serviceUserList) { + if (nextServiceUser.getAreRoleNamesAccurate()) { + // Role names from the service are accurate, so we can match them up here. + for(String nextServiceRoleName : nextServiceUser.getRoleNames()) { + Boolean personRoleExists = false; + for(IUserRole nextUserRole : updatedUserRoles) { + if (nextUserRole.getRoleName().equalsIgnoreCase(nextServiceRoleName)) { + personRoleExists = true; + break; + } + } + + if (!personRoleExists) { + IUserRole nextPersonRole = (IUserRole) userRoleEI.mappedClass.clazz.newInstance(); + userAnchorRelImpl.sourceMappedField.getSetter().invoke(nextPersonRole, userAnchor); + + nextPersonRole.setRoleName(nextServiceRoleName); + nextPersonRole.setID(UUID.randomUUID().toString()); + + syncAgentService.systemCreateObject(nextPersonRole, null); + } + } + } + } + } catch (Exception e) { + log.error("Unable to Get Person Roles", e); + } + } } diff --git a/src/main/java/com/percero/agents/auth/helpers/RoleHelper.java b/src/main/java/com/percero/agents/auth/helpers/RoleHelper.java new file mode 100644 index 0000000..1b2c5b1 --- /dev/null +++ b/src/main/java/com/percero/agents/auth/helpers/RoleHelper.java @@ -0,0 +1,76 @@ +package com.percero.agents.auth.helpers; + +import com.percero.agents.auth.vo.IUserAnchor; +import com.percero.agents.auth.vo.IUserRole; +import com.percero.agents.sync.dao.DAORegistry; +import com.percero.agents.sync.dao.IDataAccessObject; +import com.percero.agents.sync.metadata.EntityImplementation; +import com.percero.agents.sync.metadata.MappedClass; +import com.percero.agents.sync.metadata.RelationshipImplementation; +import com.percero.framework.bl.ManifestHelper; +import com.percero.framework.vo.IPerceroObject; + +import java.util.ArrayList; +import java.util.List; + +/** + * Created by jonnysamps on 10/2/15. + */ +public class RoleHelper { + + /** + * Gets a list of role names for a user + * @param userAnchor + * @return + */ + public static List getUserRoleNames(IUserAnchor userAnchor){ + List result = new ArrayList<>(); + List roles = getUserRoles(userAnchor); + + for(IUserRole role : roles) + result.add(role.getRoleName()); + + return result; + } + + public static List getUserRoles(IUserAnchor userAnchor){ + List result = new ArrayList<>(); + Class userRoleClass = ManifestHelper.findImplementingClass(IUserRole.class); + RelationshipImplementation userAnchorRelImpl = getUserAnchorRI(); + + try { + // Now find the roles + IDataAccessObject userRoleDao = + (IDataAccessObject) DAORegistry.getInstance().getDataAccessObject(userRoleClass.getName()); + IUserRole exampleRole = (IUserRole) userRoleClass.newInstance(); + + userAnchorRelImpl.sourceMappedField.getSetter().invoke(exampleRole, userAnchor); + + List roleList = userRoleDao.findByExample(exampleRole, null, null, false); + for (IPerceroObject role : roleList) { + result.add((IUserRole) role); + } + }catch(Exception e){} + + return result; + } + + /** + * Finds the UserRole implementation + * @return + */ + public static EntityImplementation getUserRoleEntityImplementation(){ + EntityImplementation userRoleEI = null; + List userRoleMappedClasses = MappedClass.findEntityImplementation(IUserRole.class); + if (userRoleMappedClasses.size() > 0) + userRoleEI = userRoleMappedClasses.get(0); + + return userRoleEI; + } + + public static RelationshipImplementation getUserAnchorRI(){ + RelationshipImplementation userAnchorRelImpl = + getUserRoleEntityImplementation().findRelationshipImplementationBySourceVarName(IUserRole.USER_ANCHOR_FIELD_NAME); + return userAnchorRelImpl; + } +} diff --git a/src/main/java/com/percero/agents/auth/helpers/UserAnchorHelper.java b/src/main/java/com/percero/agents/auth/helpers/UserAnchorHelper.java new file mode 100644 index 0000000..ceaa41e --- /dev/null +++ b/src/main/java/com/percero/agents/auth/helpers/UserAnchorHelper.java @@ -0,0 +1,43 @@ +package com.percero.agents.auth.helpers; + +import com.percero.agents.auth.vo.IUserAnchor; +import com.percero.agents.sync.dao.DAORegistry; +import com.percero.agents.sync.dao.IDataAccessObject; +import com.percero.agents.sync.metadata.EntityImplementation; +import com.percero.agents.sync.metadata.MappedClass; +import com.percero.framework.bl.ManifestHelper; +import com.percero.framework.vo.IPerceroObject; + +import java.util.List; + +/** + * Created by jonnysamps on 10/2/15. + */ +public class UserAnchorHelper { + public static IUserAnchor getUserAnchor(String userId){ + IUserAnchor result = null; + Class userAnchorClass = ManifestHelper.findImplementingClass(IUserAnchor.class); + + IDataAccessObject userAnchorDao = + (IDataAccessObject) DAORegistry.getInstance().getDataAccessObject(userAnchorClass.getName()); + + try { + IUserAnchor example = (IUserAnchor) userAnchorClass.newInstance(); + example.setUserId(userId); + List list = userAnchorDao.findByExample((IPerceroObject)example, null, null, false); + if(list.size() > 0) + result = (IUserAnchor)list.get(0); + }catch(Exception e){} + + return result; + } + + public static EntityImplementation getUserAnchorEntityImplementation(){ + EntityImplementation userAnchorEI = null; + List userAnchorMappedClasses = MappedClass.findEntityImplementation(IUserAnchor.class); + if (userAnchorMappedClasses.size() > 0) { + userAnchorEI = userAnchorMappedClasses.get(0); + } + return userAnchorEI; + } +} diff --git a/src/main/java/com/percero/agents/auth/services/AuthService2.java b/src/main/java/com/percero/agents/auth/services/AuthService2.java index 09a62a1..b6cc038 100644 --- a/src/main/java/com/percero/agents/auth/services/AuthService2.java +++ b/src/main/java/com/percero/agents/auth/services/AuthService2.java @@ -1,5 +1,7 @@ package com.percero.agents.auth.services; +import com.percero.agents.auth.helpers.RoleHelper; +import com.percero.agents.auth.helpers.UserAnchorHelper; import com.percero.agents.auth.hibernate.AssociationExample; import com.percero.agents.auth.hibernate.AuthHibernateUtils; import com.percero.agents.auth.hibernate.BaseDataObjectPropertySelector; @@ -32,7 +34,7 @@ public class AuthService2 { private static Logger logger = Logger.getLogger(AuthService2.class); @Autowired - private AuthProviderRegistry authProviderRegistry; + AuthProviderRegistry authProviderRegistry; @Autowired SessionFactory sessionFactoryAuth; @@ -336,7 +338,7 @@ private static String getRandomId() { public void ensureAnchorUserExists(ServiceUser serviceUser, User user){ IUserAnchor result = null; - Session s = appSessionFactory.openSession(); +// Session s = appSessionFactory.openSession(); EntityImplementation userAnchorEI = getUserAnchorEntityImplementation(); @@ -354,9 +356,11 @@ public void ensureAnchorUserExists(ServiceUser serviceUser, User user){ IUserAnchor example = (IUserAnchor) anchorUserClass.newInstance(); example.setUserId(user.getID()); foundUserAnchors = dao.findByExample((IPerceroObject) example, null, null, false); - }catch(Exception e){} + }catch(Exception e){ + logger.warn(e.getMessage(), e); + } - if (foundUserAnchors != null && foundUserAnchors.size() <= 0) + if (foundUserAnchors == null || foundUserAnchors.size() <= 0) addOrUpdateUserAnchorFromServiceUserList(serviceUser, user, null); else { if(foundUserAnchors.size() > 1) @@ -366,8 +370,8 @@ public void ensureAnchorUserExists(ServiceUser serviceUser, User user){ handleUserAnchorFound(serviceUser, user, result); } - if (s != null && s.isOpen()) - s.close(); +// if (s != null && s.isOpen()) +// s.close(); } private EntityImplementation getUserAnchorEntityImplementation(){ @@ -622,27 +626,16 @@ else if (result.getUserId() == null || result.getUserId().isEmpty()) { @SuppressWarnings({ "unchecked" }) public void setupUserRoles(User user, ServiceUser serviceUser) throws Exception { - Session s = appSessionFactory.openSession(); +// Session s = appSessionFactory.openSession(); try { - EntityImplementation userAnchorEI = getUserAnchorEntityImplementation(); EntityImplementation userRoleEI = getUserRoleEntityImplementation(); - RelationshipImplementation userAnchorRelImpl = userRoleEI.findRelationshipImplementationBySourceVarName(IUserRole.USER_ANCHOR_FIELD_NAME); - - String userAnchorQueryString = "SELECT ua FROM " + userAnchorEI.mappedClass.tableName + " ua WHERE ua.userId=:userId"; - Query userAnchorQuery = s.createQuery(userAnchorQueryString); - userAnchorQuery.setString("userId", user.getID()); + RelationshipImplementation userAnchorRelImpl = RoleHelper.getUserAnchorRI(); - IUserAnchor userAnchor = (IUserAnchor) userAnchorQuery.uniqueResult(); + IUserAnchor userAnchor = UserAnchorHelper.getUserAnchor(user.getID()); - //IUserRoleA userRoleAnnotation = getUserRoleAnnotation(userRoleClass); - String personRoleQueryString = "SELECT ur FROM " + userRoleEI.mappedClass.tableName - + " ur WHERE ur." + userAnchorRelImpl.sourceMappedField.getField().getName() + " IN (SELECT ua FROM " + userAnchorEI.mappedClass.tableName + " ua WHERE ua.userId=:userId)"; - Query personRoleQuery = s.createQuery(personRoleQueryString); - personRoleQuery.setString("userId", user.getID()); - - List userRoles = (List) personRoleQuery.list(); + List userRoles = RoleHelper.getUserRoles(userAnchor); List updatedUserRoles = new ArrayList(); // First, remove all roles that a Person has that are not in the serviceUserList. @@ -652,13 +645,9 @@ public void setupUserRoles(User user, ServiceUser serviceUser) throws Exception if (!serviceUser.getAreRoleNamesAccurate()) { logger.debug("Ignoring role names from " + serviceUser.getAuthProviderID().toString()); isInaccurateList = true; - break; } - else { - if (serviceUser.getRoleNames().contains(nextUserRole.getRoleName())) { - serviceUserRoleExists = true; - break; - } + else if (serviceUser.getRoleNames().contains(nextUserRole.getRoleName())) { + serviceUserRoleExists = true; } @@ -695,8 +684,6 @@ public void setupUserRoles(User user, ServiceUser serviceUser) throws Exception } catch (Exception e) { logger.error("Unable to Get Person Roles", e); - } finally { - s.close(); } } } diff --git a/src/main/java/com/percero/agents/sync/metadata/MappedField.java b/src/main/java/com/percero/agents/sync/metadata/MappedField.java index be8e245..7f09ce9 100644 --- a/src/main/java/com/percero/agents/sync/metadata/MappedField.java +++ b/src/main/java/com/percero/agents/sync/metadata/MappedField.java @@ -1,22 +1,18 @@ package com.percero.agents.sync.metadata; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.percero.framework.metadata.IMappedQuery; + import java.io.IOException; import java.io.ObjectInput; import java.io.ObjectOutput; import java.lang.reflect.Field; import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.Iterator; -import java.util.List; -import java.util.Map; +import java.util.*; import java.util.Map.Entry; -import com.google.gson.JsonElement; -import com.google.gson.JsonObject; -import com.percero.framework.metadata.IMappedQuery; - public class MappedField { private MappedClass mappedClass; @@ -166,6 +162,7 @@ public void writeExternalField(ObjectOutput output, Object anObject) public Object getValue(Object anObject) throws IllegalArgumentException, IllegalAccessException, InvocationTargetException { + if(anObject == null) return null; Object result = getGetter().invoke(anObject); return result; } diff --git a/src/main/resources/spring/percero-spring-config.xml b/src/main/resources/spring/percero-spring-config.xml index 4c4ebc1..57f1f8d 100644 --- a/src/main/resources/spring/percero-spring-config.xml +++ b/src/main/resources/spring/percero-spring-config.xml @@ -219,7 +219,8 @@ - hibernate.dialect=org.hibernate.dialect.MySQLDialect + + hibernate.dialect=org.hibernate.dialect.Oracle10gDialect hibernate.show_sql=false @@ -298,7 +299,9 @@ - hibernate.dialect=org.hibernate.dialect.MySQLDialect + + hibernate.dialect=org.hibernate.dialect.Oracle10gDialect + hibernate.show_sql=false From c80d5072cfe2278b769d4d95f4586ac3981f9262 Mon Sep 17 00:00:00 2001 From: Jonathan Samples Date: Sun, 4 Oct 2015 13:16:12 -0700 Subject: [PATCH 007/105] Added round-robin weighting to the update table processor --- .../jobs/UpdateTableConnectionFactory.java | 20 +- .../agents/sync/jobs/UpdateTablePoller.java | 39 ++- .../sync/jobs/UpdateTableProcessor.java | 327 +++++++++--------- 3 files changed, 213 insertions(+), 173 deletions(-) diff --git a/src/main/java/com/percero/agents/sync/jobs/UpdateTableConnectionFactory.java b/src/main/java/com/percero/agents/sync/jobs/UpdateTableConnectionFactory.java index b4457f6..c0f2d2f 100644 --- a/src/main/java/com/percero/agents/sync/jobs/UpdateTableConnectionFactory.java +++ b/src/main/java/com/percero/agents/sync/jobs/UpdateTableConnectionFactory.java @@ -1,13 +1,12 @@ package com.percero.agents.sync.jobs; +import com.mchange.v2.c3p0.ComboPooledDataSource; +import org.apache.log4j.Logger; + import java.beans.PropertyVetoException; import java.sql.Connection; import java.sql.SQLException; -import org.apache.log4j.Logger; - -import com.mchange.v2.c3p0.ComboPooledDataSource; - /** * Created by jonnysamps on 9/2/15. */ @@ -72,7 +71,7 @@ public void setStoredProcedureDefinition(String storedProcedureDefinition) { this.storedProcedureDefinition = storedProcedureDefinition; } - private String updateStatementSql = "update `:tableName` set lock_id=:lockId, lock_date=NOW() " + + private String updateStatementSql = "update :tableName set lock_id=:lockId, lock_date=NOW() " + "where lock_id is null or " + "lock_date < ':expireThreshold' " + "order by time_stamp limit 1"; @@ -83,6 +82,17 @@ public void setUpdateStatementSql(String updateStatementSql) { this.updateStatementSql = updateStatementSql; } + /** + * How much should we favor this connection compared to another + */ + private int weight = UpdateTableProcessor.INFINITE_ROWS; + public void setWeight(int weight){ + this.weight = weight; + } + public int getWeight(){ + return this.weight; + } + private ComboPooledDataSource cpds; public void init() throws PropertyVetoException{ diff --git a/src/main/java/com/percero/agents/sync/jobs/UpdateTablePoller.java b/src/main/java/com/percero/agents/sync/jobs/UpdateTablePoller.java index 04b5499..8b2ff1c 100644 --- a/src/main/java/com/percero/agents/sync/jobs/UpdateTablePoller.java +++ b/src/main/java/com/percero/agents/sync/jobs/UpdateTablePoller.java @@ -48,21 +48,34 @@ public class UpdateTablePoller { */ @Scheduled(fixedDelay=5000, initialDelay=10000) // Every 5 seconds public void pollUpdateTables(){ + logger.debug("|> Starting poller"); if(enabled) { - for(UpdateTableConnectionFactory updateTableConnectionFactory : updateTableRegistry.getConnectionFactories()) { - for(String tableName : updateTableConnectionFactory.getTableNames()){ - UpdateTableProcessor processor = getProcessor(updateTableConnectionFactory, tableName); - ProcessorResult result = processor.process(); - if(result.isSuccess()){ - logger.debug("Update table processor ("+tableName+") finished successfully. Total rows ("+result.getTotal()+")"); - } - else{ - logger.warn("Update table processor ("+tableName+") failed. Details:"); - logger.warn(result); - } - } - } + // Loop until we didn't find any rows to work with + while(true) { + int count = 0; + for (UpdateTableConnectionFactory updateTableConnectionFactory : updateTableRegistry.getConnectionFactories()) { + for (String tableName : updateTableConnectionFactory.getTableNames()) { + count += doProcessingForTable(updateTableConnectionFactory, tableName); + } + } + if(count <= 0) break; + } } + logger.debug("[] Stopping poller"); + } + + public int doProcessingForTable(UpdateTableConnectionFactory connectionFactory, String tableName){ + UpdateTableProcessor processor = getProcessor(connectionFactory, tableName); + ProcessorResult result = processor.process(); + if(result.isSuccess()){ + logger.debug("Update table processor ("+tableName+") finished successfully. Total rows ("+result.getTotal()+")"); + } + else{ + logger.warn("Update table processor ("+tableName+") failed. Details:"); + logger.warn(result); + } + + return result.getTotal(); } public UpdateTableProcessor getProcessor(UpdateTableConnectionFactory connectionFactory, String tableName){ diff --git a/src/main/java/com/percero/agents/sync/jobs/UpdateTableProcessor.java b/src/main/java/com/percero/agents/sync/jobs/UpdateTableProcessor.java index 9d54613..3e1c45d 100644 --- a/src/main/java/com/percero/agents/sync/jobs/UpdateTableProcessor.java +++ b/src/main/java/com/percero/agents/sync/jobs/UpdateTableProcessor.java @@ -1,36 +1,10 @@ package com.percero.agents.sync.jobs; -import java.sql.CallableStatement; -import java.sql.Connection; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.sql.SQLSyntaxErrorException; -import java.sql.Statement; -import java.sql.Types; -import java.util.ArrayList; -import java.util.Collection; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Iterator; -import java.util.Map; -import java.util.Random; -import java.util.Set; - -import javax.persistence.Table; - -import org.apache.log4j.Logger; -import org.joda.time.DateTime; -import org.springframework.util.StringUtils; - import com.percero.agents.sync.access.IAccessManager; import com.percero.agents.sync.cache.CacheManager; import com.percero.agents.sync.helpers.PostDeleteHelper; import com.percero.agents.sync.helpers.PostPutHelper; -import com.percero.agents.sync.metadata.IMappedClassManager; -import com.percero.agents.sync.metadata.MappedClass; -import com.percero.agents.sync.metadata.MappedClassManagerFactory; -import com.percero.agents.sync.metadata.MappedField; -import com.percero.agents.sync.metadata.MappedFieldPerceroObject; +import com.percero.agents.sync.metadata.*; import com.percero.agents.sync.services.DataProviderManager; import com.percero.agents.sync.services.IDataProvider; import com.percero.agents.sync.vo.BaseDataObject; @@ -38,6 +12,13 @@ import com.percero.framework.bl.IManifest; import com.percero.framework.vo.IPerceroObject; import com.percero.framework.vo.PerceroList; +import org.apache.log4j.Logger; +import org.joda.time.DateTime; +import org.springframework.util.StringUtils; + +import javax.persistence.Table; +import java.sql.*; +import java.util.*; /** * Responsible for querying an update table and processing the rows. @@ -46,6 +27,7 @@ public class UpdateTableProcessor { protected static Logger logger = Logger.getLogger(UpdateTableProcessor.class); + public static final int INFINITE_ROWS = -1; protected static final int EXPIRATION_TIME = 1000*60*30; // 30 minutes protected String tableName; @@ -56,6 +38,7 @@ public class UpdateTableProcessor { protected CacheManager cacheManager; protected DataProviderManager dataProviderManager; protected IAccessManager accessManager; + protected int maxRowsToProcess = INFINITE_ROWS; // No max public UpdateTableProcessor(String tableName, UpdateTableConnectionFactory connectionFactory, @@ -74,9 +57,10 @@ public UpdateTableProcessor(String tableName, this.cacheManager = cacheManager; this.dataProviderManager= dataProviderManager; this.accessManager = accessManager; + this.maxRowsToProcess = connectionFactory.getWeight(); } - - + + /** * Update table schema looks like this * @@ -93,7 +77,8 @@ public UpdateTableProcessor(String tableName, public ProcessorResult process(){ ProcessorResult result = new ProcessorResult(); - while(true) { + int numRowsProcessed = 0; + while(numRowsProcessed < maxRowsToProcess || maxRowsToProcess == INFINITE_ROWS) { UpdateTableRow row = getRow(); if(row == null) break; @@ -108,7 +93,7 @@ public ProcessorResult process(){ logger.warn("Failed to process update: "+ e.getMessage(), e); result.addResult(row.getType().toString(), false, e.getMessage()); } - + numRowsProcessed++; } return result; @@ -116,7 +101,7 @@ public ProcessorResult process(){ protected boolean processRow(UpdateTableRow row) throws Exception{ boolean result = true; - + logger.debug("UpdateTableProcessor: processRow"); if(row.getRowId() != null) switch (row.getType()){ case DELETE: @@ -159,19 +144,24 @@ protected boolean processRow(UpdateTableRow row) throws Exception{ @SuppressWarnings("rawtypes") protected boolean processUpdateSingle(UpdateTableRow row) throws Exception{ Class clazz = getClassForTableName(row.getTableName()); - String className = clazz.getCanonicalName(); - IMappedClassManager mcm = MappedClassManagerFactory.getMappedClassManager(); - MappedClass mappedClass = mcm.getMappedClassByClassName(className); - IDataProvider dataProvider = dataProviderManager.getDataProviderByName(mappedClass.dataProviderName); - - ClassIDPair pair = new ClassIDPair(row.getRowId(), className); - handleUpdateClassIdPair(dataProvider, pair); - - // We need to update ALL referencing objects in the case that a - // relationship was updated. Since we don't have the OLD object, we have - // no way of telling what may have changed. - updateReferences(className); + // If we found the class then we care about this row, otherwise return true and the row will be deleted + if(clazz != null) { + String className = clazz.getCanonicalName(); + + IMappedClassManager mcm = MappedClassManagerFactory.getMappedClassManager(); + MappedClass mappedClass = mcm.getMappedClassByClassName(className); + IDataProvider dataProvider = dataProviderManager.getDataProviderByName(mappedClass.dataProviderName); + + ClassIDPair pair = new ClassIDPair(row.getRowId(), className); + handleUpdateClassIdPair(dataProvider, pair); + + // We need to update ALL referencing objects in the case that a + // relationship was updated. Since we don't have the OLD object, we have + // no way of telling what may have changed. + updateReferences(className); + } + return true; } @@ -183,42 +173,44 @@ protected boolean processUpdateSingle(UpdateTableRow row) throws Exception{ @SuppressWarnings("rawtypes") protected boolean processUpdateTable(UpdateTableRow row) throws Exception{ Class clazz = getClassForTableName(row.getTableName()); - String className = clazz.getCanonicalName(); - // If there are any clients that have asked for all objects in a class then we have to push everything - if(accessManager.getNumClientsInterestedInWholeClass(className) > 0) { + if(clazz != null) { + String className = clazz.getCanonicalName(); - IMappedClassManager mcm = MappedClassManagerFactory.getMappedClassManager(); - MappedClass mappedClass = mcm.getMappedClassByClassName(className); - IDataProvider dataProvider = dataProviderManager.getDataProviderByName(mappedClass.dataProviderName); + // If there are any clients that have asked for all objects in a class then we have to push everything + if (accessManager.getNumClientsInterestedInWholeClass(className) > 0) { - Integer pageNumber = 0; - Integer pageSize = 25; - Integer total = -1; - - while (total < 0 || pageNumber * pageSize <= total) { - PerceroList objectsToUpdate = dataProvider.getAllByName(className, pageNumber, pageSize, true, null); - pageNumber++; - total = objectsToUpdate.getTotalLength(); - if (total <= 0) { - break; - } - - Set objectIds = new HashSet(objectsToUpdate.size()); - Iterator itrObjectsToUpdate = objectsToUpdate.iterator(); - while (itrObjectsToUpdate.hasNext()) { - IPerceroObject nextObjectToUpdate = itrObjectsToUpdate.next(); - objectIds.add(nextObjectToUpdate.getID()); - } - - processUpdates(className, objectIds); - } - } - else { - processUpdates(className, accessManager.getClassAccessJournalIDs(className)); - } + IMappedClassManager mcm = MappedClassManagerFactory.getMappedClassManager(); + MappedClass mappedClass = mcm.getMappedClassByClassName(className); + IDataProvider dataProvider = dataProviderManager.getDataProviderByName(mappedClass.dataProviderName); + + Integer pageNumber = 0; + Integer pageSize = 25; + Integer total = -1; + + while (total < 0 || pageNumber * pageSize <= total) { + PerceroList objectsToUpdate = dataProvider.getAllByName(className, pageNumber, pageSize, true, null); + pageNumber++; + total = objectsToUpdate.getTotalLength(); + if (total <= 0) { + break; + } - updateReferences(className); + Set objectIds = new HashSet(objectsToUpdate.size()); + Iterator itrObjectsToUpdate = objectsToUpdate.iterator(); + while (itrObjectsToUpdate.hasNext()) { + IPerceroObject nextObjectToUpdate = itrObjectsToUpdate.next(); + objectIds.add(nextObjectToUpdate.getID()); + } + + processUpdates(className, objectIds); + } + } else { + processUpdates(className, accessManager.getClassAccessJournalIDs(className)); + } + + updateReferences(className); + } return true; } @@ -239,10 +231,10 @@ protected void processUpdates(String className, Collection Ids) throws E handleUpdateClassIdPair(dataProvider, pair); } } - + protected void handleUpdateClassIdPair(IDataProvider dataProvider, ClassIDPair pair) throws Exception { IPerceroObject object = dataProvider.findById(pair, null, true); - + if (object != null) { cacheManager.updateCachedObject(object, null); postPutHelper.postPutObject(pair, null, null, true, null); @@ -257,13 +249,18 @@ protected void handleUpdateClassIdPair(IDataProvider dataProvider, ClassIDPair p @SuppressWarnings("rawtypes") protected boolean processInsertSingle(UpdateTableRow row) throws Exception{ Class clazz = getClassForTableName(row.getTableName()); - String className = clazz.getCanonicalName(); - - // We do not use PostCreateHelper here because we are going to do all - // that extra work for the whole class in updateReferences. - postPutHelper.postPutObject(new ClassIDPair(row.getRowId(), className), null, null, true, null); - updateReferences(className); + // If clazz is not null then we care about this row, otherwise throw the row away + if(clazz != null) { + String className = clazz.getCanonicalName(); + + // We do not use PostCreateHelper here because we are going to do all + // that extra work for the whole class in updateReferences. + postPutHelper.postPutObject(new ClassIDPair(row.getRowId(), className), null, null, true, null); + + updateReferences(className); + } + return true; } @@ -275,20 +272,23 @@ protected boolean processInsertSingle(UpdateTableRow row) throws Exception{ @SuppressWarnings("rawtypes") protected boolean processInsertTable(UpdateTableRow row) throws Exception { Class clazz = getClassForTableName(row.getTableName()); - String className = clazz.getCanonicalName(); - // if any client needs all of this class then the only choice we have is to push everything - if(accessManager.getNumClientsInterestedInWholeClass(className) > 0 /* || true */){ - Set allClassIdPairs = getAllClassIdPairsForTable(row.getTableName()); - for(ClassIDPair classIdPair : allClassIdPairs) { - // We do not use PostCreateHelper here because we are going to - // do all that extra work for the whole class in - // updateReferences. - postPutHelper.postPutObject(classIdPair,null, null, true, null); + if(clazz != null) { + String className = clazz.getCanonicalName(); + + // if any client needs all of this class then the only choice we have is to push everything + if (accessManager.getNumClientsInterestedInWholeClass(className) > 0 /* || true */) { + Set allClassIdPairs = getAllClassIdPairsForTable(row.getTableName()); + for (ClassIDPair classIdPair : allClassIdPairs) { + // We do not use PostCreateHelper here because we are going to + // do all that extra work for the whole class in + // updateReferences. + postPutHelper.postPutObject(classIdPair, null, null, true, null); + } } - } - updateReferences(className); + updateReferences(className); + } return true; } @@ -301,18 +301,22 @@ protected boolean processInsertTable(UpdateTableRow row) throws Exception { @SuppressWarnings("rawtypes") protected boolean processDeleteSingle(UpdateTableRow row) throws Exception{ Class clazz = getClassForTableName(row.getTableName()); - String className = clazz.getCanonicalName(); - - // See if this object is in the cache. If so, it will help us know which related objects to update. - IMappedClassManager mcm = MappedClassManagerFactory.getMappedClassManager(); - MappedClass mappedClass = mcm.getMappedClassByClassName(className); - IDataProvider dataProvider = dataProviderManager.getDataProviderByName(mappedClass.dataProviderName); - ClassIDPair pair = new ClassIDPair(row.getRowId(), className); - IPerceroObject cachedObject = dataProvider.findById(pair, null, false); // We are hoping to find this object in the cache... - handleDeletedObject(cachedObject, clazz, className, row.getRowId()); - - updateReferences(className); + if(clazz != null) { + String className = clazz.getCanonicalName(); + + // See if this object is in the cache. If so, it will help us know which related objects to update. + IMappedClassManager mcm = MappedClassManagerFactory.getMappedClassManager(); + MappedClass mappedClass = mcm.getMappedClassByClassName(className); + IDataProvider dataProvider = dataProviderManager.getDataProviderByName(mappedClass.dataProviderName); + ClassIDPair pair = new ClassIDPair(row.getRowId(), className); + IPerceroObject cachedObject = dataProvider.findById(pair, null, false); // We are hoping to find this object in the cache... + + handleDeletedObject(cachedObject, clazz, className, row.getRowId()); + + updateReferences(className); + } + return true; } @@ -324,48 +328,51 @@ protected boolean processDeleteSingle(UpdateTableRow row) throws Exception{ @SuppressWarnings("rawtypes") protected boolean processDeleteTable(UpdateTableRow row) throws Exception{ Class clazz = getClassForTableName(row.getTableName()); - String className = clazz.getCanonicalName(); - - IMappedClassManager mcm = MappedClassManagerFactory.getMappedClassManager(); - MappedClass mappedClass = mcm.getMappedClassByClassName(className); - IDataProvider dataProvider = dataProviderManager.getDataProviderByName(mappedClass.dataProviderName); - - // Get the list of ALL ID's of this class type that have been accessed. - Set accessedIds = accessManager.getClassAccessJournalIDs(className); - - if (accessedIds != null && !accessedIds.isEmpty()) { - // TODO: If ID "0", then that means someone wants to know about ALL - // records of this type. How do we do this? - - // Get a list of ALL ID's of this class type. - Set allClassIdPairs = getAllClassIdPairsForTable(row.getTableName()); - - // Remove ALL existing/current ID's from our list of accessed ID's. - for(ClassIDPair nextClassIdPair : allClassIdPairs) { - accessedIds.remove(nextClassIdPair.getID()); - } - - // Now we have the list of ID's that have actually been deleted. - for(String id : accessedIds){ - // Find the cached object first so that it is NOT removed if the same object is NOT found in the data store. - IPerceroObject cachedObject = dataProvider.findById(new ClassIDPair(id, className), null, false); - // We will know an object has been deleted IFF it does NOT exist in the data store. - IPerceroObject dataStoreObject = dataProvider.findById(new ClassIDPair(id, className), null, true); - - if (dataStoreObject != null) { - // Object has NOT been deleted. - continue; - } - - handleDeletedObject(cachedObject, clazz, className, id); - } - } - updateReferences(className); + if(clazz != null) { + String className = clazz.getCanonicalName(); + + IMappedClassManager mcm = MappedClassManagerFactory.getMappedClassManager(); + MappedClass mappedClass = mcm.getMappedClassByClassName(className); + IDataProvider dataProvider = dataProviderManager.getDataProviderByName(mappedClass.dataProviderName); + + // Get the list of ALL ID's of this class type that have been accessed. + Set accessedIds = accessManager.getClassAccessJournalIDs(className); + + if (accessedIds != null && !accessedIds.isEmpty()) { + // TODO: If ID "0", then that means someone wants to know about ALL + // records of this type. How do we do this? + + // Get a list of ALL ID's of this class type. + Set allClassIdPairs = getAllClassIdPairsForTable(row.getTableName()); + + // Remove ALL existing/current ID's from our list of accessed ID's. + for (ClassIDPair nextClassIdPair : allClassIdPairs) { + accessedIds.remove(nextClassIdPair.getID()); + } + + // Now we have the list of ID's that have actually been deleted. + for (String id : accessedIds) { + // Find the cached object first so that it is NOT removed if the same object is NOT found in the data store. + IPerceroObject cachedObject = dataProvider.findById(new ClassIDPair(id, className), null, false); + // We will know an object has been deleted IFF it does NOT exist in the data store. + IPerceroObject dataStoreObject = dataProvider.findById(new ClassIDPair(id, className), null, true); + + if (dataStoreObject != null) { + // Object has NOT been deleted. + continue; + } + + handleDeletedObject(cachedObject, clazz, className, id); + } + } + + updateReferences(className); + } return true; } - + @SuppressWarnings("rawtypes") protected void handleDeletedObject(IPerceroObject cachedObject, Class clazz, String className, String id) throws Exception { boolean isShellObject = false; @@ -376,7 +383,7 @@ protected void handleDeletedObject(IPerceroObject cachedObject, Class clazz, Str } cacheManager.handleDeletedObject(cachedObject, className, isShellObject); - + postDeleteHelper.postDeleteObject(new ClassIDPair(id, className), null, null, true); } @@ -384,12 +391,12 @@ protected void handleDeletedObject(IPerceroObject cachedObject, Class clazz, Str protected Set getAllClassIdPairsForTable(String tableName) throws Exception{ Class clazz = getClassForTableName(tableName); String className = clazz.getCanonicalName(); - + IMappedClassManager mcm = MappedClassManagerFactory.getMappedClassManager(); MappedClass mappedClass = mcm.getMappedClassByClassName(className); IDataProvider dataProvider = dataProviderManager.getDataProviderByName(mappedClass.dataProviderName); Set results = dataProvider.getAllClassIdPairsByName(className); - + return results; } @@ -415,13 +422,13 @@ protected void updateReferences(String className){ if (mappedField != null) { // Find all of this type and push down an update to all Set ids = accessManager.getClassAccessJournalIDs(mappedField.getMappedClass().className); - + if (ids.contains("0")) { // If there is a 0 ID in the list, then we need to update ALL records of this type. Integer pageNumber = 0; Integer pageSize = 25; Integer total = -1; - + while (total < 0 || pageNumber * pageSize <= total) { PerceroList objectsToUpdate = mappedField.getMappedClass().getDataProvider().getAllByName(mappedField.getMappedClass().className, pageNumber, pageSize, true, null); pageNumber++; @@ -429,7 +436,7 @@ protected void updateReferences(String className){ if (total <= 0) { break; } - + Iterator itrObjectsToUpdate = objectsToUpdate.iterator(); while (itrObjectsToUpdate.hasNext()) { IPerceroObject nextObjectToUpdate = itrObjectsToUpdate.next(); @@ -487,7 +494,11 @@ public UpdateTableRow getRow(){ DateTime expireThreshold = new DateTime(now - EXPIRATION_TIME); if (StringUtils.hasText(connectionFactory.getStoredProcedureName())) { - row = getStoredProcRow(row, conn, statement, lockId, expireThreshold); + try { + row = getStoredProcRow(row, conn, statement, lockId, expireThreshold); + }catch(Exception e){ + row = getUpdateSelectRow(row, statement, lockId, expireThreshold); + } } else { row = getUpdateSelectRow(row, statement, lockId, expireThreshold); @@ -560,23 +571,29 @@ private UpdateTableRow getStoredProcRow(UpdateTableRow row, } catch(SQLException e){ // return null; logger.warn(e.getMessage(), e); - + // If the stored proc doesn't exist, let's try and create it. - if (StringUtils.hasText(connectionFactory.getStoredProcedureDefinition()) && e.getMessage().toLowerCase().contains(connectionFactory.getStoredProcedureName().toLowerCase()) && e.getMessage().toLowerCase().contains("must be declared")) { + if (StringUtils.hasText(connectionFactory.getStoredProcedureDefinition()) && + e.getMessage().toLowerCase().contains(connectionFactory.getStoredProcedureName().toLowerCase()) + && e.getMessage().toLowerCase().contains("must be declared")) { try { Statement stmtCreateStoredProcedure = conn.createStatement(); - boolean createResult = stmtCreateStoredProcedure.execute(connectionFactory.getStoredProcedureDefinition().replaceAll(":tableName", tableName).replaceAll(":storedProcedureName", connectionFactory.getStoredProcedureName())); + boolean createResult = stmtCreateStoredProcedure.execute(connectionFactory.getStoredProcedureDefinition() + .replaceAll(":tableName", tableName) + .replaceAll(":storedProcedureName", connectionFactory.getStoredProcedureName())); System.out.println(createResult); } catch(SQLSyntaxErrorException ssee) { logger.warn("Unable to create UpdateTable stored procedure: " + ssee.getMessage()); + throw ssee; } catch(Exception e1) { logger.warn("Unable to create UpdateTable stored procedure: " + e1.getMessage()); + throw e1; } } } - + if (updateTableId != null && updateTableId > -1) { String sql = "select * from :tableName where id=:id"; sql = sql.replace(":tableName", tableName); From cd7d368f722ac6131963d9045bbb06f2bf64390d Mon Sep 17 00:00:00 2001 From: Jonathan Samples Date: Mon, 5 Oct 2015 07:27:57 -0700 Subject: [PATCH 008/105] Disabled running tests automatically --- pom.xml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/pom.xml b/pom.xml index b10b5fd..a706955 100644 --- a/pom.xml +++ b/pom.xml @@ -342,7 +342,16 @@ 1.7 + + org.apache.maven.plugins + maven-surefire-plugin + 2.18.1 + + true + + + From 9f066defedd2aa787a54949980cd437a77bbdb32 Mon Sep 17 00:00:00 2001 From: Jonathan Samples Date: Wed, 7 Oct 2015 14:30:18 -0700 Subject: [PATCH 009/105] Fixed a bug that caused all UserRoles to be deleted when logging in via new auth pathway. --- .../java/com/percero/agents/auth/helpers/AccountHelper.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/percero/agents/auth/helpers/AccountHelper.java b/src/main/java/com/percero/agents/auth/helpers/AccountHelper.java index 81984fe..faada08 100644 --- a/src/main/java/com/percero/agents/auth/helpers/AccountHelper.java +++ b/src/main/java/com/percero/agents/auth/helpers/AccountHelper.java @@ -612,7 +612,7 @@ else if (result.getUserId() == null || result.getUserId().isEmpty()) { @SuppressWarnings({ "unchecked" }) public void setupUserRoles(String userId, List serviceUserList) throws Exception { - + if(serviceUserList.size() <= 0) return; // Need this to keep the UserRoles from getting deleted, with the new auth stuff try { ManifestHelper.setManifest(manifest); EntityImplementation userRoleEI = RoleHelper.getUserRoleEntityImplementation(); From 66ab75f21fd1b02b502c8b48a484cc0eff6d51d7 Mon Sep 17 00:00:00 2001 From: Jonathan Samples Date: Wed, 7 Oct 2015 08:58:55 -0700 Subject: [PATCH 010/105] Started removing hibernate deps --- .../agents/auth/helpers/AccountHelper.java | 6 - .../spring/percero-spring-config.xml | 168 +++++++++--------- 2 files changed, 84 insertions(+), 90 deletions(-) diff --git a/src/main/java/com/percero/agents/auth/helpers/AccountHelper.java b/src/main/java/com/percero/agents/auth/helpers/AccountHelper.java index faada08..c1dd7cd 100644 --- a/src/main/java/com/percero/agents/auth/helpers/AccountHelper.java +++ b/src/main/java/com/percero/agents/auth/helpers/AccountHelper.java @@ -53,12 +53,6 @@ public void setAccessManager(IAccessManager value) { accessManager = value; } - @Autowired - protected SessionFactory appSessionFactory; - public void setAppSessionFactory(SessionFactory value) { - appSessionFactory = value; - } - @Autowired protected ISyncAgentService syncAgentService; public void setSyncAgentService(ISyncAgentService value) { diff --git a/src/main/resources/spring/percero-spring-config.xml b/src/main/resources/spring/percero-spring-config.xml index 57f1f8d..612a478 100644 --- a/src/main/resources/spring/percero-spring-config.xml +++ b/src/main/resources/spring/percero-spring-config.xml @@ -241,89 +241,89 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - hibernate.dialect=org.hibernate.dialect.Oracle10gDialect - - - - hibernate.show_sql=false - hibernate.format_sql=false - - hibernate.connection.aggressive_release=true - hibernate.jdbc.batch_size=20 - hibernate.connection.autocommit=false - hibernate.connection.autoReconnect=true - hibernate.enable_lazy_load_no_trans=true - - - - - $pf{domain.packageToScan} - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + From addfe9b852b2fa22a16c9814c96a0276246d6247 Mon Sep 17 00:00:00 2001 From: Jonathan Samples Date: Wed, 7 Oct 2015 10:19:07 -0700 Subject: [PATCH 011/105] More extracting hibernate --- .../agents/auth/helpers/AccountHelper.java | 296 ++++++++---------- .../auth/helpers/UserIdentifierHelper.java | 23 ++ 2 files changed, 151 insertions(+), 168 deletions(-) create mode 100644 src/main/java/com/percero/agents/auth/helpers/UserIdentifierHelper.java diff --git a/src/main/java/com/percero/agents/auth/helpers/AccountHelper.java b/src/main/java/com/percero/agents/auth/helpers/AccountHelper.java index c1dd7cd..92eef4e 100644 --- a/src/main/java/com/percero/agents/auth/helpers/AccountHelper.java +++ b/src/main/java/com/percero/agents/auth/helpers/AccountHelper.java @@ -4,8 +4,6 @@ import com.percero.agents.auth.services.IAuthService; import com.percero.agents.auth.vo.*; import com.percero.agents.sync.access.IAccessManager; -import com.percero.agents.sync.dao.DAORegistry; -import com.percero.agents.sync.dao.IDataAccessObject; import com.percero.agents.sync.helpers.PostCreateHelper; import com.percero.agents.sync.helpers.PostDeleteHelper; import com.percero.agents.sync.helpers.PostPutHelper; @@ -18,7 +16,10 @@ import com.percero.framework.bl.ManifestHelper; import com.percero.framework.vo.IPerceroObject; import org.apache.log4j.Logger; -import org.hibernate.*; +import org.hibernate.HibernateException; +import org.hibernate.NonUniqueResultException; +import org.hibernate.Query; +import org.hibernate.Session; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; import org.springframework.util.StringUtils; @@ -159,48 +160,18 @@ public Principal authenticateOAuth(String regAppKey, String svcOAuthKey, String public IUserAnchor validateUser(String regAppKey, String userId, IAuthService authService) throws HibernateException { IUserAnchor result = null; - Session s = appSessionFactory.openSession(); try { ManifestHelper.setManifest(manifest); - //Class userAnchorClass = ManifestHelper.findImplementingClass(IUserAnchor.class); - EntityImplementation userAnchorEI = null; - List userAnchorMappedClasses = MappedClass.findEntityImplementation(IUserAnchor.class); - if (userAnchorMappedClasses.size() > 0) { - userAnchorEI = userAnchorMappedClasses.get(0); - } + IUserAnchor foundUserAnchor = UserAnchorHelper.getUserAnchor(userId); - if (userAnchorEI != null) { - //IMappedClassManager mcm = MappedClassManagerFactory.getMappedClassManager(); - MappedClass mc = userAnchorEI.mappedClass; - if (mc != null) { - IUserAnchor foundUserAnchor = null; - try { - IDataAccessObject dao = (IDataAccessObject) DAORegistry.getInstance().getDataAccessObject(mc.className); - Class userAnchorClass = Class.forName(mc.className); - IUserAnchor example = (IUserAnchor) userAnchorClass.newInstance(); - example.setUserId(userId); - List list = dao.findByExample((IPerceroObject)example, null, null, false); - if(list.size() > 0) - foundUserAnchor = (IUserAnchor) list.get(0); -// String userAnchorQueryString = "SELECT ua FROM " + mc.tableName + " ua WHERE ua.userId=:userId"; -// Query userAnchorQuery = s.createQuery(userAnchorQueryString); -// userAnchorQuery.setString("userId", userId); -// IUserAnchor foundUserAnchor = (IUserAnchor) userAnchorQuery.uniqueResult(); - }catch(Exception e){} - - if (foundUserAnchor == null) { - result = handleUserAnchorNotFound(regAppKey, userId, authService); - } else { - handleUserAnchorFound(regAppKey, userId, authService, foundUserAnchor); - result = foundUserAnchor; - } - } + if (foundUserAnchor == null) { + result = handleUserAnchorNotFound(regAppKey, userId, authService); + } else { + handleUserAnchorFound(regAppKey, userId, authService, foundUserAnchor); + result = foundUserAnchor; } } catch (NonUniqueResultException nre) { log.error("More than one UserAnchor objects found for userId " + userId, nre); - } finally { - if (s != null && s.isOpen()) - s.close(); } return result; } @@ -398,81 +369,70 @@ protected PropertyInterface getEntityPropertyInterfaceAnnotation(Class entityInt }*/ protected IUserAnchor addOrUpdateUserAnchorFromServiceUserList(String userId, List serviceUserList, IUserAnchor result) { - Session s = null; try { + // TODO: why are we doing this everywhere? ManifestHelper.setManifest(manifest); - //Set userIdentifierClasses = ManifestHelper.findImplementingClasses(IUserIdentifier.class); - //Class userAnchorClass = ManifestHelper.findImplementingClass(IUserAnchor.class); - EntityImplementation eiUserAnchor = null; - List userAnchorEntityImplementations = MappedClass.findEntityImplementation(IUserAnchor.class); - if (userAnchorEntityImplementations.size() > 0) { - eiUserAnchor = userAnchorEntityImplementations.get(0); - } + + EntityImplementation eiUserAnchor = UserAnchorHelper.getUserAnchorEntityImplementation(); List userIdentifierEntityImplementations = MappedClass.findEntityImplementation(IUserIdentifier.class); - if (userIdentifierEntityImplementations.size() > 0) { - List identifiersToSave = new ArrayList(); + List identifiersToSave = new ArrayList(); - Iterator itrUserIdentifierEntityImplementations = userIdentifierEntityImplementations.iterator(); - while (itrUserIdentifierEntityImplementations.hasNext()) { - EntityImplementation userIdentifierEntityImplementation = itrUserIdentifierEntityImplementations.next(); + Iterator itrUserIdentifierEntityImplementations = userIdentifierEntityImplementations.iterator(); + while (itrUserIdentifierEntityImplementations.hasNext()) { + EntityImplementation userIdentifierEntityImplementation = itrUserIdentifierEntityImplementations.next(); - //MappedField userAnchorMappedField = userIdentifierEntityImplementation.getMappedFieldByName(userIdentifierAnnotation.userAnchorFieldName()); - PropertyImplementation userIdentifierPropImpl = userIdentifierEntityImplementation.findPropertyImplementationByName(IUserIdentifier.USER_IDENTIFIER_FIELD_NAME); - RelationshipImplementation userAnchorRelImpl = userIdentifierEntityImplementation.findRelationshipImplementationBySourceVarName(IUserIdentifier.USER_ANCHOR_FIELD_NAME); - //IUserIdentifierA userIdentifierAnnotation = getUserIdentifierAnnotation(userIdentifierEntityImplementation); - if (userAnchorRelImpl != null) { + PropertyImplementation userIdentifierPropImpl = userIdentifierEntityImplementation.findPropertyImplementationByName(IUserIdentifier.USER_IDENTIFIER_FIELD_NAME); + RelationshipImplementation userAnchorRelImpl = userIdentifierEntityImplementation.findRelationshipImplementationBySourceVarName(IUserIdentifier.USER_ANCHOR_FIELD_NAME); - if (s == null) { - s = appSessionFactory.openSession(); - } + if (userAnchorRelImpl != null) { - // Get this userAnchor's identifier(s). - String userIdentifierQueryString = "SELECT ui FROM " + userIdentifierEntityImplementation.mappedClass.tableName - + " ui WHERE ui." + userIdentifierPropImpl.mappedField.getField().getName() + "=:value AND (ui." + - userAnchorRelImpl.sourceMappedField.getField().getName() + " IS NULL OR ui." + userAnchorRelImpl.sourceMappedField.getField().getName() + - " IN (SELECT ua FROM " + eiUserAnchor.mappedClass.tableName + " ua WHERE (ua.userId=null OR ua.userId='' OR ua.userId=:userId)))"; - for (ServiceUser nextServiceUser : serviceUserList) { - for (ServiceIdentifier nextIdentifier : nextServiceUser.getIdentifiers()) { - try { - // Make sure this Identifier is in the same paradigm. - String paradigm = null; - Iterator itrParams = userIdentifierPropImpl.params.iterator(); - while (itrParams.hasNext()) { - PropertyImplementationParam nextParam = itrParams.next(); - if (nextParam.name.equalsIgnoreCase(IUserIdentifier.PARADIGM_PARAM_NAME)) { - paradigm = nextParam.value; - break; - } - } - if (nextIdentifier.getParadigm() == null || !nextIdentifier.getParadigm().equalsIgnoreCase(paradigm)) { - continue; + // Get this userAnchor's identifier(s). + String userIdentifierQueryString = "SELECT ui FROM " + userIdentifierEntityImplementation.mappedClass.tableName + + " ui WHERE ui." + userIdentifierPropImpl.mappedField.getField().getName() + "=:value AND (ui." + + userAnchorRelImpl.sourceMappedField.getField().getName() + " IS NULL OR ui." + userAnchorRelImpl.sourceMappedField.getField().getName() + + " IN (SELECT ua FROM " + eiUserAnchor.mappedClass.tableName + " ua WHERE (ua.userId=null OR ua.userId='' OR ua.userId=:userId)))"; + for (ServiceUser nextServiceUser : serviceUserList) { + for (ServiceIdentifier nextIdentifier : nextServiceUser.getIdentifiers()) { + try { + // Make sure this Identifier is in the same paradigm. + String paradigm = null; + Iterator itrParams = userIdentifierPropImpl.params.iterator(); + while (itrParams.hasNext()) { + PropertyImplementationParam nextParam = itrParams.next(); + if (nextParam.name.equalsIgnoreCase(IUserIdentifier.PARADIGM_PARAM_NAME)) { + paradigm = nextParam.value; + break; } + } + if (nextIdentifier.getParadigm() == null || !nextIdentifier.getParadigm().equalsIgnoreCase(paradigm)) { + continue; + } - // Look for this existing identifier. - Query userIdentifierQuery = s.createQuery(userIdentifierQueryString); - userIdentifierQuery.setString("userId", userId); - userIdentifierQuery.setString("value", nextIdentifier.getValue()); - IUserIdentifier foundIdentifier = (IUserIdentifier) userIdentifierQuery.uniqueResult(); - - if (foundIdentifier != null) { - // If the email does not have a Person, then associate email with this Person. - foundIdentifier = (IUserIdentifier) SyncHibernateUtils.cleanObject(foundIdentifier, s); - IUserAnchor existingUserAnchor = (IUserAnchor) SyncHibernateUtils.cleanObject(userAnchorRelImpl.sourceMappedField.getGetter().invoke(foundIdentifier), s); - if (existingUserAnchor == null) { - if (result != null) { - userAnchorRelImpl.sourceMappedField.getSetter().invoke(foundIdentifier, result); - syncAgentService.systemPutObject((IPerceroObject) foundIdentifier, null, null, null, true); - } - else { - identifiersToSave.add(foundIdentifier); - } - } else if (result == null) { - result = existingUserAnchor; + // Look for this existing identifier. + Query userIdentifierQuery = s.createQuery(userIdentifierQueryString); + userIdentifierQuery.setString("userId", userId); + userIdentifierQuery.setString("value", nextIdentifier.getValue()); + IUserIdentifier foundIdentifier = (IUserIdentifier) userIdentifierQuery.uniqueResult(); + + if (foundIdentifier != null) { + // If the email does not have a Person, then associate email with this Person. + foundIdentifier = (IUserIdentifier) SyncHibernateUtils.cleanObject(foundIdentifier, s); + IUserAnchor existingUserAnchor = (IUserAnchor) SyncHibernateUtils.cleanObject(userAnchorRelImpl.sourceMappedField.getGetter().invoke(foundIdentifier), s); + if (existingUserAnchor == null) { + if (result != null) { + userAnchorRelImpl.sourceMappedField.getSetter().invoke(foundIdentifier, result); + syncAgentService.systemPutObject((IPerceroObject) foundIdentifier, null, null, null, true); + } + else { + identifiersToSave.add(foundIdentifier); } - } else { // Identifier object NOT found, need to add. - // Identifier.identifierValue will be set later, once that is determined. - IUserIdentifier newIdentifier = (IUserIdentifier) userIdentifierEntityImplementation.mappedClass.clazz.newInstance(); + } else if (result == null) { + result = existingUserAnchor; + } + } else { // Identifier object NOT found, need to add. + // Identifier.identifierValue will be set later, once that is determined. + IUserIdentifier newIdentifier = (IUserIdentifier) userIdentifierEntityImplementation.mappedClass.clazz.newInstance(); /* IUserIdentifierA userIdentifierA = null; Class nextClass = newIdentifier.getClass(); @@ -488,65 +448,65 @@ protected IUserAnchor addOrUpdateUserAnchorFromServiceUserList(String userId, Li MappedField mappedField = mappedClass.getExternalizeFieldByName(userIdentifierA.userIdentifierFieldName()); mappedField.getSetter().invoke(newIdentifier, nextIdentifier.getValue()); */ - userIdentifierPropImpl.mappedField.getSetter().invoke(newIdentifier, nextIdentifier.getValue()); - identifiersToSave.add(newIdentifier); - } - } catch(NonUniqueResultException nure) { - log.warn("Non-unique User Identifier: " + nextIdentifier, nure); + userIdentifierPropImpl.mappedField.getSetter().invoke(newIdentifier, nextIdentifier.getValue()); + identifiersToSave.add(newIdentifier); } + } catch(NonUniqueResultException nure) { + log.warn("Non-unique User Identifier: " + nextIdentifier, nure); } } } } + } - if (s != null && s.isOpen()) { - s.close(); - } + if (s != null && s.isOpen()) { + s.close(); + } - // Need to create a new IUserAnchor - if (result == null) { - result = (IUserAnchor) eiUserAnchor.mappedClass.clazz.newInstance(); - ((IPerceroObject) result).setID(UUID.randomUUID().toString()); - result.setUserId(userId); - if (serviceUserList.size() > 0) { - /** - IUserAnchorA userAnchorAnnotation = getUserAnchorAnnotation(userAnchorClass); - - if (userAnchorAnnotation.firstNameFieldName() != null && !userAnchorAnnotation.firstNameFieldName().isEmpty()) - firstNameMappedField = mcUserAnchor.getMappedFieldByName(userAnchorAnnotation.firstNameFieldName()); - MappedField lastNameMappedField = null; - if (userAnchorAnnotation.lastNameFieldName() != null && !userAnchorAnnotation.lastNameFieldName().isEmpty()) - lastNameMappedField = mcUserAnchor.getMappedFieldByName(userAnchorAnnotation.lastNameFieldName()); - */ - - // Find field that has firstName and lastName PropertyImplementations. - PropertyImplementation firstNamePropImpl = eiUserAnchor.findPropertyImplementationByName(IUserAnchor.FIRST_NAME_FIELD); - PropertyImplementation lastNamePropImpl = eiUserAnchor.findPropertyImplementationByName(IUserAnchor.LAST_NAME_FIELD); - - ServiceUser firstServiceUser = serviceUserList.get(0); - - if (firstNamePropImpl != null) - firstNamePropImpl.mappedField.getSetter().invoke(result, firstServiceUser.getFirstName()); - if (lastNamePropImpl != null) - lastNamePropImpl.mappedField.getSetter().invoke(result, firstServiceUser.getLastName()); - } + // Need to create a new IUserAnchor + if (result == null) { + result = (IUserAnchor) eiUserAnchor.mappedClass.clazz.newInstance(); + ((IPerceroObject) result).setID(UUID.randomUUID().toString()); + result.setUserId(userId); + if (serviceUserList.size() > 0) { + /** + IUserAnchorA userAnchorAnnotation = getUserAnchorAnnotation(userAnchorClass); - syncAgentService.systemCreateObject((IPerceroObject) result, null); + if (userAnchorAnnotation.firstNameFieldName() != null && !userAnchorAnnotation.firstNameFieldName().isEmpty()) + firstNameMappedField = mcUserAnchor.getMappedFieldByName(userAnchorAnnotation.firstNameFieldName()); + MappedField lastNameMappedField = null; + if (userAnchorAnnotation.lastNameFieldName() != null && !userAnchorAnnotation.lastNameFieldName().isEmpty()) + lastNameMappedField = mcUserAnchor.getMappedFieldByName(userAnchorAnnotation.lastNameFieldName()); + */ - } - else if (result.getUserId() == null || result.getUserId().isEmpty()) { - result.setUserId(userId); + // Find field that has firstName and lastName PropertyImplementations. + PropertyImplementation firstNamePropImpl = eiUserAnchor.findPropertyImplementationByName(IUserAnchor.FIRST_NAME_FIELD); + PropertyImplementation lastNamePropImpl = eiUserAnchor.findPropertyImplementationByName(IUserAnchor.LAST_NAME_FIELD); - syncAgentService.systemPutObject((IPerceroObject) result, null, null, null, true); + ServiceUser firstServiceUser = serviceUserList.get(0); + + if (firstNamePropImpl != null) + firstNamePropImpl.mappedField.getSetter().invoke(result, firstServiceUser.getFirstName()); + if (lastNamePropImpl != null) + lastNamePropImpl.mappedField.getSetter().invoke(result, firstServiceUser.getLastName()); } - for (IUserIdentifier nextUserIdentifier : identifiersToSave) { - // Set Email.person now that we have a firm handle on that Person object. - Boolean isNewObject = false; - if (((IPerceroObject)nextUserIdentifier).getID() == null || ((IPerceroObject)nextUserIdentifier).getID().isEmpty()) { - isNewObject = true; - ((IPerceroObject)nextUserIdentifier).setID(UUID.randomUUID().toString()); - } + syncAgentService.systemCreateObject((IPerceroObject) result, null); + + } + else if (result.getUserId() == null || result.getUserId().isEmpty()) { + result.setUserId(userId); + + syncAgentService.systemPutObject((IPerceroObject) result, null, null, null, true); + } + + for (IUserIdentifier nextUserIdentifier : identifiersToSave) { + // Set Email.person now that we have a firm handle on that Person object. + Boolean isNewObject = false; + if (((IPerceroObject)nextUserIdentifier).getID() == null || ((IPerceroObject)nextUserIdentifier).getID().isEmpty()) { + isNewObject = true; + ((IPerceroObject)nextUserIdentifier).setID(UUID.randomUUID().toString()); + } /* IUserIdentifierA userIdentifierAnnotation = getUserIdentifierAnnotation(nextUserIdentifier.getClass()); @@ -562,32 +522,32 @@ else if (result.getUserId() == null || result.getUserId().isEmpty()) { } */ - IMappedClassManager mcm = MappedClassManagerFactory.getMappedClassManager(); - MappedClass mc = mcm.getMappedClassByClassName(nextUserIdentifier.getClass().getName()); - EntityImplementation entityImpl = mc.entityImplementations.get(IUserIdentifier.class); - if (entityImpl != null) { - RelationshipImplementation relImpl = entityImpl.findRelationshipImplementationBySourceVarName(IUserIdentifier.USER_ANCHOR_FIELD_NAME); - if (relImpl != null) { - relImpl.sourceMappedField.getSetter().invoke(nextUserIdentifier, result); - if (isNewObject) { - syncAgentService.systemCreateObject((IPerceroObject) nextUserIdentifier, null); - } - else { - syncAgentService.systemPutObject((IPerceroObject) nextUserIdentifier, null, null, null, true); - } + IMappedClassManager mcm = MappedClassManagerFactory.getMappedClassManager(); + MappedClass mc = mcm.getMappedClassByClassName(nextUserIdentifier.getClass().getName()); + EntityImplementation entityImpl = mc.entityImplementations.get(IUserIdentifier.class); + if (entityImpl != null) { + RelationshipImplementation relImpl = entityImpl.findRelationshipImplementationBySourceVarName(IUserIdentifier.USER_ANCHOR_FIELD_NAME); + if (relImpl != null) { + relImpl.sourceMappedField.getSetter().invoke(nextUserIdentifier, result); + if (isNewObject) { + syncAgentService.systemCreateObject((IPerceroObject) nextUserIdentifier, null); } else { - log.warn("No UserAnchor found for IUserIdentifier class " + nextUserIdentifier.getClass().getCanonicalName()); + syncAgentService.systemPutObject((IPerceroObject) nextUserIdentifier, null, null, null, true); } } + else { + log.warn("No UserAnchor found for IUserIdentifier class " + nextUserIdentifier.getClass().getCanonicalName()); + } } + } - if (result != null) { - setupUserRoles(userId, serviceUserList); - return result; - } + if (result != null) { + setupUserRoles(userId, serviceUserList); + return result; } + } catch (IllegalArgumentException e) { log.error(e.getMessage(), e); } catch (IllegalAccessException e) { diff --git a/src/main/java/com/percero/agents/auth/helpers/UserIdentifierHelper.java b/src/main/java/com/percero/agents/auth/helpers/UserIdentifierHelper.java new file mode 100644 index 0000000..c9a705c --- /dev/null +++ b/src/main/java/com/percero/agents/auth/helpers/UserIdentifierHelper.java @@ -0,0 +1,23 @@ +package com.percero.agents.auth.helpers; + +import com.percero.agents.auth.vo.IUserAnchor; +import com.percero.agents.auth.vo.IUserIdentifier; + +import java.util.ArrayList; +import java.util.List; + +/** + * Created by jonnysamps on 10/7/15. + */ +public class UserIdentifierHelper { + public static List getUserIdentifiersForUser(IUserAnchor user){ + List result = new ArrayList<>(); + + return result; + } + + public static IUserIdentifier getUserIdentifierForUserAndValue(String userId, String value){ + + } +} + From 15f0ca3ff9c3c2d9cb5455995df30aedf148f3df Mon Sep 17 00:00:00 2001 From: Jonathan Samples Date: Wed, 7 Oct 2015 16:14:56 -0700 Subject: [PATCH 012/105] A little further --- .../agents/auth/helpers/AccountHelper.java | 103 ++++++++---------- .../auth/helpers/UserIdentifierHelper.java | 30 ++++- 2 files changed, 76 insertions(+), 57 deletions(-) diff --git a/src/main/java/com/percero/agents/auth/helpers/AccountHelper.java b/src/main/java/com/percero/agents/auth/helpers/AccountHelper.java index 92eef4e..32e87c4 100644 --- a/src/main/java/com/percero/agents/auth/helpers/AccountHelper.java +++ b/src/main/java/com/percero/agents/auth/helpers/AccountHelper.java @@ -385,54 +385,53 @@ protected IUserAnchor addOrUpdateUserAnchorFromServiceUserList(String userId, Li PropertyImplementation userIdentifierPropImpl = userIdentifierEntityImplementation.findPropertyImplementationByName(IUserIdentifier.USER_IDENTIFIER_FIELD_NAME); RelationshipImplementation userAnchorRelImpl = userIdentifierEntityImplementation.findRelationshipImplementationBySourceVarName(IUserIdentifier.USER_ANCHOR_FIELD_NAME); - if (userAnchorRelImpl != null) { - // Get this userAnchor's identifier(s). - String userIdentifierQueryString = "SELECT ui FROM " + userIdentifierEntityImplementation.mappedClass.tableName - + " ui WHERE ui." + userIdentifierPropImpl.mappedField.getField().getName() + "=:value AND (ui." + - userAnchorRelImpl.sourceMappedField.getField().getName() + " IS NULL OR ui." + userAnchorRelImpl.sourceMappedField.getField().getName() + - " IN (SELECT ua FROM " + eiUserAnchor.mappedClass.tableName + " ua WHERE (ua.userId=null OR ua.userId='' OR ua.userId=:userId)))"; - for (ServiceUser nextServiceUser : serviceUserList) { - for (ServiceIdentifier nextIdentifier : nextServiceUser.getIdentifiers()) { - try { - // Make sure this Identifier is in the same paradigm. - String paradigm = null; - Iterator itrParams = userIdentifierPropImpl.params.iterator(); - while (itrParams.hasNext()) { - PropertyImplementationParam nextParam = itrParams.next(); - if (nextParam.name.equalsIgnoreCase(IUserIdentifier.PARADIGM_PARAM_NAME)) { - paradigm = nextParam.value; - break; - } - } - if (nextIdentifier.getParadigm() == null || !nextIdentifier.getParadigm().equalsIgnoreCase(paradigm)) { - continue; + // Get this userAnchor's identifier(s). + String userIdentifierQueryString = "SELECT ui FROM " + userIdentifierEntityImplementation.mappedClass.tableName + + " ui WHERE ui." + userIdentifierPropImpl.mappedField.getField().getName() + "=:value AND (ui." + + userAnchorRelImpl.sourceMappedField.getField().getName() + " IS NULL OR ui." + userAnchorRelImpl.sourceMappedField.getField().getName() + + " IN (SELECT ua FROM " + eiUserAnchor.mappedClass.tableName + " ua WHERE (ua.userId=null OR ua.userId='' OR ua.userId=:userId)))"; + for (ServiceUser nextServiceUser : serviceUserList) { + for (ServiceIdentifier nextIdentifier : nextServiceUser.getIdentifiers()) { + try { + // Make sure this Identifier is in the same paradigm. + String paradigm = null; + Iterator itrParams = userIdentifierPropImpl.params.iterator(); + while (itrParams.hasNext()) { + PropertyImplementationParam nextParam = itrParams.next(); + if (nextParam.name.equalsIgnoreCase(IUserIdentifier.PARADIGM_PARAM_NAME)) { + paradigm = nextParam.value; + break; } + } + if (nextIdentifier.getParadigm() == null || !nextIdentifier.getParadigm().equalsIgnoreCase(paradigm)) { + continue; + } - // Look for this existing identifier. - Query userIdentifierQuery = s.createQuery(userIdentifierQueryString); - userIdentifierQuery.setString("userId", userId); - userIdentifierQuery.setString("value", nextIdentifier.getValue()); - IUserIdentifier foundIdentifier = (IUserIdentifier) userIdentifierQuery.uniqueResult(); - - if (foundIdentifier != null) { - // If the email does not have a Person, then associate email with this Person. - foundIdentifier = (IUserIdentifier) SyncHibernateUtils.cleanObject(foundIdentifier, s); - IUserAnchor existingUserAnchor = (IUserAnchor) SyncHibernateUtils.cleanObject(userAnchorRelImpl.sourceMappedField.getGetter().invoke(foundIdentifier), s); - if (existingUserAnchor == null) { - if (result != null) { - userAnchorRelImpl.sourceMappedField.getSetter().invoke(foundIdentifier, result); - syncAgentService.systemPutObject((IPerceroObject) foundIdentifier, null, null, null, true); - } - else { - identifiersToSave.add(foundIdentifier); - } - } else if (result == null) { - result = existingUserAnchor; + // Look for this existing identifier. + Query userIdentifierQuery = s.createQuery(userIdentifierQueryString); + userIdentifierQuery.setString("userId", userId); + userIdentifierQuery.setString("value", nextIdentifier.getValue()); + IUserIdentifier foundIdentifier = (IUserIdentifier) userIdentifierQuery.uniqueResult(); + + if (foundIdentifier != null) { + // If the email does not have a Person, then associate email with this Person. + foundIdentifier = (IUserIdentifier) SyncHibernateUtils.cleanObject(foundIdentifier, s); + IUserAnchor existingUserAnchor = (IUserAnchor) SyncHibernateUtils.cleanObject(userAnchorRelImpl.sourceMappedField.getGetter().invoke(foundIdentifier), s); + if (existingUserAnchor == null) { + if (result != null) { + userAnchorRelImpl.sourceMappedField.getSetter().invoke(foundIdentifier, result); + syncAgentService.systemPutObject((IPerceroObject) foundIdentifier, null, null, null, true); } - } else { // Identifier object NOT found, need to add. - // Identifier.identifierValue will be set later, once that is determined. - IUserIdentifier newIdentifier = (IUserIdentifier) userIdentifierEntityImplementation.mappedClass.clazz.newInstance(); + else { + identifiersToSave.add(foundIdentifier); + } + } else if (result == null) { + result = existingUserAnchor; + } + } else { // Identifier object NOT found, need to add. + // Identifier.identifierValue will be set later, once that is determined. + IUserIdentifier newIdentifier = (IUserIdentifier) userIdentifierEntityImplementation.mappedClass.clazz.newInstance(); /* IUserIdentifierA userIdentifierA = null; Class nextClass = newIdentifier.getClass(); @@ -448,21 +447,16 @@ protected IUserAnchor addOrUpdateUserAnchorFromServiceUserList(String userId, Li MappedField mappedField = mappedClass.getExternalizeFieldByName(userIdentifierA.userIdentifierFieldName()); mappedField.getSetter().invoke(newIdentifier, nextIdentifier.getValue()); */ - userIdentifierPropImpl.mappedField.getSetter().invoke(newIdentifier, nextIdentifier.getValue()); - identifiersToSave.add(newIdentifier); - } - } catch(NonUniqueResultException nure) { - log.warn("Non-unique User Identifier: " + nextIdentifier, nure); + userIdentifierPropImpl.mappedField.getSetter().invoke(newIdentifier, nextIdentifier.getValue()); + identifiersToSave.add(newIdentifier); } + } catch(NonUniqueResultException nure) { + log.warn("Non-unique User Identifier: " + nextIdentifier, nure); } } } } - if (s != null && s.isOpen()) { - s.close(); - } - // Need to create a new IUserAnchor if (result == null) { result = (IUserAnchor) eiUserAnchor.mappedClass.clazz.newInstance(); @@ -556,9 +550,6 @@ else if (result.getUserId() == null || result.getUserId().isEmpty()) { log.error(e.getMessage(), e); } catch (Exception e) { log.error(e.getMessage(), e); - } finally { - if (s != null && s.isOpen()) - s.close(); } return null; diff --git a/src/main/java/com/percero/agents/auth/helpers/UserIdentifierHelper.java b/src/main/java/com/percero/agents/auth/helpers/UserIdentifierHelper.java index c9a705c..1386da2 100644 --- a/src/main/java/com/percero/agents/auth/helpers/UserIdentifierHelper.java +++ b/src/main/java/com/percero/agents/auth/helpers/UserIdentifierHelper.java @@ -2,7 +2,14 @@ import com.percero.agents.auth.vo.IUserAnchor; import com.percero.agents.auth.vo.IUserIdentifier; +import com.percero.agents.sync.dao.DAORegistry; +import com.percero.agents.sync.dao.IDataAccessObject; +import com.percero.agents.sync.metadata.EntityImplementation; +import com.percero.agents.sync.metadata.PropertyImplementation; +import com.percero.agents.sync.metadata.RelationshipImplementation; +import com.percero.framework.vo.IPerceroObject; +import java.lang.reflect.Method; import java.util.ArrayList; import java.util.List; @@ -16,8 +23,29 @@ public static List getUserIdentifiersForUser(IUserAnchor user){ return result; } - public static IUserIdentifier getUserIdentifierForUserAndValue(String userId, String value){ + public static IUserIdentifier getUserIdentifierForUserAndValue(EntityImplementation ei, IUserAnchor userAnchor, String value){ + IUserIdentifier result = null; + IDataAccessObject dao = + (IDataAccessObject) DAORegistry.getInstance().getDataAccessObject(ei.mappedClass.className); + PropertyImplementation userIdentifierPropImpl = ei.findPropertyImplementationByName(IUserIdentifier.USER_IDENTIFIER_FIELD_NAME); + RelationshipImplementation userAnchorRelImpl = ei.findRelationshipImplementationBySourceVarName(IUserIdentifier.USER_ANCHOR_FIELD_NAME); + + try { + IUserIdentifier example = (IUserIdentifier) ei.mappedClass.clazz.newInstance(); + Method valueSetter = userIdentifierPropImpl.mappedField.getSetter(); + valueSetter.invoke(example, value); + Method userSetter = userAnchorRelImpl.sourceMappedField.getSetter(); + userSetter.invoke(example, userAnchor); + List list = dao.findByExample((IPerceroObject) example, null, null, false); + if(list.size() > 0) + result = (IUserIdentifier) list.get(0); + + }catch(Exception e){} + + return result; } + + } From 6dffa60c6307865c0e6c877f2094f69e0a99f1f1 Mon Sep 17 00:00:00 2001 From: Jonathan Samples Date: Wed, 7 Oct 2015 17:35:34 -0700 Subject: [PATCH 013/105] Got hibernate out of AccountHelper --- .../agents/auth/helpers/AccountHelper.java | 35 +++---------------- 1 file changed, 5 insertions(+), 30 deletions(-) diff --git a/src/main/java/com/percero/agents/auth/helpers/AccountHelper.java b/src/main/java/com/percero/agents/auth/helpers/AccountHelper.java index 32e87c4..385c17b 100644 --- a/src/main/java/com/percero/agents/auth/helpers/AccountHelper.java +++ b/src/main/java/com/percero/agents/auth/helpers/AccountHelper.java @@ -7,7 +7,6 @@ import com.percero.agents.sync.helpers.PostCreateHelper; import com.percero.agents.sync.helpers.PostDeleteHelper; import com.percero.agents.sync.helpers.PostPutHelper; -import com.percero.agents.sync.hibernate.SyncHibernateUtils; import com.percero.agents.sync.metadata.*; import com.percero.agents.sync.services.IDataProviderManager; import com.percero.agents.sync.services.ISyncAgentService; @@ -18,8 +17,6 @@ import org.apache.log4j.Logger; import org.hibernate.HibernateException; import org.hibernate.NonUniqueResultException; -import org.hibernate.Query; -import org.hibernate.Session; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; import org.springframework.util.StringUtils; @@ -385,12 +382,6 @@ protected IUserAnchor addOrUpdateUserAnchorFromServiceUserList(String userId, Li PropertyImplementation userIdentifierPropImpl = userIdentifierEntityImplementation.findPropertyImplementationByName(IUserIdentifier.USER_IDENTIFIER_FIELD_NAME); RelationshipImplementation userAnchorRelImpl = userIdentifierEntityImplementation.findRelationshipImplementationBySourceVarName(IUserIdentifier.USER_ANCHOR_FIELD_NAME); - - // Get this userAnchor's identifier(s). - String userIdentifierQueryString = "SELECT ui FROM " + userIdentifierEntityImplementation.mappedClass.tableName - + " ui WHERE ui." + userIdentifierPropImpl.mappedField.getField().getName() + "=:value AND (ui." + - userAnchorRelImpl.sourceMappedField.getField().getName() + " IS NULL OR ui." + userAnchorRelImpl.sourceMappedField.getField().getName() + - " IN (SELECT ua FROM " + eiUserAnchor.mappedClass.tableName + " ua WHERE (ua.userId=null OR ua.userId='' OR ua.userId=:userId)))"; for (ServiceUser nextServiceUser : serviceUserList) { for (ServiceIdentifier nextIdentifier : nextServiceUser.getIdentifiers()) { try { @@ -409,27 +400,11 @@ protected IUserAnchor addOrUpdateUserAnchorFromServiceUserList(String userId, Li } // Look for this existing identifier. - Query userIdentifierQuery = s.createQuery(userIdentifierQueryString); - userIdentifierQuery.setString("userId", userId); - userIdentifierQuery.setString("value", nextIdentifier.getValue()); - IUserIdentifier foundIdentifier = (IUserIdentifier) userIdentifierQuery.uniqueResult(); - - if (foundIdentifier != null) { - // If the email does not have a Person, then associate email with this Person. - foundIdentifier = (IUserIdentifier) SyncHibernateUtils.cleanObject(foundIdentifier, s); - IUserAnchor existingUserAnchor = (IUserAnchor) SyncHibernateUtils.cleanObject(userAnchorRelImpl.sourceMappedField.getGetter().invoke(foundIdentifier), s); - if (existingUserAnchor == null) { - if (result != null) { - userAnchorRelImpl.sourceMappedField.getSetter().invoke(foundIdentifier, result); - syncAgentService.systemPutObject((IPerceroObject) foundIdentifier, null, null, null, true); - } - else { - identifiersToSave.add(foundIdentifier); - } - } else if (result == null) { - result = existingUserAnchor; - } - } else { // Identifier object NOT found, need to add. + IUserIdentifier foundIdentifier = (IUserIdentifier) UserIdentifierHelper.getUserIdentifierForUserAndValue(userIdentifierEntityImplementation,result,nextIdentifier.getValue()); + + + if (foundIdentifier == null) { + // Identifier object NOT found, need to add. // Identifier.identifierValue will be set later, once that is determined. IUserIdentifier newIdentifier = (IUserIdentifier) userIdentifierEntityImplementation.mappedClass.clazz.newInstance(); /* From 3e8b2ed8ff1473163d48323fe97371c3350c3229 Mon Sep 17 00:00:00 2001 From: Jonathan Samples Date: Wed, 7 Oct 2015 21:09:00 -0700 Subject: [PATCH 014/105] Stuff --- .../agents/auth/services/AuthService.java | 40 ++++--------------- .../agents/auth/services/AuthService2.java | 34 +--------------- 2 files changed, 10 insertions(+), 64 deletions(-) diff --git a/src/main/java/com/percero/agents/auth/services/AuthService.java b/src/main/java/com/percero/agents/auth/services/AuthService.java index c8bea97..2ae6d8e 100644 --- a/src/main/java/com/percero/agents/auth/services/AuthService.java +++ b/src/main/java/com/percero/agents/auth/services/AuthService.java @@ -1,23 +1,13 @@ package com.percero.agents.auth.services; -import java.util.ArrayList; -import java.util.Date; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Iterator; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.UUID; - +import com.percero.agents.auth.helpers.IAccountHelper; +import com.percero.agents.auth.hibernate.AssociationExample; +import com.percero.agents.auth.hibernate.AuthHibernateUtils; +import com.percero.agents.auth.hibernate.BaseDataObjectPropertySelector; +import com.percero.agents.auth.vo.*; +import com.percero.agents.sync.access.IAccessManager; import org.apache.log4j.Logger; -import org.hibernate.Criteria; -import org.hibernate.Query; -import org.hibernate.Session; -import org.hibernate.SessionFactory; -import org.hibernate.StaleStateException; -import org.hibernate.Transaction; +import org.hibernate.*; import org.hibernate.exception.LockAcquisitionException; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; @@ -25,21 +15,7 @@ import org.springframework.stereotype.Component; import org.springframework.util.StringUtils; -import com.percero.agents.auth.helpers.IAccountHelper; -import com.percero.agents.auth.hibernate.AssociationExample; -import com.percero.agents.auth.hibernate.AuthHibernateUtils; -import com.percero.agents.auth.hibernate.BaseDataObjectPropertySelector; -import com.percero.agents.auth.vo.AuthProvider; -import com.percero.agents.auth.vo.OAuthResponse; -import com.percero.agents.auth.vo.OAuthToken; -import com.percero.agents.auth.vo.ServiceIdentifier; -import com.percero.agents.auth.vo.ServiceUser; -import com.percero.agents.auth.vo.SvcAppRole; -import com.percero.agents.auth.vo.User; -import com.percero.agents.auth.vo.UserAccount; -import com.percero.agents.auth.vo.UserIdentifier; -import com.percero.agents.auth.vo.UserToken; -import com.percero.agents.sync.access.IAccessManager; +import java.util.*; /** * The AuthService is responsible for managing authentication of users within the Percero framework. The AuthService diff --git a/src/main/java/com/percero/agents/auth/services/AuthService2.java b/src/main/java/com/percero/agents/auth/services/AuthService2.java index b6cc038..a227d11 100644 --- a/src/main/java/com/percero/agents/auth/services/AuthService2.java +++ b/src/main/java/com/percero/agents/auth/services/AuthService2.java @@ -6,8 +6,6 @@ import com.percero.agents.auth.hibernate.AuthHibernateUtils; import com.percero.agents.auth.hibernate.BaseDataObjectPropertySelector; import com.percero.agents.auth.vo.*; -import com.percero.agents.sync.dao.DAORegistry; -import com.percero.agents.sync.dao.IDataAccessObject; import com.percero.agents.sync.hibernate.SyncHibernateUtils; import com.percero.agents.sync.metadata.*; import com.percero.agents.sync.services.ISyncAgentService; @@ -337,41 +335,13 @@ private static String getRandomId() { } public void ensureAnchorUserExists(ServiceUser serviceUser, User user){ - IUserAnchor result = null; -// Session s = appSessionFactory.openSession(); - - EntityImplementation userAnchorEI = getUserAnchorEntityImplementation(); - - MappedClass mc = userAnchorEI.mappedClass; - DAORegistry daoRegistry = DAORegistry.getInstance(); - IDataAccessObject dao = (IDataAccessObject) daoRegistry.getDataAccessObject(mc.className); -// String userAnchorQueryString = "SELECT ua FROM " + mc.tableName + " ua WHERE ua.userId=:userId"; -// Query userAnchorQuery = s.createQuery(userAnchorQueryString); -// userAnchorQuery.setString("userId", user.getID()); - -// List foundUserAnchors = (List) userAnchorQuery.list(); - List foundUserAnchors = null; - try { - Class anchorUserClass = Class.forName(mc.className); - IUserAnchor example = (IUserAnchor) anchorUserClass.newInstance(); - example.setUserId(user.getID()); - foundUserAnchors = dao.findByExample((IPerceroObject) example, null, null, false); - }catch(Exception e){ - logger.warn(e.getMessage(), e); - } + IUserAnchor result = UserAnchorHelper.getUserAnchor(user.getID()); - if (foundUserAnchors == null || foundUserAnchors.size() <= 0) + if (result == null) addOrUpdateUserAnchorFromServiceUserList(serviceUser, user, null); else { - if(foundUserAnchors.size() > 1) - logger.warn("Found more than one IUserAnchor for userId: "+ user.getID()); - - result = (IUserAnchor) foundUserAnchors.get(0); // Just take the first one handleUserAnchorFound(serviceUser, user, result); } - -// if (s != null && s.isOpen()) -// s.close(); } private EntityImplementation getUserAnchorEntityImplementation(){ From 8e4c512e11d64452aeacfa3ec6ea5a96d03185fa Mon Sep 17 00:00:00 2001 From: Jonathan Samples Date: Wed, 7 Oct 2015 23:38:41 -0700 Subject: [PATCH 015/105] Merged the bye_bye_hibernate branch in --- .../agents/auth/helpers/AccountHelper.java | 9 - .../agents/auth/services/AuthService2.java | 93 +++------- .../spring/percero-spring-config.xml | 168 +++++++++--------- 3 files changed, 110 insertions(+), 160 deletions(-) diff --git a/src/main/java/com/percero/agents/auth/helpers/AccountHelper.java b/src/main/java/com/percero/agents/auth/helpers/AccountHelper.java index 385c17b..f14fc66 100644 --- a/src/main/java/com/percero/agents/auth/helpers/AccountHelper.java +++ b/src/main/java/com/percero/agents/auth/helpers/AccountHelper.java @@ -438,15 +438,6 @@ protected IUserAnchor addOrUpdateUserAnchorFromServiceUserList(String userId, Li ((IPerceroObject) result).setID(UUID.randomUUID().toString()); result.setUserId(userId); if (serviceUserList.size() > 0) { - /** - IUserAnchorA userAnchorAnnotation = getUserAnchorAnnotation(userAnchorClass); - - if (userAnchorAnnotation.firstNameFieldName() != null && !userAnchorAnnotation.firstNameFieldName().isEmpty()) - firstNameMappedField = mcUserAnchor.getMappedFieldByName(userAnchorAnnotation.firstNameFieldName()); - MappedField lastNameMappedField = null; - if (userAnchorAnnotation.lastNameFieldName() != null && !userAnchorAnnotation.lastNameFieldName().isEmpty()) - lastNameMappedField = mcUserAnchor.getMappedFieldByName(userAnchorAnnotation.lastNameFieldName()); - */ // Find field that has firstName and lastName PropertyImplementations. PropertyImplementation firstNamePropImpl = eiUserAnchor.findPropertyImplementationByName(IUserAnchor.FIRST_NAME_FIELD); diff --git a/src/main/java/com/percero/agents/auth/services/AuthService2.java b/src/main/java/com/percero/agents/auth/services/AuthService2.java index a227d11..f4bf2ae 100644 --- a/src/main/java/com/percero/agents/auth/services/AuthService2.java +++ b/src/main/java/com/percero/agents/auth/services/AuthService2.java @@ -2,11 +2,11 @@ import com.percero.agents.auth.helpers.RoleHelper; import com.percero.agents.auth.helpers.UserAnchorHelper; +import com.percero.agents.auth.helpers.UserIdentifierHelper; import com.percero.agents.auth.hibernate.AssociationExample; import com.percero.agents.auth.hibernate.AuthHibernateUtils; import com.percero.agents.auth.hibernate.BaseDataObjectPropertySelector; import com.percero.agents.auth.vo.*; -import com.percero.agents.sync.hibernate.SyncHibernateUtils; import com.percero.agents.sync.metadata.*; import com.percero.agents.sync.services.ISyncAgentService; import com.percero.framework.bl.IManifest; @@ -37,9 +37,6 @@ public class AuthService2 { @Autowired SessionFactory sessionFactoryAuth; - @Autowired - SessionFactory appSessionFactory; - @Autowired ISyncAgentService syncAgentService; @@ -421,7 +418,6 @@ protected void handleUserAnchorFound(ServiceUser serviceUser, User user, IUserAn } protected IUserAnchor addOrUpdateUserAnchorFromServiceUserList(ServiceUser serviceUser, User user, IUserAnchor result) { - Session s = null; try { EntityImplementation eiUserAnchor = getUserAnchorEntityImplementation(); @@ -434,62 +430,32 @@ protected IUserAnchor addOrUpdateUserAnchorFromServiceUserList(ServiceUser servi while (itrUserIdentifierEntityImplementations.hasNext()) { EntityImplementation userIdentifierEntityImplementation = itrUserIdentifierEntityImplementations.next(); - //MappedField userAnchorMappedField = userIdentifierEntityImplementation.getMappedFieldByName(userIdentifierAnnotation.userAnchorFieldName()); PropertyImplementation userIdentifierPropImpl = userIdentifierEntityImplementation.findPropertyImplementationByName(IUserIdentifier.USER_IDENTIFIER_FIELD_NAME); RelationshipImplementation userAnchorRelImpl = userIdentifierEntityImplementation.findRelationshipImplementationBySourceVarName(IUserIdentifier.USER_ANCHOR_FIELD_NAME); - //IUserIdentifierA userIdentifierAnnotation = getUserIdentifierAnnotation(userIdentifierEntityImplementation); - if (userAnchorRelImpl != null) { - - if (s == null) { - s = appSessionFactory.openSession(); - } - // Get this userAnchor's identifier(s). - String userIdentifierQueryString = "SELECT ui FROM " + userIdentifierEntityImplementation.mappedClass.tableName - + " ui WHERE ui." + userIdentifierPropImpl.mappedField.getField().getName() + "=:value AND (ui." + - userAnchorRelImpl.sourceMappedField.getField().getName() + " IS NULL OR ui." + userAnchorRelImpl.sourceMappedField.getField().getName() + - " IN (SELECT ua FROM " + eiUserAnchor.mappedClass.tableName + " ua WHERE (ua.userId=null OR ua.userId='' OR ua.userId=:userId)))"; - - for (ServiceIdentifier nextIdentifier : serviceUser.getIdentifiers()) { - try { - // Make sure this Identifier is in the same paradigm. - String paradigm = null; - Iterator itrParams = userIdentifierPropImpl.params.iterator(); - while (itrParams.hasNext()) { - PropertyImplementationParam nextParam = itrParams.next(); - if (nextParam.name.equalsIgnoreCase(IUserIdentifier.PARADIGM_PARAM_NAME)) { - paradigm = nextParam.value; - break; - } - } - if (nextIdentifier.getParadigm() == null || !nextIdentifier.getParadigm().equalsIgnoreCase(paradigm)) { - continue; + for (ServiceIdentifier nextIdentifier : serviceUser.getIdentifiers()) { + try { + // Make sure this Identifier is in the same paradigm. + String paradigm = null; + Iterator itrParams = userIdentifierPropImpl.params.iterator(); + while (itrParams.hasNext()) { + PropertyImplementationParam nextParam = itrParams.next(); + if (nextParam.name.equalsIgnoreCase(IUserIdentifier.PARADIGM_PARAM_NAME)) { + paradigm = nextParam.value; + break; } + } + if (nextIdentifier.getParadigm() == null || !nextIdentifier.getParadigm().equalsIgnoreCase(paradigm)) { + continue; + } + + // Look for this existing identifier. + IUserIdentifier foundIdentifier = UserIdentifierHelper.getUserIdentifierForUserAndValue(userIdentifierEntityImplementation, result, nextIdentifier.getValue()); - // Look for this existing identifier. - Query userIdentifierQuery = s.createQuery(userIdentifierQueryString); - userIdentifierQuery.setString("userId", user.getID()); - userIdentifierQuery.setString("value", nextIdentifier.getValue()); - IUserIdentifier foundIdentifier = (IUserIdentifier) userIdentifierQuery.uniqueResult(); - - if (foundIdentifier != null) { - // If the email does not have a Person, then associate email with this Person. - foundIdentifier = (IUserIdentifier) SyncHibernateUtils.cleanObject(foundIdentifier, s); - IUserAnchor existingUserAnchor = (IUserAnchor) SyncHibernateUtils.cleanObject(userAnchorRelImpl.sourceMappedField.getGetter().invoke(foundIdentifier), s); - if (existingUserAnchor == null) { - if (result != null) { - userAnchorRelImpl.sourceMappedField.getSetter().invoke(foundIdentifier, result); - syncAgentService.systemPutObject((IPerceroObject) foundIdentifier, null, null, null, true); - } - else { - identifiersToSave.add(foundIdentifier); - } - } else if (result == null) { - result = existingUserAnchor; - } - } else { // Identifier object NOT found, need to add. - // Identifier.identifierValue will be set later, once that is determined. - IUserIdentifier newIdentifier = (IUserIdentifier) userIdentifierEntityImplementation.mappedClass.clazz.newInstance(); + + if (foundIdentifier == null) { + // Identifier.identifierValue will be set later, once that is determined. + IUserIdentifier newIdentifier = (IUserIdentifier) userIdentifierEntityImplementation.mappedClass.clazz.newInstance(); /* IUserIdentifierA userIdentifierA = null; Class nextClass = newIdentifier.getClass(); @@ -505,18 +471,14 @@ protected IUserAnchor addOrUpdateUserAnchorFromServiceUserList(ServiceUser servi MappedField mappedField = mappedClass.getExternalizeFieldByName(userIdentifierA.userIdentifierFieldName()); mappedField.getSetter().invoke(newIdentifier, nextIdentifier.getValue()); */ - userIdentifierPropImpl.mappedField.getSetter().invoke(newIdentifier, nextIdentifier.getValue()); - identifiersToSave.add(newIdentifier); - } - } catch(NonUniqueResultException nure) { - logger.warn("Non-unique User Identifier: " + nextIdentifier, nure); + userIdentifierPropImpl.mappedField.getSetter().invoke(newIdentifier, nextIdentifier.getValue()); + identifiersToSave.add(newIdentifier); } + } catch(NonUniqueResultException nure) { + logger.warn("Non-unique User Identifier: " + nextIdentifier, nure); } } - } - if (s != null && s.isOpen()) { - s.close(); } // Need to create a new IUserAnchor @@ -586,9 +548,6 @@ else if (result.getUserId() == null || result.getUserId().isEmpty()) { logger.error(e.getMessage(), e); } catch (Exception e) { logger.error(e.getMessage(), e); - } finally { - if (s != null && s.isOpen()) - s.close(); } return null; diff --git a/src/main/resources/spring/percero-spring-config.xml b/src/main/resources/spring/percero-spring-config.xml index 612a478..81e9b89 100644 --- a/src/main/resources/spring/percero-spring-config.xml +++ b/src/main/resources/spring/percero-spring-config.xml @@ -241,89 +241,89 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + hibernate.dialect=org.hibernate.dialect.Oracle10gDialect + + + + hibernate.show_sql=false + hibernate.format_sql=false + + hibernate.connection.aggressive_release=true + hibernate.jdbc.batch_size=20 + hibernate.connection.autocommit=false + hibernate.connection.autoReconnect=true + hibernate.enable_lazy_load_no_trans=true + + + + + $pf{domain.packageToScan} + + + + + + + From 6376c8eec03571f9f803207fdcaf6382f1ca717e Mon Sep 17 00:00:00 2001 From: Jonathan Samples Date: Thu, 8 Oct 2015 02:03:17 -0700 Subject: [PATCH 016/105] Modified the updatetable processing so that it scales threads for weight --- .../agents/sync/jobs/UpdateTablePoller.java | 64 ++++++++++--------- .../sync/jobs/UpdateTableProcessor.java | 16 +++-- .../sync/jobs/UpdateTableProcessorTest.java | 24 +++---- 3 files changed, 55 insertions(+), 49 deletions(-) diff --git a/src/main/java/com/percero/agents/sync/jobs/UpdateTablePoller.java b/src/main/java/com/percero/agents/sync/jobs/UpdateTablePoller.java index 8b2ff1c..341c47c 100644 --- a/src/main/java/com/percero/agents/sync/jobs/UpdateTablePoller.java +++ b/src/main/java/com/percero/agents/sync/jobs/UpdateTablePoller.java @@ -1,16 +1,18 @@ package com.percero.agents.sync.jobs; -import org.apache.log4j.Logger; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.scheduling.annotation.Scheduled; -import org.springframework.stereotype.Component; - import com.percero.agents.sync.access.IAccessManager; import com.percero.agents.sync.cache.CacheManager; import com.percero.agents.sync.helpers.PostDeleteHelper; import com.percero.agents.sync.helpers.PostPutHelper; import com.percero.agents.sync.services.DataProviderManager; import com.percero.framework.bl.IManifest; +import org.apache.log4j.Logger; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Component; + +import java.util.ArrayList; +import java.util.List; /** * Created by jonnysamps on 8/31/15. @@ -41,47 +43,47 @@ public class UpdateTablePoller { @Autowired IAccessManager accessManager; + List threads = new ArrayList(); + public boolean enabled = true; /** * Run every minute */ @Scheduled(fixedDelay=5000, initialDelay=10000) // Every 5 seconds - public void pollUpdateTables(){ - logger.debug("|> Starting poller"); - if(enabled) { - // Loop until we didn't find any rows to work with - while(true) { - int count = 0; - for (UpdateTableConnectionFactory updateTableConnectionFactory : updateTableRegistry.getConnectionFactories()) { - for (String tableName : updateTableConnectionFactory.getTableNames()) { - count += doProcessingForTable(updateTableConnectionFactory, tableName); - } + public void pollUpdateTables() { + boolean lastDone = true; + for (Thread thread : threads){ + if (thread.isAlive()) { + lastDone = false; + break; + } + } + + if(enabled && lastDone) { + threads.clear(); + for (UpdateTableConnectionFactory updateTableConnectionFactory : updateTableRegistry.getConnectionFactories()) { + for (String tableName : updateTableConnectionFactory.getTableNames()) { + doProcessingForTable(updateTableConnectionFactory, tableName); } - if(count <= 0) break; } } - logger.debug("[] Stopping poller"); } - public int doProcessingForTable(UpdateTableConnectionFactory connectionFactory, String tableName){ - UpdateTableProcessor processor = getProcessor(connectionFactory, tableName); - ProcessorResult result = processor.process(); - if(result.isSuccess()){ - logger.debug("Update table processor ("+tableName+") finished successfully. Total rows ("+result.getTotal()+")"); + public void doProcessingForTable(UpdateTableConnectionFactory connectionFactory, String tableName){ + // Spin `weight` new threads... weight is supposed to be a balancing scale.. but right now we + // Use it to see how many threads to create. + for(int i = 0; i < connectionFactory.getWeight(); i++) { + logger.info("Creating new processor thread"); + UpdateTableProcessor processor = getProcessor(connectionFactory, tableName); + Thread thread = new Thread(processor); + threads.add(thread); + thread.start(); } - else{ - logger.warn("Update table processor ("+tableName+") failed. Details:"); - logger.warn(result); - } - - return result.getTotal(); } public UpdateTableProcessor getProcessor(UpdateTableConnectionFactory connectionFactory, String tableName){ return new UpdateTableProcessor(tableName, connectionFactory, manifest, - postDeleteHelper, postPutHelper, cacheManager, dataProviderManager, accessManager); + postDeleteHelper, postPutHelper, cacheManager, dataProviderManager, accessManager); } - - } diff --git a/src/main/java/com/percero/agents/sync/jobs/UpdateTableProcessor.java b/src/main/java/com/percero/agents/sync/jobs/UpdateTableProcessor.java index 3e1c45d..1c7eaa7 100644 --- a/src/main/java/com/percero/agents/sync/jobs/UpdateTableProcessor.java +++ b/src/main/java/com/percero/agents/sync/jobs/UpdateTableProcessor.java @@ -24,7 +24,7 @@ * Responsible for querying an update table and processing the rows. * Created by Jonathan Samples on 8/31/15. */ -public class UpdateTableProcessor { +public class UpdateTableProcessor implements Runnable{ protected static Logger logger = Logger.getLogger(UpdateTableProcessor.class); public static final int INFINITE_ROWS = -1; @@ -74,11 +74,10 @@ public UpdateTableProcessor(String tableName, * * @return */ - public ProcessorResult process(){ + public void run(){ ProcessorResult result = new ProcessorResult(); - int numRowsProcessed = 0; - while(numRowsProcessed < maxRowsToProcess || maxRowsToProcess == INFINITE_ROWS) { + while(true) { UpdateTableRow row = getRow(); if(row == null) break; @@ -93,10 +92,15 @@ public ProcessorResult process(){ logger.warn("Failed to process update: "+ e.getMessage(), e); result.addResult(row.getType().toString(), false, e.getMessage()); } - numRowsProcessed++; } - return result; + if(result.isSuccess()){ + logger.debug("Update table processor ("+tableName+") finished successfully. Total rows ("+result.getTotal()+")"); + } + else{ + logger.warn("Update table processor ("+tableName+") failed. Details:"); + logger.warn(result); + } } protected boolean processRow(UpdateTableRow row) throws Exception{ diff --git a/src/test/java/com/percero/agents/sync/jobs/UpdateTableProcessorTest.java b/src/test/java/com/percero/agents/sync/jobs/UpdateTableProcessorTest.java index 1d867e0..ecee091 100644 --- a/src/test/java/com/percero/agents/sync/jobs/UpdateTableProcessorTest.java +++ b/src/test/java/com/percero/agents/sync/jobs/UpdateTableProcessorTest.java @@ -116,18 +116,18 @@ public void processMultipleRows() throws Exception { UpdateTableConnectionFactory connectionFactory = new UpdateTableConnectionFactory(); UpdateTableProcessor processor = poller.getProcessor(connectionFactory, tableName); - ProcessorResult result = processor.process(); - Assert.assertEquals(3, result.getTotal()); - Assert.assertEquals(0, result.getNumFailed()); - Assert.assertTrue(result.isSuccess()); - try(Connection connection = connectionFactory.getConnection(); - Statement statement = connection.createStatement()) - { - String sql = "select count(*) as 'count' from " + tableName; - ResultSet resultSet = statement.executeQuery(sql); - Assert.assertTrue(resultSet.next()); - Assert.assertEquals(0, resultSet.getInt("count")); - } +// ProcessorResult result = processor.run(); +// Assert.assertEquals(3, result.getTotal()); +// Assert.assertEquals(0, result.getNumFailed()); +// Assert.assertTrue(result.isSuccess()); +// try(Connection connection = connectionFactory.getConnection(); +// Statement statement = connection.createStatement()) +// { +// String sql = "select count(*) as 'count' from " + tableName; +// ResultSet resultSet = statement.executeQuery(sql); +// Assert.assertTrue(resultSet.next()); +// Assert.assertEquals(0, resultSet.getInt("count")); +// } } @Test From aceffa76e9a7cd5a04f56bb7cb2ba0c237622291 Mon Sep 17 00:00:00 2001 From: Jonathan Samples Date: Thu, 8 Oct 2015 02:27:34 -0700 Subject: [PATCH 017/105] Moved the dialect back to mysql for auth --- src/main/resources/spring/percero-spring-config.xml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/main/resources/spring/percero-spring-config.xml b/src/main/resources/spring/percero-spring-config.xml index 81e9b89..1a2105e 100644 --- a/src/main/resources/spring/percero-spring-config.xml +++ b/src/main/resources/spring/percero-spring-config.xml @@ -219,8 +219,7 @@ - - hibernate.dialect=org.hibernate.dialect.Oracle10gDialect + hibernate.dialect=org.hibernate.dialect.MySQLDialect hibernate.show_sql=false From 2ee16fd8f870a9686205a52f57e80687e8509ab3 Mon Sep 17 00:00:00 2001 From: Jonathan Samples Date: Thu, 8 Oct 2015 10:07:07 -0700 Subject: [PATCH 018/105] Turned back on auto db create for auth --- src/main/resources/spring/percero-spring-config.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/resources/spring/percero-spring-config.xml b/src/main/resources/spring/percero-spring-config.xml index 1a2105e..d237308 100644 --- a/src/main/resources/spring/percero-spring-config.xml +++ b/src/main/resources/spring/percero-spring-config.xml @@ -220,7 +220,7 @@ hibernate.dialect=org.hibernate.dialect.MySQLDialect - + hibernate.hbm2ddl.auto=update hibernate.show_sql=false hibernate.format_sql=false From 3558db290d97cea8f406cf5174f41bb6bfa56ec2 Mon Sep 17 00:00:00 2001 From: Jonathan Samples Date: Fri, 9 Oct 2015 08:42:48 -0700 Subject: [PATCH 019/105] Changed setting ons the update table connection factory to use the weight --- .../agents/sync/jobs/UpdateTableConnectionFactory.java | 8 ++++---- .../percero/agents/sync/jobs/UpdateTableProcessor.java | 4 ++-- .../agents/sync/jobs/UpdateTableProcessorTest.java | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/main/java/com/percero/agents/sync/jobs/UpdateTableConnectionFactory.java b/src/main/java/com/percero/agents/sync/jobs/UpdateTableConnectionFactory.java index c0f2d2f..4fb2760 100644 --- a/src/main/java/com/percero/agents/sync/jobs/UpdateTableConnectionFactory.java +++ b/src/main/java/com/percero/agents/sync/jobs/UpdateTableConnectionFactory.java @@ -104,9 +104,9 @@ public void init() throws PropertyVetoException{ cpds.setPassword(password); // the settings below are optional -- c3p0 can work with defaults - cpds.setMinPoolSize(5); + cpds.setMinPoolSize(10); cpds.setAcquireIncrement(5); - + cpds.setMaxPoolSize(this.weight); }catch(PropertyVetoException pve){ logger.error(pve.getMessage(), pve); throw pve; @@ -116,8 +116,8 @@ public void init() throws PropertyVetoException{ public Connection getConnection() throws SQLException{ try{ if (cpds == null) { - init(); - } + init(); + } return cpds.getConnection(); } catch(PropertyVetoException e){ diff --git a/src/main/java/com/percero/agents/sync/jobs/UpdateTableProcessor.java b/src/main/java/com/percero/agents/sync/jobs/UpdateTableProcessor.java index 1c7eaa7..f41683e 100644 --- a/src/main/java/com/percero/agents/sync/jobs/UpdateTableProcessor.java +++ b/src/main/java/com/percero/agents/sync/jobs/UpdateTableProcessor.java @@ -78,7 +78,7 @@ public void run(){ ProcessorResult result = new ProcessorResult(); while(true) { - UpdateTableRow row = getRow(); + UpdateTableRow row = getRows(); if(row == null) break; try { @@ -483,7 +483,7 @@ protected void updateReferences(String className){ * processors don't duplicate the work * @return */ - public UpdateTableRow getRow(){ + public UpdateTableRow getRows(){ UpdateTableRow row = null; try(Connection conn = connectionFactory.getConnection(); diff --git a/src/test/java/com/percero/agents/sync/jobs/UpdateTableProcessorTest.java b/src/test/java/com/percero/agents/sync/jobs/UpdateTableProcessorTest.java index ecee091..765cbdb 100644 --- a/src/test/java/com/percero/agents/sync/jobs/UpdateTableProcessorTest.java +++ b/src/test/java/com/percero/agents/sync/jobs/UpdateTableProcessorTest.java @@ -91,7 +91,7 @@ public void getRow() throws Exception { setupThreeRowsInUpdateTable(); UpdateTableConnectionFactory connectionFactory = new UpdateTableConnectionFactory(); UpdateTableProcessor processor = poller.getProcessor(connectionFactory, tableName); - UpdateTableRow row = processor.getRow(); + UpdateTableRow row = processor.getRows(); Assert.assertNotNull(row); Assert.assertNotNull(row.getLockId()); From c14a308bcd987484eb37ad804923efe86c091809 Mon Sep 17 00:00:00 2001 From: Jonathan Samples Date: Fri, 9 Oct 2015 22:39:33 -0700 Subject: [PATCH 020/105] Optimized the connection aquisition --- .../sync/jobs/UpdateTableProcessor.java | 72 ++++++++++++------- 1 file changed, 47 insertions(+), 25 deletions(-) diff --git a/src/main/java/com/percero/agents/sync/jobs/UpdateTableProcessor.java b/src/main/java/com/percero/agents/sync/jobs/UpdateTableProcessor.java index f41683e..69012cd 100644 --- a/src/main/java/com/percero/agents/sync/jobs/UpdateTableProcessor.java +++ b/src/main/java/com/percero/agents/sync/jobs/UpdateTableProcessor.java @@ -40,6 +40,8 @@ public class UpdateTableProcessor implements Runnable{ protected IAccessManager accessManager; protected int maxRowsToProcess = INFINITE_ROWS; // No max + private Connection connection; + public UpdateTableProcessor(String tableName, UpdateTableConnectionFactory connectionFactory, IManifest manifest, @@ -75,31 +77,37 @@ public UpdateTableProcessor(String tableName, * @return */ public void run(){ - ProcessorResult result = new ProcessorResult(); + try { + ProcessorResult result = new ProcessorResult(); - while(true) { - UpdateTableRow row = getRows(); - if(row == null) break; + while (true) { + UpdateTableRow row = getRows(); + if (row == null) break; - try { - if (processRow(row)) { - result.addResult(row.getType().toString()); - deleteRow(row); - } else { - result.addResult(row.getType().toString(), false, ""); + try { + if (processRow(row)) { + result.addResult(row.getType().toString()); + deleteRow(row); + } else { + result.addResult(row.getType().toString(), false, ""); + } + } catch (Exception e) { + logger.warn("Failed to process update: " + e.getMessage(), e); + result.addResult(row.getType().toString(), false, e.getMessage()); } - }catch(Exception e){ - logger.warn("Failed to process update: "+ e.getMessage(), e); - result.addResult(row.getType().toString(), false, e.getMessage()); } - } - if(result.isSuccess()){ - logger.debug("Update table processor ("+tableName+") finished successfully. Total rows ("+result.getTotal()+")"); - } - else{ - logger.warn("Update table processor ("+tableName+") failed. Details:"); - logger.warn(result); + if (result.isSuccess()) { + logger.debug("Update table processor (" + tableName + ") finished successfully. Total rows (" + result.getTotal() + ")"); + } else { + logger.warn("Update table processor (" + tableName + ") failed. Details:"); + logger.warn(result); + } + }finally{ + try { + Connection conn = getConnection(); + conn.close(); + }catch(Exception e){} } } @@ -486,10 +494,10 @@ protected void updateReferences(String className){ public UpdateTableRow getRows(){ UpdateTableRow row = null; - try(Connection conn = connectionFactory.getConnection(); - Statement statement = conn.createStatement()) - { - + Statement statement = null; + try{ + Connection conn = getConnection(); + statement = conn.createStatement(); Random rand = new Random(); int lockId = rand.nextInt(); @@ -510,6 +518,11 @@ public UpdateTableRow getRows(){ } catch(SQLException e){ logger.warn(e.getMessage(), e); + } finally { + try { + if (statement != null) + statement.close(); + }catch(Exception e){} } return row; @@ -619,7 +632,9 @@ private UpdateTableRow getStoredProcRow(UpdateTableRow row, * @param row */ protected void deleteRow(UpdateTableRow row){ - try(Connection conn = connectionFactory.getConnection()){ + + try{ + Connection conn = getConnection(); String sql = "delete from :tableName where ID=:ID"; sql = sql.replace(":tableName", tableName); sql = sql.replace(":ID", row.getID()+""); @@ -677,4 +692,11 @@ protected UpdateTableRow fromResultSet(ResultSet resultSet) throws SQLException{ return row; } + + private Connection getConnection() throws SQLException{ + if(this.connection == null || this.connection.isClosed()) + this.connection = this.connectionFactory.getConnection(); + + return this.connection; + } } From d7b91b3a047f585493ce6efb4674f019a62930da Mon Sep 17 00:00:00 2001 From: Jonathan Samples Date: Sat, 10 Oct 2015 22:06:23 -0700 Subject: [PATCH 021/105] Optimized update table processing, added unified reporter for logging. --- .../jobs/UpdateTableConnectionFactory.java | 2 +- .../agents/sync/jobs/UpdateTablePoller.java | 8 +- .../sync/jobs/UpdateTableProcessReporter.java | 125 +++++ .../sync/jobs/UpdateTableProcessor.java | 430 +++++++++--------- .../sync/jobs/UpdateTableProcessorTest.java | 4 +- 5 files changed, 360 insertions(+), 209 deletions(-) create mode 100644 src/main/java/com/percero/agents/sync/jobs/UpdateTableProcessReporter.java diff --git a/src/main/java/com/percero/agents/sync/jobs/UpdateTableConnectionFactory.java b/src/main/java/com/percero/agents/sync/jobs/UpdateTableConnectionFactory.java index 4fb2760..79167db 100644 --- a/src/main/java/com/percero/agents/sync/jobs/UpdateTableConnectionFactory.java +++ b/src/main/java/com/percero/agents/sync/jobs/UpdateTableConnectionFactory.java @@ -74,7 +74,7 @@ public void setStoredProcedureDefinition(String storedProcedureDefinition) { private String updateStatementSql = "update :tableName set lock_id=:lockId, lock_date=NOW() " + "where lock_id is null or " + "lock_date < ':expireThreshold' " + - "order by time_stamp limit 1"; + "order by time_stamp limit :limit"; public String getUpdateStatementSql() { return updateStatementSql; } diff --git a/src/main/java/com/percero/agents/sync/jobs/UpdateTablePoller.java b/src/main/java/com/percero/agents/sync/jobs/UpdateTablePoller.java index 341c47c..ebf664c 100644 --- a/src/main/java/com/percero/agents/sync/jobs/UpdateTablePoller.java +++ b/src/main/java/com/percero/agents/sync/jobs/UpdateTablePoller.java @@ -11,6 +11,7 @@ import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Component; +import javax.annotation.PostConstruct; import java.util.ArrayList; import java.util.List; @@ -47,6 +48,12 @@ public class UpdateTablePoller { public boolean enabled = true; + @PostConstruct + public void init(){ + // Get the reporter going + UpdateTableProcessReporter.getInstance(); + } + /** * Run every minute */ @@ -74,7 +81,6 @@ public void doProcessingForTable(UpdateTableConnectionFactory connectionFactory, // Spin `weight` new threads... weight is supposed to be a balancing scale.. but right now we // Use it to see how many threads to create. for(int i = 0; i < connectionFactory.getWeight(); i++) { - logger.info("Creating new processor thread"); UpdateTableProcessor processor = getProcessor(connectionFactory, tableName); Thread thread = new Thread(processor); threads.add(thread); diff --git a/src/main/java/com/percero/agents/sync/jobs/UpdateTableProcessReporter.java b/src/main/java/com/percero/agents/sync/jobs/UpdateTableProcessReporter.java new file mode 100644 index 0000000..be491dd --- /dev/null +++ b/src/main/java/com/percero/agents/sync/jobs/UpdateTableProcessReporter.java @@ -0,0 +1,125 @@ +package com.percero.agents.sync.jobs; + +import org.apache.log4j.Logger; + +import java.util.HashMap; +import java.util.Map; + +/** + * Created by jonnysamps on 10/10/15. + */ +public class UpdateTableProcessReporter { + + private static Logger logger = Logger.getLogger(UpdateTableProcessReporter.class); + private static final int WINDOW_LENGTH = 10000; + private static final double WINDOW_LENGTH_SECONDS = roundToOnePlace(WINDOW_LENGTH/1000.0); + + private static UpdateTableProcessReporter instance; + + private UTPReportStat totalStat = new UTPReportStat(); + private Map stats = new HashMap<>(); + + private UpdateTableProcessReporter(){ + new Thread(this.windowTimer).start(); // Start the reporting thread + } + + public static UpdateTableProcessReporter getInstance(){ + if(instance == null){ + instance = new UpdateTableProcessReporter(); + } + return instance; + } + + public void submitCountAndTime(String key, int count, long time){ + synchronized (this) { + if(!stats.containsKey(key)) + stats.put(key, new UTPReportStat()); + + totalStat.totalCount += count; + totalStat.totalTime += time; + totalStat.windowCount += count; + totalStat.windowTime += time; + stats.get(key).totalCount += count; + stats.get(key).totalTime += time; + stats.get(key).windowCount += count; + stats.get(key).windowTime += time; + } + } + + private static double roundToOnePlace(double number){ + number *= 10; + int rounded = (int) number; + number = rounded / 10.0; + return number; + } + + private void resetwindow(){ + synchronized (this) { + totalStat.windowCount = 0; + totalStat.windowTime = 0; + for(UTPReportStat stat : stats.values()){ + stat.windowCount = 0; + stat.windowTime = 0; + } + } + } + + private void printStats(){ + synchronized (this){ + printStat("Totals", totalStat); + for(String key : stats.keySet()) + printStat(key, stats.get(key)); + } + } + + private static void printStat(String label, UTPReportStat stat){ + double totalTimeSeconds = roundToOnePlace(stat.totalTime / 1000.0); + double totalOpsPerSecond = roundToOnePlace(stat.totalCount / totalTimeSeconds); + double windowTimeSeconds = roundToOnePlace(stat.windowTime/1000.0); + double windowOpsPerSecond = roundToOnePlace(stat.windowCount/windowTimeSeconds); + logger.debug( + "UT: "+pad(20, label)+ + " : Last "+pad(3,WINDOW_LENGTH_SECONDS+"")+"s" + + "("+pad(5, stat.windowCount+"")+ + ", "+pad(6, windowOpsPerSecond+"")+"/s) " + + "Total("+pad(7, stat.totalCount+"")+", " + + pad(6, totalOpsPerSecond+"")+"/s)" + ); + } + + private static String pad(int space, String content){ + StringBuilder sb = new StringBuilder(); + for(int i = 0; i < space-content.length(); i++){ + sb.append(" "); + } + sb.append(content); + return sb.toString(); + } + + /** + * Window timer will sleep for the window length, wake up print some stats, + * reset the window counters and then sleep again. + */ + private Runnable windowTimer = new Runnable() { + @Override + public void run() { + while(true){ + try { + Thread.sleep(WINDOW_LENGTH); + instance.printStats(); + instance.resetwindow(); + }catch(InterruptedException e){ + logger.error(e.getMessage(), e); + break; + } + } + } + }; + + private class UTPReportStat{ + public int totalCount = 0; + public long totalTime = 0; + public int windowCount = 0; + public long windowTime = 0; + } +} diff --git a/src/main/java/com/percero/agents/sync/jobs/UpdateTableProcessor.java b/src/main/java/com/percero/agents/sync/jobs/UpdateTableProcessor.java index 69012cd..40956b6 100644 --- a/src/main/java/com/percero/agents/sync/jobs/UpdateTableProcessor.java +++ b/src/main/java/com/percero/agents/sync/jobs/UpdateTableProcessor.java @@ -19,6 +19,7 @@ import javax.persistence.Table; import java.sql.*; import java.util.*; +import java.util.Date; /** * Responsible for querying an update table and processing the rows. @@ -81,25 +82,33 @@ public void run(){ ProcessorResult result = new ProcessorResult(); while (true) { - UpdateTableRow row = getRows(); - if (row == null) break; - - try { - if (processRow(row)) { - result.addResult(row.getType().toString()); - deleteRow(row); - } else { - result.addResult(row.getType().toString(), false, ""); + Date startTime = new Date(); + int numRows = 160; + List rows = getRows(numRows); + if (rows.size() <= 0) break; + + List successfulRows = new ArrayList<>(); + for(UpdateTableRow row : rows) { + try { + if (processRow(row)) { + result.addResult(row.getType().toString()); + successfulRows.add(row); + } else { + result.addResult(row.getType().toString(), false, ""); + } + } catch (Exception e) { + logger.warn("Failed to process update: " + e.getMessage(), e); + result.addResult(row.getType().toString(), false, e.getMessage()); } - } catch (Exception e) { - logger.warn("Failed to process update: " + e.getMessage(), e); - result.addResult(row.getType().toString(), false, e.getMessage()); } + deleteRows(successfulRows); + + Date endTime = new Date(); + UpdateTableProcessReporter.getInstance() + .submitCountAndTime(tableName, successfulRows.size(), endTime.getTime()-startTime.getTime()); } - if (result.isSuccess()) { - logger.debug("Update table processor (" + tableName + ") finished successfully. Total rows (" + result.getTotal() + ")"); - } else { + if (!result.isSuccess()) { logger.warn("Update table processor (" + tableName + ") failed. Details:"); logger.warn(result); } @@ -113,7 +122,6 @@ public void run(){ protected boolean processRow(UpdateTableRow row) throws Exception{ boolean result = true; - logger.debug("UpdateTableProcessor: processRow"); if(row.getRowId() != null) switch (row.getType()){ case DELETE: @@ -183,8 +191,8 @@ protected boolean processUpdateSingle(UpdateTableRow row) throws Exception{ * @return */ @SuppressWarnings("rawtypes") - protected boolean processUpdateTable(UpdateTableRow row) throws Exception{ - Class clazz = getClassForTableName(row.getTableName()); + protected boolean processUpdateTable(UpdateTableRow row) throws Exception{ + Class clazz = getClassForTableName(row.getTableName()); if(clazz != null) { String className = clazz.getCanonicalName(); @@ -245,12 +253,12 @@ protected void processUpdates(String className, Collection Ids) throws E } protected void handleUpdateClassIdPair(IDataProvider dataProvider, ClassIDPair pair) throws Exception { - IPerceroObject object = dataProvider.findById(pair, null, true); + IPerceroObject object = dataProvider.findById(pair, null, true); - if (object != null) { - cacheManager.updateCachedObject(object, null); - postPutHelper.postPutObject(pair, null, null, true, null); - } + if (object != null) { + cacheManager.updateCachedObject(object, null); + postPutHelper.postPutObject(pair, null, null, true, null); + } } /** @@ -259,7 +267,7 @@ protected void handleUpdateClassIdPair(IDataProvider dataProvider, ClassIDPair p * @return */ @SuppressWarnings("rawtypes") - protected boolean processInsertSingle(UpdateTableRow row) throws Exception{ + protected boolean processInsertSingle(UpdateTableRow row) throws Exception{ Class clazz = getClassForTableName(row.getTableName()); // If clazz is not null then we care about this row, otherwise throw the row away @@ -282,7 +290,7 @@ protected boolean processInsertSingle(UpdateTableRow row) throws Exception{ * @return */ @SuppressWarnings("rawtypes") - protected boolean processInsertTable(UpdateTableRow row) throws Exception { + protected boolean processInsertTable(UpdateTableRow row) throws Exception { Class clazz = getClassForTableName(row.getTableName()); if(clazz != null) { @@ -311,7 +319,7 @@ protected boolean processInsertTable(UpdateTableRow row) throws Exception { * @return */ @SuppressWarnings("rawtypes") - protected boolean processDeleteSingle(UpdateTableRow row) throws Exception{ + protected boolean processDeleteSingle(UpdateTableRow row) throws Exception{ Class clazz = getClassForTableName(row.getTableName()); if(clazz != null) { @@ -338,7 +346,7 @@ protected boolean processDeleteSingle(UpdateTableRow row) throws Exception{ * @return */ @SuppressWarnings("rawtypes") - protected boolean processDeleteTable(UpdateTableRow row) throws Exception{ + protected boolean processDeleteTable(UpdateTableRow row) throws Exception{ Class clazz = getClassForTableName(row.getTableName()); if(clazz != null) { @@ -386,21 +394,21 @@ protected boolean processDeleteTable(UpdateTableRow row) throws Exception{ } @SuppressWarnings("rawtypes") - protected void handleDeletedObject(IPerceroObject cachedObject, Class clazz, String className, String id) throws Exception { - boolean isShellObject = false; + protected void handleDeletedObject(IPerceroObject cachedObject, Class clazz, String className, String id) throws Exception { + boolean isShellObject = false; if (cachedObject == null) { - cachedObject = (IPerceroObject) clazz.newInstance(); - cachedObject.setID(id); - isShellObject = true; + cachedObject = (IPerceroObject) clazz.newInstance(); + cachedObject.setID(id); + isShellObject = true; } - cacheManager.handleDeletedObject(cachedObject, className, isShellObject); + cacheManager.handleDeletedObject(cachedObject, className, isShellObject); postDeleteHelper.postDeleteObject(new ClassIDPair(id, className), null, null, true); } @SuppressWarnings("rawtypes") - protected Set getAllClassIdPairsForTable(String tableName) throws Exception{ + protected Set getAllClassIdPairsForTable(String tableName) throws Exception{ Class clazz = getClassForTableName(tableName); String className = clazz.getCanonicalName(); @@ -420,13 +428,13 @@ protected Set getAllClassIdPairsForTable(String tableName) throws E protected void updateReferences(String className){ IMappedClassManager mcm = MappedClassManagerFactory.getMappedClassManager(); MappedClass mappedClass = mcm.getMappedClassByClassName(className); - // Go through each mapped field and push all objects of that associated - // type (just in case any has a reference to a new row - // in the updated table) + // Go through each mapped field and push all objects of that associated + // type (just in case any has a reference to a new row + // in the updated table) // -- - // TODO: is this right? Is it enough to only check the relationships on - // this class or do we need to look - // through all of the mapped classes? + // TODO: is this right? Is it enough to only check the relationships on + // this class or do we need to look + // through all of the mapped classes? for(MappedFieldPerceroObject nextMappedField : mappedClass.externalizablePerceroObjectFields) { try { // Only care about it if it has a reverse relationship @@ -436,48 +444,48 @@ protected void updateReferences(String className){ Set ids = accessManager.getClassAccessJournalIDs(mappedField.getMappedClass().className); if (ids.contains("0")) { - // If there is a 0 ID in the list, then we need to update ALL records of this type. - Integer pageNumber = 0; - Integer pageSize = 25; - Integer total = -1; - - while (total < 0 || pageNumber * pageSize <= total) { - PerceroList objectsToUpdate = mappedField.getMappedClass().getDataProvider().getAllByName(mappedField.getMappedClass().className, pageNumber, pageSize, true, null); - pageNumber++; - total = objectsToUpdate.getTotalLength(); - if (total <= 0) { - break; - } - - Iterator itrObjectsToUpdate = objectsToUpdate.iterator(); - while (itrObjectsToUpdate.hasNext()) { - IPerceroObject nextObjectToUpdate = itrObjectsToUpdate.next(); - ClassIDPair pair = BaseDataObject.toClassIdPair(nextObjectToUpdate); - Map> changedFields = new HashMap>(); - Collection changedMappedFields = new ArrayList(1); - changedMappedFields.add(mappedField); - changedFields.put(pair, changedMappedFields); - - // Remove from the cache. - cacheManager.deleteObjectFromCache(pair); - postPutHelper.postPutObject(pair, null, null, true, changedFields); - } - } + // If there is a 0 ID in the list, then we need to update ALL records of this type. + Integer pageNumber = 0; + Integer pageSize = 25; + Integer total = -1; + + while (total < 0 || pageNumber * pageSize <= total) { + PerceroList objectsToUpdate = mappedField.getMappedClass().getDataProvider().getAllByName(mappedField.getMappedClass().className, pageNumber, pageSize, true, null); + pageNumber++; + total = objectsToUpdate.getTotalLength(); + if (total <= 0) { + break; + } + + Iterator itrObjectsToUpdate = objectsToUpdate.iterator(); + while (itrObjectsToUpdate.hasNext()) { + IPerceroObject nextObjectToUpdate = itrObjectsToUpdate.next(); + ClassIDPair pair = BaseDataObject.toClassIdPair(nextObjectToUpdate); + Map> changedFields = new HashMap>(); + Collection changedMappedFields = new ArrayList(1); + changedMappedFields.add(mappedField); + changedFields.put(pair, changedMappedFields); + + // Remove from the cache. + cacheManager.deleteObjectFromCache(pair); + postPutHelper.postPutObject(pair, null, null, true, changedFields); + } + } } else { - Iterator itrIdsToUpdate = ids.iterator(); - while (itrIdsToUpdate.hasNext()) { - String nextIdToUpdate = itrIdsToUpdate.next(); - ClassIDPair pair = new ClassIDPair(nextIdToUpdate, mappedField.getMappedClass().className); - Map> changedFields = new HashMap>(); - Collection changedMappedFields = new ArrayList(1); - changedMappedFields.add(mappedField); - changedFields.put(pair, changedMappedFields); - - // Remove from the cache. - cacheManager.deleteObjectFromCache(pair); - postPutHelper.postPutObject(pair, null, null, true, changedFields); - } + Iterator itrIdsToUpdate = ids.iterator(); + while (itrIdsToUpdate.hasNext()) { + String nextIdToUpdate = itrIdsToUpdate.next(); + ClassIDPair pair = new ClassIDPair(nextIdToUpdate, mappedField.getMappedClass().className); + Map> changedFields = new HashMap>(); + Collection changedMappedFields = new ArrayList(1); + changedMappedFields.add(mappedField); + changedFields.put(pair, changedMappedFields); + + // Remove from the cache. + cacheManager.deleteObjectFromCache(pair); + postPutHelper.postPutObject(pair, null, null, true, changedFields); + } } } } catch(Exception e) { @@ -491,13 +499,10 @@ protected void updateReferences(String className){ * processors don't duplicate the work * @return */ - public UpdateTableRow getRows(){ - UpdateTableRow row = null; + public List getRows(int numRows){ + List rows = null; - Statement statement = null; - try{ - Connection conn = getConnection(); - statement = conn.createStatement(); + try { Random rand = new Random(); int lockId = rand.nextInt(); @@ -507,149 +512,162 @@ public UpdateTableRow getRows(){ if (StringUtils.hasText(connectionFactory.getStoredProcedureName())) { try { - row = getStoredProcRow(row, conn, statement, lockId, expireThreshold); - }catch(Exception e){ - row = getUpdateSelectRow(row, statement, lockId, expireThreshold); + rows = getStoredProcRow(lockId, expireThreshold, numRows); + } catch (Exception e) { + rows = getUpdateSelectRow(lockId, expireThreshold, numRows); } + } else { + rows = getUpdateSelectRow(lockId, expireThreshold, numRows); } - else { - row = getUpdateSelectRow(row, statement, lockId, expireThreshold); - } + }catch(Exception e){} - } catch(SQLException e){ - logger.warn(e.getMessage(), e); - } finally { - try { - if (statement != null) - statement.close(); - }catch(Exception e){} - } - return row; + return rows; + } + + /** + * @param lockId + * @param expireThreshold + * @return + * @throws SQLException + */ + private List getUpdateSelectRow(int lockId, DateTime expireThreshold, int numRows)throws SQLException { + List list = new ArrayList<>(); + try(Statement statement = getConnection().createStatement()) { + /** + * First try to lock a row + */ + String sql = connectionFactory.getUpdateStatementSql(); + sql = sql.replace(":tableName", tableName); + sql = sql.replace(":lockId", lockId + ""); + sql = sql.replace(":expireThreshold", expireThreshold.toString("Y-MM-dd HH:mm:ss")); + sql = sql.replace(":limit", numRows+""); + + int numUpdated = statement.executeUpdate(sql); + + // Found a row to process + if (numUpdated > 0) { + sql = "select * from :tableName where lock_id=:lockId"; + sql = sql.replace(":tableName", tableName); + sql = sql.replace(":lockId", lockId + ""); + + try (ResultSet rs = statement.executeQuery(sql)) { + int count = 0; + while(rs.next()) { + UpdateTableRow row = fromResultSet(rs); + list.add(row); + count++; + } + + if(count != numUpdated) + logger.warn("Locked a "+numUpdated+" rows but found "+count); + } + } + } + return list; } - /** - * @param row - * @param statement - * @param lockId - * @param expireThreshold - * @return - * @throws SQLException - */ - private UpdateTableRow getUpdateSelectRow(UpdateTableRow row, - Statement statement, int lockId, DateTime expireThreshold) - throws SQLException { - /** - * First try to lock a row - */ - String sql = connectionFactory.getUpdateStatementSql(); - sql = sql.replace(":tableName", tableName); - sql = sql.replace(":lockId", lockId+""); - sql = sql.replace(":expireThreshold", expireThreshold.toString("Y-MM-dd HH:mm:ss")); - - int numUpdated = statement.executeUpdate(sql); - - // Found a row to process - if(numUpdated > 0){ - sql = "select * from :tableName where lock_id=:lockId limit 1"; - sql = sql.replace(":tableName", tableName); - sql = sql.replace(":lockId", lockId+""); - - try(ResultSet rs = statement.executeQuery(sql)){ - // If got a row back - if(rs.next()) - row = fromResultSet(rs); - else - logger.warn("Locked a row but couldn't retrieve"); - } - } - return row; - } - - /** - * @param row - * @param conn - * @param lockId - * @param expireThreshold - * @return - * @throws SQLException - */ - private UpdateTableRow getStoredProcRow(UpdateTableRow row, - Connection conn, Statement statement, int lockId, DateTime expireThreshold) - throws SQLException { - Integer updateTableId = null; - try { - CallableStatement cstmt = conn.prepareCall("{call " + connectionFactory.getStoredProcedureName() + "(?, ?, ?)}"); - cstmt.setInt(1, lockId); - cstmt.setString(2, expireThreshold.toString("Y-MM-dd HH:mm:ss")); - cstmt.registerOutParameter(3, Types.INTEGER); - cstmt.executeUpdate(); - updateTableId = cstmt.getInt(3); - } catch(SQLException e){ + /** + * @param lockId + * @param expireThreshold + * @return + * @throws SQLException + */ + private List getStoredProcRow(int lockId, DateTime expireThreshold, int numRows) throws SQLException { + List list = new ArrayList<>(); + Integer updateNum = 0; + try(CallableStatement cstmt = getConnection().prepareCall("{call " + connectionFactory.getStoredProcedureName() + "(?, ?, ?, ?)}")) { + cstmt.setInt(1, lockId); + cstmt.setString(2, expireThreshold.toString("Y-MM-dd HH:mm:ss")); + cstmt.setInt(3, numRows); + cstmt.registerOutParameter(4, Types.INTEGER); + cstmt.executeUpdate(); + updateNum = cstmt.getInt(4); + } catch(SQLException e){ // return null; - logger.warn(e.getMessage(), e); + logger.warn(e.getMessage(), e); - // If the stored proc doesn't exist, let's try and create it. - if (StringUtils.hasText(connectionFactory.getStoredProcedureDefinition()) && + // If the stored proc doesn't exist, let's try and create it. + if (StringUtils.hasText(connectionFactory.getStoredProcedureDefinition()) && e.getMessage().toLowerCase().contains(connectionFactory.getStoredProcedureName().toLowerCase()) && e.getMessage().toLowerCase().contains("must be declared")) { - try { - Statement stmtCreateStoredProcedure = conn.createStatement(); - boolean createResult = stmtCreateStoredProcedure.execute(connectionFactory.getStoredProcedureDefinition() + try(Statement stmtCreateStoredProcedure = getConnection().createStatement()) { + boolean createResult = stmtCreateStoredProcedure.execute(connectionFactory.getStoredProcedureDefinition() .replaceAll(":tableName", tableName) .replaceAll(":storedProcedureName", connectionFactory.getStoredProcedureName())); - System.out.println(createResult); - } - catch(SQLSyntaxErrorException ssee) { - logger.warn("Unable to create UpdateTable stored procedure: " + ssee.getMessage()); + System.out.println(createResult); + } + catch(SQLSyntaxErrorException ssee) { + logger.warn("Unable to create UpdateTable stored procedure: " + ssee.getMessage()); throw ssee; - } - catch(Exception e1) { - logger.warn("Unable to create UpdateTable stored procedure: " + e1.getMessage()); + } + catch(Exception e1) { + logger.warn("Unable to create UpdateTable stored procedure: " + e1.getMessage()); throw e1; - } - } - } - - if (updateTableId != null && updateTableId > -1) { - String sql = "select * from :tableName where id=:id"; - sql = sql.replace(":tableName", tableName); - sql = sql.replace(":id", updateTableId.toString()); - - try(ResultSet rs = statement.executeQuery(sql)){ - // If got a row back - if(rs.next()) - row = fromResultSet(rs); - else - logger.warn("Locked a row but couldn't retrieve"); - } - } - return row; - } - - /** + } + } + } + + if (updateNum != null && updateNum > 0) { + String sql = "select * from :tableName where lock_id=:lock_id"; + sql = sql.replace(":tableName", tableName); + sql = sql.replace(":lock_id", lockId+""); + + try(Statement statement = getConnection().createStatement(); ResultSet rs = statement.executeQuery(sql)){ + rs.setFetchSize(updateNum); + int count = 0; + while(rs.next()) { + UpdateTableRow row = fromResultSet(rs); + list.add(row); + count++; + } + + if(count != updateNum) + logger.warn("Locked a "+updateNum+" rows but found "+count); + } + } + + return list; + } + + /** * Deletes the row - * @param row + * @param rows */ - protected void deleteRow(UpdateTableRow row){ + protected void deleteRows(List rows){ + try(Statement statement = getConnection().createStatement()){ - try{ - Connection conn = getConnection(); - String sql = "delete from :tableName where ID=:ID"; + String sql = "delete from :tableName where ID in (:idList)"; sql = sql.replace(":tableName", tableName); - sql = sql.replace(":ID", row.getID()+""); - Statement statement = conn.createStatement(); + sql = sql.replace(":idList", listIdJoin(rows)); + int numUpdated = statement.executeUpdate(sql); - if(numUpdated != 1){ - logger.warn("Expected to delete 1, instead "+numUpdated); + if(numUpdated != rows.size()){ + logger.warn("Expected to delete "+rows.size()+", instead "+numUpdated); } }catch(SQLException e){ logger.warn(e.getMessage(), e); } } + /** + * Utility function that takes a list of rows and returns a comma separated string + * @param list + */ + private String listIdJoin(List list){ + StringBuilder sb = new StringBuilder(); + String separator = ","; + for (UpdateTableRow row : list) { + if(sb.length() != 0) + sb.append(separator); + + sb.append(row.getID()); + } + return sb.toString(); + } + @SuppressWarnings({ "rawtypes", "unchecked" }) - public Class getClassForTableName(String tableName){ + public Class getClassForTableName(String tableName){ Class result = null; // First look for the @Table annotation @@ -683,10 +701,10 @@ protected UpdateTableRow fromResultSet(ResultSet resultSet) throws SQLException{ row.lockId = resultSet.getInt("lock_id"); row.lockDate = resultSet.getDate("lock_date"); try { - row.type = UpdateTableRowType.valueOf(resultSet.getString("type")); + row.type = UpdateTableRowType.valueOf(resultSet.getString("type")); } catch(IllegalArgumentException iae) { - logger.warn("Invalid UpdateTableRow TYPE, ignoring"); - row.type = UpdateTableRowType.NONE; + logger.warn("Invalid UpdateTableRow TYPE, ignoring"); + row.type = UpdateTableRowType.NONE; } row.timestamp = resultSet.getDate("time_stamp"); diff --git a/src/test/java/com/percero/agents/sync/jobs/UpdateTableProcessorTest.java b/src/test/java/com/percero/agents/sync/jobs/UpdateTableProcessorTest.java index 765cbdb..56e1053 100644 --- a/src/test/java/com/percero/agents/sync/jobs/UpdateTableProcessorTest.java +++ b/src/test/java/com/percero/agents/sync/jobs/UpdateTableProcessorTest.java @@ -16,6 +16,7 @@ import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Statement; +import java.util.List; /** * Created by Jonathan Samples on 9/4/15. @@ -91,7 +92,8 @@ public void getRow() throws Exception { setupThreeRowsInUpdateTable(); UpdateTableConnectionFactory connectionFactory = new UpdateTableConnectionFactory(); UpdateTableProcessor processor = poller.getProcessor(connectionFactory, tableName); - UpdateTableRow row = processor.getRows(); + List rows = processor.getRows(1); + UpdateTableRow row = rows.get(0); Assert.assertNotNull(row); Assert.assertNotNull(row.getLockId()); From 8fc713c32b5e8617d168913c210d5103b71c52df Mon Sep 17 00:00:00 2001 From: Jonathan Samples Date: Sun, 11 Oct 2015 00:42:30 -0700 Subject: [PATCH 022/105] Added UpdateTable mapping support. Added connection testing to the Connection factory. --- .../jobs/UpdateTableConnectionFactory.java | 2 + .../agents/sync/jobs/UpdateTableMapping.java | 34 ++++++ .../sync/jobs/UpdateTableProcessor.java | 70 +++++------ .../agents/sync/jobs/UpdateTableRegistry.java | 109 ++++++++++++------ .../sync/jobs/UpdateTableProcessorTest.java | 6 +- 5 files changed, 143 insertions(+), 78 deletions(-) create mode 100644 src/main/java/com/percero/agents/sync/jobs/UpdateTableMapping.java diff --git a/src/main/java/com/percero/agents/sync/jobs/UpdateTableConnectionFactory.java b/src/main/java/com/percero/agents/sync/jobs/UpdateTableConnectionFactory.java index 79167db..01b5688 100644 --- a/src/main/java/com/percero/agents/sync/jobs/UpdateTableConnectionFactory.java +++ b/src/main/java/com/percero/agents/sync/jobs/UpdateTableConnectionFactory.java @@ -107,6 +107,8 @@ public void init() throws PropertyVetoException{ cpds.setMinPoolSize(10); cpds.setAcquireIncrement(5); cpds.setMaxPoolSize(this.weight); + cpds.setTestConnectionOnCheckout(true); + }catch(PropertyVetoException pve){ logger.error(pve.getMessage(), pve); throw pve; diff --git a/src/main/java/com/percero/agents/sync/jobs/UpdateTableMapping.java b/src/main/java/com/percero/agents/sync/jobs/UpdateTableMapping.java new file mode 100644 index 0000000..5e5d90d --- /dev/null +++ b/src/main/java/com/percero/agents/sync/jobs/UpdateTableMapping.java @@ -0,0 +1,34 @@ +package com.percero.agents.sync.jobs; + +import org.apache.log4j.Logger; + +import java.util.ArrayList; +import java.util.List; + +/** + * Maintains a mapping from a table to a list of classes that + * are affected by a change in it. + * + * Created by jonnysamps on 10/10/15. + */ +public class UpdateTableMapping { + + private static Logger logger = Logger.getLogger(UpdateTableMapping.class); + public String tableName; + public List classNames; + public List classes = new ArrayList<>(); + + /** + * Look up the classes and cache references to them + */ + public void init(){ + for(String className: classNames){ + try{ + Class c = Class.forName(className); + classes.add(c); + }catch(ClassNotFoundException e){ + logger.warn("Class in updateTableMap.yml not found: "+className); + } + } + } +} diff --git a/src/main/java/com/percero/agents/sync/jobs/UpdateTableProcessor.java b/src/main/java/com/percero/agents/sync/jobs/UpdateTableProcessor.java index 40956b6..349c446 100644 --- a/src/main/java/com/percero/agents/sync/jobs/UpdateTableProcessor.java +++ b/src/main/java/com/percero/agents/sync/jobs/UpdateTableProcessor.java @@ -16,7 +16,6 @@ import org.joda.time.DateTime; import org.springframework.util.StringUtils; -import javax.persistence.Table; import java.sql.*; import java.util.*; import java.util.Date; @@ -105,7 +104,7 @@ public void run(){ Date endTime = new Date(); UpdateTableProcessReporter.getInstance() - .submitCountAndTime(tableName, successfulRows.size(), endTime.getTime()-startTime.getTime()); + .submitCountAndTime(tableName, successfulRows.size(), endTime.getTime() - startTime.getTime()); } if (!result.isSuccess()) { @@ -163,10 +162,9 @@ protected boolean processRow(UpdateTableRow row) throws Exception{ */ @SuppressWarnings("rawtypes") protected boolean processUpdateSingle(UpdateTableRow row) throws Exception{ - Class clazz = getClassForTableName(row.getTableName()); + List classes = getClassesForTableName(row.getTableName()); - // If we found the class then we care about this row, otherwise return true and the row will be deleted - if(clazz != null) { + for(Class clazz : classes) { String className = clazz.getCanonicalName(); IMappedClassManager mcm = MappedClassManagerFactory.getMappedClassManager(); @@ -192,9 +190,9 @@ protected boolean processUpdateSingle(UpdateTableRow row) throws Exception{ */ @SuppressWarnings("rawtypes") protected boolean processUpdateTable(UpdateTableRow row) throws Exception{ - Class clazz = getClassForTableName(row.getTableName()); + List classes = getClassesForTableName(row.getTableName()); - if(clazz != null) { + for(Class clazz : classes) { String className = clazz.getCanonicalName(); // If there are any clients that have asked for all objects in a class then we have to push everything @@ -268,10 +266,9 @@ protected void handleUpdateClassIdPair(IDataProvider dataProvider, ClassIDPair p */ @SuppressWarnings("rawtypes") protected boolean processInsertSingle(UpdateTableRow row) throws Exception{ - Class clazz = getClassForTableName(row.getTableName()); + List classes = getClassesForTableName(row.getTableName()); - // If clazz is not null then we care about this row, otherwise throw the row away - if(clazz != null) { + for(Class clazz : classes) { String className = clazz.getCanonicalName(); // We do not use PostCreateHelper here because we are going to do all @@ -291,9 +288,9 @@ protected boolean processInsertSingle(UpdateTableRow row) throws Exception{ */ @SuppressWarnings("rawtypes") protected boolean processInsertTable(UpdateTableRow row) throws Exception { - Class clazz = getClassForTableName(row.getTableName()); + List classes = getClassesForTableName(row.getTableName()); - if(clazz != null) { + for(Class clazz : classes) { String className = clazz.getCanonicalName(); // if any client needs all of this class then the only choice we have is to push everything @@ -320,9 +317,9 @@ protected boolean processInsertTable(UpdateTableRow row) throws Exception { */ @SuppressWarnings("rawtypes") protected boolean processDeleteSingle(UpdateTableRow row) throws Exception{ - Class clazz = getClassForTableName(row.getTableName()); + List classes = getClassesForTableName(row.getTableName()); - if(clazz != null) { + for(Class clazz : classes){ String className = clazz.getCanonicalName(); // See if this object is in the cache. If so, it will help us know which related objects to update. @@ -347,9 +344,9 @@ protected boolean processDeleteSingle(UpdateTableRow row) throws Exception{ */ @SuppressWarnings("rawtypes") protected boolean processDeleteTable(UpdateTableRow row) throws Exception{ - Class clazz = getClassForTableName(row.getTableName()); + List classes = getClassesForTableName(row.getTableName()); - if(clazz != null) { + for(Class clazz : classes){ String className = clazz.getCanonicalName(); IMappedClassManager mcm = MappedClassManagerFactory.getMappedClassManager(); @@ -409,13 +406,16 @@ protected void handleDeletedObject(IPerceroObject cachedObject, Class clazz, Str @SuppressWarnings("rawtypes") protected Set getAllClassIdPairsForTable(String tableName) throws Exception{ - Class clazz = getClassForTableName(tableName); - String className = clazz.getCanonicalName(); + Set results = new HashSet<>(); + List classes = getClassesForTableName(tableName); - IMappedClassManager mcm = MappedClassManagerFactory.getMappedClassManager(); - MappedClass mappedClass = mcm.getMappedClassByClassName(className); - IDataProvider dataProvider = dataProviderManager.getDataProviderByName(mappedClass.dataProviderName); - Set results = dataProvider.getAllClassIdPairsByName(className); + for(Class clazz : classes) { + String className = clazz.getCanonicalName(); + IMappedClassManager mcm = MappedClassManagerFactory.getMappedClassManager(); + MappedClass mappedClass = mcm.getMappedClassByClassName(className); + IDataProvider dataProvider = dataProviderManager.getDataProviderByName(mappedClass.dataProviderName); + results.addAll(dataProvider.getAllClassIdPairsByName(className)); + } return results; } @@ -667,26 +667,12 @@ private String listIdJoin(List list){ } @SuppressWarnings({ "rawtypes", "unchecked" }) - public Class getClassForTableName(String tableName){ - Class result = null; - - // First look for the @Table annotation - for(Class c : manifest.getClassList()){ - Table table = (Table) c.getAnnotation(Table.class); - if(table != null && tableName.equals(table.name())) { - result = c; - break; - } - } + public List getClassesForTableName(String tableName){ + List result = new ArrayList<>(); - // If we didn't find that now look for the simple class name to match - if(result == null){ - for(Class c : manifest.getClassList()){ - if(tableName.equals(c.getSimpleName())) { - result = c; - break; - } - } + UpdateTableMapping mapping = UpdateTableRegistry.getInstance().getTableMapping(tableName); + if(mapping != null){ + result.addAll(mapping.classes); } return result; @@ -712,7 +698,7 @@ protected UpdateTableRow fromResultSet(ResultSet resultSet) throws SQLException{ } private Connection getConnection() throws SQLException{ - if(this.connection == null || this.connection.isClosed()) + if(this.connection == null || !this.connection.isValid(10)) // 10 second timeout this.connection = this.connectionFactory.getConnection(); return this.connection; diff --git a/src/main/java/com/percero/agents/sync/jobs/UpdateTableRegistry.java b/src/main/java/com/percero/agents/sync/jobs/UpdateTableRegistry.java index 3c90751..cc58480 100644 --- a/src/main/java/com/percero/agents/sync/jobs/UpdateTableRegistry.java +++ b/src/main/java/com/percero/agents/sync/jobs/UpdateTableRegistry.java @@ -1,22 +1,21 @@ package com.percero.agents.sync.jobs; +import com.esotericsoftware.yamlbeans.YamlException; +import com.esotericsoftware.yamlbeans.YamlReader; +import edu.emory.mathcs.backport.java.util.Collections; +import org.apache.log4j.Logger; +import org.springframework.stereotype.Component; + +import javax.annotation.PostConstruct; import java.beans.PropertyVetoException; import java.io.File; import java.io.FileNotFoundException; import java.io.FileReader; import java.net.URL; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; - -import javax.annotation.PostConstruct; - -import org.apache.log4j.Logger; -import org.springframework.stereotype.Component; - -import com.esotericsoftware.yamlbeans.YamlException; -import com.esotericsoftware.yamlbeans.YamlReader; - -import edu.emory.mathcs.backport.java.util.Collections; +import java.util.Map; /** * @author Collin Brown @@ -27,39 +26,83 @@ public class UpdateTableRegistry { private static Logger logger = Logger.getLogger(UpdateTableRegistry.class); + private static UpdateTableRegistry instance; + @SuppressWarnings("unchecked") private List connectionFactories = Collections.synchronizedList(new ArrayList()); public List getConnectionFactories() { return connectionFactories; } + + private Map tableMap = new HashMap<>(); + public UpdateTableMapping getTableMapping(String tableName){ + return tableMap.get(tableName); + } + + public UpdateTableRegistry(){ + instance = this; + } + + public static UpdateTableRegistry getInstance(){ + return instance; + } @PostConstruct public void init() throws YamlException { + loadConnectionFactories(); + loadTableMappings(); + } + + private void loadTableMappings() throws YamlException{ + try{ + URL ymlUrl = UpdateTableRegistry.class.getClassLoader().getResource("updateTableMap.yml"); + if (ymlUrl == null) { + logger.warn("No configuration found for UpdateTableMapping (updateTableMap.yml), skipping UpdateTables"); + return; + } + File configFile = new File(ymlUrl.getFile()); + YamlReader reader = new YamlReader(new FileReader(configFile)); + while (true) { + UpdateTableMapping updateTableMapping = reader.read(UpdateTableMapping.class); + if (updateTableMapping == null) { + break; + } + + updateTableMapping.init(); + tableMap.put(updateTableMapping.tableName, updateTableMapping); + } + }catch (FileNotFoundException e) { + logger.warn("No configuration found for UpdateTables (updateTables.yml), skipping UpdateTables"); + } + } + + private void loadConnectionFactories() throws YamlException{ try { - URL ymlUrl = UpdateTableRegistry.class.getClassLoader().getResource("updateTables.yml"); - if (ymlUrl == null) { - logger.warn("No configuration found for UpdateTables (updateTables.yml), skipping UpdateTables"); - return; - } - File configFile = new File(ymlUrl.getFile()); - YamlReader reader = new YamlReader(new FileReader(configFile)); - while (true) { - UpdateTableConnectionFactory updateTableConnectionFactory = reader.read(UpdateTableConnectionFactory.class); - if (updateTableConnectionFactory == null) { - break; - } - - try { - updateTableConnectionFactory.init(); - connectionFactories.add(updateTableConnectionFactory); - } catch (PropertyVetoException e) { - logger.warn("Unable to initialize Update Table Connection Factory", e); - } - - } + URL ymlUrl = UpdateTableRegistry.class.getClassLoader().getResource("updateTables.yml"); + if (ymlUrl == null) { + logger.warn("No configuration found for UpdateTables (updateTables.yml), skipping UpdateTables"); + return; + } + File configFile = new File(ymlUrl.getFile()); + YamlReader reader = new YamlReader(new FileReader(configFile)); + while (true) { + UpdateTableConnectionFactory updateTableConnectionFactory = reader.read(UpdateTableConnectionFactory.class); + if (updateTableConnectionFactory == null) { + break; + } + + try { + updateTableConnectionFactory.init(); + connectionFactories.add(updateTableConnectionFactory); + } catch (PropertyVetoException e) { + logger.warn("Unable to initialize Update Table Connection Factory", e); + } + + } } catch (FileNotFoundException e) { - logger.warn("No configuration found for UpdateTables (updateTables.yml), skipping UpdateTables"); - } + logger.warn("No configuration found for UpdateTables (updateTables.yml), skipping UpdateTables"); + } + } } diff --git a/src/test/java/com/percero/agents/sync/jobs/UpdateTableProcessorTest.java b/src/test/java/com/percero/agents/sync/jobs/UpdateTableProcessorTest.java index 56e1053..1d57a6f 100644 --- a/src/test/java/com/percero/agents/sync/jobs/UpdateTableProcessorTest.java +++ b/src/test/java/com/percero/agents/sync/jobs/UpdateTableProcessorTest.java @@ -54,7 +54,7 @@ public void before() throws Exception{ public void getClassForTableName_NoTableAnnotation() throws Exception{ UpdateTableConnectionFactory connectionFactory = new UpdateTableConnectionFactory(); UpdateTableProcessor processor = poller.getProcessor(connectionFactory, tableName); - Class clazz = processor.getClassForTableName("Email"); + List clazz = processor.getClassesForTableName("Email"); Assert.assertEquals(Email.class, clazz); } @@ -62,7 +62,7 @@ public void getClassForTableName_NoTableAnnotation() throws Exception{ public void getClassForTableName_TableAnnotation() throws Exception{ UpdateTableConnectionFactory connectionFactory = new UpdateTableConnectionFactory(); UpdateTableProcessor processor = poller.getProcessor(connectionFactory, tableName); - Class clazz = processor.getClassForTableName("Person"); + List clazz = processor.getClassesForTableName("Person"); Assert.assertEquals(Person.class, clazz); } @@ -70,7 +70,7 @@ public void getClassForTableName_TableAnnotation() throws Exception{ public void getClassForTableName_NotFound() throws Exception{ UpdateTableConnectionFactory connectionFactory = new UpdateTableConnectionFactory(); UpdateTableProcessor processor = poller.getProcessor(connectionFactory, tableName); - Class clazz = processor.getClassForTableName("NotAnEntity"); + List clazz = processor.getClassesForTableName("NotAnEntity"); Assert.assertNull(clazz); } From d0e4a8a1d6e174c00a90f840901592664d2fd095 Mon Sep 17 00:00:00 2001 From: Jonathan Samples Date: Sun, 11 Oct 2015 00:56:51 -0700 Subject: [PATCH 023/105] Removed isValid check... not supported by Oracle driver I guess --- .../java/com/percero/agents/sync/jobs/UpdateTableProcessor.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/percero/agents/sync/jobs/UpdateTableProcessor.java b/src/main/java/com/percero/agents/sync/jobs/UpdateTableProcessor.java index 349c446..0895f22 100644 --- a/src/main/java/com/percero/agents/sync/jobs/UpdateTableProcessor.java +++ b/src/main/java/com/percero/agents/sync/jobs/UpdateTableProcessor.java @@ -698,7 +698,7 @@ protected UpdateTableRow fromResultSet(ResultSet resultSet) throws SQLException{ } private Connection getConnection() throws SQLException{ - if(this.connection == null || !this.connection.isValid(10)) // 10 second timeout + if(this.connection == null || this.connection.isClosed()) // 10 second timeout this.connection = this.connectionFactory.getConnection(); return this.connection; From fcf09ef0ac01169da0f7b73a42306c43fddd9e11 Mon Sep 17 00:00:00 2001 From: Jonathan Samples Date: Sun, 11 Oct 2015 21:10:16 -0700 Subject: [PATCH 024/105] Fixed some object construction issues --- .../percero/agents/sync/dao/DAORegistry.java | 10 +- .../agents/sync/services/DAODataProvider.java | 2355 ++++++++--------- 2 files changed, 1175 insertions(+), 1190 deletions(-) diff --git a/src/main/java/com/percero/agents/sync/dao/DAORegistry.java b/src/main/java/com/percero/agents/sync/dao/DAORegistry.java index aa6d584..c1401ee 100644 --- a/src/main/java/com/percero/agents/sync/dao/DAORegistry.java +++ b/src/main/java/com/percero/agents/sync/dao/DAORegistry.java @@ -1,14 +1,11 @@ package com.percero.agents.sync.dao; -import java.util.HashMap; -import java.util.Map; - -import com.percero.agents.sync.services.DAODataProvider; -import com.percero.agents.sync.services.DataProviderManager; import com.percero.framework.vo.IPerceroObject; - import edu.emory.mathcs.backport.java.util.Collections; +import java.util.HashMap; +import java.util.Map; + public class DAORegistry { private static DAORegistry instance = null; @@ -22,7 +19,6 @@ public static DAORegistry getInstance() { public DAORegistry() { instance = this; - DataProviderManager.getInstance().addDataProvider(DAODataProvider.getInstance()); } @SuppressWarnings({ "unchecked" }) diff --git a/src/main/java/com/percero/agents/sync/services/DAODataProvider.java b/src/main/java/com/percero/agents/sync/services/DAODataProvider.java index fe25bb5..5c578a0 100644 --- a/src/main/java/com/percero/agents/sync/services/DAODataProvider.java +++ b/src/main/java/com/percero/agents/sync/services/DAODataProvider.java @@ -1,24 +1,22 @@ package com.percero.agents.sync.services; -import java.lang.reflect.InvocationTargetException; -import java.lang.reflect.Method; -import java.util.ArrayList; -import java.util.Collection; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.Map.Entry; -import java.util.Set; -import java.util.UUID; -import java.util.concurrent.TimeUnit; - +import com.percero.agents.sync.access.RedisKeyUtils; +import com.percero.agents.sync.dao.DAORegistry; +import com.percero.agents.sync.dao.IDataAccessObject; +import com.percero.agents.sync.datastore.ICacheDataStore; +import com.percero.agents.sync.exceptions.SyncDataException; +import com.percero.agents.sync.exceptions.SyncException; +import com.percero.agents.sync.metadata.*; +import com.percero.agents.sync.vo.BaseDataObject; +import com.percero.agents.sync.vo.ClassIDPair; +import com.percero.agents.sync.vo.ClassIDPairs; +import com.percero.agents.sync.vo.IJsonObject; +import com.percero.framework.vo.IPerceroObject; +import com.percero.framework.vo.PerceroList; import javassist.ClassPool; import javassist.CtClass; import javassist.CtField; import javassist.CtMethod; - import org.apache.log4j.Logger; import org.codehaus.jackson.map.ObjectMapper; import org.hibernate.PropertyValueException; @@ -28,387 +26,377 @@ import org.springframework.stereotype.Component; import org.springframework.util.StringUtils; -import com.percero.agents.sync.access.RedisKeyUtils; -import com.percero.agents.sync.dao.DAORegistry; -import com.percero.agents.sync.dao.IDataAccessObject; -import com.percero.agents.sync.datastore.ICacheDataStore; -import com.percero.agents.sync.exceptions.SyncDataException; -import com.percero.agents.sync.exceptions.SyncException; -import com.percero.agents.sync.metadata.IMappedClassManager; -import com.percero.agents.sync.metadata.MappedClass; -import com.percero.agents.sync.metadata.MappedClassManagerFactory; -import com.percero.agents.sync.metadata.MappedField; -import com.percero.agents.sync.metadata.MappedFieldList; -import com.percero.agents.sync.metadata.MappedFieldPerceroObject; -import com.percero.agents.sync.vo.BaseDataObject; -import com.percero.agents.sync.vo.ClassIDPair; -import com.percero.agents.sync.vo.ClassIDPairs; -import com.percero.agents.sync.vo.IJsonObject; -import com.percero.framework.vo.IPerceroObject; -import com.percero.framework.vo.PerceroList; +import javax.annotation.PostConstruct; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.util.*; +import java.util.Map.Entry; +import java.util.concurrent.TimeUnit; @Component public class DAODataProvider implements IDataProvider { - // TODO: Better manage Hibernate Sessions (opening and closing). + // TODO: Better manage Hibernate Sessions (opening and closing). - private static final Logger log = Logger.getLogger(DAODataProvider.class); - - private static DAODataProvider instance = null; - - public static DAODataProvider getInstance() { -// if (instance == null) { -// instance = new DAODataProvider(); -// instance.initialize(); -// } - return instance; - } - - public DAODataProvider() { - instance = this; - initialize(); - } - - public void initialize() - { - // Do nothing. - } - - public String getName() { - return "daoDataProvider"; - } - - @Autowired - IDataProviderManager dataProviderManager; - public void setDataProviderManager(IDataProviderManager value) { - dataProviderManager = value; - } - - @Autowired - ICacheDataStore cacheDataStore; - - @Autowired - Long cacheTimeout = Long.valueOf(60 * 60 * 24 * 14); // Two weeks + private static final Logger log = Logger.getLogger(DAODataProvider.class); - @Autowired - ObjectMapper safeObjectMapper; + private static DAODataProvider instance = null; - - @SuppressWarnings({ "unchecked" }) - // TODO: @Transactional(readOnly=true) - public PerceroList getAllByName(String className, Integer pageNumber, Integer pageSize, Boolean returnTotal, String userId) throws Exception { - IDataAccessObject dao = (IDataAccessObject) DAORegistry.getInstance().getDataAccessObject(className); - PerceroList results = dao.getAll(pageNumber, pageSize, returnTotal, userId, false); - - if (results != null && !results.isEmpty()) { - Iterator itrResults = results.iterator(); - while (itrResults.hasNext()) { - IPerceroObject nextResult = itrResults.next(); - try { - populateToManyRelationships(nextResult, true, null); - populateToOneRelationships(nextResult, true, null); - } catch (IllegalArgumentException e) { - e.printStackTrace(); - throw new SyncDataException(e); - } catch (IllegalAccessException e) { - e.printStackTrace(); - throw new SyncDataException(e); - } catch (InvocationTargetException e) { - e.printStackTrace(); - throw new SyncDataException(e); - } - } - } - - putObjectsInRedisCache(results); - - // Now clean the objects for the user. - List cleanedObjects = cleanObject(results, userId); - results.clear(); - results.addAll(cleanedObjects); - return results; - } - - @SuppressWarnings("unchecked") - public Set getAllClassIdPairsByName(String className) throws Exception { - IDataAccessObject dao = (IDataAccessObject) DAORegistry.getInstance().getDataAccessObject(className); - PerceroList allObjects = dao.getAll(null, null, false, null, true); - - Set results = new HashSet(allObjects == null ? 0 : allObjects.size()); - - if (allObjects != null && !allObjects.isEmpty()) { - Iterator itrResults = allObjects.iterator(); - while (itrResults.hasNext()) { - IPerceroObject nextResult = itrResults.next(); - results.add(BaseDataObject.toClassIdPair(nextResult)); - } - } - - return results; - } - - @SuppressWarnings({ "unchecked" }) - // TODO: @Transactional(readOnly=true) - public Integer countAllByName(String className, String userId) throws Exception { - IDataAccessObject dao = (IDataAccessObject) DAORegistry.getInstance().getDataAccessObject(className); - Integer result = dao.countAll(userId); - return result; - } - - @SuppressWarnings("unchecked") - public List runQuery(MappedClass mappedClass, String queryName, Object[] queryArguments, String userId) throws SyncException { - IDataAccessObject dao = (IDataAccessObject) DAORegistry.getInstance().getDataAccessObject(mappedClass.className); - List result = dao.runQuery(queryName, queryArguments, userId); - return result; - } - - // TODO: This method has not been tested and is most likely broken. - @SuppressWarnings({ "rawtypes", "unchecked" }) - protected static List processQueryResults(String resultClassName, Query updateFilter, List updateFilterResult) throws Exception { - String[] returnAliases = updateFilter.getReturnAliases(); - Type[] returnTypes = updateFilter.getReturnTypes(); - - String[] fieldNames = new String[returnAliases.length]; - String[] getMethodNames = new String[returnAliases.length]; - String[] setMethodNames = new String[returnAliases.length]; - Class[] fieldClasses = new Class[returnAliases.length]; - - Class clazz = null; - ClassPool pool = null; - CtClass evalClass = null; - - if (returnAliases.length > 1) { - try { - clazz = MappedClass.forName(resultClassName); - } catch(Exception e) { - // Class must not yet exist, so let's create it. - } - - if (clazz == null) { - pool = ClassPool.getDefault(); - evalClass = pool.makeClass(resultClassName); - } - - // Create a new Class based on the result set. - for(int i = 0; i < returnAliases.length; i++) { - Type nextType = returnTypes[i]; - String nextTypeCanonicalName = nextType.getReturnedClass().getCanonicalName(); - String nextFieldName = returnAliases[i]; - try { - Integer.parseInt(nextFieldName); - nextFieldName = "field" + i; - } catch(NumberFormatException nfe) { - // Do nothing. Simply means the field name is not a Number. - } - String nextUpperFieldName = nextFieldName.substring(0, 1).toUpperCase() + nextFieldName.substring(1); - - fieldNames[i] = nextFieldName; - getMethodNames[i] = "get" + nextUpperFieldName; - setMethodNames[i] = "set" + nextUpperFieldName; - fieldClasses[i] = nextType.getReturnedClass(); - - if (evalClass != null) { - evalClass.addField(CtField.make("private " + fieldClasses[i].getCanonicalName() + " " + nextFieldName + ";", evalClass)); - evalClass.addMethod(CtMethod.make("public void " + setMethodNames[i] + "(" + fieldClasses[i].getCanonicalName() + " value) {this." + nextFieldName + " = value;}", evalClass)); - evalClass.addMethod(CtMethod.make("public " + nextTypeCanonicalName +" " + getMethodNames[i] + "() {return this." + nextFieldName + ";}", evalClass)); - } - } - - if (clazz == null && evalClass != null) { - clazz = evalClass.toClass(); - } - } - - List results = new ArrayList(); - - // Now populate the newly created objects. - for(Object nextResult : (List)updateFilterResult) { - - if (nextResult instanceof Object[]) { - Object nextObject = clazz.newInstance(); - for(int i = 0; i < returnAliases.length; i++) { - Class[] formalParams = new Class[] { fieldClasses[i] }; - Method setMethod = clazz.getDeclaredMethod(setMethodNames[i], formalParams); - setMethod.invoke(nextObject, ((Object[])nextResult)[i]); - } - - results.add(nextObject); - } else - results.add(nextResult); - } - - return results; - } - - public IPerceroObject findById(ClassIDPair classIdPair, String userId) { - return findById(classIdPair, userId, false); - } - @SuppressWarnings("unchecked") - public IPerceroObject findById(ClassIDPair classIdPair, String userId, Boolean ignoreCache) { - - try { - IPerceroObject result = null; - if (!ignoreCache) { - result = retrieveFromRedisCache(classIdPair); - } - - if (result == null) { - - IDataAccessObject dao = (IDataAccessObject) DAORegistry.getInstance().getDataAccessObject(classIdPair.getClassName()); - // Retrieve results BEFORE applying access rules so that our cached value represents the full object. - result = dao.retrieveObject(classIdPair, null, false); - populateToManyRelationships(result, true, null); - populateToOneRelationships(result, true, null); - - // Now put the object in the cache. - if (result != null) { - putObjectInRedisCache(result); - } - else { - // Not necessarily a problem but could be helpful when debugging. - log.debug("Unable to retrieve object from database: " + classIdPair.toJson()); - } - } - else { - // (Re)Set the expiration. - setObjectExpiration(result); - } - - result = cleanObject(result, userId); - - return result; - } catch(Exception e) { - log.error("Unable to findById: "+classIdPair.toJson(), e); - } - - return null; - } - + public static DAODataProvider getInstance() { + return instance; + } - private void putObjectInRedisCache(IPerceroObject perceroObject) { - // Now put the object in the cache. - if (cacheTimeout > 0 && perceroObject != null) { - String key = RedisKeyUtils.classIdPair(perceroObject.getClass().getCanonicalName(), perceroObject.getID()); - cacheDataStore.setValue(key, ((BaseDataObject)perceroObject).toJson()); - setObjectExpiration(key); - } - } - - private void putObjectsInRedisCache(List results) { - if (cacheTimeout > 0) { - Map mapJsonObjectStrings = new HashMap(results.size()); - Iterator itrDatabaseObjects = results.iterator(); - while (itrDatabaseObjects.hasNext()) { - IPerceroObject nextDatabaseObject = itrDatabaseObjects.next(); - String nextCacheKey = RedisKeyUtils.classIdPair(nextDatabaseObject.getClass().getCanonicalName(), nextDatabaseObject.getID()); - - mapJsonObjectStrings.put(nextCacheKey, ((BaseDataObject)nextDatabaseObject).toJson()); - } - - // Store the objects in redis. - cacheDataStore.setValues(mapJsonObjectStrings); - // (Re)Set the expiration. - cacheDataStore.expire(mapJsonObjectStrings.keySet(), cacheTimeout, TimeUnit.SECONDS); - } - } - - private void setObjectExpiration(IPerceroObject perceroObject) { - setObjectExpiration(RedisKeyUtils.classIdPair(perceroObject.getClass().getCanonicalName(), perceroObject.getID())); - } - - private void setObjectExpiration(String key) { - // (Re)Set the expiration. - if (cacheTimeout > 0 && key != null) { - cacheDataStore.expire(key, cacheTimeout, TimeUnit.SECONDS); - } - } + public DAODataProvider() { + instance = this; + } + + @PostConstruct + public void init() + { + dataProviderManager.addDataProvider(this); + ((DataProviderManager)dataProviderManager).setDefaultDataProvider(this); + } + + public void initialize(){ + // Do nothing + } + + public String getName() { + return "daoDataProvider"; + } + + @Autowired + IDataProviderManager dataProviderManager; + public void setDataProviderManager(IDataProviderManager value) { + dataProviderManager = value; + } + + @Autowired + ICacheDataStore cacheDataStore; + + @Autowired + Long cacheTimeout = Long.valueOf(60 * 60 * 24 * 14); // Two weeks + + @Autowired + ObjectMapper safeObjectMapper; + + + + @SuppressWarnings({ "unchecked" }) + // TODO: @Transactional(readOnly=true) + public PerceroList getAllByName(String className, Integer pageNumber, Integer pageSize, Boolean returnTotal, String userId) throws Exception { + IDataAccessObject dao = (IDataAccessObject) DAORegistry.getInstance().getDataAccessObject(className); + PerceroList results = dao.getAll(pageNumber, pageSize, returnTotal, userId, false); + + if (results != null && !results.isEmpty()) { + Iterator itrResults = results.iterator(); + while (itrResults.hasNext()) { + IPerceroObject nextResult = itrResults.next(); + try { + populateToManyRelationships(nextResult, true, null); + populateToOneRelationships(nextResult, true, null); + } catch (IllegalArgumentException e) { + e.printStackTrace(); + throw new SyncDataException(e); + } catch (IllegalAccessException e) { + e.printStackTrace(); + throw new SyncDataException(e); + } catch (InvocationTargetException e) { + e.printStackTrace(); + throw new SyncDataException(e); + } + } + } + + putObjectsInRedisCache(results); + + // Now clean the objects for the user. + List cleanedObjects = cleanObject(results, userId); + results.clear(); + results.addAll(cleanedObjects); + return results; + } + + @SuppressWarnings("unchecked") + public Set getAllClassIdPairsByName(String className) throws Exception { + IDataAccessObject dao = (IDataAccessObject) DAORegistry.getInstance().getDataAccessObject(className); + PerceroList allObjects = dao.getAll(null, null, false, null, true); + + Set results = new HashSet(allObjects == null ? 0 : allObjects.size()); + + if (allObjects != null && !allObjects.isEmpty()) { + Iterator itrResults = allObjects.iterator(); + while (itrResults.hasNext()) { + IPerceroObject nextResult = itrResults.next(); + results.add(BaseDataObject.toClassIdPair(nextResult)); + } + } + + return results; + } + + @SuppressWarnings({ "unchecked" }) + // TODO: @Transactional(readOnly=true) + public Integer countAllByName(String className, String userId) throws Exception { + IDataAccessObject dao = (IDataAccessObject) DAORegistry.getInstance().getDataAccessObject(className); + Integer result = dao.countAll(userId); + return result; + } + + @SuppressWarnings("unchecked") + public List runQuery(MappedClass mappedClass, String queryName, Object[] queryArguments, String userId) throws SyncException { + IDataAccessObject dao = (IDataAccessObject) DAORegistry.getInstance().getDataAccessObject(mappedClass.className); + List result = dao.runQuery(queryName, queryArguments, userId); + return result; + } + + // TODO: This method has not been tested and is most likely broken. + @SuppressWarnings({ "rawtypes", "unchecked" }) + protected static List processQueryResults(String resultClassName, Query updateFilter, List updateFilterResult) throws Exception { + String[] returnAliases = updateFilter.getReturnAliases(); + Type[] returnTypes = updateFilter.getReturnTypes(); + + String[] fieldNames = new String[returnAliases.length]; + String[] getMethodNames = new String[returnAliases.length]; + String[] setMethodNames = new String[returnAliases.length]; + Class[] fieldClasses = new Class[returnAliases.length]; + + Class clazz = null; + ClassPool pool = null; + CtClass evalClass = null; + + if (returnAliases.length > 1) { + try { + clazz = MappedClass.forName(resultClassName); + } catch(Exception e) { + // Class must not yet exist, so let's create it. + } + + if (clazz == null) { + pool = ClassPool.getDefault(); + evalClass = pool.makeClass(resultClassName); + } + + // Create a new Class based on the result set. + for(int i = 0; i < returnAliases.length; i++) { + Type nextType = returnTypes[i]; + String nextTypeCanonicalName = nextType.getReturnedClass().getCanonicalName(); + String nextFieldName = returnAliases[i]; + try { + Integer.parseInt(nextFieldName); + nextFieldName = "field" + i; + } catch(NumberFormatException nfe) { + // Do nothing. Simply means the field name is not a Number. + } + String nextUpperFieldName = nextFieldName.substring(0, 1).toUpperCase() + nextFieldName.substring(1); + + fieldNames[i] = nextFieldName; + getMethodNames[i] = "get" + nextUpperFieldName; + setMethodNames[i] = "set" + nextUpperFieldName; + fieldClasses[i] = nextType.getReturnedClass(); + + if (evalClass != null) { + evalClass.addField(CtField.make("private " + fieldClasses[i].getCanonicalName() + " " + nextFieldName + ";", evalClass)); + evalClass.addMethod(CtMethod.make("public void " + setMethodNames[i] + "(" + fieldClasses[i].getCanonicalName() + " value) {this." + nextFieldName + " = value;}", evalClass)); + evalClass.addMethod(CtMethod.make("public " + nextTypeCanonicalName +" " + getMethodNames[i] + "() {return this." + nextFieldName + ";}", evalClass)); + } + } + + if (clazz == null && evalClass != null) { + clazz = evalClass.toClass(); + } + } + + List results = new ArrayList(); + + // Now populate the newly created objects. + for(Object nextResult : (List)updateFilterResult) { + + if (nextResult instanceof Object[]) { + Object nextObject = clazz.newInstance(); + for(int i = 0; i < returnAliases.length; i++) { + Class[] formalParams = new Class[] { fieldClasses[i] }; + Method setMethod = clazz.getDeclaredMethod(setMethodNames[i], formalParams); + setMethod.invoke(nextObject, ((Object[])nextResult)[i]); + } + + results.add(nextObject); + } else + results.add(nextResult); + } + + return results; + } + + public IPerceroObject findById(ClassIDPair classIdPair, String userId) { + return findById(classIdPair, userId, false); + } + @SuppressWarnings("unchecked") + public IPerceroObject findById(ClassIDPair classIdPair, String userId, Boolean ignoreCache) { + + try { + IPerceroObject result = null; + if (!ignoreCache) { + result = retrieveFromRedisCache(classIdPair); + } + + if (result == null) { + + IDataAccessObject dao = (IDataAccessObject) DAORegistry.getInstance().getDataAccessObject(classIdPair.getClassName()); + // Retrieve results BEFORE applying access rules so that our cached value represents the full object. + result = dao.retrieveObject(classIdPair, null, false); + populateToManyRelationships(result, true, null); + populateToOneRelationships(result, true, null); + + // Now put the object in the cache. + if (result != null) { + putObjectInRedisCache(result); + } + else { + // Not necessarily a problem but could be helpful when debugging. + log.debug("Unable to retrieve object from database: " + classIdPair.toJson()); + } + } + else { + // (Re)Set the expiration. + setObjectExpiration(result); + } + + result = cleanObject(result, userId); + + return result; + } catch(Exception e) { + log.error("Unable to findById: "+classIdPair.toJson(), e); + } + + return null; + } + + + private void putObjectInRedisCache(IPerceroObject perceroObject) { + // Now put the object in the cache. + if (cacheTimeout > 0 && perceroObject != null) { + String key = RedisKeyUtils.classIdPair(perceroObject.getClass().getCanonicalName(), perceroObject.getID()); + cacheDataStore.setValue(key, ((BaseDataObject)perceroObject).toJson()); + setObjectExpiration(key); + } + } + + private void putObjectsInRedisCache(List results) { + if (cacheTimeout > 0) { + Map mapJsonObjectStrings = new HashMap(results.size()); + Iterator itrDatabaseObjects = results.iterator(); + while (itrDatabaseObjects.hasNext()) { + IPerceroObject nextDatabaseObject = itrDatabaseObjects.next(); + String nextCacheKey = RedisKeyUtils.classIdPair(nextDatabaseObject.getClass().getCanonicalName(), nextDatabaseObject.getID()); + + mapJsonObjectStrings.put(nextCacheKey, ((BaseDataObject)nextDatabaseObject).toJson()); + } + + // Store the objects in redis. + cacheDataStore.setValues(mapJsonObjectStrings); + // (Re)Set the expiration. + cacheDataStore.expire(mapJsonObjectStrings.keySet(), cacheTimeout, TimeUnit.SECONDS); + } + } + + private void setObjectExpiration(IPerceroObject perceroObject) { + setObjectExpiration(RedisKeyUtils.classIdPair(perceroObject.getClass().getCanonicalName(), perceroObject.getID())); + } + + private void setObjectExpiration(String key) { + // (Re)Set the expiration. + if (cacheTimeout > 0 && key != null) { + cacheDataStore.expire(key, cacheTimeout, TimeUnit.SECONDS); + } + } + + + @SuppressWarnings({ "rawtypes", "unchecked" }) + private IPerceroObject retrieveFromRedisCache(ClassIDPair classIdPair) throws Exception { + IPerceroObject result = null; + if (cacheTimeout > 0) { + IMappedClassManager mcm = MappedClassManagerFactory.getMappedClassManager(); + Class theClass = MappedClass.forName(classIdPair.getClassName()); + MappedClass mc = mcm.getMappedClassByClassName(classIdPair.getClassName()); + + String key = RedisKeyUtils.classIdPair(classIdPair.getClassName(), classIdPair.getID()); + String jsonObjectString = (String) cacheDataStore.getValue(key); + if (jsonObjectString != null) { + if (IJsonObject.class.isAssignableFrom(theClass)) { + IJsonObject jsonObject = (IJsonObject) theClass.newInstance(); + jsonObject.fromJson(jsonObjectString); + result = (IPerceroObject) jsonObject; + } + else { + result = (IPerceroObject) safeObjectMapper.readValue(jsonObjectString, theClass); + } + } + else { + // Check MappedClass' child classes. + Iterator itrChildMappedClasses = mc.childMappedClasses.iterator(); + while (itrChildMappedClasses.hasNext()) { + MappedClass nextChildMc = itrChildMappedClasses.next(); + key = RedisKeyUtils.classIdPair(nextChildMc.className, classIdPair.getID()); + jsonObjectString = (String) cacheDataStore.getValue(key); + if (jsonObjectString != null) { + result = (IPerceroObject) safeObjectMapper.readValue(jsonObjectString, theClass); + return result; + } + } + } + } + + return result; + } + + @SuppressWarnings({ "rawtypes", "unchecked" }) + private Map retrieveFromRedisCache(ClassIDPairs classIdPairs, Boolean pleaseSetTimeout) throws Exception { + Map result = new HashMap(classIdPairs.getIds().size()); + + if (cacheTimeout > 0) { + IMappedClassManager mcm = MappedClassManagerFactory.getMappedClassManager(); + Class theClass = MappedClass.forName(classIdPairs.getClassName()); + MappedClass mc = mcm.getMappedClassByClassName(classIdPairs.getClassName()); + + Set keys = new HashSet(classIdPairs.getIds().size()); + Iterator itrIds = classIdPairs.getIds().iterator(); + while (itrIds.hasNext()) { + String nextId = itrIds.next(); + String nextKey = RedisKeyUtils.classIdPair(classIdPairs.getClassName(), nextId); + keys.add(nextKey); + + // Check MappedClass' child classes. + Iterator itrChildMappedClasses = mc.childMappedClasses.iterator(); + while (itrChildMappedClasses.hasNext()) { + MappedClass nextChildMc = itrChildMappedClasses.next(); + if (nextChildMc.clazz == BaseDataObject.class) { + // Reached the top level, so break. + break; + } + nextKey = RedisKeyUtils.classIdPair(nextChildMc.className, nextId); + keys.add(nextKey); + } + } - - @SuppressWarnings({ "rawtypes", "unchecked" }) - private IPerceroObject retrieveFromRedisCache(ClassIDPair classIdPair) throws Exception { - IPerceroObject result = null; - if (cacheTimeout > 0) { - IMappedClassManager mcm = MappedClassManagerFactory.getMappedClassManager(); - Class theClass = MappedClass.forName(classIdPair.getClassName()); - MappedClass mc = mcm.getMappedClassByClassName(classIdPair.getClassName()); - - String key = RedisKeyUtils.classIdPair(classIdPair.getClassName(), classIdPair.getID()); - String jsonObjectString = (String) cacheDataStore.getValue(key); - if (jsonObjectString != null) { - if (IJsonObject.class.isAssignableFrom(theClass)) { - IJsonObject jsonObject = (IJsonObject) theClass.newInstance(); - jsonObject.fromJson(jsonObjectString); - result = (IPerceroObject) jsonObject; - } - else { - result = (IPerceroObject) safeObjectMapper.readValue(jsonObjectString, theClass); - } - } - else { - // Check MappedClass' child classes. - Iterator itrChildMappedClasses = mc.childMappedClasses.iterator(); - while (itrChildMappedClasses.hasNext()) { - MappedClass nextChildMc = itrChildMappedClasses.next(); - key = RedisKeyUtils.classIdPair(nextChildMc.className, classIdPair.getID()); - jsonObjectString = (String) cacheDataStore.getValue(key); - if (jsonObjectString != null) { - result = (IPerceroObject) safeObjectMapper.readValue(jsonObjectString, theClass); - return result; - } - } - } - } - - return result; - } - - @SuppressWarnings({ "rawtypes", "unchecked" }) - private Map retrieveFromRedisCache(ClassIDPairs classIdPairs, Boolean pleaseSetTimeout) throws Exception { - Map result = new HashMap(classIdPairs.getIds().size()); - - if (cacheTimeout > 0) { - IMappedClassManager mcm = MappedClassManagerFactory.getMappedClassManager(); - Class theClass = MappedClass.forName(classIdPairs.getClassName()); - MappedClass mc = mcm.getMappedClassByClassName(classIdPairs.getClassName()); - - Set keys = new HashSet(classIdPairs.getIds().size()); - Iterator itrIds = classIdPairs.getIds().iterator(); - while (itrIds.hasNext()) { - String nextId = itrIds.next(); - String nextKey = RedisKeyUtils.classIdPair(classIdPairs.getClassName(), nextId); - keys.add(nextKey); - - // Check MappedClass' child classes. - Iterator itrChildMappedClasses = mc.childMappedClasses.iterator(); - while (itrChildMappedClasses.hasNext()) { - MappedClass nextChildMc = itrChildMappedClasses.next(); - if (nextChildMc.clazz == BaseDataObject.class) { - // Reached the top level, so break. - break; - } - nextKey = RedisKeyUtils.classIdPair(nextChildMc.className, nextId); - keys.add(nextKey); - } - } - // String key = RedisKeyUtils.classIdPair(classIdPair.getClassName(), classIdPair.getID()); - List jsonObjectStrings = cacheDataStore.getValues(keys); - Iterator itrJsonObjectStrings = jsonObjectStrings.iterator(); - while (itrJsonObjectStrings.hasNext()) { - String jsonObjectString = (String) itrJsonObjectStrings.next(); - if (jsonObjectString != null) { - if (IJsonObject.class.isAssignableFrom(theClass)) { - IJsonObject jsonObject = (IJsonObject) theClass.newInstance(); - jsonObject.fromJson(jsonObjectString); - result.put( (((IPerceroObject) jsonObject).getID()), (IPerceroObject) jsonObject ); - } - else { - IPerceroObject nextPerceroObject = (IPerceroObject) safeObjectMapper.readValue(jsonObjectString, theClass); - result.put( nextPerceroObject.getID(), nextPerceroObject); - } - - } + List jsonObjectStrings = cacheDataStore.getValues(keys); + Iterator itrJsonObjectStrings = jsonObjectStrings.iterator(); + while (itrJsonObjectStrings.hasNext()) { + String jsonObjectString = (String) itrJsonObjectStrings.next(); + if (jsonObjectString != null) { + if (IJsonObject.class.isAssignableFrom(theClass)) { + IJsonObject jsonObject = (IJsonObject) theClass.newInstance(); + jsonObject.fromJson(jsonObjectString); + result.put( (((IPerceroObject) jsonObject).getID()), (IPerceroObject) jsonObject ); + } + else { + IPerceroObject nextPerceroObject = (IPerceroObject) safeObjectMapper.readValue(jsonObjectString, theClass); + result.put( nextPerceroObject.getID(), nextPerceroObject); + } + + } // else { // // Check MappedClass' child classes. // Iterator itrChildMappedClasses = mc.childMappedClasses.iterator(); @@ -422,787 +410,788 @@ private Map retrieveFromRedisCache(ClassIDPairs classIdP // } // } // } - } - - if (pleaseSetTimeout) { - cacheDataStore.expire(keys, cacheTimeout, TimeUnit.SECONDS); - } - } - - return result; - } - - @SuppressWarnings({ }) - public Boolean getReadAccess(ClassIDPair classIdPair, String userId) { - IDataAccessObject dao = DAORegistry.getInstance().getDataAccessObject(classIdPair.getClassName()); - return dao.hasReadAccess(classIdPair, userId); - } - - @SuppressWarnings({ }) - public Boolean getDeleteAccess(ClassIDPair classIdPair, String userId) { - IDataAccessObject dao = DAORegistry.getInstance().getDataAccessObject(classIdPair.getClassName()); - return dao.hasDeleteAccess(classIdPair, userId); - } - - public List findByIds(ClassIDPairs classIdPairs, String userId) { - return findByIds(classIdPairs, userId, false); - } - @SuppressWarnings({ }) - public List findByIds(ClassIDPairs classIdPairs, String userId, Boolean ignoreCache) { - IDataAccessObject dao = DAORegistry.getInstance().getDataAccessObject(classIdPairs.getClassName()); - List results = new ArrayList(); - - try { - // Copy the ClassIDPairs to find object since we remove any of the - // ID's from the list that we find in the cache. - ClassIDPairs classIdPairsCopy = new ClassIDPairs(); - classIdPairsCopy.setClassName(classIdPairs.getClassName()); - List idsToFind = new ArrayList(classIdPairs.getIds().size()); - idsToFind.addAll(classIdPairs.getIds()); - classIdPairsCopy.setIds(idsToFind); - - Map cachedResults = null; - if (!ignoreCache) { - cachedResults = retrieveFromRedisCache(classIdPairs, true); - if (cachedResults != null &&!cachedResults.isEmpty()) { - // Add the cached results - - Iterator itrCachedResults = cachedResults.values().iterator(); - while (itrCachedResults.hasNext()) { - IPerceroObject nextCachedResult = itrCachedResults.next(); - if (nextCachedResult != null) { - idsToFind.remove(nextCachedResult.getID()); - results.add(nextCachedResult); - setObjectExpiration(nextCachedResult); - } - } - } - } - - List daoObjects = null; - if (classIdPairsCopy.getIds() != null && !classIdPairsCopy.getIds().isEmpty()) { - daoObjects = dao.retrieveObjects(classIdPairsCopy, userId, false); - - for(IPerceroObject nextResult : daoObjects) { - populateToManyRelationships(nextResult, true, null); - populateToOneRelationships(nextResult, true, null); - } - - putObjectsInRedisCache(daoObjects); // Only need to put objects in cache that were not already found in cache. - results.addAll(daoObjects); - } - - // Now clean the objects for the user. - results = cleanObject(results, userId); - - } catch(Exception e) { - log.error(e); - e.printStackTrace(); - } - - return results; - } - - - @SuppressWarnings("unchecked") - public List findByExample(IPerceroObject theQueryObject, List excludeProperties, String userId, Boolean shellOnly) throws SyncException { - IDataAccessObject dao = (IDataAccessObject) DAORegistry.getInstance().getDataAccessObject(theQueryObject.getClass().getCanonicalName()); - List results = dao.findByExample(theQueryObject, excludeProperties, userId, shellOnly); - - if (results != null && !results.isEmpty()) { - Iterator itrResults = results.iterator(); - while (itrResults.hasNext()) { - IPerceroObject nextResult = itrResults.next(); - try { - populateToManyRelationships(nextResult, true, null); - populateToOneRelationships(nextResult, true, null); - } catch (IllegalArgumentException e) { - e.printStackTrace(); - throw new SyncDataException(e); - } catch (IllegalAccessException e) { - e.printStackTrace(); - throw new SyncDataException(e); - } catch (InvocationTargetException e) { - e.printStackTrace(); - throw new SyncDataException(e); - } - } - } - - putObjectsInRedisCache(results); - - // Now clean the objects for the user. - results = cleanObject(results, userId); - - return results; - } - - @SuppressWarnings({ "unchecked", "rawtypes" }) - public T createObject(T perceroObject, String userId) throws SyncException { - - try { - IDataAccessObject dao = (IDataAccessObject) DAORegistry.getInstance().getDataAccessObject(perceroObject.getClass().getCanonicalName()); - // Make sure object has an ID. - IMappedClassManager mcm = MappedClassManagerFactory.getMappedClassManager(); - MappedClass mappedClass = mcm.getMappedClassByClassName(perceroObject.getClass().getName()); - if (!mappedClass.hasGeneratedId && !StringUtils.hasText(perceroObject.getID())) - perceroObject.setID(UUID.randomUUID().toString()); - else { - // Check to see if item already exists. - IPerceroObject existingObject = dao.retrieveObject(BaseDataObject.toClassIdPair(perceroObject), null, false); - if (existingObject != null) - { - populateToManyRelationships(perceroObject, true, null); - populateToOneRelationships(perceroObject, true, null); - return (T) cleanObject(perceroObject, userId); - } - } - - perceroObject = (T) dao.createObject(perceroObject, userId); - if (perceroObject == null) { - return perceroObject; - } - populateToManyRelationships(perceroObject, true, null); - populateToOneRelationships(perceroObject, true, null); - - // Now update the cache. - // TODO: Field-level updates could be REALLY useful here. Would avoid A TON of UNNECESSARY work... - if (cacheTimeout > 0) { - String key = RedisKeyUtils.classIdPair(perceroObject.getClass().getCanonicalName(), perceroObject.getID()); - cacheDataStore.setValue(key, ((BaseDataObject)perceroObject).toJson()); - cacheDataStore.expire(key, cacheTimeout, TimeUnit.SECONDS); - - Set keysToDelete = new HashSet(); - MappedClass nextMappedClass = mcm.getMappedClassByClassName(perceroObject.getClass().getName()); - Iterator itrToManyFields = nextMappedClass.toManyFields.iterator(); - while(itrToManyFields.hasNext()) { - MappedField nextMappedField = itrToManyFields.next(); - Object fieldObject = nextMappedField.getGetter().invoke(perceroObject); - if (fieldObject != null) { - if (fieldObject instanceof IPerceroObject) { - String nextKey = RedisKeyUtils.classIdPair(fieldObject.getClass().getCanonicalName(), ((IPerceroObject)fieldObject).getID()); - keysToDelete.add(nextKey); - } - else if (fieldObject instanceof Collection) { - Iterator itrFieldObject = ((Collection) fieldObject).iterator(); - while(itrFieldObject.hasNext()) { - Object nextListObject = itrFieldObject.next(); - if (nextListObject instanceof IPerceroObject) { - String nextKey = RedisKeyUtils.classIdPair(nextListObject.getClass().getCanonicalName(), ((IPerceroObject)nextListObject).getID()); - keysToDelete.add(nextKey); - } - } - } - } - } - Iterator itrToOneFields = mappedClass.toOneFields.iterator(); - while(itrToOneFields.hasNext()) { - MappedFieldPerceroObject nextMappedField = itrToOneFields.next(); - Object fieldObject = nextMappedField.getGetter().invoke(perceroObject); - if (fieldObject != null) { - if (fieldObject instanceof IPerceroObject) { - String nextKey = RedisKeyUtils.classIdPair(fieldObject.getClass().getCanonicalName(), ((IPerceroObject)fieldObject).getID()); - keysToDelete.add(nextKey); - } - else if (fieldObject instanceof Collection) { - Iterator itrFieldObject = ((Collection) fieldObject).iterator(); - while(itrFieldObject.hasNext()) { - Object nextListObject = itrFieldObject.next(); - if (nextListObject instanceof IPerceroObject) { - String nextKey = RedisKeyUtils.classIdPair(nextListObject.getClass().getCanonicalName(), ((IPerceroObject)nextListObject).getID()); - keysToDelete.add(nextKey); - } - } - } - } - } - - if (!keysToDelete.isEmpty()) { - cacheDataStore.deleteKeys(keysToDelete); - // TODO: Do we simply delete the key? Or do we refetch the object here and update the key? - //redisDataStore.setValue(nextKey, ((BaseDataObject)perceroObject).toJson()); - } - } - - return (T) cleanObject(perceroObject, userId); - } - catch(PropertyValueException pve) { - log.error("Error creating object", pve); - - SyncDataException sde = new SyncDataException(SyncDataException.MISSING_REQUIRED_FIELD, SyncDataException.MISSING_REQUIRED_FIELD_CODE, "Missing required field " + pve.getPropertyName()); - sde.fieldName = pve.getPropertyName(); - throw sde; - } - catch(Exception e) { - log.error("Error creating object", e); - - SyncDataException sde = new SyncDataException(SyncDataException.CREATE_OBJECT_ERROR, SyncDataException.CREATE_OBJECT_ERROR_CODE); - throw sde; - } - } - + } - //////////////////////////////////////////////////// - // PUT - //////////////////////////////////////////////////// - @SuppressWarnings("unchecked") - public T putObject(T perceroObject, Map> changedFields, String userId) throws SyncException { - IDataAccessObject dao = (IDataAccessObject) DAORegistry.getInstance().getDataAccessObject(perceroObject.getClass().getCanonicalName()); - perceroObject = (T) dao.updateObject(perceroObject, changedFields, userId); - - try { - populateToManyRelationships(perceroObject, true, null); - populateToOneRelationships(perceroObject, true, null); - } catch (IllegalArgumentException e) { - e.printStackTrace(); - throw new SyncException(e); - } catch (IllegalAccessException e) { - e.printStackTrace(); - throw new SyncException(e); - } catch (InvocationTargetException e) { - e.printStackTrace(); - throw new SyncException(e); - } - // Now update the cache. - if (cacheTimeout > 0) { - // TODO: Also need to update the caches of anything object that is related to this object. - String key = RedisKeyUtils.classIdPair(perceroObject.getClass().getCanonicalName(), perceroObject.getID()); - if (cacheDataStore.hasKey(key)) { - cacheDataStore.setValue(key, ((BaseDataObject)perceroObject).toJson()); - } - - // Iterate through each changed object and reset the cache for that object. - if (changedFields != null) { - Iterator itrChangedFieldKeyset = changedFields.keySet().iterator(); - Set keysToDelete = new HashSet(); - while (itrChangedFieldKeyset.hasNext()) { - ClassIDPair thePair = itrChangedFieldKeyset.next(); - if (!thePair.comparePerceroObject(perceroObject)) { - String nextKey = RedisKeyUtils.classIdPair(thePair.getClassName(), thePair.getID()); - keysToDelete.add(nextKey); - } - } - - if (!keysToDelete.isEmpty()) { - cacheDataStore.deleteKeys(keysToDelete); - // TODO: Do we simply delete the key? Or do we refetch the object here and update the key? - //redisDataStore.setValue(nextKey, ((BaseDataObject)perceroObject).toJson()); - } - } - else { - // No changedFields? We should never get here? - IMappedClassManager mcm = MappedClassManagerFactory.getMappedClassManager(); - MappedClass mappedClass = mcm.getMappedClassByClassName(perceroObject.getClass().getName()); - Iterator itrToManyFields = mappedClass.toManyFields.iterator(); - while(itrToManyFields.hasNext()) { - MappedField nextMappedField = itrToManyFields.next(); - Object fieldObject = null; - try { - fieldObject = nextMappedField.getGetter().invoke(perceroObject); - } catch (IllegalAccessException e) { - e.printStackTrace(); - } catch (IllegalArgumentException e) { - e.printStackTrace(); - } catch (InvocationTargetException e) { - e.printStackTrace(); - } - if (fieldObject != null) { - if (fieldObject instanceof IPerceroObject) { - String nextKey = RedisKeyUtils.classIdPair(fieldObject.getClass().getCanonicalName(), ((IPerceroObject)fieldObject).getID()); - if (cacheDataStore.hasKey(nextKey)) { - cacheDataStore.deleteKey(nextKey); - // TODO: Do we simply delete the key? Or do we refetch the object here and update the key? - //redisDataStore.setValue(nextKey, ((BaseDataObject)perceroObject).toJson()); - } - } - else if (fieldObject instanceof Collection) { - Iterator itrFieldObject = ((Collection) fieldObject).iterator(); - while(itrFieldObject.hasNext()) { - Object nextListObject = itrFieldObject.next(); - if (nextListObject instanceof IPerceroObject) { - String nextKey = RedisKeyUtils.classIdPair(nextListObject.getClass().getCanonicalName(), ((IPerceroObject)nextListObject).getID()); - if (cacheDataStore.hasKey(nextKey)) { - cacheDataStore.deleteKey(nextKey); - // TODO: Do we simply delete the key? Or do we refetch the object here and update the key? - //redisDataStore.setValue(nextKey, ((BaseDataObject)perceroObject).toJson()); - } - } - } - } - } - } - Iterator itrToOneFields = mappedClass.toOneFields.iterator(); - while(itrToOneFields.hasNext()) { - MappedFieldPerceroObject nextMappedField = itrToOneFields.next(); - Object fieldObject = null; - try { - fieldObject = nextMappedField.getGetter().invoke(perceroObject); - } catch (IllegalAccessException e) { - e.printStackTrace(); - } catch (IllegalArgumentException e) { - e.printStackTrace(); - } catch (InvocationTargetException e) { - e.printStackTrace(); - } - if (fieldObject != null) { - if (fieldObject instanceof IPerceroObject) { - String nextKey = RedisKeyUtils.classIdPair(fieldObject.getClass().getCanonicalName(), ((IPerceroObject)fieldObject).getID()); - if (cacheDataStore.hasKey(nextKey)) { - cacheDataStore.deleteKey(nextKey); - // TODO: Do we simply delete the key? Or do we refetch the object here and update the key? - //redisDataStore.setValue(nextKey, ((BaseDataObject)perceroObject).toJson()); - } - } - else if (fieldObject instanceof Collection) { - Iterator itrFieldObject = ((Collection) fieldObject).iterator(); - while(itrFieldObject.hasNext()) { - Object nextListObject = itrFieldObject.next(); - if (nextListObject instanceof IPerceroObject) { - String nextKey = RedisKeyUtils.classIdPair(nextListObject.getClass().getCanonicalName(), ((IPerceroObject)nextListObject).getID()); - if (cacheDataStore.hasKey(nextKey)) { - cacheDataStore.deleteKey(nextKey); - // TODO: Do we simply delete the key? Or do we refetch the object here and update the key? - //redisDataStore.setValue(nextKey, ((BaseDataObject)perceroObject).toJson()); - } - } - } - } - } - } - } - } - - return (T) cleanObject(perceroObject, userId); - } - - - //////////////////////////////////////////////////// - // DELETE - //////////////////////////////////////////////////// - @SuppressWarnings({ "unchecked", "rawtypes" }) - public Boolean deleteObject(ClassIDPair theClassIdPair, String userId) throws SyncException { - - IDataAccessObject dao = (IDataAccessObject) DAORegistry.getInstance().getDataAccessObject(theClassIdPair.getClassName()); - IPerceroObject perceroObject = dao.retrieveObject(theClassIdPair, null, false); // Retrieve the full object so we can update the cache if the delete is successful. - Boolean result = dao.deleteObject(theClassIdPair, userId); - - try { - MappedClass mappedClass = MappedClassManagerFactory.getMappedClassManager().getMappedClassByClassName(perceroObject.getClass().getCanonicalName()); - if (mappedClass == null) { - log.warn("Missing MappedClass for " + perceroObject.getClass().getCanonicalName()); - throw new SyncException(SyncException.MISSING_MAPPED_CLASS_ERROR, SyncException.MISSING_MAPPED_CLASS_ERROR_CODE ); - } - - // Now delete from cache. - // Now update the cache. - // TODO: Field-level updates could be REALLY useful here. Would avoid A TON of UNNECESSARY work... - if (result && cacheTimeout > 0) { - Set keysToDelete = new HashSet(); - - String key = RedisKeyUtils.classIdPair(perceroObject.getClass().getCanonicalName(), perceroObject.getID()); - keysToDelete.add(key); - - Iterator itrToManyFields = mappedClass.toManyFields.iterator(); - while(itrToManyFields.hasNext()) { - MappedField nextMappedField = itrToManyFields.next(); - Object fieldObject = nextMappedField.getGetter().invoke(perceroObject); - if (fieldObject != null) { - if (fieldObject instanceof IPerceroObject) { - String nextKey = RedisKeyUtils.classIdPair(fieldObject.getClass().getCanonicalName(), ((IPerceroObject)fieldObject).getID()); - keysToDelete.add(nextKey); - } - else if (fieldObject instanceof Collection) { - Iterator itrFieldObject = ((Collection) fieldObject).iterator(); - while(itrFieldObject.hasNext()) { - Object nextListObject = itrFieldObject.next(); - if (nextListObject instanceof IPerceroObject) { - String nextKey = RedisKeyUtils.classIdPair(nextListObject.getClass().getCanonicalName(), ((IPerceroObject)nextListObject).getID()); - keysToDelete.add(nextKey); - } - } - } - } - } - Iterator itrToOneFields = mappedClass.toOneFields.iterator(); - while(itrToOneFields.hasNext()) { - MappedFieldPerceroObject nextMappedField = itrToOneFields.next(); - Object fieldObject = nextMappedField.getGetter().invoke(perceroObject); - if (fieldObject != null) { - if (fieldObject instanceof IPerceroObject) { - String nextKey = RedisKeyUtils.classIdPair(fieldObject.getClass().getCanonicalName(), ((IPerceroObject)fieldObject).getID()); - keysToDelete.add(nextKey); - } - else if (fieldObject instanceof Collection) { - Iterator itrFieldObject = ((Collection) fieldObject).iterator(); - while(itrFieldObject.hasNext()) { - Object nextListObject = itrFieldObject.next(); - if (nextListObject instanceof IPerceroObject) { - String nextKey = RedisKeyUtils.classIdPair(nextListObject.getClass().getCanonicalName(), ((IPerceroObject)nextListObject).getID()); - keysToDelete.add(nextKey); - } - } - } - } - } - - if (!keysToDelete.isEmpty()) { - cacheDataStore.deleteKeys(keysToDelete); - } - } - - } catch(Exception e) { - if (perceroObject != null) { - log.error("Unable to delete record from database: " + perceroObject.getClass().getCanonicalName() + ":" + perceroObject.getID(), e); - } - else { - log.error("Unable to delete record from database: NULL Object", e); - } - throw new SyncDataException(SyncDataException.DELETE_OBJECT_ERROR, SyncDataException.DELETE_OBJECT_ERROR_CODE); - } - - return result; - } - - - //////////////////////////////////////////////////// - // CLEAN - //////////////////////////////////////////////////// - @SuppressWarnings("unchecked") - public IPerceroObject cleanObject(IPerceroObject perceroObject, String userId) throws SyncException { - IDataAccessObject dao = (IDataAccessObject) DAORegistry.getInstance().getDataAccessObject(perceroObject.getClass().getCanonicalName()); - perceroObject = dao.cleanObjectForUser(perceroObject, userId); - return perceroObject; - } - - @Override - @SuppressWarnings({ "rawtypes", "unchecked" }) - public List cleanObject(List perceroObjects, String userId) throws SyncException { - try { - List results = new ArrayList(perceroObjects.size()); - - Map> classPairs = new HashMap>(); - - Iterator itrPerceroObjects = perceroObjects.iterator(); - while (itrPerceroObjects.hasNext()) { - IPerceroObject nextPerceroObject = itrPerceroObjects.next(); - if ( ((BaseDataObject)nextPerceroObject).getIsClean()) { - results.add(nextPerceroObject); - } - else { - List classObjects = classPairs.get(nextPerceroObject.getClass()); - if (classObjects == null) { - classObjects = new ArrayList(); - classPairs.put(nextPerceroObject.getClass(), classObjects); - } - classObjects.add(nextPerceroObject); - } - } - - Iterator>> itrClassPairs = classPairs.entrySet().iterator(); - while (itrClassPairs.hasNext()) { - Entry> nextEntrySet = itrClassPairs.next(); - IDataAccessObject dao = (IDataAccessObject) DAORegistry.getInstance().getDataAccessObject(nextEntrySet.getKey().getCanonicalName()); - - List nextClassObjects = nextEntrySet.getValue(); - if (nextClassObjects != null && !nextClassObjects.isEmpty()) { - Iterator itrClassObjects = nextClassObjects.iterator(); - while (itrClassObjects.hasNext()) { - IPerceroObject nextObject = itrClassObjects.next(); - IPerceroObject nextResult = dao.cleanObjectForUser(nextObject, userId); - if (nextResult != null) { - results.add(nextResult); - } - } - } - } - - return results; - } catch(Exception e) { - throw new SyncException(e); - } - } + if (pleaseSetTimeout) { + cacheDataStore.expire(keys, cacheTimeout, TimeUnit.SECONDS); + } + } - - @SuppressWarnings("rawtypes") - public Map> getChangedMappedFields(IPerceroObject newObject) { - Map> result = new HashMap>(); - Collection baseObjectResult = null; - ClassIDPair basePair = new ClassIDPair(newObject.getID(), newObject.getClass().getCanonicalName()); - - String className = newObject.getClass().getCanonicalName(); - IPerceroObject oldObject = findById(new ClassIDPair(newObject.getID(), className), null); - IMappedClassManager mcm = MappedClassManagerFactory.getMappedClassManager(); - MappedClass mc = mcm.getMappedClassByClassName(className); - Iterator itrMappedFields = mc.externalizableFields.iterator(); - while (itrMappedFields.hasNext()) { - MappedField nextMappedField = itrMappedFields.next(); - try { - Boolean fieldIsEqual = nextMappedField.compareObjects(oldObject, newObject); - if (!fieldIsEqual) { - if (baseObjectResult == null) { - baseObjectResult = new HashSet(); - result.put(basePair, baseObjectResult); - } - baseObjectResult.add(nextMappedField); - - // If this is a relationship field, then need to grab the old and new values. - if (nextMappedField.getReverseMappedField() != null) { - if (nextMappedField instanceof MappedFieldPerceroObject) { - MappedFieldPerceroObject nextMappedFieldPerceroObject = (MappedFieldPerceroObject) nextMappedField; - - IPerceroObject oldReversePerceroObject = (IPerceroObject) nextMappedFieldPerceroObject.getGetter().invoke(oldObject); - if (oldReversePerceroObject != null) { - ClassIDPair oldReversePair = new ClassIDPair(oldReversePerceroObject.getID(), oldReversePerceroObject.getClass().getCanonicalName()); - Collection oldReverseChangedFields = result.get(oldReversePair); - if (oldReverseChangedFields == null) { - oldReverseChangedFields = new HashSet(); - result.put(oldReversePair, oldReverseChangedFields); - } - oldReverseChangedFields.add(nextMappedField.getReverseMappedField()); - } - - IPerceroObject newReversePerceroObject = (IPerceroObject) nextMappedFieldPerceroObject.getGetter().invoke(newObject); - if (newReversePerceroObject != null) { - ClassIDPair newReversePair = new ClassIDPair(newReversePerceroObject.getID(), newReversePerceroObject.getClass().getCanonicalName()); - Collection changedFields = result.get(newReversePair); - if (changedFields == null) { - changedFields = new HashSet(); - result.put(newReversePair, changedFields); - } - changedFields.add(nextMappedField.getReverseMappedField()); - } - } - else if (nextMappedField instanceof MappedFieldList) { - MappedFieldList nextMappedFieldList = (MappedFieldList) nextMappedField; - - List oldReverseList = (List) nextMappedFieldList.getGetter().invoke(oldObject); - if (oldReverseList == null) - oldReverseList = new ArrayList(); - - List newReverseList = (List) nextMappedFieldList.getGetter().invoke(newObject); - if (newReverseList == null) - newReverseList = new ArrayList(); - - // Compare each item in the lists. - Collection oldChangedList = retrieveObjectsNotInCollection(oldReverseList, newReverseList); - Iterator itrOldChangedList = oldChangedList.iterator(); - while (itrOldChangedList.hasNext()) { - BaseDataObject nextOldChangedObject = (BaseDataObject) itrOldChangedList.next(); - ClassIDPair nextOldReversePair = BaseDataObject.toClassIdPair(nextOldChangedObject); - - // Old object is not in new list, so add to list of changed fields. - Collection changedFields = result.get(nextOldReversePair); - if (changedFields == null) { - changedFields = new HashSet(); - result.put(nextOldReversePair, changedFields); - } - changedFields.add(nextMappedField.getReverseMappedField()); - } - - Collection newChangedList = retrieveObjectsNotInCollection(newReverseList, oldReverseList); - Iterator itrNewChangedList = newChangedList.iterator(); - while (itrNewChangedList.hasNext()) { - BaseDataObject nextNewChangedObject = (BaseDataObject) itrNewChangedList.next(); - ClassIDPair nextNewReversePair = BaseDataObject.toClassIdPair(nextNewChangedObject); - - // Old object is not in new list, so add to list of changed fields. - Collection changedFields = result.get(nextNewReversePair); - if (changedFields == null) { - changedFields = new HashSet(); - result.put(nextNewReversePair, changedFields); - } - changedFields.add(nextMappedField.getReverseMappedField()); - } - } - } - } - } catch(Exception e) { - log.warn("Error getting changed field: " + nextMappedField.getField().getName(), e); - baseObjectResult.add(nextMappedField); - } - } - - return result; - } - - @SuppressWarnings({ "rawtypes", "unchecked" }) - private Collection retrieveObjectsNotInCollection(Collection baseList, Collection compareToList) { - Collection result = new HashSet(); - Iterator itrBaseList = baseList.iterator(); - Iterator itrCompareToList = null; - boolean matchFound = false; - - while (itrBaseList.hasNext()) { - BaseDataObject nextBasePerceroObject = (BaseDataObject) itrBaseList.next(); - ClassIDPair nextBasePair = BaseDataObject.toClassIdPair(nextBasePerceroObject); - nextBasePerceroObject = (BaseDataObject) findById(nextBasePair, null); - - itrCompareToList = compareToList.iterator(); - matchFound = false; - while (itrCompareToList.hasNext()) { - BaseDataObject nextCompareToPerceroObject = (BaseDataObject) itrCompareToList.next(); - nextCompareToPerceroObject = (BaseDataObject) findById(BaseDataObject.toClassIdPair(nextCompareToPerceroObject), null); - - if (nextBasePerceroObject.getClass() == nextCompareToPerceroObject.getClass() && nextBasePerceroObject.getID().equalsIgnoreCase(nextCompareToPerceroObject.getID())) { - matchFound = true; - break; - } - } - - if (!matchFound) { - result.add(nextBasePerceroObject); - } - } - - return result; - } + return result; + } + + @SuppressWarnings({ }) + public Boolean getReadAccess(ClassIDPair classIdPair, String userId) { + IDataAccessObject dao = DAORegistry.getInstance().getDataAccessObject(classIdPair.getClassName()); + return dao.hasReadAccess(classIdPair, userId); + } + + @SuppressWarnings({ }) + public Boolean getDeleteAccess(ClassIDPair classIdPair, String userId) { + IDataAccessObject dao = DAORegistry.getInstance().getDataAccessObject(classIdPair.getClassName()); + return dao.hasDeleteAccess(classIdPair, userId); + } + + public List findByIds(ClassIDPairs classIdPairs, String userId) { + return findByIds(classIdPairs, userId, false); + } + @SuppressWarnings({ }) + public List findByIds(ClassIDPairs classIdPairs, String userId, Boolean ignoreCache) { + IDataAccessObject dao = DAORegistry.getInstance().getDataAccessObject(classIdPairs.getClassName()); + List results = new ArrayList(); + + try { + // Copy the ClassIDPairs to find object since we remove any of the + // ID's from the list that we find in the cache. + ClassIDPairs classIdPairsCopy = new ClassIDPairs(); + classIdPairsCopy.setClassName(classIdPairs.getClassName()); + List idsToFind = new ArrayList(classIdPairs.getIds().size()); + idsToFind.addAll(classIdPairs.getIds()); + classIdPairsCopy.setIds(idsToFind); + + Map cachedResults = null; + if (!ignoreCache) { + cachedResults = retrieveFromRedisCache(classIdPairs, true); + if (cachedResults != null &&!cachedResults.isEmpty()) { + // Add the cached results + + Iterator itrCachedResults = cachedResults.values().iterator(); + while (itrCachedResults.hasNext()) { + IPerceroObject nextCachedResult = itrCachedResults.next(); + if (nextCachedResult != null) { + idsToFind.remove(nextCachedResult.getID()); + results.add(nextCachedResult); + setObjectExpiration(nextCachedResult); + } + } + } + } + + List daoObjects = null; + if (classIdPairsCopy.getIds() != null && !classIdPairsCopy.getIds().isEmpty()) { + daoObjects = dao.retrieveObjects(classIdPairsCopy, userId, false); + + for(IPerceroObject nextResult : daoObjects) { + populateToManyRelationships(nextResult, true, null); + populateToOneRelationships(nextResult, true, null); + } + + putObjectsInRedisCache(daoObjects); // Only need to put objects in cache that were not already found in cache. + results.addAll(daoObjects); + } + + // Now clean the objects for the user. + results = cleanObject(results, userId); + + } catch(Exception e) { + log.error(e); + e.printStackTrace(); + } + + return results; + } + + + @SuppressWarnings("unchecked") + public List findByExample(IPerceroObject theQueryObject, List excludeProperties, String userId, Boolean shellOnly) throws SyncException { + IDataAccessObject dao = (IDataAccessObject) DAORegistry.getInstance().getDataAccessObject(theQueryObject.getClass().getCanonicalName()); + List results = dao.findByExample(theQueryObject, excludeProperties, userId, shellOnly); + + if (results != null && !results.isEmpty()) { + Iterator itrResults = results.iterator(); + while (itrResults.hasNext()) { + IPerceroObject nextResult = itrResults.next(); + try { + populateToManyRelationships(nextResult, true, null); + populateToOneRelationships(nextResult, true, null); + } catch (IllegalArgumentException e) { + e.printStackTrace(); + throw new SyncDataException(e); + } catch (IllegalAccessException e) { + e.printStackTrace(); + throw new SyncDataException(e); + } catch (InvocationTargetException e) { + e.printStackTrace(); + throw new SyncDataException(e); + } + } + } + + putObjectsInRedisCache(results); + + // Now clean the objects for the user. + results = cleanObject(results, userId); + + return results; + } + + @SuppressWarnings({ "unchecked", "rawtypes" }) + public T createObject(T perceroObject, String userId) throws SyncException { + + try { + IDataAccessObject dao = (IDataAccessObject) DAORegistry.getInstance().getDataAccessObject(perceroObject.getClass().getCanonicalName()); + // Make sure object has an ID. + IMappedClassManager mcm = MappedClassManagerFactory.getMappedClassManager(); + MappedClass mappedClass = mcm.getMappedClassByClassName(perceroObject.getClass().getName()); + if (!mappedClass.hasGeneratedId && !StringUtils.hasText(perceroObject.getID())) + perceroObject.setID(UUID.randomUUID().toString()); + else { + // Check to see if item already exists. + IPerceroObject existingObject = dao.retrieveObject(BaseDataObject.toClassIdPair(perceroObject), null, false); + if (existingObject != null) + { + populateToManyRelationships(perceroObject, true, null); + populateToOneRelationships(perceroObject, true, null); + return (T) cleanObject(perceroObject, userId); + } + } + + perceroObject = (T) dao.createObject(perceroObject, userId); + if (perceroObject == null) { + return perceroObject; + } + populateToManyRelationships(perceroObject, true, null); + populateToOneRelationships(perceroObject, true, null); + + // Now update the cache. + // TODO: Field-level updates could be REALLY useful here. Would avoid A TON of UNNECESSARY work... + if (cacheTimeout > 0) { + String key = RedisKeyUtils.classIdPair(perceroObject.getClass().getCanonicalName(), perceroObject.getID()); + cacheDataStore.setValue(key, ((BaseDataObject)perceroObject).toJson()); + cacheDataStore.expire(key, cacheTimeout, TimeUnit.SECONDS); + + Set keysToDelete = new HashSet(); + MappedClass nextMappedClass = mcm.getMappedClassByClassName(perceroObject.getClass().getName()); + Iterator itrToManyFields = nextMappedClass.toManyFields.iterator(); + while(itrToManyFields.hasNext()) { + MappedField nextMappedField = itrToManyFields.next(); + Object fieldObject = nextMappedField.getGetter().invoke(perceroObject); + if (fieldObject != null) { + if (fieldObject instanceof IPerceroObject) { + String nextKey = RedisKeyUtils.classIdPair(fieldObject.getClass().getCanonicalName(), ((IPerceroObject)fieldObject).getID()); + keysToDelete.add(nextKey); + } + else if (fieldObject instanceof Collection) { + Iterator itrFieldObject = ((Collection) fieldObject).iterator(); + while(itrFieldObject.hasNext()) { + Object nextListObject = itrFieldObject.next(); + if (nextListObject instanceof IPerceroObject) { + String nextKey = RedisKeyUtils.classIdPair(nextListObject.getClass().getCanonicalName(), ((IPerceroObject)nextListObject).getID()); + keysToDelete.add(nextKey); + } + } + } + } + } + Iterator itrToOneFields = mappedClass.toOneFields.iterator(); + while(itrToOneFields.hasNext()) { + MappedFieldPerceroObject nextMappedField = itrToOneFields.next(); + Object fieldObject = nextMappedField.getGetter().invoke(perceroObject); + if (fieldObject != null) { + if (fieldObject instanceof IPerceroObject) { + String nextKey = RedisKeyUtils.classIdPair(fieldObject.getClass().getCanonicalName(), ((IPerceroObject)fieldObject).getID()); + keysToDelete.add(nextKey); + } + else if (fieldObject instanceof Collection) { + Iterator itrFieldObject = ((Collection) fieldObject).iterator(); + while(itrFieldObject.hasNext()) { + Object nextListObject = itrFieldObject.next(); + if (nextListObject instanceof IPerceroObject) { + String nextKey = RedisKeyUtils.classIdPair(nextListObject.getClass().getCanonicalName(), ((IPerceroObject)nextListObject).getID()); + keysToDelete.add(nextKey); + } + } + } + } + } + + if (!keysToDelete.isEmpty()) { + cacheDataStore.deleteKeys(keysToDelete); + // TODO: Do we simply delete the key? Or do we refetch the object here and update the key? + //redisDataStore.setValue(nextKey, ((BaseDataObject)perceroObject).toJson()); + } + } + + return (T) cleanObject(perceroObject, userId); + } + catch(PropertyValueException pve) { + log.error("Error creating object", pve); + + SyncDataException sde = new SyncDataException(SyncDataException.MISSING_REQUIRED_FIELD, SyncDataException.MISSING_REQUIRED_FIELD_CODE, "Missing required field " + pve.getPropertyName()); + sde.fieldName = pve.getPropertyName(); + throw sde; + } + catch(Exception e) { + log.error("Error creating object", e); + + SyncDataException sde = new SyncDataException(SyncDataException.CREATE_OBJECT_ERROR, SyncDataException.CREATE_OBJECT_ERROR_CODE); + throw sde; + } + } + + + //////////////////////////////////////////////////// + // PUT + //////////////////////////////////////////////////// + @SuppressWarnings("unchecked") + public T putObject(T perceroObject, Map> changedFields, String userId) throws SyncException { + IDataAccessObject dao = (IDataAccessObject) DAORegistry.getInstance().getDataAccessObject(perceroObject.getClass().getCanonicalName()); + perceroObject = (T) dao.updateObject(perceroObject, changedFields, userId); + + try { + populateToManyRelationships(perceroObject, true, null); + populateToOneRelationships(perceroObject, true, null); + } catch (IllegalArgumentException e) { + e.printStackTrace(); + throw new SyncException(e); + } catch (IllegalAccessException e) { + e.printStackTrace(); + throw new SyncException(e); + } catch (InvocationTargetException e) { + e.printStackTrace(); + throw new SyncException(e); + } + // Now update the cache. + if (cacheTimeout > 0) { + // TODO: Also need to update the caches of anything object that is related to this object. + String key = RedisKeyUtils.classIdPair(perceroObject.getClass().getCanonicalName(), perceroObject.getID()); + if (cacheDataStore.hasKey(key)) { + cacheDataStore.setValue(key, ((BaseDataObject)perceroObject).toJson()); + } + + // Iterate through each changed object and reset the cache for that object. + if (changedFields != null) { + Iterator itrChangedFieldKeyset = changedFields.keySet().iterator(); + Set keysToDelete = new HashSet(); + while (itrChangedFieldKeyset.hasNext()) { + ClassIDPair thePair = itrChangedFieldKeyset.next(); + if (!thePair.comparePerceroObject(perceroObject)) { + String nextKey = RedisKeyUtils.classIdPair(thePair.getClassName(), thePair.getID()); + keysToDelete.add(nextKey); + } + } + + if (!keysToDelete.isEmpty()) { + cacheDataStore.deleteKeys(keysToDelete); + // TODO: Do we simply delete the key? Or do we refetch the object here and update the key? + //redisDataStore.setValue(nextKey, ((BaseDataObject)perceroObject).toJson()); + } + } + else { + // No changedFields? We should never get here? + IMappedClassManager mcm = MappedClassManagerFactory.getMappedClassManager(); + MappedClass mappedClass = mcm.getMappedClassByClassName(perceroObject.getClass().getName()); + Iterator itrToManyFields = mappedClass.toManyFields.iterator(); + while(itrToManyFields.hasNext()) { + MappedField nextMappedField = itrToManyFields.next(); + Object fieldObject = null; + try { + fieldObject = nextMappedField.getGetter().invoke(perceroObject); + } catch (IllegalAccessException e) { + e.printStackTrace(); + } catch (IllegalArgumentException e) { + e.printStackTrace(); + } catch (InvocationTargetException e) { + e.printStackTrace(); + } + if (fieldObject != null) { + if (fieldObject instanceof IPerceroObject) { + String nextKey = RedisKeyUtils.classIdPair(fieldObject.getClass().getCanonicalName(), ((IPerceroObject)fieldObject).getID()); + if (cacheDataStore.hasKey(nextKey)) { + cacheDataStore.deleteKey(nextKey); + // TODO: Do we simply delete the key? Or do we refetch the object here and update the key? + //redisDataStore.setValue(nextKey, ((BaseDataObject)perceroObject).toJson()); + } + } + else if (fieldObject instanceof Collection) { + Iterator itrFieldObject = ((Collection) fieldObject).iterator(); + while(itrFieldObject.hasNext()) { + Object nextListObject = itrFieldObject.next(); + if (nextListObject instanceof IPerceroObject) { + String nextKey = RedisKeyUtils.classIdPair(nextListObject.getClass().getCanonicalName(), ((IPerceroObject)nextListObject).getID()); + if (cacheDataStore.hasKey(nextKey)) { + cacheDataStore.deleteKey(nextKey); + // TODO: Do we simply delete the key? Or do we refetch the object here and update the key? + //redisDataStore.setValue(nextKey, ((BaseDataObject)perceroObject).toJson()); + } + } + } + } + } + } + Iterator itrToOneFields = mappedClass.toOneFields.iterator(); + while(itrToOneFields.hasNext()) { + MappedFieldPerceroObject nextMappedField = itrToOneFields.next(); + Object fieldObject = null; + try { + fieldObject = nextMappedField.getGetter().invoke(perceroObject); + } catch (IllegalAccessException e) { + e.printStackTrace(); + } catch (IllegalArgumentException e) { + e.printStackTrace(); + } catch (InvocationTargetException e) { + e.printStackTrace(); + } + if (fieldObject != null) { + if (fieldObject instanceof IPerceroObject) { + String nextKey = RedisKeyUtils.classIdPair(fieldObject.getClass().getCanonicalName(), ((IPerceroObject)fieldObject).getID()); + if (cacheDataStore.hasKey(nextKey)) { + cacheDataStore.deleteKey(nextKey); + // TODO: Do we simply delete the key? Or do we refetch the object here and update the key? + //redisDataStore.setValue(nextKey, ((BaseDataObject)perceroObject).toJson()); + } + } + else if (fieldObject instanceof Collection) { + Iterator itrFieldObject = ((Collection) fieldObject).iterator(); + while(itrFieldObject.hasNext()) { + Object nextListObject = itrFieldObject.next(); + if (nextListObject instanceof IPerceroObject) { + String nextKey = RedisKeyUtils.classIdPair(nextListObject.getClass().getCanonicalName(), ((IPerceroObject)nextListObject).getID()); + if (cacheDataStore.hasKey(nextKey)) { + cacheDataStore.deleteKey(nextKey); + // TODO: Do we simply delete the key? Or do we refetch the object here and update the key? + //redisDataStore.setValue(nextKey, ((BaseDataObject)perceroObject).toJson()); + } + } + } + } + } + } + } + } + + return (T) cleanObject(perceroObject, userId); + } + + + //////////////////////////////////////////////////// + // DELETE + //////////////////////////////////////////////////// + @SuppressWarnings({ "unchecked", "rawtypes" }) + public Boolean deleteObject(ClassIDPair theClassIdPair, String userId) throws SyncException { + + IDataAccessObject dao = (IDataAccessObject) DAORegistry.getInstance().getDataAccessObject(theClassIdPair.getClassName()); + IPerceroObject perceroObject = dao.retrieveObject(theClassIdPair, null, false); // Retrieve the full object so we can update the cache if the delete is successful. + Boolean result = dao.deleteObject(theClassIdPair, userId); + + try { + MappedClass mappedClass = MappedClassManagerFactory.getMappedClassManager().getMappedClassByClassName(perceroObject.getClass().getCanonicalName()); + if (mappedClass == null) { + log.warn("Missing MappedClass for " + perceroObject.getClass().getCanonicalName()); + throw new SyncException(SyncException.MISSING_MAPPED_CLASS_ERROR, SyncException.MISSING_MAPPED_CLASS_ERROR_CODE ); + } + + // Now delete from cache. + // Now update the cache. + // TODO: Field-level updates could be REALLY useful here. Would avoid A TON of UNNECESSARY work... + if (result && cacheTimeout > 0) { + Set keysToDelete = new HashSet(); + + String key = RedisKeyUtils.classIdPair(perceroObject.getClass().getCanonicalName(), perceroObject.getID()); + keysToDelete.add(key); + + Iterator itrToManyFields = mappedClass.toManyFields.iterator(); + while(itrToManyFields.hasNext()) { + MappedField nextMappedField = itrToManyFields.next(); + Object fieldObject = nextMappedField.getGetter().invoke(perceroObject); + if (fieldObject != null) { + if (fieldObject instanceof IPerceroObject) { + String nextKey = RedisKeyUtils.classIdPair(fieldObject.getClass().getCanonicalName(), ((IPerceroObject)fieldObject).getID()); + keysToDelete.add(nextKey); + } + else if (fieldObject instanceof Collection) { + Iterator itrFieldObject = ((Collection) fieldObject).iterator(); + while(itrFieldObject.hasNext()) { + Object nextListObject = itrFieldObject.next(); + if (nextListObject instanceof IPerceroObject) { + String nextKey = RedisKeyUtils.classIdPair(nextListObject.getClass().getCanonicalName(), ((IPerceroObject)nextListObject).getID()); + keysToDelete.add(nextKey); + } + } + } + } + } + Iterator itrToOneFields = mappedClass.toOneFields.iterator(); + while(itrToOneFields.hasNext()) { + MappedFieldPerceroObject nextMappedField = itrToOneFields.next(); + Object fieldObject = nextMappedField.getGetter().invoke(perceroObject); + if (fieldObject != null) { + if (fieldObject instanceof IPerceroObject) { + String nextKey = RedisKeyUtils.classIdPair(fieldObject.getClass().getCanonicalName(), ((IPerceroObject)fieldObject).getID()); + keysToDelete.add(nextKey); + } + else if (fieldObject instanceof Collection) { + Iterator itrFieldObject = ((Collection) fieldObject).iterator(); + while(itrFieldObject.hasNext()) { + Object nextListObject = itrFieldObject.next(); + if (nextListObject instanceof IPerceroObject) { + String nextKey = RedisKeyUtils.classIdPair(nextListObject.getClass().getCanonicalName(), ((IPerceroObject)nextListObject).getID()); + keysToDelete.add(nextKey); + } + } + } + } + } + + if (!keysToDelete.isEmpty()) { + cacheDataStore.deleteKeys(keysToDelete); + } + } + + } catch(Exception e) { + if (perceroObject != null) { + log.error("Unable to delete record from database: " + perceroObject.getClass().getCanonicalName() + ":" + perceroObject.getID(), e); + } + else { + log.error("Unable to delete record from database: NULL Object", e); + } + throw new SyncDataException(SyncDataException.DELETE_OBJECT_ERROR, SyncDataException.DELETE_OBJECT_ERROR_CODE); + } + + return result; + } + + + //////////////////////////////////////////////////// + // CLEAN + //////////////////////////////////////////////////// + @SuppressWarnings("unchecked") + public IPerceroObject cleanObject(IPerceroObject perceroObject, String userId) throws SyncException { + if(perceroObject == null) return null; + IDataAccessObject dao = (IDataAccessObject) DAORegistry.getInstance().getDataAccessObject(perceroObject.getClass().getCanonicalName()); + perceroObject = dao.cleanObjectForUser(perceroObject, userId); + return perceroObject; + } + + @Override + @SuppressWarnings({ "rawtypes", "unchecked" }) + public List cleanObject(List perceroObjects, String userId) throws SyncException { + try { + List results = new ArrayList(perceroObjects.size()); + + Map> classPairs = new HashMap>(); + + Iterator itrPerceroObjects = perceroObjects.iterator(); + while (itrPerceroObjects.hasNext()) { + IPerceroObject nextPerceroObject = itrPerceroObjects.next(); + if ( ((BaseDataObject)nextPerceroObject).getIsClean()) { + results.add(nextPerceroObject); + } + else { + List classObjects = classPairs.get(nextPerceroObject.getClass()); + if (classObjects == null) { + classObjects = new ArrayList(); + classPairs.put(nextPerceroObject.getClass(), classObjects); + } + classObjects.add(nextPerceroObject); + } + } + + Iterator>> itrClassPairs = classPairs.entrySet().iterator(); + while (itrClassPairs.hasNext()) { + Entry> nextEntrySet = itrClassPairs.next(); + IDataAccessObject dao = (IDataAccessObject) DAORegistry.getInstance().getDataAccessObject(nextEntrySet.getKey().getCanonicalName()); + + List nextClassObjects = nextEntrySet.getValue(); + if (nextClassObjects != null && !nextClassObjects.isEmpty()) { + Iterator itrClassObjects = nextClassObjects.iterator(); + while (itrClassObjects.hasNext()) { + IPerceroObject nextObject = itrClassObjects.next(); + IPerceroObject nextResult = dao.cleanObjectForUser(nextObject, userId); + if (nextResult != null) { + results.add(nextResult); + } + } + } + } + + return results; + } catch(Exception e) { + throw new SyncException(e); + } + } + + + @SuppressWarnings("rawtypes") + public Map> getChangedMappedFields(IPerceroObject newObject) { + Map> result = new HashMap>(); + Collection baseObjectResult = null; + ClassIDPair basePair = new ClassIDPair(newObject.getID(), newObject.getClass().getCanonicalName()); + + String className = newObject.getClass().getCanonicalName(); + IPerceroObject oldObject = findById(new ClassIDPair(newObject.getID(), className), null); + IMappedClassManager mcm = MappedClassManagerFactory.getMappedClassManager(); + MappedClass mc = mcm.getMappedClassByClassName(className); + Iterator itrMappedFields = mc.externalizableFields.iterator(); + while (itrMappedFields.hasNext()) { + MappedField nextMappedField = itrMappedFields.next(); + try { + Boolean fieldIsEqual = nextMappedField.compareObjects(oldObject, newObject); + if (!fieldIsEqual) { + if (baseObjectResult == null) { + baseObjectResult = new HashSet(); + result.put(basePair, baseObjectResult); + } + baseObjectResult.add(nextMappedField); + + // If this is a relationship field, then need to grab the old and new values. + if (nextMappedField.getReverseMappedField() != null) { + if (nextMappedField instanceof MappedFieldPerceroObject) { + MappedFieldPerceroObject nextMappedFieldPerceroObject = (MappedFieldPerceroObject) nextMappedField; + + IPerceroObject oldReversePerceroObject = (IPerceroObject) nextMappedFieldPerceroObject.getGetter().invoke(oldObject); + if (oldReversePerceroObject != null) { + ClassIDPair oldReversePair = new ClassIDPair(oldReversePerceroObject.getID(), oldReversePerceroObject.getClass().getCanonicalName()); + Collection oldReverseChangedFields = result.get(oldReversePair); + if (oldReverseChangedFields == null) { + oldReverseChangedFields = new HashSet(); + result.put(oldReversePair, oldReverseChangedFields); + } + oldReverseChangedFields.add(nextMappedField.getReverseMappedField()); + } + + IPerceroObject newReversePerceroObject = (IPerceroObject) nextMappedFieldPerceroObject.getGetter().invoke(newObject); + if (newReversePerceroObject != null) { + ClassIDPair newReversePair = new ClassIDPair(newReversePerceroObject.getID(), newReversePerceroObject.getClass().getCanonicalName()); + Collection changedFields = result.get(newReversePair); + if (changedFields == null) { + changedFields = new HashSet(); + result.put(newReversePair, changedFields); + } + changedFields.add(nextMappedField.getReverseMappedField()); + } + } + else if (nextMappedField instanceof MappedFieldList) { + MappedFieldList nextMappedFieldList = (MappedFieldList) nextMappedField; + + List oldReverseList = (List) nextMappedFieldList.getGetter().invoke(oldObject); + if (oldReverseList == null) + oldReverseList = new ArrayList(); + + List newReverseList = (List) nextMappedFieldList.getGetter().invoke(newObject); + if (newReverseList == null) + newReverseList = new ArrayList(); + + // Compare each item in the lists. + Collection oldChangedList = retrieveObjectsNotInCollection(oldReverseList, newReverseList); + Iterator itrOldChangedList = oldChangedList.iterator(); + while (itrOldChangedList.hasNext()) { + BaseDataObject nextOldChangedObject = (BaseDataObject) itrOldChangedList.next(); + ClassIDPair nextOldReversePair = BaseDataObject.toClassIdPair(nextOldChangedObject); + + // Old object is not in new list, so add to list of changed fields. + Collection changedFields = result.get(nextOldReversePair); + if (changedFields == null) { + changedFields = new HashSet(); + result.put(nextOldReversePair, changedFields); + } + changedFields.add(nextMappedField.getReverseMappedField()); + } + + Collection newChangedList = retrieveObjectsNotInCollection(newReverseList, oldReverseList); + Iterator itrNewChangedList = newChangedList.iterator(); + while (itrNewChangedList.hasNext()) { + BaseDataObject nextNewChangedObject = (BaseDataObject) itrNewChangedList.next(); + ClassIDPair nextNewReversePair = BaseDataObject.toClassIdPair(nextNewChangedObject); + + // Old object is not in new list, so add to list of changed fields. + Collection changedFields = result.get(nextNewReversePair); + if (changedFields == null) { + changedFields = new HashSet(); + result.put(nextNewReversePair, changedFields); + } + changedFields.add(nextMappedField.getReverseMappedField()); + } + } + } + } + } catch(Exception e) { + log.warn("Error getting changed field: " + nextMappedField.getField().getName(), e); + baseObjectResult.add(nextMappedField); + } + } + + return result; + } + + @SuppressWarnings({ "rawtypes", "unchecked" }) + private Collection retrieveObjectsNotInCollection(Collection baseList, Collection compareToList) { + Collection result = new HashSet(); + Iterator itrBaseList = baseList.iterator(); + Iterator itrCompareToList = null; + boolean matchFound = false; + + while (itrBaseList.hasNext()) { + BaseDataObject nextBasePerceroObject = (BaseDataObject) itrBaseList.next(); + ClassIDPair nextBasePair = BaseDataObject.toClassIdPair(nextBasePerceroObject); + nextBasePerceroObject = (BaseDataObject) findById(nextBasePair, null); + + itrCompareToList = compareToList.iterator(); + matchFound = false; + while (itrCompareToList.hasNext()) { + BaseDataObject nextCompareToPerceroObject = (BaseDataObject) itrCompareToList.next(); + nextCompareToPerceroObject = (BaseDataObject) findById(BaseDataObject.toClassIdPair(nextCompareToPerceroObject), null); + + if (nextBasePerceroObject.getClass() == nextCompareToPerceroObject.getClass() && nextBasePerceroObject.getID().equalsIgnoreCase(nextCompareToPerceroObject.getID())) { + matchFound = true; + break; + } + } + + if (!matchFound) { + result.add(nextBasePerceroObject); + } + } + + return result; + } /* (non-Javadoc) * @see com.percero.agents.sync.services.IDataProvider#findAllRelatedObjects(com.percero.framework.vo.IPerceroObject, com.percero.agents.sync.metadata.MappedField, java.lang.Boolean, java.lang.String) */ - -// @Override - @SuppressWarnings("unchecked") - public List findAllRelatedObjects(IPerceroObject perceroObject, MappedField mappedField, Boolean shellOnly, String userId) throws SyncException { - List result = new ArrayList(); - - if (!StringUtils.hasText(perceroObject.getID())) { - // No valid ID on the object, so can't search for it. - return result; - } - - if (mappedField.getMappedClass().getSourceMappedFields().contains(mappedField)) { - // This object is the source. - IDataAccessObject dao = (IDataAccessObject) DAORegistry.getInstance().getDataAccessObject(mappedField.getMappedClass().className); - IPerceroObject thisObject = dao.retrieveObject(BaseDataObject.toClassIdPair(perceroObject), userId, false); - IPerceroObject relatedObject; - try { - relatedObject = (IPerceroObject) mappedField.getGetter().invoke(thisObject); - } catch (Exception e) { - throw new SyncException(e); - } - if (relatedObject != null) { - if (!shellOnly) { - MappedClass relatedMappedClass = MappedClassManagerFactory.getMappedClassManager().getMappedClassByClassName(relatedObject.getClass().getCanonicalName()); - relatedObject = relatedMappedClass.getDataProvider().findById(BaseDataObject.toClassIdPair(relatedObject), userId); - } - result.add(relatedObject); - } - result.add(relatedObject); - } - else { - // This object is the target. - // The reverse mapped field should be the MappedField on the target object, the one that this MUST be the data provider for. - MappedField reverseMappedField = mappedField.getReverseMappedField(); - if (reverseMappedField == null) { - // No reverse mapped field, meaning there is nothing to do. - return result; - } - - IDataProvider dataProvider = reverseMappedField.getMappedClass().getDataProvider(); - result = dataProvider.getAllByRelationship(reverseMappedField, BaseDataObject.toClassIdPair(perceroObject), shellOnly, userId); - } - - return result; - } - - @SuppressWarnings("unchecked") - public List getAllByRelationship(MappedField mappedField, ClassIDPair targetClassIdPair, Boolean shellOnly, String userId) throws SyncException { - if (mappedField.getMappedClass().getSourceMappedFields().contains(mappedField)) { - // This object is the source. - IDataAccessObject dao = (IDataAccessObject) DAORegistry.getInstance().getDataAccessObject(mappedField.getMappedClass().className); - return dao.retrieveAllByRelationship(mappedField, targetClassIdPair, shellOnly, userId); - } - else { - // This object is the target. - if (mappedField.getReverseMappedField() != null) { - IDataProvider dataProvider = mappedField.getReverseMappedField().getMappedClass().getDataProvider(); - try { - IPerceroObject targetObject = (IPerceroObject) Class.forName(targetClassIdPair.getClassName()).newInstance(); - return dataProvider.findAllRelatedObjects(targetObject, mappedField.getReverseMappedField(), shellOnly, userId); - } catch(Exception e) { - throw new SyncException(e); - } - } - else { - return new ArrayList(0); - } - } - - - } - - - protected boolean isMappedFieldInChangedFields(IPerceroObject perceroObject, MappedField nextMappedField, Map> changedFields) { - Iterator>> itrChangedFields = changedFields.entrySet().iterator(); - while (itrChangedFields.hasNext()) { - Entry> nextChangedFieldEntry = itrChangedFields.next(); - - if (!nextChangedFieldEntry.getKey().comparePerceroObject(perceroObject)) { - // Not the same object. - continue; - } - - for(MappedField nextChangedField : nextChangedFieldEntry.getValue()) { - if ( StringUtils.hasText(nextMappedField.getColumnName()) && nextMappedField.getColumnName().equalsIgnoreCase(nextChangedField.getColumnName()) ) { - // We have found our field. - return true; - } - else if ( StringUtils.hasText(nextMappedField.getJoinColumnName()) && nextMappedField.getJoinColumnName().equalsIgnoreCase(nextChangedField.getJoinColumnName()) ) { - // We have found our field. - return true; - } - } - } - - return false; - } - - - /** - * Populates all the *-TO-MANY relationships on the specified object. All TO-MANY relationships have their - * relationship data stored on the other side of the relationship, so we have to go those dataProvider and - * ask for the objects. - * - * @param perceroObject - * @param userId - * @throws SyncException - * @throws IllegalArgumentException - * @throws IllegalAccessException - * @throws InvocationTargetException - */ - public void populateToManyRelationships(IPerceroObject perceroObject, Boolean shellOnly, - String userId) throws SyncException, IllegalArgumentException, - IllegalAccessException, InvocationTargetException { - if (perceroObject == null || !StringUtils.hasText(perceroObject.getID())) { - // Invalid object. - log.warn("Invalid object in populateToManyRelationships"); - return; - } - - MappedClass mappedClass = MappedClassManagerFactory.getMappedClassManager().getMappedClassByClassName(perceroObject.getClass().getCanonicalName()); - for(MappedField nextToManyMappedField : mappedClass.toManyFields) { - - // TODO: Take into account Access Rights. - List allRelatedObjects = findAllRelatedObjects(perceroObject, nextToManyMappedField, shellOnly, userId); - nextToManyMappedField.getSetter().invoke(perceroObject, allRelatedObjects); - } - } - - public void populateToOneRelationships(IPerceroObject perceroObject, Boolean shellOnly, - String userId) throws SyncException, IllegalArgumentException, - IllegalAccessException, InvocationTargetException { - if (perceroObject == null || !StringUtils.hasText(perceroObject.getID())) { - // Invalid object. - log.warn("Invalid object in populateToOneRelationships"); - return; - } - - MappedClass mappedClass = MappedClassManagerFactory.getMappedClassManager().getMappedClassByClassName(perceroObject.getClass().getCanonicalName()); - for(MappedFieldPerceroObject nextToOneMappedField : mappedClass.toOneFields) { - // If perceroObject is the "owner" of this relationship, then we have all the data necessary here. - if (nextToOneMappedField.isSourceEntity()) { + + // @Override + @SuppressWarnings("unchecked") + public List findAllRelatedObjects(IPerceroObject perceroObject, MappedField mappedField, Boolean shellOnly, String userId) throws SyncException { + List result = new ArrayList(); + + if (!StringUtils.hasText(perceroObject.getID())) { + // No valid ID on the object, so can't search for it. + return result; + } + + if (mappedField.getMappedClass().getSourceMappedFields().contains(mappedField)) { + // This object is the source. + IDataAccessObject dao = (IDataAccessObject) DAORegistry.getInstance().getDataAccessObject(mappedField.getMappedClass().className); + IPerceroObject thisObject = dao.retrieveObject(BaseDataObject.toClassIdPair(perceroObject), userId, false); + IPerceroObject relatedObject; + try { + relatedObject = (IPerceroObject) mappedField.getGetter().invoke(thisObject); + } catch (Exception e) { + throw new SyncException(e); + } + if (relatedObject != null) { + if (!shellOnly) { + MappedClass relatedMappedClass = MappedClassManagerFactory.getMappedClassManager().getMappedClassByClassName(relatedObject.getClass().getCanonicalName()); + relatedObject = relatedMappedClass.getDataProvider().findById(BaseDataObject.toClassIdPair(relatedObject), userId); + } + result.add(relatedObject); + } + result.add(relatedObject); + } + else { + // This object is the target. + // The reverse mapped field should be the MappedField on the target object, the one that this MUST be the data provider for. + MappedField reverseMappedField = mappedField.getReverseMappedField(); + if (reverseMappedField == null) { + // No reverse mapped field, meaning there is nothing to do. + return result; + } + + IDataProvider dataProvider = reverseMappedField.getMappedClass().getDataProvider(); + result = dataProvider.getAllByRelationship(reverseMappedField, BaseDataObject.toClassIdPair(perceroObject), shellOnly, userId); + } + + return result; + } + + @SuppressWarnings("unchecked") + public List getAllByRelationship(MappedField mappedField, ClassIDPair targetClassIdPair, Boolean shellOnly, String userId) throws SyncException { + if (mappedField.getMappedClass().getSourceMappedFields().contains(mappedField)) { + // This object is the source. + IDataAccessObject dao = (IDataAccessObject) DAORegistry.getInstance().getDataAccessObject(mappedField.getMappedClass().className); + return dao.retrieveAllByRelationship(mappedField, targetClassIdPair, shellOnly, userId); + } + else { + // This object is the target. + if (mappedField.getReverseMappedField() != null) { + IDataProvider dataProvider = mappedField.getReverseMappedField().getMappedClass().getDataProvider(); + try { + IPerceroObject targetObject = (IPerceroObject) Class.forName(targetClassIdPair.getClassName()).newInstance(); + return dataProvider.findAllRelatedObjects(targetObject, mappedField.getReverseMappedField(), shellOnly, userId); + } catch(Exception e) { + throw new SyncException(e); + } + } + else { + return new ArrayList(0); + } + } + + + } + + + protected boolean isMappedFieldInChangedFields(IPerceroObject perceroObject, MappedField nextMappedField, Map> changedFields) { + Iterator>> itrChangedFields = changedFields.entrySet().iterator(); + while (itrChangedFields.hasNext()) { + Entry> nextChangedFieldEntry = itrChangedFields.next(); + + if (!nextChangedFieldEntry.getKey().comparePerceroObject(perceroObject)) { + // Not the same object. + continue; + } + + for(MappedField nextChangedField : nextChangedFieldEntry.getValue()) { + if ( StringUtils.hasText(nextMappedField.getColumnName()) && nextMappedField.getColumnName().equalsIgnoreCase(nextChangedField.getColumnName()) ) { + // We have found our field. + return true; + } + else if ( StringUtils.hasText(nextMappedField.getJoinColumnName()) && nextMappedField.getJoinColumnName().equalsIgnoreCase(nextChangedField.getJoinColumnName()) ) { + // We have found our field. + return true; + } + } + } + + return false; + } + + + /** + * Populates all the *-TO-MANY relationships on the specified object. All TO-MANY relationships have their + * relationship data stored on the other side of the relationship, so we have to go those dataProvider and + * ask for the objects. + * + * @param perceroObject + * @param userId + * @throws SyncException + * @throws IllegalArgumentException + * @throws IllegalAccessException + * @throws InvocationTargetException + */ + public void populateToManyRelationships(IPerceroObject perceroObject, Boolean shellOnly, + String userId) throws SyncException, IllegalArgumentException, + IllegalAccessException, InvocationTargetException { + if (perceroObject == null || !StringUtils.hasText(perceroObject.getID())) { + // Invalid object. + log.warn("Invalid object in populateToManyRelationships"); + return; + } + + MappedClass mappedClass = MappedClassManagerFactory.getMappedClassManager().getMappedClassByClassName(perceroObject.getClass().getCanonicalName()); + for(MappedField nextToManyMappedField : mappedClass.toManyFields) { + + // TODO: Take into account Access Rights. + List allRelatedObjects = findAllRelatedObjects(perceroObject, nextToManyMappedField, shellOnly, userId); + nextToManyMappedField.getSetter().invoke(perceroObject, allRelatedObjects); + } + } + + public void populateToOneRelationships(IPerceroObject perceroObject, Boolean shellOnly, + String userId) throws SyncException, IllegalArgumentException, + IllegalAccessException, InvocationTargetException { + if (perceroObject == null || !StringUtils.hasText(perceroObject.getID())) { + // Invalid object. + log.warn("Invalid object in populateToOneRelationships"); + return; + } + + MappedClass mappedClass = MappedClassManagerFactory.getMappedClassManager().getMappedClassByClassName(perceroObject.getClass().getCanonicalName()); + for(MappedFieldPerceroObject nextToOneMappedField : mappedClass.toOneFields) { + // If perceroObject is the "owner" of this relationship, then we have all the data necessary here. + if (nextToOneMappedField.isSourceEntity()) { // if (!shellOnly || true) { // // We want more than a Shell object, so ask the dataProvider of the mappedField for that object. // IDataProvider dataProvider = nextToOneMappedField.getReverseMappedField().getMappedClass().getDataProvider(); @@ -1251,29 +1240,29 @@ public void populateToOneRelationships(IPerceroObject perceroObject, Boolean she // } // } // } - } - else { - // We need to go to the dataProvider for the related object's class and ask for it. - // Though this returns a List, we expect there to be only one result in the list. - List allRelatedObjects = findAllRelatedObjects(perceroObject, nextToOneMappedField, shellOnly, userId); - IPerceroObject relatedPerceroObject = null; - if (allRelatedObjects != null && !allRelatedObjects.isEmpty()) { - relatedPerceroObject = allRelatedObjects.get(0); - } - nextToOneMappedField.getSetter().invoke(perceroObject, relatedPerceroObject); - } - } - } - - protected void addObjectToCache(IPerceroObject nextPerceroObject) { - String key = RedisKeyUtils.classIdPair(nextPerceroObject.getClass().getCanonicalName(), nextPerceroObject.getID()); - if (cacheTimeout > 0) - cacheDataStore.setValue(key, ((BaseDataObject)nextPerceroObject).toJson()); - - // (Re)Set the expiration. - if (cacheTimeout > 0 && key != null) { - cacheDataStore.expire(key, cacheTimeout, TimeUnit.SECONDS); - } - } + } + else { + // We need to go to the dataProvider for the related object's class and ask for it. + // Though this returns a List, we expect there to be only one result in the list. + List allRelatedObjects = findAllRelatedObjects(perceroObject, nextToOneMappedField, shellOnly, userId); + IPerceroObject relatedPerceroObject = null; + if (allRelatedObjects != null && !allRelatedObjects.isEmpty()) { + relatedPerceroObject = allRelatedObjects.get(0); + } + nextToOneMappedField.getSetter().invoke(perceroObject, relatedPerceroObject); + } + } + } + + protected void addObjectToCache(IPerceroObject nextPerceroObject) { + String key = RedisKeyUtils.classIdPair(nextPerceroObject.getClass().getCanonicalName(), nextPerceroObject.getID()); + if (cacheTimeout > 0) + cacheDataStore.setValue(key, ((BaseDataObject)nextPerceroObject).toJson()); + + // (Re)Set the expiration. + if (cacheTimeout > 0 && key != null) { + cacheDataStore.expire(key, cacheTimeout, TimeUnit.SECONDS); + } + } } From a462bc5fab2ea6074722e523b48023cfc6af9dcd Mon Sep 17 00:00:00 2001 From: Collin Brown Date: Mon, 12 Oct 2015 08:12:15 -0700 Subject: [PATCH 025/105] Fix for null clientId --- .../com/percero/agents/sync/access/RedisAccessManager.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/main/java/com/percero/agents/sync/access/RedisAccessManager.java b/src/main/java/com/percero/agents/sync/access/RedisAccessManager.java index c392dec..eb29328 100644 --- a/src/main/java/com/percero/agents/sync/access/RedisAccessManager.java +++ b/src/main/java/com/percero/agents/sync/access/RedisAccessManager.java @@ -760,6 +760,9 @@ public void saveUpdateJournalClients(ClassIDPair pair, Collection client String classValue = RedisKeyUtils.classIdPair(pair.getClassName(), pair.getID()); for(String nextClient : clientIds) { + if (!StringUtils.hasText(nextClient)) { + continue; + } if (!sendToPusher && pusherClient != null) { // Don't send to pushing Client if (nextClient.equals(pusherClient)) { From cc9e5fd1023f08294bf1ef5bcb34cc73d31253d3 Mon Sep 17 00:00:00 2001 From: Collin Brown Date: Mon, 12 Oct 2015 19:53:55 -0700 Subject: [PATCH 026/105] Batching updateReferences for each class name used in the batch loader. Required some enhancements to the DataProviders to retrieve an object directly (and only) from the cache. Also changes the way updates to related objects are handled to allow the updateTableProcessor to run a bit more efficiently. --- .../agents/sync/helpers/PostPutHelper.java | 28 ++-- .../sync/jobs/UpdateTableProcessor.java | 130 ++++++++++++++---- .../agents/sync/services/DAODataProvider.java | 17 ++- .../agents/sync/services/IDataProvider.java | 3 + .../sync/services/RedisDataProvider.java | 21 +++ .../sync/services/SyncAgentDataProvider.java | 56 +++++--- .../sync/services/SyncAgentService.java | 43 +++++- 7 files changed, 241 insertions(+), 57 deletions(-) diff --git a/src/main/java/com/percero/agents/sync/helpers/PostPutHelper.java b/src/main/java/com/percero/agents/sync/helpers/PostPutHelper.java index 50c5a0d..1f45942 100644 --- a/src/main/java/com/percero/agents/sync/helpers/PostPutHelper.java +++ b/src/main/java/com/percero/agents/sync/helpers/PostPutHelper.java @@ -81,7 +81,6 @@ public void postPutObject(ClassIDPair pair, String pusherUserId, String pusherCl pushObjectUpdateJournals(clientIds, pair, pairChangedFields); // Now run past the ChangeWatcher. - if (changedFields == null || changedFields.isEmpty()) { accessManager.checkChangeWatchers(pair, null, null); } @@ -92,15 +91,26 @@ public void postPutObject(ClassIDPair pair, String pusherUserId, String pusherCl Map.Entry> nextEntry = itrChangedFieldEntryset.next(); ClassIDPair thePair = nextEntry.getKey(); Collection changedMappedFields = nextEntry.getValue(); - Iterator itrChangedFields = changedMappedFields.iterator(); - String[] fieldNames = new String[changedMappedFields.size()]; - int i = 0; - while (itrChangedFields.hasNext()) { - MappedField nextChangedField = itrChangedFields.next(); - fieldNames[i] = nextChangedField.getField().getName(); - i++; + + // If thePair is NOT the object being updated, then need to run the postPutHelper for the Pair object as well. + if (!thePair.equals(pair)) { + Map> thePairChangedFields = new HashMap>(1); + thePairChangedFields.put(thePair, changedMappedFields); + + // This will also run thePair through the ChangeWatcher check below. + this.postPutObject(thePair, pusherUserId, pusherClientId, pushToUser, thePairChangedFields); + } + else { + Iterator itrChangedFields = changedMappedFields.iterator(); + String[] fieldNames = new String[changedMappedFields.size()]; + int i = 0; + while (itrChangedFields.hasNext()) { + MappedField nextChangedField = itrChangedFields.next(); + fieldNames[i] = nextChangedField.getField().getName(); + i++; + } + accessManager.checkChangeWatchers(thePair, fieldNames, null); } - accessManager.checkChangeWatchers(thePair, fieldNames, null); } // Iterator itrChangedFieldKeyset = changedFields.keySet().iterator(); // while (itrChangedFieldKeyset.hasNext()) { diff --git a/src/main/java/com/percero/agents/sync/jobs/UpdateTableProcessor.java b/src/main/java/com/percero/agents/sync/jobs/UpdateTableProcessor.java index 0895f22..7e656b4 100644 --- a/src/main/java/com/percero/agents/sync/jobs/UpdateTableProcessor.java +++ b/src/main/java/com/percero/agents/sync/jobs/UpdateTableProcessor.java @@ -1,10 +1,36 @@ package com.percero.agents.sync.jobs; +import java.sql.CallableStatement; +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.SQLSyntaxErrorException; +import java.sql.Statement; +import java.sql.Types; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Date; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Random; +import java.util.Set; + +import org.apache.log4j.Logger; +import org.joda.time.DateTime; +import org.springframework.util.StringUtils; + import com.percero.agents.sync.access.IAccessManager; import com.percero.agents.sync.cache.CacheManager; import com.percero.agents.sync.helpers.PostDeleteHelper; import com.percero.agents.sync.helpers.PostPutHelper; -import com.percero.agents.sync.metadata.*; +import com.percero.agents.sync.metadata.IMappedClassManager; +import com.percero.agents.sync.metadata.MappedClass; +import com.percero.agents.sync.metadata.MappedClassManagerFactory; +import com.percero.agents.sync.metadata.MappedField; +import com.percero.agents.sync.metadata.MappedFieldPerceroObject; import com.percero.agents.sync.services.DataProviderManager; import com.percero.agents.sync.services.IDataProvider; import com.percero.agents.sync.vo.BaseDataObject; @@ -12,13 +38,6 @@ import com.percero.framework.bl.IManifest; import com.percero.framework.vo.IPerceroObject; import com.percero.framework.vo.PerceroList; -import org.apache.log4j.Logger; -import org.joda.time.DateTime; -import org.springframework.util.StringUtils; - -import java.sql.*; -import java.util.*; -import java.util.Date; /** * Responsible for querying an update table and processing the rows. @@ -60,8 +79,15 @@ public UpdateTableProcessor(String tableName, this.dataProviderManager= dataProviderManager; this.accessManager = accessManager; this.maxRowsToProcess = connectionFactory.getWeight(); + + classNamesToUpdateReferences = new HashSet(); } + + // TODO: This should be moved to be stored in a database table. Could + // possibly use the existing "entire table update" mechanism to handle this + // (for each reverse mapped class) + private Set classNamesToUpdateReferences = null; /** * Update table schema looks like this @@ -106,7 +132,11 @@ public void run(){ UpdateTableProcessReporter.getInstance() .submitCountAndTime(tableName, successfulRows.size(), endTime.getTime() - startTime.getTime()); } - + + for(String className : classNamesToUpdateReferences) { + updateReferences(className); + } + if (!result.isSuccess()) { logger.warn("Update table processor (" + tableName + ") failed. Details:"); logger.warn(result); @@ -163,23 +193,18 @@ protected boolean processRow(UpdateTableRow row) throws Exception{ @SuppressWarnings("rawtypes") protected boolean processUpdateSingle(UpdateTableRow row) throws Exception{ List classes = getClassesForTableName(row.getTableName()); - + for(Class clazz : classes) { String className = clazz.getCanonicalName(); IMappedClassManager mcm = MappedClassManagerFactory.getMappedClassManager(); MappedClass mappedClass = mcm.getMappedClassByClassName(className); IDataProvider dataProvider = dataProviderManager.getDataProviderByName(mappedClass.dataProviderName); - + ClassIDPair pair = new ClassIDPair(row.getRowId(), className); handleUpdateClassIdPair(dataProvider, pair); - - // We need to update ALL referencing objects in the case that a - // relationship was updated. Since we don't have the OLD object, we have - // no way of telling what may have changed. - updateReferences(className); } - + return true; } @@ -251,12 +276,66 @@ protected void processUpdates(String className, Collection Ids) throws E } protected void handleUpdateClassIdPair(IDataProvider dataProvider, ClassIDPair pair) throws Exception { - IPerceroObject object = dataProvider.findById(pair, null, true); + // Attempt to retrieve from the cache so that we have the "OLD" value. + IPerceroObject oldValue = dataProvider.retrieveCachedObject(pair); + // Now retrieve the object directly from the data source so that we + // can compare it to the cached/old value. + IPerceroObject perceroObject = dataProvider.findById(pair, null, true); + // If PerceroObject is NULL, then it no longer exists and we can drop this update. + if (perceroObject == null) { + return; + } - if (object != null) { - cacheManager.updateCachedObject(object, null); - postPutHelper.postPutObject(pair, null, null, true, null); + Map> changedFields = null; + if (oldValue != null) { + // dataProvider.getChangedMappedFields is typically used to + // compare a new object, but we can use the cached object in + // this case (we just need to tell getChangedMappedFields to NOT + // use the cache). + changedFields = dataProvider + .getChangedMappedFields(perceroObject, oldValue); + if (changedFields == null || changedFields.size() > 0) { + // Something has changed. + cacheManager.updateCachedObject(perceroObject, changedFields); + postPutHelper.postPutObject(pair, null, null, true, changedFields); + + Iterator>> itrChangedFieldEntryset = changedFields.entrySet().iterator(); + while (itrChangedFieldEntryset.hasNext()) { + Map.Entry> nextEntry = itrChangedFieldEntryset.next(); + ClassIDPair thePair = nextEntry.getKey(); + Collection changedMappedFields = nextEntry.getValue(); + + // If thePair is NOT the object being updated, then need to run the postPutHelper for the Pair object as well. + if (!thePair.equals(pair)) { + Map> thePairChangedFields = new HashMap>(1); + thePairChangedFields.put(thePair, changedMappedFields); + + postPutHelper.postPutObject(thePair, null, null, true, thePairChangedFields); + } + else { + Iterator itrChangedFields = changedMappedFields.iterator(); + String[] fieldNames = new String[changedMappedFields.size()]; + int i = 0; + while (itrChangedFields.hasNext()) { + MappedField nextChangedField = itrChangedFields.next(); + fieldNames[i] = nextChangedField.getField().getName(); + i++; + } + accessManager.checkChangeWatchers(thePair, fieldNames, null); + } + } + } } + else { + cacheManager.updateCachedObject(perceroObject, null); + postPutHelper.postPutObject(pair, null, null, true, null); + + // We don't have any record of the old value, so we need to + // update ALL referencing objects in the case that a + // relationship was updated. Since we don't have the OLD object, + // we have no way of telling what may have changed. + classNamesToUpdateReferences.add(pair.getClassName()); + } } /** @@ -275,7 +354,7 @@ protected boolean processInsertSingle(UpdateTableRow row) throws Exception{ // that extra work for the whole class in updateReferences. postPutHelper.postPutObject(new ClassIDPair(row.getRowId(), className), null, null, true, null); - updateReferences(className); + classNamesToUpdateReferences.add(className); } return true; @@ -304,7 +383,7 @@ protected boolean processInsertTable(UpdateTableRow row) throws Exception { } } - updateReferences(className); + classNamesToUpdateReferences.add(className); } return true; @@ -331,7 +410,7 @@ protected boolean processDeleteSingle(UpdateTableRow row) throws Exception{ handleDeletedObject(cachedObject, clazz, className, row.getRowId()); - updateReferences(className); + classNamesToUpdateReferences.add(className); } return true; @@ -384,7 +463,7 @@ protected boolean processDeleteTable(UpdateTableRow row) throws Exception{ } } - updateReferences(className); + classNamesToUpdateReferences.add(className); } return true; @@ -425,6 +504,7 @@ protected Set getAllClassIdPairsForTable(String tableName) throws E * Finds all back references to this class and pushes updates to all of them. * @param className */ + // TODO: Need to batch this. protected void updateReferences(String className){ IMappedClassManager mcm = MappedClassManagerFactory.getMappedClassManager(); MappedClass mappedClass = mcm.getMappedClassByClassName(className); diff --git a/src/main/java/com/percero/agents/sync/services/DAODataProvider.java b/src/main/java/com/percero/agents/sync/services/DAODataProvider.java index 5c578a0..ca423ae 100644 --- a/src/main/java/com/percero/agents/sync/services/DAODataProvider.java +++ b/src/main/java/com/percero/agents/sync/services/DAODataProvider.java @@ -312,6 +312,10 @@ private void setObjectExpiration(String key) { } } + @Override + public IPerceroObject retrieveCachedObject(ClassIDPair classIdPair) throws Exception { + return retrieveFromRedisCache(classIdPair); + } @SuppressWarnings({ "rawtypes", "unchecked" }) private IPerceroObject retrieveFromRedisCache(ClassIDPair classIdPair) throws Exception { @@ -911,15 +915,24 @@ public List cleanObject(List perceroObjects, Str } } + public Map> getChangedMappedFields(IPerceroObject newObject) { + return getChangedMappedFields(newObject, false); + } + + public Map> getChangedMappedFields(IPerceroObject newObject, boolean ignoreCache) { + String className = newObject.getClass().getCanonicalName(); + IPerceroObject oldObject = findById(new ClassIDPair(newObject.getID(), className), null, ignoreCache); + return getChangedMappedFields(oldObject, newObject); + } + @SuppressWarnings("rawtypes") - public Map> getChangedMappedFields(IPerceroObject newObject) { + public Map> getChangedMappedFields(IPerceroObject oldObject, IPerceroObject newObject) { Map> result = new HashMap>(); Collection baseObjectResult = null; ClassIDPair basePair = new ClassIDPair(newObject.getID(), newObject.getClass().getCanonicalName()); String className = newObject.getClass().getCanonicalName(); - IPerceroObject oldObject = findById(new ClassIDPair(newObject.getID(), className), null); IMappedClassManager mcm = MappedClassManagerFactory.getMappedClassManager(); MappedClass mc = mcm.getMappedClassByClassName(className); Iterator itrMappedFields = mc.externalizableFields.iterator(); diff --git a/src/main/java/com/percero/agents/sync/services/IDataProvider.java b/src/main/java/com/percero/agents/sync/services/IDataProvider.java index dc40989..2573174 100644 --- a/src/main/java/com/percero/agents/sync/services/IDataProvider.java +++ b/src/main/java/com/percero/agents/sync/services/IDataProvider.java @@ -25,6 +25,7 @@ public interface IDataProvider { public List runQuery(MappedClass mappedClass, String queryName, Object[] queryArguments, String clientId) throws SyncException; public IPerceroObject findById(ClassIDPair classIdPair, String userId); public IPerceroObject findById(ClassIDPair classIdPair, String userId, Boolean ignoreCache); + public IPerceroObject retrieveCachedObject(ClassIDPair classIdPair) throws Exception; public List findByIds(ClassIDPairs classIdPairs, String userId); public List findByIds(ClassIDPairs classIdPairs, String userId, Boolean ignoreCache); // public IPerceroObject findUnique(IPerceroObject theQueryObject, String userId); @@ -51,6 +52,8 @@ public interface IDataProvider { * @return */ public Map> getChangedMappedFields(IPerceroObject newObject); + public Map> getChangedMappedFields(IPerceroObject newObject, boolean ignoreCache); + public Map> getChangedMappedFields(IPerceroObject oldObject, IPerceroObject compareObject); /** * Given the mappedField, returns ALL objects in the relationship described by the mappedField. diff --git a/src/main/java/com/percero/agents/sync/services/RedisDataProvider.java b/src/main/java/com/percero/agents/sync/services/RedisDataProvider.java index 0922ba2..98c9611 100644 --- a/src/main/java/com/percero/agents/sync/services/RedisDataProvider.java +++ b/src/main/java/com/percero/agents/sync/services/RedisDataProvider.java @@ -426,4 +426,25 @@ public List findByIds(ClassIDPairs classIdPairs, // TODO Auto-generated method stub return null; } + + @Override + public IPerceroObject retrieveCachedObject(ClassIDPair classIdPair) + throws Exception { + // TODO Auto-generated method stub + return null; + } + + @Override + public Map> getChangedMappedFields( + IPerceroObject newObject, boolean ignoreCache) { + // TODO Auto-generated method stub + return null; + } + + @Override + public Map> getChangedMappedFields( + IPerceroObject oldObject, IPerceroObject compareObject) { + // TODO Auto-generated method stub + return null; + } } diff --git a/src/main/java/com/percero/agents/sync/services/SyncAgentDataProvider.java b/src/main/java/com/percero/agents/sync/services/SyncAgentDataProvider.java index adf8efa..62a248b 100644 --- a/src/main/java/com/percero/agents/sync/services/SyncAgentDataProvider.java +++ b/src/main/java/com/percero/agents/sync/services/SyncAgentDataProvider.java @@ -592,6 +592,11 @@ public IPerceroObject systemGetByIdWithClassAndSession(ClassIDPair classIdPair, return null; } + @Override + public IPerceroObject retrieveCachedObject(ClassIDPair classIdPair) throws Exception { + return retrieveFromRedisCache(classIdPair); + } + @SuppressWarnings({ "rawtypes", "unchecked" }) private IPerceroObject retrieveFromRedisCache(ClassIDPair classIdPair) throws Exception { IPerceroObject result = null; @@ -1457,33 +1462,48 @@ public Object cleanObject(Object object, Session s, String userId) { } + public Map> getChangedMappedFields(IPerceroObject compareObject) { + return getChangedMappedFields(compareObject, false); + } + public Map> getChangedMappedFields(IPerceroObject compareObject, boolean ignoreCache) { + String className = compareObject.getClass().getCanonicalName(); + IPerceroObject oldObject = systemGetById(new ClassIDPair(compareObject.getID(), className), ignoreCache); + + return getChangedMappedFields(oldObject, compareObject); + } @SuppressWarnings("rawtypes") - public Map> getChangedMappedFields(IPerceroObject newObject) { + public Map> getChangedMappedFields(IPerceroObject oldObject, IPerceroObject compareObject) { Map> result = new HashMap>(); - Collection baseObjectResult = null; - ClassIDPair basePair = new ClassIDPair(newObject.getID(), newObject.getClass().getCanonicalName()); - String className = newObject.getClass().getCanonicalName(); - IPerceroObject oldObject = systemGetById(new ClassIDPair(newObject.getID(), className)); + // Validate both objects. + if (oldObject == null || compareObject == null || oldObject.getID() == null || compareObject.getID() == null || !oldObject.getID().equals(compareObject.getID())) { + return result; + } + + Collection baseObjectResult = null; + ClassIDPair basePair = new ClassIDPair(compareObject.getID(), compareObject.getClass().getCanonicalName()); + + String className = compareObject.getClass().getCanonicalName(); + IMappedClassManager mcm = MappedClassManagerFactory.getMappedClassManager(); MappedClass mc = mcm.getMappedClassByClassName(className); Iterator itrMappedFields = mc.externalizableFields.iterator(); while (itrMappedFields.hasNext()) { MappedField nextMappedField = itrMappedFields.next(); try { - Boolean fieldIsEqual = nextMappedField.compareObjects(oldObject, newObject); + Boolean fieldIsEqual = nextMappedField.compareObjects(oldObject, compareObject); if (!fieldIsEqual) { if (baseObjectResult == null) { baseObjectResult = new HashSet(); result.put(basePair, baseObjectResult); } baseObjectResult.add(nextMappedField); - + // If this is a relationship field, then need to grab the old and new values. if (nextMappedField.getReverseMappedField() != null) { if (nextMappedField instanceof MappedFieldPerceroObject) { MappedFieldPerceroObject nextMappedFieldPerceroObject = (MappedFieldPerceroObject) nextMappedField; - + IPerceroObject oldReversePerceroObject = (IPerceroObject) nextMappedFieldPerceroObject.getGetter().invoke(oldObject); if (oldReversePerceroObject != null) { ClassIDPair oldReversePair = new ClassIDPair(oldReversePerceroObject.getID(), oldReversePerceroObject.getClass().getCanonicalName()); @@ -1494,8 +1514,8 @@ public Map> getChangedMappedFields(IPercero } oldReverseChangedFields.add(nextMappedField.getReverseMappedField()); } - - IPerceroObject newReversePerceroObject = (IPerceroObject) nextMappedFieldPerceroObject.getGetter().invoke(newObject); + + IPerceroObject newReversePerceroObject = (IPerceroObject) nextMappedFieldPerceroObject.getGetter().invoke(compareObject); if (newReversePerceroObject != null) { ClassIDPair newReversePair = new ClassIDPair(newReversePerceroObject.getID(), newReversePerceroObject.getClass().getCanonicalName()); Collection changedFields = result.get(newReversePair); @@ -1508,22 +1528,22 @@ public Map> getChangedMappedFields(IPercero } else if (nextMappedField instanceof MappedFieldList) { MappedFieldList nextMappedFieldList = (MappedFieldList) nextMappedField; - + List oldReverseList = (List) nextMappedFieldList.getGetter().invoke(oldObject); if (oldReverseList == null) oldReverseList = new ArrayList(); - - List newReverseList = (List) nextMappedFieldList.getGetter().invoke(newObject); + + List newReverseList = (List) nextMappedFieldList.getGetter().invoke(compareObject); if (newReverseList == null) newReverseList = new ArrayList(); - + // Compare each item in the lists. Collection oldChangedList = retrieveObjectsNotInCollection(oldReverseList, newReverseList); Iterator itrOldChangedList = oldChangedList.iterator(); while (itrOldChangedList.hasNext()) { BaseDataObject nextOldChangedObject = (BaseDataObject) itrOldChangedList.next(); ClassIDPair nextOldReversePair = BaseDataObject.toClassIdPair(nextOldChangedObject); - + // Old object is not in new list, so add to list of changed fields. Collection changedFields = result.get(nextOldReversePair); if (changedFields == null) { @@ -1532,13 +1552,13 @@ else if (nextMappedField instanceof MappedFieldList) { } changedFields.add(nextMappedField.getReverseMappedField()); } - + Collection newChangedList = retrieveObjectsNotInCollection(newReverseList, oldReverseList); Iterator itrNewChangedList = newChangedList.iterator(); while (itrNewChangedList.hasNext()) { BaseDataObject nextNewChangedObject = (BaseDataObject) itrNewChangedList.next(); ClassIDPair nextNewReversePair = BaseDataObject.toClassIdPair(nextNewChangedObject); - + // Old object is not in new list, so add to list of changed fields. Collection changedFields = result.get(nextNewReversePair); if (changedFields == null) { @@ -1555,7 +1575,7 @@ else if (nextMappedField instanceof MappedFieldList) { baseObjectResult.add(nextMappedField); } } - + return result; } diff --git a/src/main/java/com/percero/agents/sync/services/SyncAgentService.java b/src/main/java/com/percero/agents/sync/services/SyncAgentService.java index 91dce80..947b25b 100644 --- a/src/main/java/com/percero/agents/sync/services/SyncAgentService.java +++ b/src/main/java/com/percero/agents/sync/services/SyncAgentService.java @@ -744,10 +744,47 @@ public ServerResponse putObject(IPerceroObject perceroObject, String transaction cacheDataStore.lpushListValue(RedisKeyUtils.historicalObject(result.getClass().getCanonicalName(), result.getID()), historyObject); } - if (taskExecutor != null && false) { - taskExecutor.execute(new PostPutTask(postPutHelper, BaseDataObject.toClassIdPair((BaseDataObject) result), userId, clientId, pushToClient, changedFields)); - } else { +// if (taskExecutor != null && false) { +// taskExecutor.execute(new PostPutTask(postPutHelper, BaseDataObject.toClassIdPair((BaseDataObject) result), userId, clientId, pushToClient, changedFields)); +// } else { postPutHelper.postPutObject(BaseDataObject.toClassIdPair((BaseDataObject) result), userId, clientId, pushToClient, changedFields); +// } + + // For each changed field, we look at the reverse mapped + // relationship (if one exists) and update that object + // (this was already done in the cache by the + // cacheManager). + if (changedFields != null && !changedFields.isEmpty()) { + Iterator>> itrChangedFieldEntryset = changedFields.entrySet().iterator(); + while (itrChangedFieldEntryset.hasNext()) { + Map.Entry> nextEntry = itrChangedFieldEntryset.next(); + ClassIDPair thePair = nextEntry.getKey(); + Collection changedMappedFields = nextEntry.getValue(); + + // If thePair is NOT the object being updated, then need to run the postPutHelper for the Pair object as well. + if (!thePair.equals(classIdPair)) { + Map> thePairChangedFields = new HashMap>(1); + thePairChangedFields.put(thePair, changedMappedFields); + + // This will also run thePair through the ChangeWatcher check below. +// if (taskExecutor != null && false) { +// taskExecutor.execute(new PostPutTask(postPutHelper, thePair, userId, clientId, pushToClient, thePairChangedFields)); +// } else { + postPutHelper.postPutObject(thePair, userId, clientId, pushToClient, thePairChangedFields); +// } + } + else { + Iterator itrChangedFields = changedMappedFields.iterator(); + String[] fieldNames = new String[changedMappedFields.size()]; + int i = 0; + while (itrChangedFields.hasNext()) { + MappedField nextChangedField = itrChangedFields.next(); + fieldNames[i] = nextChangedField.getField().getName(); + i++; + } + accessManager.checkChangeWatchers(thePair, fieldNames, null); + } + } } } } From 10bab195fe8a039d1120b2403eae51814f90caee Mon Sep 17 00:00:00 2001 From: Collin Brown Date: Tue, 13 Oct 2015 07:59:06 -0700 Subject: [PATCH 027/105] Added logging warning for failed login --- .../java/com/percero/agents/auth/services/AuthService2.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/main/java/com/percero/agents/auth/services/AuthService2.java b/src/main/java/com/percero/agents/auth/services/AuthService2.java index f4bf2ae..940da7c 100644 --- a/src/main/java/com/percero/agents/auth/services/AuthService2.java +++ b/src/main/java/com/percero/agents/auth/services/AuthService2.java @@ -63,6 +63,9 @@ public AuthenticationResponse authenticate(AuthenticationRequest request) throws userToken = (UserToken) AuthHibernateUtils.cleanObject(userToken); response.setResult(userToken); } + else { + logger.warn("LOGIN FAILED (" + provider.getID() + "): Unable to retrieve valid Service User"); + } return response; } From 17be2283b25b411bd28345992ac30b2400c0433d Mon Sep 17 00:00:00 2001 From: Collin Brown Date: Tue, 13 Oct 2015 14:19:06 -0700 Subject: [PATCH 028/105] RunProcessRequest can now handle key/value dictionary as queryArguments --- pom.xml | 2 +- .../java/com/percero/agents/sync/vo/RunProcessRequest.java | 6 +++--- .../java/com/percero/agents/sync/vo/RunShardedProcess.java | 6 +++--- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/pom.xml b/pom.xml index a706955..45f6661 100644 --- a/pom.xml +++ b/pom.xml @@ -7,7 +7,7 @@ org.activestack syncengine - 1.1.19-SNAPSHOT + 1.1.20-SNAPSHOT 3.2.4.RELEASE diff --git a/src/main/java/com/percero/agents/sync/vo/RunProcessRequest.java b/src/main/java/com/percero/agents/sync/vo/RunProcessRequest.java index f226230..a9d6031 100644 --- a/src/main/java/com/percero/agents/sync/vo/RunProcessRequest.java +++ b/src/main/java/com/percero/agents/sync/vo/RunProcessRequest.java @@ -10,11 +10,11 @@ public void setQueryName(String queryName) { this.queryName = queryName; } - private Object[] queryArguments; - public Object[] getQueryArguments() { + private Object queryArguments; + public Object getQueryArguments() { return queryArguments; } - public void setQueryArguments(Object[] queryArguments) { + public void setQueryArguments(Object queryArguments) { this.queryArguments = queryArguments; } diff --git a/src/main/java/com/percero/agents/sync/vo/RunShardedProcess.java b/src/main/java/com/percero/agents/sync/vo/RunShardedProcess.java index 5b24625..046700d 100644 --- a/src/main/java/com/percero/agents/sync/vo/RunShardedProcess.java +++ b/src/main/java/com/percero/agents/sync/vo/RunShardedProcess.java @@ -13,7 +13,7 @@ public RunShardedProcess() { private String clientId; private String serviceGroupId; private String processId; - private Object[] arguments; + private Object arguments; public String getOriginatingClientId() { return originatingClientId; @@ -39,10 +39,10 @@ public String getProcessId() { public void setProcessId(String processId) { this.processId = processId; } - public Object[] getArguments() { + public Object getArguments() { return arguments; } - public void setArguments(Object[] arguments) { + public void setArguments(Object arguments) { this.arguments = arguments; } } From 60c6696bc077d7a4f61d745a642ccf193756a277 Mon Sep 17 00:00:00 2001 From: Collin Brown Date: Wed, 14 Oct 2015 12:00:05 -0700 Subject: [PATCH 029/105] Move populate relationships in findById to not attempt to access null object --- .../com/percero/agents/sync/services/DAODataProvider.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/percero/agents/sync/services/DAODataProvider.java b/src/main/java/com/percero/agents/sync/services/DAODataProvider.java index ca423ae..358d620 100644 --- a/src/main/java/com/percero/agents/sync/services/DAODataProvider.java +++ b/src/main/java/com/percero/agents/sync/services/DAODataProvider.java @@ -246,11 +246,11 @@ public IPerceroObject findById(ClassIDPair classIdPair, String userId, Boolean i IDataAccessObject dao = (IDataAccessObject) DAORegistry.getInstance().getDataAccessObject(classIdPair.getClassName()); // Retrieve results BEFORE applying access rules so that our cached value represents the full object. result = dao.retrieveObject(classIdPair, null, false); - populateToManyRelationships(result, true, null); - populateToOneRelationships(result, true, null); // Now put the object in the cache. if (result != null) { + populateToManyRelationships(result, true, null); + populateToOneRelationships(result, true, null); putObjectInRedisCache(result); } else { From 6bb7191c891c001dca1c2beaeb516005719ac9da Mon Sep 17 00:00:00 2001 From: Collin Brown Date: Wed, 14 Oct 2015 12:00:28 -0700 Subject: [PATCH 030/105] 0.0.21-SNAPSHOT --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 45f6661..81a104f 100644 --- a/pom.xml +++ b/pom.xml @@ -7,7 +7,7 @@ org.activestack syncengine - 1.1.20-SNAPSHOT + 1.1.21-SNAPSHOT 3.2.4.RELEASE From aa01f28aa794ba07f9128b1ade3ae32647b801c4 Mon Sep 17 00:00:00 2001 From: Collin Brown Date: Wed, 14 Oct 2015 15:23:49 -0700 Subject: [PATCH 031/105] Added configurable JedisPoolConfig for Spring redis connection pool --- pom.xml | 2 +- src/main/resources/env.properties.sample | 8 ++++++++ .../spring/changeWatcher-spring-config.xml | 19 ++++++++++++++++++- .../spring/percero-spring-config.xml | 15 +++++++++++++++ 4 files changed, 42 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index 81a104f..753cb7b 100644 --- a/pom.xml +++ b/pom.xml @@ -7,7 +7,7 @@ org.activestack syncengine - 1.1.21-SNAPSHOT + 1.1.22-SNAPSHOT 3.2.4.RELEASE diff --git a/src/main/resources/env.properties.sample b/src/main/resources/env.properties.sample index b720680..5bed2db 100644 --- a/src/main/resources/env.properties.sample +++ b/src/main/resources/env.properties.sample @@ -30,6 +30,14 @@ databaseProject.password=root redis.host=localhost redis.port=12345 redis.password=password +redis.maxTotal=10 +redis.maxIdle=8 +redis.minIdle=1 +redis.testOnBorrow=true +redis.testOnReturn=true +redis.testWhileIdle=true +redis.timeBetweenEvictionRunsMillis=60000 +redis.numTestsPerEvictionRun=10 #gateway.rabbitmq.login=root #gateway.rabbitmq.password=root diff --git a/src/main/resources/spring/changeWatcher-spring-config.xml b/src/main/resources/spring/changeWatcher-spring-config.xml index 35ce648..8fd2755 100644 --- a/src/main/resources/spring/changeWatcher-spring-config.xml +++ b/src/main/resources/spring/changeWatcher-spring-config.xml @@ -73,8 +73,20 @@ + + @@ -170,6 +182,10 @@ hibernate.dialect=org.hibernate.dialect.MySQLDialect + + hibernate.hbm2ddl.auto=update hibernate.show_sql=false @@ -248,7 +264,8 @@ - hibernate.dialect=org.hibernate.dialect.MySQLDialect + + hibernate.dialect=org.hibernate.dialect.Oracle10gDialect hibernate.hbm2ddl.auto=update hibernate.show_sql=false diff --git a/src/main/resources/spring/percero-spring-config.xml b/src/main/resources/spring/percero-spring-config.xml index d237308..07096da 100644 --- a/src/main/resources/spring/percero-spring-config.xml +++ b/src/main/resources/spring/percero-spring-config.xml @@ -75,8 +75,20 @@ + + @@ -220,6 +232,9 @@ hibernate.dialect=org.hibernate.dialect.MySQLDialect + hibernate.hbm2ddl.auto=update hibernate.show_sql=false From d89e021e52a00cdeb93e33899a3eefa67bafa1fd Mon Sep 17 00:00:00 2001 From: Collin Brown Date: Thu, 15 Oct 2015 08:18:43 -0700 Subject: [PATCH 032/105] Adding ClassIDPair as param for CUSTOM change watcher --- .../com/percero/agents/sync/access/RedisAccessManager.java | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/main/java/com/percero/agents/sync/access/RedisAccessManager.java b/src/main/java/com/percero/agents/sync/access/RedisAccessManager.java index eb29328..6dee518 100644 --- a/src/main/java/com/percero/agents/sync/access/RedisAccessManager.java +++ b/src/main/java/com/percero/agents/sync/access/RedisAccessManager.java @@ -1331,6 +1331,12 @@ public void checkChangeWatchers(ClassIDPair classIdPair, String[] fieldNames, St String nextChangeWatcher = itrChangeWatchers.next(); // TODO: Test this optimization. // if (getChangeWatcherResultExists(nextChangeWatcher)) { + // If the type is CUSTOM, need to append the ClassIDPair, if it exists. + if (nextChangeWatcher.startsWith("cw:cf:CUSTOM:")) { + if (classIdPair != null) { + nextChangeWatcher += ":" + classIdPair.getClassName() + ":" + classIdPair.getID(); + } + } setupRecalculateChangeWatcher(nextChangeWatcher); // } // else { @@ -1457,6 +1463,7 @@ protected Long removeChangeWatcherForField(ClassIDPair classIdPair, String[] fie } protected void setupRecalculateChangeWatcher(String changeWatcherId) { + ChangeWatcherReporting.internalRequestsCounter++; if (useChangeWatcherQueue && pushSyncHelper != null) { pushSyncHelper.pushStringToRoute( (new StringBuilder(changeWatcherId).append(":TS:").append(System.currentTimeMillis())).toString(), changeWatcherRouteName); From a89443adf30024d24650eb571df9a097fe64db68 Mon Sep 17 00:00:00 2001 From: Jonathan Samples Date: Thu, 15 Oct 2015 08:26:31 -0700 Subject: [PATCH 033/105] Fixed AccountHelper dependency on DAO... bad --- .../percero/agents/auth/helpers/UserAnchorHelper.java | 7 ++++--- .../percero/agents/sync/services/DAODataProvider.java | 10 ---------- src/main/resources/log4j.properties | 2 +- 3 files changed, 5 insertions(+), 14 deletions(-) diff --git a/src/main/java/com/percero/agents/auth/helpers/UserAnchorHelper.java b/src/main/java/com/percero/agents/auth/helpers/UserAnchorHelper.java index ceaa41e..a1e5ee3 100644 --- a/src/main/java/com/percero/agents/auth/helpers/UserAnchorHelper.java +++ b/src/main/java/com/percero/agents/auth/helpers/UserAnchorHelper.java @@ -5,6 +5,8 @@ import com.percero.agents.sync.dao.IDataAccessObject; import com.percero.agents.sync.metadata.EntityImplementation; import com.percero.agents.sync.metadata.MappedClass; +import com.percero.agents.sync.services.DataProviderManager; +import com.percero.agents.sync.services.IDataProvider; import com.percero.framework.bl.ManifestHelper; import com.percero.framework.vo.IPerceroObject; @@ -18,13 +20,12 @@ public static IUserAnchor getUserAnchor(String userId){ IUserAnchor result = null; Class userAnchorClass = ManifestHelper.findImplementingClass(IUserAnchor.class); - IDataAccessObject userAnchorDao = - (IDataAccessObject) DAORegistry.getInstance().getDataAccessObject(userAnchorClass.getName()); + IDataProvider dataProvider = DataProviderManager.getInstance().getDefaultDataProvider(); try { IUserAnchor example = (IUserAnchor) userAnchorClass.newInstance(); example.setUserId(userId); - List list = userAnchorDao.findByExample((IPerceroObject)example, null, null, false); + List list = dataProvider.findByExample((IPerceroObject)example, null, null, false); if(list.size() > 0) result = (IUserAnchor)list.get(0); }catch(Exception e){} diff --git a/src/main/java/com/percero/agents/sync/services/DAODataProvider.java b/src/main/java/com/percero/agents/sync/services/DAODataProvider.java index ca423ae..52473fe 100644 --- a/src/main/java/com/percero/agents/sync/services/DAODataProvider.java +++ b/src/main/java/com/percero/agents/sync/services/DAODataProvider.java @@ -36,13 +36,10 @@ @Component public class DAODataProvider implements IDataProvider { - // TODO: Better manage Hibernate Sessions (opening and closing). - private static final Logger log = Logger.getLogger(DAODataProvider.class); private static DAODataProvider instance = null; - public static DAODataProvider getInstance() { return instance; } @@ -51,13 +48,6 @@ public DAODataProvider() { instance = this; } - @PostConstruct - public void init() - { - dataProviderManager.addDataProvider(this); - ((DataProviderManager)dataProviderManager).setDefaultDataProvider(this); - } - public void initialize(){ // Do nothing } diff --git a/src/main/resources/log4j.properties b/src/main/resources/log4j.properties index 6a6b6bf..1962384 100644 --- a/src/main/resources/log4j.properties +++ b/src/main/resources/log4j.properties @@ -1,6 +1,6 @@ log4j.rootLogger=info, stdout -log4j.category.com.percero.agents.sync.jobs=DEBUG +#log4j.category.com.percero.agents.sync.jobs=DEBUG log4j.appender.stdout=org.apache.log4j.ConsoleAppender log4j.appender.stdout.layout=org.apache.log4j.PatternLayout From 2e08fc588c5e38dd204d855c7509720d2e55dea6 Mon Sep 17 00:00:00 2001 From: Collin Brown Date: Fri, 16 Oct 2015 23:32:59 -0700 Subject: [PATCH 034/105] Changing batching on client management --- pom.xml | 2 +- .../agents/auth/services/AuthService.java | 315 ++++++++++-------- .../sync/access/RedisAccessManager.java | 16 +- .../agents/sync/access/RedisKeyUtils.java | 4 + .../agents/sync/jobs/UpdateTablePoller.java | 2 +- .../sync/jobs/UpdateTableProcessReporter.java | 250 +++++++------- .../sync/jobs/UpdateTableProcessor.java | 4 +- .../percero/amqp/RabbitMQPushSyncHelper.java | 25 +- 8 files changed, 329 insertions(+), 289 deletions(-) diff --git a/pom.xml b/pom.xml index 753cb7b..06eb736 100644 --- a/pom.xml +++ b/pom.xml @@ -7,7 +7,7 @@ org.activestack syncengine - 1.1.22-SNAPSHOT + 1.1.25-SNAPSHOT 3.2.4.RELEASE diff --git a/src/main/java/com/percero/agents/auth/services/AuthService.java b/src/main/java/com/percero/agents/auth/services/AuthService.java index 2ae6d8e..effce71 100644 --- a/src/main/java/com/percero/agents/auth/services/AuthService.java +++ b/src/main/java/com/percero/agents/auth/services/AuthService.java @@ -479,24 +479,6 @@ private UserAccount updateUserAccountToken(UserAccount theQueryObject, Boolean c } } - /* - if (serviceUser.getEmails() != null && serviceUser.getEmails().size() > 0) { - String strFindUserIdentifier = "SELECT ui.user FROM UserIdentifier ui WHERE ui.type='email' AND ("; - int counter = 0; - for(String nextEmail : serviceUser.getEmails()) { - if (counter > 0) - strFindUserIdentifier += " OR "; - strFindUserIdentifier += "ui.userIdentifier='" + nextEmail + "'"; - counter++; - } - strFindUserIdentifier += ")"; - Query q = s.createQuery(strFindUserIdentifier); - List userList = (List) q.list(); - if (userList.size() > 0) { - theUser = userList.get(0); - } - }*/ - Transaction tx = s.beginTransaction(); tx.begin(); Date currentDate = new Date(); @@ -563,33 +545,6 @@ private UserAccount updateUserAccountToken(UserAccount theQueryObject, Boolean c } tx.commit(); } - /* - if (serviceUser.getEmails() != null && serviceUser.getEmails().size() > 0) { - if (s == null) - s = sessionFactoryAuth.openSession(); - Transaction tx = s.beginTransaction(); - Query q = null; - for(String nextEmail : serviceUser.getEmails()) { - q = s.createQuery("FROM UserIdentifier ui WHERE ui.userIdentifier=:uid AND ui.type='email'"); - q.setString("uid", nextEmail); - - List userIdenditifierList = (List) q.list(); - - if (userIdenditifierList.size() == 0) { - try { - UserIdentifier userIdentifier = new UserIdentifier(); - userIdentifier.setType("email"); - userIdentifier.setUser(theFoundUserAccount.getUser()); - userIdentifier.setUserIdentifier(nextEmail); - s.saveOrUpdate(userIdentifier); - } catch(Exception e) { - log.warn("Unable to save Email UserIdentifier for " + serviceUser.getName(), e); - } - } - } - tx.commit(); - } - */ } } catch (Exception e) { log.error("Unable to run authenticate UserAccount", e); @@ -690,57 +645,68 @@ private UserToken loginUserAccount(UserAccount theUserAccount, String clientId, /* (non-Javadoc) * @see com.com.percero.agents.auth.services.IAuthService#logoutUser(java.lang.String, java.lang.String, java.lang.String) */ - @SuppressWarnings("rawtypes") public Boolean logoutUser(String aUserId, String aToken, String aClientId) { Boolean result = false; - Boolean validUser = StringUtils.hasText(aUserId); - Boolean validClient = StringUtils.hasText(aClientId); + boolean validUser = StringUtils.hasText(aUserId); + boolean validClient = StringUtils.hasText(aClientId); + boolean validToken = StringUtils.hasText(aToken); // If neither a valid user or a valid client, then no one to logout. - if (!validUser && !validClient) { + if (!validUser && !validClient && !validToken) { + log.warn("Invalid user/client/token on AuthService.logoutUser"); return false; } - log.debug("Logging out User: " + aUserId + ", Client: " + aClientId); - User theQueryUser = new User(); - theQueryUser.setID(aUserId); - - UserToken theQueryUserToken = new UserToken(); - if (validUser) { - theQueryUserToken.setUser(theQueryUser); + String deleteUserTokenSql = "DELETE FROM UserToken WHERE "; + + // Match EITHER the ClientID OR the Token + if (validClient && validToken) { + log.debug("Logging out Client: " + aClientId + ", Token: " + aToken); + deleteUserTokenSql += " (clientId=:clientId OR token=:token) "; } - if (validClient) { - theQueryUserToken.setClientId(aClientId); + else if (validToken) { + log.debug("Logging out Token: " + aToken); + log.debug("Logging out Token: " + aToken); + deleteUserTokenSql += " token=:token "; } - - if (StringUtils.hasText(aToken)) { - theQueryUserToken.setToken(aToken); + else if (validClient) { + log.debug("Logging out Client: " + aClientId); + deleteUserTokenSql += " clientId=:clientId "; + } + else if (validUser) { + // This will log out ALL of the User's devices, logging them out completely. + log.warn("Logging out ALL User " + aUserId + " devices!"); + deleteUserTokenSql += " user_ID=:user_ID "; } - result = true; - List userTokenResults = findByExample(theQueryUserToken, null); - Iterator itrUserTokenResults = userTokenResults.iterator(); Session s = null; - while (itrUserTokenResults.hasNext()) { - UserToken userTokenResult = (UserToken) itrUserTokenResults.next(); + try { + s = sessionFactoryAuth.openSession(); - try { - if (s == null) { - s = sessionFactoryAuth.openSession(); - } - log.debug("Deleting UserToken: " + userTokenResult.getID() + ", Client: " + userTokenResult.getClientId()); - Transaction tx = s.beginTransaction(); - tx.begin(); - s.delete(userTokenResult); - tx.commit(); - } catch (StaleStateException e) { - // Most likely this failed because the userToken has already been deleted from the database. - log.debug("Unable to delete UserToken due to StaleStateException: " + e.getMessage()); - result = false; - } catch (Exception e) { - log.error("Unable to delete UserToken", e); - result = false; + Query deleteQuery = s.createSQLQuery(deleteUserTokenSql); + + if (validClient && validToken) { + deleteQuery.setString("token", aToken); + deleteQuery.setString("clientId", aClientId); + } + else if (validToken) { + deleteQuery.setString("token", aToken); } + else if (validClient) { + deleteQuery.setString("clientId", aClientId); + } + else if (validUser) { + deleteQuery.setString("user_ID", aUserId); + } + + deleteQuery.executeUpdate(); + } catch (StaleStateException e) { + // Most likely this failed because the userToken has already been deleted from the database. + log.debug("Unable to delete UserToken due to StaleStateException: " + e.getMessage()); + result = false; + } catch (Exception e) { + log.error("Unable to delete UserToken", e); + result = false; } if (s != null && s.isOpen()) { @@ -768,8 +734,10 @@ public boolean validateUserByToken(String regAppKey, String aUserId, String aTok query.setString("clientId", aClientId); Long uniqueResultCount = (Long) query.uniqueResult(); - if (uniqueResultCount != null && uniqueResultCount > 0) + if (uniqueResultCount != null && uniqueResultCount > 0) { +// result = false; result = true; + } else log.warn("Invalid User in validateUserByToken: User " + aUserId + ", Token " + aToken + ", Client " + aClientId); } catch (Exception e) { @@ -1019,38 +987,44 @@ private static String getRandomId() { **********************************************/ static final String DELETE_USER_TOKENS_COLLECTION_SQL = "DELETE FROM UserToken WHERE clientId IN (:clientIds)"; - @SuppressWarnings("unchecked") + private Set previousInvalidClientIds = new HashSet(); + +// @SuppressWarnings("unchecked") /** * This method checks for any rogue/ghost UserTokens and removes them. */ // @Scheduled(fixedRate=30000) // 30 Seconds @Scheduled(fixedDelay=300000, initialDelay=120000) // 5 Minutes, 2 Minutes private void cleanupUserTokens() { - Session s = null; +// Session s = null; try { - s = sessionFactoryAuth.openSession(); +// s = sessionFactoryAuth.openSession(); int firstResultCounter = 0; - int maxResults = 3; - String userTokenQueryString = "SELECT DISTINCT(ut.clientId), ut.deviceId FROM UserToken ut ORDER BY ut.clientId"; - Query userTokenQuery = s.createQuery(userTokenQueryString); - userTokenQuery.setMaxResults(maxResults); - userTokenQuery.setFirstResult(firstResultCounter); + int maxResults = 30; - // Gather up all clientIds to remove, then delete at the end. List clientIdsToDelete = new LinkedList(); - List userTokenClientIds = userTokenQuery.list(); - Map clientDevicesMap = new HashMap(userTokenClientIds.size()); - Iterator itrUserTokenClientIds = userTokenClientIds.iterator(); - if (userTokenClientIds != null) { - while (itrUserTokenClientIds.hasNext()) { - Object[] nextClientDevice = itrUserTokenClientIds.next(); - if (nextClientDevice != null && nextClientDevice.length >= 2) { - clientDevicesMap.put((String)nextClientDevice[0], (String)nextClientDevice[1]); - } - } - } + Map clientDevicesMap = retrieveListOfClientDevicesFromUserTokens(maxResults, firstResultCounter); +// String userTokenQueryString = "SELECT DISTINCT(ut.clientId), ut.deviceId FROM UserToken ut ORDER BY ut.clientId"; +// Query userTokenQuery = s.createQuery(userTokenQueryString); +// userTokenQuery.setMaxResults(maxResults); +// userTokenQuery.setFirstResult(firstResultCounter); +// +// // Gather up all clientIds to remove, then delete at the end. +// List clientIdsToDelete = new LinkedList(); +// List userTokenClientIds = userTokenQuery.list(); +// Map clientDevicesMap = new HashMap(userTokenClientIds.size()); +// if (userTokenClientIds != null) { +// Iterator itrUserTokenClientIds = userTokenClientIds.iterator(); +// while (itrUserTokenClientIds.hasNext()) { +// Object[] nextClientDevice = itrUserTokenClientIds.next(); +// if (nextClientDevice != null && nextClientDevice.length >= 2) { +// clientDevicesMap.put((String)nextClientDevice[0], (String)nextClientDevice[1]); +// } +// } +// } +// while (clientDevicesMap != null && !clientDevicesMap.isEmpty()) { Set validClients = accessManager.validateClientsIncludeFromDeviceHistory(clientDevicesMap); @@ -1058,58 +1032,113 @@ private void cleanupUserTokens() { clientIdsToDelete.addAll(clientDevicesMap.keySet()); // If countToDelete is greater than Max User Token Cleanup Count, execute delete and start again. - if (clientIdsToDelete.size() > maxUserTokenCleanupCount) { - log.warn("Deleting " + clientIdsToDelete.size() + " client UserTokens"); - Transaction tx = s.beginTransaction(); - tx.begin(); - Query deleteQuery = s.createQuery(DELETE_USER_TOKENS_COLLECTION_SQL); - deleteQuery.setParameterList("clientIds", clientIdsToDelete); - deleteQuery.executeUpdate(); - tx.commit(); + if (!clientIdsToDelete.isEmpty()) { + log.warn("Cleaning up " + clientIdsToDelete.size() + " client UserTokens"); - clientIdsToDelete.clear(); - firstResultCounter = 0; - } - else { - firstResultCounter += maxResults; - } - - userTokenQuery.setFirstResult(firstResultCounter); - userTokenClientIds = userTokenQuery.list(); - if (userTokenClientIds != null) { - clientDevicesMap = new HashMap(userTokenClientIds.size()); - itrUserTokenClientIds = userTokenClientIds.iterator(); - if (userTokenClientIds != null) { - while (itrUserTokenClientIds.hasNext()) { - Object[] nextClientDevice = itrUserTokenClientIds.next(); - if (nextClientDevice != null && nextClientDevice.length >= 2) { - clientDevicesMap.put((String)nextClientDevice[0], (String)nextClientDevice[1]); - } + // Logout ALL of these Clients. + Iterator itrClientIdsToDelete = clientIdsToDelete.iterator(); + while (itrClientIdsToDelete.hasNext()) { + String clientId = itrClientIdsToDelete.next(); + + // Want to give each client a bit of time to login, so + // pend the client Id to remove the first time, then + // actually remove it the second time. + if (previousInvalidClientIds.contains(clientId)) { + accessManager.logoutClient(clientId, true); // Force the deletion, since this client is no longer valid. + previousInvalidClientIds.remove(clientId); + } + else { + previousInvalidClientIds.add(clientId); } } + + clientIdsToDelete.clear(); +// firstResultCounter = 0; } - else { - clientDevicesMap = null; - } +// else { + firstResultCounter += maxResults; +// } + + clientDevicesMap = retrieveListOfClientDevicesFromUserTokens(maxResults, firstResultCounter); +// userTokenQuery.setFirstResult(firstResultCounter); +// userTokenClientIds = userTokenQuery.list(); +// if (userTokenClientIds != null) { +// clientDevicesMap = new HashMap(userTokenClientIds.size()); +// Iterator itrUserTokenClientIds = userTokenClientIds.iterator(); +// if (userTokenClientIds != null) { +// while (itrUserTokenClientIds.hasNext()) { +// Object[] nextClientDevice = itrUserTokenClientIds.next(); +// if (nextClientDevice != null && nextClientDevice.length >= 2) { +// clientDevicesMap.put((String)nextClientDevice[0], (String)nextClientDevice[1]); +// } +// } +// } +// } +// else { +// clientDevicesMap = null; +// } } - if (clientIdsToDelete.size() > 0) { - log.warn("Deleting " + clientIdsToDelete.size() + " client UserTokens"); - log.debug( "Deleting Client IDs: " + StringUtils.arrayToCommaDelimitedString(clientIdsToDelete.toArray()) ); - Transaction tx = s.beginTransaction(); - tx.begin(); - Query deleteQuery = s.createQuery(DELETE_USER_TOKENS_COLLECTION_SQL); - deleteQuery.setParameterList("clientIds", clientIdsToDelete); - deleteQuery.executeUpdate(); - tx.commit(); +// if (clientIdsToDelete.size() > 0) { +// log.warn("Cleaning up " + clientIdsToDelete.size() + " client UserTokens"); +// +// // Logout ALL of these Clients. +// Iterator itrClientIdsToDelete = clientIdsToDelete.iterator(); +// while (itrClientIdsToDelete.hasNext()) { +// String clientId = itrClientIdsToDelete.next(); +// accessManager.logoutClient(clientId, true); // Force the deletion, since this client is no longer valid. +// } +//// log.warn("Deleting " + clientIdsToDelete.size() + " client UserTokens"); +//// log.debug( "Deleting Client IDs: " + StringUtils.arrayToCommaDelimitedString(clientIdsToDelete.toArray()) ); +//// Transaction tx = s.beginTransaction(); +//// tx.begin(); +//// Query deleteQuery = s.createQuery(DELETE_USER_TOKENS_COLLECTION_SQL); +//// deleteQuery.setParameterList("clientIds", clientIdsToDelete); +//// deleteQuery.executeUpdate(); +//// tx.commit(); +// +// clientIdsToDelete.clear(); +// } + } catch (Exception e) { + log.error("Unable to get cleanup UserTokens", e); +// } finally { +// if (s != null) +// s.close(); + } + } + + @SuppressWarnings("unchecked") + private Map retrieveListOfClientDevicesFromUserTokens(int maxResults, int firstResultCounter) { + Map result = null; - clientIdsToDelete.clear(); + Session s = null; + try { + s = sessionFactoryAuth.openSession(); + String userTokenQueryString = "SELECT DISTINCT(ut.clientId), ut.deviceId FROM UserToken ut ORDER BY ut.clientId"; + Query userTokenQuery = s.createQuery(userTokenQueryString); + userTokenQuery.setMaxResults(maxResults); + userTokenQuery.setFirstResult(firstResultCounter); + + // Gather up all clientIds to remove, then delete at the end. + List userTokenClientIds = userTokenQuery.list(); + result = new HashMap(userTokenClientIds.size()); + + if (userTokenClientIds != null) { + Iterator itrUserTokenClientIds = userTokenClientIds.iterator(); + while (itrUserTokenClientIds.hasNext()) { + Object[] nextClientDevice = itrUserTokenClientIds.next(); + if (nextClientDevice != null && nextClientDevice.length >= 2) { + result.put((String)nextClientDevice[0], (String)nextClientDevice[1]); + } + } } } catch (Exception e) { - log.error("Unable to get cleanup UserTokens", e); + log.error("Unable to retrieve list of Client Devices from UserTokens", e); } finally { if (s != null) s.close(); } + + return result; } } diff --git a/src/main/java/com/percero/agents/sync/access/RedisAccessManager.java b/src/main/java/com/percero/agents/sync/access/RedisAccessManager.java index 6dee518..8757792 100644 --- a/src/main/java/com/percero/agents/sync/access/RedisAccessManager.java +++ b/src/main/java/com/percero/agents/sync/access/RedisAccessManager.java @@ -178,21 +178,21 @@ public Boolean validateClientByClientId(String clientId, Boolean setClientTimeou return false; } - private static final Set CLIENT_KEYS_SEY = new HashSet(2); + private static final Set CLIENT_KEYS_SET = new HashSet(2); static { - CLIENT_KEYS_SEY.add(RedisKeyUtils.clientsPersistent()); - CLIENT_KEYS_SEY.add(RedisKeyUtils.clientsNonPersistent()); + CLIENT_KEYS_SET.add(RedisKeyUtils.clientsPersistent()); + CLIENT_KEYS_SET.add(RedisKeyUtils.clientsNonPersistent()); } @SuppressWarnings("unchecked") public Set validateClients(Collection clientIds) throws Exception { - Set validClients = (Set) cacheDataStore.getSetsContainsMembers(CLIENT_KEYS_SEY, clientIds.toArray()); + Set validClients = (Set) cacheDataStore.getSetsContainsMembers(CLIENT_KEYS_SET, clientIds.toArray()); return validClients; } @SuppressWarnings("unchecked") public Set validateClientsIncludeFromDeviceHistory(Map clientDevices) throws Exception { - Set validClients = (Set) cacheDataStore.getSetsContainsMembers(CLIENT_KEYS_SEY, clientDevices.keySet().toArray()); + Set validClients = (Set) cacheDataStore.getSetsContainsMembers(CLIENT_KEYS_SET, clientDevices.keySet().toArray()); // Now check each device to see if it has a corresponding clientId. Iterator> itrClientDevices = clientDevices.entrySet().iterator(); @@ -609,7 +609,7 @@ private void deleteClientWatchers(String clientId) { @Scheduled(fixedRate=60000) // 60 Seconds // @Scheduled(fixedRate=300000) // 5 Minutes public void postAccessJournals() { - log.info("Posting " + postAccessJournals.size() + " Access Journal" + (postAccessJournals.size() == 1 ? "" : "s")); + log.debug("Posting " + postAccessJournals.size() + " Access Journal" + (postAccessJournals.size() == 1 ? "" : "s")); /**if (taskExecutor != null) { taskExecutor.execute(new RedisPostClientTask(postClientHelper, postClientIds)); } else {*/ @@ -668,7 +668,7 @@ else if (!findClientByClientId(nextClientId) || !StringUtils.hasText(getClientUs @Scheduled(fixedRate=300000) // 5 Minutes // @Scheduled(fixedRate=120000) // 2 Minutes public void postClients() { - log.info("Posting " + postClientIds.size() + " client" + (postClientIds.size() == 1 ? "" : "s")); + log.debug("Posting " + postClientIds.size() + " client" + (postClientIds.size() == 1 ? "" : "s")); /**if (taskExecutor != null) { taskExecutor.execute(new RedisPostClientTask(postClientHelper, postClientIds)); } else {*/ @@ -846,7 +846,7 @@ private Long upsertRedisAccessJournal(String userId, String clientId, String cla // Add to the class's AccessJournals set if(classId != null && !classId.isEmpty()) { // && !classId.equals("0")) { - log.info("Adding to class AccessJournals: "+classId); + log.debug("Adding to class AccessJournals: "+classId); String classAccessJournalKey = RedisKeyUtils.classAccessJournal(className); cacheDataStore.addSetValue(classAccessJournalKey, classId); } diff --git a/src/main/java/com/percero/agents/sync/access/RedisKeyUtils.java b/src/main/java/com/percero/agents/sync/access/RedisKeyUtils.java index 8dd3df0..71e20ac 100644 --- a/src/main/java/com/percero/agents/sync/access/RedisKeyUtils.java +++ b/src/main/java/com/percero/agents/sync/access/RedisKeyUtils.java @@ -38,6 +38,10 @@ public static String clientsNonPersistent() { return "c:np"; } + public static String eolClients() { + return "c:eol"; + } + public static String client(String clientId) { return (new StringBuilder(INT_64).append("c:").append(clientId)).toString(); } diff --git a/src/main/java/com/percero/agents/sync/jobs/UpdateTablePoller.java b/src/main/java/com/percero/agents/sync/jobs/UpdateTablePoller.java index ebf664c..008bd73 100644 --- a/src/main/java/com/percero/agents/sync/jobs/UpdateTablePoller.java +++ b/src/main/java/com/percero/agents/sync/jobs/UpdateTablePoller.java @@ -51,7 +51,7 @@ public class UpdateTablePoller { @PostConstruct public void init(){ // Get the reporter going - UpdateTableProcessReporter.getInstance(); +// UpdateTableProcessReporter.getInstance(); } /** diff --git a/src/main/java/com/percero/agents/sync/jobs/UpdateTableProcessReporter.java b/src/main/java/com/percero/agents/sync/jobs/UpdateTableProcessReporter.java index be491dd..a918b2e 100644 --- a/src/main/java/com/percero/agents/sync/jobs/UpdateTableProcessReporter.java +++ b/src/main/java/com/percero/agents/sync/jobs/UpdateTableProcessReporter.java @@ -1,125 +1,125 @@ -package com.percero.agents.sync.jobs; - -import org.apache.log4j.Logger; - -import java.util.HashMap; -import java.util.Map; - -/** - * Created by jonnysamps on 10/10/15. - */ -public class UpdateTableProcessReporter { - - private static Logger logger = Logger.getLogger(UpdateTableProcessReporter.class); - private static final int WINDOW_LENGTH = 10000; - private static final double WINDOW_LENGTH_SECONDS = roundToOnePlace(WINDOW_LENGTH/1000.0); - - private static UpdateTableProcessReporter instance; - - private UTPReportStat totalStat = new UTPReportStat(); - private Map stats = new HashMap<>(); - - private UpdateTableProcessReporter(){ - new Thread(this.windowTimer).start(); // Start the reporting thread - } - - public static UpdateTableProcessReporter getInstance(){ - if(instance == null){ - instance = new UpdateTableProcessReporter(); - } - return instance; - } - - public void submitCountAndTime(String key, int count, long time){ - synchronized (this) { - if(!stats.containsKey(key)) - stats.put(key, new UTPReportStat()); - - totalStat.totalCount += count; - totalStat.totalTime += time; - totalStat.windowCount += count; - totalStat.windowTime += time; - stats.get(key).totalCount += count; - stats.get(key).totalTime += time; - stats.get(key).windowCount += count; - stats.get(key).windowTime += time; - } - } - - private static double roundToOnePlace(double number){ - number *= 10; - int rounded = (int) number; - number = rounded / 10.0; - return number; - } - - private void resetwindow(){ - synchronized (this) { - totalStat.windowCount = 0; - totalStat.windowTime = 0; - for(UTPReportStat stat : stats.values()){ - stat.windowCount = 0; - stat.windowTime = 0; - } - } - } - - private void printStats(){ - synchronized (this){ - printStat("Totals", totalStat); - for(String key : stats.keySet()) - printStat(key, stats.get(key)); - } - } - - private static void printStat(String label, UTPReportStat stat){ - double totalTimeSeconds = roundToOnePlace(stat.totalTime / 1000.0); - double totalOpsPerSecond = roundToOnePlace(stat.totalCount / totalTimeSeconds); - double windowTimeSeconds = roundToOnePlace(stat.windowTime/1000.0); - double windowOpsPerSecond = roundToOnePlace(stat.windowCount/windowTimeSeconds); - logger.debug( - "UT: "+pad(20, label)+ - " : Last "+pad(3,WINDOW_LENGTH_SECONDS+"")+"s" + - "("+pad(5, stat.windowCount+"")+ - ", "+pad(6, windowOpsPerSecond+"")+"/s) " + - "Total("+pad(7, stat.totalCount+"")+", " + - pad(6, totalOpsPerSecond+"")+"/s)" - ); - } - - private static String pad(int space, String content){ - StringBuilder sb = new StringBuilder(); - for(int i = 0; i < space-content.length(); i++){ - sb.append(" "); - } - sb.append(content); - return sb.toString(); - } - - /** - * Window timer will sleep for the window length, wake up print some stats, - * reset the window counters and then sleep again. - */ - private Runnable windowTimer = new Runnable() { - @Override - public void run() { - while(true){ - try { - Thread.sleep(WINDOW_LENGTH); - instance.printStats(); - instance.resetwindow(); - }catch(InterruptedException e){ - logger.error(e.getMessage(), e); - break; - } - } - } - }; - - private class UTPReportStat{ - public int totalCount = 0; - public long totalTime = 0; - public int windowCount = 0; - public long windowTime = 0; - } -} +//package com.percero.agents.sync.jobs; +// +//import org.apache.log4j.Logger; +// +//import java.util.HashMap; +//import java.util.Map; +// +///** +// * Created by jonnysamps on 10/10/15. +// */ +//public class UpdateTableProcessReporter { +// +// private static Logger logger = Logger.getLogger(UpdateTableProcessReporter.class); +// private static final int WINDOW_LENGTH = 10000; +// private static final double WINDOW_LENGTH_SECONDS = roundToOnePlace(WINDOW_LENGTH/1000.0); +// +// private static UpdateTableProcessReporter instance; +// +// private UTPReportStat totalStat = new UTPReportStat(); +// private Map stats = new HashMap<>(); +// +// private UpdateTableProcessReporter(){ +// new Thread(this.windowTimer).start(); // Start the reporting thread +// } +// +// public static UpdateTableProcessReporter getInstance(){ +// if(instance == null){ +// instance = new UpdateTableProcessReporter(); +// } +// return instance; +// } +// +// public void submitCountAndTime(String key, int count, long time){ +// synchronized (this) { +// if(!stats.containsKey(key)) +// stats.put(key, new UTPReportStat()); +// +// totalStat.totalCount += count; +// totalStat.totalTime += time; +// totalStat.windowCount += count; +// totalStat.windowTime += time; +// stats.get(key).totalCount += count; +// stats.get(key).totalTime += time; +// stats.get(key).windowCount += count; +// stats.get(key).windowTime += time; +// } +// } +// +// private static double roundToOnePlace(double number){ +// number *= 10; +// int rounded = (int) number; +// number = rounded / 10.0; +// return number; +// } +// +// private void resetwindow(){ +// synchronized (this) { +// totalStat.windowCount = 0; +// totalStat.windowTime = 0; +// for(UTPReportStat stat : stats.values()){ +// stat.windowCount = 0; +// stat.windowTime = 0; +// } +// } +// } +// +// private void printStats(){ +// synchronized (this){ +// printStat("Totals", totalStat); +// for(String key : stats.keySet()) +// printStat(key, stats.get(key)); +// } +// } +// +// private static void printStat(String label, UTPReportStat stat){ +// double totalTimeSeconds = roundToOnePlace(stat.totalTime / 1000.0); +// double totalOpsPerSecond = roundToOnePlace(stat.totalCount / totalTimeSeconds); +// double windowTimeSeconds = roundToOnePlace(stat.windowTime/1000.0); +// double windowOpsPerSecond = roundToOnePlace(stat.windowCount/windowTimeSeconds); +// logger.debug( +// "UT: "+pad(20, label)+ +// " : Last "+pad(3,WINDOW_LENGTH_SECONDS+"")+"s" + +// "("+pad(5, stat.windowCount+"")+ +// ", "+pad(6, windowOpsPerSecond+"")+"/s) " + +// "Total("+pad(7, stat.totalCount+"")+", " + +// pad(6, totalOpsPerSecond+"")+"/s)" +// ); +// } +// +// private static String pad(int space, String content){ +// StringBuilder sb = new StringBuilder(); +// for(int i = 0; i < space-content.length(); i++){ +// sb.append(" "); +// } +// sb.append(content); +// return sb.toString(); +// } +// +// /** +// * Window timer will sleep for the window length, wake up print some stats, +// * reset the window counters and then sleep again. +// */ +// private Runnable windowTimer = new Runnable() { +// @Override +// public void run() { +// while(true){ +// try { +// Thread.sleep(WINDOW_LENGTH); +// instance.printStats(); +// instance.resetwindow(); +// }catch(InterruptedException e){ +// logger.error(e.getMessage(), e); +// break; +// } +// } +// } +// }; +// +// private class UTPReportStat{ +// public int totalCount = 0; +// public long totalTime = 0; +// public int windowCount = 0; +// public long windowTime = 0; +// } +//} diff --git a/src/main/java/com/percero/agents/sync/jobs/UpdateTableProcessor.java b/src/main/java/com/percero/agents/sync/jobs/UpdateTableProcessor.java index 7e656b4..55448db 100644 --- a/src/main/java/com/percero/agents/sync/jobs/UpdateTableProcessor.java +++ b/src/main/java/com/percero/agents/sync/jobs/UpdateTableProcessor.java @@ -129,8 +129,8 @@ public void run(){ deleteRows(successfulRows); Date endTime = new Date(); - UpdateTableProcessReporter.getInstance() - .submitCountAndTime(tableName, successfulRows.size(), endTime.getTime() - startTime.getTime()); +// UpdateTableProcessReporter.getInstance() +// .submitCountAndTime(tableName, successfulRows.size(), endTime.getTime() - startTime.getTime()); } for(String className : classNamesToUpdateReferences) { diff --git a/src/main/java/com/percero/amqp/RabbitMQPushSyncHelper.java b/src/main/java/com/percero/amqp/RabbitMQPushSyncHelper.java index 5dcc070..61c3268 100644 --- a/src/main/java/com/percero/amqp/RabbitMQPushSyncHelper.java +++ b/src/main/java/com/percero/amqp/RabbitMQPushSyncHelper.java @@ -49,6 +49,8 @@ import com.google.gson.JsonObject; import com.google.gson.JsonParser; import com.percero.agents.sync.access.IAccessManager; +import com.percero.agents.sync.access.RedisKeyUtils; +import com.percero.agents.sync.datastore.ICacheDataStore; import com.percero.agents.sync.services.IPushSyncHelper; import com.percero.agents.sync.vo.BaseDataObject; import com.percero.agents.sync.vo.IJsonObject; @@ -100,6 +102,12 @@ public class RabbitMQPushSyncHelper implements IPushSyncHelper, ApplicationConte @Autowired @Value("$pf{gateway.rabbitmq.queue_timeout:43200000}") // 8 Hours long rabbitQueueTimeout = 43200000; + @Autowired + ICacheDataStore cacheDataStore; + public void setCacheDataStore(ICacheDataStore cacheDataStore) { + this.cacheDataStore = cacheDataStore; + } + @SuppressWarnings("rawtypes") protected void pushJsonToRouting(String objectJson, Class objectClass, String routingKey) { try{ @@ -228,7 +236,7 @@ public Boolean removeClient(String clientId) { // Send EOL message down the pipe. pushJsonToRouting("{\"EOL\":true, \"clientId\":\"" + clientId + "\"}", String.class, clientId); - eolClients.add(clientId); + cacheDataStore.addSetValue(RedisKeyUtils.eolClients(), clientId); } catch(AmqpIOException e) { // Most likely due to queue already being deleted. if (e.getCause() instanceof IOException && e.getCause().getCause() instanceof ShutdownSignalException) { @@ -254,7 +262,6 @@ public Boolean removeClient(String clientId) { protected Boolean deleteQueue(String queue) { try { logger.debug("RabbitMQ Deleting Queue " + queue); - eolClients.remove(queue); Queue clientQueue = new Queue(queue, durableQueues); amqpAdmin.declareQueue(clientQueue); amqpAdmin.deleteQueue(queue); @@ -263,13 +270,13 @@ protected Boolean deleteQueue(String queue) { logger.debug("Unable to clear out AMQP queue: " + queue + " (most likely because it no longer exists)", e); return false; } + + // Remove queue name from list of EOL Clients. + cacheDataStore.removeSetValue(RedisKeyUtils.eolClients(), queue); return true; } - - @SuppressWarnings("unchecked") - private Set eolClients = Collections.synchronizedSet(new HashSet()); - + @Override public Boolean renameClient(String thePreviousClientId, String clientId) { if (!StringUtils.hasText(thePreviousClientId)) { @@ -340,7 +347,7 @@ public void validateQueues() { String host = rabbitHost; if (!StringUtils.hasText(host)) { // No Rabbit host configured? Very strange, but no sense in moving forward here... - logger.debug("No RabbitMQ host configured?"); + logger.error("No RabbitMQ host configured?"); return; } @@ -390,7 +397,7 @@ public void validateQueues() { continue; } - if (eolClients.contains(nextQueueName)) { + if (cacheDataStore.getSetIsMember(RedisKeyUtils.eolClients(), nextQueueName)) { JsonElement nextJsonQueueMessages = nextJsonQueueObject.get("messages"); int nextQueueMessages = 0; if (nextJsonQueueMessages != null) { @@ -410,7 +417,7 @@ public void validateQueues() { int numConsumers = nextJsonQueueConsumers.getAsInt(); if (numConsumers == 0) { // If this queue is in the EOL list, then it can simply be deleted. - if (eolClients.contains(nextQueueName)) { + if (cacheDataStore.getSetIsMember(RedisKeyUtils.eolClients(), nextQueueName)) { logger.debug("Deleting EOL no consumers queue " + nextQueueName); deleteQueue(nextQueueName); continue; From 253081620e91b2fa9e7119c1076f50bcd630ef64 Mon Sep 17 00:00:00 2001 From: Collin Brown Date: Fri, 16 Oct 2015 23:53:32 -0700 Subject: [PATCH 035/105] Version bump --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 06eb736..aca307a 100644 --- a/pom.xml +++ b/pom.xml @@ -7,7 +7,7 @@ org.activestack syncengine - 1.1.25-SNAPSHOT + 1.1.26-SNAPSHOT 3.2.4.RELEASE From 23deb7be7c3dbb10170adfcc6f542d4204be62ec Mon Sep 17 00:00:00 2001 From: Collin Brown Date: Sun, 18 Oct 2015 00:09:48 -0700 Subject: [PATCH 036/105] Cleanup how RabbitMQ clients are removed. Added base updateTable-spring-config.xml --- pom.xml | 2 +- .../agents/auth/services/AuthService.java | 1 - .../percero/amqp/RabbitMQPushSyncHelper.java | 25 +- .../spring/updateTable-spring-config.xml | 290 ++++++++++++++++++ 4 files changed, 306 insertions(+), 12 deletions(-) create mode 100644 src/main/resources/spring/updateTable-spring-config.xml diff --git a/pom.xml b/pom.xml index aca307a..6ef0bfb 100644 --- a/pom.xml +++ b/pom.xml @@ -7,7 +7,7 @@ org.activestack syncengine - 1.1.26-SNAPSHOT + 1.1.27-SNAPSHOT 3.2.4.RELEASE diff --git a/src/main/java/com/percero/agents/auth/services/AuthService.java b/src/main/java/com/percero/agents/auth/services/AuthService.java index effce71..e2b5735 100644 --- a/src/main/java/com/percero/agents/auth/services/AuthService.java +++ b/src/main/java/com/percero/agents/auth/services/AuthService.java @@ -735,7 +735,6 @@ public boolean validateUserByToken(String regAppKey, String aUserId, String aTok Long uniqueResultCount = (Long) query.uniqueResult(); if (uniqueResultCount != null && uniqueResultCount > 0) { -// result = false; result = true; } else diff --git a/src/main/java/com/percero/amqp/RabbitMQPushSyncHelper.java b/src/main/java/com/percero/amqp/RabbitMQPushSyncHelper.java index 61c3268..b14dc91 100644 --- a/src/main/java/com/percero/amqp/RabbitMQPushSyncHelper.java +++ b/src/main/java/com/percero/amqp/RabbitMQPushSyncHelper.java @@ -59,7 +59,6 @@ import com.rabbitmq.client.ShutdownSignalException; import edu.emory.mathcs.backport.java.util.Arrays; -import edu.emory.mathcs.backport.java.util.Collections; @Component public class RabbitMQPushSyncHelper implements IPushSyncHelper, ApplicationContextAware { @@ -229,14 +228,18 @@ public void pushStringToRoute(String aString, String routeName) { @Override public Boolean removeClient(String clientId) { try { - logger.debug("RabbitMQ Removing Client " + clientId); - Queue clientQueue = new Queue(clientId, durableQueues); - amqpAdmin.declareQueue(clientQueue); - amqpAdmin.purgeQueue(clientId, true); - - // Send EOL message down the pipe. - pushJsonToRouting("{\"EOL\":true, \"clientId\":\"" + clientId + "\"}", String.class, clientId); - cacheDataStore.addSetValue(RedisKeyUtils.eolClients(), clientId); + if (!cacheDataStore.getSetIsMember(RedisKeyUtils.eolClients(), clientId)) { + logger.debug("RabbitMQ Removing Client " + clientId); + Queue clientQueue = new Queue(clientId, durableQueues); + amqpAdmin.declareQueue(clientQueue); + + // Remove ALL the messages from the queue, since this client is dead and gone. + amqpAdmin.purgeQueue(clientId, true); + + // If this client hasn't already received an EOL message, send it now. + pushJsonToRouting("{\"EOL\":true, \"clientId\":\"" + clientId + "\"}", String.class, clientId); + cacheDataStore.addSetValue(RedisKeyUtils.eolClients(), clientId); + } } catch(AmqpIOException e) { // Most likely due to queue already being deleted. if (e.getCause() instanceof IOException && e.getCause().getCause() instanceof ShutdownSignalException) { @@ -245,6 +248,8 @@ public Boolean removeClient(String clientId) { msg = sse.getMessage(); if (msg.contains("reply-text=NOT_FOUND")) { + // This would indicate that the queue no longer exists, so we can also remove it from the cache for final termination + cacheDataStore.removeSetValue(RedisKeyUtils.eolClients(), clientId); return true; } } @@ -276,7 +281,7 @@ protected Boolean deleteQueue(String queue) { return true; } - + @Override public Boolean renameClient(String thePreviousClientId, String clientId) { if (!StringUtils.hasText(thePreviousClientId)) { diff --git a/src/main/resources/spring/updateTable-spring-config.xml b/src/main/resources/spring/updateTable-spring-config.xml new file mode 100644 index 0000000..670a111 --- /dev/null +++ b/src/main/resources/spring/updateTable-spring-config.xml @@ -0,0 +1,290 @@ + + + + + + + + + + + + classpath*:*.properties + classpath*:properties/*.properties + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + hibernate.dialect=org.hibernate.dialect.MySQLDialect + + + hibernate.hbm2ddl.auto=update + + hibernate.show_sql=false + hibernate.format_sql=false + + hibernate.connection.aggressive_release=true + hibernate.jdbc.batch_size=20 + hibernate.connection.autocommit=false + hibernate.enable_lazy_load_no_trans=true + + + + + com.percero.amqp + com.percero.agents.auth.vo + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + hibernate.dialect=org.hibernate.dialect.Oracle10gDialect + hibernate.hbm2ddl.auto=update + + hibernate.show_sql=false + hibernate.format_sql=false + + hibernate.connection.aggressive_release=true + hibernate.jdbc.batch_size=20 + hibernate.connection.autocommit=false + hibernate.connection.autoReconnect=true + hibernate.enable_lazy_load_no_trans=true + + + + + $pf{domain.packageToScan} + + + + + + + + + From 6a4417c4efcd7ff7b664a2482671362b74df736c Mon Sep 17 00:00:00 2001 From: Collin Brown Date: Mon, 19 Oct 2015 02:45:38 -0700 Subject: [PATCH 037/105] If exception retrieving object on create, just ignore --- pom.xml | 2 +- .../agents/sync/services/DAODataProvider.java | 18 +++++++++++------- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/pom.xml b/pom.xml index 6ef0bfb..f686d4e 100644 --- a/pom.xml +++ b/pom.xml @@ -7,7 +7,7 @@ org.activestack syncengine - 1.1.27-SNAPSHOT + 1.1.28-SNAPSHOT 3.2.4.RELEASE diff --git a/src/main/java/com/percero/agents/sync/services/DAODataProvider.java b/src/main/java/com/percero/agents/sync/services/DAODataProvider.java index 25b6685..494812f 100644 --- a/src/main/java/com/percero/agents/sync/services/DAODataProvider.java +++ b/src/main/java/com/percero/agents/sync/services/DAODataProvider.java @@ -531,13 +531,17 @@ public T createObject(T perceroObject, String userId) perceroObject.setID(UUID.randomUUID().toString()); else { // Check to see if item already exists. - IPerceroObject existingObject = dao.retrieveObject(BaseDataObject.toClassIdPair(perceroObject), null, false); - if (existingObject != null) - { - populateToManyRelationships(perceroObject, true, null); - populateToOneRelationships(perceroObject, true, null); - return (T) cleanObject(perceroObject, userId); - } + try { + IPerceroObject existingObject = dao.retrieveObject(BaseDataObject.toClassIdPair(perceroObject), null, false); + if (existingObject != null) + { + populateToManyRelationships(perceroObject, true, null); + populateToOneRelationships(perceroObject, true, null); + return (T) cleanObject(perceroObject, userId); + } + } catch( Exception e) { + log.debug("Error retrieving object on create", e); + } } perceroObject = (T) dao.createObject(perceroObject, userId); From 91b6a2e4412cee55c7369e66ff99c0e35a760f8e Mon Sep 17 00:00:00 2001 From: Collin Brown Date: Mon, 19 Oct 2015 21:17:40 -0700 Subject: [PATCH 038/105] Add postGetHelper hook for ChangeWatchers --- pom.xml | 2 +- .../agents/sync/helpers/PostGetHelper.java | 103 +++++++++++++++++- .../sync/services/SyncAgentService.java | 24 ++++ 3 files changed, 127 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index f686d4e..73c96db 100644 --- a/pom.xml +++ b/pom.xml @@ -7,7 +7,7 @@ org.activestack syncengine - 1.1.28-SNAPSHOT + 1.1.29-SNAPSHOT 3.2.4.RELEASE diff --git a/src/main/java/com/percero/agents/sync/helpers/PostGetHelper.java b/src/main/java/com/percero/agents/sync/helpers/PostGetHelper.java index d6c1a60..e264576 100644 --- a/src/main/java/com/percero/agents/sync/helpers/PostGetHelper.java +++ b/src/main/java/com/percero/agents/sync/helpers/PostGetHelper.java @@ -1,30 +1,51 @@ package com.percero.agents.sync.helpers; +import java.net.URLDecoder; import java.util.ArrayList; +import java.util.Collection; import java.util.Iterator; import java.util.List; +import java.util.Set; +import org.apache.log4j.Logger; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; import com.percero.agents.sync.access.IAccessManager; +import com.percero.agents.sync.access.RedisAccessManager; +import com.percero.agents.sync.access.RedisKeyUtils; +import com.percero.agents.sync.cw.ChangeWatcherReporting; +import com.percero.agents.sync.cw.IChangeWatcherHelper; +import com.percero.agents.sync.cw.IChangeWatcherHelperFactory; import com.percero.agents.sync.vo.ClassIDPair; import com.percero.framework.vo.IPerceroObject; @Component public class PostGetHelper { - //private static final Logger log = Logger.getLogger(PostGetHelper.class); + private static Logger log = Logger.getLogger(PostGetHelper.class); + + + public static final String CATEGORY = "POST_GET"; @Autowired IAccessManager accessManager; public void setAccessManager(IAccessManager value) { accessManager = value; } + + @Autowired + IChangeWatcherHelperFactory changeWatcherHelperFactory; + public void setChangeWatcherHelperFactory(IChangeWatcherHelperFactory value) { + this.changeWatcherHelperFactory = value; + } + public void postGetObject(IPerceroObject perceroObject, String userId, String clientId) throws Exception { ClassIDPair pair = new ClassIDPair(perceroObject.getID(), perceroObject.getClass().getCanonicalName()); accessManager.saveAccessJournal(pair, userId, clientId); + + postGet(pair); } public void postGetObject(List perceroObjects, String userId, String clientId) throws Exception { @@ -36,4 +57,84 @@ public void postGetObject(List perceroObjects, String userId, St } accessManager.saveAccessJournal(classIdPairs, userId, clientId); } + + protected void postGet(ClassIDPair classIdPair) { + IChangeWatcherHelper cwh = changeWatcherHelperFactory.getHelper(CATEGORY); + if (cwh != null) { + String changeWatcherId = "cw:cf:" + CATEGORY + ":" + classIdPair.getClassName() + ":" + classIdPair.getID(); + setupRecalculateChangeWatcher(changeWatcherId); + } + } + + protected void setupRecalculateChangeWatcher(String changeWatcherId) { + + ChangeWatcherReporting.internalRequestsCounter++; +// if (useChangeWatcherQueue && pushSyncHelper != null) { +// pushSyncHelper.pushStringToRoute( (new StringBuilder(changeWatcherId).append(":TS:").append(System.currentTimeMillis())).toString(), changeWatcherRouteName); +// } +// else { + recalculateChangeWatcher(changeWatcherId); +// } + } + + @SuppressWarnings("unchecked") + public void recalculateChangeWatcher(String changeWatcherId) { + try { + // Check to see if a timestamp has been included. + String[] changeWatcherTsArray = changeWatcherId.split(":TS:"); + Long requestTimestamp = System.currentTimeMillis(); + if (changeWatcherTsArray.length > 1) { + requestTimestamp = Long.valueOf(changeWatcherTsArray[1]); + changeWatcherId = changeWatcherTsArray[0]; + } + + // Need to break up + String[] params = changeWatcherId.split(":"); + + // 3rd var should be category. + String category = params[2]; + + // 4th var should be subCategory. + String subCategory = params[3]; + + // 5th var should be fieldName. + String fieldName = params[4]; + + String[] otherParams = null; + if (params.length > 5) { + otherParams = new String[params.length-5]; + for(int i=5; i < params.length; i++) { + otherParams[i-5] = URLDecoder.decode(params[i], "UTF-8"); + } + } + + if (changeWatcherHelperFactory != null) + { + IChangeWatcherHelper cwh = changeWatcherHelperFactory.getHelper(category); + cwh.reprocess(category, subCategory, fieldName, null, otherParams, requestTimestamp); + + /** + // If no clients interested in this value, then remove it from the cache. + if (clientIds == null || clientIds.isEmpty()) { + removeChangeWatcherResult(pair, fieldName, otherParams); + updateWatcherFields(pair, fieldName, fieldsToWatch, params); + + if (watchedFields != null) { + Iterator itrOldWatchedFields = watchedFields.iterator(); + + while (itrOldWatchedFields.hasNext()) { + String nextOldField = (String) itrOldWatchedFields.next(); + redisDataStore.removeSetValue(watcherField, nextOldField); + + // Also remove this ChangeWatcher from the FieldWatcher. + redisDataStore.removeSetValue(nextOldField, changeWatcherId); + } + } + }**/ + } + + } catch(Exception e) { + log.error("Error recalculating Change Watcher: " + changeWatcherId, e); + } + } } diff --git a/src/main/java/com/percero/agents/sync/services/SyncAgentService.java b/src/main/java/com/percero/agents/sync/services/SyncAgentService.java index 947b25b..ade655d 100644 --- a/src/main/java/com/percero/agents/sync/services/SyncAgentService.java +++ b/src/main/java/com/percero/agents/sync/services/SyncAgentService.java @@ -550,6 +550,14 @@ public IPerceroObject systemGetById(String aClassName, String anId) {//throws Ex IDataProvider dataProvider = dataProviderManager.getDataProviderByName(mappedClass.dataProviderName); IPerceroObject result = dataProvider.findById(new ClassIDPair(anId, aClassName), null); + if (result != null) { + try { + postGetHelper.postGetObject(result, null, null); + } catch (Exception e) { + log.error(e.getMessage(), e); + } + } + return result; } @@ -561,6 +569,14 @@ public IPerceroObject systemGetById(ClassIDPair cip) { //throws Exception { IDataProvider dataProvider = dataProviderManager.getDataProviderByName(mappedClass.dataProviderName); IPerceroObject result = dataProvider.findById(cip, null); + if (result != null) { + try { + postGetHelper.postGetObject(result, null, null); + } catch (Exception e) { + log.error(e.getMessage(), e); + } + } + return result; } @@ -576,6 +592,14 @@ public T systemGetByObject(T perceroObject) {//throws IDataProvider dataProvider = dataProviderManager.getDataProviderByName(mappedClass.dataProviderName); T result = (T) dataProvider.findById(new ClassIDPair(perceroObject.getID(), className), null); + + if (result != null) { + try { + postGetHelper.postGetObject(result, null, null); + } catch (Exception e) { + log.error(e.getMessage(), e); + } + } return result; } From eb3b4e909cb8edfc9f9be2907f7e47ec69512465 Mon Sep 17 00:00:00 2001 From: Collin Brown Date: Mon, 19 Oct 2015 21:36:43 -0700 Subject: [PATCH 039/105] Fix infinite loop in new postget hook --- pom.xml | 2 +- .../sync/services/SyncAgentService.java | 48 +++++++++---------- 2 files changed, 25 insertions(+), 25 deletions(-) diff --git a/pom.xml b/pom.xml index 73c96db..08f9cfa 100644 --- a/pom.xml +++ b/pom.xml @@ -7,7 +7,7 @@ org.activestack syncengine - 1.1.29-SNAPSHOT + 1.1.30-SNAPSHOT 3.2.4.RELEASE diff --git a/src/main/java/com/percero/agents/sync/services/SyncAgentService.java b/src/main/java/com/percero/agents/sync/services/SyncAgentService.java index ade655d..b3d3961 100644 --- a/src/main/java/com/percero/agents/sync/services/SyncAgentService.java +++ b/src/main/java/com/percero/agents/sync/services/SyncAgentService.java @@ -549,14 +549,14 @@ public IPerceroObject systemGetById(String aClassName, String anId) {//throws Ex IDataProvider dataProvider = dataProviderManager.getDataProviderByName(mappedClass.dataProviderName); IPerceroObject result = dataProvider.findById(new ClassIDPair(anId, aClassName), null); - - if (result != null) { - try { - postGetHelper.postGetObject(result, null, null); - } catch (Exception e) { - log.error(e.getMessage(), e); - } - } +// +// if (result != null) { +// try { +// postGetHelper.postGetObject(result, null, null); +// } catch (Exception e) { +// log.error(e.getMessage(), e); +// } +// } return result; } @@ -568,14 +568,14 @@ public IPerceroObject systemGetById(ClassIDPair cip) { //throws Exception { IDataProvider dataProvider = dataProviderManager.getDataProviderByName(mappedClass.dataProviderName); IPerceroObject result = dataProvider.findById(cip, null); - - if (result != null) { - try { - postGetHelper.postGetObject(result, null, null); - } catch (Exception e) { - log.error(e.getMessage(), e); - } - } +// +// if (result != null) { +// try { +// postGetHelper.postGetObject(result, null, null); +// } catch (Exception e) { +// log.error(e.getMessage(), e); +// } +// } return result; } @@ -592,14 +592,14 @@ public T systemGetByObject(T perceroObject) {//throws IDataProvider dataProvider = dataProviderManager.getDataProviderByName(mappedClass.dataProviderName); T result = (T) dataProvider.findById(new ClassIDPair(perceroObject.getID(), className), null); - - if (result != null) { - try { - postGetHelper.postGetObject(result, null, null); - } catch (Exception e) { - log.error(e.getMessage(), e); - } - } +// +// if (result != null) { +// try { +// postGetHelper.postGetObject(result, null, null); +// } catch (Exception e) { +// log.error(e.getMessage(), e); +// } +// } return result; } From 0dd65e8781294653c816ed20cd65eed28c0846e7 Mon Sep 17 00:00:00 2001 From: Jonathan Samples Date: Mon, 19 Oct 2015 23:20:41 -0700 Subject: [PATCH 040/105] Added some authentication messages and codes to the response --- pom.xml | 2 +- .../auth/services/AnonAuthProvider.java | 13 +++++---- .../agents/auth/services/AuthService2.java | 8 +++-- .../auth/services/GoogleAuthProvider.java | 13 +++++++-- .../agents/auth/services/IAuthProvider.java | 3 +- .../auth/services/InMemoryAuthProvider.java | 29 ++++++++++--------- .../com/percero/agents/auth/vo/AuthCode.java | 20 +++++++++++++ .../agents/auth/vo/AuthProviderResponse.java | 9 ++++++ .../percero/agents/auth/vo/AuthResponse.java | 16 ++++++++++ 9 files changed, 86 insertions(+), 27 deletions(-) create mode 100644 src/main/java/com/percero/agents/auth/vo/AuthCode.java create mode 100644 src/main/java/com/percero/agents/auth/vo/AuthProviderResponse.java diff --git a/pom.xml b/pom.xml index 73c96db..08f9cfa 100644 --- a/pom.xml +++ b/pom.xml @@ -7,7 +7,7 @@ org.activestack syncengine - 1.1.29-SNAPSHOT + 1.1.30-SNAPSHOT 3.2.4.RELEASE diff --git a/src/main/java/com/percero/agents/auth/services/AnonAuthProvider.java b/src/main/java/com/percero/agents/auth/services/AnonAuthProvider.java index 629131c..6346ad4 100644 --- a/src/main/java/com/percero/agents/auth/services/AnonAuthProvider.java +++ b/src/main/java/com/percero/agents/auth/services/AnonAuthProvider.java @@ -1,9 +1,6 @@ package com.percero.agents.auth.services; -import com.percero.agents.auth.vo.AuthProvider; -import com.percero.agents.auth.vo.ServiceIdentifier; -import com.percero.agents.auth.vo.ServiceOrganization; -import com.percero.agents.auth.vo.ServiceUser; +import com.percero.agents.auth.vo.*; import com.percero.util.RandomStringGenerator; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; @@ -31,7 +28,10 @@ public String getID() { return ID; } - public ServiceUser authenticate(String credential) { + public AuthProviderResponse authenticate(String credential) { + AuthProviderResponse result = new AuthProviderResponse(); + result.authCode = AuthCode.SUCCESS; + ServiceUser serviceUser = new ServiceUser(); serviceUser.setFirstName("ANON"); serviceUser.setLastName("ANON"); @@ -51,6 +51,7 @@ public ServiceUser authenticate(String credential) { serviceUser.getEmails().add(email); serviceUser.getIdentifiers().add(new ServiceIdentifier("email", email)); - return serviceUser; + result.serviceUser = serviceUser; + return result; } } diff --git a/src/main/java/com/percero/agents/auth/services/AuthService2.java b/src/main/java/com/percero/agents/auth/services/AuthService2.java index 940da7c..e5bdb34 100644 --- a/src/main/java/com/percero/agents/auth/services/AuthService2.java +++ b/src/main/java/com/percero/agents/auth/services/AuthService2.java @@ -51,7 +51,10 @@ public AuthenticationResponse authenticate(AuthenticationRequest request) throws IAuthProvider provider = authProviderRegistry.getProvider(request.getAuthProvider()); - ServiceUser serviceUser = provider.authenticate(request.getCredential()); + AuthProviderResponse apResponse = provider.authenticate(request.getCredential()); + ServiceUser serviceUser = apResponse.serviceUser; + response.setStatusCode(apResponse.authCode.getCode()); + response.setMessage(apResponse.authCode.getMessage()); // Login successful if(serviceUser != null) { @@ -64,7 +67,8 @@ public AuthenticationResponse authenticate(AuthenticationRequest request) throws response.setResult(userToken); } else { - logger.warn("LOGIN FAILED (" + provider.getID() + "): Unable to retrieve valid Service User"); + logger.warn("LOGIN FAILED: (" + provider.getID() + "): Unable to retrieve valid Service User"); + logger.warn(" ERROR: ("+response.getStatusCode()+") "+response.getMessage()); } return response; diff --git a/src/main/java/com/percero/agents/auth/services/GoogleAuthProvider.java b/src/main/java/com/percero/agents/auth/services/GoogleAuthProvider.java index 58e21db..b24b45e 100644 --- a/src/main/java/com/percero/agents/auth/services/GoogleAuthProvider.java +++ b/src/main/java/com/percero/agents/auth/services/GoogleAuthProvider.java @@ -177,7 +177,8 @@ public void init(){ } } - public ServiceUser authenticate(String credential) { + public AuthProviderResponse authenticate(String credential) { + AuthProviderResponse result = new AuthProviderResponse(); try { OAuthCredential decodedCredential = om.readValue(credential, OAuthCredential.class); @@ -196,12 +197,18 @@ public ServiceUser authenticate(String credential) { googleCredential.setFromTokenResponse(authResponse); ServiceUser serviceUser = getServiceUser(googleCredential); - - return serviceUser; + if(serviceUser == null) + result.authCode = AuthCode.FORBIDDEN; + else{ + result.authCode = AuthCode.SUCCESS; + result.serviceUser = serviceUser; + } } catch(Exception e) { log.error("Unable to get authenticate oauth code", e); return null; } + + return result; } private ServiceUser getServiceUser(Credential credential) { diff --git a/src/main/java/com/percero/agents/auth/services/IAuthProvider.java b/src/main/java/com/percero/agents/auth/services/IAuthProvider.java index 5b32aad..ebf1b22 100644 --- a/src/main/java/com/percero/agents/auth/services/IAuthProvider.java +++ b/src/main/java/com/percero/agents/auth/services/IAuthProvider.java @@ -1,5 +1,6 @@ package com.percero.agents.auth.services; +import com.percero.agents.auth.vo.AuthProviderResponse; import com.percero.agents.auth.vo.ServiceUser; /** @@ -17,6 +18,6 @@ public interface IAuthProvider { * @param credential - A String to be interpreted by the provider as an authentication credential * @return ServiceUser */ - ServiceUser authenticate(String credential); + AuthProviderResponse authenticate(String credential); } diff --git a/src/main/java/com/percero/agents/auth/services/InMemoryAuthProvider.java b/src/main/java/com/percero/agents/auth/services/InMemoryAuthProvider.java index 1e0ed34..743fe68 100644 --- a/src/main/java/com/percero/agents/auth/services/InMemoryAuthProvider.java +++ b/src/main/java/com/percero/agents/auth/services/InMemoryAuthProvider.java @@ -1,9 +1,6 @@ package com.percero.agents.auth.services; -import com.percero.agents.auth.vo.BasicAuthCredential; -import com.percero.agents.auth.vo.InMemoryAuthProviderUser; -import com.percero.agents.auth.vo.ServiceIdentifier; -import com.percero.agents.auth.vo.ServiceUser; +import com.percero.agents.auth.vo.*; import org.apache.commons.codec.digest.DigestUtils; import java.util.HashMap; @@ -21,23 +18,27 @@ public String getID() { return ID; } - public ServiceUser authenticate(String credential) { - ServiceUser result = null; + public AuthProviderResponse authenticate(String credential) { + AuthProviderResponse result = new AuthProviderResponse(); BasicAuthCredential cred = BasicAuthCredential.fromString(credential); String hashPass = DigestUtils.sha1Hex(cred.getPassword()); InMemoryAuthProviderUser user = users.get(cred.getUsername()); if(user != null && user.getPassHash().equals(hashPass)) { - result = new ServiceUser(); - result.setAuthProviderID(getID()); - result.setId(cred.getUsername()); - result.setFirstName(user.getFirstName()); - result.setLastName(user.getLastName()); - result.getEmails().add(user.getEmail()); - result.setAreRoleNamesAccurate(true); - result.getIdentifiers().add(new ServiceIdentifier("email", user.getEmail())); + ServiceUser serviceUser = new ServiceUser(); + serviceUser.setAuthProviderID(getID()); + serviceUser.setId(cred.getUsername()); + serviceUser.setFirstName(user.getFirstName()); + serviceUser.setLastName(user.getLastName()); + serviceUser.getEmails().add(user.getEmail()); + serviceUser.setAreRoleNamesAccurate(true); + serviceUser.getIdentifiers().add(new ServiceIdentifier("email", user.getEmail())); + result.serviceUser = serviceUser; + result.authCode = AuthCode.SUCCESS; } + else + result.authCode = AuthCode.FORBIDDEN; return result; } diff --git a/src/main/java/com/percero/agents/auth/vo/AuthCode.java b/src/main/java/com/percero/agents/auth/vo/AuthCode.java new file mode 100644 index 0000000..8ba3cc1 --- /dev/null +++ b/src/main/java/com/percero/agents/auth/vo/AuthCode.java @@ -0,0 +1,20 @@ +package com.percero.agents.auth.vo; + +/** + * Created by jonnysamps on 10/19/15. + */ +public class AuthCode { + public static final AuthCode SUCCESS = new AuthCode(200, "Success"); + public static final AuthCode UNAUTHORIZED = new AuthCode(401, "Unauthorized"); + public static final AuthCode FORBIDDEN = new AuthCode(401, "Forbidden"); + + private int code; + private String message; + public AuthCode(int code, String message){ + this.code = code; + this.message = message; + } + + public int getCode(){ return this.code; } + public String getMessage(){ return this.message; } +} diff --git a/src/main/java/com/percero/agents/auth/vo/AuthProviderResponse.java b/src/main/java/com/percero/agents/auth/vo/AuthProviderResponse.java new file mode 100644 index 0000000..068e5ba --- /dev/null +++ b/src/main/java/com/percero/agents/auth/vo/AuthProviderResponse.java @@ -0,0 +1,9 @@ +package com.percero.agents.auth.vo; + +/** + * Created by jonnysamps on 10/19/15. + */ +public class AuthProviderResponse { + public AuthCode authCode; + public ServiceUser serviceUser; +} diff --git a/src/main/java/com/percero/agents/auth/vo/AuthResponse.java b/src/main/java/com/percero/agents/auth/vo/AuthResponse.java index fabdcfb..8e08299 100644 --- a/src/main/java/com/percero/agents/auth/vo/AuthResponse.java +++ b/src/main/java/com/percero/agents/auth/vo/AuthResponse.java @@ -28,4 +28,20 @@ public String getCorrespondingMessageId() { public void setCorrespondingMessageId(String correspondingMessageId) { this.correspondingMessageId = correspondingMessageId; } + + private String message = "OK"; + public String getMessage(){ + return this.message; + } + public void setMessage(String message){ + this.message = message; + } + + private int statusCode = 200; + public int getStatusCode(){ + return this.statusCode; + } + public void setStatusCode(int statusCode){ + this.statusCode = statusCode; + } } From 3768712ee92022e9d33808b4fbd7b030f938443f Mon Sep 17 00:00:00 2001 From: Collin Brown Date: Tue, 20 Oct 2015 21:04:12 -0700 Subject: [PATCH 041/105] Fix reverse mapped field --- .../services/FileAuthProviderFactory.java | 2 +- .../sync/cw/ChangeWatcherHelperFactory.java | 5 +++- .../agents/sync/metadata/MappedClass.java | 30 +++++++++++-------- .../sync/services/SyncAgentService.java | 2 +- 4 files changed, 24 insertions(+), 15 deletions(-) diff --git a/src/main/java/com/percero/agents/auth/services/FileAuthProviderFactory.java b/src/main/java/com/percero/agents/auth/services/FileAuthProviderFactory.java index 53d8916..25593ae 100644 --- a/src/main/java/com/percero/agents/auth/services/FileAuthProviderFactory.java +++ b/src/main/java/com/percero/agents/auth/services/FileAuthProviderFactory.java @@ -48,7 +48,7 @@ public void init(){ logger.info("Using FileAuthProvider ("+providerID+"). Found "+list.size()+" users."); }catch(IOException e){ logger.info("Not using FileAuthProvider"); - logger.debug(e.getMessage(),e); +// logger.debug(e.getMessage(),e); } } } diff --git a/src/main/java/com/percero/agents/sync/cw/ChangeWatcherHelperFactory.java b/src/main/java/com/percero/agents/sync/cw/ChangeWatcherHelperFactory.java index 8783440..8d15e12 100644 --- a/src/main/java/com/percero/agents/sync/cw/ChangeWatcherHelperFactory.java +++ b/src/main/java/com/percero/agents/sync/cw/ChangeWatcherHelperFactory.java @@ -72,9 +72,12 @@ public IChangeWatcherHelper getHelper(String category) { clazz = clazz.getSuperclass(); } } catch(Exception e) { - log.error("Unable to get ChangeWatcherHelper for " + category, e); +// log.error("Unable to get ChangeWatcherHelper for " + category, e); } + // If no result, then set that up in the helperCache. + helperCache.put(category, null); + return null; } else { diff --git a/src/main/java/com/percero/agents/sync/metadata/MappedClass.java b/src/main/java/com/percero/agents/sync/metadata/MappedClass.java index c8c23c4..0ab45af 100644 --- a/src/main/java/com/percero/agents/sync/metadata/MappedClass.java +++ b/src/main/java/com/percero/agents/sync/metadata/MappedClass.java @@ -918,8 +918,9 @@ public void initializeRelationships() { if (refOneToMany != null) { - Boolean inheritsFrom = inheritsFrom(this.clazz, refOneToMany.targetEntity()); - if (inheritsFrom && nextMappedField.getField().getName().equals(refOneToMany.mappedBy())) { +// Boolean inheritsFrom = inheritsFrom(this.clazz, refOneToMany.targetEntity()); +// if (inheritsFrom && nextMappedField.getField().getName().equals(refOneToMany.mappedBy())) { + if (this.clazz == refOneToMany.targetEntity() && nextMappedField.getField().getName().equals(refOneToMany.mappedBy())) { // Found the referenced field. reverseMappedField = nextRefMappedField; break; @@ -932,8 +933,9 @@ else if (nextRefMappedField instanceof MappedFieldPerceroObject) { refOneToOne = nextRefMappedField.getGetter().getAnnotation(OneToOne.class); if (refOneToOne != null) { - Boolean inheritsFrom = inheritsFrom(this.clazz, refOneToOne.targetEntity()); - if (inheritsFrom && nextMappedField.getField().getName().equals(refOneToOne.mappedBy())) { +// Boolean inheritsFrom = inheritsFrom(this.clazz, refOneToOne.targetEntity()); +// if (inheritsFrom && nextMappedField.getField().getName().equals(refOneToOne.mappedBy())) { + if (this.clazz == refOneToOne.targetEntity() && nextMappedField.getField().getName().equals(refOneToOne.mappedBy())) { // Found the referenced field. reverseMappedField = nextRefMappedField; break; @@ -951,8 +953,9 @@ else if (nextRefMappedField instanceof MappedFieldPerceroObject) { refOneToOne = nextRefMappedField.getGetter().getAnnotation(OneToOne.class); if (refOneToOne != null) { - Boolean inheritsFrom = inheritsFrom(this.clazz, nextRefMappedField.getField().getType()); - if (inheritsFrom && nextMappedField.getField().getName().equals(refOneToOne.mappedBy())) { +// Boolean inheritsFrom = inheritsFrom(this.clazz, nextRefMappedField.getField().getType()); +// if (inheritsFrom && nextMappedField.getField().getName().equals(refOneToOne.mappedBy())) { + if (this.clazz == nextRefMappedField.getField().getType() && nextMappedField.getField().getName().equals(refOneToOne.mappedBy())) { // Found the referenced field. reverseMappedField = nextRefMappedField; break; @@ -999,8 +1002,9 @@ else if (nextRefMappedField instanceof MappedFieldPerceroObject) { refOneToMany = nextRefMappedField.getGetter().getAnnotation(OneToMany.class); if (refOneToMany != null) { - Boolean inheritsFrom = inheritsFrom(this.clazz, refOneToMany.targetEntity()); - if (inheritsFrom && nextMappedField.getField().getName().equals(refOneToMany.mappedBy())) { +// Boolean inheritsFrom = inheritsFrom(this.clazz, refOneToMany.targetEntity()); +// if (inheritsFrom && nextMappedField.getField().getName().equals(refOneToMany.mappedBy())) { + if (this.clazz == refOneToMany.targetEntity() && nextMappedField.getField().getName().equals(refOneToMany.mappedBy())) { // Found the referenced field. reverseMappedField = nextRefMappedField; break; @@ -1013,8 +1017,9 @@ else if (nextRefMappedField instanceof MappedFieldPerceroObject) { refOneToOne = nextRefMappedField.getGetter().getAnnotation(OneToOne.class); if (refOneToOne != null) { - Boolean inheritsFrom = inheritsFrom(this.clazz, refOneToOne.targetEntity()); - if (inheritsFrom && nextMappedField.getField().getName().equals(refOneToOne.mappedBy())) { +// Boolean inheritsFrom = inheritsFrom(this.clazz, refOneToOne.targetEntity()); +// if (inheritsFrom && nextMappedField.getField().getName().equals(refOneToOne.mappedBy())) { + if (this.clazz == refOneToOne.targetEntity() && nextMappedField.getField().getName().equals(refOneToOne.mappedBy())) { // Found the referenced field. reverseMappedField = nextRefMappedField; break; @@ -1032,8 +1037,9 @@ else if (nextRefMappedField instanceof MappedFieldPerceroObject) { refOneToOne = nextRefMappedField.getGetter().getAnnotation(OneToOne.class); if (refOneToOne != null) { - Boolean inheritsFrom = inheritsFrom(this.clazz, nextRefMappedField.getField().getType()); - if (inheritsFrom && nextMappedField.getField().getName().equals(refOneToOne.mappedBy())) { +// Boolean inheritsFrom = inheritsFrom(this.clazz, nextRefMappedField.getField().getType()); +// if (inheritsFrom && nextMappedField.getField().getName().equals(refOneToOne.mappedBy())) { + if (this.clazz == nextRefMappedField.getField().getType() && nextMappedField.getField().getName().equals(refOneToOne.mappedBy())) { // Found the referenced field. reverseMappedField = nextRefMappedField; break; diff --git a/src/main/java/com/percero/agents/sync/services/SyncAgentService.java b/src/main/java/com/percero/agents/sync/services/SyncAgentService.java index b3d3961..d1876e7 100644 --- a/src/main/java/com/percero/agents/sync/services/SyncAgentService.java +++ b/src/main/java/com/percero/agents/sync/services/SyncAgentService.java @@ -350,7 +350,7 @@ public Object runProcess(String processName, Object queryArguments, String clien Object result = null; Boolean isValidClient = accessManager.validateClientByClientId(clientId); - if (!isValidClient && false) + if (!isValidClient) throw new ClientException(ClientException.INVALID_CLIENT, ClientException.INVALID_CLIENT_CODE); if (!StringUtils.hasText(processName)) From b6dfb048d1028fced8c4cce8bc7145d6b11145d1 Mon Sep 17 00:00:00 2001 From: Collin Brown Date: Wed, 21 Oct 2015 19:11:22 -0700 Subject: [PATCH 042/105] Version bump --- pom.xml | 2 +- src/main/java/com/percero/util/DateUtils.java | 10 ++++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 08f9cfa..8f94d9e 100644 --- a/pom.xml +++ b/pom.xml @@ -7,7 +7,7 @@ org.activestack syncengine - 1.1.30-SNAPSHOT + 1.1.31-SNAPSHOT 3.2.4.RELEASE diff --git a/src/main/java/com/percero/util/DateUtils.java b/src/main/java/com/percero/util/DateUtils.java index 46b5bf3..18aa308 100644 --- a/src/main/java/com/percero/util/DateUtils.java +++ b/src/main/java/com/percero/util/DateUtils.java @@ -1,5 +1,8 @@ package com.percero.util; +import java.sql.Timestamp; +import java.util.Date; + public class DateUtils { public static java.sql.Date utilDateToSqlDate(java.util.Date utilDate) { @@ -10,5 +13,12 @@ public static java.sql.Date utilDateToSqlDate(java.util.Date utilDate) { return null; } } + + public static Date utilDateFromSqlTimestamp(Timestamp timestamp) { + if (timestamp != null) { + return new Date(timestamp.getTime()); + } + return null; + } } From 76fbf08fa779a4abf99579741d00a0835442f062 Mon Sep 17 00:00:00 2001 From: Collin Brown Date: Thu, 22 Oct 2015 01:43:38 -0700 Subject: [PATCH 043/105] Wrap get all related objects so that exception does not stop retrieval of rest of object. --- pom.xml | 2 +- .../com/percero/agents/sync/services/DAODataProvider.java | 8 ++++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/pom.xml b/pom.xml index 8f94d9e..365e767 100644 --- a/pom.xml +++ b/pom.xml @@ -7,7 +7,7 @@ org.activestack syncengine - 1.1.31-SNAPSHOT + 1.1.32-SNAPSHOT 3.2.4.RELEASE diff --git a/src/main/java/com/percero/agents/sync/services/DAODataProvider.java b/src/main/java/com/percero/agents/sync/services/DAODataProvider.java index 494812f..59e1bcc 100644 --- a/src/main/java/com/percero/agents/sync/services/DAODataProvider.java +++ b/src/main/java/com/percero/agents/sync/services/DAODataProvider.java @@ -1181,8 +1181,12 @@ public void populateToManyRelationships(IPerceroObject perceroObject, Boolean sh for(MappedField nextToManyMappedField : mappedClass.toManyFields) { // TODO: Take into account Access Rights. - List allRelatedObjects = findAllRelatedObjects(perceroObject, nextToManyMappedField, shellOnly, userId); - nextToManyMappedField.getSetter().invoke(perceroObject, allRelatedObjects); + try { + List allRelatedObjects = findAllRelatedObjects(perceroObject, nextToManyMappedField, shellOnly, userId); + nextToManyMappedField.getSetter().invoke(perceroObject, allRelatedObjects); + } catch(Exception e) { + log.error("Unable to retrieve related objects for " + perceroObject.getClass().getCanonicalName() + "::" + nextToManyMappedField.getField().getName()); + } } } From 4f303c0ac62d7dea7aeabfb927c3cd4b2df49ed4 Mon Sep 17 00:00:00 2001 From: Collin Brown Date: Sat, 24 Oct 2015 19:48:38 -0700 Subject: [PATCH 044/105] Temporary fix for postGet updating retrieved object If postGet updates the retrieved object, then the process would return the "old" object. Added code for postGet to optionally return an updated object that is instead passed back to the requestor. --- pom.xml | 2 +- .../agents/sync/cw/ChangeWatcherHelper.java | 13 +- .../cw/DerivedValueChangeWatcherHelper.java | 18 +- .../agents/sync/cw/IChangeWatcherHelper.java | 6 +- .../agents/sync/helpers/PostGetHelper.java | 23 +- .../agents/sync/metadata/MappedClass.java | 1453 ++++++++++------- .../agents/sync/metadata/MappedField.java | 2 +- .../agents/sync/metadata/MappedFieldList.java | 12 + .../metadata/MappedFieldPerceroObject.java | 24 + .../sync/services/SyncAgentService.java | 14 +- .../spring/changeWatcher-spring-config.xml | 13 +- 11 files changed, 993 insertions(+), 587 deletions(-) diff --git a/pom.xml b/pom.xml index 365e767..9efce3f 100644 --- a/pom.xml +++ b/pom.xml @@ -7,7 +7,7 @@ org.activestack syncengine - 1.1.32-SNAPSHOT + 1.1.33-SNAPSHOT 3.2.4.RELEASE diff --git a/src/main/java/com/percero/agents/sync/cw/ChangeWatcherHelper.java b/src/main/java/com/percero/agents/sync/cw/ChangeWatcherHelper.java index 75d8704..4605103 100644 --- a/src/main/java/com/percero/agents/sync/cw/ChangeWatcherHelper.java +++ b/src/main/java/com/percero/agents/sync/cw/ChangeWatcherHelper.java @@ -36,27 +36,30 @@ public void setAccessManager(IAccessManager value) { accessManager = value; } - public void process(String category, String subCategory, String fieldName) { + public Object process(String category, String subCategory, String fieldName) { // This is really an error. StringBuilder strBuilder = new StringBuilder("No value calculate method found for: ").append(category).append(":").append(subCategory).append(":").append(fieldName); log.error(strBuilder.toString()); + + return null; } - public void process(String category, String subCategory, String fieldName, String[] params) { + public Object process(String category, String subCategory, String fieldName, String[] params) { // This is really an error. StringBuilder strBuilder = new StringBuilder("No value process method found for: ").append(category).append(":").append(subCategory).append(":").append(fieldName); for(String nextString : params) { strBuilder.append(".").append(nextString); } log.error(strBuilder.toString()); + + return null; } - public void reprocess(String category, String subCategory, String fieldName, Collection clientIds, String[] params, Long requestTimestamp) { + public Object reprocess(String category, String subCategory, String fieldName, Collection clientIds, String[] params, Long requestTimestamp) { ChangeWatcherReporting.reprocessCounter++; - process(category, subCategory, fieldName, params); - log.debug(ChangeWatcherReporting.stringResults()); + return process(category, subCategory, fieldName, params); } diff --git a/src/main/java/com/percero/agents/sync/cw/DerivedValueChangeWatcherHelper.java b/src/main/java/com/percero/agents/sync/cw/DerivedValueChangeWatcherHelper.java index 8f28c61..5cfb97c 100644 --- a/src/main/java/com/percero/agents/sync/cw/DerivedValueChangeWatcherHelper.java +++ b/src/main/java/com/percero/agents/sync/cw/DerivedValueChangeWatcherHelper.java @@ -95,13 +95,13 @@ public Object calculate(String fieldName, ClassIDPair classIdPair, String[] para } @Override - public void process(String category, String subCategory, String fieldName) { - calculate(fieldName, new ClassIDPair(subCategory, category)); + public Object process(String category, String subCategory, String fieldName) { + return calculate(fieldName, new ClassIDPair(subCategory, category)); } @Override - public void process(String category, String subCategory, String fieldName, String[] params) { - calculate(fieldName, new ClassIDPair(subCategory, category), params); + public Object process(String category, String subCategory, String fieldName, String[] params) { + return calculate(fieldName, new ClassIDPair(subCategory, category), params); } /**protected void postCalculate(String fieldName, ClassIDPair classIdPair, Object result, Object oldValue) { @@ -141,7 +141,7 @@ protected void postCalculate(String fieldName, ClassIDPair classIdPair, String[] } - public void recalculate(String fieldName, ClassIDPair classIdPair, Collection clientIds, String[] params, Long requestTimestamp) { + public Object recalculate(String fieldName, ClassIDPair classIdPair, Collection clientIds, String[] params, Long requestTimestamp) { Object value = null; // Check to see if this value has already been calculated. @@ -155,7 +155,7 @@ public void recalculate(String fieldName, ClassIDPair classIdPair, Collection clientIds, String[] params, Long requestTimestamp) { + public Object reprocess(String category, String subCategory, String fieldName, Collection clientIds, String[] params, Long requestTimestamp) { ChangeWatcherReporting.reprocessCounter++; - this.recalculate(fieldName, new ClassIDPair(subCategory, category), clientIds, params, requestTimestamp); + return this.recalculate(fieldName, new ClassIDPair(subCategory, category), clientIds, params, requestTimestamp); } diff --git a/src/main/java/com/percero/agents/sync/cw/IChangeWatcherHelper.java b/src/main/java/com/percero/agents/sync/cw/IChangeWatcherHelper.java index 9f1a02c..a61f297 100644 --- a/src/main/java/com/percero/agents/sync/cw/IChangeWatcherHelper.java +++ b/src/main/java/com/percero/agents/sync/cw/IChangeWatcherHelper.java @@ -4,8 +4,8 @@ public interface IChangeWatcherHelper { - public void process(String category, String subCategory, String fieldName); - public void process(String category, String subCategory, String fieldName, String[] params); - public void reprocess(String category, String subCategory, String fieldName, Collection clientIds, String[] params, Long requestTimestamp); + public Object process(String category, String subCategory, String fieldName); + public Object process(String category, String subCategory, String fieldName, String[] params); + public Object reprocess(String category, String subCategory, String fieldName, Collection clientIds, String[] params, Long requestTimestamp); } diff --git a/src/main/java/com/percero/agents/sync/helpers/PostGetHelper.java b/src/main/java/com/percero/agents/sync/helpers/PostGetHelper.java index e264576..0b97d4e 100644 --- a/src/main/java/com/percero/agents/sync/helpers/PostGetHelper.java +++ b/src/main/java/com/percero/agents/sync/helpers/PostGetHelper.java @@ -41,11 +41,11 @@ public void setChangeWatcherHelperFactory(IChangeWatcherHelperFactory value) { } - public void postGetObject(IPerceroObject perceroObject, String userId, String clientId) throws Exception { + public IPerceroObject postGetObject(IPerceroObject perceroObject, String userId, String clientId) throws Exception { ClassIDPair pair = new ClassIDPair(perceroObject.getID(), perceroObject.getClass().getCanonicalName()); accessManager.saveAccessJournal(pair, userId, clientId); - postGet(pair); + return postGet(pair); } public void postGetObject(List perceroObjects, String userId, String clientId) throws Exception { @@ -58,27 +58,29 @@ public void postGetObject(List perceroObjects, String userId, St accessManager.saveAccessJournal(classIdPairs, userId, clientId); } - protected void postGet(ClassIDPair classIdPair) { + protected IPerceroObject postGet(ClassIDPair classIdPair) { IChangeWatcherHelper cwh = changeWatcherHelperFactory.getHelper(CATEGORY); if (cwh != null) { String changeWatcherId = "cw:cf:" + CATEGORY + ":" + classIdPair.getClassName() + ":" + classIdPair.getID(); - setupRecalculateChangeWatcher(changeWatcherId); + return setupRecalculateChangeWatcher(changeWatcherId); } + + return null; } - protected void setupRecalculateChangeWatcher(String changeWatcherId) { + protected IPerceroObject setupRecalculateChangeWatcher(String changeWatcherId) { ChangeWatcherReporting.internalRequestsCounter++; // if (useChangeWatcherQueue && pushSyncHelper != null) { // pushSyncHelper.pushStringToRoute( (new StringBuilder(changeWatcherId).append(":TS:").append(System.currentTimeMillis())).toString(), changeWatcherRouteName); // } // else { - recalculateChangeWatcher(changeWatcherId); + return recalculateChangeWatcher(changeWatcherId); // } } @SuppressWarnings("unchecked") - public void recalculateChangeWatcher(String changeWatcherId) { + public IPerceroObject recalculateChangeWatcher(String changeWatcherId) { try { // Check to see if a timestamp has been included. String[] changeWatcherTsArray = changeWatcherId.split(":TS:"); @@ -111,7 +113,10 @@ public void recalculateChangeWatcher(String changeWatcherId) { if (changeWatcherHelperFactory != null) { IChangeWatcherHelper cwh = changeWatcherHelperFactory.getHelper(category); - cwh.reprocess(category, subCategory, fieldName, null, otherParams, requestTimestamp); + Object result = cwh.reprocess(category, subCategory, fieldName, null, otherParams, requestTimestamp); + if (result != null && result instanceof IPerceroObject) { + return (IPerceroObject) result; + } /** // If no clients interested in this value, then remove it from the cache. @@ -136,5 +141,7 @@ public void recalculateChangeWatcher(String changeWatcherId) { } catch(Exception e) { log.error("Error recalculating Change Watcher: " + changeWatcherId, e); } + + return null; } } diff --git a/src/main/java/com/percero/agents/sync/metadata/MappedClass.java b/src/main/java/com/percero/agents/sync/metadata/MappedClass.java index 0ab45af..d3657ba 100644 --- a/src/main/java/com/percero/agents/sync/metadata/MappedClass.java +++ b/src/main/java/com/percero/agents/sync/metadata/MappedClass.java @@ -59,31 +59,36 @@ @SuppressWarnings("unchecked") public class MappedClass implements IMappedClass { - + private static Logger logger = Logger.getLogger(MappedClass.class); - - public static Map,List> entityInterfacesMappedClasses = Collections.synchronizedMap(new HashMap, List>()); - + + public static Map, List> entityInterfacesMappedClasses = Collections + .synchronizedMap(new HashMap, List>()); + public static Boolean allMappedClassesInitialized = false; @SuppressWarnings("rawtypes") public static void processManifest(IManifest manifest) { if (!allMappedClassesInitialized) { try { - IMappedClassManager mcm = MappedClassManagerFactory.getMappedClassManager(); + IMappedClassManager mcm = MappedClassManagerFactory + .getMappedClassManager(); Set mappedClasses = new HashSet(); ManifestHelper.setManifest(manifest); - Iterator itrUuidMap = manifest.getUuidMap().keySet().iterator(); - while(itrUuidMap.hasNext()) { + Iterator itrUuidMap = manifest.getUuidMap().keySet() + .iterator(); + while (itrUuidMap.hasNext()) { String nextUuid = itrUuidMap.next(); Class nextClass = manifest.getUuidMap().get(nextUuid); - - MappedClass mc = mcm.getMappedClassByClassName(nextClass.getCanonicalName()); + + MappedClass mc = mcm.getMappedClassByClassName(nextClass + .getCanonicalName()); mappedClasses.add(mc); } - + // Initialize the Fields - Iterator itrMappedClasses = mappedClasses.iterator(); + Iterator itrMappedClasses = mappedClasses + .iterator(); while (itrMappedClasses.hasNext()) { MappedClass mc = itrMappedClasses.next(); mc.initializeFields(); @@ -100,7 +105,7 @@ public static void processManifest(IManifest manifest) { MappedClass mc = itrMappedClasses.next(); mc.initializeRelationships(); } - + // Now set the childMappedClasses. itrMappedClasses = mappedClasses.iterator(); while (itrMappedClasses.hasNext()) { @@ -109,7 +114,9 @@ public static void processManifest(IManifest manifest) { int classLevel = 0; while (clazz != null) { if (manifest.getClassList().contains(clazz)) { - MappedClass nextMc = mcm.getMappedClassByClassName(clazz.getCanonicalName()); + MappedClass nextMc = mcm + .getMappedClassByClassName(clazz + .getCanonicalName()); nextMc.childMappedClasses.add(mc); if (classLevel == 0) { mc.parentMappedClass = nextMc; @@ -119,140 +126,165 @@ public static void processManifest(IManifest manifest) { clazz = clazz.getSuperclass(); } } - - + MappedClass.allMappedClassesInitialized = true; - - } catch(Exception e) { + + } catch (Exception e) { logger.error("Unable to process manifest.", e); } } } - + @SuppressWarnings("rawtypes") - public static List findEntityImplementation(Class interfaceClazz) { + public static List findEntityImplementation( + Class interfaceClazz) { return entityInterfacesMappedClasses.get(interfaceClazz); } private static Comparator fieldComparator; - + static { fieldComparator = new Comparator() { public int compare(MappedField o1, MappedField o2) { - return o1.getField().getName().compareToIgnoreCase(o2.getField().getName()); + return o1.getField().getName() + .compareToIgnoreCase(o2.getField().getName()); } }; } - + public int ID = 0; public Boolean needsReadCleaning = false; - public Boolean getNeedsReadCleaning () { - if (getReadAccessRightsFieldReferences() != null && !getReadAccessRightsFieldReferences().isEmpty()) { + + public Boolean getNeedsReadCleaning() { + if (getReadAccessRightsFieldReferences() != null + && !getReadAccessRightsFieldReferences().isEmpty()) { return true; - } - else if (getReadQuery() != null) { + } else if (getReadQuery() != null) { return true; - } - else if (getReadAllQuery() != null) { + } else if (getReadAllQuery() != null) { return true; - } - else { + } else { return false; } } - + public Boolean fieldsInitialized = false; public Boolean queriesInitialized = false; public Boolean relationshipsInitialized = false; -// private Boolean isInitializing = false; - + // private Boolean isInitializing = false; + public String dataProviderName = ""; + public String getDataProviderName() { return dataProviderName; } + public void setDataProviderName(String dataProviderName) { this.dataProviderName = dataProviderName; - + // Reset the DataProvider if (dataProvider != null) { dataProvider = null; } } - + public String className = ""; public String tableName = ""; public String tableSchema = ""; public MappedField idMappedField = null; - public List requiredFields = Collections.synchronizedList(new ArrayList()); - - public Set toManyFields = Collections.synchronizedSet(new HashSet()); - public Set toOneFields = Collections.synchronizedSet(new HashSet()); + public List requiredFields = Collections + .synchronizedList(new ArrayList()); + + public Set toManyFields = Collections + .synchronizedSet(new HashSet()); + public Set toOneFields = Collections + .synchronizedSet(new HashSet()); + + private Set sourceMappedFields = Collections + .synchronizedSet(new HashSet()); - private Set sourceMappedFields = Collections.synchronizedSet(new HashSet()); public Set getSourceMappedFields() { return sourceMappedFields; } - private Set targetMappedFields = Collections.synchronizedSet(new HashSet()); + + private Set targetMappedFields = Collections + .synchronizedSet(new HashSet()); + public Set getTargetMappedFields() { return targetMappedFields; } - public Set propertyFields = Collections.synchronizedSet(new HashSet()); - public List entityFields = Collections.synchronizedList(new ArrayList()); - public List mapFields = Collections.synchronizedList(new ArrayList()); - public List listFields = Collections.synchronizedList(new ArrayList()); - public List nonLazyLoadingFields = Collections.synchronizedList(new ArrayList()); - public Set externalizablePerceroObjectFields = Collections.synchronizedSet(new HashSet()); - public Set externalizableFields = Collections.synchronizedSet(new TreeSet(fieldComparator)); - public Map cascadeRemoveFieldReferences = Collections.synchronizedMap(new HashMap()); - public Map nulledOnRemoveFieldReferences = Collections.synchronizedMap(new HashMap()); + public Set propertyFields = Collections + .synchronizedSet(new HashSet()); + public List entityFields = Collections + .synchronizedList(new ArrayList()); + public List mapFields = Collections + .synchronizedList(new ArrayList()); + public List listFields = Collections + .synchronizedList(new ArrayList()); + public List nonLazyLoadingFields = Collections + .synchronizedList(new ArrayList()); + public Set externalizablePerceroObjectFields = Collections + .synchronizedSet(new HashSet()); + public Set externalizableFields = Collections + .synchronizedSet(new TreeSet(fieldComparator)); + public Map cascadeRemoveFieldReferences = Collections + .synchronizedMap(new HashMap()); + public Map nulledOnRemoveFieldReferences = Collections + .synchronizedMap(new HashMap()); @SuppressWarnings("rawtypes") - public Map entityImplementations = Collections.synchronizedMap(new HashMap()); + public Map entityImplementations = Collections + .synchronizedMap(new HashMap()); public MappedClass parentMappedClass = null; - public List childMappedClasses = Collections.synchronizedList(new ArrayList()); + public List childMappedClasses = Collections + .synchronizedList(new ArrayList()); + public List getChildMappedClasses() { return this.childMappedClasses; } - - + IDataProvider dataProvider = null; + public IDataProvider getDataProvider() { if (dataProvider == null) { - dataProvider = DataProviderManager.getInstance().getDataProviderByName(dataProviderName); + dataProvider = DataProviderManager.getInstance() + .getDataProviderByName(dataProviderName); } return dataProvider; } - - + /** - * readAccessRightsFieldReferences holds all MappedFields that have some sort of - * readAccessRights associated with that field. This means that this field needs - * to be recalculated for each User. - * If a MappedClass has NO fields in this list AND NO readQuery, then all objects - * of that type do NOT need to be recalculated for each User. + * readAccessRightsFieldReferences holds all MappedFields that have some + * sort of readAccessRights associated with that field. This means that this + * field needs to be recalculated for each User. If a MappedClass has NO + * fields in this list AND NO readQuery, then all objects of that type do + * NOT need to be recalculated for each User. */ public Set readAccessRightsFieldReferences = new HashSet(); + public Set getReadAccessRightsFieldReferences() { return readAccessRightsFieldReferences; } + public List> uniqueConstraints = new ArrayList>(); public List queries = new ArrayList(); public Boolean hasGeneratedId = false; public Boolean hasNonLazyLoadProperties = false; - + @SuppressWarnings("rawtypes") public Class clazz = null; - + public MappedField getExternalizeFieldByName(String fieldName) { for (MappedField mappedField : externalizableFields) { if (mappedField.getField().getName().equals(fieldName)) return mappedField; } - + return null; } - + private IMappedQuery readAllQuery = null; + public IMappedQuery getReadAllQuery() { return readAllQuery; } @@ -260,8 +292,9 @@ public IMappedQuery getReadAllQuery() { public void setReadAllQuery(IMappedQuery query) { this.readAllQuery = query; } - + private IMappedQuery createQuery = null; + public IMappedQuery getCreateQuery() { return createQuery; } @@ -271,6 +304,7 @@ public void setCreateQuery(IMappedQuery createQuery) { } private IMappedQuery updateQuery = null; + public IMappedQuery getUpdateQuery() { return updateQuery; } @@ -280,15 +314,17 @@ public void setUpdateQuery(IMappedQuery updateQuery) { } private IMappedQuery readQuery = null; + public IMappedQuery getReadQuery() { return readQuery; } - + public void setReadQuery(IMappedQuery readQuery) { this.readQuery = readQuery; } - + private IMappedQuery deleteQuery = null; + public IMappedQuery getDeleteQuery() { return deleteQuery; } @@ -297,123 +333,141 @@ public void setDeleteQuery(IMappedQuery deleteQuery) { this.deleteQuery = deleteQuery; } - @SuppressWarnings("rawtypes") public MappedClass(int theId, String aClassName) { ID = theId; className = aClassName; try { clazz = MappedClass.forName(className); - } catch(Exception e) { + } catch (Exception e) { logger.error("Unable to instantiate class " + className, e); } - + MappedClassManagerFactory.getMappedClassManager(); - + // Look for EntityInterfaces Class nextClazz = clazz; while (nextClazz != null) { - EntityInterface entityInterface = (EntityInterface) nextClazz.getAnnotation(EntityInterface.class); + EntityInterface entityInterface = (EntityInterface) nextClazz + .getAnnotation(EntityInterface.class); if (entityInterface != null) { if (entityInterface.interfaceClass() != null) { - List allEntityImplementations = MappedClass.entityInterfacesMappedClasses.get(entityInterface.interfaceClass()); + List allEntityImplementations = MappedClass.entityInterfacesMappedClasses + .get(entityInterface.interfaceClass()); if (allEntityImplementations == null) { allEntityImplementations = new ArrayList(); - MappedClass.entityInterfacesMappedClasses.put(entityInterface.interfaceClass(), allEntityImplementations); + MappedClass.entityInterfacesMappedClasses.put( + entityInterface.interfaceClass(), + allEntityImplementations); } - + EntityImplementation entityImpl = processEntityInterface(entityInterface); allEntityImplementations.add(entityImpl); } } - - EntityInterfaces entityInterfaces = (EntityInterfaces) nextClazz.getAnnotation(EntityInterfaces.class); + + EntityInterfaces entityInterfaces = (EntityInterfaces) nextClazz + .getAnnotation(EntityInterfaces.class); if (entityInterfaces != null) { - for(EntityInterface nextEntityInterface : entityInterfaces.entityInterfaces()) { + for (EntityInterface nextEntityInterface : entityInterfaces + .entityInterfaces()) { if (nextEntityInterface.interfaceClass() != null) { - List allEntityImplementations = MappedClass.entityInterfacesMappedClasses.get(nextEntityInterface.interfaceClass()); + List allEntityImplementations = MappedClass.entityInterfacesMappedClasses + .get(nextEntityInterface.interfaceClass()); if (allEntityImplementations == null) { allEntityImplementations = new ArrayList(); - MappedClass.entityInterfacesMappedClasses.put(nextEntityInterface.interfaceClass(), allEntityImplementations); + MappedClass.entityInterfacesMappedClasses.put( + nextEntityInterface.interfaceClass(), + allEntityImplementations); } - + EntityImplementation entityImpl = processEntityInterface(entityInterface); if (!allEntityImplementations.contains(entityImpl)) { allEntityImplementations.add(entityImpl); } } - + } } nextClazz = nextClazz.getSuperclass(); } } - - protected EntityImplementation processEntityInterface(EntityInterface entityInterface) { + + protected EntityImplementation processEntityInterface( + EntityInterface entityInterface) { if (entityInterface == null || entityInterface.interfaceClass() == null) { logger.warn("Invalid EntityInterface on class " + className); return null; } // Check to see if interface has already been processed. - EntityImplementation entityImpl = entityImplementations.get(entityInterface.interfaceClass()); + EntityImplementation entityImpl = entityImplementations + .get(entityInterface.interfaceClass()); if (entityImpl != null) { // This EntityInterface has already been processed. return entityImpl; } - + entityImpl = new EntityImplementation(); entityImpl.mappedClass = this; entityImpl.entityInterfaceClass = entityInterface.interfaceClass(); - + entityImplementations.put(entityInterface.interfaceClass(), entityImpl); return entityImpl; } - + @SuppressWarnings("rawtypes") public void initializeFields() { if (fieldsInitialized) { return; } - + try { Class clazz = MappedClass.forName(className); - + List fields = SyncHibernateUtils.getClassFields(clazz); - for(Field nextField : fields) { + for (Field nextField : fields) { // Ignore this field if marked as Transient. - Transient transientAnno = (Transient) nextField.getAnnotation(Transient.class); + Transient transientAnno = (Transient) nextField + .getAnnotation(Transient.class); if (transientAnno != null) continue; - - Externalize externalize = (Externalize) nextField.getAnnotation(Externalize.class); + + Externalize externalize = (Externalize) nextField + .getAnnotation(Externalize.class); if (externalize == null) { // Only process Externalizeable fields. continue; } - + // Check to see if this Field has already been processed. - Iterator itrExternalizeFields = externalizableFields.iterator(); + Iterator itrExternalizeFields = externalizableFields + .iterator(); Boolean fieldAlreadyMapped = false; while (itrExternalizeFields.hasNext()) { - MappedField nextExistingMappedField = itrExternalizeFields.next(); - if (nextExistingMappedField.getField().getName().equals(nextField.getName())) { + MappedField nextExistingMappedField = itrExternalizeFields + .next(); + if (nextExistingMappedField.getField().getName() + .equals(nextField.getName())) { // This field has already been mapped. fieldAlreadyMapped = true; break; } } - + if (fieldAlreadyMapped) { continue; } - - Method theGetter = SyncHibernateUtils.getFieldGetters(clazz, nextField); - transientAnno = (Transient) theGetter.getAnnotation(Transient.class); + + Method theGetter = SyncHibernateUtils.getFieldGetters(clazz, + nextField); + transientAnno = (Transient) theGetter + .getAnnotation(Transient.class); if (transientAnno != null) continue; - Method theSetter = SyncHibernateUtils.getFieldSetters(clazz, nextField); - + Method theSetter = SyncHibernateUtils.getFieldSetters(clazz, + nextField); + MappedField nextMappedField = null; Class nextFieldClass = nextField.getType(); if (nextFieldClass == int.class) @@ -436,11 +490,13 @@ else if (nextFieldClass == String.class) nextMappedField = new MappedFieldString(); else if (nextFieldClass == Date.class) nextMappedField = new MappedFieldDate(); -/** else if (nextFieldClass == Map.class) - nextMappedField = new MappedFieldMap(); - else if (nextFieldClass == List.class) - nextMappedField = new MappedFieldList();**/ - else if (implementsInterface(nextFieldClass, IPerceroObject.class)) + /** + * else if (nextFieldClass == Map.class) nextMappedField = new + * MappedFieldMap(); else if (nextFieldClass == List.class) + * nextMappedField = new MappedFieldList(); + **/ + else if (implementsInterface(nextFieldClass, + IPerceroObject.class)) nextMappedField = new MappedFieldPerceroObject(); else if (implementsInterface(nextFieldClass, Map.class)) nextMappedField = new MappedFieldMap(); @@ -454,105 +510,121 @@ else if (implementsInterface(nextFieldClass, List.class)) nextMappedField.setField(nextField); nextMappedField.setGetter(theGetter); nextMappedField.setSetter(theSetter); - + nextMappedField.setUseLazyLoading(externalize.useLazyLoading()); if (!nextMappedField.getUseLazyLoading()) this.hasNonLazyLoadProperties = true; - + if (!externalize.useLazyLoading()) nonLazyLoadingFields.add(nextMappedField); externalizableFields.add(nextMappedField); if (nextMappedField instanceof MappedFieldPerceroObject) - externalizablePerceroObjectFields.add((MappedFieldPerceroObject) nextMappedField); - - OneToMany oneToMany = (OneToMany) theGetter.getAnnotation(OneToMany.class); + externalizablePerceroObjectFields + .add((MappedFieldPerceroObject) nextMappedField); + + OneToMany oneToMany = (OneToMany) theGetter + .getAnnotation(OneToMany.class); if (oneToMany == null) - oneToMany = (OneToMany) nextField.getAnnotation(OneToMany.class); + oneToMany = (OneToMany) nextField + .getAnnotation(OneToMany.class); if (oneToMany != null) { toManyFields.add(nextMappedField); getTargetMappedFields().add(nextMappedField); } + + if (oneToMany != null) { + ((MappedFieldList) nextMappedField).setListClass(oneToMany.targetEntity()); + } - ManyToOne manyToOne = (ManyToOne) theGetter.getAnnotation(ManyToOne.class); + ManyToOne manyToOne = (ManyToOne) theGetter + .getAnnotation(ManyToOne.class); if (manyToOne == null) - manyToOne = (ManyToOne) nextField.getAnnotation(ManyToOne.class); + manyToOne = (ManyToOne) nextField + .getAnnotation(ManyToOne.class); if (manyToOne != null) { - getSourceMappedFields().add((MappedFieldPerceroObject) nextMappedField); + getSourceMappedFields().add( + (MappedFieldPerceroObject) nextMappedField); } - - OneToOne oneToOne = (OneToOne) theGetter.getAnnotation(OneToOne.class); + + OneToOne oneToOne = (OneToOne) theGetter + .getAnnotation(OneToOne.class); if (oneToOne == null) - oneToOne = (OneToOne) nextField.getAnnotation(OneToOne.class); + oneToOne = (OneToOne) nextField + .getAnnotation(OneToOne.class); if (oneToOne != null) { if (StringUtils.hasText(oneToOne.mappedBy())) { getTargetMappedFields().add(nextMappedField); - } - else { - getSourceMappedFields().add((MappedFieldPerceroObject) nextMappedField); + } else { + getSourceMappedFields().add( + (MappedFieldPerceroObject) nextMappedField); } } - + Boolean isPropertyField = true; - Entity nextEntity = (Entity) nextField.getType().getAnnotation(Entity.class); - if (nextEntity != null) - { + Entity nextEntity = (Entity) nextField.getType().getAnnotation( + Entity.class); + if (nextEntity != null) { entityFields.add(nextField); - toOneFields.add( (MappedFieldPerceroObject) nextMappedField ); + toOneFields.add((MappedFieldPerceroObject) nextMappedField); isPropertyField = false; } - - if (inheritsFrom(nextField.getType(), Map.class)) - { + + if (inheritsFrom(nextField.getType(), Map.class)) { mapFields.add(nextField); toManyFields.add(nextMappedField); isPropertyField = false; } - - if (inheritsFrom(nextField.getType(), List.class)) - { + + if (inheritsFrom(nextField.getType(), List.class)) { listFields.add(nextField); toManyFields.add(nextMappedField); isPropertyField = false; } - + if (isPropertyField) { propertyFields.add(nextMappedField); } - + Id id = (Id) theGetter.getAnnotation(Id.class); if (id == null) id = (Id) nextField.getAnnotation(Id.class); if (id != null) { idMappedField = nextMappedField; - List uniqueConstraintList = new ArrayList(1); + List uniqueConstraintList = new ArrayList( + 1); uniqueConstraintList.add(nextMappedField); uniqueConstraints.add(uniqueConstraintList); - + // Check to see if this class has a Generated ID. - GeneratedValue generatedValue = (GeneratedValue) nextField.getAnnotation(GeneratedValue.class); + GeneratedValue generatedValue = (GeneratedValue) nextField + .getAnnotation(GeneratedValue.class); hasGeneratedId = (generatedValue != null); } - + Column column = (Column) theGetter.getAnnotation(Column.class); if (column == null) column = (Column) nextField.getAnnotation(Column.class); if (column != null) { if (column.unique()) { - List uniqueConstraintList = new ArrayList(1); + List uniqueConstraintList = new ArrayList( + 1); uniqueConstraintList.add(nextMappedField); uniqueConstraints.add(uniqueConstraintList); } - - if (column.name() != null && column.name().trim().length() > 0) + + if (column.name() != null + && column.name().trim().length() > 0) nextMappedField.setColumnName(column.name()); else nextMappedField.setColumnName(nextField.getName()); } - - JoinColumn joinColumn = (JoinColumn) theGetter.getAnnotation(JoinColumn.class); + + JoinColumn joinColumn = (JoinColumn) theGetter + .getAnnotation(JoinColumn.class); if (joinColumn == null) - joinColumn = (JoinColumn) nextField.getAnnotation(JoinColumn.class); + joinColumn = (JoinColumn) nextField + .getAnnotation(JoinColumn.class); if (joinColumn != null) { if (StringUtils.hasText(joinColumn.name())) { nextMappedField.setJoinColumnName(joinColumn.name()); @@ -560,131 +632,160 @@ else if (implementsInterface(nextFieldClass, List.class)) } // Get NamedQueries for handling Access Rights. - AccessRights accessRights = (AccessRights) nextField.getAnnotation(AccessRights.class); + AccessRights accessRights = (AccessRights) nextField + .getAnnotation(AccessRights.class); if (accessRights == null) - accessRights = (AccessRights) nextMappedField.getGetter().getAnnotation(AccessRights.class); + accessRights = (AccessRights) nextMappedField.getGetter() + .getAnnotation(AccessRights.class); if (accessRights != null) { - for(AccessRight nextAccessRight : accessRights.value()) { - /*if (nextAccessRight.type().equalsIgnoreCase("createQuery")) { - if (nextAccessRight.query().indexOf("jpql:") >= 0) - nextMappedField.createQuery = new JpqlQuery(); - else - nextMappedField.createQuery = new MappedQuery(); - nextMappedField.createQuery.setQuery(nextAccessRight.query()); - } else if (nextAccessRight.type().equalsIgnoreCase("updateQuery")) { - if (nextAccessRight.query().indexOf("jpql:") >= 0) - nextMappedField.updateQuery = new JpqlQuery(); - else - nextMappedField.updateQuery = new MappedQuery(); - nextMappedField.updateQuery.setQuery(nextAccessRight.query()); - } else*/ - if (nextAccessRight.type().equalsIgnoreCase("readQuery")) { + for (AccessRight nextAccessRight : accessRights.value()) { + /* + * if + * (nextAccessRight.type().equalsIgnoreCase("createQuery" + * )) { if (nextAccessRight.query().indexOf("jpql:") >= + * 0) nextMappedField.createQuery = new JpqlQuery(); + * else nextMappedField.createQuery = new MappedQuery(); + * nextMappedField + * .createQuery.setQuery(nextAccessRight.query()); } + * else if + * (nextAccessRight.type().equalsIgnoreCase("updateQuery" + * )) { if (nextAccessRight.query().indexOf("jpql:") >= + * 0) nextMappedField.updateQuery = new JpqlQuery(); + * else nextMappedField.updateQuery = new MappedQuery(); + * nextMappedField + * .updateQuery.setQuery(nextAccessRight.query()); } + * else + */ + if (nextAccessRight.type() + .equalsIgnoreCase("readQuery")) { if (nextAccessRight.query().indexOf("jpql:") >= 0) { nextMappedField.setReadQuery(new JpqlQuery()); - nextMappedField.getReadQuery().setQuery(nextAccessRight.query()); - } - else if (nextAccessRight.query().indexOf("sql:") >= 0) { - nextMappedField.setReadQuery(new SqlQuery(nextAccessRight.query().substring(nextAccessRight.query().indexOf("sql:")+4))); - } - else { - // Whatever type of Query this is, it is not supported. + nextMappedField.getReadQuery().setQuery( + nextAccessRight.query()); + } else if (nextAccessRight.query().indexOf("sql:") >= 0) { + nextMappedField.setReadQuery(new SqlQuery( + nextAccessRight.query().substring( + nextAccessRight.query() + .indexOf("sql:") + 4))); + } else { + // Whatever type of Query this is, it is not + // supported. continue; -// nextMappedField.setReadQuery(new MappedQuery()); + // nextMappedField.setReadQuery(new + // MappedQuery()); } nextMappedField.setHasReadAccessRights(true); - readAccessRightsFieldReferences.add(nextMappedField); - } /*else if (nextAccessRight.type().equalsIgnoreCase("deleteQuery")) { - if (nextAccessRight.query().indexOf("jpql:") >= 0) - nextMappedField.deleteQuery = new JpqlQuery(); - else - nextMappedField.deleteQuery = new MappedQuery(); - nextMappedField.deleteQuery.setQuery(nextAccessRight.query()); - }*/ + readAccessRightsFieldReferences + .add(nextMappedField); + } /* + * else if + * (nextAccessRight.type().equalsIgnoreCase("deleteQuery" + * )) { if (nextAccessRight.query().indexOf("jpql:") >= + * 0) nextMappedField.deleteQuery = new JpqlQuery(); + * else nextMappedField.deleteQuery = new MappedQuery(); + * nextMappedField + * .deleteQuery.setQuery(nextAccessRight.query()); } + */ // Add to queries list. IMappedQuery nextQuery = null; if (nextAccessRight.query().indexOf("jpql:") >= 0) { nextQuery = new JpqlQuery(); nextQuery.setQuery(nextAccessRight.query()); - } - else if (nextAccessRight.query().indexOf("sql:") >= 0) { - nextQuery = new SqlQuery(nextAccessRight.query().substring(nextAccessRight.query().indexOf("sql:")+4)); - } - else { + } else if (nextAccessRight.query().indexOf("sql:") >= 0) { + nextQuery = new SqlQuery(nextAccessRight.query() + .substring( + nextAccessRight.query().indexOf( + "sql:") + 4)); + } else { // Unsupported Query type continue; -// nextQuery = new MappedQuery(); + // nextQuery = new MappedQuery(); } nextQuery.setQueryName(nextAccessRight.type()); - + nextMappedField.queries.add(nextQuery); } } - - // Check to see if this has any PropertyInterfaces that need to be addresed. - PropertyInterface propInterface = (PropertyInterface) nextField.getAnnotation(PropertyInterface.class); + + // Check to see if this has any PropertyInterfaces that need to + // be addresed. + PropertyInterface propInterface = (PropertyInterface) nextField + .getAnnotation(PropertyInterface.class); if (propInterface != null) { - processMappedFieldPropertyInterface(propInterface, nextMappedField); + processMappedFieldPropertyInterface(propInterface, + nextMappedField); } - PropertyInterfaces propertyInterfaces = (PropertyInterfaces) nextField.getAnnotation(PropertyInterfaces.class); + PropertyInterfaces propertyInterfaces = (PropertyInterfaces) nextField + .getAnnotation(PropertyInterfaces.class); if (propertyInterfaces != null) { - for(PropertyInterface nextPropInterface : propertyInterfaces.propertyInterfaces()) { - processMappedFieldPropertyInterface(nextPropInterface, nextMappedField); + for (PropertyInterface nextPropInterface : propertyInterfaces + .propertyInterfaces()) { + processMappedFieldPropertyInterface(nextPropInterface, + nextMappedField); } } } - } catch(Exception e) { + } catch (Exception e) { logger.error("Error parsing MappedClass " + this.className, e); } fieldsInitialized = true; } - - private void processMappedFieldPropertyInterface(PropertyInterface propInterface, MappedField mappedField) { - if (propInterface == null || !StringUtils.hasText(propInterface.propertyName())) { - logger.warn("Invalid PropertyInterface for " + className + "." + mappedField.getField().getName()); + + private void processMappedFieldPropertyInterface( + PropertyInterface propInterface, MappedField mappedField) { + if (propInterface == null + || !StringUtils.hasText(propInterface.propertyName())) { + logger.warn("Invalid PropertyInterface for " + className + "." + + mappedField.getField().getName()); return; } - - EntityImplementation entityImpl = entityImplementations.get(propInterface.entityInterfaceClass()); + + EntityImplementation entityImpl = entityImplementations + .get(propInterface.entityInterfaceClass()); if (entityImpl != null) { - Iterator itrPropImpls = entityImpl.propertyImplementations.iterator(); + Iterator itrPropImpls = entityImpl.propertyImplementations + .iterator(); while (itrPropImpls.hasNext()) { PropertyImplementation nextPropImpl = itrPropImpls.next(); - if (nextPropImpl.propertyName.equals(propInterface.propertyName())) { + if (nextPropImpl.propertyName.equals(propInterface + .propertyName())) { // Found property implementation. nextPropImpl.mappedField = mappedField; return; } } - + // No valid PropertyIntrerface found so create one. PropertyImplementation propImpl = new PropertyImplementation(); propImpl.entityImplementation = entityImpl; propImpl.propertyName = propInterface.propertyName(); propImpl.mappedField = mappedField; entityImpl.propertyImplementations.add(propImpl); - - for(PropertyInterfaceParam nextPropInterfaceParam : propInterface.params()) { + + for (PropertyInterfaceParam nextPropInterfaceParam : propInterface + .params()) { PropertyImplementationParam param = new PropertyImplementationParam(); param.propertyImplementation = propImpl; param.name = nextPropInterfaceParam.name(); param.value = nextPropInterfaceParam.value(); propImpl.params.add(param); } - } - else { + } else { // Need to crearte EntityImplementation as well. entityImpl = new EntityImplementation(); - entityImpl.entityInterfaceClass = propInterface.entityInterfaceClass(); + entityImpl.entityInterfaceClass = propInterface + .entityInterfaceClass(); entityImpl.mappedClass = this; - + PropertyImplementation propImpl = new PropertyImplementation(); propImpl.entityImplementation = entityImpl; propImpl.propertyName = propInterface.propertyName(); propImpl.mappedField = mappedField; entityImpl.propertyImplementations.add(propImpl); - - for(PropertyInterfaceParam nextPropInterfaceParam : propInterface.params()) { + + for (PropertyInterfaceParam nextPropInterfaceParam : propInterface + .params()) { PropertyImplementationParam param = new PropertyImplementationParam(); param.propertyImplementation = propImpl; param.name = nextPropInterfaceParam.name(); @@ -693,123 +794,177 @@ private void processMappedFieldPropertyInterface(PropertyInterface propInterface } } } - + @SuppressWarnings("rawtypes") public void initializeQueries() { if (queriesInitialized) { return; } - IMappedClassManager mcm = MappedClassManagerFactory.getMappedClassManager(); + IMappedClassManager mcm = MappedClassManagerFactory + .getMappedClassManager(); try { Class clazz = MappedClass.forName(className); Class tempClazz = clazz; - while(!tempClazz.equals(Object.class)) { + while (!tempClazz.equals(Object.class)) { // Get Entity name, if exists. if (!StringUtils.hasText(tableName)) { - Entity entity = (Entity) tempClazz.getAnnotation(Entity.class); + Entity entity = (Entity) tempClazz + .getAnnotation(Entity.class); if (entity != null) { tableName = entity.name(); } } - + if (!StringUtils.hasText(tableSchema)) { Table table = (Table) tempClazz.getAnnotation(Table.class); if (table != null && StringUtils.hasText(table.schema())) { tableSchema = table.schema(); } } - + if (!StringUtils.hasText(dataProviderName)) { - DataProvider dataProvider = (DataProvider) tempClazz.getAnnotation(DataProvider.class); + DataProvider dataProvider = (DataProvider) tempClazz + .getAnnotation(DataProvider.class); if (dataProvider != null) { setDataProviderName(dataProvider.name()); } } - + // Get native queries for handling AccessRights /** - * readAllQuery - * Added by Jonathan Samples because initial download for certain entities was disasterously - * slow using the old style + * readAllQuery Added by Jonathan Samples because initial + * download for certain entities was disasterously slow using + * the old style */ - PerceroNamedNativeQueries nativeQueries = (PerceroNamedNativeQueries) tempClazz.getAnnotation(PerceroNamedNativeQueries.class); - if(nativeQueries != null){ - for(PerceroNamedNativeQuery q : nativeQueries.value()){ + PerceroNamedNativeQueries nativeQueries = (PerceroNamedNativeQueries) tempClazz + .getAnnotation(PerceroNamedNativeQueries.class); + if (nativeQueries != null) { + for (PerceroNamedNativeQuery q : nativeQueries.value()) { if (q.name().equalsIgnoreCase("readAllQuery")) { - logger.debug("Adding readAllQuery to mappedClass: "+className); + logger.debug("Adding readAllQuery to mappedClass: " + + className); readAllQuery = new SqlQuery(q.query()); } } } - + // Get NamedQueries for handling Access Rights. - NamedQueries namedQueries = (NamedQueries) tempClazz.getAnnotation(NamedQueries.class); + NamedQueries namedQueries = (NamedQueries) tempClazz + .getAnnotation(NamedQueries.class); if (namedQueries != null) { - for(NamedQuery nextNamedQuery : namedQueries.value()) { - - if (nextNamedQuery.name().equalsIgnoreCase("createQuery")) { - createQuery = QueryFactory.createQuery(nextNamedQuery.query()); - } else if (nextNamedQuery.name().equalsIgnoreCase("updateQuery")) { - updateQuery = QueryFactory.createQuery(nextNamedQuery.query()); - } else if (nextNamedQuery.name().equalsIgnoreCase("readQuery")) { - readQuery = QueryFactory.createQuery(nextNamedQuery.query()); - - Iterator itrToOneFields = toOneFields.iterator(); - while(itrToOneFields.hasNext()) { - MappedFieldPerceroObject nextMappedField = itrToOneFields.next(); - MappedClass referencedMappedClass = mcm.getMappedClassByClassName(nextMappedField.getField().getType().getCanonicalName()); + for (NamedQuery nextNamedQuery : namedQueries.value()) { + + if (nextNamedQuery.name().equalsIgnoreCase( + "createQuery")) { + createQuery = QueryFactory + .createQuery(nextNamedQuery.query()); + } else if (nextNamedQuery.name().equalsIgnoreCase( + "updateQuery")) { + updateQuery = QueryFactory + .createQuery(nextNamedQuery.query()); + } else if (nextNamedQuery.name().equalsIgnoreCase( + "readQuery")) { + readQuery = QueryFactory.createQuery(nextNamedQuery + .query()); + + Iterator itrToOneFields = toOneFields + .iterator(); + while (itrToOneFields.hasNext()) { + MappedFieldPerceroObject nextMappedField = itrToOneFields + .next(); + MappedClass referencedMappedClass = mcm + .getMappedClassByClassName(nextMappedField + .getField().getType() + .getCanonicalName()); // Need to find the corresponding field. - for(MappedField nextRefMappedField : referencedMappedClass.toManyFields) { + for (MappedField nextRefMappedField : referencedMappedClass.toManyFields) { if (nextRefMappedField instanceof MappedFieldList) { - OneToMany refOneToMany = nextRefMappedField.getField().getAnnotation(OneToMany.class); + OneToMany refOneToMany = nextRefMappedField + .getField().getAnnotation( + OneToMany.class); if (refOneToMany == null) - refOneToMany = nextRefMappedField.getGetter().getAnnotation(OneToMany.class); - - if (refOneToMany != null && refOneToMany.targetEntity().getCanonicalName().equals(this.className) && nextMappedField.getField().getName().equals(refOneToMany.mappedBy())) { + refOneToMany = nextRefMappedField + .getGetter().getAnnotation( + OneToMany.class); + + if (refOneToMany != null + && refOneToMany.targetEntity() + .getCanonicalName() + .equals(this.className) + && nextMappedField + .getField() + .getName() + .equals(refOneToMany + .mappedBy())) { // Found the referenced field. - nextRefMappedField.setHasReadAccessRights(true); - referencedMappedClass.readAccessRightsFieldReferences.add(nextRefMappedField); + nextRefMappedField + .setHasReadAccessRights(true); + referencedMappedClass.readAccessRightsFieldReferences + .add(nextRefMappedField); break; } - } - else if (nextRefMappedField instanceof MappedFieldPerceroObject) { - OneToOne refOneToOne = nextRefMappedField.getField().getAnnotation(OneToOne.class); + } else if (nextRefMappedField instanceof MappedFieldPerceroObject) { + OneToOne refOneToOne = nextRefMappedField + .getField().getAnnotation( + OneToOne.class); if (refOneToOne == null) - refOneToOne = nextRefMappedField.getGetter().getAnnotation(OneToOne.class); - - if (refOneToOne != null && refOneToOne.targetEntity().getCanonicalName().equals(this.className) && nextMappedField.getField().getName().equals(refOneToOne.mappedBy())) { + refOneToOne = nextRefMappedField + .getGetter().getAnnotation( + OneToOne.class); + + if (refOneToOne != null + && refOneToOne.targetEntity() + .getCanonicalName() + .equals(this.className) + && nextMappedField + .getField() + .getName() + .equals(refOneToOne + .mappedBy())) { // Found the referenced field. - nextRefMappedField.setHasReadAccessRights(true); - referencedMappedClass.readAccessRightsFieldReferences.add(nextRefMappedField); + nextRefMappedField + .setHasReadAccessRights(true); + referencedMappedClass.readAccessRightsFieldReferences + .add(nextRefMappedField); break; } } } } - - } else if (nextNamedQuery.name().equalsIgnoreCase("deleteQuery")) { - deleteQuery = QueryFactory.createQuery(nextNamedQuery.query()); + + } else if (nextNamedQuery.name().equalsIgnoreCase( + "deleteQuery")) { + deleteQuery = QueryFactory + .createQuery(nextNamedQuery.query()); } // Add to queries list. - IMappedQuery nextQuery = QueryFactory.createQuery(nextNamedQuery.query()); + IMappedQuery nextQuery = QueryFactory + .createQuery(nextNamedQuery.query()); nextQuery.setQueryName(nextNamedQuery.name()); - + queries.add(nextQuery); } } Table table = (Table) tempClazz.getAnnotation(Table.class); - + if (table != null) { - for(UniqueConstraint nextUniqueConstraint : table.uniqueConstraints()) { - // TODO: Add an Array of MappedFields instead of a UniqueConstraint. + for (UniqueConstraint nextUniqueConstraint : table + .uniqueConstraints()) { + // TODO: Add an Array of MappedFields instead of a + // UniqueConstraint. List listMappedFields = new ArrayList(); - for(String nextColumnName : nextUniqueConstraint.columnNames()) { - Iterator itrAllMappedFields = externalizableFields.iterator(); - while(itrAllMappedFields.hasNext()) { - MappedField nextMappedField = itrAllMappedFields.next(); - if (nextColumnName.equalsIgnoreCase(nextMappedField.getColumnName())) { + for (String nextColumnName : nextUniqueConstraint + .columnNames()) { + Iterator itrAllMappedFields = externalizableFields + .iterator(); + while (itrAllMappedFields.hasNext()) { + MappedField nextMappedField = itrAllMappedFields + .next(); + if (nextColumnName + .equalsIgnoreCase(nextMappedField + .getColumnName())) { listMappedFields.add(nextMappedField); break; } @@ -818,124 +973,161 @@ else if (nextRefMappedField instanceof MappedFieldPerceroObject) { uniqueConstraints.add(listMappedFields); } } - + tempClazz = tempClazz.getSuperclass(); } - - + if (tableName == null || tableName.isEmpty()) { tableName = ClassUtils.getShortName(clazz); } - } catch(Exception e) { + } catch (Exception e) { logger.error("Error parsing MappedClass " + this.className, e); } - + queriesInitialized = true; } - + public void initializeRelationships() { if (relationshipsInitialized) { return; } - IMappedClassManager mcm = MappedClassManagerFactory.getMappedClassManager(); + IMappedClassManager mcm = MappedClassManagerFactory + .getMappedClassManager(); try { - - // NOTE: These sections have to be done after the code above so that we know all MappedClasses in the toManyFields list have been initialized - Iterator itrExternalizeFields = externalizableFields.iterator(); - while(itrExternalizeFields.hasNext()) { + + // NOTE: These sections have to be done after the code above so that + // we know all MappedClasses in the toManyFields list have been + // initialized + Iterator itrExternalizeFields = externalizableFields + .iterator(); + while (itrExternalizeFields.hasNext()) { MappedField nextMappedField = itrExternalizeFields.next(); - OneToMany oneToMany = (OneToMany) nextMappedField.getGetter().getAnnotation(OneToMany.class); + OneToMany oneToMany = (OneToMany) nextMappedField.getGetter() + .getAnnotation(OneToMany.class); if (oneToMany == null) - oneToMany = (OneToMany) nextMappedField.getField().getAnnotation(OneToMany.class); - + oneToMany = (OneToMany) nextMappedField.getField() + .getAnnotation(OneToMany.class); + if (oneToMany != null) { -// // This must be a source MappedField -// sourceMappedFields.add((MappedFieldPerceroObject) nextMappedField); + // // This must be a source MappedField + // sourceMappedFields.add((MappedFieldPerceroObject) + // nextMappedField); - //toManyFields.add(nextMappedField); - ParameterizedType listType = (ParameterizedType) nextMappedField.getField().getGenericType(); - Class listClass = (Class) listType.getActualTypeArguments()[0]; - MappedClass referencedMappedClass = mcm.getMappedClassByClassName(listClass.getCanonicalName()); + // toManyFields.add(nextMappedField); + ParameterizedType listType = (ParameterizedType) nextMappedField + .getField().getGenericType(); + Class listClass = (Class) listType + .getActualTypeArguments()[0]; + MappedClass referencedMappedClass = mcm + .getMappedClassByClassName(listClass + .getCanonicalName()); if (!referencedMappedClass.fieldsInitialized) { referencedMappedClass.initializeFields(); } - if (referencedMappedClass.getReadQuery() != null && StringUtils.hasText(referencedMappedClass.getReadQuery().getQuery())) { + if (referencedMappedClass.getReadQuery() != null + && StringUtils.hasText(referencedMappedClass + .getReadQuery().getQuery())) { nextMappedField.setHasReadAccessRights(true); readAccessRightsFieldReferences.add(nextMappedField); } - + } - ManyToOne manyToOne = (ManyToOne) nextMappedField.getGetter().getAnnotation(ManyToOne.class); + ManyToOne manyToOne = (ManyToOne) nextMappedField.getGetter() + .getAnnotation(ManyToOne.class); if (manyToOne == null) - manyToOne = (ManyToOne) nextMappedField.getField().getAnnotation(ManyToOne.class); - -// if (manyToOne != null) { -// // This must be a target MappedField -// targetMappedFields.add(nextMappedField); -// } - - OneToOne oneToOne = (OneToOne) nextMappedField.getGetter().getAnnotation(OneToOne.class); + manyToOne = (ManyToOne) nextMappedField.getField() + .getAnnotation(ManyToOne.class); + + // if (manyToOne != null) { + // // This must be a target MappedField + // targetMappedFields.add(nextMappedField); + // } + + OneToOne oneToOne = (OneToOne) nextMappedField.getGetter() + .getAnnotation(OneToOne.class); if (oneToOne == null) - oneToOne = (OneToOne) nextMappedField.getField().getAnnotation(OneToOne.class); - -// if (oneToOne != null) { -// // Not sure if this is source or target, let's find out... -// if(StringUtils.hasText(oneToOne.mappedBy())) { -// // This must be a target MappedField -// targetMappedFields.add(nextMappedField); -// } -// else { -// // This must be a source MappedField -// sourceMappedFields.add((MappedFieldPerceroObject) nextMappedField); -// } -// } - + oneToOne = (OneToOne) nextMappedField.getField() + .getAnnotation(OneToOne.class); + + // if (oneToOne != null) { + // // Not sure if this is source or target, let's find out... + // if(StringUtils.hasText(oneToOne.mappedBy())) { + // // This must be a target MappedField + // targetMappedFields.add(nextMappedField); + // } + // else { + // // This must be a source MappedField + // sourceMappedFields.add((MappedFieldPerceroObject) + // nextMappedField); + // } + // } + if (manyToOne != null && !manyToOne.optional() - || - oneToOne != null && !oneToOne.optional()) { + || oneToOne != null && !oneToOne.optional()) { requiredFields.add(nextMappedField); - - MappedClass referencedMappedClass = mcm.getMappedClassByClassName(nextMappedField.getField().getType().getName()); + + MappedClass referencedMappedClass = mcm + .getMappedClassByClassName(nextMappedField + .getField().getType().getName()); if (!referencedMappedClass.fieldsInitialized) { referencedMappedClass.initializeFields(); } - - if (referencedMappedClass.getReadQuery() != null && StringUtils.hasText(referencedMappedClass.getReadQuery().getQuery())) { + + if (referencedMappedClass.getReadQuery() != null + && StringUtils.hasText(referencedMappedClass + .getReadQuery().getQuery())) { nextMappedField.setHasReadAccessRights(true); readAccessRightsFieldReferences.add(nextMappedField); } - + MappedField reverseMappedField = null; - + // Find the reverse field. - for(MappedField nextRefMappedField : referencedMappedClass.toManyFields) { + for (MappedField nextRefMappedField : referencedMappedClass.toManyFields) { if (nextRefMappedField instanceof MappedFieldList) { - OneToMany refOneToMany = nextRefMappedField.getField().getAnnotation(OneToMany.class); + OneToMany refOneToMany = nextRefMappedField + .getField().getAnnotation(OneToMany.class); if (refOneToMany == null) - refOneToMany = nextRefMappedField.getGetter().getAnnotation(OneToMany.class); - + refOneToMany = nextRefMappedField.getGetter() + .getAnnotation(OneToMany.class); + if (refOneToMany != null) { - -// Boolean inheritsFrom = inheritsFrom(this.clazz, refOneToMany.targetEntity()); -// if (inheritsFrom && nextMappedField.getField().getName().equals(refOneToMany.mappedBy())) { - if (this.clazz == refOneToMany.targetEntity() && nextMappedField.getField().getName().equals(refOneToMany.mappedBy())) { + + Boolean inheritsFrom = + inheritsFrom(this.clazz, + refOneToMany.targetEntity()); + if (inheritsFrom && + nextMappedField.getField().getName().equals(refOneToMany.mappedBy())) + { +// if (this.clazz == refOneToMany.targetEntity() +// && nextMappedField +// .getField() +// .getName() +// .equals(refOneToMany.mappedBy())) { // Found the referenced field. reverseMappedField = nextRefMappedField; break; } } - } - else if (nextRefMappedField instanceof MappedFieldPerceroObject) { - OneToOne refOneToOne = nextRefMappedField.getField().getAnnotation(OneToOne.class); + } else if (nextRefMappedField instanceof MappedFieldPerceroObject) { + OneToOne refOneToOne = nextRefMappedField + .getField().getAnnotation(OneToOne.class); if (refOneToOne == null) - refOneToOne = nextRefMappedField.getGetter().getAnnotation(OneToOne.class); - + refOneToOne = nextRefMappedField.getGetter() + .getAnnotation(OneToOne.class); + if (refOneToOne != null) { -// Boolean inheritsFrom = inheritsFrom(this.clazz, refOneToOne.targetEntity()); -// if (inheritsFrom && nextMappedField.getField().getName().equals(refOneToOne.mappedBy())) { - if (this.clazz == refOneToOne.targetEntity() && nextMappedField.getField().getName().equals(refOneToOne.mappedBy())) { + Boolean inheritsFrom = + inheritsFrom(this.clazz, + refOneToOne.targetEntity()); + if (inheritsFrom && + nextMappedField.getField().getName().equals(refOneToOne.mappedBy())) + { +// if (this.clazz == refOneToOne.targetEntity() +// && nextMappedField.getField().getName() +// .equals(refOneToOne.mappedBy())) { // Found the referenced field. reverseMappedField = nextRefMappedField; break; @@ -943,103 +1135,175 @@ else if (nextRefMappedField instanceof MappedFieldPerceroObject) { } } } - + if (reverseMappedField == null) { // Find the reverse field. - for(MappedField nextRefMappedField : referencedMappedClass.toOneFields) { + for (MappedField nextRefMappedField : referencedMappedClass.toOneFields) { if (nextRefMappedField instanceof MappedFieldPerceroObject) { - OneToOne refOneToOne = nextRefMappedField.getField().getAnnotation(OneToOne.class); + OneToOne refOneToOne = nextRefMappedField + .getField().getAnnotation( + OneToOne.class); if (refOneToOne == null) - refOneToOne = nextRefMappedField.getGetter().getAnnotation(OneToOne.class); - + refOneToOne = nextRefMappedField + .getGetter().getAnnotation( + OneToOne.class); + if (refOneToOne != null) { -// Boolean inheritsFrom = inheritsFrom(this.clazz, nextRefMappedField.getField().getType()); -// if (inheritsFrom && nextMappedField.getField().getName().equals(refOneToOne.mappedBy())) { - if (this.clazz == nextRefMappedField.getField().getType() && nextMappedField.getField().getName().equals(refOneToOne.mappedBy())) { - // Found the referenced field. - reverseMappedField = nextRefMappedField; - break; + if (StringUtils.hasText(refOneToOne.mappedBy())) { + Boolean inheritsFrom = inheritsFrom(this.clazz, + nextRefMappedField.getField().getType()); + if (inheritsFrom && + nextMappedField.getField().getName().equals(refOneToOne.mappedBy())) + { + // Found the referenced field. + reverseMappedField = nextRefMappedField; + break; + } + } + else { + if (this.clazz == nextRefMappedField + .getField().getType() + && nextMappedField + .getField() + .getName() + .equals(refOneToOne + .mappedBy())) { + // Found the referenced field. + reverseMappedField = nextRefMappedField; + break; + } } } } } } - - if (!referencedMappedClass.cascadeRemoveFieldReferences.keySet().contains(nextMappedField)) { + + if (!referencedMappedClass.cascadeRemoveFieldReferences + .keySet().contains(nextMappedField)) { if (reverseMappedField == null) { if (manyToOne != null) { - if (!manyToOne.targetEntity().getName().equalsIgnoreCase("void")) { + if (!manyToOne.targetEntity().getName() + .equalsIgnoreCase("void")) { System.out.println("IS THIS CORRECT?"); } } if (oneToOne != null) { - if (!oneToOne.targetEntity().getName().equalsIgnoreCase("void")) { + if (!oneToOne.targetEntity().getName() + .equalsIgnoreCase("void")) { System.out.println("IS THIS CORRECT?"); } } } - referencedMappedClass.cascadeRemoveFieldReferences.put(nextMappedField, reverseMappedField); + referencedMappedClass.cascadeRemoveFieldReferences.put( + nextMappedField, reverseMappedField); } - + nextMappedField.setReverseMappedField(reverseMappedField); - } else if (manyToOne != null && manyToOne.optional() - || - oneToOne != null && oneToOne.optional() && (oneToOne.mappedBy() == null || oneToOne.mappedBy().isEmpty()) - ) { - MappedClass referencedMappedClass = mcm.getMappedClassByClassName(nextMappedField.getField().getType().getName()); - - if (referencedMappedClass.getReadQuery() != null && StringUtils.hasText(referencedMappedClass.getReadQuery().getQuery())) - nextMappedField.setHasReadAccessRights(true); - + } else if (manyToOne != null + && manyToOne.optional() + || oneToOne != null + && oneToOne.optional() + && (oneToOne.mappedBy() == null || oneToOne.mappedBy() + .isEmpty())) { + MappedClass referencedMappedClass = mcm + .getMappedClassByClassName(nextMappedField + .getField().getType().getName()); + + if (referencedMappedClass.getReadQuery() != null + && StringUtils.hasText(referencedMappedClass + .getReadQuery().getQuery())) + nextMappedField.setHasReadAccessRights(true); + MappedField reverseMappedField = null; - + // Find the reverse field. - for(MappedField nextRefMappedField : referencedMappedClass.toManyFields) { + for (MappedField nextRefMappedField : referencedMappedClass.toManyFields) { if (nextRefMappedField instanceof MappedFieldList) { - OneToMany refOneToMany = nextRefMappedField.getField().getAnnotation(OneToMany.class); + OneToMany refOneToMany = nextRefMappedField + .getField().getAnnotation(OneToMany.class); if (refOneToMany == null) - refOneToMany = nextRefMappedField.getGetter().getAnnotation(OneToMany.class); - + refOneToMany = nextRefMappedField.getGetter() + .getAnnotation(OneToMany.class); + if (refOneToMany != null) { -// Boolean inheritsFrom = inheritsFrom(this.clazz, refOneToMany.targetEntity()); -// if (inheritsFrom && nextMappedField.getField().getName().equals(refOneToMany.mappedBy())) { - if (this.clazz == refOneToMany.targetEntity() && nextMappedField.getField().getName().equals(refOneToMany.mappedBy())) { + Boolean inheritsFrom = + inheritsFrom(this.clazz, + refOneToMany.targetEntity()); + if (inheritsFrom && + nextMappedField.getField().getName().equals(refOneToMany.mappedBy())) + { +// if (this.clazz == refOneToMany.targetEntity() +// && nextMappedField +// .getField() +// .getName() +// .equals(refOneToMany.mappedBy())) { // Found the referenced field. reverseMappedField = nextRefMappedField; break; } } - } - else if (nextRefMappedField instanceof MappedFieldPerceroObject) { - OneToOne refOneToOne = nextRefMappedField.getField().getAnnotation(OneToOne.class); + } else if (nextRefMappedField instanceof MappedFieldPerceroObject) { + OneToOne refOneToOne = nextRefMappedField + .getField().getAnnotation(OneToOne.class); if (refOneToOne == null) - refOneToOne = nextRefMappedField.getGetter().getAnnotation(OneToOne.class); - + refOneToOne = nextRefMappedField.getGetter() + .getAnnotation(OneToOne.class); + if (refOneToOne != null) { -// Boolean inheritsFrom = inheritsFrom(this.clazz, refOneToOne.targetEntity()); -// if (inheritsFrom && nextMappedField.getField().getName().equals(refOneToOne.mappedBy())) { - if (this.clazz == refOneToOne.targetEntity() && nextMappedField.getField().getName().equals(refOneToOne.mappedBy())) { - // Found the referenced field. - reverseMappedField = nextRefMappedField; - break; + if (StringUtils.hasText(refOneToOne.mappedBy())) { + Boolean inheritsFrom = inheritsFrom(this.clazz, + refOneToOne.targetEntity()); + if (inheritsFrom && nextMappedField.getField().getName().equals(refOneToOne.mappedBy())) + { + if (this.clazz == refOneToOne.targetEntity() + && nextMappedField.getField().getName() + .equals(refOneToOne.mappedBy())) { + // Found the referenced field. + reverseMappedField = nextRefMappedField; + break; + } + } + } + else { + if (this.clazz == refOneToOne.targetEntity() + && nextMappedField.getField().getName() + .equals(refOneToOne.mappedBy())) { + // Found the referenced field. + reverseMappedField = nextRefMappedField; + break; + } } } } } - + if (reverseMappedField == null) { // Find the reverse field. - for(MappedField nextRefMappedField : referencedMappedClass.toOneFields) { + for (MappedField nextRefMappedField : referencedMappedClass.toOneFields) { if (nextRefMappedField instanceof MappedFieldPerceroObject) { - OneToOne refOneToOne = nextRefMappedField.getField().getAnnotation(OneToOne.class); + OneToOne refOneToOne = nextRefMappedField + .getField().getAnnotation( + OneToOne.class); if (refOneToOne == null) - refOneToOne = nextRefMappedField.getGetter().getAnnotation(OneToOne.class); - + refOneToOne = nextRefMappedField + .getGetter().getAnnotation( + OneToOne.class); + if (refOneToOne != null) { -// Boolean inheritsFrom = inheritsFrom(this.clazz, nextRefMappedField.getField().getType()); -// if (inheritsFrom && nextMappedField.getField().getName().equals(refOneToOne.mappedBy())) { - if (this.clazz == nextRefMappedField.getField().getType() && nextMappedField.getField().getName().equals(refOneToOne.mappedBy())) { + // Boolean inheritsFrom = + // inheritsFrom(this.clazz, + // nextRefMappedField.getField().getType()); + // if (inheritsFrom && + // nextMappedField.getField().getName().equals(refOneToOne.mappedBy())) + // { + if (this.clazz == nextRefMappedField + .getField().getType() + && nextMappedField + .getField() + .getName() + .equals(refOneToOne + .mappedBy())) { // Found the referenced field. reverseMappedField = nextRefMappedField; break; @@ -1048,90 +1312,111 @@ else if (nextRefMappedField instanceof MappedFieldPerceroObject) { } } } - - if (!referencedMappedClass.nulledOnRemoveFieldReferences.keySet().contains(nextMappedField)) { + + if (!referencedMappedClass.nulledOnRemoveFieldReferences + .keySet().contains(nextMappedField)) { if (reverseMappedField == null) { if (manyToOne != null) { - if (!manyToOne.targetEntity().getName().equalsIgnoreCase("void")) { + if (!manyToOne.targetEntity().getName() + .equalsIgnoreCase("void")) { System.out.println("IS THIS CORRECT?"); } } if (oneToOne != null) { - if (!oneToOne.targetEntity().getName().equalsIgnoreCase("void")) { + if (!oneToOne.targetEntity().getName() + .equalsIgnoreCase("void")) { System.out.println("IS THIS CORRECT?"); } } } - referencedMappedClass.nulledOnRemoveFieldReferences.put(nextMappedField, reverseMappedField); + referencedMappedClass.nulledOnRemoveFieldReferences + .put(nextMappedField, reverseMappedField); readAccessRightsFieldReferences.add(nextMappedField); } nextMappedField.setReverseMappedField(reverseMappedField); } - - // Check to see if this has any RelationshipInterfaces that need to be addresed. - RelationshipInterface propInterface = (RelationshipInterface) nextMappedField.getField().getAnnotation(RelationshipInterface.class); + + // Check to see if this has any RelationshipInterfaces that need + // to be addresed. + RelationshipInterface propInterface = (RelationshipInterface) nextMappedField + .getField().getAnnotation(RelationshipInterface.class); if (propInterface != null) { - processMappedFieldRelationshipInterface(propInterface, nextMappedField); + processMappedFieldRelationshipInterface(propInterface, + nextMappedField); } - RelationshipInterfaces relationshipInterfaces = (RelationshipInterfaces) nextMappedField.getField().getAnnotation(RelationshipInterfaces.class); + RelationshipInterfaces relationshipInterfaces = (RelationshipInterfaces) nextMappedField + .getField().getAnnotation(RelationshipInterfaces.class); if (relationshipInterfaces != null) { - for(RelationshipInterface nextPropInterface : relationshipInterfaces.relationshipInterfaces()) { - processMappedFieldRelationshipInterface(nextPropInterface, nextMappedField); + for (RelationshipInterface nextPropInterface : relationshipInterfaces + .relationshipInterfaces()) { + processMappedFieldRelationshipInterface( + nextPropInterface, nextMappedField); } } } - + // Now check Read AccessRights on all toMany fields. Iterator itrToManyFields = toManyFields.iterator(); - while(itrToManyFields.hasNext()) { + while (itrToManyFields.hasNext()) { MappedField nextMappedField = itrToManyFields.next(); - ParameterizedType listType = (ParameterizedType) nextMappedField.getField().getGenericType(); - Class listClass = (Class) listType.getActualTypeArguments()[0]; - MappedClass referencedMappedClass = mcm.getMappedClassByClassName(listClass.getCanonicalName()); - if (referencedMappedClass.getReadQuery() != null && StringUtils.hasText(referencedMappedClass.getReadQuery().getQuery())) { + ParameterizedType listType = (ParameterizedType) nextMappedField + .getField().getGenericType(); + Class listClass = (Class) listType + .getActualTypeArguments()[0]; + MappedClass referencedMappedClass = mcm + .getMappedClassByClassName(listClass.getCanonicalName()); + if (referencedMappedClass.getReadQuery() != null + && StringUtils.hasText(referencedMappedClass + .getReadQuery().getQuery())) { nextMappedField.setHasReadAccessRights(true); readAccessRightsFieldReferences.add(nextMappedField); } } - } catch(Exception e) { + } catch (Exception e) { logger.error("Error parsing MappedClass " + this.className, e); } - + relationshipsInitialized = true; } - - private void processMappedFieldRelationshipInterface(RelationshipInterface relInterface, MappedField mappedField) { - if (relInterface == null || !StringUtils.hasText(relInterface.sourceVarName())) { - logger.warn("Invalid RelationshipInterface for " + className + "." + mappedField.getField().getName()); + + private void processMappedFieldRelationshipInterface( + RelationshipInterface relInterface, MappedField mappedField) { + if (relInterface == null + || !StringUtils.hasText(relInterface.sourceVarName())) { + logger.warn("Invalid RelationshipInterface for " + className + "." + + mappedField.getField().getName()); return; } - - EntityImplementation entityImpl = entityImplementations.get(relInterface.entityInterfaceClass()); + + EntityImplementation entityImpl = entityImplementations + .get(relInterface.entityInterfaceClass()); if (entityImpl != null) { - Iterator itrRelImpls = entityImpl.relationshipImplementations.iterator(); + Iterator itrRelImpls = entityImpl.relationshipImplementations + .iterator(); while (itrRelImpls.hasNext()) { RelationshipImplementation nextRelImpl = itrRelImpls.next(); - if (nextRelImpl.sourceVarName.equals(relInterface.sourceVarName())) { + if (nextRelImpl.sourceVarName.equals(relInterface + .sourceVarName())) { // Found relationship implementation. nextRelImpl.sourceMappedField = mappedField; return; } } - + // No valid RelationshipIntrerface found so create one. RelationshipImplementation relImpl = new RelationshipImplementation(); relImpl.entityImplementation = entityImpl; relImpl.sourceVarName = relInterface.sourceVarName(); relImpl.sourceMappedField = mappedField; entityImpl.relationshipImplementations.add(relImpl); - } - else { + } else { // Need to crearte EntityImplementation as well. entityImpl = new EntityImplementation(); - entityImpl.entityInterfaceClass = relInterface.entityInterfaceClass(); + entityImpl.entityInterfaceClass = relInterface + .entityInterfaceClass(); entityImpl.mappedClass = this; - + RelationshipImplementation relImpl = new RelationshipImplementation(); relImpl.entityImplementation = entityImpl; relImpl.sourceVarName = relInterface.sourceVarName(); @@ -1140,36 +1425,35 @@ private void processMappedFieldRelationshipInterface(RelationshipInterface relIn } /** - // Find the existing RelationshipImplementation and set its mappedField. - Iterator itrClasses = entityImplementations.keySet().iterator(); - while (itrClasses.hasNext()) { - Class nextInterfaceClazz = itrClasses.next(); - Iterator itrEntityImpls = entityImplementations.get(nextInterfaceClazz).iterator(); - while (itrEntityImpls.hasNext()) { - EntityImplementation nextEntityImpl = itrEntityImpls.next(); - Iterator itrRelImpls = nextEntityImpl.relationshipImplementations.iterator(); - while (itrRelImpls.hasNext()) { - RelationshipImplementation nextRelImpl = itrRelImpls.next(); - if (nextRelImpl.sourceVarName.equals(relInterface.sourceVarName())) { - // Found relationship implementation. - nextRelImpl.sourceMappedField = mappedField; - return; - } - } - } - } - - // No corresponding Relationship Implementation. - logger.warn("No RelationshipImplementation found on " + className + "." + mappedField.getField().getName() + " of type " + relInterface.sourceVarName());*/ + * // Find the existing RelationshipImplementation and set its + * mappedField. Iterator itrClasses = + * entityImplementations.keySet().iterator(); while + * (itrClasses.hasNext()) { Class nextInterfaceClazz = + * itrClasses.next(); Iterator itrEntityImpls = + * entityImplementations.get(nextInterfaceClazz).iterator(); while + * (itrEntityImpls.hasNext()) { EntityImplementation nextEntityImpl = + * itrEntityImpls.next(); Iterator + * itrRelImpls = nextEntityImpl.relationshipImplementations.iterator(); + * while (itrRelImpls.hasNext()) { RelationshipImplementation + * nextRelImpl = itrRelImpls.next(); if + * (nextRelImpl.sourceVarName.equals(relInterface.sourceVarName())) { // + * Found relationship implementation. nextRelImpl.sourceMappedField = + * mappedField; return; } } } } + * + * // No corresponding Relationship Implementation. + * logger.warn("No RelationshipImplementation found on " + className + + * "." + mappedField.getField().getName() + " of type " + + * relInterface.sourceVarName()); + */ } - + @SuppressWarnings("rawtypes") private boolean inheritsFrom(Class a, Class b) { if (a.equals(b)) return true; - + Class s = a.getSuperclass(); - while(true) { + while (true) { if (s != null && s.equals(b)) return true; else if (s == null || s.equals(Object.class)) @@ -1178,51 +1462,52 @@ else if (s == null || s.equals(Object.class)) s = s.getSuperclass(); } } - + @SuppressWarnings("rawtypes") private boolean implementsInterface(Class a, Class b) { if (a.equals(b)) return true; - + Class[] interfaces = a.getInterfaces(); - - for(Class nextInterface : interfaces) { + + for (Class nextInterface : interfaces) { if (nextInterface.equals(b)) return true; } - + Class s = a.getSuperclass(); if (s != null) return implementsInterface(s, b); - + return false; } - + @SuppressWarnings("rawtypes") public MappedClass getSuperMappedClass() { - IMappedClassManager mcm = MappedClassManagerFactory.getMappedClassManager(); - + IMappedClassManager mcm = MappedClassManagerFactory + .getMappedClassManager(); + try { Class clazz = MappedClass.forName(className); - MappedClass superMappedClass = mcm.getMappedClassByClassName(clazz.getSuperclass().getName()); + MappedClass superMappedClass = mcm.getMappedClassByClassName(clazz + .getSuperclass().getName()); return superMappedClass; - } catch(Exception e) { + } catch (Exception e) { return null; } } - + public MappedField getMappedFieldByName(String fieldName) { MappedField result = null; - for(MappedField nextMappedField : externalizableFields) { + for (MappedField nextMappedField : externalizableFields) { if (nextMappedField.getField().getName().equals(fieldName)) { result = nextMappedField; break; } } - + return result; } - public boolean isFieldRequired(String fieldName) { Iterator itr = requiredFields.iterator(); @@ -1233,7 +1518,7 @@ public boolean isFieldRequired(String fieldName) { } return false; } - + @Override public boolean equals(Object obj) { if (obj == null) @@ -1242,146 +1527,202 @@ else if (obj.getClass() != this.getClass()) return false; else { MappedClass mcObj = (MappedClass) obj; - if ((this.className == null || this.className.trim().length() == 0) && (mcObj.className == null || mcObj.className.trim().length() == 0)) + if ((this.className == null || this.className.trim().length() == 0) + && (mcObj.className == null || mcObj.className.trim() + .length() == 0)) return true; - else if ((this.className == null || this.className.trim().length() == 0) || (mcObj.className == null || mcObj.className.trim().length() == 0)) + else if ((this.className == null || this.className.trim().length() == 0) + || (mcObj.className == null || mcObj.className.trim() + .length() == 0)) return false; else return this.className.equals(mcObj.className); } } - - - - + /*************************************** * Helper Methods ***************************************/ - public Map getRelatedClassIdPairMappedFieldMap(IPerceroObject perceroObject, Boolean isShellObject) throws Exception { + public Map getRelatedClassIdPairMappedFieldMap( + IPerceroObject perceroObject, Boolean isShellObject) + throws Exception { Map results = new HashMap(); - - Iterator itrToOneFieldsToUpdate = toOneFields.iterator(); + + Iterator itrToOneFieldsToUpdate = toOneFields + .iterator(); while (itrToOneFieldsToUpdate.hasNext()) { - MappedFieldPerceroObject nextToOneField = itrToOneFieldsToUpdate.next(); - + MappedFieldPerceroObject nextToOneField = itrToOneFieldsToUpdate + .next(); + // If no reverse mapped field, then nothing to do. if (nextToOneField.getReverseMappedField() != null) { - - // If no PerceroObject, then we need to retrieve ALL objects of this type. + + // If no PerceroObject, then we need to retrieve ALL objects of + // this type. if (perceroObject == null) { - MappedClass reverseMappedClass = nextToOneField.getReverseMappedField().getMappedClass(); - IDataProvider reverseDataProvider = reverseMappedClass.getDataProvider(); - List relatedObjects = reverseDataProvider.getAllByName(reverseMappedClass.className, null, null, false, null); - Iterator itrRelatedObjects = relatedObjects.iterator(); + MappedClass reverseMappedClass = nextToOneField + .getReverseMappedField().getMappedClass(); + IDataProvider reverseDataProvider = reverseMappedClass + .getDataProvider(); + List relatedObjects = reverseDataProvider + .getAllByName(reverseMappedClass.className, null, + null, false, null); + Iterator itrRelatedObjects = relatedObjects + .iterator(); while (itrRelatedObjects.hasNext()) { - IPerceroObject nextRelatedObject = itrRelatedObjects.next(); - results.put(BaseDataObject.toClassIdPair(nextRelatedObject), nextToOneField.getReverseMappedField()); + IPerceroObject nextRelatedObject = itrRelatedObjects + .next(); + results.put( + BaseDataObject.toClassIdPair(nextRelatedObject), + nextToOneField.getReverseMappedField()); } - } - else if (isShellObject) { - // If this is a Shell Object, then we need to ask the IDataProvider to get the related objects. - List relatedObjects = getDataProvider().findAllRelatedObjects(perceroObject, nextToOneField, true, null); - Iterator itrRelatedObjects = relatedObjects.iterator(); + } else if (isShellObject) { + // If this is a Shell Object, then we need to ask the + // IDataProvider to get the related objects. + List relatedObjects = getDataProvider() + .findAllRelatedObjects(perceroObject, + nextToOneField, true, null); + Iterator itrRelatedObjects = relatedObjects + .iterator(); while (itrRelatedObjects.hasNext()) { - IPerceroObject nextRelatedObject = itrRelatedObjects.next(); - results.put(BaseDataObject.toClassIdPair(nextRelatedObject), nextToOneField.getReverseMappedField()); + IPerceroObject nextRelatedObject = itrRelatedObjects + .next(); + results.put( + BaseDataObject.toClassIdPair(nextRelatedObject), + nextToOneField.getReverseMappedField()); } - } - else { - IPerceroObject toOneObject = (IPerceroObject) nextToOneField.getGetter().invoke(perceroObject); + } else { + IPerceroObject toOneObject = (IPerceroObject) nextToOneField + .getGetter().invoke(perceroObject); if (toOneObject != null) { - results.put(BaseDataObject.toClassIdPair(toOneObject), nextToOneField.getReverseMappedField()); + results.put(BaseDataObject.toClassIdPair(toOneObject), + nextToOneField.getReverseMappedField()); } } } } - + Iterator itrToManyFieldsToUpdate = toManyFields.iterator(); while (itrToManyFieldsToUpdate.hasNext()) { MappedField nextToManyField = itrToManyFieldsToUpdate.next(); if (nextToManyField instanceof MappedFieldPerceroObject) { MappedFieldPerceroObject nextPerceroObjectField = (MappedFieldPerceroObject) nextToManyField; - + if (nextPerceroObjectField.getReverseMappedField() != null) { - // If no PerceroObject, then we need to retrieve ALL objects of this type. + // If no PerceroObject, then we need to retrieve ALL objects + // of this type. if (perceroObject == null) { - MappedClass reverseMappedClass = nextToManyField.getReverseMappedField().getMappedClass(); - IDataProvider reverseDataProvider = reverseMappedClass.getDataProvider(); - List relatedObjects = reverseDataProvider.getAllByName(reverseMappedClass.className, null, null, false, null); - Iterator itrRelatedObjects = relatedObjects.iterator(); + MappedClass reverseMappedClass = nextToManyField + .getReverseMappedField().getMappedClass(); + IDataProvider reverseDataProvider = reverseMappedClass + .getDataProvider(); + List relatedObjects = reverseDataProvider + .getAllByName(reverseMappedClass.className, + null, null, false, null); + Iterator itrRelatedObjects = relatedObjects + .iterator(); while (itrRelatedObjects.hasNext()) { - IPerceroObject nextRelatedObject = itrRelatedObjects.next(); - results.put(BaseDataObject.toClassIdPair(nextRelatedObject), nextToManyField.getReverseMappedField()); + IPerceroObject nextRelatedObject = itrRelatedObjects + .next(); + results.put(BaseDataObject + .toClassIdPair(nextRelatedObject), + nextToManyField.getReverseMappedField()); } - } - else if (isShellObject) { - // If this is a Shell Object, then we need to ask the IDataProvider to get the related objects. - List relatedObjects = getDataProvider().findAllRelatedObjects(perceroObject, nextToManyField, true, null); - Iterator itrRelatedObjects = relatedObjects.iterator(); + } else if (isShellObject) { + // If this is a Shell Object, then we need to ask the + // IDataProvider to get the related objects. + List relatedObjects = getDataProvider() + .findAllRelatedObjects(perceroObject, + nextToManyField, true, null); + Iterator itrRelatedObjects = relatedObjects + .iterator(); while (itrRelatedObjects.hasNext()) { - IPerceroObject nextRelatedObject = itrRelatedObjects.next(); - results.put(BaseDataObject.toClassIdPair(nextRelatedObject), nextToManyField.getReverseMappedField()); + IPerceroObject nextRelatedObject = itrRelatedObjects + .next(); + results.put(BaseDataObject + .toClassIdPair(nextRelatedObject), + nextToManyField.getReverseMappedField()); } - } - else { - IPerceroObject toOneObject = (IPerceroObject) nextPerceroObjectField.getGetter().invoke(perceroObject); + } else { + IPerceroObject toOneObject = (IPerceroObject) nextPerceroObjectField + .getGetter().invoke(perceroObject); if (toOneObject != null) { - results.put(BaseDataObject.toClassIdPair(toOneObject), nextPerceroObjectField.getReverseMappedField()); + results.put(BaseDataObject + .toClassIdPair(toOneObject), + nextPerceroObjectField + .getReverseMappedField()); } } } - } - else if (nextToManyField instanceof MappedFieldList) { + } else if (nextToManyField instanceof MappedFieldList) { MappedFieldList nextListField = (MappedFieldList) nextToManyField; - + if (nextListField.getReverseMappedField() != null) { - // If no PerceroObject, then we need to retrieve ALL objects of this type. + // If no PerceroObject, then we need to retrieve ALL objects + // of this type. if (perceroObject == null) { - MappedClass reverseMappedClass = nextToManyField.getReverseMappedField().getMappedClass(); - IDataProvider reverseDataProvider = reverseMappedClass.getDataProvider(); - List relatedObjects = reverseDataProvider.getAllByName(reverseMappedClass.className, null, null, false, null); - Iterator itrRelatedObjects = relatedObjects.iterator(); + MappedClass reverseMappedClass = nextToManyField + .getReverseMappedField().getMappedClass(); + IDataProvider reverseDataProvider = reverseMappedClass + .getDataProvider(); + List relatedObjects = reverseDataProvider + .getAllByName(reverseMappedClass.className, + null, null, false, null); + Iterator itrRelatedObjects = relatedObjects + .iterator(); while (itrRelatedObjects.hasNext()) { - IPerceroObject nextRelatedObject = itrRelatedObjects.next(); - results.put(BaseDataObject.toClassIdPair(nextRelatedObject), nextToManyField.getReverseMappedField()); + IPerceroObject nextRelatedObject = itrRelatedObjects + .next(); + results.put(BaseDataObject + .toClassIdPair(nextRelatedObject), + nextToManyField.getReverseMappedField()); } - } - else if (isShellObject) { - // If this is a Shell Object, then we need to ask the IDataProvider to get the related objects. - List relatedObjects = getDataProvider().findAllRelatedObjects(perceroObject, nextToManyField, true, null); - Iterator itrRelatedObjects = relatedObjects.iterator(); + } else if (isShellObject) { + // If this is a Shell Object, then we need to ask the + // IDataProvider to get the related objects. + List relatedObjects = getDataProvider() + .findAllRelatedObjects(perceroObject, + nextToManyField, true, null); + Iterator itrRelatedObjects = relatedObjects + .iterator(); while (itrRelatedObjects.hasNext()) { - IPerceroObject nextRelatedObject = itrRelatedObjects.next(); - results.put(BaseDataObject.toClassIdPair(nextRelatedObject), nextToManyField.getReverseMappedField()); + IPerceroObject nextRelatedObject = itrRelatedObjects + .next(); + results.put(BaseDataObject + .toClassIdPair(nextRelatedObject), + nextToManyField.getReverseMappedField()); } - } - else { - List listObjects = (List) nextListField.getGetter().invoke(perceroObject); + } else { + List listObjects = (List) nextListField + .getGetter().invoke(perceroObject); if (listObjects != null && !listObjects.isEmpty()) { - Iterator itrListObjects = listObjects.iterator(); + Iterator itrListObjects = listObjects + .iterator(); while (itrListObjects.hasNext()) { - IPerceroObject nextListObject = itrListObjects.next(); - results.put(BaseDataObject.toClassIdPair(nextListObject), nextListField.getReverseMappedField()); + IPerceroObject nextListObject = itrListObjects + .next(); + results.put(BaseDataObject + .toClassIdPair(nextListObject), + nextListField.getReverseMappedField()); } } } } } } - + return results; } - - - /*************************************** * Static Helper Methods ***************************************/ @SuppressWarnings("rawtypes") - private static Map CLASS_MAP = Collections.synchronizedMap(new HashMap()); + private static Map CLASS_MAP = Collections + .synchronizedMap(new HashMap()); + @SuppressWarnings("rawtypes") public static Class forName(String className) throws ClassNotFoundException { Class clazz = CLASS_MAP.get(className); @@ -1391,30 +1732,34 @@ public static Class forName(String className) throws ClassNotFoundException { CLASS_MAP.put(className, clazz); } } - + return clazz; } private static IMappedClassManager mcm = null; + @SuppressWarnings("rawtypes") - public static MappedClassMethodPair getFieldSetters(Class theClass, String theFieldName) { + public static MappedClassMethodPair getFieldSetters(Class theClass, + String theFieldName) { Method theMethod = null; Method[] theMethods = theClass.getMethods(); String theModifiedFieldName = theFieldName; if (theModifiedFieldName.indexOf("_") == 0) theModifiedFieldName = theModifiedFieldName.substring(1); - - for(Method nextMethod : theMethods) { - if (nextMethod.getName().equalsIgnoreCase("set" + theModifiedFieldName)) { + + for (Method nextMethod : theMethods) { + if (nextMethod.getName().equalsIgnoreCase( + "set" + theModifiedFieldName)) { theMethod = nextMethod; break; } } - + if (mcm == null) { mcm = MappedClassManagerFactory.getMappedClassManager(); } - MappedClass mc = mcm.getMappedClassByClassName(theClass.getCanonicalName()); + MappedClass mc = mcm.getMappedClassByClassName(theClass + .getCanonicalName()); MappedClassMethodPair result = new MappedClassMethodPair(mc, theMethod); return result; @@ -1423,13 +1768,11 @@ public static MappedClassMethodPair getFieldSetters(Class theClass, String theFi public static class MappedClassMethodPair { public Method method; public MappedClass mappedClass; - + public MappedClassMethodPair(MappedClass mappedClass, Method method) { this.mappedClass = mappedClass; this.method = method; } } - - } diff --git a/src/main/java/com/percero/agents/sync/metadata/MappedField.java b/src/main/java/com/percero/agents/sync/metadata/MappedField.java index 7f09ce9..726655b 100644 --- a/src/main/java/com/percero/agents/sync/metadata/MappedField.java +++ b/src/main/java/com/percero/agents/sync/metadata/MappedField.java @@ -25,7 +25,7 @@ public class MappedField { public List queries = new ArrayList(); private Boolean hasReadAccessRights = false; - private MappedField reverseMappedField = null; + protected MappedField reverseMappedField = null; public MappedField getReverseMappedField() { return reverseMappedField; } diff --git a/src/main/java/com/percero/agents/sync/metadata/MappedFieldList.java b/src/main/java/com/percero/agents/sync/metadata/MappedFieldList.java index 27d1587..f4d314a 100644 --- a/src/main/java/com/percero/agents/sync/metadata/MappedFieldList.java +++ b/src/main/java/com/percero/agents/sync/metadata/MappedFieldList.java @@ -13,6 +13,17 @@ import com.percero.framework.vo.IPerceroObject; public class MappedFieldList extends MappedField { + + private Class listClass = null; + public Class getListClass() { + return listClass; + } + + public void setListClass(Class listClass) { + this.listClass = listClass; + } + + @SuppressWarnings("rawtypes") @Override @@ -104,4 +115,5 @@ else if (nextValueB instanceof IPerceroObject) { return true; } } + } diff --git a/src/main/java/com/percero/agents/sync/metadata/MappedFieldPerceroObject.java b/src/main/java/com/percero/agents/sync/metadata/MappedFieldPerceroObject.java index 009aac9..a30598b 100644 --- a/src/main/java/com/percero/agents/sync/metadata/MappedFieldPerceroObject.java +++ b/src/main/java/com/percero/agents/sync/metadata/MappedFieldPerceroObject.java @@ -107,4 +107,28 @@ else if (valueA == null || valueB == null) return (valueA.getID().equals(valueB.getID()) && valueA.getClass().getCanonicalName().equals(valueB.getClass().getCanonicalName())); } + + @Override + public void setReverseMappedField(MappedField reverseMappedField) { + this.reverseMappedField = reverseMappedField; + + if (reverseMappedField != null) { + Class declaringClass = getField().getDeclaringClass(); + Class mappedDeclaringClass = this.getMappedClass().clazz.getSuperclass(); +// Class listClass = null; +// if (reverseMappedField instanceof MappedFieldList) { +// listClass = ((MappedFieldList) reverseMappedField).getListClass().getSuperclass(); +// } +// else { +// listClass = reverseMappedField.getField().getDeclaringClass(); +// } + + // if (this.reverseMappedField != null) { + if (declaringClass == mappedDeclaringClass) { + if(this.reverseMappedField.getReverseMappedField() == null) { + this.reverseMappedField.setReverseMappedField(this); + } + } + } + } } diff --git a/src/main/java/com/percero/agents/sync/services/SyncAgentService.java b/src/main/java/com/percero/agents/sync/services/SyncAgentService.java index d1876e7..2b21eb1 100644 --- a/src/main/java/com/percero/agents/sync/services/SyncAgentService.java +++ b/src/main/java/com/percero/agents/sync/services/SyncAgentService.java @@ -292,6 +292,7 @@ public PerceroList getAllByName(String className, Integer pageNu Object newInstance = theClass.newInstance(); if (newInstance instanceof IPerceroObject) { ((IPerceroObject) newInstance).setID("0"); + postGetHelper.postGetObject((IPerceroObject) newInstance, userId, clientId); } } @@ -480,7 +481,10 @@ public Object findUnique(Object theQueryObject, String clientId) throws Exceptio } if (result != null) { - postGetHelper.postGetObject(result, userId, clientId); + IPerceroObject postGetResult = postGetHelper.postGetObject(result, userId, clientId); + if (postGetResult != null) { + result = postGetResult; + } } return result; @@ -508,8 +512,9 @@ public Object searchByExample(Object theQueryObject, IDataProvider dataProvider = dataProviderManager.getDataProviderByName(mappedClass.dataProviderName); List result = dataProvider.findByExample((IPerceroObject) theQueryObject, excludeProperties, userId, false); - if (result != null && result.size() > 0) + if (result != null && result.size() > 0) { postGetHelper.postGetObject(result, userId, clientId); + } return result; } @@ -536,7 +541,10 @@ public Object findById(String aClassName, String anId, String clientId) throws E IPerceroObject result = dataProvider.findById(new ClassIDPair(anId, aClassName), userId); if (result != null) { - postGetHelper.postGetObject(result, userId, clientId); + IPerceroObject postGetResult = postGetHelper.postGetObject(result, userId, clientId); + if (postGetResult != null) { + result = postGetResult; + } } return result; diff --git a/src/main/resources/spring/changeWatcher-spring-config.xml b/src/main/resources/spring/changeWatcher-spring-config.xml index 8fd2755..82dffd1 100644 --- a/src/main/resources/spring/changeWatcher-spring-config.xml +++ b/src/main/resources/spring/changeWatcher-spring-config.xml @@ -45,6 +45,8 @@ + + @@ -108,6 +110,11 @@ + @@ -185,7 +192,6 @@ - hibernate.hbm2ddl.auto=update hibernate.show_sql=false @@ -264,9 +270,10 @@ - + hibernate.dialect=org.hibernate.dialect.Oracle10gDialect - hibernate.hbm2ddl.auto=update + + hibernate.show_sql=false hibernate.format_sql=false From 069bb8f041dd651f8717ac81935d269116cd2089 Mon Sep 17 00:00:00 2001 From: Jonathan Samples Date: Tue, 27 Oct 2015 07:31:46 -0700 Subject: [PATCH 045/105] Added more granular spring config files for the various ways to run it --- .../agents/sync/jobs/UpdateTablePoller.java | 2 +- .../sync/jobs/UpdateTableProcessReporter.java | 252 ++++++------ .../sync/jobs/UpdateTableProcessor.java | 4 +- src/main/resources/log4j.properties | 2 +- .../spring/basic_components_spring_config.xml | 294 ++++++++++++++ .../spring/changeWatcher-spring-config.xml | 286 +------------ .../spring/messageListener-spring-config.xml | 69 ++++ .../spring/percero-spring-config.xml | 376 +++--------------- .../spring/updateTable-spring-config.xml | 275 +------------ 9 files changed, 558 insertions(+), 1002 deletions(-) create mode 100644 src/main/resources/spring/basic_components_spring_config.xml create mode 100644 src/main/resources/spring/messageListener-spring-config.xml diff --git a/src/main/java/com/percero/agents/sync/jobs/UpdateTablePoller.java b/src/main/java/com/percero/agents/sync/jobs/UpdateTablePoller.java index 008bd73..ebf664c 100644 --- a/src/main/java/com/percero/agents/sync/jobs/UpdateTablePoller.java +++ b/src/main/java/com/percero/agents/sync/jobs/UpdateTablePoller.java @@ -51,7 +51,7 @@ public class UpdateTablePoller { @PostConstruct public void init(){ // Get the reporter going -// UpdateTableProcessReporter.getInstance(); + UpdateTableProcessReporter.getInstance(); } /** diff --git a/src/main/java/com/percero/agents/sync/jobs/UpdateTableProcessReporter.java b/src/main/java/com/percero/agents/sync/jobs/UpdateTableProcessReporter.java index a918b2e..1251283 100644 --- a/src/main/java/com/percero/agents/sync/jobs/UpdateTableProcessReporter.java +++ b/src/main/java/com/percero/agents/sync/jobs/UpdateTableProcessReporter.java @@ -1,125 +1,127 @@ -//package com.percero.agents.sync.jobs; -// -//import org.apache.log4j.Logger; -// -//import java.util.HashMap; -//import java.util.Map; -// -///** -// * Created by jonnysamps on 10/10/15. -// */ -//public class UpdateTableProcessReporter { -// -// private static Logger logger = Logger.getLogger(UpdateTableProcessReporter.class); -// private static final int WINDOW_LENGTH = 10000; -// private static final double WINDOW_LENGTH_SECONDS = roundToOnePlace(WINDOW_LENGTH/1000.0); -// -// private static UpdateTableProcessReporter instance; -// -// private UTPReportStat totalStat = new UTPReportStat(); -// private Map stats = new HashMap<>(); -// -// private UpdateTableProcessReporter(){ -// new Thread(this.windowTimer).start(); // Start the reporting thread -// } -// -// public static UpdateTableProcessReporter getInstance(){ -// if(instance == null){ -// instance = new UpdateTableProcessReporter(); -// } -// return instance; -// } -// -// public void submitCountAndTime(String key, int count, long time){ -// synchronized (this) { -// if(!stats.containsKey(key)) -// stats.put(key, new UTPReportStat()); -// -// totalStat.totalCount += count; -// totalStat.totalTime += time; -// totalStat.windowCount += count; -// totalStat.windowTime += time; -// stats.get(key).totalCount += count; -// stats.get(key).totalTime += time; -// stats.get(key).windowCount += count; -// stats.get(key).windowTime += time; -// } -// } -// -// private static double roundToOnePlace(double number){ -// number *= 10; -// int rounded = (int) number; -// number = rounded / 10.0; -// return number; -// } -// -// private void resetwindow(){ -// synchronized (this) { -// totalStat.windowCount = 0; -// totalStat.windowTime = 0; -// for(UTPReportStat stat : stats.values()){ -// stat.windowCount = 0; -// stat.windowTime = 0; -// } -// } -// } -// -// private void printStats(){ -// synchronized (this){ -// printStat("Totals", totalStat); -// for(String key : stats.keySet()) -// printStat(key, stats.get(key)); -// } -// } -// -// private static void printStat(String label, UTPReportStat stat){ -// double totalTimeSeconds = roundToOnePlace(stat.totalTime / 1000.0); -// double totalOpsPerSecond = roundToOnePlace(stat.totalCount / totalTimeSeconds); -// double windowTimeSeconds = roundToOnePlace(stat.windowTime/1000.0); -// double windowOpsPerSecond = roundToOnePlace(stat.windowCount/windowTimeSeconds); -// logger.debug( -// "UT: "+pad(20, label)+ -// " : Last "+pad(3,WINDOW_LENGTH_SECONDS+"")+"s" + -// "("+pad(5, stat.windowCount+"")+ -// ", "+pad(6, windowOpsPerSecond+"")+"/s) " + -// "Total("+pad(7, stat.totalCount+"")+", " + -// pad(6, totalOpsPerSecond+"")+"/s)" -// ); -// } -// -// private static String pad(int space, String content){ -// StringBuilder sb = new StringBuilder(); -// for(int i = 0; i < space-content.length(); i++){ -// sb.append(" "); -// } -// sb.append(content); -// return sb.toString(); -// } -// -// /** -// * Window timer will sleep for the window length, wake up print some stats, -// * reset the window counters and then sleep again. -// */ -// private Runnable windowTimer = new Runnable() { -// @Override -// public void run() { -// while(true){ -// try { -// Thread.sleep(WINDOW_LENGTH); -// instance.printStats(); -// instance.resetwindow(); -// }catch(InterruptedException e){ -// logger.error(e.getMessage(), e); -// break; -// } -// } -// } -// }; -// -// private class UTPReportStat{ -// public int totalCount = 0; -// public long totalTime = 0; -// public int windowCount = 0; -// public long windowTime = 0; -// } -//} +package com.percero.agents.sync.jobs; + +import org.apache.log4j.Logger; + +import java.util.HashMap; +import java.util.Map; + +/** + * Created by jonnysamps on 10/10/15. + */ +public class UpdateTableProcessReporter { + + private static Logger logger = Logger.getLogger(UpdateTableProcessReporter.class); + private static final int WINDOW_LENGTH = 10000; + private static final double WINDOW_LENGTH_SECONDS = roundToOnePlace(WINDOW_LENGTH/1000.0); + + private static UpdateTableProcessReporter instance; + + private UTPReportStat totalStat = new UTPReportStat(); + private Map stats = new HashMap<>(); + + private UpdateTableProcessReporter(){ + new Thread(this.windowTimer).start(); // Start the reporting thread + } + + public static UpdateTableProcessReporter getInstance(){ + if(instance == null){ + instance = new UpdateTableProcessReporter(); + } + return instance; + } + + public void submitCountAndTime(String key, int count, long time){ + synchronized (this) { + if(!stats.containsKey(key)) + stats.put(key, new UTPReportStat()); + + totalStat.totalCount += count; + totalStat.totalTime += time; + totalStat.windowCount += count; + totalStat.windowTime += time; + stats.get(key).totalCount += count; + stats.get(key).totalTime += time; + stats.get(key).windowCount += count; + stats.get(key).windowTime += time; + } + } + + private static double roundToOnePlace(double number){ + number *= 10; + int rounded = (int) number; + number = rounded / 10.0; + return number; + } + + private void resetwindow(){ + synchronized (this) { + totalStat.windowCount = 0; + totalStat.windowTime = 0; + for(UTPReportStat stat : stats.values()){ + stat.windowCount = 0; + stat.windowTime = 0; + } + } + } + + private void printStats(){ + if(logger.isDebugEnabled()) { + synchronized (this) { + printStat("Totals", totalStat); + for (String key : stats.keySet()) + printStat(key, stats.get(key)); + } + } + } + + private static void printStat(String label, UTPReportStat stat){ + double totalTimeSeconds = roundToOnePlace(stat.totalTime / 1000.0); + double totalOpsPerSecond = roundToOnePlace(stat.totalCount / totalTimeSeconds); + double windowTimeSeconds = roundToOnePlace(stat.windowTime/1000.0); + double windowOpsPerSecond = roundToOnePlace(stat.windowCount/windowTimeSeconds); + logger.debug( + "UT: " + pad(20, label) + + " : Last " + pad(3, WINDOW_LENGTH_SECONDS + "") + "s" + + "(" + pad(5, stat.windowCount + "") + + ", " + pad(6, windowOpsPerSecond + "") + "/s) " + + "Total(" + pad(7, stat.totalCount + "") + ", " + + pad(6, totalOpsPerSecond + "") + "/s)" + ); + } + + private static String pad(int space, String content){ + StringBuilder sb = new StringBuilder(); + for(int i = 0; i < space-content.length(); i++){ + sb.append(" "); + } + sb.append(content); + return sb.toString(); + } + + /** + * Window timer will sleep for the window length, wake up print some stats, + * reset the window counters and then sleep again. + */ + private Runnable windowTimer = new Runnable() { + @Override + public void run() { + while(true){ + try { + Thread.sleep(WINDOW_LENGTH); + instance.printStats(); + instance.resetwindow(); + }catch(InterruptedException e){ + logger.error(e.getMessage(), e); + break; + } + } + } + }; + + private class UTPReportStat{ + public int totalCount = 0; + public long totalTime = 0; + public int windowCount = 0; + public long windowTime = 0; + } +} diff --git a/src/main/java/com/percero/agents/sync/jobs/UpdateTableProcessor.java b/src/main/java/com/percero/agents/sync/jobs/UpdateTableProcessor.java index 55448db..7e656b4 100644 --- a/src/main/java/com/percero/agents/sync/jobs/UpdateTableProcessor.java +++ b/src/main/java/com/percero/agents/sync/jobs/UpdateTableProcessor.java @@ -129,8 +129,8 @@ public void run(){ deleteRows(successfulRows); Date endTime = new Date(); -// UpdateTableProcessReporter.getInstance() -// .submitCountAndTime(tableName, successfulRows.size(), endTime.getTime() - startTime.getTime()); + UpdateTableProcessReporter.getInstance() + .submitCountAndTime(tableName, successfulRows.size(), endTime.getTime() - startTime.getTime()); } for(String className : classNamesToUpdateReferences) { diff --git a/src/main/resources/log4j.properties b/src/main/resources/log4j.properties index 1962384..6c6c509 100644 --- a/src/main/resources/log4j.properties +++ b/src/main/resources/log4j.properties @@ -1,6 +1,6 @@ log4j.rootLogger=info, stdout -#log4j.category.com.percero.agents.sync.jobs=DEBUG +log4j.category.com.percero.agents.sync.jobs=INFO log4j.appender.stdout=org.apache.log4j.ConsoleAppender log4j.appender.stdout.layout=org.apache.log4j.PatternLayout diff --git a/src/main/resources/spring/basic_components_spring_config.xml b/src/main/resources/spring/basic_components_spring_config.xml new file mode 100644 index 0000000..13a90ae --- /dev/null +++ b/src/main/resources/spring/basic_components_spring_config.xml @@ -0,0 +1,294 @@ + + + + + + + + + + + + classpath*:*.properties + classpath*:properties/*.properties + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + hibernate.dialect=org.hibernate.dialect.MySQLDialect + + hibernate.hbm2ddl.auto=update + + hibernate.show_sql=false + hibernate.format_sql=false + + hibernate.connection.aggressive_release=true + hibernate.jdbc.batch_size=20 + hibernate.connection.autocommit=false + hibernate.enable_lazy_load_no_trans=true + + + + + com.percero.amqp + com.percero.agents.auth.vo + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + hibernate.dialect=org.hibernate.dialect.Oracle10gDialect + + + + hibernate.show_sql=false + hibernate.format_sql=false + + hibernate.connection.aggressive_release=true + hibernate.jdbc.batch_size=20 + hibernate.connection.autocommit=false + hibernate.connection.autoReconnect=true + hibernate.enable_lazy_load_no_trans=true + + + + + $pf{domain.packageToScan} + + + + + + + + + diff --git a/src/main/resources/spring/changeWatcher-spring-config.xml b/src/main/resources/spring/changeWatcher-spring-config.xml index 82dffd1..ea8a6bf 100644 --- a/src/main/resources/spring/changeWatcher-spring-config.xml +++ b/src/main/resources/spring/changeWatcher-spring-config.xml @@ -15,286 +15,12 @@ http://www.springframework.org/schema/aop http://www.springframework.org/schema/aop/spring-aop-3.0.xsd http://www.springframework.org/schema/cache http://www.springframework.org/schema/cache/spring-cache.xsd"> - - - - - - - - - classpath*:*.properties - classpath*:properties/*.properties - - - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - hibernate.dialect=org.hibernate.dialect.MySQLDialect - - hibernate.hbm2ddl.auto=update - - hibernate.show_sql=false - hibernate.format_sql=false - - hibernate.connection.aggressive_release=true - hibernate.jdbc.batch_size=20 - hibernate.connection.autocommit=false - hibernate.enable_lazy_load_no_trans=true - - - - - com.percero.amqp - com.percero.agents.auth.vo - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - hibernate.dialect=org.hibernate.dialect.Oracle10gDialect - - - - hibernate.show_sql=false - hibernate.format_sql=false - - hibernate.connection.aggressive_release=true - hibernate.jdbc.batch_size=20 - hibernate.connection.autocommit=false - hibernate.connection.autoReconnect=true - hibernate.enable_lazy_load_no_trans=true - - - - - $pf{domain.packageToScan} - - - - - - - + + + diff --git a/src/main/resources/spring/messageListener-spring-config.xml b/src/main/resources/spring/messageListener-spring-config.xml new file mode 100644 index 0000000..ac2c921 --- /dev/null +++ b/src/main/resources/spring/messageListener-spring-config.xml @@ -0,0 +1,69 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/main/resources/spring/percero-spring-config.xml b/src/main/resources/spring/percero-spring-config.xml index 07096da..4a748e2 100644 --- a/src/main/resources/spring/percero-spring-config.xml +++ b/src/main/resources/spring/percero-spring-config.xml @@ -15,329 +15,57 @@ http://www.springframework.org/schema/aop http://www.springframework.org/schema/aop/spring-aop-3.0.xsd http://www.springframework.org/schema/cache http://www.springframework.org/schema/cache/spring-cache.xsd"> - - - - - - - - - classpath*:*.properties - classpath*:properties/*.properties - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - hibernate.dialect=org.hibernate.dialect.MySQLDialect - - hibernate.hbm2ddl.auto=update - - hibernate.show_sql=false - hibernate.format_sql=false - - hibernate.connection.aggressive_release=true - hibernate.jdbc.batch_size=20 - hibernate.connection.autocommit=false - hibernate.enable_lazy_load_no_trans=true - - - - - com.percero.amqp - com.percero.agents.auth.vo - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - hibernate.dialect=org.hibernate.dialect.Oracle10gDialect - - - - hibernate.show_sql=false - hibernate.format_sql=false - - hibernate.connection.aggressive_release=true - hibernate.jdbc.batch_size=20 - hibernate.connection.autocommit=false - hibernate.connection.autoReconnect=true - hibernate.enable_lazy_load_no_trans=true - - - - - $pf{domain.packageToScan} - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/main/resources/spring/updateTable-spring-config.xml b/src/main/resources/spring/updateTable-spring-config.xml index 670a111..54cfe54 100644 --- a/src/main/resources/spring/updateTable-spring-config.xml +++ b/src/main/resources/spring/updateTable-spring-config.xml @@ -15,276 +15,13 @@ http://www.springframework.org/schema/aop http://www.springframework.org/schema/aop/spring-aop-3.0.xsd http://www.springframework.org/schema/cache http://www.springframework.org/schema/cache/spring-cache.xsd"> - - - - - - - - - classpath*:*.properties - classpath*:properties/*.properties - - - + - + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - hibernate.dialect=org.hibernate.dialect.MySQLDialect - - - hibernate.hbm2ddl.auto=update - - hibernate.show_sql=false - hibernate.format_sql=false - - hibernate.connection.aggressive_release=true - hibernate.jdbc.batch_size=20 - hibernate.connection.autocommit=false - hibernate.enable_lazy_load_no_trans=true - - - - - com.percero.amqp - com.percero.agents.auth.vo - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - hibernate.dialect=org.hibernate.dialect.Oracle10gDialect - hibernate.hbm2ddl.auto=update - - hibernate.show_sql=false - hibernate.format_sql=false - - hibernate.connection.aggressive_release=true - hibernate.jdbc.batch_size=20 - hibernate.connection.autocommit=false - hibernate.connection.autoReconnect=true - hibernate.enable_lazy_load_no_trans=true - - - - - $pf{domain.packageToScan} - - - - - - - + From 1523603c6cadf505f77d7afeca3267ed4add893b Mon Sep 17 00:00:00 2001 From: Jonathan Samples Date: Thu, 29 Oct 2015 07:24:56 -0700 Subject: [PATCH 046/105] Small change to updateTable spring config to declare the dummy queue --- src/main/resources/spring/updateTable-spring-config.xml | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/resources/spring/updateTable-spring-config.xml b/src/main/resources/spring/updateTable-spring-config.xml index 54cfe54..b8e5241 100644 --- a/src/main/resources/spring/updateTable-spring-config.xml +++ b/src/main/resources/spring/updateTable-spring-config.xml @@ -18,6 +18,7 @@ + From 0dc7b41aad566b4256a13b5e231adb61b8f3cf97 Mon Sep 17 00:00:00 2001 From: Collin Brown Date: Fri, 30 Oct 2015 09:35:33 -0700 Subject: [PATCH 047/105] Beginning removal of Hibernate --- pom.xml | 2 +- .../sync/cw/ChangeWatcherHelperFactory.java | 6 ++--- .../spring/percero-spring-config.xml | 25 +++++++++++-------- 3 files changed, 19 insertions(+), 14 deletions(-) diff --git a/pom.xml b/pom.xml index 9efce3f..d2d3189 100644 --- a/pom.xml +++ b/pom.xml @@ -7,7 +7,7 @@ org.activestack syncengine - 1.1.33-SNAPSHOT + 1.1.34-SNAPSHOT 3.2.4.RELEASE diff --git a/src/main/java/com/percero/agents/sync/cw/ChangeWatcherHelperFactory.java b/src/main/java/com/percero/agents/sync/cw/ChangeWatcherHelperFactory.java index 8d15e12..c34dde5 100644 --- a/src/main/java/com/percero/agents/sync/cw/ChangeWatcherHelperFactory.java +++ b/src/main/java/com/percero/agents/sync/cw/ChangeWatcherHelperFactory.java @@ -45,9 +45,8 @@ public void registerChangeWatcherHelper(String category, IChangeWatcherHelper ch private Map helperCache = new HashMap(); @SuppressWarnings({ "rawtypes", "unchecked" }) public IChangeWatcherHelper getHelper(String category) { - IChangeWatcherHelper helper = helperCache.get(category); - - if (helper == null) { + IChangeWatcherHelper helper = null; + if (!helperCache.containsKey(category)) { // Get the appropriate Helper Component. try { Class clazz = MappedClass.forName(category); @@ -81,6 +80,7 @@ public IChangeWatcherHelper getHelper(String category) { return null; } else { + helper = helperCache.get(category); return helper; } } diff --git a/src/main/resources/spring/percero-spring-config.xml b/src/main/resources/spring/percero-spring-config.xml index 07096da..ca22c23 100644 --- a/src/main/resources/spring/percero-spring-config.xml +++ b/src/main/resources/spring/percero-spring-config.xml @@ -254,7 +254,11 @@ - + + + + + @@ -287,7 +292,7 @@ - + + < ! - - these are C3P0 properties - - > - + name="debugUnreturnedConnectionStackTraces" value="true" /> - - > - + < ! - - hibernate.dialect=org.hibernate.dialect.MySQLDialect- - > hibernate.dialect=org.hibernate.dialect.Oracle10gDialect - - + < ! - -hibernate.hbm2ddl.auto=update- - > + < ! - - hibernate.ejb.naming_strategy" value="org.hibernate.cfg.ImprovedNamingStrategy"/> - - > hibernate.show_sql=false hibernate.format_sql=false - + < ! - - hibernate.current_session_context_class=org.hibernate.context.ThreadLocalSessionContext - - > hibernate.connection.aggressive_release=true hibernate.jdbc.batch_size=20 hibernate.connection.autocommit=false @@ -339,5 +344,5 @@ - +--> From f0a097b62cb6ef232efa4c6db5b623e4c7fd8a8f Mon Sep 17 00:00:00 2001 From: Collin Brown Date: Fri, 30 Oct 2015 09:40:36 -0700 Subject: [PATCH 048/105] whip removal of Hibernate --- pom.xml | 2 +- .../spring/basic_components_spring_config.xml | 23 +++++++++++-------- 2 files changed, 15 insertions(+), 10 deletions(-) diff --git a/pom.xml b/pom.xml index d2d3189..9f8fc0f 100644 --- a/pom.xml +++ b/pom.xml @@ -7,7 +7,7 @@ org.activestack syncengine - 1.1.34-SNAPSHOT + 1.1.35-SNAPSHOT 3.2.4.RELEASE diff --git a/src/main/resources/spring/basic_components_spring_config.xml b/src/main/resources/spring/basic_components_spring_config.xml index 13a90ae..a02ddfb 100644 --- a/src/main/resources/spring/basic_components_spring_config.xml +++ b/src/main/resources/spring/basic_components_spring_config.xml @@ -205,7 +205,10 @@ - + + + + @@ -238,7 +242,7 @@ - + + < ! - - these are C3P0 properties - - > - + name="debugUnreturnedConnectionStackTraces" value="true" /> - - > - + < ! - - hibernate.dialect=org.hibernate.dialect.MySQLDialect - - > hibernate.dialect=org.hibernate.dialect.Oracle10gDialect - - + < ! - -hibernate.hbm2ddl.auto=update- - > + < ! - - hibernate.ejb.naming_strategy" value="org.hibernate.cfg.ImprovedNamingStrategy"/> - - > hibernate.show_sql=false hibernate.format_sql=false - + < ! - - hibernate.current_session_context_class=org.hibernate.context.ThreadLocalSessionContext - - > hibernate.connection.aggressive_release=true hibernate.jdbc.batch_size=20 hibernate.connection.autocommit=false @@ -290,5 +294,6 @@ + --> From 8fcba69fb1862112f2a83588087e7d7a451098aa Mon Sep 17 00:00:00 2001 From: Collin Brown Date: Mon, 2 Nov 2015 09:19:36 -0800 Subject: [PATCH 049/105] Adding more debug log statements around auth. Throw an exception if more than one UserAccount is linked to a single User --- pom.xml | 2 +- .../agents/auth/helpers/AccountHelper.java | 35 ++++++++++++------- .../agents/auth/services/AuthService.java | 19 ++++++---- .../agents/auth/services/AuthService2.java | 28 ++++++++++----- 4 files changed, 56 insertions(+), 28 deletions(-) diff --git a/pom.xml b/pom.xml index 9f8fc0f..847d9eb 100644 --- a/pom.xml +++ b/pom.xml @@ -7,7 +7,7 @@ org.activestack syncengine - 1.1.35-SNAPSHOT + 1.1.36-SNAPSHOT 3.2.4.RELEASE diff --git a/src/main/java/com/percero/agents/auth/helpers/AccountHelper.java b/src/main/java/com/percero/agents/auth/helpers/AccountHelper.java index f14fc66..11e6d05 100644 --- a/src/main/java/com/percero/agents/auth/helpers/AccountHelper.java +++ b/src/main/java/com/percero/agents/auth/helpers/AccountHelper.java @@ -85,7 +85,9 @@ public Principal authenticateOAuth(String regAppKey, String svcOAuthKey, String } public Principal authenticateOAuth(String regAppKey, String svcOAuthKey, String userId, String userToken, String clientId, String clientType, String deviceId, Set existingClientIds) { - if (!StringUtils.hasText(clientType)) { + log.debug("[AccountHelper] Authenticating user " + userId + ", token " + userToken + ", client " + clientId + ", device " + deviceId + " " + (existingClientIds != null && !existingClientIds.isEmpty() ? "NO existing clients" : existingClientIds.size() + " existing client(s)")); + + if (!StringUtils.hasText(clientType)) { clientType = Client.NON_PERSISTENT_TYPE; } @@ -98,10 +100,9 @@ public Principal authenticateOAuth(String regAppKey, String svcOAuthKey, String } else { validated = authService.validateUserByToken(regAppKey, userId, userToken, clientId); } - log.debug("ValidateUserByToken Result: " + userId + " / " + userToken + " / " + clientId + " = " + (validated ? "true" : "false")); + log.debug("[AccountHelper] ValidateUserByToken Result: " + userId + " / " + userToken + " / " + clientId + " = " + (validated ? "true" : "false")); } catch (Exception e) { - log.error("Error validating User by Token", e); - e.printStackTrace(); + log.error("[AccountHelper] Error validating User by Token", e); validated = false; } @@ -112,7 +113,7 @@ public Principal authenticateOAuth(String regAppKey, String svcOAuthKey, String if (userAnchor == null) { // Something went wrong here. - throw new Error("Invalid UserAnchor object."); + throw new Error("[AccountHelper] Invalid UserAnchor object."); } List roleList = getUserRoles(userId); @@ -168,12 +169,13 @@ public IUserAnchor validateUser(String regAppKey, String userId, IAuthService au result = foundUserAnchor; } } catch (NonUniqueResultException nre) { - log.error("More than one UserAnchor objects found for userId " + userId, nre); + log.error("[AccountHelper] More than one UserAnchor objects found for userId " + userId, nre); } return result; } protected IUserAnchor handleUserAnchorNotFound(String regAppKey, String userId, IAuthService authService) { + log.debug("[AccountHelper] Handling UserAnchor NOT Found for user " + userId); // Get this person's email addresses from the AuthManager. List serviceUserList = authService.getServiceUsers(userId); IUserAnchor result = addOrUpdateUserAnchorFromServiceUserList(userId, serviceUserList, null); @@ -183,6 +185,7 @@ protected IUserAnchor handleUserAnchorNotFound(String regAppKey, String userId, } protected void handleUserAnchorFound(String regAppKey, String userId, IAuthService authService, IUserAnchor userAnchor) { + log.debug("[AccountHelper] Handling UserAnchor Found for user " + userId); // Get this person's email addresses from the AuthManager. List serviceUserList = authService.getServiceUsers(userId); @@ -366,7 +369,8 @@ protected PropertyInterface getEntityPropertyInterfaceAnnotation(Class entityInt }*/ protected IUserAnchor addOrUpdateUserAnchorFromServiceUserList(String userId, List serviceUserList, IUserAnchor result) { - try { + log.debug("[AccountHelper] Add/Update UserAnchor for user " + userId); + try { // TODO: why are we doing this everywhere? ManifestHelper.setManifest(manifest); @@ -434,8 +438,11 @@ protected IUserAnchor addOrUpdateUserAnchorFromServiceUserList(String userId, Li // Need to create a new IUserAnchor if (result == null) { - result = (IUserAnchor) eiUserAnchor.mappedClass.clazz.newInstance(); - ((IPerceroObject) result).setID(UUID.randomUUID().toString()); + String id = UUID.randomUUID().toString(); + log.debug("[AccountHelper] NO UserAnchor found for user " + userId + ", creating new UserAnchor with ID " + id); + + result = (IUserAnchor) eiUserAnchor.mappedClass.clazz.newInstance(); + ((IPerceroObject) result).setID(id); result.setUserId(userId); if (serviceUserList.size() > 0) { @@ -452,12 +459,14 @@ protected IUserAnchor addOrUpdateUserAnchorFromServiceUserList(String userId, Li } syncAgentService.systemCreateObject((IPerceroObject) result, null); + log.debug("[AccountHelper] UserAnchor " + id + " created"); } else if (result.getUserId() == null || result.getUserId().isEmpty()) { + log.debug("[AccountHelper] Valid UserAnchor found but UserId is NOT set, updating UserAnchor for user " + userId); result.setUserId(userId); - syncAgentService.systemPutObject((IPerceroObject) result, null, null, null, true); + log.debug("[AccountHelper] UserAnchor updated"); } for (IUserIdentifier nextUserIdentifier : identifiersToSave) { @@ -490,14 +499,16 @@ else if (result.getUserId() == null || result.getUserId().isEmpty()) { if (relImpl != null) { relImpl.sourceMappedField.getSetter().invoke(nextUserIdentifier, result); if (isNewObject) { + log.debug("[AccountHelper] Creating " + nextUserIdentifier.getClass().getCanonicalName() + ":" + ((IPerceroObject) nextUserIdentifier).getID() + " for user " + userId); syncAgentService.systemCreateObject((IPerceroObject) nextUserIdentifier, null); } else { - syncAgentService.systemPutObject((IPerceroObject) nextUserIdentifier, null, null, null, true); + log.debug("[AccountHelper] Updating " + nextUserIdentifier.getClass().getCanonicalName() + ":" + ((IPerceroObject) nextUserIdentifier).getID() + " for user " + userId); + syncAgentService.systemPutObject((IPerceroObject) nextUserIdentifier, null, null, null, true); } } else { - log.warn("No UserAnchor found for IUserIdentifier class " + nextUserIdentifier.getClass().getCanonicalName()); + log.warn("[AccountHelper] No UserAnchor found for IUserIdentifier class " + nextUserIdentifier.getClass().getCanonicalName()); } } } diff --git a/src/main/java/com/percero/agents/auth/services/AuthService.java b/src/main/java/com/percero/agents/auth/services/AuthService.java index e2b5735..71b09bf 100644 --- a/src/main/java/com/percero/agents/auth/services/AuthService.java +++ b/src/main/java/com/percero/agents/auth/services/AuthService.java @@ -721,6 +721,8 @@ else if (validUser) { */ // TODO: This function should also validate that the user is valid against the ServiceProvider's API. public boolean validateUserByToken(String regAppKey, String aUserId, String aToken, String aClientId) { + log.debug("[AuthService] Validating user " + aUserId + " by token " + aToken + ", client " + aClientId + " NO existing clients"); + boolean result = false; if (/*StringUtils.hasText(regAppKey) && */StringUtils.hasText(aUserId) && StringUtils.hasText(aToken)) { @@ -735,12 +737,15 @@ public boolean validateUserByToken(String regAppKey, String aUserId, String aTok Long uniqueResultCount = (Long) query.uniqueResult(); if (uniqueResultCount != null && uniqueResultCount > 0) { + if (uniqueResultCount > 1) { + log.error("[AuthService] " + uniqueResultCount + " UserTokens found for user " + aUserId + ", token " + aToken + ", client " + aClientId); + } result = true; } else - log.warn("Invalid User in validateUserByToken: User " + aUserId + ", Token " + aToken + ", Client " + aClientId); + log.warn("[AuthService] Invalid User in validateUserByToken: User " + aUserId + ", Token " + aToken + ", Client " + aClientId); } catch (Exception e) { - log.error("Unable to validateUserByToken", e); + log.error("[AuthService] Unable to validateUserByToken", e); result = false; } finally { if (s != null) @@ -757,6 +762,8 @@ public boolean validateUserByToken(String regAppKey, String aUserId, String aTok * @see com.com.percero.agents.auth.services.IAuthService#validateUserByToken(java.lang.String, java.lang.String, java.lang.String, java.lang.String, java.lang.String) */ public boolean validateUserByToken(String regAppKey, String aUserId, String aToken, String aClientId, Set existingClientIds) { + log.debug("[AuthService] Validating user " + aUserId + " by token " + aToken + ", client " + aClientId + " " + (existingClientIds != null && !existingClientIds.isEmpty() ? existingClientIds.size() + " existing clients" : " 0 existing clients")); + boolean result = false; if (/*StringUtils.hasText(regAppKey) && */StringUtils.hasText(aUserId) && StringUtils.hasText(aToken)) { @@ -783,21 +790,21 @@ public boolean validateUserByToken(String regAppKey, String aUserId, String aTok result = true; } else { - log.warn("Unable to update UserToken in validateUserByToken: User " + aUserId + ", Token " + aToken + ", Client " + aClientId); + log.warn("[AuthService] Unable to update UserToken in validateUserByToken: User " + aUserId + ", Token " + aToken + ", Client " + aClientId); } } else { - log.warn("Invalid User in validateUserByToken: User " + aUserId + ", Token " + aToken + ", Client " + aClientId); + log.warn("[AuthService] Invalid User in validateUserByToken: User " + aUserId + ", Token " + aToken + ", Client " + aClientId); } } catch (Exception e) { - log.error("Unable to validateUserByToken", e); + log.error("[AuthService] Unable to validateUserByToken", e); result = false; } finally { if (s != null) s.close(); } } else { - log.warn("Invalid User in validateUserByToken"); + log.warn("[AuthService] Invalid User in validateUserByToken"); } return result; diff --git a/src/main/java/com/percero/agents/auth/services/AuthService2.java b/src/main/java/com/percero/agents/auth/services/AuthService2.java index e5bdb34..2f0ded0 100644 --- a/src/main/java/com/percero/agents/auth/services/AuthService2.java +++ b/src/main/java/com/percero/agents/auth/services/AuthService2.java @@ -7,6 +7,7 @@ import com.percero.agents.auth.hibernate.AuthHibernateUtils; import com.percero.agents.auth.hibernate.BaseDataObjectPropertySelector; import com.percero.agents.auth.vo.*; +import com.percero.agents.sync.exceptions.SyncDataException; import com.percero.agents.sync.metadata.*; import com.percero.agents.sync.services.ISyncAgentService; import com.percero.framework.bl.IManifest; @@ -104,7 +105,7 @@ public AuthenticationResponse reauthenticate(ReauthenticationRequest request){ * @param serviceUser * @return */ - private UserAccount findUserAccount(ServiceUser serviceUser){ + private UserAccount findUserAccount(ServiceUser serviceUser) throws SyncDataException { UserAccount theFoundUserAccount = null; UserAccount theQueryObject = new UserAccount(); theQueryObject.setAccountId(serviceUser.getId()); @@ -118,11 +119,15 @@ private UserAccount findUserAccount(ServiceUser serviceUser){ List userAccounts = findByExample(theQueryObject, excludeProperties); - if ((userAccounts instanceof List) - && ((List) userAccounts).size() > 0) { - // Found a valid UserAccount. - List userAccountList = (List) userAccounts; - theFoundUserAccount = (UserAccount) userAccountList.get(0); + if ((userAccounts instanceof List)) { + List listUserAccounts = (List) userAccounts; + if (listUserAccounts.size() == 1) { + // Found a valid UserAccount. + theFoundUserAccount = (UserAccount) listUserAccounts.get(0); + } + else if (listUserAccounts.size() > 1) { + throw new SyncDataException(listUserAccounts.size() + " UserAccounts found for serviceUser " + serviceUser.getId(), 1001); + } } return theFoundUserAccount; @@ -133,7 +138,8 @@ private UserAccount findUserAccount(ServiceUser serviceUser){ * @param serviceUser * @return */ - private User findUser(ServiceUser serviceUser){ + @SuppressWarnings("unchecked") + private User findUser(ServiceUser serviceUser) throws SyncDataException { Session s = sessionFactoryAuth.openSession(); User theUser = null; @@ -151,9 +157,12 @@ private User findUser(ServiceUser serviceUser){ Query q = s.createQuery(strFindUserIdentifier); List userList = (List) q.list(); - if (userList.size() > 0) { + if (userList.size() == 1) { theUser = userList.get(0); } + else if (userList.size() > 1) { + throw new SyncDataException(userList.size() + " UserIdentifiers found for serviceUser " + serviceUser.getId(), 1001); + } } return theUser; @@ -165,7 +174,8 @@ private User findUser(ServiceUser serviceUser){ * @param serviceUser * @returns UserAccount */ - private UserAccount getOrCreateUserAccount(ServiceUser serviceUser, IAuthProvider provider, AuthenticationRequest request){ + @SuppressWarnings("unchecked") + private UserAccount getOrCreateUserAccount(ServiceUser serviceUser, IAuthProvider provider, AuthenticationRequest request){ UserAccount theFoundUserAccount = null; Session s = null; From 4b7ac99afd5feb8ea469f7d9a211ffa7c8ca783a Mon Sep 17 00:00:00 2001 From: Collin Brown Date: Mon, 2 Nov 2015 09:45:03 -0800 Subject: [PATCH 050/105] Fixed logging statement --- .../java/com/percero/agents/auth/helpers/AccountHelper.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/percero/agents/auth/helpers/AccountHelper.java b/src/main/java/com/percero/agents/auth/helpers/AccountHelper.java index 11e6d05..7febf4d 100644 --- a/src/main/java/com/percero/agents/auth/helpers/AccountHelper.java +++ b/src/main/java/com/percero/agents/auth/helpers/AccountHelper.java @@ -85,7 +85,7 @@ public Principal authenticateOAuth(String regAppKey, String svcOAuthKey, String } public Principal authenticateOAuth(String regAppKey, String svcOAuthKey, String userId, String userToken, String clientId, String clientType, String deviceId, Set existingClientIds) { - log.debug("[AccountHelper] Authenticating user " + userId + ", token " + userToken + ", client " + clientId + ", device " + deviceId + " " + (existingClientIds != null && !existingClientIds.isEmpty() ? "NO existing clients" : existingClientIds.size() + " existing client(s)")); + log.debug("[AccountHelper] Authenticating user " + userId + ", token " + userToken + ", client " + clientId + ", device " + deviceId + " " + (existingClientIds == null || existingClientIds.isEmpty() ? "NO existing clients" : existingClientIds.size() + " existing client(s)")); if (!StringUtils.hasText(clientType)) { clientType = Client.NON_PERSISTENT_TYPE; From 63c96755ecf161d7be632ff56c622b9c9eaeddb2 Mon Sep 17 00:00:00 2001 From: Collin Brown Date: Tue, 3 Nov 2015 17:19:33 -0800 Subject: [PATCH 051/105] Added changeWatcher queue to default spring config --- pom.xml | 2 +- src/main/resources/spring/percero-spring-config.xml | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index 847d9eb..b305a2f 100644 --- a/pom.xml +++ b/pom.xml @@ -7,7 +7,7 @@ org.activestack syncengine - 1.1.36-SNAPSHOT + 1.1.37-SNAPSHOT 3.2.4.RELEASE diff --git a/src/main/resources/spring/percero-spring-config.xml b/src/main/resources/spring/percero-spring-config.xml index 4a748e2..cf2710e 100644 --- a/src/main/resources/spring/percero-spring-config.xml +++ b/src/main/resources/spring/percero-spring-config.xml @@ -60,9 +60,10 @@ + - + From a137afc1b2a0a4e2b39c4fa5b3cdd76cf48f94bf Mon Sep 17 00:00:00 2001 From: Jonathan Samples Date: Tue, 3 Nov 2015 20:59:35 -0800 Subject: [PATCH 052/105] Made the yml file configurable --- .../agents/sync/jobs/UpdateTableRegistry.java | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/percero/agents/sync/jobs/UpdateTableRegistry.java b/src/main/java/com/percero/agents/sync/jobs/UpdateTableRegistry.java index cc58480..05e1dd5 100644 --- a/src/main/java/com/percero/agents/sync/jobs/UpdateTableRegistry.java +++ b/src/main/java/com/percero/agents/sync/jobs/UpdateTableRegistry.java @@ -2,8 +2,11 @@ import com.esotericsoftware.yamlbeans.YamlException; import com.esotericsoftware.yamlbeans.YamlReader; +import com.google.api.client.util.Value; import edu.emory.mathcs.backport.java.util.Collections; import org.apache.log4j.Logger; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.stereotype.Component; import javax.annotation.PostConstruct; @@ -46,6 +49,12 @@ public UpdateTableRegistry(){ public static UpdateTableRegistry getInstance(){ return instance; } + + @Autowired(required=false) @Qualifier("updateTableMapFile") + private String updateTableMapFile = "updateTableMap.yml"; + + @Autowired(required=false) @Qualifier("updateTableConfigFile") + private String updateTableConfigFile = "updateTables.yml"; @PostConstruct public void init() throws YamlException { @@ -55,7 +64,7 @@ public void init() throws YamlException { private void loadTableMappings() throws YamlException{ try{ - URL ymlUrl = UpdateTableRegistry.class.getClassLoader().getResource("updateTableMap.yml"); + URL ymlUrl = UpdateTableRegistry.class.getClassLoader().getResource(updateTableMapFile); if (ymlUrl == null) { logger.warn("No configuration found for UpdateTableMapping (updateTableMap.yml), skipping UpdateTables"); return; @@ -78,7 +87,7 @@ private void loadTableMappings() throws YamlException{ private void loadConnectionFactories() throws YamlException{ try { - URL ymlUrl = UpdateTableRegistry.class.getClassLoader().getResource("updateTables.yml"); + URL ymlUrl = UpdateTableRegistry.class.getClassLoader().getResource(updateTableConfigFile); if (ymlUrl == null) { logger.warn("No configuration found for UpdateTables (updateTables.yml), skipping UpdateTables"); return; From 9b2a168c03e5a0394c8302196d70e1a4c60b3652 Mon Sep 17 00:00:00 2001 From: Collin Brown Date: Thu, 5 Nov 2015 08:25:19 -0800 Subject: [PATCH 053/105] Use variables for update table lock_id and lock_date columns --- .../jobs/UpdateTableConnectionFactory.java | 26 +++++++++++++++---- .../sync/jobs/UpdateTableProcessor.java | 8 +++--- 2 files changed, 25 insertions(+), 9 deletions(-) diff --git a/src/main/java/com/percero/agents/sync/jobs/UpdateTableConnectionFactory.java b/src/main/java/com/percero/agents/sync/jobs/UpdateTableConnectionFactory.java index 01b5688..119bd0c 100644 --- a/src/main/java/com/percero/agents/sync/jobs/UpdateTableConnectionFactory.java +++ b/src/main/java/com/percero/agents/sync/jobs/UpdateTableConnectionFactory.java @@ -56,14 +56,14 @@ public String[] getTableNames() { } private String storedProcedureName; - private String storedProcedureDefinition; - public String getStoredProcedureName() { return storedProcedureName; } public void setStoredProcedureName(String storedProcedureName) { this.storedProcedureName = storedProcedureName; } + + private String storedProcedureDefinition; public String getStoredProcedureDefinition() { return storedProcedureDefinition; } @@ -71,9 +71,25 @@ public void setStoredProcedureDefinition(String storedProcedureDefinition) { this.storedProcedureDefinition = storedProcedureDefinition; } - private String updateStatementSql = "update :tableName set lock_id=:lockId, lock_date=NOW() " + - "where lock_id is null or " + - "lock_date < ':expireThreshold' " + + private String lockIdColumnName = "lock_id"; + public String getLockIdColumnName() { + return lockIdColumnName; + } + public void setLockIdColumnName(String lockIdColumnName) { + this.lockIdColumnName = lockIdColumnName; + } + + private String lockDateColumnName = "lock_date"; + public String getLockDateColumnName() { + return lockDateColumnName; + } + public void setLockDateColumnName(String lockDateColumnName) { + this.lockDateColumnName = lockDateColumnName; + } + + private String updateStatementSql = "update :tableName set " + getLockIdColumnName() + "=:lockId, " + getLockDateColumnName() + "=NOW() " + + "where " + getLockIdColumnName() + " is null or " + + getLockDateColumnName() + " < ':expireThreshold' " + "order by time_stamp limit :limit"; public String getUpdateStatementSql() { return updateStatementSql; diff --git a/src/main/java/com/percero/agents/sync/jobs/UpdateTableProcessor.java b/src/main/java/com/percero/agents/sync/jobs/UpdateTableProcessor.java index 7e656b4..cc6cd02 100644 --- a/src/main/java/com/percero/agents/sync/jobs/UpdateTableProcessor.java +++ b/src/main/java/com/percero/agents/sync/jobs/UpdateTableProcessor.java @@ -627,7 +627,7 @@ private List getUpdateSelectRow(int lockId, DateTime expireThres // Found a row to process if (numUpdated > 0) { - sql = "select * from :tableName where lock_id=:lockId"; + sql = "select * from :tableName where " + connectionFactory.getLockIdColumnName() + "=:lockId"; sql = sql.replace(":tableName", tableName); sql = sql.replace(":lockId", lockId + ""); @@ -689,7 +689,7 @@ private List getStoredProcRow(int lockId, DateTime expireThresho } if (updateNum != null && updateNum > 0) { - String sql = "select * from :tableName where lock_id=:lock_id"; + String sql = "select * from :tableName where " + connectionFactory.getLockIdColumnName() + "=:lock_id"; sql = sql.replace(":tableName", tableName); sql = sql.replace(":lock_id", lockId+""); @@ -764,8 +764,8 @@ protected UpdateTableRow fromResultSet(ResultSet resultSet) throws SQLException{ row.ID = resultSet.getInt("ID"); row.tableName = resultSet.getString("tableName"); row.rowId = resultSet.getString("row_id"); - row.lockId = resultSet.getInt("lock_id"); - row.lockDate = resultSet.getDate("lock_date"); + row.lockId = resultSet.getInt(connectionFactory.getLockIdColumnName()); + row.lockDate = resultSet.getDate(connectionFactory.getLockDateColumnName()); try { row.type = UpdateTableRowType.valueOf(resultSet.getString("type")); } catch(IllegalArgumentException iae) { From afc67fa81905da0cd0ffe709f63c193114176585 Mon Sep 17 00:00:00 2001 From: Collin Brown Date: Thu, 5 Nov 2015 16:30:44 -0800 Subject: [PATCH 054/105] Version bump --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index b305a2f..de6dcb0 100644 --- a/pom.xml +++ b/pom.xml @@ -7,7 +7,7 @@ org.activestack syncengine - 1.1.37-SNAPSHOT + 1.1.38-SNAPSHOT 3.2.4.RELEASE From 58fad17850745a56e05f585863236f848a058c64 Mon Sep 17 00:00:00 2001 From: Collin Brown Date: Mon, 9 Nov 2015 08:34:02 -0800 Subject: [PATCH 055/105] Add hibernate.hbm2ddl.auto param to env.properties to allow for prod config --- pom.xml | 2 +- src/main/resources/env.properties.sample | 4 ++++ src/main/resources/spring/basic_components_spring_config.xml | 2 +- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index de6dcb0..d5c7f74 100644 --- a/pom.xml +++ b/pom.xml @@ -7,7 +7,7 @@ org.activestack syncengine - 1.1.38-SNAPSHOT + 1.1.39-SNAPSHOT 3.2.4.RELEASE diff --git a/src/main/resources/env.properties.sample b/src/main/resources/env.properties.sample index 5bed2db..102a49d 100644 --- a/src/main/resources/env.properties.sample +++ b/src/main/resources/env.properties.sample @@ -10,6 +10,10 @@ databaseAuth.driverClassName=com.mysql.jdbc.Driver databaseAuth.jdbcUrl=jdbc:mysql://localhost:3306/defaultAuthManager?autoReconnect=true databaseAuth.username=root databaseAuth.password=root +# Recommended setting for DEV/TESTING +databaseAuth.hibernate.hbm2ddl.auto=update +# Recommended setting for PROD +# databaseAuth.hibernate.hbm2ddl.auto=validate #Anonymous Properties anonAuth.enabled=false diff --git a/src/main/resources/spring/basic_components_spring_config.xml b/src/main/resources/spring/basic_components_spring_config.xml index a02ddfb..1a55e2d 100644 --- a/src/main/resources/spring/basic_components_spring_config.xml +++ b/src/main/resources/spring/basic_components_spring_config.xml @@ -186,7 +186,7 @@ - hibernate.hbm2ddl.auto=update + hibernate.hbm2ddl.auto=$pf{databaseAuth.hibernate.hbm2ddl.auto:update} hibernate.show_sql=false hibernate.format_sql=false From 4e76d4669ea10adbc392a6c98ea0e359accca3f4 Mon Sep 17 00:00:00 2001 From: Collin Brown Date: Tue, 10 Nov 2015 06:45:50 -0800 Subject: [PATCH 056/105] Updated Joda 2.1 -> 2.9 --- pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index d5c7f74..b152862 100644 --- a/pom.xml +++ b/pom.xml @@ -7,7 +7,7 @@ org.activestack syncengine - 1.1.39-SNAPSHOT + 1.1.40-SNAPSHOT 3.2.4.RELEASE @@ -246,7 +246,7 @@ joda-time joda-time - 2.1 + 2.9 com.google.apis From 70031c6b808f30d8353a8820143c533df5b3c147 Mon Sep 17 00:00:00 2001 From: Collin Brown Date: Tue, 10 Nov 2015 18:06:22 -0800 Subject: [PATCH 057/105] Adding redis cache set of class ID's in cache for each class name --- pom.xml | 2 +- .../agents/auth/services/AuthService2.java | 4 +- .../agents/sync/access/RedisKeyUtils.java | 8 + .../agents/sync/cache/CacheManager.java | 168 ++++++++----- .../sync/datastore/ICacheDataStore.java | 3 + .../sync/datastore/RedisCacheDataStore.java | 26 ++ .../jobs/UpdateTableConnectionFactory.java | 10 +- .../sync/jobs/UpdateTableProcessor.java | 40 ++-- .../agents/sync/services/DAODataProvider.java | 224 ++++++++++++------ 9 files changed, 333 insertions(+), 152 deletions(-) diff --git a/pom.xml b/pom.xml index b152862..f8f3f50 100644 --- a/pom.xml +++ b/pom.xml @@ -7,7 +7,7 @@ org.activestack syncengine - 1.1.40-SNAPSHOT + 1.1.41-SNAPSHOT 3.2.4.RELEASE diff --git a/src/main/java/com/percero/agents/auth/services/AuthService2.java b/src/main/java/com/percero/agents/auth/services/AuthService2.java index 2f0ded0..298648f 100644 --- a/src/main/java/com/percero/agents/auth/services/AuthService2.java +++ b/src/main/java/com/percero/agents/auth/services/AuthService2.java @@ -61,7 +61,7 @@ public AuthenticationResponse authenticate(AuthenticationRequest request) throws if(serviceUser != null) { logger.debug(provider.getID() + " Authentication success"); serviceUser.setAuthProviderID(provider.getID()); // Set the provider ID just in case the provider didn't - UserAccount userAccount = getOrCreateUserAccount(serviceUser, provider, request); + UserAccount userAccount = getOrCreateUserAccount(serviceUser, provider); ensureAnchorUserExists(serviceUser, userAccount.getUser()); UserToken userToken = loginUserAccount(userAccount, request.getClientId(), request.getDeviceId()); userToken = (UserToken) AuthHibernateUtils.cleanObject(userToken); @@ -175,7 +175,7 @@ else if (userList.size() > 1) { * @returns UserAccount */ @SuppressWarnings("unchecked") - private UserAccount getOrCreateUserAccount(ServiceUser serviceUser, IAuthProvider provider, AuthenticationRequest request){ + public UserAccount getOrCreateUserAccount(ServiceUser serviceUser, IAuthProvider provider){ UserAccount theFoundUserAccount = null; Session s = null; diff --git a/src/main/java/com/percero/agents/sync/access/RedisKeyUtils.java b/src/main/java/com/percero/agents/sync/access/RedisKeyUtils.java index 71e20ac..ffcd0e0 100644 --- a/src/main/java/com/percero/agents/sync/access/RedisKeyUtils.java +++ b/src/main/java/com/percero/agents/sync/access/RedisKeyUtils.java @@ -136,6 +136,10 @@ public static String dataRecord() { return "dr"; } + public static String changeWatcherCustom(String subName) { + return (new StringBuilder(INT_64).append(CHANGE_WATCHER_PREFIX).append("CUSTOM")).toString(); + } + public static String changeWatcherClass(String className, String classId) { return (new StringBuilder(INT_128).append(CHANGE_WATCHER_PREFIX).append(className).append(":").append(classId)).toString(); } @@ -220,6 +224,10 @@ public static String changeWatcherTimestamp(String changeWatcherId) { return (new StringBuilder(INT_128).append("cw:time:").append(changeWatcherId)).toString(); } + public static String classIds(String className) { + return (new StringBuilder(INT_64).append("ids:").append(className)).toString(); + } + public static String classIdPair(String className, String classId) { return (new StringBuilder(INT_128).append(className).append(":").append(classId)).toString(); } diff --git a/src/main/java/com/percero/agents/sync/cache/CacheManager.java b/src/main/java/com/percero/agents/sync/cache/CacheManager.java index 83f2cf1..f55c50b 100644 --- a/src/main/java/com/percero/agents/sync/cache/CacheManager.java +++ b/src/main/java/com/percero/agents/sync/cache/CacheManager.java @@ -1,8 +1,20 @@ package com.percero.agents.sync.cache; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import org.apache.log4j.Logger; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + import com.percero.agents.sync.access.RedisKeyUtils; import com.percero.agents.sync.datastore.ICacheDataStore; -import com.percero.agents.sync.hibernate.SyncHibernateUtils; import com.percero.agents.sync.metadata.IMappedClassManager; import com.percero.agents.sync.metadata.MappedClass; import com.percero.agents.sync.metadata.MappedClassManagerFactory; @@ -13,13 +25,6 @@ import com.percero.agents.sync.vo.BaseDataObject; import com.percero.agents.sync.vo.ClassIDPair; import com.percero.framework.vo.IPerceroObject; -import org.apache.log4j.Logger; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Component; -import sun.misc.Cache; - -import java.lang.reflect.InvocationTargetException; -import java.util.*; /** * Created by jonnysamps on 9/5/15. @@ -44,6 +49,8 @@ public void updateCachedObject(IPerceroObject perceroObject, Map pairsToDelete = new ArrayList(); // Iterate through each changed object and reset the cache for that object. if (changedFields != null) { @@ -58,23 +65,18 @@ public void updateCachedObject(IPerceroObject perceroObject, Map itrChangedFieldKeyset = changedFields.keySet().iterator(); - Set keysToDelete = new HashSet(); while (itrChangedFieldKeyset.hasNext()) { ClassIDPair thePair = itrChangedFieldKeyset.next(); if (!thePair.comparePerceroObject(perceroObject)) { - String nextKey = RedisKeyUtils.classIdPair(thePair.getClassName(), thePair.getID()); - keysToDelete.add(nextKey); + pairsToDelete.add(thePair); +// String nextKey = RedisKeyUtils.classIdPair(thePair.getClassName(), thePair.getID()); } } - - if (!keysToDelete.isEmpty()) { - cacheDataStore.deleteKeys(keysToDelete); - // TODO: Do we simply delete the key? Or do we refetch the object here and update the key? - //redisDataStore.setValue(nextKey, ((BaseDataObject)perceroObject).toJson()); - } } else { - // No changedFields? We should never get here? + // No changedFields -> VERY INEFFICIENT. + // This is typically only reached when using UpdateTables + // that have update records that are NOT in the cache. IMappedClassManager mcm = MappedClassManagerFactory.getMappedClassManager(); MappedClass mappedClass = mcm.getMappedClassByClassName(perceroObject.getClass().getName()); Iterator itrToManyFields = mappedClass.toManyFields.iterator(); @@ -83,24 +85,26 @@ public void updateCachedObject(IPerceroObject perceroObject, Map itrFieldObject = ((Collection) fieldObject).iterator(); while(itrFieldObject.hasNext()) { Object nextListObject = itrFieldObject.next(); if (nextListObject instanceof IPerceroObject) { - String nextKey = RedisKeyUtils.classIdPair(nextListObject.getClass().getCanonicalName(), ((IPerceroObject)nextListObject).getID()); - if (cacheDataStore.hasKey(nextKey)) { - cacheDataStore.deleteKey(nextKey); - // TODO: Do we simply delete the key? Or do we refetch the object here and update the key? - //redisDataStore.setValue(nextKey, ((BaseDataObject)perceroObject).toJson()); - } + pairsToDelete.add(BaseDataObject.toClassIdPair((IPerceroObject) nextListObject)); +// String nextKey = RedisKeyUtils.classIdPair(nextListObject.getClass().getCanonicalName(), ((IPerceroObject)nextListObject).getID()); +// if (cacheDataStore.hasKey(nextKey)) { +// cacheDataStore.deleteKey(nextKey); +// // TODO: Do we simply delete the key? Or do we refetch the object here and update the key? +// //redisDataStore.setValue(nextKey, ((BaseDataObject)perceroObject).toJson()); +// } } } } @@ -112,30 +116,39 @@ else if (fieldObject instanceof Collection) { Object fieldObject = nextMappedField.getGetter().invoke(perceroObject); if (fieldObject != null) { if (fieldObject instanceof IPerceroObject) { - String nextKey = RedisKeyUtils.classIdPair(fieldObject.getClass().getCanonicalName(), ((IPerceroObject)fieldObject).getID()); - if (cacheDataStore.hasKey(nextKey)) { - cacheDataStore.deleteKey(nextKey); - // TODO: Do we simply delete the key? Or do we refetch the object here and update the key? - //redisDataStore.setValue(nextKey, ((BaseDataObject)perceroObject).toJson()); - } + pairsToDelete.add(BaseDataObject.toClassIdPair((IPerceroObject) fieldObject)); +// String nextKey = RedisKeyUtils.classIdPair(fieldObject.getClass().getCanonicalName(), ((IPerceroObject)fieldObject).getID()); +// if (cacheDataStore.hasKey(nextKey)) { +// cacheDataStore.deleteKey(nextKey); +// // TODO: Do we simply delete the key? Or do we refetch the object here and update the key? +// //redisDataStore.setValue(nextKey, ((BaseDataObject)perceroObject).toJson()); +// } } else if (fieldObject instanceof Collection) { Iterator itrFieldObject = ((Collection) fieldObject).iterator(); while(itrFieldObject.hasNext()) { Object nextListObject = itrFieldObject.next(); if (nextListObject instanceof IPerceroObject) { - String nextKey = RedisKeyUtils.classIdPair(nextListObject.getClass().getCanonicalName(), ((IPerceroObject)nextListObject).getID()); - if (cacheDataStore.hasKey(nextKey)) { - cacheDataStore.deleteKey(nextKey); - // TODO: Do we simply delete the key? Or do we refetch the object here and update the key? - //redisDataStore.setValue(nextKey, ((BaseDataObject)perceroObject).toJson()); - } + pairsToDelete.add(BaseDataObject.toClassIdPair((IPerceroObject) nextListObject)); +// String nextKey = RedisKeyUtils.classIdPair(nextListObject.getClass().getCanonicalName(), ((IPerceroObject)nextListObject).getID()); +// if (cacheDataStore.hasKey(nextKey)) { +// cacheDataStore.deleteKey(nextKey); +// // TODO: Do we simply delete the key? Or do we refetch the object here and update the key? +// //redisDataStore.setValue(nextKey, ((BaseDataObject)perceroObject).toJson()); +// } } } } } } } + + if (!pairsToDelete.isEmpty()) { + deleteObjectsFromRedisCache(pairsToDelete); +// cacheDataStore.deleteKeys(pairsToDelete); + // TODO: Do we simply delete the key? Or do we refetch the object here and update the key? + //redisDataStore.setValue(nextKey, ((BaseDataObject)perceroObject).toJson()); + } } } catch (Exception e){ logger.warn(e.getMessage(), e); @@ -145,23 +158,24 @@ else if (fieldObject instanceof Collection) { public void handleDeletedObject(IPerceroObject perceroObject, String className, Boolean isShellObject) { if (cacheTimeout > 0) { try { - Set keysToDelete = new HashSet(); + List pairsToDelete = new ArrayList(); - String key = RedisKeyUtils.classIdPair(className, perceroObject.getID()); - keysToDelete.add(key); +// String key = RedisKeyUtils.classIdPair(className, perceroObject.getID()); + pairsToDelete.add(BaseDataObject.toClassIdPair(perceroObject)); Set relatedClassIdPairs = getRelatedClassIdPairs(perceroObject, className, isShellObject); Iterator itrRelatedClassIdPairs = relatedClassIdPairs.iterator(); while (itrRelatedClassIdPairs.hasNext()) { ClassIDPair nextRelatedClassIdPair = itrRelatedClassIdPairs.next(); - String nextKey = RedisKeyUtils.classIdPair(nextRelatedClassIdPair.getClassName(), nextRelatedClassIdPair.getID()); - keysToDelete.add(nextKey); + pairsToDelete.add(nextRelatedClassIdPair); +// String nextKey = RedisKeyUtils.classIdPair(nextRelatedClassIdPair.getClassName(), nextRelatedClassIdPair.getID()); } - if (!keysToDelete.isEmpty()) { - cacheDataStore.deleteKeys(keysToDelete); - // TODO: Do we simply delete the key? Or do we refetch the object here and update the key? - //redisDataStore.setValue(nextKey, ((BaseDataObject)perceroObject).toJson()); + if (!pairsToDelete.isEmpty()) { + deleteObjectsFromRedisCache(pairsToDelete); +// cacheDataStore.deleteKeys(pairsToDelete); +// // TODO: Do we simply delete the key? Or do we refetch the object here and update the key? +// //redisDataStore.setValue(nextKey, ((BaseDataObject)perceroObject).toJson()); } } catch (Exception e) { logger.error(e.getMessage(), e); @@ -169,14 +183,17 @@ public void handleDeletedObject(IPerceroObject perceroObject, String className, } } - public void deleteObjectFromCache(ClassIDPair classIdPair) { - String key = RedisKeyUtils.classIdPair(classIdPair.getClassName(), classIdPair.getID()); - cacheDataStore.deleteKey(key); - } +// public void deleteObjectFromCache(ClassIDPair classIdPair) { +// String key = RedisKeyUtils.classIdPair(classIdPair.getClassName(), classIdPair.getID()); +// cacheDataStore.deleteKey(key); +// +// String classKey = RedisKeyUtils.classIds(classIdPair.getClassName()); +// cacheDataStore.removeSetValue(classKey, classIdPair.getID()); +// } @SuppressWarnings({ "unchecked", "rawtypes" }) - public Set getRelatedClassIdPairs(IPerceroObject perceroObject, String className, Boolean isShellObject) throws Exception { + private Set getRelatedClassIdPairs(IPerceroObject perceroObject, String className, Boolean isShellObject) throws Exception { Set results = new HashSet(); IMappedClassManager mcm = MappedClassManagerFactory.getMappedClassManager(); @@ -186,7 +203,8 @@ public Set getRelatedClassIdPairs(IPerceroObject perceroObject, Str while(itrToManyFields.hasNext()) { MappedField nextMappedField = itrToManyFields.next(); - // If this is a SHELL object, then we need to get ALL objects of this MappedField type. +// // If this is a SHELL object, then we need to get ALL objects of this MappedField type. + // We only care about Cached values, so if this is a shell object, then only retrieve object keys that are in the cache if (isShellObject) { // If NO reverse mapped field, then nothing to update. if (nextMappedField.getReverseMappedField() != null) { @@ -257,4 +275,40 @@ else if (fieldObject instanceof Collection) { return results; } + + public void deleteObjectFromCache(ClassIDPair pair) { + // Now put the object in the cache. + if (cacheTimeout > 0 && pair != null) { + String key = RedisKeyUtils.classIdPair(pair.getClassName(), pair.getID()); + cacheDataStore.deleteKey(key); + + String classKey = RedisKeyUtils.classIds(pair.getClassName()); + cacheDataStore.removeSetValue(classKey, pair.getID()); + } + } + + public void deleteObjectsFromRedisCache(List results) { + if (cacheTimeout > 0) { + Set objectStrings = new HashSet(results.size()); + Map> mapJsonClassIdStrings = new HashMap>(); + Iterator itrDatabaseObjects = results.iterator(); + while (itrDatabaseObjects.hasNext()) { + ClassIDPair nextDatabaseObject = itrDatabaseObjects.next(); + String nextCacheKey = RedisKeyUtils.classIdPair(nextDatabaseObject.getClassName(), nextDatabaseObject.getID()); + objectStrings.add(nextCacheKey); + + Set classIdList = mapJsonClassIdStrings.get(nextDatabaseObject.getClassName()); + if (classIdList == null) { + classIdList = new HashSet(); + mapJsonClassIdStrings.put(nextDatabaseObject.getClassName(), classIdList); + } + classIdList.add(nextDatabaseObject.getID()); + } + + // Store the objects in redis. + cacheDataStore.deleteKeys(objectStrings); + // Store the class Id's list in redis. + cacheDataStore.removeSetsValues(mapJsonClassIdStrings); + } + } } diff --git a/src/main/java/com/percero/agents/sync/datastore/ICacheDataStore.java b/src/main/java/com/percero/agents/sync/datastore/ICacheDataStore.java index ff2d7fe..ca3b1a6 100644 --- a/src/main/java/com/percero/agents/sync/datastore/ICacheDataStore.java +++ b/src/main/java/com/percero/agents/sync/datastore/ICacheDataStore.java @@ -161,6 +161,9 @@ Set getSetUnion(String key, @Transactional Long setSetValue(String key, Object value); + Long setSetsValues(Map> value); + Long removeSetsValues(Map> value); + SetOperations getSet(String key); RedisTemplate getTemplate(); diff --git a/src/main/java/com/percero/agents/sync/datastore/RedisCacheDataStore.java b/src/main/java/com/percero/agents/sync/datastore/RedisCacheDataStore.java index 5fbf1f9..e222820 100644 --- a/src/main/java/com/percero/agents/sync/datastore/RedisCacheDataStore.java +++ b/src/main/java/com/percero/agents/sync/datastore/RedisCacheDataStore.java @@ -588,6 +588,32 @@ public Long setSetValue(String key, Object value) { return template.opsForSet().add(key, value);// ? Long.valueOf(1) : Long.valueOf(0); } + @Override + @Transactional + public Long setSetsValues(Map> values) { + Long result = new Long(0); + Iterator>> itr = values.entrySet().iterator(); + while (itr.hasNext()) { + Entry> nextEntry = itr.next(); + result += template.opsForSet().add(nextEntry.getKey(), nextEntry.getValue().toArray());// ? Long.valueOf(1) : Long.valueOf(0); + + } + return result; + } + + @Override + @Transactional + public Long removeSetsValues(Map> values) { + Long result = new Long(0); + Iterator>> itr = values.entrySet().iterator(); + while (itr.hasNext()) { + Entry> nextEntry = itr.next(); + result += template.opsForSet().remove(nextEntry.getKey(), nextEntry.getValue().toArray());// ? Long.valueOf(1) : Long.valueOf(0); + + } + return result; + } + /* (non-Javadoc) * @see com.percero.agents.sync.datastore.IRedisCacheDataStore#getSet(java.lang.String) */ diff --git a/src/main/java/com/percero/agents/sync/jobs/UpdateTableConnectionFactory.java b/src/main/java/com/percero/agents/sync/jobs/UpdateTableConnectionFactory.java index 119bd0c..8576242 100644 --- a/src/main/java/com/percero/agents/sync/jobs/UpdateTableConnectionFactory.java +++ b/src/main/java/com/percero/agents/sync/jobs/UpdateTableConnectionFactory.java @@ -87,10 +87,18 @@ public void setLockDateColumnName(String lockDateColumnName) { this.lockDateColumnName = lockDateColumnName; } + private String timestampColumnName = "time_stamp"; + public String getTimestampColumnName() { + return timestampColumnName; + } + public void setTimestampColumnName(String timestampColumnName) { + this.timestampColumnName = timestampColumnName; + } + private String updateStatementSql = "update :tableName set " + getLockIdColumnName() + "=:lockId, " + getLockDateColumnName() + "=NOW() " + "where " + getLockIdColumnName() + " is null or " + getLockDateColumnName() + " < ':expireThreshold' " + - "order by time_stamp limit :limit"; + "order by " + getTimestampColumnName() + " limit :limit"; public String getUpdateStatementSql() { return updateStatementSql; } diff --git a/src/main/java/com/percero/agents/sync/jobs/UpdateTableProcessor.java b/src/main/java/com/percero/agents/sync/jobs/UpdateTableProcessor.java index cc6cd02..e06ef93 100644 --- a/src/main/java/com/percero/agents/sync/jobs/UpdateTableProcessor.java +++ b/src/main/java/com/percero/agents/sync/jobs/UpdateTableProcessor.java @@ -582,24 +582,31 @@ protected void updateReferences(String className){ public List getRows(int numRows){ List rows = null; - try { - Random rand = new Random(); + Random rand = new Random(); - int lockId = rand.nextInt(); - long now = System.currentTimeMillis(); + int lockId = rand.nextInt(); + long now = System.currentTimeMillis(); - DateTime expireThreshold = new DateTime(now - EXPIRATION_TIME); + DateTime expireThreshold = new DateTime(now - EXPIRATION_TIME); - if (StringUtils.hasText(connectionFactory.getStoredProcedureName())) { - try { - rows = getStoredProcRow(lockId, expireThreshold, numRows); - } catch (Exception e) { - rows = getUpdateSelectRow(lockId, expireThreshold, numRows); - } - } else { - rows = getUpdateSelectRow(lockId, expireThreshold, numRows); + if (StringUtils.hasText(connectionFactory.getStoredProcedureName())) { + try { + rows = getStoredProcRow(lockId, expireThreshold, numRows); + } catch (Exception e) { + logger.error("Error running Update Table stored procedure for " + connectionFactory.getJdbcUrl() + "\n Failing over to get UpdateTable with SELECT statement", e); + try { + rows = getUpdateSelectRow(lockId, expireThreshold, numRows); + } catch (SQLException e1) { + logger.error("Error running Update Table SELECT statement as backup to Stored Procedure\n UPDATE TABLE NON_FUNCTIONAL " + connectionFactory.getJdbcUrl(), e1); + } } - }catch(Exception e){} + } else { + try { + rows = getUpdateSelectRow(lockId, expireThreshold, numRows); + } catch (SQLException e1) { + logger.error("Error running Update Table SELECT statement as backup to Stored Procedure\n UPDATE TABLE NON_FUNCTIONAL " + connectionFactory.getJdbcUrl(), e1); + } + } return rows; @@ -656,7 +663,8 @@ private List getUpdateSelectRow(int lockId, DateTime expireThres private List getStoredProcRow(int lockId, DateTime expireThreshold, int numRows) throws SQLException { List list = new ArrayList<>(); Integer updateNum = 0; - try(CallableStatement cstmt = getConnection().prepareCall("{call " + connectionFactory.getStoredProcedureName() + "(?, ?, ?, ?)}")) { + String storedProcSql = connectionFactory.getStoredProcedureName(); + try(CallableStatement cstmt = getConnection().prepareCall("{call " + storedProcSql + "(?, ?, ?, ?)}")) { cstmt.setInt(1, lockId); cstmt.setString(2, expireThreshold.toString("Y-MM-dd HH:mm:ss")); cstmt.setInt(3, numRows); @@ -772,7 +780,7 @@ protected UpdateTableRow fromResultSet(ResultSet resultSet) throws SQLException{ logger.warn("Invalid UpdateTableRow TYPE, ignoring"); row.type = UpdateTableRowType.NONE; } - row.timestamp = resultSet.getDate("time_stamp"); + row.timestamp = resultSet.getDate(connectionFactory.getTimestampColumnName()); return row; } diff --git a/src/main/java/com/percero/agents/sync/services/DAODataProvider.java b/src/main/java/com/percero/agents/sync/services/DAODataProvider.java index 59e1bcc..4e7bebe 100644 --- a/src/main/java/com/percero/agents/sync/services/DAODataProvider.java +++ b/src/main/java/com/percero/agents/sync/services/DAODataProvider.java @@ -1,22 +1,24 @@ package com.percero.agents.sync.services; -import com.percero.agents.sync.access.RedisKeyUtils; -import com.percero.agents.sync.dao.DAORegistry; -import com.percero.agents.sync.dao.IDataAccessObject; -import com.percero.agents.sync.datastore.ICacheDataStore; -import com.percero.agents.sync.exceptions.SyncDataException; -import com.percero.agents.sync.exceptions.SyncException; -import com.percero.agents.sync.metadata.*; -import com.percero.agents.sync.vo.BaseDataObject; -import com.percero.agents.sync.vo.ClassIDPair; -import com.percero.agents.sync.vo.ClassIDPairs; -import com.percero.agents.sync.vo.IJsonObject; -import com.percero.framework.vo.IPerceroObject; -import com.percero.framework.vo.PerceroList; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.TimeUnit; + import javassist.ClassPool; import javassist.CtClass; import javassist.CtField; import javassist.CtMethod; + import org.apache.log4j.Logger; import org.codehaus.jackson.map.ObjectMapper; import org.hibernate.PropertyValueException; @@ -26,12 +28,24 @@ import org.springframework.stereotype.Component; import org.springframework.util.StringUtils; -import javax.annotation.PostConstruct; -import java.lang.reflect.InvocationTargetException; -import java.lang.reflect.Method; -import java.util.*; -import java.util.Map.Entry; -import java.util.concurrent.TimeUnit; +import com.percero.agents.sync.access.RedisKeyUtils; +import com.percero.agents.sync.dao.DAORegistry; +import com.percero.agents.sync.dao.IDataAccessObject; +import com.percero.agents.sync.datastore.ICacheDataStore; +import com.percero.agents.sync.exceptions.SyncDataException; +import com.percero.agents.sync.exceptions.SyncException; +import com.percero.agents.sync.metadata.IMappedClassManager; +import com.percero.agents.sync.metadata.MappedClass; +import com.percero.agents.sync.metadata.MappedClassManagerFactory; +import com.percero.agents.sync.metadata.MappedField; +import com.percero.agents.sync.metadata.MappedFieldList; +import com.percero.agents.sync.metadata.MappedFieldPerceroObject; +import com.percero.agents.sync.vo.BaseDataObject; +import com.percero.agents.sync.vo.ClassIDPair; +import com.percero.agents.sync.vo.ClassIDPairs; +import com.percero.agents.sync.vo.IJsonObject; +import com.percero.framework.vo.IPerceroObject; +import com.percero.framework.vo.PerceroList; @Component public class DAODataProvider implements IDataProvider { @@ -270,27 +284,76 @@ private void putObjectInRedisCache(IPerceroObject perceroObject) { String key = RedisKeyUtils.classIdPair(perceroObject.getClass().getCanonicalName(), perceroObject.getID()); cacheDataStore.setValue(key, ((BaseDataObject)perceroObject).toJson()); setObjectExpiration(key); + + String classKey = RedisKeyUtils.classIds(perceroObject.getClass().getCanonicalName()); + cacheDataStore.setSetValue(classKey, perceroObject.getID()); } } private void putObjectsInRedisCache(List results) { if (cacheTimeout > 0) { Map mapJsonObjectStrings = new HashMap(results.size()); + Map> mapJsonClassIdStrings = new HashMap>(); Iterator itrDatabaseObjects = results.iterator(); while (itrDatabaseObjects.hasNext()) { IPerceroObject nextDatabaseObject = itrDatabaseObjects.next(); String nextCacheKey = RedisKeyUtils.classIdPair(nextDatabaseObject.getClass().getCanonicalName(), nextDatabaseObject.getID()); + + Set classIdList = mapJsonClassIdStrings.get(nextDatabaseObject.getClass().getCanonicalName()); + if (classIdList == null) { + classIdList = new HashSet(); + mapJsonClassIdStrings.put(nextDatabaseObject.getClass().getCanonicalName(), classIdList); + } + classIdList.add(nextDatabaseObject.getID()); mapJsonObjectStrings.put(nextCacheKey, ((BaseDataObject)nextDatabaseObject).toJson()); } // Store the objects in redis. cacheDataStore.setValues(mapJsonObjectStrings); + // Store the class Id's list in redis. + cacheDataStore.setSetsValues(mapJsonClassIdStrings); // (Re)Set the expiration. cacheDataStore.expire(mapJsonObjectStrings.keySet(), cacheTimeout, TimeUnit.SECONDS); } } + private void deleteObjectFromRedisCache(ClassIDPair pair) { + // Now put the object in the cache. + if (cacheTimeout > 0 && pair != null) { + String key = RedisKeyUtils.classIdPair(pair.getClassName(), pair.getID()); + cacheDataStore.deleteKey(key); + + String classKey = RedisKeyUtils.classIds(pair.getClassName()); + cacheDataStore.removeSetValue(classKey, pair.getID()); + } + } + + private void deleteObjectsFromRedisCache(List results) { + if (cacheTimeout > 0) { + Set objectStrings = new HashSet(results.size()); + Map> mapJsonClassIdStrings = new HashMap>(); + Iterator itrDatabaseObjects = results.iterator(); + while (itrDatabaseObjects.hasNext()) { + ClassIDPair nextDatabaseObject = itrDatabaseObjects.next(); + String nextCacheKey = RedisKeyUtils.classIdPair(nextDatabaseObject.getClassName(), nextDatabaseObject.getID()); + objectStrings.add(nextCacheKey); + + Set classIdList = mapJsonClassIdStrings.get(nextDatabaseObject.getClassName()); + if (classIdList == null) { + classIdList = new HashSet(); + mapJsonClassIdStrings.put(nextDatabaseObject.getClassName(), classIdList); + } + classIdList.add(nextDatabaseObject.getID()); + } + + // Store the objects in redis. + cacheDataStore.deleteKeys(objectStrings); + // Store the class Id's list in redis. + cacheDataStore.removeSetsValues(mapJsonClassIdStrings); + } + } + private void setObjectExpiration(IPerceroObject perceroObject) { setObjectExpiration(RedisKeyUtils.classIdPair(perceroObject.getClass().getCanonicalName(), perceroObject.getID())); } @@ -657,23 +720,18 @@ public T putObject(T perceroObject, Map pairsToDelete = new ArrayList(); // Iterate through each changed object and reset the cache for that object. if (changedFields != null) { Iterator itrChangedFieldKeyset = changedFields.keySet().iterator(); - Set keysToDelete = new HashSet(); while (itrChangedFieldKeyset.hasNext()) { ClassIDPair thePair = itrChangedFieldKeyset.next(); if (!thePair.comparePerceroObject(perceroObject)) { - String nextKey = RedisKeyUtils.classIdPair(thePair.getClassName(), thePair.getID()); - keysToDelete.add(nextKey); + pairsToDelete.add(thePair); +// String nextKey = RedisKeyUtils.classIdPair(thePair.getClassName(), thePair.getID()); +// pairsToDelete.add(nextKey); } } - - if (!keysToDelete.isEmpty()) { - cacheDataStore.deleteKeys(keysToDelete); - // TODO: Do we simply delete the key? Or do we refetch the object here and update the key? - //redisDataStore.setValue(nextKey, ((BaseDataObject)perceroObject).toJson()); - } } else { // No changedFields? We should never get here? @@ -694,24 +752,26 @@ public T putObject(T perceroObject, Map itrFieldObject = ((Collection) fieldObject).iterator(); while(itrFieldObject.hasNext()) { Object nextListObject = itrFieldObject.next(); if (nextListObject instanceof IPerceroObject) { - String nextKey = RedisKeyUtils.classIdPair(nextListObject.getClass().getCanonicalName(), ((IPerceroObject)nextListObject).getID()); - if (cacheDataStore.hasKey(nextKey)) { - cacheDataStore.deleteKey(nextKey); - // TODO: Do we simply delete the key? Or do we refetch the object here and update the key? - //redisDataStore.setValue(nextKey, ((BaseDataObject)perceroObject).toJson()); - } + pairsToDelete.add(BaseDataObject.toClassIdPair((IPerceroObject) nextListObject)); +// String nextKey = RedisKeyUtils.classIdPair(nextListObject.getClass().getCanonicalName(), ((IPerceroObject)nextListObject).getID()); +// if (cacheDataStore.hasKey(nextKey)) { +// cacheDataStore.deleteKey(nextKey); +// // TODO: Do we simply delete the key? Or do we refetch the object here and update the key? +// //redisDataStore.setValue(nextKey, ((BaseDataObject)perceroObject).toJson()); +// } } } } @@ -732,30 +792,39 @@ else if (fieldObject instanceof Collection) { } if (fieldObject != null) { if (fieldObject instanceof IPerceroObject) { - String nextKey = RedisKeyUtils.classIdPair(fieldObject.getClass().getCanonicalName(), ((IPerceroObject)fieldObject).getID()); - if (cacheDataStore.hasKey(nextKey)) { - cacheDataStore.deleteKey(nextKey); - // TODO: Do we simply delete the key? Or do we refetch the object here and update the key? - //redisDataStore.setValue(nextKey, ((BaseDataObject)perceroObject).toJson()); - } + pairsToDelete.add(BaseDataObject.toClassIdPair((IPerceroObject) fieldObject)); +// String nextKey = RedisKeyUtils.classIdPair(fieldObject.getClass().getCanonicalName(), ((IPerceroObject)fieldObject).getID()); +// if (cacheDataStore.hasKey(nextKey)) { +// cacheDataStore.deleteKey(nextKey); +// // TODO: Do we simply delete the key? Or do we refetch the object here and update the key? +// //redisDataStore.setValue(nextKey, ((BaseDataObject)perceroObject).toJson()); +// } } else if (fieldObject instanceof Collection) { Iterator itrFieldObject = ((Collection) fieldObject).iterator(); while(itrFieldObject.hasNext()) { Object nextListObject = itrFieldObject.next(); if (nextListObject instanceof IPerceroObject) { - String nextKey = RedisKeyUtils.classIdPair(nextListObject.getClass().getCanonicalName(), ((IPerceroObject)nextListObject).getID()); - if (cacheDataStore.hasKey(nextKey)) { - cacheDataStore.deleteKey(nextKey); - // TODO: Do we simply delete the key? Or do we refetch the object here and update the key? - //redisDataStore.setValue(nextKey, ((BaseDataObject)perceroObject).toJson()); - } + pairsToDelete.add(BaseDataObject.toClassIdPair((IPerceroObject) nextListObject)); +// String nextKey = RedisKeyUtils.classIdPair(nextListObject.getClass().getCanonicalName(), ((IPerceroObject)nextListObject).getID()); +// if (cacheDataStore.hasKey(nextKey)) { +// cacheDataStore.deleteKey(nextKey); +// // TODO: Do we simply delete the key? Or do we refetch the object here and update the key? +// //redisDataStore.setValue(nextKey, ((BaseDataObject)perceroObject).toJson()); +// } } } } } } } + + if (!pairsToDelete.isEmpty()) { + deleteObjectsFromRedisCache(pairsToDelete); +// cacheDataStore.deleteKeys(pairsToDelete); + // TODO: Do we simply delete the key? Or do we refetch the object here and update the key? + //redisDataStore.setValue(nextKey, ((BaseDataObject)perceroObject).toJson()); + } } return (T) cleanObject(perceroObject, userId); @@ -783,10 +852,10 @@ public Boolean deleteObject(ClassIDPair theClassIdPair, String userId) throws Sy // Now update the cache. // TODO: Field-level updates could be REALLY useful here. Would avoid A TON of UNNECESSARY work... if (result && cacheTimeout > 0) { - Set keysToDelete = new HashSet(); + List objectsToDelete = new ArrayList(); - String key = RedisKeyUtils.classIdPair(perceroObject.getClass().getCanonicalName(), perceroObject.getID()); - keysToDelete.add(key); +// String key = RedisKeyUtils.classIdPair(perceroObject.getClass().getCanonicalName(), perceroObject.getID()); + objectsToDelete.add(BaseDataObject.toClassIdPair(perceroObject)); Iterator itrToManyFields = mappedClass.toManyFields.iterator(); while(itrToManyFields.hasNext()) { @@ -794,16 +863,18 @@ public Boolean deleteObject(ClassIDPair theClassIdPair, String userId) throws Sy Object fieldObject = nextMappedField.getGetter().invoke(perceroObject); if (fieldObject != null) { if (fieldObject instanceof IPerceroObject) { - String nextKey = RedisKeyUtils.classIdPair(fieldObject.getClass().getCanonicalName(), ((IPerceroObject)fieldObject).getID()); - keysToDelete.add(nextKey); + objectsToDelete.add(BaseDataObject.toClassIdPair((IPerceroObject)fieldObject)); +// String nextKey = RedisKeyUtils.classIdPair(fieldObject.getClass().getCanonicalName(), ((IPerceroObject)fieldObject).getID()); +// objectsToDelete.add(nextKey); } else if (fieldObject instanceof Collection) { Iterator itrFieldObject = ((Collection) fieldObject).iterator(); while(itrFieldObject.hasNext()) { Object nextListObject = itrFieldObject.next(); if (nextListObject instanceof IPerceroObject) { - String nextKey = RedisKeyUtils.classIdPair(nextListObject.getClass().getCanonicalName(), ((IPerceroObject)nextListObject).getID()); - keysToDelete.add(nextKey); + objectsToDelete.add(BaseDataObject.toClassIdPair((IPerceroObject) nextListObject)); +// String nextKey = RedisKeyUtils.classIdPair(nextListObject.getClass().getCanonicalName(), ((IPerceroObject)nextListObject).getID()); +// objectsToDelete.add(nextKey); } } } @@ -815,24 +886,27 @@ else if (fieldObject instanceof Collection) { Object fieldObject = nextMappedField.getGetter().invoke(perceroObject); if (fieldObject != null) { if (fieldObject instanceof IPerceroObject) { - String nextKey = RedisKeyUtils.classIdPair(fieldObject.getClass().getCanonicalName(), ((IPerceroObject)fieldObject).getID()); - keysToDelete.add(nextKey); + objectsToDelete.add(BaseDataObject.toClassIdPair((IPerceroObject) fieldObject)); +// String nextKey = RedisKeyUtils.classIdPair(fieldObject.getClass().getCanonicalName(), ((IPerceroObject)fieldObject).getID()); +// objectsToDelete.add(nextKey); } else if (fieldObject instanceof Collection) { Iterator itrFieldObject = ((Collection) fieldObject).iterator(); while(itrFieldObject.hasNext()) { Object nextListObject = itrFieldObject.next(); if (nextListObject instanceof IPerceroObject) { - String nextKey = RedisKeyUtils.classIdPair(nextListObject.getClass().getCanonicalName(), ((IPerceroObject)nextListObject).getID()); - keysToDelete.add(nextKey); + objectsToDelete.add(BaseDataObject.toClassIdPair((IPerceroObject) nextListObject)); +// String nextKey = RedisKeyUtils.classIdPair(nextListObject.getClass().getCanonicalName(), ((IPerceroObject)nextListObject).getID()); +// objectsToDelete.add(nextKey); } } } } } - if (!keysToDelete.isEmpty()) { - cacheDataStore.deleteKeys(keysToDelete); + if (!objectsToDelete.isEmpty()) { + deleteObjectsFromRedisCache(objectsToDelete); +// cacheDataStore.deleteKeys(objectsToDelete); } } @@ -1265,15 +1339,15 @@ public void populateToOneRelationships(IPerceroObject perceroObject, Boolean she } } - protected void addObjectToCache(IPerceroObject nextPerceroObject) { - String key = RedisKeyUtils.classIdPair(nextPerceroObject.getClass().getCanonicalName(), nextPerceroObject.getID()); - if (cacheTimeout > 0) - cacheDataStore.setValue(key, ((BaseDataObject)nextPerceroObject).toJson()); - - // (Re)Set the expiration. - if (cacheTimeout > 0 && key != null) { - cacheDataStore.expire(key, cacheTimeout, TimeUnit.SECONDS); - } - } +// protected void addObjectToCache(IPerceroObject nextPerceroObject) { +// String key = RedisKeyUtils.classIdPair(nextPerceroObject.getClass().getCanonicalName(), nextPerceroObject.getID()); +// if (cacheTimeout > 0) +// cacheDataStore.setValue(key, ((BaseDataObject)nextPerceroObject).toJson()); +// +// // (Re)Set the expiration. +// if (cacheTimeout > 0 && key != null) { +// cacheDataStore.expire(key, cacheTimeout, TimeUnit.SECONDS); +// } +// } } From 6809d95eb864c7247ac8f05066c6e0644cbde1d5 Mon Sep 17 00:00:00 2001 From: Collin Brown Date: Tue, 10 Nov 2015 20:42:02 -0800 Subject: [PATCH 058/105] wip - UpdateTable performance work --- .../agents/sync/cache/CacheManager.java | 54 +++++++++++++------ .../agents/sync/services/DAODataProvider.java | 8 +-- 2 files changed, 42 insertions(+), 20 deletions(-) diff --git a/src/main/java/com/percero/agents/sync/cache/CacheManager.java b/src/main/java/com/percero/agents/sync/cache/CacheManager.java index f55c50b..58944d3 100644 --- a/src/main/java/com/percero/agents/sync/cache/CacheManager.java +++ b/src/main/java/com/percero/agents/sync/cache/CacheManager.java @@ -22,6 +22,7 @@ import com.percero.agents.sync.metadata.MappedFieldPerceroObject; import com.percero.agents.sync.services.DataProviderManager; import com.percero.agents.sync.services.IDataProvider; +import com.percero.agents.sync.services.SyncAgentService; import com.percero.agents.sync.vo.BaseDataObject; import com.percero.agents.sync.vo.ClassIDPair; import com.percero.framework.vo.IPerceroObject; @@ -163,7 +164,7 @@ public void handleDeletedObject(IPerceroObject perceroObject, String className, // String key = RedisKeyUtils.classIdPair(className, perceroObject.getID()); pairsToDelete.add(BaseDataObject.toClassIdPair(perceroObject)); - Set relatedClassIdPairs = getRelatedClassIdPairs(perceroObject, className, isShellObject); + Set relatedClassIdPairs = getRelatedCachedClassIdPairs(perceroObject, className, isShellObject); Iterator itrRelatedClassIdPairs = relatedClassIdPairs.iterator(); while (itrRelatedClassIdPairs.hasNext()) { ClassIDPair nextRelatedClassIdPair = itrRelatedClassIdPairs.next(); @@ -193,7 +194,7 @@ public void handleDeletedObject(IPerceroObject perceroObject, String className, @SuppressWarnings({ "unchecked", "rawtypes" }) - private Set getRelatedClassIdPairs(IPerceroObject perceroObject, String className, Boolean isShellObject) throws Exception { + private Set getRelatedCachedClassIdPairs(IPerceroObject perceroObject, String className, Boolean isShellObject) throws Exception { Set results = new HashSet(); IMappedClassManager mcm = MappedClassManagerFactory.getMappedClassManager(); @@ -209,13 +210,24 @@ private Set getRelatedClassIdPairs(IPerceroObject perceroObject, St // If NO reverse mapped field, then nothing to update. if (nextMappedField.getReverseMappedField() != null) { MappedClass reverseMappedClass = nextMappedField.getReverseMappedField().getMappedClass(); - IDataProvider reverseDataProvider = DataProviderManager.getInstance().getDataProviderByName(reverseMappedClass.className); - Set allClassIdPairs = reverseDataProvider.getAllClassIdPairsByName(reverseMappedClass.className); - Iterator itrAllClassIdPairs = allClassIdPairs.iterator(); - while (itrAllClassIdPairs.hasNext()) { - ClassIDPair nextAllClassIdPair = itrAllClassIdPairs.next(); - results.add(nextAllClassIdPair); +// IDataProvider reverseDataProvider = DataProviderManager.getInstance().getDataProviderByName(reverseMappedClass.className); + + MappedClass nextReverseMappedClass = reverseMappedClass; + while (nextReverseMappedClass != null) { + String nextClassName = nextReverseMappedClass.className; + String key = RedisKeyUtils.classIds(nextClassName); + Set classIds = (Set) cacheDataStore.getSetValue(key); + for(String nextClassId : classIds) { + results.add(new ClassIDPair(nextClassId, nextClassName)); + } + nextReverseMappedClass = nextReverseMappedClass.parentMappedClass; } +// Set allClassIdPairs = reverseDataProvider.getAllClassIdPairsByName(reverseMappedClass.className); +// Iterator itrAllClassIdPairs = allClassIdPairs.iterator(); +// while (itrAllClassIdPairs.hasNext()) { +// ClassIDPair nextAllClassIdPair = itrAllClassIdPairs.next(); +// results.add(nextAllClassIdPair); +// } } } else { @@ -245,13 +257,23 @@ else if (fieldObject instanceof Collection) { // If NO reverse mapped field, then nothing to update. if (nextMappedField.getReverseMappedField() != null) { MappedClass reverseMappedClass = nextMappedField.getReverseMappedField().getMappedClass(); - IDataProvider reverseDataProvider = DataProviderManager.getInstance().getDataProviderByName(reverseMappedClass.className); - Set allClassIdPairs = reverseDataProvider.getAllClassIdPairsByName(reverseMappedClass.className); - Iterator itrAllClassIdPairs = allClassIdPairs.iterator(); - while (itrAllClassIdPairs.hasNext()) { - ClassIDPair nextAllClassIdPair = itrAllClassIdPairs.next(); - results.add(nextAllClassIdPair); + MappedClass nextReverseMappedClass = reverseMappedClass; + while (nextReverseMappedClass != null) { + String nextClassName = nextReverseMappedClass.className; + String key = RedisKeyUtils.classIds(nextClassName); + Set classIds = (Set) cacheDataStore.getSetValue(key); + for(String nextClassId : classIds) { + results.add(new ClassIDPair(nextClassId, nextClassName)); + } + nextReverseMappedClass = nextReverseMappedClass.parentMappedClass; } +// IDataProvider reverseDataProvider = DataProviderManager.getInstance().getDataProviderByName(reverseMappedClass.className); +// Set allClassIdPairs = reverseDataProvider.getAllClassIdPairsByName(reverseMappedClass.className); +// Iterator itrAllClassIdPairs = allClassIdPairs.iterator(); +// while (itrAllClassIdPairs.hasNext()) { +// ClassIDPair nextAllClassIdPair = itrAllClassIdPairs.next(); +// results.add(nextAllClassIdPair); +// } } } else { @@ -297,10 +319,10 @@ public void deleteObjectsFromRedisCache(List results) { String nextCacheKey = RedisKeyUtils.classIdPair(nextDatabaseObject.getClassName(), nextDatabaseObject.getID()); objectStrings.add(nextCacheKey); - Set classIdList = mapJsonClassIdStrings.get(nextDatabaseObject.getClassName()); + Set classIdList = mapJsonClassIdStrings.get(RedisKeyUtils.classIds(nextDatabaseObject.getClassName())); if (classIdList == null) { classIdList = new HashSet(); - mapJsonClassIdStrings.put(nextDatabaseObject.getClassName(), classIdList); + mapJsonClassIdStrings.put(RedisKeyUtils.classIds(nextDatabaseObject.getClassName()), classIdList); } classIdList.add(nextDatabaseObject.getID()); } diff --git a/src/main/java/com/percero/agents/sync/services/DAODataProvider.java b/src/main/java/com/percero/agents/sync/services/DAODataProvider.java index 4e7bebe..ae72f79 100644 --- a/src/main/java/com/percero/agents/sync/services/DAODataProvider.java +++ b/src/main/java/com/percero/agents/sync/services/DAODataProvider.java @@ -299,10 +299,10 @@ private void putObjectsInRedisCache(List results) { IPerceroObject nextDatabaseObject = itrDatabaseObjects.next(); String nextCacheKey = RedisKeyUtils.classIdPair(nextDatabaseObject.getClass().getCanonicalName(), nextDatabaseObject.getID()); - Set classIdList = mapJsonClassIdStrings.get(nextDatabaseObject.getClass().getCanonicalName()); + Set classIdList = mapJsonClassIdStrings.get(RedisKeyUtils.classIds(nextDatabaseObject.getClass().getCanonicalName())); if (classIdList == null) { classIdList = new HashSet(); - mapJsonClassIdStrings.put(nextDatabaseObject.getClass().getCanonicalName(), classIdList); + mapJsonClassIdStrings.put(RedisKeyUtils.classIds(nextDatabaseObject.getClass().getCanonicalName()), classIdList); } classIdList.add(nextDatabaseObject.getID()); @@ -339,10 +339,10 @@ private void deleteObjectsFromRedisCache(List results) { String nextCacheKey = RedisKeyUtils.classIdPair(nextDatabaseObject.getClassName(), nextDatabaseObject.getID()); objectStrings.add(nextCacheKey); - Set classIdList = mapJsonClassIdStrings.get(nextDatabaseObject.getClassName()); + Set classIdList = mapJsonClassIdStrings.get(RedisKeyUtils.classIds(nextDatabaseObject.getClassName())); if (classIdList == null) { classIdList = new HashSet(); - mapJsonClassIdStrings.put(nextDatabaseObject.getClassName(), classIdList); + mapJsonClassIdStrings.put(RedisKeyUtils.classIds(nextDatabaseObject.getClassName()), classIdList); } classIdList.add(nextDatabaseObject.getID()); } From b0dfb8a86dd31290a303b975bfd23b696061bd94 Mon Sep 17 00:00:00 2001 From: Collin Brown Date: Tue, 10 Nov 2015 20:43:24 -0800 Subject: [PATCH 059/105] Snapshot bump --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index f8f3f50..1d0ace5 100644 --- a/pom.xml +++ b/pom.xml @@ -7,7 +7,7 @@ org.activestack syncengine - 1.1.41-SNAPSHOT + 1.1.42-SNAPSHOT 3.2.4.RELEASE From 62b5ef57b82a385b38492cc423534d8df89c2580 Mon Sep 17 00:00:00 2001 From: Jonathan Samples Date: Tue, 10 Nov 2015 21:43:36 -0800 Subject: [PATCH 060/105] Better logging for update table stuff --- .../com/percero/agents/sync/jobs/UpdateTableRegistry.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/main/java/com/percero/agents/sync/jobs/UpdateTableRegistry.java b/src/main/java/com/percero/agents/sync/jobs/UpdateTableRegistry.java index 05e1dd5..b4012ae 100644 --- a/src/main/java/com/percero/agents/sync/jobs/UpdateTableRegistry.java +++ b/src/main/java/com/percero/agents/sync/jobs/UpdateTableRegistry.java @@ -66,7 +66,7 @@ private void loadTableMappings() throws YamlException{ try{ URL ymlUrl = UpdateTableRegistry.class.getClassLoader().getResource(updateTableMapFile); if (ymlUrl == null) { - logger.warn("No configuration found for UpdateTableMapping (updateTableMap.yml), skipping UpdateTables"); + logger.warn("No table mapping found for UpdateTables ("+updateTableMapFile+"), skipping UpdateTables"); return; } File configFile = new File(ymlUrl.getFile()); @@ -81,7 +81,7 @@ private void loadTableMappings() throws YamlException{ tableMap.put(updateTableMapping.tableName, updateTableMapping); } }catch (FileNotFoundException e) { - logger.warn("No configuration found for UpdateTables (updateTables.yml), skipping UpdateTables"); + logger.warn("No table mapping found for UpdateTables ("+updateTableMapFile+"), skipping UpdateTables"); } } @@ -89,7 +89,7 @@ private void loadConnectionFactories() throws YamlException{ try { URL ymlUrl = UpdateTableRegistry.class.getClassLoader().getResource(updateTableConfigFile); if (ymlUrl == null) { - logger.warn("No configuration found for UpdateTables (updateTables.yml), skipping UpdateTables"); + logger.warn("No configuration found for UpdateTables ("+updateTableConfigFile+"), skipping UpdateTables"); return; } File configFile = new File(ymlUrl.getFile()); @@ -110,7 +110,7 @@ private void loadConnectionFactories() throws YamlException{ } } catch (FileNotFoundException e) { - logger.warn("No configuration found for UpdateTables (updateTables.yml), skipping UpdateTables"); + logger.warn("No configuration found for UpdateTables ("+updateTableConfigFile+"), skipping UpdateTables"); } } From 55329e5d9e6c7b668b52abcc16c58c94c0c0e34f Mon Sep 17 00:00:00 2001 From: Collin Brown Date: Wed, 11 Nov 2015 07:12:50 -0800 Subject: [PATCH 061/105] Fix NPE in pushUpdate to clients --- pom.xml | 2 +- .../agents/sync/cw/DerivedValueChangeWatcherHelper.java | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index 1d0ace5..d27e99e 100644 --- a/pom.xml +++ b/pom.xml @@ -7,7 +7,7 @@ org.activestack syncengine - 1.1.42-SNAPSHOT + 1.1.43-SNAPSHOT 3.2.4.RELEASE diff --git a/src/main/java/com/percero/agents/sync/cw/DerivedValueChangeWatcherHelper.java b/src/main/java/com/percero/agents/sync/cw/DerivedValueChangeWatcherHelper.java index 5cfb97c..d28e7c4 100644 --- a/src/main/java/com/percero/agents/sync/cw/DerivedValueChangeWatcherHelper.java +++ b/src/main/java/com/percero/agents/sync/cw/DerivedValueChangeWatcherHelper.java @@ -306,6 +306,10 @@ protected Boolean validateReadAccess(ClassIDPair classIdPair, String userId) { protected void pushUpdate(ClassIDPair classIdPair, String fieldName, String[] params, Object update, Collection clientIds) { // Now push the result to all interested clients. + if (clientIds == null || clientIds.isEmpty()) { + return; + } + try { Iterator itrClientIds = clientIds.iterator(); while(itrClientIds.hasNext()) { @@ -320,7 +324,7 @@ protected void pushUpdate(ClassIDPair classIdPair, String fieldName, String[] pa public void pushChangeWatcherUpdate(Collection clientIds, ClassIDPair classIdPair, String fieldName, String[] params, Object value) { - if (clientIds != null && clientIds.size() > 0) { + if (clientIds != null && !clientIds.isEmpty()) { try { PushCWUpdateResponse pushUpdateResponse = new PushCWUpdateResponse(); pushUpdateResponse.setFieldName(fieldName); From 771e452177a3c8a4ae37450397f2b4090e744ee3 Mon Sep 17 00:00:00 2001 From: Collin Brown Date: Wed, 11 Nov 2015 17:27:22 -0800 Subject: [PATCH 062/105] Fix NPE if object is no longer in database --- pom.xml | 2 +- .../java/com/percero/agents/sync/services/DAODataProvider.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index d27e99e..76033b6 100644 --- a/pom.xml +++ b/pom.xml @@ -7,7 +7,7 @@ org.activestack syncengine - 1.1.43-SNAPSHOT + 1.1.44-SNAPSHOT 3.2.4.RELEASE diff --git a/src/main/java/com/percero/agents/sync/services/DAODataProvider.java b/src/main/java/com/percero/agents/sync/services/DAODataProvider.java index ae72f79..24ad39c 100644 --- a/src/main/java/com/percero/agents/sync/services/DAODataProvider.java +++ b/src/main/java/com/percero/agents/sync/services/DAODataProvider.java @@ -1113,7 +1113,7 @@ private Collection retrieveObjectsNotInCollection(Collection baseList, Collectio BaseDataObject nextCompareToPerceroObject = (BaseDataObject) itrCompareToList.next(); nextCompareToPerceroObject = (BaseDataObject) findById(BaseDataObject.toClassIdPair(nextCompareToPerceroObject), null); - if (nextBasePerceroObject.getClass() == nextCompareToPerceroObject.getClass() && nextBasePerceroObject.getID().equalsIgnoreCase(nextCompareToPerceroObject.getID())) { + if (nextCompareToPerceroObject != null && nextBasePerceroObject != null && nextBasePerceroObject.getClass() == nextCompareToPerceroObject.getClass() && nextBasePerceroObject.getID().equalsIgnoreCase(nextCompareToPerceroObject.getID())) { matchFound = true; break; } From 03a5030309aaed6a82a800bed464a764f2d3a0d7 Mon Sep 17 00:00:00 2001 From: Collin Brown Date: Fri, 13 Nov 2015 16:06:08 -0800 Subject: [PATCH 063/105] Optimization for UpdateTable LOCK Table --- pom.xml | 2 +- .../sync/jobs/UpdateTableProcessor.java | 118 +++++++++--------- 2 files changed, 60 insertions(+), 60 deletions(-) diff --git a/pom.xml b/pom.xml index 76033b6..30cbe5a 100644 --- a/pom.xml +++ b/pom.xml @@ -7,7 +7,7 @@ org.activestack syncengine - 1.1.44-SNAPSHOT + 1.1.45-SNAPSHOT 3.2.4.RELEASE diff --git a/src/main/java/com/percero/agents/sync/jobs/UpdateTableProcessor.java b/src/main/java/com/percero/agents/sync/jobs/UpdateTableProcessor.java index e06ef93..e031da1 100644 --- a/src/main/java/com/percero/agents/sync/jobs/UpdateTableProcessor.java +++ b/src/main/java/com/percero/agents/sync/jobs/UpdateTableProcessor.java @@ -2,6 +2,7 @@ import java.sql.CallableStatement; import java.sql.Connection; +import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.SQLException; import java.sql.SQLSyntaxErrorException; @@ -115,15 +116,13 @@ public void run(){ List successfulRows = new ArrayList<>(); for(UpdateTableRow row : rows) { try { - if (processRow(row)) { - result.addResult(row.getType().toString()); - successfulRows.add(row); - } else { - result.addResult(row.getType().toString(), false, ""); - } + processRow(row); + result.addResult(row.getType().toString()); + successfulRows.add(row); } catch (Exception e) { logger.warn("Failed to process update: " + e.getMessage(), e); result.addResult(row.getType().toString(), false, e.getMessage()); + releaseRowLock(row); } } deleteRows(successfulRows); @@ -149,40 +148,35 @@ public void run(){ } } - protected boolean processRow(UpdateTableRow row) throws Exception{ - boolean result = true; + protected void processRow(UpdateTableRow row) throws Exception{ if(row.getRowId() != null) switch (row.getType()){ case DELETE: - result = processDeleteSingle(row); + processDeleteSingle(row); break; case UPDATE: - result = processUpdateSingle(row); + processUpdateSingle(row); break; case INSERT: - result = processInsertSingle(row); + processInsertSingle(row); break; default: // Don't know how to process - result = true; break; } else switch (row.getType()){ case DELETE: - result = processDeleteTable(row); + processDeleteTable(row); break; case UPDATE: - result = processUpdateTable(row); + processUpdateTable(row); break; case INSERT: - result = processInsertTable(row); + processInsertTable(row); break; default: // Don't know how to process - result = true; break; } - - return result; } /** @@ -191,7 +185,7 @@ protected boolean processRow(UpdateTableRow row) throws Exception{ * @return */ @SuppressWarnings("rawtypes") - protected boolean processUpdateSingle(UpdateTableRow row) throws Exception{ + protected void processUpdateSingle(UpdateTableRow row) throws Exception{ List classes = getClassesForTableName(row.getTableName()); for(Class clazz : classes) { @@ -204,8 +198,6 @@ protected boolean processUpdateSingle(UpdateTableRow row) throws Exception{ ClassIDPair pair = new ClassIDPair(row.getRowId(), className); handleUpdateClassIdPair(dataProvider, pair); } - - return true; } /** @@ -214,7 +206,7 @@ protected boolean processUpdateSingle(UpdateTableRow row) throws Exception{ * @return */ @SuppressWarnings("rawtypes") - protected boolean processUpdateTable(UpdateTableRow row) throws Exception{ + protected void processUpdateTable(UpdateTableRow row) throws Exception{ List classes = getClassesForTableName(row.getTableName()); for(Class clazz : classes) { @@ -254,8 +246,6 @@ protected boolean processUpdateTable(UpdateTableRow row) throws Exception{ updateReferences(className); } - - return true; } /** @@ -344,7 +334,7 @@ protected void handleUpdateClassIdPair(IDataProvider dataProvider, ClassIDPair p * @return */ @SuppressWarnings("rawtypes") - protected boolean processInsertSingle(UpdateTableRow row) throws Exception{ + protected void processInsertSingle(UpdateTableRow row) throws Exception{ List classes = getClassesForTableName(row.getTableName()); for(Class clazz : classes) { @@ -356,8 +346,6 @@ protected boolean processInsertSingle(UpdateTableRow row) throws Exception{ classNamesToUpdateReferences.add(className); } - - return true; } /** @@ -366,7 +354,7 @@ protected boolean processInsertSingle(UpdateTableRow row) throws Exception{ * @return */ @SuppressWarnings("rawtypes") - protected boolean processInsertTable(UpdateTableRow row) throws Exception { + protected void processInsertTable(UpdateTableRow row) throws Exception { List classes = getClassesForTableName(row.getTableName()); for(Class clazz : classes) { @@ -385,8 +373,6 @@ protected boolean processInsertTable(UpdateTableRow row) throws Exception { classNamesToUpdateReferences.add(className); } - - return true; } /** @@ -395,7 +381,7 @@ protected boolean processInsertTable(UpdateTableRow row) throws Exception { * @return */ @SuppressWarnings("rawtypes") - protected boolean processDeleteSingle(UpdateTableRow row) throws Exception{ + protected void processDeleteSingle(UpdateTableRow row) throws Exception{ List classes = getClassesForTableName(row.getTableName()); for(Class clazz : classes){ @@ -412,8 +398,6 @@ protected boolean processDeleteSingle(UpdateTableRow row) throws Exception{ classNamesToUpdateReferences.add(className); } - - return true; } /** @@ -422,7 +406,7 @@ protected boolean processDeleteSingle(UpdateTableRow row) throws Exception{ * @return */ @SuppressWarnings("rawtypes") - protected boolean processDeleteTable(UpdateTableRow row) throws Exception{ + protected void processDeleteTable(UpdateTableRow row) throws Exception{ List classes = getClassesForTableName(row.getTableName()); for(Class clazz : classes){ @@ -465,8 +449,6 @@ protected boolean processDeleteTable(UpdateTableRow row) throws Exception{ classNamesToUpdateReferences.add(className); } - - return true; } @SuppressWarnings("rawtypes") @@ -723,13 +705,14 @@ private List getStoredProcRow(int lockId, DateTime expireThresho * @param rows */ protected void deleteRows(List rows){ - try(Statement statement = getConnection().createStatement()){ - - String sql = "delete from :tableName where ID in (:idList)"; - sql = sql.replace(":tableName", tableName); - sql = sql.replace(":idList", listIdJoin(rows)); - - int numUpdated = statement.executeUpdate(sql); + if (rows == null || rows.isEmpty()) { + return; + } + String sql = "delete from " + tableName + " where " + connectionFactory.getLockIdColumnName() + "=" + rows.get(0).getLockId(); + + try(PreparedStatement statement = getConnection().prepareStatement(sql)){ + + int numUpdated = statement.executeUpdate(sql); if(numUpdated != rows.size()){ logger.warn("Expected to delete "+rows.size()+", instead "+numUpdated); } @@ -738,23 +721,40 @@ protected void deleteRows(List rows){ } } - /** - * Utility function that takes a list of rows and returns a comma separated string - * @param list - */ - private String listIdJoin(List list){ - StringBuilder sb = new StringBuilder(); - String separator = ","; - for (UpdateTableRow row : list) { - if(sb.length() != 0) - sb.append(separator); - - sb.append(row.getID()); - } - return sb.toString(); + protected void releaseRowLock(UpdateTableRow row){ + if (row == null) { + return; + } + String sql = "UPDATE " + tableName + " SET " + connectionFactory.getLockIdColumnName() + "=NULL, " + connectionFactory.getLockDateColumnName() + "=NULL where ID=" + row.getID(); + + try(PreparedStatement statement = getConnection().prepareStatement(sql)){ + + int numUpdated = statement.executeUpdate(sql); + if(numUpdated != 1){ + logger.warn("Expected to update 1, instead "+numUpdated); + } + }catch(SQLException e){ + logger.warn(e.getMessage(), e); + } } - - @SuppressWarnings({ "rawtypes", "unchecked" }) + +// /** +// * Utility function that takes a list of rows and returns a comma separated string +// * @param list +// */ +// private String listIdJoin(List list){ +// StringBuilder sb = new StringBuilder(); +// String separator = ","; +// for (UpdateTableRow row : list) { +// if(sb.length() != 0) +// sb.append(separator); +// +// sb.append(row.getID()); +// } +// return sb.toString(); +// } + + @SuppressWarnings({ "rawtypes" }) public List getClassesForTableName(String tableName){ List result = new ArrayList<>(); From b6d45e0c7a0e21c629c475536cc365ffe7044651 Mon Sep 17 00:00:00 2001 From: Collin Brown Date: Mon, 16 Nov 2015 09:13:52 -0800 Subject: [PATCH 064/105] Added null checks in the update table path to handle the case where an "update" record has been deleted --- pom.xml | 2 +- .../percero/agents/sync/helpers/PostPutHelper.java | 14 +++++++++++--- .../agents/sync/jobs/UpdateTableProcessor.java | 12 +++++++++--- .../agents/sync/services/DAODataProvider.java | 7 +++++-- 4 files changed, 26 insertions(+), 9 deletions(-) diff --git a/pom.xml b/pom.xml index 30cbe5a..bddcd72 100644 --- a/pom.xml +++ b/pom.xml @@ -7,7 +7,7 @@ org.activestack syncengine - 1.1.45-SNAPSHOT + 1.1.46-SNAPSHOT 3.2.4.RELEASE diff --git a/src/main/java/com/percero/agents/sync/helpers/PostPutHelper.java b/src/main/java/com/percero/agents/sync/helpers/PostPutHelper.java index 1f45942..4ec1529 100644 --- a/src/main/java/com/percero/agents/sync/helpers/PostPutHelper.java +++ b/src/main/java/com/percero/agents/sync/helpers/PostPutHelper.java @@ -64,7 +64,7 @@ public void postPutObject(ClassIDPair pair, String pusherUserId, String pusherCl Map.Entry> nextEntry = itrChangedFieldEntryset.next(); ClassIDPair nextPair = nextEntry.getKey(); - if (nextPair.equals(pair)) { + if (nextPair != null && nextPair.equals(pair)) { pairChangedFields = nextEntry.getValue(); break; } @@ -90,6 +90,10 @@ public void postPutObject(ClassIDPair pair, String pusherUserId, String pusherCl while (itrChangedFieldEntryset.hasNext()) { Map.Entry> nextEntry = itrChangedFieldEntryset.next(); ClassIDPair thePair = nextEntry.getKey(); + if (thePair == null) { + continue; + } + Collection changedMappedFields = nextEntry.getValue(); // If thePair is NOT the object being updated, then need to run the postPutHelper for the Pair object as well. @@ -149,7 +153,9 @@ public void pushObjectUpdateJournals(Collection clientIds, ClassIDPair c Iterator itrChangedFields = changedFields.iterator(); while (itrChangedFields.hasNext()) { MappedField nextChangedField = itrChangedFields.next(); - pushUpdateResponse.addUpdatedField(nextChangedField.getField().getName()); + if (nextChangedField != null) { + pushUpdateResponse.addUpdatedField(nextChangedField.getField().getName()); + } } } @@ -182,7 +188,9 @@ public void pushObjectUpdateJournals(Collection clientIds, ClassIDPair c Iterator itrChangedFields = changedFields.iterator(); while (itrChangedFields.hasNext()) { MappedField nextChangedField = itrChangedFields.next(); - pushUpdateResponse.addUpdatedField(nextChangedField.getField().getName()); + if (nextChangedField != null) { + pushUpdateResponse.addUpdatedField(nextChangedField.getField().getName()); + } } } diff --git a/src/main/java/com/percero/agents/sync/jobs/UpdateTableProcessor.java b/src/main/java/com/percero/agents/sync/jobs/UpdateTableProcessor.java index e031da1..70e0c6c 100644 --- a/src/main/java/com/percero/agents/sync/jobs/UpdateTableProcessor.java +++ b/src/main/java/com/percero/agents/sync/jobs/UpdateTableProcessor.java @@ -196,7 +196,10 @@ protected void processUpdateSingle(UpdateTableRow row) throws Exception{ IDataProvider dataProvider = dataProviderManager.getDataProviderByName(mappedClass.dataProviderName); ClassIDPair pair = new ClassIDPair(row.getRowId(), className); - handleUpdateClassIdPair(dataProvider, pair); + + if (pair != null && StringUtils.hasText(pair.getID()) && StringUtils.hasText(pair.getClassName())) { + handleUpdateClassIdPair(dataProvider, pair); + } } } @@ -261,7 +264,10 @@ protected void processUpdates(String className, Collection Ids) throws E for(String ID : Ids) { ClassIDPair pair = new ClassIDPair(ID, className); - handleUpdateClassIdPair(dataProvider, pair); + + if (pair != null && StringUtils.hasText(pair.getID()) && StringUtils.hasText(pair.getClassName())) { + handleUpdateClassIdPair(dataProvider, pair); + } } } @@ -284,7 +290,7 @@ protected void handleUpdateClassIdPair(IDataProvider dataProvider, ClassIDPair p // use the cache). changedFields = dataProvider .getChangedMappedFields(perceroObject, oldValue); - if (changedFields == null || changedFields.size() > 0) { + if (changedFields != null && !changedFields.isEmpty()) { // Something has changed. cacheManager.updateCachedObject(perceroObject, changedFields); postPutHelper.postPutObject(pair, null, null, true, changedFields); diff --git a/src/main/java/com/percero/agents/sync/services/DAODataProvider.java b/src/main/java/com/percero/agents/sync/services/DAODataProvider.java index 24ad39c..f7b6e5d 100644 --- a/src/main/java/com/percero/agents/sync/services/DAODataProvider.java +++ b/src/main/java/com/percero/agents/sync/services/DAODataProvider.java @@ -1021,7 +1021,7 @@ public Map> getChangedMappedFields(IPercero MappedFieldPerceroObject nextMappedFieldPerceroObject = (MappedFieldPerceroObject) nextMappedField; IPerceroObject oldReversePerceroObject = (IPerceroObject) nextMappedFieldPerceroObject.getGetter().invoke(oldObject); - if (oldReversePerceroObject != null) { + if (oldReversePerceroObject != null && StringUtils.hasText(oldReversePerceroObject.getID())) { ClassIDPair oldReversePair = new ClassIDPair(oldReversePerceroObject.getID(), oldReversePerceroObject.getClass().getCanonicalName()); Collection oldReverseChangedFields = result.get(oldReversePair); if (oldReverseChangedFields == null) { @@ -1032,7 +1032,7 @@ public Map> getChangedMappedFields(IPercero } IPerceroObject newReversePerceroObject = (IPerceroObject) nextMappedFieldPerceroObject.getGetter().invoke(newObject); - if (newReversePerceroObject != null) { + if (newReversePerceroObject != null && StringUtils.hasText(newReversePerceroObject.getID())) { ClassIDPair newReversePair = new ClassIDPair(newReversePerceroObject.getID(), newReversePerceroObject.getClass().getCanonicalName()); Collection changedFields = result.get(newReversePair); if (changedFields == null) { @@ -1073,6 +1073,9 @@ else if (nextMappedField instanceof MappedFieldList) { Iterator itrNewChangedList = newChangedList.iterator(); while (itrNewChangedList.hasNext()) { BaseDataObject nextNewChangedObject = (BaseDataObject) itrNewChangedList.next(); + if (!StringUtils.hasText(nextNewChangedObject.getID())) { + continue; + } ClassIDPair nextNewReversePair = BaseDataObject.toClassIdPair(nextNewChangedObject); // Old object is not in new list, so add to list of changed fields. From 59a4aa6e8d17dbb01d92ca9bcc9d5ff508dd87aa Mon Sep 17 00:00:00 2001 From: Collin Brown Date: Mon, 16 Nov 2015 14:17:49 -0800 Subject: [PATCH 065/105] More NPE checks for UpdateTable processing --- pom.xml | 2 +- .../com/percero/agents/sync/services/DAODataProvider.java | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index bddcd72..1c7f8fd 100644 --- a/pom.xml +++ b/pom.xml @@ -7,7 +7,7 @@ org.activestack syncengine - 1.1.46-SNAPSHOT + 1.1.47-SNAPSHOT 3.2.4.RELEASE diff --git a/src/main/java/com/percero/agents/sync/services/DAODataProvider.java b/src/main/java/com/percero/agents/sync/services/DAODataProvider.java index f7b6e5d..c9a64c8 100644 --- a/src/main/java/com/percero/agents/sync/services/DAODataProvider.java +++ b/src/main/java/com/percero/agents/sync/services/DAODataProvider.java @@ -1073,7 +1073,7 @@ else if (nextMappedField instanceof MappedFieldList) { Iterator itrNewChangedList = newChangedList.iterator(); while (itrNewChangedList.hasNext()) { BaseDataObject nextNewChangedObject = (BaseDataObject) itrNewChangedList.next(); - if (!StringUtils.hasText(nextNewChangedObject.getID())) { + if (nextNewChangedObject == null || !StringUtils.hasText(nextNewChangedObject.getID())) { continue; } ClassIDPair nextNewReversePair = BaseDataObject.toClassIdPair(nextNewChangedObject); @@ -1107,6 +1107,9 @@ private Collection retrieveObjectsNotInCollection(Collection baseList, Collectio while (itrBaseList.hasNext()) { BaseDataObject nextBasePerceroObject = (BaseDataObject) itrBaseList.next(); + if (nextBasePerceroObject == null || !StringUtils.hasText(nextBasePerceroObject.getID())) { + continue; + } ClassIDPair nextBasePair = BaseDataObject.toClassIdPair(nextBasePerceroObject); nextBasePerceroObject = (BaseDataObject) findById(nextBasePair, null); From e5d7d6de4605e50b92193755ef2b8ed6d262d918 Mon Sep 17 00:00:00 2001 From: Collin Brown Date: Mon, 16 Nov 2015 14:21:52 -0800 Subject: [PATCH 066/105] More NULL checks for UpdateTable --- .../percero/agents/sync/services/DAODataProvider.java | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/main/java/com/percero/agents/sync/services/DAODataProvider.java b/src/main/java/com/percero/agents/sync/services/DAODataProvider.java index c9a64c8..bca995a 100644 --- a/src/main/java/com/percero/agents/sync/services/DAODataProvider.java +++ b/src/main/java/com/percero/agents/sync/services/DAODataProvider.java @@ -836,10 +836,19 @@ else if (fieldObject instanceof Collection) { //////////////////////////////////////////////////// @SuppressWarnings({ "unchecked", "rawtypes" }) public Boolean deleteObject(ClassIDPair theClassIdPair, String userId) throws SyncException { + + if (theClassIdPair == null || !StringUtils.hasText(theClassIdPair.getID())) { + // Invalid object. + return false; + } IDataAccessObject dao = (IDataAccessObject) DAORegistry.getInstance().getDataAccessObject(theClassIdPair.getClassName()); IPerceroObject perceroObject = dao.retrieveObject(theClassIdPair, null, false); // Retrieve the full object so we can update the cache if the delete is successful. Boolean result = dao.deleteObject(theClassIdPair, userId); + + if (perceroObject == null) { + return true; + } try { MappedClass mappedClass = MappedClassManagerFactory.getMappedClassManager().getMappedClassByClassName(perceroObject.getClass().getCanonicalName()); From 2d9054bee026331ad50113269da2f984a080b5e1 Mon Sep 17 00:00:00 2001 From: Collin Brown Date: Wed, 18 Nov 2015 11:58:11 -0800 Subject: [PATCH 067/105] Fix for getting changed relationship values of UpdateTable records --- pom.xml | 3 +- .../agents/sync/cache/CacheManager.java | 3 +- .../sync/jobs/UpdateTableProcessor.java | 4 +- .../agents/sync/jobs/UpdateTableRegistry.java | 22 +++---- .../agents/sync/services/DAODataProvider.java | 60 +++++++++---------- .../agents/sync/services/IDataProvider.java | 2 +- .../sync/services/RedisDataProvider.java | 2 +- .../sync/services/SyncAgentDataProvider.java | 4 +- 8 files changed, 51 insertions(+), 49 deletions(-) diff --git a/pom.xml b/pom.xml index 1c7f8fd..7dbb6bf 100644 --- a/pom.xml +++ b/pom.xml @@ -7,7 +7,7 @@ org.activestack syncengine - 1.1.47-SNAPSHOT + 1.1.48-SNAPSHOT 3.2.4.RELEASE @@ -193,6 +193,7 @@ jedis 2.5.2 + org.springframework spring-beans diff --git a/src/main/java/com/percero/agents/sync/cache/CacheManager.java b/src/main/java/com/percero/agents/sync/cache/CacheManager.java index 58944d3..125a4e9 100644 --- a/src/main/java/com/percero/agents/sync/cache/CacheManager.java +++ b/src/main/java/com/percero/agents/sync/cache/CacheManager.java @@ -12,6 +12,7 @@ import org.apache.log4j.Logger; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; +import org.springframework.util.StringUtils; import com.percero.agents.sync.access.RedisKeyUtils; import com.percero.agents.sync.datastore.ICacheDataStore; @@ -68,7 +69,7 @@ public void updateCachedObject(IPerceroObject perceroObject, Map itrChangedFieldKeyset = changedFields.keySet().iterator(); while (itrChangedFieldKeyset.hasNext()) { ClassIDPair thePair = itrChangedFieldKeyset.next(); - if (!thePair.comparePerceroObject(perceroObject)) { + if (thePair != null && StringUtils.hasText(thePair.getID()) && !thePair.comparePerceroObject(perceroObject)) { pairsToDelete.add(thePair); // String nextKey = RedisKeyUtils.classIdPair(thePair.getClassName(), thePair.getID()); } diff --git a/src/main/java/com/percero/agents/sync/jobs/UpdateTableProcessor.java b/src/main/java/com/percero/agents/sync/jobs/UpdateTableProcessor.java index 70e0c6c..0eae5ef 100644 --- a/src/main/java/com/percero/agents/sync/jobs/UpdateTableProcessor.java +++ b/src/main/java/com/percero/agents/sync/jobs/UpdateTableProcessor.java @@ -281,7 +281,7 @@ protected void handleUpdateClassIdPair(IDataProvider dataProvider, ClassIDPair p if (perceroObject == null) { return; } - + Map> changedFields = null; if (oldValue != null) { // dataProvider.getChangedMappedFields is typically used to @@ -289,7 +289,7 @@ protected void handleUpdateClassIdPair(IDataProvider dataProvider, ClassIDPair p // this case (we just need to tell getChangedMappedFields to NOT // use the cache). changedFields = dataProvider - .getChangedMappedFields(perceroObject, oldValue); + .getChangedMappedFields(perceroObject, oldValue, oldValue != null); if (changedFields != null && !changedFields.isEmpty()) { // Something has changed. cacheManager.updateCachedObject(perceroObject, changedFields); diff --git a/src/main/java/com/percero/agents/sync/jobs/UpdateTableRegistry.java b/src/main/java/com/percero/agents/sync/jobs/UpdateTableRegistry.java index b4012ae..d39eafe 100644 --- a/src/main/java/com/percero/agents/sync/jobs/UpdateTableRegistry.java +++ b/src/main/java/com/percero/agents/sync/jobs/UpdateTableRegistry.java @@ -1,15 +1,5 @@ package com.percero.agents.sync.jobs; -import com.esotericsoftware.yamlbeans.YamlException; -import com.esotericsoftware.yamlbeans.YamlReader; -import com.google.api.client.util.Value; -import edu.emory.mathcs.backport.java.util.Collections; -import org.apache.log4j.Logger; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.stereotype.Component; - -import javax.annotation.PostConstruct; import java.beans.PropertyVetoException; import java.io.File; import java.io.FileNotFoundException; @@ -20,6 +10,18 @@ import java.util.List; import java.util.Map; +import javax.annotation.PostConstruct; + +import org.apache.log4j.Logger; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.stereotype.Component; + +import com.esotericsoftware.yamlbeans.YamlException; +import com.esotericsoftware.yamlbeans.YamlReader; + +import edu.emory.mathcs.backport.java.util.Collections; + /** * @author Collin Brown * diff --git a/src/main/java/com/percero/agents/sync/services/DAODataProvider.java b/src/main/java/com/percero/agents/sync/services/DAODataProvider.java index bca995a..b7ef6f2 100644 --- a/src/main/java/com/percero/agents/sync/services/DAODataProvider.java +++ b/src/main/java/com/percero/agents/sync/services/DAODataProvider.java @@ -1000,11 +1000,11 @@ public Map> getChangedMappedFields(IPercero String className = newObject.getClass().getCanonicalName(); IPerceroObject oldObject = findById(new ClassIDPair(newObject.getID(), className), null, ignoreCache); - return getChangedMappedFields(oldObject, newObject); + return getChangedMappedFields(oldObject, newObject, ignoreCache); } - @SuppressWarnings("rawtypes") - public Map> getChangedMappedFields(IPerceroObject oldObject, IPerceroObject newObject) { + @SuppressWarnings({ "unchecked" }) + public Map> getChangedMappedFields(IPerceroObject oldObject, IPerceroObject newObject, boolean ignoreCache) { Map> result = new HashMap>(); Collection baseObjectResult = null; ClassIDPair basePair = new ClassIDPair(newObject.getID(), newObject.getClass().getCanonicalName()); @@ -1012,6 +1012,7 @@ public Map> getChangedMappedFields(IPercero String className = newObject.getClass().getCanonicalName(); IMappedClassManager mcm = MappedClassManagerFactory.getMappedClassManager(); MappedClass mc = mcm.getMappedClassByClassName(className); + Iterator itrMappedFields = mc.externalizableFields.iterator(); while (itrMappedFields.hasNext()) { MappedField nextMappedField = itrMappedFields.next(); @@ -1054,20 +1055,19 @@ public Map> getChangedMappedFields(IPercero else if (nextMappedField instanceof MappedFieldList) { MappedFieldList nextMappedFieldList = (MappedFieldList) nextMappedField; - List oldReverseList = (List) nextMappedFieldList.getGetter().invoke(oldObject); + List oldReverseList = (List) nextMappedFieldList.getGetter().invoke(oldObject); if (oldReverseList == null) - oldReverseList = new ArrayList(); + oldReverseList = new ArrayList(); - List newReverseList = (List) nextMappedFieldList.getGetter().invoke(newObject); + List newReverseList = (List) nextMappedFieldList.getGetter().invoke(newObject); if (newReverseList == null) - newReverseList = new ArrayList(); + newReverseList = new ArrayList(); // Compare each item in the lists. - Collection oldChangedList = retrieveObjectsNotInCollection(oldReverseList, newReverseList); - Iterator itrOldChangedList = oldChangedList.iterator(); + Collection oldChangedList = retrieveObjectsNotInCollection(oldReverseList, newReverseList); + Iterator itrOldChangedList = oldChangedList.iterator(); while (itrOldChangedList.hasNext()) { - BaseDataObject nextOldChangedObject = (BaseDataObject) itrOldChangedList.next(); - ClassIDPair nextOldReversePair = BaseDataObject.toClassIdPair(nextOldChangedObject); + ClassIDPair nextOldReversePair = itrOldChangedList.next(); // Old object is not in new list, so add to list of changed fields. Collection changedFields = result.get(nextOldReversePair); @@ -1078,14 +1078,10 @@ else if (nextMappedField instanceof MappedFieldList) { changedFields.add(nextMappedField.getReverseMappedField()); } - Collection newChangedList = retrieveObjectsNotInCollection(newReverseList, oldReverseList); - Iterator itrNewChangedList = newChangedList.iterator(); + Collection newChangedList = retrieveObjectsNotInCollection(newReverseList, oldReverseList); + Iterator itrNewChangedList = newChangedList.iterator(); while (itrNewChangedList.hasNext()) { - BaseDataObject nextNewChangedObject = (BaseDataObject) itrNewChangedList.next(); - if (nextNewChangedObject == null || !StringUtils.hasText(nextNewChangedObject.getID())) { - continue; - } - ClassIDPair nextNewReversePair = BaseDataObject.toClassIdPair(nextNewChangedObject); + ClassIDPair nextNewReversePair = itrNewChangedList.next(); // Old object is not in new list, so add to list of changed fields. Collection changedFields = result.get(nextNewReversePair); @@ -1107,35 +1103,37 @@ else if (nextMappedField instanceof MappedFieldList) { return result; } - @SuppressWarnings({ "rawtypes", "unchecked" }) - private Collection retrieveObjectsNotInCollection(Collection baseList, Collection compareToList) { - Collection result = new HashSet(); - Iterator itrBaseList = baseList.iterator(); - Iterator itrCompareToList = null; + private Collection retrieveObjectsNotInCollection(Collection baseList, Collection compareToList) { + Collection result = new HashSet(); + Iterator itrBaseList = baseList.iterator(); + Iterator itrCompareToList = null; boolean matchFound = false; while (itrBaseList.hasNext()) { - BaseDataObject nextBasePerceroObject = (BaseDataObject) itrBaseList.next(); + IPerceroObject nextBasePerceroObject = itrBaseList.next(); if (nextBasePerceroObject == null || !StringUtils.hasText(nextBasePerceroObject.getID())) { continue; } - ClassIDPair nextBasePair = BaseDataObject.toClassIdPair(nextBasePerceroObject); - nextBasePerceroObject = (BaseDataObject) findById(nextBasePair, null); itrCompareToList = compareToList.iterator(); matchFound = false; while (itrCompareToList.hasNext()) { - BaseDataObject nextCompareToPerceroObject = (BaseDataObject) itrCompareToList.next(); - nextCompareToPerceroObject = (BaseDataObject) findById(BaseDataObject.toClassIdPair(nextCompareToPerceroObject), null); - - if (nextCompareToPerceroObject != null && nextBasePerceroObject != null && nextBasePerceroObject.getClass() == nextCompareToPerceroObject.getClass() && nextBasePerceroObject.getID().equalsIgnoreCase(nextCompareToPerceroObject.getID())) { + IPerceroObject nextCompareToPerceroObject = itrCompareToList.next(); + + if (nextCompareToPerceroObject != null + && nextBasePerceroObject != null + && nextBasePerceroObject.getClass() == nextCompareToPerceroObject + .getClass() + && nextBasePerceroObject.getID().equalsIgnoreCase( + nextCompareToPerceroObject.getID())) { matchFound = true; break; } } if (!matchFound) { - result.add(nextBasePerceroObject); + ClassIDPair nextBasePair = BaseDataObject.toClassIdPair(nextBasePerceroObject); + result.add(nextBasePair); } } diff --git a/src/main/java/com/percero/agents/sync/services/IDataProvider.java b/src/main/java/com/percero/agents/sync/services/IDataProvider.java index 2573174..ace4c59 100644 --- a/src/main/java/com/percero/agents/sync/services/IDataProvider.java +++ b/src/main/java/com/percero/agents/sync/services/IDataProvider.java @@ -53,7 +53,7 @@ public interface IDataProvider { */ public Map> getChangedMappedFields(IPerceroObject newObject); public Map> getChangedMappedFields(IPerceroObject newObject, boolean ignoreCache); - public Map> getChangedMappedFields(IPerceroObject oldObject, IPerceroObject compareObject); + public Map> getChangedMappedFields(IPerceroObject oldObject, IPerceroObject compareObject, boolean ignoreCache); /** * Given the mappedField, returns ALL objects in the relationship described by the mappedField. diff --git a/src/main/java/com/percero/agents/sync/services/RedisDataProvider.java b/src/main/java/com/percero/agents/sync/services/RedisDataProvider.java index 98c9611..a5634c3 100644 --- a/src/main/java/com/percero/agents/sync/services/RedisDataProvider.java +++ b/src/main/java/com/percero/agents/sync/services/RedisDataProvider.java @@ -443,7 +443,7 @@ public Map> getChangedMappedFields( @Override public Map> getChangedMappedFields( - IPerceroObject oldObject, IPerceroObject compareObject) { + IPerceroObject oldObject, IPerceroObject compareObject, boolean ignoreCache) { // TODO Auto-generated method stub return null; } diff --git a/src/main/java/com/percero/agents/sync/services/SyncAgentDataProvider.java b/src/main/java/com/percero/agents/sync/services/SyncAgentDataProvider.java index 62a248b..cee4f85 100644 --- a/src/main/java/com/percero/agents/sync/services/SyncAgentDataProvider.java +++ b/src/main/java/com/percero/agents/sync/services/SyncAgentDataProvider.java @@ -1469,10 +1469,10 @@ public Map> getChangedMappedFields(IPercero String className = compareObject.getClass().getCanonicalName(); IPerceroObject oldObject = systemGetById(new ClassIDPair(compareObject.getID(), className), ignoreCache); - return getChangedMappedFields(oldObject, compareObject); + return getChangedMappedFields(oldObject, compareObject, ignoreCache); } @SuppressWarnings("rawtypes") - public Map> getChangedMappedFields(IPerceroObject oldObject, IPerceroObject compareObject) { + public Map> getChangedMappedFields(IPerceroObject oldObject, IPerceroObject compareObject, boolean ignoreCache) { Map> result = new HashMap>(); // Validate both objects. From 881cab6a108c13fa71967e1db5c3dfa8a9d340e1 Mon Sep 17 00:00:00 2001 From: Collin Brown Date: Thu, 19 Nov 2015 08:27:26 -0800 Subject: [PATCH 068/105] Added debug log statements for UpdateTableProcessor warning if no class names are mapped for UpdateTable.table name --- .../sync/jobs/UpdateTableProcessor.java | 284 ++++++++++-------- 1 file changed, 160 insertions(+), 124 deletions(-) diff --git a/src/main/java/com/percero/agents/sync/jobs/UpdateTableProcessor.java b/src/main/java/com/percero/agents/sync/jobs/UpdateTableProcessor.java index 0eae5ef..d6c1c87 100644 --- a/src/main/java/com/percero/agents/sync/jobs/UpdateTableProcessor.java +++ b/src/main/java/com/percero/agents/sync/jobs/UpdateTableProcessor.java @@ -188,18 +188,24 @@ protected void processRow(UpdateTableRow row) throws Exception{ protected void processUpdateSingle(UpdateTableRow row) throws Exception{ List classes = getClassesForTableName(row.getTableName()); - for(Class clazz : classes) { - String className = clazz.getCanonicalName(); - - IMappedClassManager mcm = MappedClassManagerFactory.getMappedClassManager(); - MappedClass mappedClass = mcm.getMappedClassByClassName(className); - IDataProvider dataProvider = dataProviderManager.getDataProviderByName(mappedClass.dataProviderName); - - ClassIDPair pair = new ClassIDPair(row.getRowId(), className); - - if (pair != null && StringUtils.hasText(pair.getID()) && StringUtils.hasText(pair.getClassName())) { - handleUpdateClassIdPair(dataProvider, pair); - } + if (classes == null || classes.isEmpty()) { + logger.warn("No Classes defined for UpdateTable table " + row.getTableName()); + } + else { + logger.debug("Processing " + classes.size() + " class(es) for UpdateTable UPDATE, table " + row.getTableName()); + for(Class clazz : classes) { + String className = clazz.getCanonicalName(); + + IMappedClassManager mcm = MappedClassManagerFactory.getMappedClassManager(); + MappedClass mappedClass = mcm.getMappedClassByClassName(className); + IDataProvider dataProvider = dataProviderManager.getDataProviderByName(mappedClass.dataProviderName); + + ClassIDPair pair = new ClassIDPair(row.getRowId(), className); + + if (pair != null && StringUtils.hasText(pair.getID()) && StringUtils.hasText(pair.getClassName())) { + handleUpdateClassIdPair(dataProvider, pair); + } + } } } @@ -212,42 +218,48 @@ protected void processUpdateSingle(UpdateTableRow row) throws Exception{ protected void processUpdateTable(UpdateTableRow row) throws Exception{ List classes = getClassesForTableName(row.getTableName()); - for(Class clazz : classes) { - String className = clazz.getCanonicalName(); - - // If there are any clients that have asked for all objects in a class then we have to push everything - if (accessManager.getNumClientsInterestedInWholeClass(className) > 0) { - - IMappedClassManager mcm = MappedClassManagerFactory.getMappedClassManager(); - MappedClass mappedClass = mcm.getMappedClassByClassName(className); - IDataProvider dataProvider = dataProviderManager.getDataProviderByName(mappedClass.dataProviderName); - - Integer pageNumber = 0; - Integer pageSize = 25; - Integer total = -1; - - while (total < 0 || pageNumber * pageSize <= total) { - PerceroList objectsToUpdate = dataProvider.getAllByName(className, pageNumber, pageSize, true, null); - pageNumber++; - total = objectsToUpdate.getTotalLength(); - if (total <= 0) { - break; - } - - Set objectIds = new HashSet(objectsToUpdate.size()); - Iterator itrObjectsToUpdate = objectsToUpdate.iterator(); - while (itrObjectsToUpdate.hasNext()) { - IPerceroObject nextObjectToUpdate = itrObjectsToUpdate.next(); - objectIds.add(nextObjectToUpdate.getID()); - } - - processUpdates(className, objectIds); - } - } else { - processUpdates(className, accessManager.getClassAccessJournalIDs(className)); - } - - updateReferences(className); + if (classes == null || classes.isEmpty()) { + logger.warn("No Classes defined for UpdateTable table " + row.getTableName()); + } + else { + logger.debug("Processing " + classes.size() + " class(es) for UpdateTable UPDATE, table " + row.getTableName()); + for(Class clazz : classes) { + String className = clazz.getCanonicalName(); + + // If there are any clients that have asked for all objects in a class then we have to push everything + if (accessManager.getNumClientsInterestedInWholeClass(className) > 0) { + + IMappedClassManager mcm = MappedClassManagerFactory.getMappedClassManager(); + MappedClass mappedClass = mcm.getMappedClassByClassName(className); + IDataProvider dataProvider = dataProviderManager.getDataProviderByName(mappedClass.dataProviderName); + + Integer pageNumber = 0; + Integer pageSize = 25; + Integer total = -1; + + while (total < 0 || pageNumber * pageSize <= total) { + PerceroList objectsToUpdate = dataProvider.getAllByName(className, pageNumber, pageSize, true, null); + pageNumber++; + total = objectsToUpdate.getTotalLength(); + if (total <= 0) { + break; + } + + Set objectIds = new HashSet(objectsToUpdate.size()); + Iterator itrObjectsToUpdate = objectsToUpdate.iterator(); + while (itrObjectsToUpdate.hasNext()) { + IPerceroObject nextObjectToUpdate = itrObjectsToUpdate.next(); + objectIds.add(nextObjectToUpdate.getID()); + } + + processUpdates(className, objectIds); + } + } else { + processUpdates(className, accessManager.getClassAccessJournalIDs(className)); + } + + updateReferences(className); + } } } @@ -343,14 +355,20 @@ protected void handleUpdateClassIdPair(IDataProvider dataProvider, ClassIDPair p protected void processInsertSingle(UpdateTableRow row) throws Exception{ List classes = getClassesForTableName(row.getTableName()); - for(Class clazz : classes) { - String className = clazz.getCanonicalName(); - - // We do not use PostCreateHelper here because we are going to do all - // that extra work for the whole class in updateReferences. - postPutHelper.postPutObject(new ClassIDPair(row.getRowId(), className), null, null, true, null); - - classNamesToUpdateReferences.add(className); + if (classes == null || classes.isEmpty()) { + logger.warn("No Classes defined for UpdateTable table " + row.getTableName()); + } + else { + logger.debug("Processing " + classes.size() + " class(es) for UpdateTable INSERT, table " + row.getTableName()); + for(Class clazz : classes) { + String className = clazz.getCanonicalName(); + + // We do not use PostCreateHelper here because we are going to do all + // that extra work for the whole class in updateReferences. + postPutHelper.postPutObject(new ClassIDPair(row.getRowId(), className), null, null, true, null); + + classNamesToUpdateReferences.add(className); + } } } @@ -363,21 +381,27 @@ protected void processInsertSingle(UpdateTableRow row) throws Exception{ protected void processInsertTable(UpdateTableRow row) throws Exception { List classes = getClassesForTableName(row.getTableName()); - for(Class clazz : classes) { - String className = clazz.getCanonicalName(); - - // if any client needs all of this class then the only choice we have is to push everything - if (accessManager.getNumClientsInterestedInWholeClass(className) > 0 /* || true */) { - Set allClassIdPairs = getAllClassIdPairsForTable(row.getTableName()); - for (ClassIDPair classIdPair : allClassIdPairs) { - // We do not use PostCreateHelper here because we are going to - // do all that extra work for the whole class in - // updateReferences. - postPutHelper.postPutObject(classIdPair, null, null, true, null); - } - } - - classNamesToUpdateReferences.add(className); + if (classes == null || classes.isEmpty()) { + logger.warn("No Classes defined for UpdateTable table " + row.getTableName()); + } + else { + logger.debug("Processing " + classes.size() + " class(es) for UpdateTable INSERT, table " + row.getTableName()); + for(Class clazz : classes) { + String className = clazz.getCanonicalName(); + + // if any client needs all of this class then the only choice we have is to push everything + if (accessManager.getNumClientsInterestedInWholeClass(className) > 0 /* || true */) { + Set allClassIdPairs = getAllClassIdPairsForTable(row.getTableName()); + for (ClassIDPair classIdPair : allClassIdPairs) { + // We do not use PostCreateHelper here because we are going to + // do all that extra work for the whole class in + // updateReferences. + postPutHelper.postPutObject(classIdPair, null, null, true, null); + } + } + + classNamesToUpdateReferences.add(className); + } } } @@ -389,20 +413,26 @@ protected void processInsertTable(UpdateTableRow row) throws Exception { @SuppressWarnings("rawtypes") protected void processDeleteSingle(UpdateTableRow row) throws Exception{ List classes = getClassesForTableName(row.getTableName()); - - for(Class clazz : classes){ - String className = clazz.getCanonicalName(); - - // See if this object is in the cache. If so, it will help us know which related objects to update. - IMappedClassManager mcm = MappedClassManagerFactory.getMappedClassManager(); - MappedClass mappedClass = mcm.getMappedClassByClassName(className); - IDataProvider dataProvider = dataProviderManager.getDataProviderByName(mappedClass.dataProviderName); - ClassIDPair pair = new ClassIDPair(row.getRowId(), className); - IPerceroObject cachedObject = dataProvider.findById(pair, null, false); // We are hoping to find this object in the cache... - - handleDeletedObject(cachedObject, clazz, className, row.getRowId()); - - classNamesToUpdateReferences.add(className); + + if (classes == null || classes.isEmpty()) { + logger.warn("No Classes defined for UpdateTable table " + row.getTableName()); + } + else { + logger.debug("Processing " + classes.size() + " class(es) for UpdateTable DELETE, table " + row.getTableName()); + for(Class clazz : classes){ + String className = clazz.getCanonicalName(); + + // See if this object is in the cache. If so, it will help us know which related objects to update. + IMappedClassManager mcm = MappedClassManagerFactory.getMappedClassManager(); + MappedClass mappedClass = mcm.getMappedClassByClassName(className); + IDataProvider dataProvider = dataProviderManager.getDataProviderByName(mappedClass.dataProviderName); + ClassIDPair pair = new ClassIDPair(row.getRowId(), className); + IPerceroObject cachedObject = dataProvider.findById(pair, null, false); // We are hoping to find this object in the cache... + + handleDeletedObject(cachedObject, clazz, className, row.getRowId()); + + classNamesToUpdateReferences.add(className); + } } } @@ -415,45 +445,51 @@ protected void processDeleteSingle(UpdateTableRow row) throws Exception{ protected void processDeleteTable(UpdateTableRow row) throws Exception{ List classes = getClassesForTableName(row.getTableName()); - for(Class clazz : classes){ - String className = clazz.getCanonicalName(); - - IMappedClassManager mcm = MappedClassManagerFactory.getMappedClassManager(); - MappedClass mappedClass = mcm.getMappedClassByClassName(className); - IDataProvider dataProvider = dataProviderManager.getDataProviderByName(mappedClass.dataProviderName); - - // Get the list of ALL ID's of this class type that have been accessed. - Set accessedIds = accessManager.getClassAccessJournalIDs(className); - - if (accessedIds != null && !accessedIds.isEmpty()) { - // TODO: If ID "0", then that means someone wants to know about ALL - // records of this type. How do we do this? - - // Get a list of ALL ID's of this class type. - Set allClassIdPairs = getAllClassIdPairsForTable(row.getTableName()); - - // Remove ALL existing/current ID's from our list of accessed ID's. - for (ClassIDPair nextClassIdPair : allClassIdPairs) { - accessedIds.remove(nextClassIdPair.getID()); - } - - // Now we have the list of ID's that have actually been deleted. - for (String id : accessedIds) { - // Find the cached object first so that it is NOT removed if the same object is NOT found in the data store. - IPerceroObject cachedObject = dataProvider.findById(new ClassIDPair(id, className), null, false); - // We will know an object has been deleted IFF it does NOT exist in the data store. - IPerceroObject dataStoreObject = dataProvider.findById(new ClassIDPair(id, className), null, true); - - if (dataStoreObject != null) { - // Object has NOT been deleted. - continue; - } - - handleDeletedObject(cachedObject, clazz, className, id); - } - } - - classNamesToUpdateReferences.add(className); + if (classes == null || classes.isEmpty()) { + logger.warn("No Classes defined for UpdateTable table " + row.getTableName()); + } + else { + logger.debug("Processing " + classes.size() + " class(es) for UpdateTable DELETE, table " + row.getTableName()); + for(Class clazz : classes){ + String className = clazz.getCanonicalName(); + + IMappedClassManager mcm = MappedClassManagerFactory.getMappedClassManager(); + MappedClass mappedClass = mcm.getMappedClassByClassName(className); + IDataProvider dataProvider = dataProviderManager.getDataProviderByName(mappedClass.dataProviderName); + + // Get the list of ALL ID's of this class type that have been accessed. + Set accessedIds = accessManager.getClassAccessJournalIDs(className); + + if (accessedIds != null && !accessedIds.isEmpty()) { + // TODO: If ID "0", then that means someone wants to know about ALL + // records of this type. How do we do this? + + // Get a list of ALL ID's of this class type. + Set allClassIdPairs = getAllClassIdPairsForTable(row.getTableName()); + + // Remove ALL existing/current ID's from our list of accessed ID's. + for (ClassIDPair nextClassIdPair : allClassIdPairs) { + accessedIds.remove(nextClassIdPair.getID()); + } + + // Now we have the list of ID's that have actually been deleted. + for (String id : accessedIds) { + // Find the cached object first so that it is NOT removed if the same object is NOT found in the data store. + IPerceroObject cachedObject = dataProvider.findById(new ClassIDPair(id, className), null, false); + // We will know an object has been deleted IFF it does NOT exist in the data store. + IPerceroObject dataStoreObject = dataProvider.findById(new ClassIDPair(id, className), null, true); + + if (dataStoreObject != null) { + // Object has NOT been deleted. + continue; + } + + handleDeletedObject(cachedObject, clazz, className, id); + } + } + + classNamesToUpdateReferences.add(className); + } } } From 673463b951921d5a67a7d0f9cb84ad72c2e2883d Mon Sep 17 00:00:00 2001 From: Collin Brown Date: Thu, 19 Nov 2015 15:54:35 -0800 Subject: [PATCH 069/105] Using Spring for UpdateTable Task running. --- pom.xml | 4 +- .../agents/sync/jobs/UpdateTablePoller.java | 65 +++++++++++-------- .../sync/jobs/UpdateTableProcessor.java | 12 +++- 3 files changed, 49 insertions(+), 32 deletions(-) diff --git a/pom.xml b/pom.xml index 7dbb6bf..6923e1f 100644 --- a/pom.xml +++ b/pom.xml @@ -7,7 +7,7 @@ org.activestack syncengine - 1.1.48-SNAPSHOT + 1.1.49-SNAPSHOT 3.2.4.RELEASE @@ -193,7 +193,7 @@ jedis 2.5.2 - + org.springframework spring-beans diff --git a/src/main/java/com/percero/agents/sync/jobs/UpdateTablePoller.java b/src/main/java/com/percero/agents/sync/jobs/UpdateTablePoller.java index ebf664c..33d1264 100644 --- a/src/main/java/com/percero/agents/sync/jobs/UpdateTablePoller.java +++ b/src/main/java/com/percero/agents/sync/jobs/UpdateTablePoller.java @@ -1,19 +1,22 @@ package com.percero.agents.sync.jobs; +import java.util.HashSet; +import java.util.Set; + +import javax.annotation.PostConstruct; + +import org.apache.log4j.Logger; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.core.task.TaskExecutor; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Component; + import com.percero.agents.sync.access.IAccessManager; import com.percero.agents.sync.cache.CacheManager; import com.percero.agents.sync.helpers.PostDeleteHelper; import com.percero.agents.sync.helpers.PostPutHelper; import com.percero.agents.sync.services.DataProviderManager; import com.percero.framework.bl.IManifest; -import org.apache.log4j.Logger; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.scheduling.annotation.Scheduled; -import org.springframework.stereotype.Component; - -import javax.annotation.PostConstruct; -import java.util.ArrayList; -import java.util.List; /** * Created by jonnysamps on 8/31/15. @@ -44,7 +47,10 @@ public class UpdateTablePoller { @Autowired IAccessManager accessManager; - List threads = new ArrayList(); + @Autowired + TaskExecutor taskExecutor; + + Set runningProcessors = new HashSet(); public boolean enabled = true; @@ -57,22 +63,22 @@ public void init(){ /** * Run every minute */ - @Scheduled(fixedDelay=5000, initialDelay=10000) // Every 5 seconds + @Scheduled(fixedRate=5000, initialDelay=10000) // Every 5 seconds public void pollUpdateTables() { - boolean lastDone = true; - for (Thread thread : threads){ - if (thread.isAlive()) { - lastDone = false; - break; - } - } - - if(enabled && lastDone) { - threads.clear(); - for (UpdateTableConnectionFactory updateTableConnectionFactory : updateTableRegistry.getConnectionFactories()) { - for (String tableName : updateTableConnectionFactory.getTableNames()) { - doProcessingForTable(updateTableConnectionFactory, tableName); - } + for (UpdateTableConnectionFactory updateTableConnectionFactory : updateTableRegistry.getConnectionFactories()) { + for (String tableName : updateTableConnectionFactory.getTableNames()) { + boolean processorRunning = false; + for(UpdateTableProcessor runningProcessor : runningProcessors) { + if (runningProcessor.connectionFactory == updateTableConnectionFactory) { + if (runningProcessor.tableName.equals(tableName)) { + processorRunning = true; + break; + } + } + } + if (!processorRunning) { + doProcessingForTable(updateTableConnectionFactory, tableName); + } } } } @@ -82,14 +88,17 @@ public void doProcessingForTable(UpdateTableConnectionFactory connectionFactory, // Use it to see how many threads to create. for(int i = 0; i < connectionFactory.getWeight(); i++) { UpdateTableProcessor processor = getProcessor(connectionFactory, tableName); - Thread thread = new Thread(processor); - threads.add(thread); - thread.start(); + taskExecutor.execute(processor); + runningProcessors.add(processor); } } + + public void processorCallback(UpdateTableProcessor processor) { + runningProcessors.remove(processor); + } public UpdateTableProcessor getProcessor(UpdateTableConnectionFactory connectionFactory, String tableName){ return new UpdateTableProcessor(tableName, connectionFactory, manifest, - postDeleteHelper, postPutHelper, cacheManager, dataProviderManager, accessManager); + postDeleteHelper, postPutHelper, cacheManager, dataProviderManager, accessManager, this); } } diff --git a/src/main/java/com/percero/agents/sync/jobs/UpdateTableProcessor.java b/src/main/java/com/percero/agents/sync/jobs/UpdateTableProcessor.java index d6c1c87..1bc6c0f 100644 --- a/src/main/java/com/percero/agents/sync/jobs/UpdateTableProcessor.java +++ b/src/main/java/com/percero/agents/sync/jobs/UpdateTableProcessor.java @@ -59,6 +59,7 @@ public class UpdateTableProcessor implements Runnable{ protected DataProviderManager dataProviderManager; protected IAccessManager accessManager; protected int maxRowsToProcess = INFINITE_ROWS; // No max + protected UpdateTablePoller poller; private Connection connection; @@ -69,7 +70,8 @@ public UpdateTableProcessor(String tableName, PostPutHelper postPutHelper, CacheManager cacheManager, DataProviderManager dataProviderManager, - IAccessManager accessManager) + IAccessManager accessManager, + UpdateTablePoller poller) { this.tableName = tableName; this.connectionFactory = connectionFactory; @@ -80,10 +82,11 @@ public UpdateTableProcessor(String tableName, this.dataProviderManager= dataProviderManager; this.accessManager = accessManager; this.maxRowsToProcess = connectionFactory.getWeight(); + this.poller = poller; classNamesToUpdateReferences = new HashSet(); } - + // TODO: This should be moved to be stored in a database table. Could // possibly use the existing "entire table update" mechanism to handle this @@ -146,6 +149,11 @@ public void run(){ conn.close(); }catch(Exception e){} } + + // Callback + if (poller != null) { + poller.processorCallback(this); + } } protected void processRow(UpdateTableRow row) throws Exception{ From c1d72fd6b1d1bc855eb1fd80a2ebf0a38478eead Mon Sep 17 00:00:00 2001 From: Collin Brown Date: Fri, 20 Nov 2015 08:12:08 -0800 Subject: [PATCH 070/105] Enhanced weighting of UpdateTable processing --- pom.xml | 15 +++++- .../agents/sync/jobs/UpdateTablePoller.java | 49 ++++++++++++------- .../sync/jobs/UpdateTableProcessor.java | 10 ++++ 3 files changed, 55 insertions(+), 19 deletions(-) diff --git a/pom.xml b/pom.xml index 6923e1f..4bd133c 100644 --- a/pom.xml +++ b/pom.xml @@ -7,7 +7,7 @@ org.activestack syncengine - 1.1.49-SNAPSHOT + 1.1.50-SNAPSHOT 3.2.4.RELEASE @@ -193,7 +193,18 @@ jedis 2.5.2 - + + + + org.quartz-scheduler + quartz + 2.2.2 + + + org.quartz-scheduler + quartz-jobs + 2.2.2 + org.springframework spring-beans diff --git a/src/main/java/com/percero/agents/sync/jobs/UpdateTablePoller.java b/src/main/java/com/percero/agents/sync/jobs/UpdateTablePoller.java index 33d1264..4e0d881 100644 --- a/src/main/java/com/percero/agents/sync/jobs/UpdateTablePoller.java +++ b/src/main/java/com/percero/agents/sync/jobs/UpdateTablePoller.java @@ -1,6 +1,8 @@ package com.percero.agents.sync.jobs; +import java.util.HashMap; import java.util.HashSet; +import java.util.Map; import java.util.Set; import javax.annotation.PostConstruct; @@ -18,6 +20,8 @@ import com.percero.agents.sync.services.DataProviderManager; import com.percero.framework.bl.IManifest; +import edu.emory.mathcs.backport.java.util.Collections; + /** * Created by jonnysamps on 8/31/15. */ @@ -50,7 +54,7 @@ public class UpdateTablePoller { @Autowired TaskExecutor taskExecutor; - Set runningProcessors = new HashSet(); + Map> runningProcessors = java.util.Collections.synchronizedMap(new HashMap>()); public boolean enabled = true; @@ -63,38 +67,49 @@ public void init(){ /** * Run every minute */ - @Scheduled(fixedRate=5000, initialDelay=10000) // Every 5 seconds +// @Scheduled(fixedRate=5000, initialDelay=10000) // Every 5 seconds + @Scheduled(fixedRate=5000) // Every 5 seconds public void pollUpdateTables() { for (UpdateTableConnectionFactory updateTableConnectionFactory : updateTableRegistry.getConnectionFactories()) { for (String tableName : updateTableConnectionFactory.getTableNames()) { - boolean processorRunning = false; - for(UpdateTableProcessor runningProcessor : runningProcessors) { - if (runningProcessor.connectionFactory == updateTableConnectionFactory) { - if (runningProcessor.tableName.equals(tableName)) { - processorRunning = true; - break; - } - } - } - if (!processorRunning) { +// boolean processorRunning = false; +// for(UpdateTableProcessor runningProcessor : runningProcessors) { +// if (runningProcessor.connectionFactory == updateTableConnectionFactory) { +// if (runningProcessor.tableName.equals(tableName)) { +// processorRunning = true; +// break; +// } +// } +// } +// if (!processorRunning) { doProcessingForTable(updateTableConnectionFactory, tableName); - } +// } } } } - public void doProcessingForTable(UpdateTableConnectionFactory connectionFactory, String tableName){ + @SuppressWarnings("unchecked") + public void doProcessingForTable(UpdateTableConnectionFactory connectionFactory, String tableName){ // Spin `weight` new threads... weight is supposed to be a balancing scale.. but right now we // Use it to see how many threads to create. - for(int i = 0; i < connectionFactory.getWeight(); i++) { + String processName = connectionFactory.getJdbcUrl() + "::" + tableName; + Set processSet = runningProcessors.get(processName); + if (processSet == null) { + processSet = Collections.synchronizedSet(new HashSet()); + runningProcessors.put(processName, processSet); + } + while (processSet.size() < connectionFactory.getWeight()) { UpdateTableProcessor processor = getProcessor(connectionFactory, tableName); taskExecutor.execute(processor); - runningProcessors.add(processor); + processSet.add(processor); } } public void processorCallback(UpdateTableProcessor processor) { - runningProcessors.remove(processor); + Set processSet = runningProcessors.get(processor.getProcessorName()); + if (processSet != null) { + processSet.remove(processor); + } } public UpdateTableProcessor getProcessor(UpdateTableConnectionFactory connectionFactory, String tableName){ diff --git a/src/main/java/com/percero/agents/sync/jobs/UpdateTableProcessor.java b/src/main/java/com/percero/agents/sync/jobs/UpdateTableProcessor.java index 1bc6c0f..0de90a0 100644 --- a/src/main/java/com/percero/agents/sync/jobs/UpdateTableProcessor.java +++ b/src/main/java/com/percero/agents/sync/jobs/UpdateTableProcessor.java @@ -88,6 +88,16 @@ public UpdateTableProcessor(String tableName, } + public String getProcessorName() { + if (connectionFactory != null && connectionFactory.getJdbcUrl() != null && tableName != null) { + return connectionFactory.getJdbcUrl() + "::" + tableName; + } + else { + return ""; + } + } + + // TODO: This should be moved to be stored in a database table. Could // possibly use the existing "entire table update" mechanism to handle this // (for each reverse mapped class) From 06d05a0a52f19f0b9a74c84e924005e9dbdef953 Mon Sep 17 00:00:00 2001 From: Collin Brown Date: Fri, 20 Nov 2015 10:52:43 -0800 Subject: [PATCH 071/105] Added more logging around UpdateTable processing --- pom.xml | 2 +- .../agents/sync/jobs/UpdateTablePoller.java | 35 +++++++++---------- 2 files changed, 17 insertions(+), 20 deletions(-) diff --git a/pom.xml b/pom.xml index 4bd133c..87f8d13 100644 --- a/pom.xml +++ b/pom.xml @@ -7,7 +7,7 @@ org.activestack syncengine - 1.1.50-SNAPSHOT + 1.1.51-SNAPSHOT 3.2.4.RELEASE diff --git a/src/main/java/com/percero/agents/sync/jobs/UpdateTablePoller.java b/src/main/java/com/percero/agents/sync/jobs/UpdateTablePoller.java index 4e0d881..0943da1 100644 --- a/src/main/java/com/percero/agents/sync/jobs/UpdateTablePoller.java +++ b/src/main/java/com/percero/agents/sync/jobs/UpdateTablePoller.java @@ -67,23 +67,12 @@ public void init(){ /** * Run every minute */ -// @Scheduled(fixedRate=5000, initialDelay=10000) // Every 5 seconds - @Scheduled(fixedRate=5000) // Every 5 seconds +// @Scheduled(fixedRate=5000) // Every 5 seconds + @Scheduled(fixedRate=5000, initialDelay=10000) // Every 5 seconds public void pollUpdateTables() { for (UpdateTableConnectionFactory updateTableConnectionFactory : updateTableRegistry.getConnectionFactories()) { for (String tableName : updateTableConnectionFactory.getTableNames()) { -// boolean processorRunning = false; -// for(UpdateTableProcessor runningProcessor : runningProcessors) { -// if (runningProcessor.connectionFactory == updateTableConnectionFactory) { -// if (runningProcessor.tableName.equals(tableName)) { -// processorRunning = true; -// break; -// } -// } -// } -// if (!processorRunning) { - doProcessingForTable(updateTableConnectionFactory, tableName); -// } + doProcessingForTable(updateTableConnectionFactory, tableName); } } } @@ -98,17 +87,25 @@ public void doProcessingForTable(UpdateTableConnectionFactory connectionFactory, processSet = Collections.synchronizedSet(new HashSet()); runningProcessors.put(processName, processSet); } - while (processSet.size() < connectionFactory.getWeight()) { - UpdateTableProcessor processor = getProcessor(connectionFactory, tableName); - taskExecutor.execute(processor); - processSet.add(processor); - } + + if (processSet.size() < connectionFactory.getWeight()) { + while (processSet.size() < connectionFactory.getWeight()) { + UpdateTableProcessor processor = getProcessor(connectionFactory, tableName); + taskExecutor.execute(processor); + processSet.add(processor); + logger.debug("Starting UpdateTable processor " + processor.getProcessorName() + " [" + processSet.size() + "/" + connectionFactory.getWeight() + "]"); + } + } + else { + logger.debug("Processor " + processName + " already at capacity [" + processSet.size() + "/" + connectionFactory.getWeight() + "]"); + } } public void processorCallback(UpdateTableProcessor processor) { Set processSet = runningProcessors.get(processor.getProcessorName()); if (processSet != null) { processSet.remove(processor); + logger.debug("Removing UpdateTable processor " + processor.getProcessorName() + " [" + processSet.size() + "/" + processor.connectionFactory.getWeight() + "]"); } } From e474fb921af6a68e1d81800c0cdaf0c84399b408 Mon Sep 17 00:00:00 2001 From: Jonathan Samples Date: Wed, 25 Nov 2015 07:18:51 -0800 Subject: [PATCH 072/105] Added a new listener on the change watcher process to listen for change watcher checks --- .../sync/cw/CheckChangeWatcherMessage.java | 12 +++++ .../agents/sync/helpers/PostPutHelper.java | 22 +++++++++- .../agents/sync/jobs/UpdateTablePoller.java | 4 +- .../amqp/CheckChangeWatcherListener.java | 44 +++++++++++++++++++ .../percero/amqp/RabbitMQPushSyncHelper.java | 7 ++- .../spring/basic_components_spring_config.xml | 1 + .../spring/changeWatcher-spring-config.xml | 10 +++-- .../spring/messageListener-spring-config.xml | 2 +- .../spring/percero-spring-config.xml | 2 +- .../spring/updateTable-spring-config.xml | 2 +- 10 files changed, 94 insertions(+), 12 deletions(-) create mode 100644 src/main/java/com/percero/agents/sync/cw/CheckChangeWatcherMessage.java create mode 100644 src/main/java/com/percero/amqp/CheckChangeWatcherListener.java diff --git a/src/main/java/com/percero/agents/sync/cw/CheckChangeWatcherMessage.java b/src/main/java/com/percero/agents/sync/cw/CheckChangeWatcherMessage.java new file mode 100644 index 0000000..bd97fbd --- /dev/null +++ b/src/main/java/com/percero/agents/sync/cw/CheckChangeWatcherMessage.java @@ -0,0 +1,12 @@ +package com.percero.agents.sync.cw; + +import com.percero.agents.sync.vo.ClassIDPair; + +/** + * Created by jonnysamps on 11/23/15. + */ +public class CheckChangeWatcherMessage { + public ClassIDPair classIDPair; + public String[] fieldNames; + public String[] params; +} diff --git a/src/main/java/com/percero/agents/sync/helpers/PostPutHelper.java b/src/main/java/com/percero/agents/sync/helpers/PostPutHelper.java index 4ec1529..2dcff59 100644 --- a/src/main/java/com/percero/agents/sync/helpers/PostPutHelper.java +++ b/src/main/java/com/percero/agents/sync/helpers/PostPutHelper.java @@ -6,8 +6,10 @@ import java.util.Iterator; import java.util.Map; +import com.percero.agents.sync.cw.CheckChangeWatcherMessage; import org.apache.log4j.Logger; import org.codehaus.jackson.map.ObjectMapper; +import org.springframework.amqp.core.AmqpTemplate; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; @@ -45,6 +47,9 @@ public void setAccessManager(IAccessManager value) { public void setPushSyncHelper(IPushSyncHelper value) { pushSyncHelper = value; } + + @Autowired + AmqpTemplate template; @Autowired Boolean guaranteeUpdateDelivery = true; @@ -82,7 +87,8 @@ public void postPutObject(ClassIDPair pair, String pusherUserId, String pusherCl // Now run past the ChangeWatcher. if (changedFields == null || changedFields.isEmpty()) { - accessManager.checkChangeWatchers(pair, null, null); + enqueueCheckChangeWatcher(pair, null, null); +// accessManager.checkChangeWatchers(pair, null, null); } else { // TODO: Need to somehow aggregate changes per client/object. @@ -113,7 +119,10 @@ public void postPutObject(ClassIDPair pair, String pusherUserId, String pusherCl fieldNames[i] = nextChangedField.getField().getName(); i++; } - accessManager.checkChangeWatchers(thePair, fieldNames, null); + + // Swap out inline processing for a worker queue + enqueueCheckChangeWatcher(thePair, fieldNames, null); +// accessManager.checkChangeWatchers(thePair, fieldNames, null); } } // Iterator itrChangedFieldKeyset = changedFields.keySet().iterator(); @@ -133,6 +142,15 @@ public void postPutObject(ClassIDPair pair, String pusherUserId, String pusherCl } } + private void enqueueCheckChangeWatcher(ClassIDPair classIDPair, String[] fieldNames, String[] params){ + CheckChangeWatcherMessage message = new CheckChangeWatcherMessage(); + message.classIDPair = classIDPair; + message.fieldNames = fieldNames; + message.params = params; + template.convertAndSend("checkChangeWatcher", message); + } + + public void pushObjectUpdateJournals(Collection clientIds, ClassIDPair classIdPair, Collection changedFields) { if (classIdPair != null && clientIds != null && !clientIds.isEmpty()) { IMappedClassManager mcm = MappedClassManagerFactory.getMappedClassManager(); diff --git a/src/main/java/com/percero/agents/sync/jobs/UpdateTablePoller.java b/src/main/java/com/percero/agents/sync/jobs/UpdateTablePoller.java index 0943da1..f802659 100644 --- a/src/main/java/com/percero/agents/sync/jobs/UpdateTablePoller.java +++ b/src/main/java/com/percero/agents/sync/jobs/UpdateTablePoller.java @@ -67,9 +67,9 @@ public void init(){ /** * Run every minute */ -// @Scheduled(fixedRate=5000) // Every 5 seconds - @Scheduled(fixedRate=5000, initialDelay=10000) // Every 5 seconds + @Scheduled(cron="0/5 * * * * *") // Every 5 seconds public void pollUpdateTables() { + logger.debug("*** UpdateTablePoller running..."); for (UpdateTableConnectionFactory updateTableConnectionFactory : updateTableRegistry.getConnectionFactories()) { for (String tableName : updateTableConnectionFactory.getTableNames()) { doProcessingForTable(updateTableConnectionFactory, tableName); diff --git a/src/main/java/com/percero/amqp/CheckChangeWatcherListener.java b/src/main/java/com/percero/amqp/CheckChangeWatcherListener.java new file mode 100644 index 0000000..ad52a78 --- /dev/null +++ b/src/main/java/com/percero/amqp/CheckChangeWatcherListener.java @@ -0,0 +1,44 @@ +package com.percero.amqp; + +import com.percero.agents.sync.access.IAccessManager; +import com.percero.agents.sync.cw.CheckChangeWatcherMessage; +import com.percero.agents.sync.metadata.MappedClass; +import org.apache.log4j.Logger; +import org.codehaus.jackson.JsonNode; +import org.codehaus.jackson.map.ObjectMapper; +import org.springframework.amqp.core.Message; +import org.springframework.amqp.core.MessageListener; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + + +/** + * Created by jonnysamps on 11/23/15. + */ +@Component("checkChangeWatcherListener") +public class CheckChangeWatcherListener implements MessageListener { + + private static Logger logger = Logger.getLogger(CheckChangeWatcherListener.class); + + @Autowired + ObjectMapper om; + + @Autowired + IAccessManager accessManager; + public void setAccessManager(IAccessManager value) { + accessManager = value; + } + + @Override + public void onMessage(Message message) { + try { + JsonNode node = om.readTree(message.getBody()); + CheckChangeWatcherMessage checkChangeWatcherMessage = om.readValue(node, CheckChangeWatcherMessage.class); + accessManager.checkChangeWatchers(checkChangeWatcherMessage.classIDPair, + checkChangeWatcherMessage.fieldNames, + checkChangeWatcherMessage.params); + }catch(Exception e){ + logger.error(e.getMessage(), e); + } + } +} diff --git a/src/main/java/com/percero/amqp/RabbitMQPushSyncHelper.java b/src/main/java/com/percero/amqp/RabbitMQPushSyncHelper.java index b14dc91..fdc5015 100644 --- a/src/main/java/com/percero/amqp/RabbitMQPushSyncHelper.java +++ b/src/main/java/com/percero/amqp/RabbitMQPushSyncHelper.java @@ -37,6 +37,7 @@ import org.springframework.amqp.support.converter.MessageConversionException; import org.springframework.beans.BeansException; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.beans.factory.annotation.Value; import org.springframework.context.ApplicationContext; import org.springframework.context.ApplicationContextAware; @@ -86,8 +87,12 @@ public class RabbitMQPushSyncHelper implements IPushSyncHelper, ApplicationConte // RabbitMQ Components @Resource AmqpAdmin amqpAdmin; - @Resource + AbstractMessageListenerContainer rabbitMessageListenerContainer; + @Resource @Qualifier("defaultListenerContainer") + public void setRabbitMessageListenerContainer(AbstractMessageListenerContainer container){ + rabbitMessageListenerContainer = container; + } // RabbitMQ environment variables. @Autowired @Value("$pf{gateway.rabbitmq.admin_port:15672}") diff --git a/src/main/resources/spring/basic_components_spring_config.xml b/src/main/resources/spring/basic_components_spring_config.xml index 1a55e2d..6f49312 100644 --- a/src/main/resources/spring/basic_components_spring_config.xml +++ b/src/main/resources/spring/basic_components_spring_config.xml @@ -132,6 +132,7 @@ + - - + + + + + + diff --git a/src/main/resources/spring/messageListener-spring-config.xml b/src/main/resources/spring/messageListener-spring-config.xml index ac2c921..024bd01 100644 --- a/src/main/resources/spring/messageListener-spring-config.xml +++ b/src/main/resources/spring/messageListener-spring-config.xml @@ -62,7 +62,7 @@ - + diff --git a/src/main/resources/spring/percero-spring-config.xml b/src/main/resources/spring/percero-spring-config.xml index cf2710e..ed759f5 100644 --- a/src/main/resources/spring/percero-spring-config.xml +++ b/src/main/resources/spring/percero-spring-config.xml @@ -62,7 +62,7 @@ - + diff --git a/src/main/resources/spring/updateTable-spring-config.xml b/src/main/resources/spring/updateTable-spring-config.xml index b8e5241..cce5ace 100644 --- a/src/main/resources/spring/updateTable-spring-config.xml +++ b/src/main/resources/spring/updateTable-spring-config.xml @@ -19,7 +19,7 @@ - + From bba26ad1adbc1d6c47c905e08311b895c86bc182 Mon Sep 17 00:00:00 2001 From: Jonathan Samples Date: Wed, 25 Nov 2015 07:39:55 -0800 Subject: [PATCH 073/105] bumped ther version --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 87f8d13..e1a6a5d 100644 --- a/pom.xml +++ b/pom.xml @@ -7,7 +7,7 @@ org.activestack syncengine - 1.1.51-SNAPSHOT + 1.1.52-SNAPSHOT 3.2.4.RELEASE From 0b1263fd2056ab426696c4904e0e7c4b37f7cfe8 Mon Sep 17 00:00:00 2001 From: Jonathan Samples Date: Wed, 25 Nov 2015 09:42:57 -0800 Subject: [PATCH 074/105] Added 32 concurrency to checkChangeWatcher listener --- src/main/resources/spring/changeWatcher-spring-config.xml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/main/resources/spring/changeWatcher-spring-config.xml b/src/main/resources/spring/changeWatcher-spring-config.xml index ae9491d..ff7c5e8 100644 --- a/src/main/resources/spring/changeWatcher-spring-config.xml +++ b/src/main/resources/spring/changeWatcher-spring-config.xml @@ -20,9 +20,7 @@ - + - - From fb261b8639c1905d86b6da79296a991f9a073db6 Mon Sep 17 00:00:00 2001 From: Jonathan Samples Date: Wed, 25 Nov 2015 09:44:24 -0800 Subject: [PATCH 075/105] bumped version --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index e1a6a5d..3ae1c8e 100644 --- a/pom.xml +++ b/pom.xml @@ -7,7 +7,7 @@ org.activestack syncengine - 1.1.52-SNAPSHOT + 1.1.53-SNAPSHOT 3.2.4.RELEASE From cae45d8e1d1ea63511b8e226c307946f15e94c2c Mon Sep 17 00:00:00 2001 From: Jonathan Samples Date: Tue, 1 Dec 2015 23:36:11 -0800 Subject: [PATCH 076/105] Added diff checking for update table pushes --- pom.xml | 2 +- .../agents/sync/helpers/PostPutHelper.java | 1 + .../sync/jobs/UpdateTableProcessor.java | 553 +++++++++--------- 3 files changed, 289 insertions(+), 267 deletions(-) diff --git a/pom.xml b/pom.xml index 3ae1c8e..e52e355 100644 --- a/pom.xml +++ b/pom.xml @@ -7,7 +7,7 @@ org.activestack syncengine - 1.1.53-SNAPSHOT + 1.1.54-SNAPSHOT 3.2.4.RELEASE diff --git a/src/main/java/com/percero/agents/sync/helpers/PostPutHelper.java b/src/main/java/com/percero/agents/sync/helpers/PostPutHelper.java index 2dcff59..62d34f4 100644 --- a/src/main/java/com/percero/agents/sync/helpers/PostPutHelper.java +++ b/src/main/java/com/percero/agents/sync/helpers/PostPutHelper.java @@ -125,6 +125,7 @@ public void postPutObject(ClassIDPair pair, String pusherUserId, String pusherCl // accessManager.checkChangeWatchers(thePair, fieldNames, null); } } + // Iterator itrChangedFieldKeyset = changedFields.keySet().iterator(); // while (itrChangedFieldKeyset.hasNext()) { // ClassIDPair thePair = itrChangedFieldKeyset.next(); diff --git a/src/main/java/com/percero/agents/sync/jobs/UpdateTableProcessor.java b/src/main/java/com/percero/agents/sync/jobs/UpdateTableProcessor.java index 0de90a0..fa3a675 100644 --- a/src/main/java/com/percero/agents/sync/jobs/UpdateTableProcessor.java +++ b/src/main/java/com/percero/agents/sync/jobs/UpdateTableProcessor.java @@ -83,24 +83,24 @@ public UpdateTableProcessor(String tableName, this.accessManager = accessManager; this.maxRowsToProcess = connectionFactory.getWeight(); this.poller = poller; - + classNamesToUpdateReferences = new HashSet(); } - - + + public String getProcessorName() { - if (connectionFactory != null && connectionFactory.getJdbcUrl() != null && tableName != null) { - return connectionFactory.getJdbcUrl() + "::" + tableName; - } - else { - return ""; - } + if (connectionFactory != null && connectionFactory.getJdbcUrl() != null && tableName != null) { + return connectionFactory.getJdbcUrl() + "::" + tableName; + } + else { + return ""; + } } - - - // TODO: This should be moved to be stored in a database table. Could - // possibly use the existing "entire table update" mechanism to handle this - // (for each reverse mapped class) + + + // TODO: This should be moved to be stored in a database table. Could + // possibly use the existing "entire table update" mechanism to handle this + // (for each reverse mapped class) private Set classNamesToUpdateReferences = null; /** @@ -144,11 +144,11 @@ public void run(){ UpdateTableProcessReporter.getInstance() .submitCountAndTime(tableName, successfulRows.size(), endTime.getTime() - startTime.getTime()); } - + for(String className : classNamesToUpdateReferences) { - updateReferences(className); + updateReferences(className); } - + if (!result.isSuccess()) { logger.warn("Update table processor (" + tableName + ") failed. Details:"); logger.warn(result); @@ -159,10 +159,10 @@ public void run(){ conn.close(); }catch(Exception e){} } - + // Callback if (poller != null) { - poller.processorCallback(this); + poller.processorCallback(this); } } @@ -205,25 +205,25 @@ protected void processRow(UpdateTableRow row) throws Exception{ @SuppressWarnings("rawtypes") protected void processUpdateSingle(UpdateTableRow row) throws Exception{ List classes = getClassesForTableName(row.getTableName()); - + if (classes == null || classes.isEmpty()) { - logger.warn("No Classes defined for UpdateTable table " + row.getTableName()); + logger.warn("No Classes defined for UpdateTable table " + row.getTableName()); } else { - logger.debug("Processing " + classes.size() + " class(es) for UpdateTable UPDATE, table " + row.getTableName()); - for(Class clazz : classes) { - String className = clazz.getCanonicalName(); - - IMappedClassManager mcm = MappedClassManagerFactory.getMappedClassManager(); - MappedClass mappedClass = mcm.getMappedClassByClassName(className); - IDataProvider dataProvider = dataProviderManager.getDataProviderByName(mappedClass.dataProviderName); - - ClassIDPair pair = new ClassIDPair(row.getRowId(), className); - - if (pair != null && StringUtils.hasText(pair.getID()) && StringUtils.hasText(pair.getClassName())) { - handleUpdateClassIdPair(dataProvider, pair); - } - } + logger.debug("Processing " + classes.size() + " class(es) for UpdateTable UPDATE, table " + row.getTableName()); + for(Class clazz : classes) { + String className = clazz.getCanonicalName(); + + IMappedClassManager mcm = MappedClassManagerFactory.getMappedClassManager(); + MappedClass mappedClass = mcm.getMappedClassByClassName(className); + IDataProvider dataProvider = dataProviderManager.getDataProviderByName(mappedClass.dataProviderName); + + ClassIDPair pair = new ClassIDPair(row.getRowId(), className); + + if (pair != null && StringUtils.hasText(pair.getID()) && StringUtils.hasText(pair.getClassName())) { + handleUpdateClassIdPair(dataProvider, pair); + } + } } } @@ -237,47 +237,47 @@ protected void processUpdateTable(UpdateTableRow row) throws Exception{ List classes = getClassesForTableName(row.getTableName()); if (classes == null || classes.isEmpty()) { - logger.warn("No Classes defined for UpdateTable table " + row.getTableName()); + logger.warn("No Classes defined for UpdateTable table " + row.getTableName()); } else { - logger.debug("Processing " + classes.size() + " class(es) for UpdateTable UPDATE, table " + row.getTableName()); - for(Class clazz : classes) { - String className = clazz.getCanonicalName(); - - // If there are any clients that have asked for all objects in a class then we have to push everything - if (accessManager.getNumClientsInterestedInWholeClass(className) > 0) { - - IMappedClassManager mcm = MappedClassManagerFactory.getMappedClassManager(); - MappedClass mappedClass = mcm.getMappedClassByClassName(className); - IDataProvider dataProvider = dataProviderManager.getDataProviderByName(mappedClass.dataProviderName); - - Integer pageNumber = 0; - Integer pageSize = 25; - Integer total = -1; - - while (total < 0 || pageNumber * pageSize <= total) { - PerceroList objectsToUpdate = dataProvider.getAllByName(className, pageNumber, pageSize, true, null); - pageNumber++; - total = objectsToUpdate.getTotalLength(); - if (total <= 0) { - break; - } - - Set objectIds = new HashSet(objectsToUpdate.size()); - Iterator itrObjectsToUpdate = objectsToUpdate.iterator(); - while (itrObjectsToUpdate.hasNext()) { - IPerceroObject nextObjectToUpdate = itrObjectsToUpdate.next(); - objectIds.add(nextObjectToUpdate.getID()); - } - - processUpdates(className, objectIds); - } - } else { - processUpdates(className, accessManager.getClassAccessJournalIDs(className)); - } - - updateReferences(className); - } + logger.debug("Processing " + classes.size() + " class(es) for UpdateTable UPDATE, table " + row.getTableName()); + for(Class clazz : classes) { + String className = clazz.getCanonicalName(); + + // If there are any clients that have asked for all objects in a class then we have to push everything + if (accessManager.getNumClientsInterestedInWholeClass(className) > 0) { + + IMappedClassManager mcm = MappedClassManagerFactory.getMappedClassManager(); + MappedClass mappedClass = mcm.getMappedClassByClassName(className); + IDataProvider dataProvider = dataProviderManager.getDataProviderByName(mappedClass.dataProviderName); + + Integer pageNumber = 0; + Integer pageSize = 25; + Integer total = -1; + + while (total < 0 || pageNumber * pageSize <= total) { + PerceroList objectsToUpdate = dataProvider.getAllByName(className, pageNumber, pageSize, true, null); + pageNumber++; + total = objectsToUpdate.getTotalLength(); + if (total <= 0) { + break; + } + + Set objectIds = new HashSet(objectsToUpdate.size()); + Iterator itrObjectsToUpdate = objectsToUpdate.iterator(); + while (itrObjectsToUpdate.hasNext()) { + IPerceroObject nextObjectToUpdate = itrObjectsToUpdate.next(); + objectIds.add(nextObjectToUpdate.getID()); + } + + processUpdates(className, objectIds); + } + } else { + processUpdates(className, accessManager.getClassAccessJournalIDs(className)); + } + + updateReferences(className); + } } } @@ -294,74 +294,74 @@ protected void processUpdates(String className, Collection Ids) throws E for(String ID : Ids) { ClassIDPair pair = new ClassIDPair(ID, className); - + if (pair != null && StringUtils.hasText(pair.getID()) && StringUtils.hasText(pair.getClassName())) { - handleUpdateClassIdPair(dataProvider, pair); + handleUpdateClassIdPair(dataProvider, pair); } } } protected void handleUpdateClassIdPair(IDataProvider dataProvider, ClassIDPair pair) throws Exception { // Attempt to retrieve from the cache so that we have the "OLD" value. - IPerceroObject oldValue = dataProvider.retrieveCachedObject(pair); - // Now retrieve the object directly from the data source so that we - // can compare it to the cached/old value. + IPerceroObject oldValue = dataProvider.retrieveCachedObject(pair); + // Now retrieve the object directly from the data source so that we + // can compare it to the cached/old value. IPerceroObject perceroObject = dataProvider.findById(pair, null, true); // If PerceroObject is NULL, then it no longer exists and we can drop this update. if (perceroObject == null) { - return; + return; } - + Map> changedFields = null; if (oldValue != null) { - // dataProvider.getChangedMappedFields is typically used to - // compare a new object, but we can use the cached object in - // this case (we just need to tell getChangedMappedFields to NOT - // use the cache). - changedFields = dataProvider - .getChangedMappedFields(perceroObject, oldValue, oldValue != null); - if (changedFields != null && !changedFields.isEmpty()) { - // Something has changed. - cacheManager.updateCachedObject(perceroObject, changedFields); - postPutHelper.postPutObject(pair, null, null, true, changedFields); - - Iterator>> itrChangedFieldEntryset = changedFields.entrySet().iterator(); - while (itrChangedFieldEntryset.hasNext()) { - Map.Entry> nextEntry = itrChangedFieldEntryset.next(); - ClassIDPair thePair = nextEntry.getKey(); - Collection changedMappedFields = nextEntry.getValue(); - - // If thePair is NOT the object being updated, then need to run the postPutHelper for the Pair object as well. - if (!thePair.equals(pair)) { - Map> thePairChangedFields = new HashMap>(1); - thePairChangedFields.put(thePair, changedMappedFields); - - postPutHelper.postPutObject(thePair, null, null, true, thePairChangedFields); - } - else { - Iterator itrChangedFields = changedMappedFields.iterator(); - String[] fieldNames = new String[changedMappedFields.size()]; - int i = 0; - while (itrChangedFields.hasNext()) { - MappedField nextChangedField = itrChangedFields.next(); - fieldNames[i] = nextChangedField.getField().getName(); - i++; - } - accessManager.checkChangeWatchers(thePair, fieldNames, null); - } - } - } + // dataProvider.getChangedMappedFields is typically used to + // compare a new object, but we can use the cached object in + // this case (we just need to tell getChangedMappedFields to NOT + // use the cache). + changedFields = dataProvider + .getChangedMappedFields(perceroObject, oldValue, oldValue != null); + if (changedFields != null && !changedFields.isEmpty()) { + // Something has changed. + cacheManager.updateCachedObject(perceroObject, changedFields); + postPutHelper.postPutObject(pair, null, null, true, changedFields); + + Iterator>> itrChangedFieldEntryset = changedFields.entrySet().iterator(); + while (itrChangedFieldEntryset.hasNext()) { + Map.Entry> nextEntry = itrChangedFieldEntryset.next(); + ClassIDPair thePair = nextEntry.getKey(); + Collection changedMappedFields = nextEntry.getValue(); + + // If thePair is NOT the object being updated, then need to run the postPutHelper for the Pair object as well. + if (!thePair.equals(pair)) { + Map> thePairChangedFields = new HashMap>(1); + thePairChangedFields.put(thePair, changedMappedFields); + + postPutHelper.postPutObject(thePair, null, null, true, thePairChangedFields); + } + else { + Iterator itrChangedFields = changedMappedFields.iterator(); + String[] fieldNames = new String[changedMappedFields.size()]; + int i = 0; + while (itrChangedFields.hasNext()) { + MappedField nextChangedField = itrChangedFields.next(); + fieldNames[i] = nextChangedField.getField().getName(); + i++; + } + accessManager.checkChangeWatchers(thePair, fieldNames, null); + } + } + } } else { - cacheManager.updateCachedObject(perceroObject, null); - postPutHelper.postPutObject(pair, null, null, true, null); - - // We don't have any record of the old value, so we need to - // update ALL referencing objects in the case that a - // relationship was updated. Since we don't have the OLD object, - // we have no way of telling what may have changed. - classNamesToUpdateReferences.add(pair.getClassName()); - } + cacheManager.updateCachedObject(perceroObject, null); + postPutHelper.postPutObject(pair, null, null, true, null); + + // We don't have any record of the old value, so we need to + // update ALL referencing objects in the case that a + // relationship was updated. Since we don't have the OLD object, + // we have no way of telling what may have changed. + classNamesToUpdateReferences.add(pair.getClassName()); + } } /** @@ -374,19 +374,19 @@ protected void processInsertSingle(UpdateTableRow row) throws Exception{ List classes = getClassesForTableName(row.getTableName()); if (classes == null || classes.isEmpty()) { - logger.warn("No Classes defined for UpdateTable table " + row.getTableName()); + logger.warn("No Classes defined for UpdateTable table " + row.getTableName()); } else { - logger.debug("Processing " + classes.size() + " class(es) for UpdateTable INSERT, table " + row.getTableName()); - for(Class clazz : classes) { - String className = clazz.getCanonicalName(); - - // We do not use PostCreateHelper here because we are going to do all - // that extra work for the whole class in updateReferences. - postPutHelper.postPutObject(new ClassIDPair(row.getRowId(), className), null, null, true, null); - - classNamesToUpdateReferences.add(className); - } + logger.debug("Processing " + classes.size() + " class(es) for UpdateTable INSERT, table " + row.getTableName()); + for(Class clazz : classes) { + String className = clazz.getCanonicalName(); + + // We do not use PostCreateHelper here because we are going to do all + // that extra work for the whole class in updateReferences. + postPutHelper.postPutObject(new ClassIDPair(row.getRowId(), className), null, null, true, null); + + classNamesToUpdateReferences.add(className); + } } } @@ -400,26 +400,26 @@ protected void processInsertTable(UpdateTableRow row) throws Exception { List classes = getClassesForTableName(row.getTableName()); if (classes == null || classes.isEmpty()) { - logger.warn("No Classes defined for UpdateTable table " + row.getTableName()); + logger.warn("No Classes defined for UpdateTable table " + row.getTableName()); } else { - logger.debug("Processing " + classes.size() + " class(es) for UpdateTable INSERT, table " + row.getTableName()); - for(Class clazz : classes) { - String className = clazz.getCanonicalName(); - - // if any client needs all of this class then the only choice we have is to push everything - if (accessManager.getNumClientsInterestedInWholeClass(className) > 0 /* || true */) { - Set allClassIdPairs = getAllClassIdPairsForTable(row.getTableName()); - for (ClassIDPair classIdPair : allClassIdPairs) { - // We do not use PostCreateHelper here because we are going to - // do all that extra work for the whole class in - // updateReferences. - postPutHelper.postPutObject(classIdPair, null, null, true, null); - } - } - - classNamesToUpdateReferences.add(className); - } + logger.debug("Processing " + classes.size() + " class(es) for UpdateTable INSERT, table " + row.getTableName()); + for(Class clazz : classes) { + String className = clazz.getCanonicalName(); + + // if any client needs all of this class then the only choice we have is to push everything + if (accessManager.getNumClientsInterestedInWholeClass(className) > 0 /* || true */) { + Set allClassIdPairs = getAllClassIdPairsForTable(row.getTableName()); + for (ClassIDPair classIdPair : allClassIdPairs) { + // We do not use PostCreateHelper here because we are going to + // do all that extra work for the whole class in + // updateReferences. + postPutHelper.postPutObject(classIdPair, null, null, true, null); + } + } + + classNamesToUpdateReferences.add(className); + } } } @@ -431,26 +431,26 @@ protected void processInsertTable(UpdateTableRow row) throws Exception { @SuppressWarnings("rawtypes") protected void processDeleteSingle(UpdateTableRow row) throws Exception{ List classes = getClassesForTableName(row.getTableName()); - + if (classes == null || classes.isEmpty()) { - logger.warn("No Classes defined for UpdateTable table " + row.getTableName()); + logger.warn("No Classes defined for UpdateTable table " + row.getTableName()); } else { - logger.debug("Processing " + classes.size() + " class(es) for UpdateTable DELETE, table " + row.getTableName()); - for(Class clazz : classes){ - String className = clazz.getCanonicalName(); - - // See if this object is in the cache. If so, it will help us know which related objects to update. - IMappedClassManager mcm = MappedClassManagerFactory.getMappedClassManager(); - MappedClass mappedClass = mcm.getMappedClassByClassName(className); - IDataProvider dataProvider = dataProviderManager.getDataProviderByName(mappedClass.dataProviderName); - ClassIDPair pair = new ClassIDPair(row.getRowId(), className); - IPerceroObject cachedObject = dataProvider.findById(pair, null, false); // We are hoping to find this object in the cache... - - handleDeletedObject(cachedObject, clazz, className, row.getRowId()); - - classNamesToUpdateReferences.add(className); - } + logger.debug("Processing " + classes.size() + " class(es) for UpdateTable DELETE, table " + row.getTableName()); + for(Class clazz : classes){ + String className = clazz.getCanonicalName(); + + // See if this object is in the cache. If so, it will help us know which related objects to update. + IMappedClassManager mcm = MappedClassManagerFactory.getMappedClassManager(); + MappedClass mappedClass = mcm.getMappedClassByClassName(className); + IDataProvider dataProvider = dataProviderManager.getDataProviderByName(mappedClass.dataProviderName); + ClassIDPair pair = new ClassIDPair(row.getRowId(), className); + IPerceroObject cachedObject = dataProvider.findById(pair, null, false); // We are hoping to find this object in the cache... + + handleDeletedObject(cachedObject, clazz, className, row.getRowId()); + + classNamesToUpdateReferences.add(className); + } } } @@ -464,50 +464,50 @@ protected void processDeleteTable(UpdateTableRow row) throws Exception{ List classes = getClassesForTableName(row.getTableName()); if (classes == null || classes.isEmpty()) { - logger.warn("No Classes defined for UpdateTable table " + row.getTableName()); + logger.warn("No Classes defined for UpdateTable table " + row.getTableName()); } else { - logger.debug("Processing " + classes.size() + " class(es) for UpdateTable DELETE, table " + row.getTableName()); - for(Class clazz : classes){ - String className = clazz.getCanonicalName(); - - IMappedClassManager mcm = MappedClassManagerFactory.getMappedClassManager(); - MappedClass mappedClass = mcm.getMappedClassByClassName(className); - IDataProvider dataProvider = dataProviderManager.getDataProviderByName(mappedClass.dataProviderName); - - // Get the list of ALL ID's of this class type that have been accessed. - Set accessedIds = accessManager.getClassAccessJournalIDs(className); - - if (accessedIds != null && !accessedIds.isEmpty()) { - // TODO: If ID "0", then that means someone wants to know about ALL - // records of this type. How do we do this? - - // Get a list of ALL ID's of this class type. - Set allClassIdPairs = getAllClassIdPairsForTable(row.getTableName()); - - // Remove ALL existing/current ID's from our list of accessed ID's. - for (ClassIDPair nextClassIdPair : allClassIdPairs) { - accessedIds.remove(nextClassIdPair.getID()); - } - - // Now we have the list of ID's that have actually been deleted. - for (String id : accessedIds) { - // Find the cached object first so that it is NOT removed if the same object is NOT found in the data store. - IPerceroObject cachedObject = dataProvider.findById(new ClassIDPair(id, className), null, false); - // We will know an object has been deleted IFF it does NOT exist in the data store. - IPerceroObject dataStoreObject = dataProvider.findById(new ClassIDPair(id, className), null, true); - - if (dataStoreObject != null) { - // Object has NOT been deleted. - continue; - } - - handleDeletedObject(cachedObject, clazz, className, id); - } - } - - classNamesToUpdateReferences.add(className); - } + logger.debug("Processing " + classes.size() + " class(es) for UpdateTable DELETE, table " + row.getTableName()); + for(Class clazz : classes){ + String className = clazz.getCanonicalName(); + + IMappedClassManager mcm = MappedClassManagerFactory.getMappedClassManager(); + MappedClass mappedClass = mcm.getMappedClassByClassName(className); + IDataProvider dataProvider = dataProviderManager.getDataProviderByName(mappedClass.dataProviderName); + + // Get the list of ALL ID's of this class type that have been accessed. + Set accessedIds = accessManager.getClassAccessJournalIDs(className); + + if (accessedIds != null && !accessedIds.isEmpty()) { + // TODO: If ID "0", then that means someone wants to know about ALL + // records of this type. How do we do this? + + // Get a list of ALL ID's of this class type. + Set allClassIdPairs = getAllClassIdPairsForTable(row.getTableName()); + + // Remove ALL existing/current ID's from our list of accessed ID's. + for (ClassIDPair nextClassIdPair : allClassIdPairs) { + accessedIds.remove(nextClassIdPair.getID()); + } + + // Now we have the list of ID's that have actually been deleted. + for (String id : accessedIds) { + // Find the cached object first so that it is NOT removed if the same object is NOT found in the data store. + IPerceroObject cachedObject = dataProvider.findById(new ClassIDPair(id, className), null, false); + // We will know an object has been deleted IFF it does NOT exist in the data store. + IPerceroObject dataStoreObject = dataProvider.findById(new ClassIDPair(id, className), null, true); + + if (dataStoreObject != null) { + // Object has NOT been deleted. + continue; + } + + handleDeletedObject(cachedObject, clazz, className, id); + } + } + + classNamesToUpdateReferences.add(className); + } } } @@ -546,7 +546,6 @@ protected Set getAllClassIdPairsForTable(String tableName) throws E * Finds all back references to this class and pushes updates to all of them. * @param className */ - // TODO: Need to batch this. protected void updateReferences(String className){ IMappedClassManager mcm = MappedClassManagerFactory.getMappedClassManager(); MappedClass mappedClass = mcm.getMappedClassByClassName(className); @@ -581,16 +580,18 @@ protected void updateReferences(String className){ Iterator itrObjectsToUpdate = objectsToUpdate.iterator(); while (itrObjectsToUpdate.hasNext()) { - IPerceroObject nextObjectToUpdate = itrObjectsToUpdate.next(); + IDataProvider dataProvider = dataProviderManager.getDefaultDataProvider(); + // We assume this is from the cache because some client has requested all of this class + // and since we turned off cache expiration it must be in there. + BaseDataObject nextObjectToUpdate = (BaseDataObject) itrObjectsToUpdate.next(); ClassIDPair pair = BaseDataObject.toClassIdPair(nextObjectToUpdate); - Map> changedFields = new HashMap>(); - Collection changedMappedFields = new ArrayList(1); - changedMappedFields.add(mappedField); - changedFields.put(pair, changedMappedFields); - - // Remove from the cache. - cacheManager.deleteObjectFromCache(pair); - postPutHelper.postPutObject(pair, null, null, true, changedFields); + // This call ignores the cache + BaseDataObject nextObjectToUpdateFromDB = (BaseDataObject) dataProvider.findById(pair, null, true); + + // Compare their JSON selves for "deep" equality, only push if different + if(!nextObjectToUpdate.toJson().equals(nextObjectToUpdateFromDB.toJson())) { + pushPair(pair, mappedField); + } } } } @@ -599,14 +600,18 @@ protected void updateReferences(String className){ while (itrIdsToUpdate.hasNext()) { String nextIdToUpdate = itrIdsToUpdate.next(); ClassIDPair pair = new ClassIDPair(nextIdToUpdate, mappedField.getMappedClass().className); - Map> changedFields = new HashMap>(); - Collection changedMappedFields = new ArrayList(1); - changedMappedFields.add(mappedField); - changedFields.put(pair, changedMappedFields); - - // Remove from the cache. - cacheManager.deleteObjectFromCache(pair); - postPutHelper.postPutObject(pair, null, null, true, changedFields); + + IDataProvider dataProvider = dataProviderManager.getDefaultDataProvider(); + + // First call assumes we are getting it from the cache... because we can assume that + // All objects that we care about are all cached (since we turned off cache expiration). + BaseDataObject cachedObject = (BaseDataObject) dataProvider.findById(pair, null); + // This call ignores the cache + BaseDataObject objectFromDB = (BaseDataObject) dataProvider.findById(pair, null, true); + // Only push if different + if(!cachedObject.toJson().equals(objectFromDB.toJson())) { + pushPair(pair, mappedField); + } } } } @@ -616,6 +621,22 @@ protected void updateReferences(String className){ } } + private void pushPair(ClassIDPair pair, MappedField mappedField){ + try { + Map> changedFields = new HashMap>(); + Collection changedMappedFields = new ArrayList(1); + changedMappedFields.add(mappedField); + changedFields.put(pair, changedMappedFields); + + + // Remove from the cache. + cacheManager.deleteObjectFromCache(pair); + postPutHelper.postPutObject(pair, null, null, true, changedFields); + }catch(Exception e){ + logger.error("Error pushing ClassIDPair" + mappedField.getMappedClass().className + "." + mappedField.getField().getName(), e); + } + } + /** * Pulls a row off the update table and locks it so that other * processors don't duplicate the work @@ -635,19 +656,19 @@ public List getRows(int numRows){ try { rows = getStoredProcRow(lockId, expireThreshold, numRows); } catch (Exception e) { - logger.error("Error running Update Table stored procedure for " + connectionFactory.getJdbcUrl() + "\n Failing over to get UpdateTable with SELECT statement", e); - try { - rows = getUpdateSelectRow(lockId, expireThreshold, numRows); - } catch (SQLException e1) { - logger.error("Error running Update Table SELECT statement as backup to Stored Procedure\n UPDATE TABLE NON_FUNCTIONAL " + connectionFactory.getJdbcUrl(), e1); - } + logger.error("Error running Update Table stored procedure for " + connectionFactory.getJdbcUrl() + "\n Failing over to get UpdateTable with SELECT statement", e); + try { + rows = getUpdateSelectRow(lockId, expireThreshold, numRows); + } catch (SQLException e1) { + logger.error("Error running Update Table SELECT statement as backup to Stored Procedure\n UPDATE TABLE NON_FUNCTIONAL " + connectionFactory.getJdbcUrl(), e1); + } } } else { - try { - rows = getUpdateSelectRow(lockId, expireThreshold, numRows); - } catch (SQLException e1) { - logger.error("Error running Update Table SELECT statement as backup to Stored Procedure\n UPDATE TABLE NON_FUNCTIONAL " + connectionFactory.getJdbcUrl(), e1); - } + try { + rows = getUpdateSelectRow(lockId, expireThreshold, numRows); + } catch (SQLException e1) { + logger.error("Error running Update Table SELECT statement as backup to Stored Procedure\n UPDATE TABLE NON_FUNCTIONAL " + connectionFactory.getJdbcUrl(), e1); + } } @@ -765,14 +786,14 @@ private List getStoredProcRow(int lockId, DateTime expireThresho * @param rows */ protected void deleteRows(List rows){ - if (rows == null || rows.isEmpty()) { - return; - } - String sql = "delete from " + tableName + " where " + connectionFactory.getLockIdColumnName() + "=" + rows.get(0).getLockId(); - + if (rows == null || rows.isEmpty()) { + return; + } + String sql = "delete from " + tableName + " where " + connectionFactory.getLockIdColumnName() + "=" + rows.get(0).getLockId(); + try(PreparedStatement statement = getConnection().prepareStatement(sql)){ - - int numUpdated = statement.executeUpdate(sql); + + int numUpdated = statement.executeUpdate(sql); if(numUpdated != rows.size()){ logger.warn("Expected to delete "+rows.size()+", instead "+numUpdated); } @@ -782,22 +803,22 @@ protected void deleteRows(List rows){ } protected void releaseRowLock(UpdateTableRow row){ - if (row == null) { - return; - } - String sql = "UPDATE " + tableName + " SET " + connectionFactory.getLockIdColumnName() + "=NULL, " + connectionFactory.getLockDateColumnName() + "=NULL where ID=" + row.getID(); - - try(PreparedStatement statement = getConnection().prepareStatement(sql)){ - - int numUpdated = statement.executeUpdate(sql); - if(numUpdated != 1){ - logger.warn("Expected to update 1, instead "+numUpdated); - } - }catch(SQLException e){ - logger.warn(e.getMessage(), e); - } + if (row == null) { + return; + } + String sql = "UPDATE " + tableName + " SET " + connectionFactory.getLockIdColumnName() + "=NULL, " + connectionFactory.getLockDateColumnName() + "=NULL where ID=" + row.getID(); + + try(PreparedStatement statement = getConnection().prepareStatement(sql)){ + + int numUpdated = statement.executeUpdate(sql); + if(numUpdated != 1){ + logger.warn("Expected to update 1, instead "+numUpdated); + } + }catch(SQLException e){ + logger.warn(e.getMessage(), e); + } } - + // /** // * Utility function that takes a list of rows and returns a comma separated string // * @param list From b4c103e522b2d776c308bbe07738fab6667d46e1 Mon Sep 17 00:00:00 2001 From: Nikish Parikh Date: Fri, 4 Dec 2015 03:38:58 +0530 Subject: [PATCH 077/105] Stack traced added to RabbitMQPushSyncHelper and logback supported added to remote loging. --- logback.xml | 9 ++++ pom.xml | 7 ++- .../percero/amqp/RabbitMQPushSyncHelper.java | 49 ++++++++++++++++++- 3 files changed, 63 insertions(+), 2 deletions(-) create mode 100644 logback.xml diff --git a/logback.xml b/logback.xml new file mode 100644 index 0000000..dbdd760 --- /dev/null +++ b/logback.xml @@ -0,0 +1,9 @@ + + + http://logs-01.loggly.com/inputs/7f44497d-2cbe-488c-a88e-87ca01ab5e08/tag/logback + %d{"ISO8601", UTC} %p %t %c{0}.%M - %m%n + + + + + diff --git a/pom.xml b/pom.xml index e52e355..c8d49d0 100644 --- a/pom.xml +++ b/pom.xml @@ -341,6 +341,11 @@ yamlbeans 1.09 + + org.logback-extensions + logback-ext-loggly + 0.1.2 + @@ -390,4 +395,4 @@ - \ No newline at end of file + diff --git a/src/main/java/com/percero/amqp/RabbitMQPushSyncHelper.java b/src/main/java/com/percero/amqp/RabbitMQPushSyncHelper.java index fdc5015..ea30523 100644 --- a/src/main/java/com/percero/amqp/RabbitMQPushSyncHelper.java +++ b/src/main/java/com/percero/amqp/RabbitMQPushSyncHelper.java @@ -61,10 +61,13 @@ import edu.emory.mathcs.backport.java.util.Arrays; +import org.slf4j.*; + @Component public class RabbitMQPushSyncHelper implements IPushSyncHelper, ApplicationContextAware { private static Logger logger = Logger.getLogger(RabbitMQPushSyncHelper.class); + private static final org.slf4j.Logger slf4jLogger = LoggerFactory.getLogger(RabbitMQPushSyncHelper.class); public static final String DEFAULT_CHARSET = "UTF-8"; @@ -115,6 +118,8 @@ public void setCacheDataStore(ICacheDataStore cacheDataStore) { @SuppressWarnings("rawtypes") protected void pushJsonToRouting(String objectJson, Class objectClass, String routingKey) { try{ + printCurrentStackTrace(); + Message convertedMessage = toMessage(objectJson, objectClass, MessageProperties.CONTENT_TYPE_JSON); template.send(routingKey, convertedMessage); } @@ -125,6 +130,8 @@ protected void pushJsonToRouting(String objectJson, Class objectClass, String ro protected void pushMessageToRouting(Message convertedMessage, String routingKey) { try{ + printCurrentStackTrace(); + template.send(routingKey, convertedMessage); } catch(Exception e){ @@ -135,6 +142,8 @@ protected void pushMessageToRouting(Message convertedMessage, String routingKey) @SuppressWarnings("rawtypes") protected void pushStringToRouting(String objectJson, Class objectClass, String routingKey) { try{ + printCurrentStackTrace(); + Message convertedMessage = toMessage(objectJson, objectClass, MessageProperties.CONTENT_TYPE_BYTES); template.send(routingKey, convertedMessage); } @@ -146,6 +155,8 @@ protected void pushStringToRouting(String objectJson, Class objectClass, String @SuppressWarnings("rawtypes") public final Message toMessage(String objectJson, Class objectClass, String contentEncoding) throws MessageConversionException { + printCurrentStackTrace(); + MessageProperties messageProperties = new MessageProperties(); messageProperties.setContentType(MessageProperties.CONTENT_TYPE_JSON); messageProperties.setContentEncoding(this.defaultCharset); @@ -156,6 +167,8 @@ public final Message toMessage(String objectJson, Class objectClass, String cont @SuppressWarnings("rawtypes") public final Message toMessage(String objectJson, Class objectClass, MessageProperties messageProperties, String contentEncoding) throws MessageConversionException { + printCurrentStackTrace(); + Message message = createMessage(objectJson, objectClass, messageProperties, contentEncoding); return message; } @@ -163,6 +176,9 @@ public final Message toMessage(String objectJson, Class objectClass, MessageProp @SuppressWarnings("rawtypes") protected Message createMessage(String aString, Class objectClass, MessageProperties messageProperties, String contentEncoding) throws MessageConversionException { + + printCurrentStackTrace(); + byte[] bytes = null; try { String jsonString = aString; @@ -181,6 +197,8 @@ protected Message createMessage(String aString, Class objectClass, MessageProper } public void pushSyncResponseToClient(SyncResponse anObject, String clientId) { + printCurrentStackTrace(); + if (anObject != null && StringUtils.hasText(clientId)) { pushJsonToRouting(anObject.toJson(objectMapper), anObject.getClass(), clientId); } @@ -188,6 +206,8 @@ public void pushSyncResponseToClient(SyncResponse anObject, String clientId) { @SuppressWarnings("rawtypes") public void pushSyncResponseToClients(SyncResponse syncResponse, Collection clientIds) { + printCurrentStackTrace(); + if ( syncResponse != null && clientIds != null && !clientIds.isEmpty() ) { Class objectClass = syncResponse.getClass(); @@ -202,6 +222,8 @@ public void pushSyncResponseToClients(SyncResponse syncResponse, Collection listClients) { + printCurrentStackTrace(); + if (anObject != null && listClients != null && !listClients.isEmpty() ) { // Route to specific clients. // Optimization: create the JSON string of the object. @@ -225,6 +247,8 @@ public void pushObjectToClients(Object anObject, Collection listClients) @Override public void pushStringToRoute(String aString, String routeName) { + printCurrentStackTrace(); + if (StringUtils.hasText(routeName)) { pushStringToRouting(aString, String.class, routeName); } @@ -232,6 +256,8 @@ public void pushStringToRoute(String aString, String routeName) { @Override public Boolean removeClient(String clientId) { + printCurrentStackTrace(); + try { if (!cacheDataStore.getSetIsMember(RedisKeyUtils.eolClients(), clientId)) { logger.debug("RabbitMQ Removing Client " + clientId); @@ -270,6 +296,8 @@ public Boolean removeClient(String clientId) { } protected Boolean deleteQueue(String queue) { + printCurrentStackTrace(); + try { logger.debug("RabbitMQ Deleting Queue " + queue); Queue clientQueue = new Queue(queue, durableQueues); @@ -289,6 +317,8 @@ protected Boolean deleteQueue(String queue) { @Override public Boolean renameClient(String thePreviousClientId, String clientId) { + printCurrentStackTrace(); + if (!StringUtils.hasText(thePreviousClientId)) { logger.warn("RabbitMQ renameClient previous client not set"); return false; @@ -343,7 +373,8 @@ else if (clientId.equalsIgnoreCase(thePreviousClientId)) { // @Scheduled(fixedRate=30000) // 30 Seconds @Scheduled(fixedRate=300000) // 5 Minutes public void validateQueues() { - + printCurrentStackTrace(); + synchronized (validatingQueues) { if (validatingQueues) { // Currently running. @@ -513,6 +544,8 @@ public void validateQueues() { @Override public void setApplicationContext(ApplicationContext applicationContext) throws BeansException { + printCurrentStackTrace(); + this.applicationContext = applicationContext; Map queues = this.applicationContext.getBeansOfType(Queue.class); @@ -536,5 +569,19 @@ public void setApplicationContext(ApplicationContext applicationContext) queueNames.add(nextQueue.getName()); } } + + private void printCurrentStackTrace(){ + + // System.out.println("**********************************"); + StringBuffer loggerSb = new StringBuffer(); + for(StackTraceElement ste : Thread.currentThread().getStackTrace()){ + loggerSb.append(ste.toString()); + } + slf4jLogger.info("{\"message\" : \"" + loggerSb.toString() + "\"}"); + + // System.out.println("**********************************"); + } + + } \ No newline at end of file From 93af295fe9e0334a50665a695817cdf77f4823b6 Mon Sep 17 00:00:00 2001 From: Jonathan Samples Date: Thu, 3 Dec 2015 20:40:35 -0800 Subject: [PATCH 078/105] Added an a scheduler task executor. --- pom.xml | 2 +- .../agents/sync/access/RedisAccessManager.java | 3 ++- .../agents/sync/helpers/PostPutHelper.java | 4 ++-- .../agents/sync/jobs/UpdateTablePoller.java | 3 ++- .../agents/sync/jobs/UpdateTableProcessor.java | 16 +++++++++++----- .../agents/sync/services/SyncAgentService.java | 3 ++- .../com/percero/amqp/ChangeWatcherListener.java | 3 ++- .../com/percero/amqp/PerceroAgentListener.java | 3 ++- .../spring/basic_components_spring_config.xml | 1 + 9 files changed, 25 insertions(+), 13 deletions(-) diff --git a/pom.xml b/pom.xml index e52e355..cb59e2a 100644 --- a/pom.xml +++ b/pom.xml @@ -7,7 +7,7 @@ org.activestack syncengine - 1.1.54-SNAPSHOT + 1.1.55-SNAPSHOT 3.2.4.RELEASE diff --git a/src/main/java/com/percero/agents/sync/access/RedisAccessManager.java b/src/main/java/com/percero/agents/sync/access/RedisAccessManager.java index 8757792..f2862d8 100644 --- a/src/main/java/com/percero/agents/sync/access/RedisAccessManager.java +++ b/src/main/java/com/percero/agents/sync/access/RedisAccessManager.java @@ -16,6 +16,7 @@ import org.apache.log4j.Logger; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.core.task.TaskExecutor; import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Component; @@ -59,7 +60,7 @@ public void setUserDeviceTimeout(Long value) { userDeviceTimeout = value; } - @Autowired + @Autowired @Qualifier("executorWithCallerRunsPolicy") protected TaskExecutor taskExecutor; public void setTaskExecutor(TaskExecutor value) { taskExecutor = value; diff --git a/src/main/java/com/percero/agents/sync/helpers/PostPutHelper.java b/src/main/java/com/percero/agents/sync/helpers/PostPutHelper.java index 62d34f4..50d2e2c 100644 --- a/src/main/java/com/percero/agents/sync/helpers/PostPutHelper.java +++ b/src/main/java/com/percero/agents/sync/helpers/PostPutHelper.java @@ -58,7 +58,7 @@ public void setGuaranteeUpdateDelivery(Boolean value) { } public void postPutObject(ClassIDPair pair, String pusherUserId, String pusherClientId, boolean pushToUser, Map> changedFields) throws Exception { - + Collection clientIds = accessManager.getObjectAccessJournals(pair.getClassName(), pair.getID()); /*Collection clientIds = */accessManager.saveUpdateJournalClients(pair, clientIds, guaranteeUpdateDelivery, pusherClientId, pushToUser); @@ -143,7 +143,7 @@ public void postPutObject(ClassIDPair pair, String pusherUserId, String pusherCl } } - private void enqueueCheckChangeWatcher(ClassIDPair classIDPair, String[] fieldNames, String[] params){ + public void enqueueCheckChangeWatcher(ClassIDPair classIDPair, String[] fieldNames, String[] params){ CheckChangeWatcherMessage message = new CheckChangeWatcherMessage(); message.classIDPair = classIDPair; message.fieldNames = fieldNames; diff --git a/src/main/java/com/percero/agents/sync/jobs/UpdateTablePoller.java b/src/main/java/com/percero/agents/sync/jobs/UpdateTablePoller.java index f802659..b88274b 100644 --- a/src/main/java/com/percero/agents/sync/jobs/UpdateTablePoller.java +++ b/src/main/java/com/percero/agents/sync/jobs/UpdateTablePoller.java @@ -9,6 +9,7 @@ import org.apache.log4j.Logger; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.core.task.TaskExecutor; import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Component; @@ -51,7 +52,7 @@ public class UpdateTablePoller { @Autowired IAccessManager accessManager; - @Autowired + @Autowired @Qualifier("executorWithCallerRunsPolicy") TaskExecutor taskExecutor; Map> runningProcessors = java.util.Collections.synchronizedMap(new HashMap>()); diff --git a/src/main/java/com/percero/agents/sync/jobs/UpdateTableProcessor.java b/src/main/java/com/percero/agents/sync/jobs/UpdateTableProcessor.java index fa3a675..64793b8 100644 --- a/src/main/java/com/percero/agents/sync/jobs/UpdateTableProcessor.java +++ b/src/main/java/com/percero/agents/sync/jobs/UpdateTableProcessor.java @@ -347,7 +347,7 @@ protected void handleUpdateClassIdPair(IDataProvider dataProvider, ClassIDPair p fieldNames[i] = nextChangedField.getField().getName(); i++; } - accessManager.checkChangeWatchers(thePair, fieldNames, null); + postPutHelper.enqueueCheckChangeWatcher(thePair, fieldNames, null); } } } @@ -506,7 +506,8 @@ protected void processDeleteTable(UpdateTableRow row) throws Exception{ } } - classNamesToUpdateReferences.add(className); + // Removed per a conversation with Collin +// classNamesToUpdateReferences.add(className); } } } @@ -588,8 +589,10 @@ protected void updateReferences(String className){ // This call ignores the cache BaseDataObject nextObjectToUpdateFromDB = (BaseDataObject) dataProvider.findById(pair, null, true); - // Compare their JSON selves for "deep" equality, only push if different - if(!nextObjectToUpdate.toJson().equals(nextObjectToUpdateFromDB.toJson())) { + + Map> changedFields = dataProvider + .getChangedMappedFields(nextObjectToUpdateFromDB, nextObjectToUpdate, nextObjectToUpdate != null); + if (changedFields != null && !changedFields.isEmpty()) { pushPair(pair, mappedField); } } @@ -608,8 +611,11 @@ protected void updateReferences(String className){ BaseDataObject cachedObject = (BaseDataObject) dataProvider.findById(pair, null); // This call ignores the cache BaseDataObject objectFromDB = (BaseDataObject) dataProvider.findById(pair, null, true); + + Map> changedFields = dataProvider + .getChangedMappedFields(objectFromDB, cachedObject, cachedObject != null); // Only push if different - if(!cachedObject.toJson().equals(objectFromDB.toJson())) { + if (changedFields != null && !changedFields.isEmpty()) { pushPair(pair, mappedField); } } diff --git a/src/main/java/com/percero/agents/sync/services/SyncAgentService.java b/src/main/java/com/percero/agents/sync/services/SyncAgentService.java index 2b21eb1..906c555 100644 --- a/src/main/java/com/percero/agents/sync/services/SyncAgentService.java +++ b/src/main/java/com/percero/agents/sync/services/SyncAgentService.java @@ -20,6 +20,7 @@ import org.apache.log4j.Logger; import org.codehaus.jackson.map.ObjectMapper; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.context.ApplicationEventPublisher; import org.springframework.context.ApplicationEventPublisherAware; import org.springframework.core.task.TaskExecutor; @@ -114,7 +115,7 @@ public void setAccessManager(IAccessManager value) { accessManager = value; } - @Autowired + @Autowired @Qualifier("executorWithCallerRunsPolicy") TaskExecutor taskExecutor; public void setTaskExecutor(TaskExecutor value) { taskExecutor = value; diff --git a/src/main/java/com/percero/amqp/ChangeWatcherListener.java b/src/main/java/com/percero/amqp/ChangeWatcherListener.java index 42c664d..edb3f2b 100644 --- a/src/main/java/com/percero/amqp/ChangeWatcherListener.java +++ b/src/main/java/com/percero/amqp/ChangeWatcherListener.java @@ -6,6 +6,7 @@ import org.springframework.amqp.core.Message; import org.springframework.amqp.rabbit.core.ChannelAwareMessageListener; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.context.ApplicationContext; import org.springframework.context.support.ClassPathXmlApplicationContext; import org.springframework.core.task.TaskExecutor; @@ -39,7 +40,7 @@ public class ChangeWatcherListener implements ChannelAwareMessageListener { @Autowired IAccessManager accessManager; - @Autowired + @Autowired @Qualifier("executorWithCallerRunsPolicy") TaskExecutor taskExecutor; @Autowired diff --git a/src/main/java/com/percero/amqp/PerceroAgentListener.java b/src/main/java/com/percero/amqp/PerceroAgentListener.java index ef8876a..a5f2122 100644 --- a/src/main/java/com/percero/amqp/PerceroAgentListener.java +++ b/src/main/java/com/percero/amqp/PerceroAgentListener.java @@ -20,6 +20,7 @@ import org.springframework.amqp.core.MessageListener; import org.springframework.amqp.support.converter.JsonMessageConverter; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.context.ApplicationContext; import org.springframework.context.support.ClassPathXmlApplicationContext; import org.springframework.core.task.TaskExecutor; @@ -65,7 +66,7 @@ public static PerceroAgentListener getInstance() { @Autowired IPushSyncHelper pushSyncHelper; - @Autowired + @Autowired @Qualifier("executorWithCallerRunsPolicy") TaskExecutor taskExecutor; @Autowired ConnectHandler connectHandler; diff --git a/src/main/resources/spring/basic_components_spring_config.xml b/src/main/resources/spring/basic_components_spring_config.xml index 6f49312..fe34a72 100644 --- a/src/main/resources/spring/basic_components_spring_config.xml +++ b/src/main/resources/spring/basic_components_spring_config.xml @@ -101,6 +101,7 @@ + From 68fd848e4036fbc28b7452c4940c5252d9849335 Mon Sep 17 00:00:00 2001 From: Jonathan Samples Date: Thu, 3 Dec 2015 21:22:17 -0800 Subject: [PATCH 079/105] Changed some formatting on stack traces --- pom.xml | 10 +++++----- .../java/com/percero/amqp/RabbitMQPushSyncHelper.java | 1 + 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/pom.xml b/pom.xml index e43710e..d61d0d0 100644 --- a/pom.xml +++ b/pom.xml @@ -240,11 +240,11 @@ jcl-over-slf4j ${slf4j.version} - - org.slf4j - slf4j-log4j12 - ${slf4j.version} - + + + + + commons-dbcp commons-dbcp diff --git a/src/main/java/com/percero/amqp/RabbitMQPushSyncHelper.java b/src/main/java/com/percero/amqp/RabbitMQPushSyncHelper.java index ea30523..7ab3dfc 100644 --- a/src/main/java/com/percero/amqp/RabbitMQPushSyncHelper.java +++ b/src/main/java/com/percero/amqp/RabbitMQPushSyncHelper.java @@ -576,6 +576,7 @@ private void printCurrentStackTrace(){ StringBuffer loggerSb = new StringBuffer(); for(StackTraceElement ste : Thread.currentThread().getStackTrace()){ loggerSb.append(ste.toString()); + loggerSb.append("\n"); } slf4jLogger.info("{\"message\" : \"" + loggerSb.toString() + "\"}"); From 273175cfadb43277e527e0bd4a8913604c487df5 Mon Sep 17 00:00:00 2001 From: Jonathan Samples Date: Thu, 3 Dec 2015 21:31:53 -0800 Subject: [PATCH 080/105] One more change for stack trace markers --- .../com/percero/amqp/RabbitMQPushSyncHelper.java | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/src/main/java/com/percero/amqp/RabbitMQPushSyncHelper.java b/src/main/java/com/percero/amqp/RabbitMQPushSyncHelper.java index 7ab3dfc..3cd36e6 100644 --- a/src/main/java/com/percero/amqp/RabbitMQPushSyncHelper.java +++ b/src/main/java/com/percero/amqp/RabbitMQPushSyncHelper.java @@ -544,8 +544,7 @@ public void validateQueues() { @Override public void setApplicationContext(ApplicationContext applicationContext) throws BeansException { - printCurrentStackTrace(); - + this.applicationContext = applicationContext; Map queues = this.applicationContext.getBeansOfType(Queue.class); @@ -569,20 +568,14 @@ public void setApplicationContext(ApplicationContext applicationContext) queueNames.add(nextQueue.getName()); } } - + + private static final Marker marker = MarkerFactory.getMarker("PUSH_STACK_TRACE"); private void printCurrentStackTrace(){ - - // System.out.println("**********************************"); StringBuffer loggerSb = new StringBuffer(); for(StackTraceElement ste : Thread.currentThread().getStackTrace()){ loggerSb.append(ste.toString()); loggerSb.append("\n"); } - slf4jLogger.info("{\"message\" : \"" + loggerSb.toString() + "\"}"); - - // System.out.println("**********************************"); + slf4jLogger.info(marker, "{\"message\" : \"" + loggerSb.toString() + "\"}"); } - - - } \ No newline at end of file From 2e01d1520ff87b1d8526118020ab4a1fe47b8bd6 Mon Sep 17 00:00:00 2001 From: Nikish Parikh Date: Mon, 21 Dec 2015 06:14:05 +0530 Subject: [PATCH 081/105] loggy removed --- logback.xml | 9 ----- pom.xml | 11 ------ .../percero/amqp/RabbitMQPushSyncHelper.java | 38 +++++++------------ 3 files changed, 14 insertions(+), 44 deletions(-) delete mode 100644 logback.xml diff --git a/logback.xml b/logback.xml deleted file mode 100644 index dbdd760..0000000 --- a/logback.xml +++ /dev/null @@ -1,9 +0,0 @@ - - - http://logs-01.loggly.com/inputs/7f44497d-2cbe-488c-a88e-87ca01ab5e08/tag/logback - %d{"ISO8601", UTC} %p %t %c{0}.%M - %m%n - - - - - diff --git a/pom.xml b/pom.xml index d61d0d0..b3b6176 100644 --- a/pom.xml +++ b/pom.xml @@ -12,7 +12,6 @@ 3.2.4.RELEASE 1.6.11 - 1.6.1 1.19.0 1.19.0 1.9.5 @@ -230,16 +229,6 @@ log4j 1.2.16 - - org.slf4j - slf4j-api - ${slf4j.version} - - - org.slf4j - jcl-over-slf4j - ${slf4j.version} - diff --git a/src/main/java/com/percero/amqp/RabbitMQPushSyncHelper.java b/src/main/java/com/percero/amqp/RabbitMQPushSyncHelper.java index 3cd36e6..6b56faa 100644 --- a/src/main/java/com/percero/amqp/RabbitMQPushSyncHelper.java +++ b/src/main/java/com/percero/amqp/RabbitMQPushSyncHelper.java @@ -67,7 +67,6 @@ public class RabbitMQPushSyncHelper implements IPushSyncHelper, ApplicationContextAware { private static Logger logger = Logger.getLogger(RabbitMQPushSyncHelper.class); - private static final org.slf4j.Logger slf4jLogger = LoggerFactory.getLogger(RabbitMQPushSyncHelper.class); public static final String DEFAULT_CHARSET = "UTF-8"; @@ -118,7 +117,7 @@ public void setCacheDataStore(ICacheDataStore cacheDataStore) { @SuppressWarnings("rawtypes") protected void pushJsonToRouting(String objectJson, Class objectClass, String routingKey) { try{ - printCurrentStackTrace(); + Message convertedMessage = toMessage(objectJson, objectClass, MessageProperties.CONTENT_TYPE_JSON); template.send(routingKey, convertedMessage); @@ -130,7 +129,7 @@ protected void pushJsonToRouting(String objectJson, Class objectClass, String ro protected void pushMessageToRouting(Message convertedMessage, String routingKey) { try{ - printCurrentStackTrace(); + template.send(routingKey, convertedMessage); } @@ -142,7 +141,7 @@ protected void pushMessageToRouting(Message convertedMessage, String routingKey) @SuppressWarnings("rawtypes") protected void pushStringToRouting(String objectJson, Class objectClass, String routingKey) { try{ - printCurrentStackTrace(); + Message convertedMessage = toMessage(objectJson, objectClass, MessageProperties.CONTENT_TYPE_BYTES); template.send(routingKey, convertedMessage); @@ -155,7 +154,7 @@ protected void pushStringToRouting(String objectJson, Class objectClass, String @SuppressWarnings("rawtypes") public final Message toMessage(String objectJson, Class objectClass, String contentEncoding) throws MessageConversionException { - printCurrentStackTrace(); + MessageProperties messageProperties = new MessageProperties(); messageProperties.setContentType(MessageProperties.CONTENT_TYPE_JSON); @@ -167,7 +166,7 @@ public final Message toMessage(String objectJson, Class objectClass, String cont @SuppressWarnings("rawtypes") public final Message toMessage(String objectJson, Class objectClass, MessageProperties messageProperties, String contentEncoding) throws MessageConversionException { - printCurrentStackTrace(); + Message message = createMessage(objectJson, objectClass, messageProperties, contentEncoding); return message; @@ -177,7 +176,7 @@ public final Message toMessage(String objectJson, Class objectClass, MessageProp protected Message createMessage(String aString, Class objectClass, MessageProperties messageProperties, String contentEncoding) throws MessageConversionException { - printCurrentStackTrace(); + byte[] bytes = null; try { @@ -197,7 +196,7 @@ protected Message createMessage(String aString, Class objectClass, MessageProper } public void pushSyncResponseToClient(SyncResponse anObject, String clientId) { - printCurrentStackTrace(); + if (anObject != null && StringUtils.hasText(clientId)) { pushJsonToRouting(anObject.toJson(objectMapper), anObject.getClass(), clientId); @@ -206,7 +205,7 @@ public void pushSyncResponseToClient(SyncResponse anObject, String clientId) { @SuppressWarnings("rawtypes") public void pushSyncResponseToClients(SyncResponse syncResponse, Collection clientIds) { - printCurrentStackTrace(); + if ( syncResponse != null && clientIds != null && !clientIds.isEmpty() ) { Class objectClass = syncResponse.getClass(); @@ -222,7 +221,7 @@ public void pushSyncResponseToClients(SyncResponse syncResponse, Collection listClients) { - printCurrentStackTrace(); + if (anObject != null && listClients != null && !listClients.isEmpty() ) { // Route to specific clients. @@ -247,7 +246,7 @@ public void pushObjectToClients(Object anObject, Collection listClients) @Override public void pushStringToRoute(String aString, String routeName) { - printCurrentStackTrace(); + if (StringUtils.hasText(routeName)) { pushStringToRouting(aString, String.class, routeName); @@ -256,7 +255,7 @@ public void pushStringToRoute(String aString, String routeName) { @Override public Boolean removeClient(String clientId) { - printCurrentStackTrace(); + try { if (!cacheDataStore.getSetIsMember(RedisKeyUtils.eolClients(), clientId)) { @@ -296,7 +295,7 @@ public Boolean removeClient(String clientId) { } protected Boolean deleteQueue(String queue) { - printCurrentStackTrace(); + try { logger.debug("RabbitMQ Deleting Queue " + queue); @@ -317,7 +316,7 @@ protected Boolean deleteQueue(String queue) { @Override public Boolean renameClient(String thePreviousClientId, String clientId) { - printCurrentStackTrace(); + if (!StringUtils.hasText(thePreviousClientId)) { logger.warn("RabbitMQ renameClient previous client not set"); @@ -373,7 +372,7 @@ else if (clientId.equalsIgnoreCase(thePreviousClientId)) { // @Scheduled(fixedRate=30000) // 30 Seconds @Scheduled(fixedRate=300000) // 5 Minutes public void validateQueues() { - printCurrentStackTrace(); + synchronized (validatingQueues) { if (validatingQueues) { @@ -569,13 +568,4 @@ public void setApplicationContext(ApplicationContext applicationContext) } } - private static final Marker marker = MarkerFactory.getMarker("PUSH_STACK_TRACE"); - private void printCurrentStackTrace(){ - StringBuffer loggerSb = new StringBuffer(); - for(StackTraceElement ste : Thread.currentThread().getStackTrace()){ - loggerSb.append(ste.toString()); - loggerSb.append("\n"); - } - slf4jLogger.info(marker, "{\"message\" : \"" + loggerSb.toString() + "\"}"); - } } \ No newline at end of file From 17f842c106cd8568cb693505ff395cde34ea993a Mon Sep 17 00:00:00 2001 From: Nikish Parikh Date: Mon, 21 Dec 2015 06:15:34 +0530 Subject: [PATCH 082/105] version change --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index b3b6176..7c999f4 100644 --- a/pom.xml +++ b/pom.xml @@ -7,7 +7,7 @@ org.activestack syncengine - 1.1.55-SNAPSHOT + 1.1.56-SNAPSHOT 3.2.4.RELEASE From a0686c741a38ab9f918b62bc25161cdc64366682 Mon Sep 17 00:00:00 2001 From: Jonathan Samples Date: Tue, 5 Jan 2016 07:44:19 -0800 Subject: [PATCH 083/105] Added the old (cached) value to the parameters of a change watcher --- .../agents/sync/access/AccessManager.java | 6 +++--- .../agents/sync/access/IAccessManager.java | 5 +++-- .../sync/access/RedisAccessManager.java | 15 ++++++------- .../agents/sync/cw/ChangeWatcherHelper.java | 8 +++---- .../sync/cw/CheckChangeWatcherMessage.java | 3 +++ .../cw/DerivedValueChangeWatcherHelper.java | 8 +++---- .../agents/sync/cw/IChangeWatcherHelper.java | 8 ++++--- .../agents/sync/helpers/PostDeleteHelper.java | 2 +- .../agents/sync/helpers/PostGetHelper.java | 2 +- .../agents/sync/helpers/PostPutHelper.java | 19 +++++++++++++---- .../sync/jobs/UpdateTableProcessor.java | 7 +++---- .../sync/services/SyncAgentService.java | 2 +- .../percero/amqp/ChangeWatcherListener.java | 2 +- .../amqp/CheckChangeWatcherListener.java | 21 +++++++++++++++++-- .../amqp/handlers/ChangeWatcherHandler.java | 2 +- .../spring/percero-spring-config.xml | 8 +++++++ 16 files changed, 80 insertions(+), 38 deletions(-) diff --git a/src/main/java/com/percero/agents/sync/access/AccessManager.java b/src/main/java/com/percero/agents/sync/access/AccessManager.java index b1dc5a4..bb4d880 100644 --- a/src/main/java/com/percero/agents/sync/access/AccessManager.java +++ b/src/main/java/com/percero/agents/sync/access/AccessManager.java @@ -1187,7 +1187,7 @@ public void addWatcherField(ClassIDPair classIdPair, String fieldName, } public void checkChangeWatchers(ClassIDPair classIdPair, - String fieldName, String[] params) { + String fieldName, String[] params, IPerceroObject oldValue) { // TODO Auto-generated method stub } @@ -1200,13 +1200,13 @@ public void removeChangeWatchersByObject(ClassIDPair classIdPair) { @Override public void checkChangeWatchers(ClassIDPair classIdPair, - String[] fieldNames, String[] params) { + String[] fieldNames, String[] params, IPerceroObject oldValue) { // TODO Auto-generated method stub } @Override - public void recalculateChangeWatcher(String changeWatcherId) { + public void recalculateChangeWatcher(String changeWatcherId, IPerceroObject oldValue) { // TODO Auto-generated method stub } diff --git a/src/main/java/com/percero/agents/sync/access/IAccessManager.java b/src/main/java/com/percero/agents/sync/access/IAccessManager.java index 0d3dc27..9e8b1cd 100644 --- a/src/main/java/com/percero/agents/sync/access/IAccessManager.java +++ b/src/main/java/com/percero/agents/sync/access/IAccessManager.java @@ -7,6 +7,7 @@ import com.percero.agents.sync.vo.ClassIDPair; import com.percero.agents.sync.vo.Client; +import com.percero.framework.vo.IPerceroObject; /** * Manages client registration and activation throughout the system. The scope of this client @@ -213,9 +214,9 @@ public Set findClientByUserIdDeviceId(String deviceId, String userId) public Long getChangeWatcherResultTimestamp(ClassIDPair classIdPair, String fieldName, String[] params); public Object getChangeWatcherResult(ClassIDPair classIdPair, String fieldName); public Object getChangeWatcherResult(ClassIDPair classIdPair, String fieldName, String[] params); - public void checkChangeWatchers(ClassIDPair classIdPair, String[] fieldNames, String[] params); + public void checkChangeWatchers(ClassIDPair classIdPair, String[] fieldNames, String[] params, IPerceroObject oldValue); public void removeChangeWatchersByObject(ClassIDPair classIdPair); - public void recalculateChangeWatcher(String changeWatcherId); + public void recalculateChangeWatcher(String changeWatcherId, IPerceroObject oldValue); /** * Takes in a Collection of ClientIDs and returns the sub-set that are valid Clients. diff --git a/src/main/java/com/percero/agents/sync/access/RedisAccessManager.java b/src/main/java/com/percero/agents/sync/access/RedisAccessManager.java index f2862d8..6cc03ba 100644 --- a/src/main/java/com/percero/agents/sync/access/RedisAccessManager.java +++ b/src/main/java/com/percero/agents/sync/access/RedisAccessManager.java @@ -14,6 +14,7 @@ import java.util.TreeMap; import java.util.concurrent.TimeUnit; +import com.percero.framework.vo.IPerceroObject; import org.apache.log4j.Logger; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; @@ -1319,10 +1320,10 @@ public Object getChangeWatcherResult(ClassIDPair classIdPair, String fieldName, } public void checkChangeWatchers(ClassIDPair classIdPair) { - checkChangeWatchers(classIdPair, null, null); + checkChangeWatchers(classIdPair, null, null, null); } - public void checkChangeWatchers(ClassIDPair classIdPair, String[] fieldNames, String[] params) { + public void checkChangeWatchers(ClassIDPair classIdPair, String[] fieldNames, String[] params, IPerceroObject oldValue) { Collection changeWatchers = getChangeWatchersForField(classIdPair, fieldNames, params); // If there are ChangeWatchers, then recalculate for each one. @@ -1338,7 +1339,7 @@ public void checkChangeWatchers(ClassIDPair classIdPair, String[] fieldNames, St nextChangeWatcher += ":" + classIdPair.getClassName() + ":" + classIdPair.getID(); } } - setupRecalculateChangeWatcher(nextChangeWatcher); + setupRecalculateChangeWatcher(nextChangeWatcher, oldValue); // } // else { // // Remove this change watcher from the list. @@ -1463,19 +1464,19 @@ protected Long removeChangeWatcherForField(ClassIDPair classIdPair, String[] fie return result; } - protected void setupRecalculateChangeWatcher(String changeWatcherId) { + protected void setupRecalculateChangeWatcher(String changeWatcherId, IPerceroObject oldValue) { ChangeWatcherReporting.internalRequestsCounter++; if (useChangeWatcherQueue && pushSyncHelper != null) { pushSyncHelper.pushStringToRoute( (new StringBuilder(changeWatcherId).append(":TS:").append(System.currentTimeMillis())).toString(), changeWatcherRouteName); } else { - recalculateChangeWatcher(changeWatcherId); + recalculateChangeWatcher(changeWatcherId, oldValue); } } @SuppressWarnings("unchecked") - public void recalculateChangeWatcher(String changeWatcherId) { + public void recalculateChangeWatcher(String changeWatcherId, IPerceroObject oldValue) { try { // Check to see if a timestamp has been included. String[] changeWatcherTsArray = changeWatcherId.split(":TS:"); @@ -1510,7 +1511,7 @@ public void recalculateChangeWatcher(String changeWatcherId) { Collection clientIds = (Set) cacheDataStore.getSetValue(RedisKeyUtils.clientWatcher(changeWatcherId)); // The list of ClientID's who are interested in this object. IChangeWatcherHelper cwh = changeWatcherHelperFactory.getHelper(category); - cwh.reprocess(category, subCategory, fieldName, clientIds, otherParams, requestTimestamp); + cwh.reprocess(category, subCategory, fieldName, clientIds, otherParams, requestTimestamp, oldValue); /** // If no clients interested in this value, then remove it from the cache. diff --git a/src/main/java/com/percero/agents/sync/cw/ChangeWatcherHelper.java b/src/main/java/com/percero/agents/sync/cw/ChangeWatcherHelper.java index 4605103..8627ae8 100644 --- a/src/main/java/com/percero/agents/sync/cw/ChangeWatcherHelper.java +++ b/src/main/java/com/percero/agents/sync/cw/ChangeWatcherHelper.java @@ -36,7 +36,7 @@ public void setAccessManager(IAccessManager value) { accessManager = value; } - public Object process(String category, String subCategory, String fieldName) { + public Object process(String category, String subCategory, String fieldName, IPerceroObject oldValue) { // This is really an error. StringBuilder strBuilder = new StringBuilder("No value calculate method found for: ").append(category).append(":").append(subCategory).append(":").append(fieldName); log.error(strBuilder.toString()); @@ -44,7 +44,7 @@ public Object process(String category, String subCategory, String fieldName) { return null; } - public Object process(String category, String subCategory, String fieldName, String[] params) { + public Object process(String category, String subCategory, String fieldName, String[] params, IPerceroObject oldValue) { // This is really an error. StringBuilder strBuilder = new StringBuilder("No value process method found for: ").append(category).append(":").append(subCategory).append(":").append(fieldName); for(String nextString : params) { @@ -56,10 +56,10 @@ public Object process(String category, String subCategory, String fieldName, Str } - public Object reprocess(String category, String subCategory, String fieldName, Collection clientIds, String[] params, Long requestTimestamp) { + public Object reprocess(String category, String subCategory, String fieldName, Collection clientIds, String[] params, Long requestTimestamp, IPerceroObject oldValue) { ChangeWatcherReporting.reprocessCounter++; log.debug(ChangeWatcherReporting.stringResults()); - return process(category, subCategory, fieldName, params); + return process(category, subCategory, fieldName, params, oldValue); } diff --git a/src/main/java/com/percero/agents/sync/cw/CheckChangeWatcherMessage.java b/src/main/java/com/percero/agents/sync/cw/CheckChangeWatcherMessage.java index bd97fbd..ca03025 100644 --- a/src/main/java/com/percero/agents/sync/cw/CheckChangeWatcherMessage.java +++ b/src/main/java/com/percero/agents/sync/cw/CheckChangeWatcherMessage.java @@ -1,6 +1,8 @@ package com.percero.agents.sync.cw; +import com.percero.agents.sync.vo.BaseDataObject; import com.percero.agents.sync.vo.ClassIDPair; +import com.percero.framework.vo.IPerceroObject; /** * Created by jonnysamps on 11/23/15. @@ -9,4 +11,5 @@ public class CheckChangeWatcherMessage { public ClassIDPair classIDPair; public String[] fieldNames; public String[] params; + public String oldValueJson; } diff --git a/src/main/java/com/percero/agents/sync/cw/DerivedValueChangeWatcherHelper.java b/src/main/java/com/percero/agents/sync/cw/DerivedValueChangeWatcherHelper.java index d28e7c4..9b0ba3e 100644 --- a/src/main/java/com/percero/agents/sync/cw/DerivedValueChangeWatcherHelper.java +++ b/src/main/java/com/percero/agents/sync/cw/DerivedValueChangeWatcherHelper.java @@ -95,12 +95,12 @@ public Object calculate(String fieldName, ClassIDPair classIdPair, String[] para } @Override - public Object process(String category, String subCategory, String fieldName) { + public Object process(String category, String subCategory, String fieldName, IPerceroObject oldValue) { return calculate(fieldName, new ClassIDPair(subCategory, category)); } @Override - public Object process(String category, String subCategory, String fieldName, String[] params) { + public Object process(String category, String subCategory, String fieldName, String[] params, IPerceroObject oldValue) { return calculate(fieldName, new ClassIDPair(subCategory, category), params); } @@ -133,7 +133,7 @@ protected void postCalculate(String fieldName, ClassIDPair classIdPair, String[] fieldNames = new String[1]; fieldNames[0] = fieldName; } - accessManager.checkChangeWatchers(classIdPair, fieldNames, params); + accessManager.checkChangeWatchers(classIdPair, fieldNames, params, null); } } catch (Exception e) { log.error("Unable to chech change watchers for " + classIdPair.getID() + " / " + fieldName + " in " + getClass().getCanonicalName()); @@ -182,7 +182,7 @@ public Object recalculate(String fieldName, ClassIDPair classIdPair, Collection< } @Override - public Object reprocess(String category, String subCategory, String fieldName, Collection clientIds, String[] params, Long requestTimestamp) { + public Object reprocess(String category, String subCategory, String fieldName, Collection clientIds, String[] params, Long requestTimestamp, IPerceroObject oldValue) { ChangeWatcherReporting.reprocessCounter++; return this.recalculate(fieldName, new ClassIDPair(subCategory, category), clientIds, params, requestTimestamp); } diff --git a/src/main/java/com/percero/agents/sync/cw/IChangeWatcherHelper.java b/src/main/java/com/percero/agents/sync/cw/IChangeWatcherHelper.java index a61f297..e31789f 100644 --- a/src/main/java/com/percero/agents/sync/cw/IChangeWatcherHelper.java +++ b/src/main/java/com/percero/agents/sync/cw/IChangeWatcherHelper.java @@ -1,11 +1,13 @@ package com.percero.agents.sync.cw; +import com.percero.framework.vo.IPerceroObject; + import java.util.Collection; public interface IChangeWatcherHelper { - public Object process(String category, String subCategory, String fieldName); - public Object process(String category, String subCategory, String fieldName, String[] params); - public Object reprocess(String category, String subCategory, String fieldName, Collection clientIds, String[] params, Long requestTimestamp); + public Object process(String category, String subCategory, String fieldName, IPerceroObject oldValue); + public Object process(String category, String subCategory, String fieldName, String[] params, IPerceroObject oldValue); + public Object reprocess(String category, String subCategory, String fieldName, Collection clientIds, String[] params, Long requestTimestamp, IPerceroObject oldValue); } diff --git a/src/main/java/com/percero/agents/sync/helpers/PostDeleteHelper.java b/src/main/java/com/percero/agents/sync/helpers/PostDeleteHelper.java index f089aa4..31071c4 100644 --- a/src/main/java/com/percero/agents/sync/helpers/PostDeleteHelper.java +++ b/src/main/java/com/percero/agents/sync/helpers/PostDeleteHelper.java @@ -84,7 +84,7 @@ public void postDeleteObject(ClassIDPair pair, String pusherUserId, String pushe accessManager.removeHistoricalObjectsByObject(pair); // Now run past the ChangeWatcher. - accessManager.checkChangeWatchers(pair, null, null); + accessManager.checkChangeWatchers(pair, null, null, null); // Remove ChangeWatchers associated with this object. accessManager.removeChangeWatchersByObject(pair); diff --git a/src/main/java/com/percero/agents/sync/helpers/PostGetHelper.java b/src/main/java/com/percero/agents/sync/helpers/PostGetHelper.java index 0b97d4e..a78f03d 100644 --- a/src/main/java/com/percero/agents/sync/helpers/PostGetHelper.java +++ b/src/main/java/com/percero/agents/sync/helpers/PostGetHelper.java @@ -113,7 +113,7 @@ public IPerceroObject recalculateChangeWatcher(String changeWatcherId) { if (changeWatcherHelperFactory != null) { IChangeWatcherHelper cwh = changeWatcherHelperFactory.getHelper(category); - Object result = cwh.reprocess(category, subCategory, fieldName, null, otherParams, requestTimestamp); + Object result = cwh.reprocess(category, subCategory, fieldName, null, otherParams, requestTimestamp, null); if (result != null && result instanceof IPerceroObject) { return (IPerceroObject) result; } diff --git a/src/main/java/com/percero/agents/sync/helpers/PostPutHelper.java b/src/main/java/com/percero/agents/sync/helpers/PostPutHelper.java index 50d2e2c..41607ea 100644 --- a/src/main/java/com/percero/agents/sync/helpers/PostPutHelper.java +++ b/src/main/java/com/percero/agents/sync/helpers/PostPutHelper.java @@ -1,5 +1,6 @@ package com.percero.agents.sync.helpers; +import java.io.IOException; import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; @@ -7,6 +8,7 @@ import java.util.Map; import com.percero.agents.sync.cw.CheckChangeWatcherMessage; +import com.percero.framework.vo.IPerceroObject; import org.apache.log4j.Logger; import org.codehaus.jackson.map.ObjectMapper; import org.springframework.amqp.core.AmqpTemplate; @@ -57,7 +59,11 @@ public void setGuaranteeUpdateDelivery(Boolean value) { guaranteeUpdateDelivery = value; } - public void postPutObject(ClassIDPair pair, String pusherUserId, String pusherClientId, boolean pushToUser, Map> changedFields) throws Exception { + public void postPutObject(ClassIDPair pair, String pusherUserId, String pusherClientID, boolean pushToUser, Map> changedFields) throws Exception { + postPutObject(pair, pusherUserId, pusherClientID, pushToUser, changedFields, null); + } + + public void postPutObject(ClassIDPair pair, String pusherUserId, String pusherClientId, boolean pushToUser, Map> changedFields, IPerceroObject oldValue) throws Exception { Collection clientIds = accessManager.getObjectAccessJournals(pair.getClassName(), pair.getID()); /*Collection clientIds = */accessManager.saveUpdateJournalClients(pair, clientIds, guaranteeUpdateDelivery, pusherClientId, pushToUser); @@ -87,7 +93,7 @@ public void postPutObject(ClassIDPair pair, String pusherUserId, String pusherCl // Now run past the ChangeWatcher. if (changedFields == null || changedFields.isEmpty()) { - enqueueCheckChangeWatcher(pair, null, null); + enqueueCheckChangeWatcher(pair, null, null, oldValue); // accessManager.checkChangeWatchers(pair, null, null); } else { @@ -121,7 +127,7 @@ public void postPutObject(ClassIDPair pair, String pusherUserId, String pusherCl } // Swap out inline processing for a worker queue - enqueueCheckChangeWatcher(thePair, fieldNames, null); + enqueueCheckChangeWatcher(thePair, fieldNames, null, oldValue); // accessManager.checkChangeWatchers(thePair, fieldNames, null); } } @@ -143,14 +149,19 @@ public void postPutObject(ClassIDPair pair, String pusherUserId, String pusherCl } } - public void enqueueCheckChangeWatcher(ClassIDPair classIDPair, String[] fieldNames, String[] params){ + public void enqueueCheckChangeWatcher(ClassIDPair classIDPair, String[] fieldNames, String[] params, IPerceroObject oldValue){ CheckChangeWatcherMessage message = new CheckChangeWatcherMessage(); message.classIDPair = classIDPair; message.fieldNames = fieldNames; message.params = params; + if(oldValue != null) + message.oldValueJson = ((BaseDataObject)oldValue).toJson(); template.convertAndSend("checkChangeWatcher", message); } + public void enqueueCheckChangeWatcher(ClassIDPair classIDPair, String[] fieldNames, String[] params){ + enqueueCheckChangeWatcher(classIDPair, fieldNames, params, null); + } public void pushObjectUpdateJournals(Collection clientIds, ClassIDPair classIdPair, Collection changedFields) { if (classIdPair != null && clientIds != null && !clientIds.isEmpty()) { diff --git a/src/main/java/com/percero/agents/sync/jobs/UpdateTableProcessor.java b/src/main/java/com/percero/agents/sync/jobs/UpdateTableProcessor.java index 64793b8..52111a0 100644 --- a/src/main/java/com/percero/agents/sync/jobs/UpdateTableProcessor.java +++ b/src/main/java/com/percero/agents/sync/jobs/UpdateTableProcessor.java @@ -318,12 +318,11 @@ protected void handleUpdateClassIdPair(IDataProvider dataProvider, ClassIDPair p // compare a new object, but we can use the cached object in // this case (we just need to tell getChangedMappedFields to NOT // use the cache). - changedFields = dataProvider - .getChangedMappedFields(perceroObject, oldValue, oldValue != null); + changedFields = dataProvider.getChangedMappedFields(perceroObject, oldValue, true); if (changedFields != null && !changedFields.isEmpty()) { // Something has changed. cacheManager.updateCachedObject(perceroObject, changedFields); - postPutHelper.postPutObject(pair, null, null, true, changedFields); + postPutHelper.postPutObject(pair, null, null, true, changedFields, oldValue); Iterator>> itrChangedFieldEntryset = changedFields.entrySet().iterator(); while (itrChangedFieldEntryset.hasNext()) { @@ -347,7 +346,7 @@ protected void handleUpdateClassIdPair(IDataProvider dataProvider, ClassIDPair p fieldNames[i] = nextChangedField.getField().getName(); i++; } - postPutHelper.enqueueCheckChangeWatcher(thePair, fieldNames, null); + postPutHelper.enqueueCheckChangeWatcher(thePair, fieldNames, null, oldValue); } } } diff --git a/src/main/java/com/percero/agents/sync/services/SyncAgentService.java b/src/main/java/com/percero/agents/sync/services/SyncAgentService.java index 906c555..72f5505 100644 --- a/src/main/java/com/percero/agents/sync/services/SyncAgentService.java +++ b/src/main/java/com/percero/agents/sync/services/SyncAgentService.java @@ -815,7 +815,7 @@ public ServerResponse putObject(IPerceroObject perceroObject, String transaction fieldNames[i] = nextChangedField.getField().getName(); i++; } - accessManager.checkChangeWatchers(thePair, fieldNames, null); + accessManager.checkChangeWatchers(thePair, fieldNames, null, null); } } } diff --git a/src/main/java/com/percero/amqp/ChangeWatcherListener.java b/src/main/java/com/percero/amqp/ChangeWatcherListener.java index edb3f2b..7bad2fb 100644 --- a/src/main/java/com/percero/amqp/ChangeWatcherListener.java +++ b/src/main/java/com/percero/amqp/ChangeWatcherListener.java @@ -81,7 +81,7 @@ public void onMessage(Message message, Channel channel) { if (taskExecutor != null) { taskExecutor.execute(new ChangeWatcherHandlerTask(changeWatcherId)); } else { - accessManager.recalculateChangeWatcher(changeWatcherId); + accessManager.recalculateChangeWatcher(changeWatcherId, null); } } } diff --git a/src/main/java/com/percero/amqp/CheckChangeWatcherListener.java b/src/main/java/com/percero/amqp/CheckChangeWatcherListener.java index ad52a78..c9143c6 100644 --- a/src/main/java/com/percero/amqp/CheckChangeWatcherListener.java +++ b/src/main/java/com/percero/amqp/CheckChangeWatcherListener.java @@ -2,7 +2,7 @@ import com.percero.agents.sync.access.IAccessManager; import com.percero.agents.sync.cw.CheckChangeWatcherMessage; -import com.percero.agents.sync.metadata.MappedClass; +import com.percero.agents.sync.vo.BaseDataObject; import org.apache.log4j.Logger; import org.codehaus.jackson.JsonNode; import org.codehaus.jackson.map.ObjectMapper; @@ -34,11 +34,28 @@ public void onMessage(Message message) { try { JsonNode node = om.readTree(message.getBody()); CheckChangeWatcherMessage checkChangeWatcherMessage = om.readValue(node, CheckChangeWatcherMessage.class); + BaseDataObject oldValue = getDeserializeOldValue(checkChangeWatcherMessage.classIDPair.getClassName(), + checkChangeWatcherMessage.oldValueJson); accessManager.checkChangeWatchers(checkChangeWatcherMessage.classIDPair, checkChangeWatcherMessage.fieldNames, - checkChangeWatcherMessage.params); + checkChangeWatcherMessage.params, + oldValue); }catch(Exception e){ logger.error(e.getMessage(), e); } } + + private BaseDataObject getDeserializeOldValue(String className, String json){ + if(json == null) return null; + + BaseDataObject result = null; + + try { + Class clazz = Class.forName(className); + result = (BaseDataObject) clazz.newInstance(); + result.fromJson(json); + }catch(Exception e){ logger.error(e.getMessage(), e); } + + return result; + } } diff --git a/src/main/java/com/percero/amqp/handlers/ChangeWatcherHandler.java b/src/main/java/com/percero/amqp/handlers/ChangeWatcherHandler.java index 192ae6e..069e191 100644 --- a/src/main/java/com/percero/amqp/handlers/ChangeWatcherHandler.java +++ b/src/main/java/com/percero/amqp/handlers/ChangeWatcherHandler.java @@ -29,7 +29,7 @@ public ChangeWatcherHandler() { IDecoder decoder; public Boolean handleChangeWatcher(String changeWatcherId) throws Exception { - accessManager.recalculateChangeWatcher(changeWatcherId); + accessManager.recalculateChangeWatcher(changeWatcherId, null); return true; } } diff --git a/src/main/resources/spring/percero-spring-config.xml b/src/main/resources/spring/percero-spring-config.xml index ed759f5..091f759 100644 --- a/src/main/resources/spring/percero-spring-config.xml +++ b/src/main/resources/spring/percero-spring-config.xml @@ -69,4 +69,12 @@ + + + + + + + + From 6a92fa015171c700bd68932aa4ccc10f86a4f662 Mon Sep 17 00:00:00 2001 From: Jonathan Samples Date: Tue, 5 Jan 2016 07:45:04 -0800 Subject: [PATCH 084/105] Bumped the version 1.1.57 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 7c999f4..819436f 100644 --- a/pom.xml +++ b/pom.xml @@ -7,7 +7,7 @@ org.activestack syncengine - 1.1.56-SNAPSHOT + 1.1.57-SNAPSHOT 3.2.4.RELEASE From f31b4a7f570c9012ce10075b3b76b31a1a5e6e6a Mon Sep 17 00:00:00 2001 From: Nikish Parikh Date: Sat, 9 Jan 2016 11:02:22 +0530 Subject: [PATCH 085/105] New date util function added --- src/main/java/com/percero/util/DateUtils.java | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/main/java/com/percero/util/DateUtils.java b/src/main/java/com/percero/util/DateUtils.java index 18aa308..2a3602d 100644 --- a/src/main/java/com/percero/util/DateUtils.java +++ b/src/main/java/com/percero/util/DateUtils.java @@ -20,5 +20,12 @@ public static Date utilDateFromSqlTimestamp(Timestamp timestamp) { } return null; } + + public static Timestamp sqlTimestampFromUtilDate(java.util.Date utilDate){ + if(utilDate!=null){ + return new Timestamp(utilDate.getTime()); + } + return null; + } } From e2ad97a05b8c8c2a17836953564159b97ba3b8bc Mon Sep 17 00:00:00 2001 From: Jay Eno Date: Mon, 18 Jan 2016 18:05:03 -0700 Subject: [PATCH 086/105] Switch synchronized Map to a Concurrent Map for expires --- .../sync/datastore/RedisCacheDataStore.java | 1306 ++++++++--------- 1 file changed, 647 insertions(+), 659 deletions(-) diff --git a/src/main/java/com/percero/agents/sync/datastore/RedisCacheDataStore.java b/src/main/java/com/percero/agents/sync/datastore/RedisCacheDataStore.java index e222820..07ccd6b 100644 --- a/src/main/java/com/percero/agents/sync/datastore/RedisCacheDataStore.java +++ b/src/main/java/com/percero/agents/sync/datastore/RedisCacheDataStore.java @@ -1,659 +1,647 @@ -package com.percero.agents.sync.datastore; - -import java.util.Collection; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.Map.Entry; -import java.util.Set; -import java.util.concurrent.TimeUnit; - -import org.apache.log4j.Logger; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.data.redis.core.HashOperations; -import org.springframework.data.redis.core.RedisTemplate; -import org.springframework.data.redis.core.SetOperations; -import org.springframework.scheduling.annotation.Scheduled; -import org.springframework.transaction.annotation.Transactional; - -import com.percero.agents.sync.services.PerceroRedisTemplate; - -import edu.emory.mathcs.backport.java.util.Collections; - -public class RedisCacheDataStore implements ICacheDataStore { - - private static Logger log = Logger.getLogger(RedisCacheDataStore.class); - - @Autowired - protected PerceroRedisTemplate template; - - /* (non-Javadoc) - * @see com.percero.agents.sync.datastore.IRedisCacheDataStore#expire(java.lang.String, long, java.util.concurrent.TimeUnit) - */ - @Override - public Boolean expire( final String key, final long timeout, final TimeUnit timeUnit ) { - return expire(key, timeout, timeUnit, false); - } - - /* (non-Javadoc) - * @see com.percero.agents.sync.datastore.IRedisCacheDataStore#expire(java.util.Collection, long, java.util.concurrent.TimeUnit) - */ - @Override - public Boolean expire( final Collection keys, final long timeout, final TimeUnit timeUnit ) { - return expire(keys, timeout, timeUnit, false); - } - - /* (non-Javadoc) - * @see com.percero.agents.sync.datastore.IRedisCacheDataStore#expire(java.lang.String, long, java.util.concurrent.TimeUnit, java.lang.Boolean) - */ - @Override - public Boolean expire( final String key, final long timeout, final TimeUnit timeUnit, Boolean forceNow ) { - if (forceNow) { - return template.expire(key, timeout, timeUnit); - } - else { - expiresToBeWritten.put(key, new PendingExpire(key, timeout, timeUnit)); - return true; - } - } - - /* (non-Javadoc) - * @see com.percero.agents.sync.datastore.IRedisCacheDataStore#expire(java.util.Collection, long, java.util.concurrent.TimeUnit, java.lang.Boolean) - */ - @Override - public Boolean expire( final Collection keys, final long timeout, final TimeUnit timeUnit, Boolean forceNow ) { - if (forceNow) { - Iterator itrKeys = keys.iterator(); - while (itrKeys.hasNext()) { - if (!template.expire(itrKeys.next(), timeout, timeUnit)) { - return false; - } - } - return true; - } - else { - Iterator itrKeys = keys.iterator(); - while (itrKeys.hasNext()) { - String nextKey = itrKeys.next(); - expiresToBeWritten.put(nextKey, new PendingExpire(nextKey, timeout, timeUnit)); - } - return true; - } - } - - - @SuppressWarnings("unchecked") - private Map expiresToBeWritten = Collections.synchronizedMap(new HashMap()); - /* (non-Javadoc) - * @see com.percero.agents.sync.datastore.IRedisCacheDataStore#postExpires() - */ - @Override - @Scheduled(fixedRate=600000) // 10 * 60 * 1000 -> Ten Minutes. - public void postExpires() { - log.info("Posting " + expiresToBeWritten.size() + " expire" + (expiresToBeWritten.size() == 1 ? "" : "s")); - synchronized (expiresToBeWritten) { - Collection expireKeysToRemove = new HashSet(expiresToBeWritten.size(), (float)1.0); - Iterator> itrExpiresEntrySet = expiresToBeWritten.entrySet().iterator(); - while (itrExpiresEntrySet.hasNext()) { - Map.Entry nextEntry = itrExpiresEntrySet.next(); - String nextKey = nextEntry.getKey(); - PendingExpire nextPendingExpire = nextEntry.getValue(); - expire(nextPendingExpire.key, nextPendingExpire.timeout, nextPendingExpire.timeUnit, true); - expireKeysToRemove.add(nextKey); - } -// Iterator itrExpires = expiresToBeWritten.keySet().iterator(); -// while (itrExpires.hasNext()) { -// String nextKey = itrExpires.next(); -// PendingExpire nextPendingExpire = expiresToBeWritten.get(nextKey); -// expire(nextPendingExpire.key, nextPendingExpire.timeout, nextPendingExpire.timeUnit, true); -// expireKeysToRemove.add(nextKey); -// } - - Iterator itrExpireKeysToRemove = expireKeysToRemove.iterator(); - while (itrExpireKeysToRemove.hasNext()) { - expiresToBeWritten.remove(itrExpireKeysToRemove.next()); - } - } - } - - private class PendingExpire { - String key; - long timeout; - TimeUnit timeUnit; - - public PendingExpire(String key, long timeout, TimeUnit timeUnit) { - this.key = key; - this.timeout = timeout; - this.timeUnit = timeUnit; - } - } - - - /* (non-Javadoc) - * @see com.percero.agents.sync.datastore.IRedisCacheDataStore#getValue(java.lang.String) - */ - @Override - public Object getValue( final String key ) { - return template.opsForValue().get( key ); - } - - /* (non-Javadoc) - * @see com.percero.agents.sync.datastore.IRedisCacheDataStore#getValues(java.util.Collection) - */ - @Override - public List getValues( final Collection keys ) { - return template.opsForValue().multiGet(keys); - } - - /* (non-Javadoc) - * @see com.percero.agents.sync.datastore.IRedisCacheDataStore#setValue(java.lang.String, java.lang.Object) - */ - @Override - public void setValue( final String key, final Object value ) { - template.opsForValue().set( key, value ); - } - -// public void setValues( final KeyValuePair[] pairs ) { - /* (non-Javadoc) - * @see com.percero.agents.sync.datastore.IRedisCacheDataStore#setValues(java.util.Map) - */ - @Override - public void setValues( final Map keysAndValuesMap ) { - template.opsForValue().multiSet(keysAndValuesMap); -// if (pairs != null) { -// for (KeyValuePair nextPair : pairs) { -// template.opsForValue().set( nextPair.key, nextPair.value ); -// } -// } - } - - /* (non-Javadoc) - * @see com.percero.agents.sync.datastore.IRedisCacheDataStore#addSetValue(java.lang.String, java.lang.Object) - */ - @Override - @Transactional - public Long addSetValue( final String key, final Object value ) { - return template.opsForSet().add(key, value);// ? Long.valueOf(1) : Long.valueOf(0); - } - - /* (non-Javadoc) - * @see com.percero.agents.sync.datastore.IRedisCacheDataStore#lpushListValue(java.lang.String, java.lang.Object) - */ - @Override - @Transactional - public Long lpushListValue( final String key, final Object value ) { - return template.opsForList().leftPush(key, value); - } - - /* (non-Javadoc) - * @see com.percero.agents.sync.datastore.IRedisCacheDataStore#listAll(java.lang.String) - */ - @Override - @Transactional - public List listAll( final String key) { - return listRange(key, Long.valueOf(0), Long.valueOf(-1)); - } - - /* (non-Javadoc) - * @see com.percero.agents.sync.datastore.IRedisCacheDataStore#listRange(java.lang.String, java.lang.Long, java.lang.Long) - */ - @Override - @Transactional - public List listRange( final String key, final Long start, final Long end ) { - return template.opsForList().range(key, start, end); - } - - /* (non-Javadoc) - * @see com.percero.agents.sync.datastore.IRedisCacheDataStore#listIndex(java.lang.String, java.lang.Long) - */ - @Override - @Transactional - public Object listIndex( final String key, final Long index ) { - return template.opsForList().index(key, index); - } - - /* (non-Javadoc) - * @see com.percero.agents.sync.datastore.IRedisCacheDataStore#listIndex(java.lang.String) - */ - @Override - @Transactional - public Object listIndex( final String key ) { - return template.opsForList().index(key, Long.valueOf(0)); - } - - /* (non-Javadoc) - * @see com.percero.agents.sync.datastore.IRedisCacheDataStore#getHashValue(java.lang.String, java.lang.String) - */ - @Override - @Transactional - public Object getHashValue( final String key, final String hashKey ) { - return template.opsForHash().get(key, hashKey); - } - - /* (non-Javadoc) - * @see com.percero.agents.sync.datastore.IRedisCacheDataStore#getHashKeys(java.lang.String) - */ - @Override - @Transactional - public Set getHashKeys( final String key ) { - Set keysObjects = template.opsForHash().keys(key); - Set results = new HashSet(keysObjects.size()); - for(Object nextKey : keysObjects) { - results.add((String)nextKey); - } - return results; - } - - /* (non-Javadoc) - * @see com.percero.agents.sync.datastore.IRedisCacheDataStore#getHashEntries(java.lang.String) - */ - @Override - @Transactional - public Map getHashEntries( final String key ) { - Map entries = template.opsForHash().entries(key); - Map result = new HashMap(); - Iterator> itrEntries = entries.entrySet().iterator(); - while (itrEntries.hasNext()) { - Entry nextEntry = itrEntries.next(); - result.put((String) nextEntry.getKey(), nextEntry.getValue()); - } - return result; - } - - /* (non-Javadoc) - * @see com.percero.agents.sync.datastore.IRedisCacheDataStore#getHashSize(java.lang.String) - */ - @Override - @Transactional - public Long getHashSize( final String key ) { - return template.opsForHash().size(key); - } - - @Transactional - public Long getSetSize( final String key ) { - return template.opsForSet().size(key); - } - - /* (non-Javadoc) - * @see com.percero.agents.sync.datastore.IRedisCacheDataStore#deleteHashKey(java.lang.String, java.lang.Object) - */ - @Override - @Transactional - public void deleteHashKey( final String key, final String hashKey ) { - template.opsForHash().delete(key, hashKey); - } - - /* (non-Javadoc) - * @see com.percero.agents.sync.datastore.IRedisCacheDataStore#setHashValue(java.lang.String, java.lang.String, java.lang.Object) - */ - @Override - @Transactional - public void setHashValue( final String key, final String hashKey, final Object value ) { - template.opsForHash().put(key, hashKey, value); - } - - /* (non-Javadoc) - * @see com.percero.agents.sync.datastore.IRedisCacheDataStore#setAllHashValues(java.lang.String, java.util.Map) - */ - @Override - @Transactional - public void setAllHashValues( final String key, final Map hashMap ) { - template.opsForHash().putAll(key, hashMap); - } - - /* (non-Javadoc) - * @see com.percero.agents.sync.datastore.IRedisCacheDataStore#removeSetValue(java.lang.String, java.lang.Object) - */ - @Override - @Transactional - public Long removeSetValue( final String key, final Object value ) { - return template.opsForSet().remove(key, value);// ? Long.valueOf(1) : Long.valueOf(0); - } - - /* (non-Javadoc) - * @see com.percero.agents.sync.datastore.IRedisCacheDataStore#getSetIsEmpty(java.lang.String) - */ - @Override - @Transactional - public Boolean getSetIsEmpty( final String key ) { - Long result = template.opsForSet().size(key);// ? Long.valueOf(1) : Long.valueOf(0); - return (result <= 0); - } - - /* (non-Javadoc) - * @see com.percero.agents.sync.datastore.IRedisCacheDataStore#removeSetValueAndGetSize(java.lang.String, java.lang.Object) - */ - @Override - @Transactional - public Long removeSetValueAndGetSize( final String key, final Object value ) { - SetOperations setOps = template.opsForSet(); - setOps.remove(key, value); - return setOps.size(key); - } - - /* (non-Javadoc) - * @see com.percero.agents.sync.datastore.IRedisCacheDataStore#removeSetsValue(java.util.Collection, java.lang.Object) - */ - @Override - public void removeSetsValue( final Collection keys, final Object value ) { - SetOperations setOps = template.opsForSet(); - Iterator itrKeys = keys.iterator(); - while(itrKeys.hasNext()) { - Object nextKey = itrKeys.next(); - try { - setOps.remove((String) nextKey, value); - } catch(Exception e) { - log.warn("Invalid Key", e); - } - } - } - - /* (non-Javadoc) - * @see com.percero.agents.sync.datastore.IRedisCacheDataStore#removeSetsValue(java.lang.String, java.util.Collection, java.lang.Object) - */ - @Override - public void removeSetsValue( final String keysPrefix, final Collection keys, final Object value ) { - SetOperations setOps = template.opsForSet(); - Iterator itrKeys = keys.iterator(); - while(itrKeys.hasNext()) { - Object nextKey = itrKeys.next(); - try { - setOps.remove(keysPrefix + (String) nextKey, value); - } catch(Exception e) { - log.warn("Invalid Key", e); - } - } - } - -// private static long keysCount = 0; -// public Set keys(String pattern) { -// Set result = template.keys(pattern); -// log.info("Keys: " + ++keysCount); -// -// return result; -// } - - /* (non-Javadoc) - * @see com.percero.agents.sync.datastore.IRedisCacheDataStore#hasKey(java.lang.String) - */ - @Override - public Boolean hasKey(String key) { - Boolean result = template.hasKey(key); - return result; - } - - /* (non-Javadoc) - * @see com.percero.agents.sync.datastore.IRedisCacheDataStore#hasHashKey(java.lang.String, java.lang.String) - */ - @Override - public Boolean hasHashKey(String key, String hashKey) { - Boolean result = template.opsForHash().hasKey(key, hashKey); - return result; - } - - /* (non-Javadoc) - * @see com.percero.agents.sync.datastore.IRedisCacheDataStore#swapHashKey(java.lang.String, java.lang.String, java.lang.String) - */ - @Override - @Transactional - public void swapHashKey( final String key, final String oldHashKey, final String newHashValue ) { - HashOperations hashOps = template.opsForHash(); - if (hashOps.hasKey(key, oldHashKey)) { - hashOps.put(key, newHashValue, hashOps.get(key, oldHashKey)); - hashOps.delete(key, oldHashKey); - } - } - - /* (non-Javadoc) - * @see com.percero.agents.sync.datastore.IRedisCacheDataStore#renameIfAbsent(java.lang.String, java.lang.String) - */ - @Override - public Boolean renameIfAbsent(String oldKey, String newKey) { - return template.renameIfAbsent(oldKey, newKey); - } - - /* (non-Javadoc) - * @see com.percero.agents.sync.datastore.IRedisCacheDataStore#rename(java.lang.String, java.lang.String) - */ - @Override - public void rename(String oldKey, String newKey) { - template.rename(oldKey, newKey); - } - - /* (non-Javadoc) - * @see com.percero.agents.sync.datastore.IRedisCacheDataStore#setUnionAndStore(java.lang.String, java.lang.String, java.lang.String) - */ - @Override - public Long setUnionAndStore(String key, String otherKey, String destKey) { - return template.opsForSet().unionAndStore(key, otherKey, destKey); - } - - /* (non-Javadoc) - * @see com.percero.agents.sync.datastore.IRedisCacheDataStore#setUnionAndStore(java.lang.String, java.util.Collection, java.lang.String) - */ - @Override - public Long setUnionAndStore(String key, Collection otherKeys, String destKey) { - return template.opsForSet().unionAndStore(key, otherKeys, destKey); - } - - /* (non-Javadoc) - * @see com.percero.agents.sync.datastore.IRedisDataStore#removeSetValues(java.lang.String, java.util.Collection) - */ - /* (non-Javadoc) - * @see com.percero.agents.sync.datastore.IRedisCacheDataStore#removeSetValues(java.lang.String, java.util.Collection) - */ - @Override - @Transactional - public void removeSetValues( final String key, final Collection values ) { - SetOperations setOps = template.opsForSet(); - - Iterator itrValues = values.iterator(); - while(itrValues.hasNext()) { - try { - setOps.remove(key, itrValues.next()); - } catch(Exception e) { - log.error("Unable to remove Set Value", e); - } - } - } - - /* (non-Javadoc) - * @see com.percero.agents.sync.datastore.IRedisDataStore#getSetValue(java.lang.String) - */ - /* (non-Javadoc) - * @see com.percero.agents.sync.datastore.IRedisCacheDataStore#getSetValue(java.lang.String) - */ - @Override - public Set getSetValue( final String key ) { - return template.opsForSet().members(key); - } - - /* (non-Javadoc) - * @see com.percero.agents.sync.datastore.IRedisCacheDataStore#replaceSet(java.lang.String, java.util.Collection) - */ - @Override - @Transactional - public Long replaceSet( final String key, Collection values ) { - template.delete(key); - if (values != null) { - return template.opsForSet().add(key, values.toArray()); - } - else { - return Long.valueOf(0); - } - } - - /* (non-Javadoc) - * @see com.percero.agents.sync.datastore.IRedisDataStore#swapSetValue(java.lang.String, java.lang.String, java.lang.String) - */ - /* (non-Javadoc) - * @see com.percero.agents.sync.datastore.IRedisCacheDataStore#swapSetValue(java.lang.String, java.lang.String, java.lang.String) - */ - @Override - @Transactional - public void swapSetValue( final String key, final String oldValue, final String newValue ) { - SetOperations setOps = template.opsForSet(); - if (setOps.isMember(key, oldValue)) { - setOps.remove(key, oldValue); - setOps.add(key, newValue); - } - } - - /* (non-Javadoc) - * @see com.percero.agents.sync.datastore.IRedisCacheDataStore#getSetIsMember(java.lang.String, java.lang.Object) - */ - @Override - public Boolean getSetIsMember( final String key, final Object object ) { - return template.opsForSet().isMember(key, object); - } - - /* (non-Javadoc) - * @see com.percero.agents.sync.datastore.IRedisCacheDataStore#getSetsContainsMembers(java.util.Collection, java.lang.Object[]) - */ - @Override - @Transactional - public Set getSetsContainsMembers( final Collection keys, final Object[] membersToCheck ) { - SetOperations setOps = template.opsForSet(); - String randomKey = "UserTokenCheck:" + System.currentTimeMillis(); - setOps.add(randomKey, membersToCheck); - - Set intersectingMembers = new HashSet(); - Iterator itrKeys = keys.iterator(); - while (itrKeys.hasNext()) { - String nextKey = itrKeys.next(); - intersectingMembers.addAll(setOps.intersect(randomKey, nextKey)); - } - template.delete(randomKey); - return intersectingMembers; - } - - /* (non-Javadoc) - * @see com.percero.agents.sync.datastore.IRedisCacheDataStore#getSetIntersect(java.lang.String, java.util.Collection) - */ - @Override - public Set getSetIntersect( final String key, Collection otherKeys ) { - return template.opsForSet().intersect(key, otherKeys); - } - - /* (non-Javadoc) - * @see com.percero.agents.sync.datastore.IRedisCacheDataStore#getSetIntersect(java.lang.String, java.lang.String) - */ - @Override - public Set getSetIntersect( final String key, String otherKey ) { - return template.opsForSet().intersect(key, otherKey); - } - - /* (non-Javadoc) - * @see com.percero.agents.sync.datastore.IRedisCacheDataStore#getSetUnion(java.lang.String, java.util.Collection) - */ - @Override - public Set getSetUnion( final String key, final Collection otherKeys ) { - return template.opsForSet().union(key, otherKeys); - } - - /* (non-Javadoc) - * @see com.percero.agents.sync.datastore.IRedisCacheDataStore#getSetUnion(java.lang.String, java.lang.String) - */ - @Override - public Set getSetUnion( final String key, final String otherKey ) { - return template.opsForSet().union(key, otherKey); - } - - /* (non-Javadoc) - * @see com.percero.agents.sync.datastore.IRedisCacheDataStore#deleteKey(java.lang.String) - */ - @Override - public void deleteKey( final String key ) { - template.delete(key); - } - - /* (non-Javadoc) - * @see com.percero.agents.sync.datastore.IRedisCacheDataStore#deleteKeys(java.util.Collection) - */ - @Override - public void deleteKeys( final Collection keys ) { - if (keys != null && !keys.isEmpty()) { - template.delete(keys); - } - } - - /* (non-Javadoc) - * @see com.percero.agents.sync.datastore.IRedisCacheDataStore#setSetValue(java.lang.String, java.lang.Object) - */ - @Override - @Transactional - public Long setSetValue(String key, Object value) { - return template.opsForSet().add(key, value);// ? Long.valueOf(1) : Long.valueOf(0); - } - - @Override - @Transactional - public Long setSetsValues(Map> values) { - Long result = new Long(0); - Iterator>> itr = values.entrySet().iterator(); - while (itr.hasNext()) { - Entry> nextEntry = itr.next(); - result += template.opsForSet().add(nextEntry.getKey(), nextEntry.getValue().toArray());// ? Long.valueOf(1) : Long.valueOf(0); - - } - return result; - } - - @Override - @Transactional - public Long removeSetsValues(Map> values) { - Long result = new Long(0); - Iterator>> itr = values.entrySet().iterator(); - while (itr.hasNext()) { - Entry> nextEntry = itr.next(); - result += template.opsForSet().remove(nextEntry.getKey(), nextEntry.getValue().toArray());// ? Long.valueOf(1) : Long.valueOf(0); - - } - return result; - } - - /* (non-Javadoc) - * @see com.percero.agents.sync.datastore.IRedisCacheDataStore#getSet(java.lang.String) - */ - @Override - public SetOperations getSet(String key) { - return template.opsForSet(); - } - - /* (non-Javadoc) - * @see com.percero.agents.sync.datastore.IRedisCacheDataStore#getTemplate() - */ - @Override - public RedisTemplate getTemplate() { - return template; - } - - /* (non-Javadoc) - * @see com.percero.agents.sync.datastore.IRedisCacheDataStore#keys(java.lang.String) - */ - @Override - public Collection keys(String pattern) { - return template.keys(pattern); - } - - /* (non-Javadoc) - * @see com.percero.agents.sync.datastore.IRedisCacheDataStore#removeKeysValue(java.lang.String, java.lang.Object) - */ - @Override - @Transactional - public void removeKeysValue(String pattern, Object value) { - Set keys = template.keys(pattern); - Iterator itrKeys = keys.iterator(); - - while(itrKeys.hasNext()) { - try { - template.opsForSet().remove(itrKeys.next(), value); - } catch(Exception e) { - log.error("Error removing KeysValue", e); - } - } - } - -} +package com.percero.agents.sync.datastore; + +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Set; +import java.util.concurrent.TimeUnit; + +import org.apache.log4j.Logger; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.data.redis.core.HashOperations; +import org.springframework.data.redis.core.RedisTemplate; +import org.springframework.data.redis.core.SetOperations; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.transaction.annotation.Transactional; + +import com.percero.agents.sync.services.PerceroRedisTemplate; + +import java.util.concurrent.ConcurrentHashMap; + +public class RedisCacheDataStore implements ICacheDataStore { + + private static Logger log = Logger.getLogger(RedisCacheDataStore.class); + + @Autowired + protected PerceroRedisTemplate template; + + /* (non-Javadoc) + * @see com.percero.agents.sync.datastore.IRedisCacheDataStore#expire(java.lang.String, long, java.util.concurrent.TimeUnit) + */ + @Override + public Boolean expire( final String key, final long timeout, final TimeUnit timeUnit ) { + return expire(key, timeout, timeUnit, false); + } + + /* (non-Javadoc) + * @see com.percero.agents.sync.datastore.IRedisCacheDataStore#expire(java.util.Collection, long, java.util.concurrent.TimeUnit) + */ + @Override + public Boolean expire( final Collection keys, final long timeout, final TimeUnit timeUnit ) { + return expire(keys, timeout, timeUnit, false); + } + + /* (non-Javadoc) + * @see com.percero.agents.sync.datastore.IRedisCacheDataStore#expire(java.lang.String, long, java.util.concurrent.TimeUnit, java.lang.Boolean) + */ + @Override + public Boolean expire( final String key, final long timeout, final TimeUnit timeUnit, Boolean forceNow ) { + if (forceNow) { + return template.expire(key, timeout, timeUnit); + } + else { + expiresToBeWritten.put(key, new PendingExpire(key, timeout, timeUnit)); + return true; + } + } + + /* (non-Javadoc) + * @see com.percero.agents.sync.datastore.IRedisCacheDataStore#expire(java.util.Collection, long, java.util.concurrent.TimeUnit, java.lang.Boolean) + */ + @Override + public Boolean expire( final Collection keys, final long timeout, final TimeUnit timeUnit, Boolean forceNow ) { + if (forceNow) { + Iterator itrKeys = keys.iterator(); + while (itrKeys.hasNext()) { + if (!template.expire(itrKeys.next(), timeout, timeUnit)) { + return false; + } + } + return true; + } + else { + Iterator itrKeys = keys.iterator(); + while (itrKeys.hasNext()) { + String nextKey = itrKeys.next(); + expiresToBeWritten.put(nextKey, new PendingExpire(nextKey, timeout, timeUnit)); + } + return true; + } + } + + + @SuppressWarnings("unchecked") + private Map expiresToBeWritten = new ConcurrentHashMap(); + /* (non-Javadoc) + * @see com.percero.agents.sync.datastore.IRedisCacheDataStore#postExpires() + */ + @Override + @Scheduled(fixedRate=600000) // 10 * 60 * 1000 -> Ten Minutes. + public void postExpires() { + log.info("Posting expire(s).."); + int expiredCounter = 0; + for(String key : expiresToBeWritten.keySet()){ + PendingExpire nextPendingExpire = expiresToBeWritten.get(key); + if (expire(nextPendingExpire.key, nextPendingExpire.timeout, nextPendingExpire.timeUnit, true)) { + expiresToBeWritten.remove(key); + expiredCounter++; + } else { + log.error("Failed to expire key: " + key); + } + } + log.info("Expired: " + expiredCounter); + } + + private class PendingExpire { + String key; + long timeout; + TimeUnit timeUnit; + + public PendingExpire(String key, long timeout, TimeUnit timeUnit) { + this.key = key; + this.timeout = timeout; + this.timeUnit = timeUnit; + } + } + + + /* (non-Javadoc) + * @see com.percero.agents.sync.datastore.IRedisCacheDataStore#getValue(java.lang.String) + */ + @Override + public Object getValue( final String key ) { + return template.opsForValue().get( key ); + } + + /* (non-Javadoc) + * @see com.percero.agents.sync.datastore.IRedisCacheDataStore#getValues(java.util.Collection) + */ + @Override + public List getValues( final Collection keys ) { + return template.opsForValue().multiGet(keys); + } + + /* (non-Javadoc) + * @see com.percero.agents.sync.datastore.IRedisCacheDataStore#setValue(java.lang.String, java.lang.Object) + */ + @Override + public void setValue( final String key, final Object value ) { + template.opsForValue().set( key, value ); + } + +// public void setValues( final KeyValuePair[] pairs ) { + /* (non-Javadoc) + * @see com.percero.agents.sync.datastore.IRedisCacheDataStore#setValues(java.util.Map) + */ + @Override + public void setValues( final Map keysAndValuesMap ) { + template.opsForValue().multiSet(keysAndValuesMap); +// if (pairs != null) { +// for (KeyValuePair nextPair : pairs) { +// template.opsForValue().set( nextPair.key, nextPair.value ); +// } +// } + } + + /* (non-Javadoc) + * @see com.percero.agents.sync.datastore.IRedisCacheDataStore#addSetValue(java.lang.String, java.lang.Object) + */ + @Override + @Transactional + public Long addSetValue( final String key, final Object value ) { + return template.opsForSet().add(key, value);// ? Long.valueOf(1) : Long.valueOf(0); + } + + /* (non-Javadoc) + * @see com.percero.agents.sync.datastore.IRedisCacheDataStore#lpushListValue(java.lang.String, java.lang.Object) + */ + @Override + @Transactional + public Long lpushListValue( final String key, final Object value ) { + return template.opsForList().leftPush(key, value); + } + + /* (non-Javadoc) + * @see com.percero.agents.sync.datastore.IRedisCacheDataStore#listAll(java.lang.String) + */ + @Override + @Transactional + public List listAll( final String key) { + return listRange(key, Long.valueOf(0), Long.valueOf(-1)); + } + + /* (non-Javadoc) + * @see com.percero.agents.sync.datastore.IRedisCacheDataStore#listRange(java.lang.String, java.lang.Long, java.lang.Long) + */ + @Override + @Transactional + public List listRange( final String key, final Long start, final Long end ) { + return template.opsForList().range(key, start, end); + } + + /* (non-Javadoc) + * @see com.percero.agents.sync.datastore.IRedisCacheDataStore#listIndex(java.lang.String, java.lang.Long) + */ + @Override + @Transactional + public Object listIndex( final String key, final Long index ) { + return template.opsForList().index(key, index); + } + + /* (non-Javadoc) + * @see com.percero.agents.sync.datastore.IRedisCacheDataStore#listIndex(java.lang.String) + */ + @Override + @Transactional + public Object listIndex( final String key ) { + return template.opsForList().index(key, Long.valueOf(0)); + } + + /* (non-Javadoc) + * @see com.percero.agents.sync.datastore.IRedisCacheDataStore#getHashValue(java.lang.String, java.lang.String) + */ + @Override + @Transactional + public Object getHashValue( final String key, final String hashKey ) { + return template.opsForHash().get(key, hashKey); + } + + /* (non-Javadoc) + * @see com.percero.agents.sync.datastore.IRedisCacheDataStore#getHashKeys(java.lang.String) + */ + @Override + @Transactional + public Set getHashKeys( final String key ) { + Set keysObjects = template.opsForHash().keys(key); + Set results = new HashSet(keysObjects.size()); + for(Object nextKey : keysObjects) { + results.add((String)nextKey); + } + return results; + } + + /* (non-Javadoc) + * @see com.percero.agents.sync.datastore.IRedisCacheDataStore#getHashEntries(java.lang.String) + */ + @Override + @Transactional + public Map getHashEntries( final String key ) { + Map entries = template.opsForHash().entries(key); + Map result = new HashMap(); + Iterator> itrEntries = entries.entrySet().iterator(); + while (itrEntries.hasNext()) { + Entry nextEntry = itrEntries.next(); + result.put((String) nextEntry.getKey(), nextEntry.getValue()); + } + return result; + } + + /* (non-Javadoc) + * @see com.percero.agents.sync.datastore.IRedisCacheDataStore#getHashSize(java.lang.String) + */ + @Override + @Transactional + public Long getHashSize( final String key ) { + return template.opsForHash().size(key); + } + + @Transactional + public Long getSetSize( final String key ) { + return template.opsForSet().size(key); + } + + /* (non-Javadoc) + * @see com.percero.agents.sync.datastore.IRedisCacheDataStore#deleteHashKey(java.lang.String, java.lang.Object) + */ + @Override + @Transactional + public void deleteHashKey( final String key, final String hashKey ) { + template.opsForHash().delete(key, hashKey); + } + + /* (non-Javadoc) + * @see com.percero.agents.sync.datastore.IRedisCacheDataStore#setHashValue(java.lang.String, java.lang.String, java.lang.Object) + */ + @Override + @Transactional + public void setHashValue( final String key, final String hashKey, final Object value ) { + template.opsForHash().put(key, hashKey, value); + } + + /* (non-Javadoc) + * @see com.percero.agents.sync.datastore.IRedisCacheDataStore#setAllHashValues(java.lang.String, java.util.Map) + */ + @Override + @Transactional + public void setAllHashValues( final String key, final Map hashMap ) { + template.opsForHash().putAll(key, hashMap); + } + + /* (non-Javadoc) + * @see com.percero.agents.sync.datastore.IRedisCacheDataStore#removeSetValue(java.lang.String, java.lang.Object) + */ + @Override + @Transactional + public Long removeSetValue( final String key, final Object value ) { + return template.opsForSet().remove(key, value);// ? Long.valueOf(1) : Long.valueOf(0); + } + + /* (non-Javadoc) + * @see com.percero.agents.sync.datastore.IRedisCacheDataStore#getSetIsEmpty(java.lang.String) + */ + @Override + @Transactional + public Boolean getSetIsEmpty( final String key ) { + Long result = template.opsForSet().size(key);// ? Long.valueOf(1) : Long.valueOf(0); + return (result <= 0); + } + + /* (non-Javadoc) + * @see com.percero.agents.sync.datastore.IRedisCacheDataStore#removeSetValueAndGetSize(java.lang.String, java.lang.Object) + */ + @Override + @Transactional + public Long removeSetValueAndGetSize( final String key, final Object value ) { + SetOperations setOps = template.opsForSet(); + setOps.remove(key, value); + return setOps.size(key); + } + + /* (non-Javadoc) + * @see com.percero.agents.sync.datastore.IRedisCacheDataStore#removeSetsValue(java.util.Collection, java.lang.Object) + */ + @Override + public void removeSetsValue( final Collection keys, final Object value ) { + SetOperations setOps = template.opsForSet(); + Iterator itrKeys = keys.iterator(); + while(itrKeys.hasNext()) { + Object nextKey = itrKeys.next(); + try { + setOps.remove((String) nextKey, value); + } catch(Exception e) { + log.warn("Invalid Key", e); + } + } + } + + /* (non-Javadoc) + * @see com.percero.agents.sync.datastore.IRedisCacheDataStore#removeSetsValue(java.lang.String, java.util.Collection, java.lang.Object) + */ + @Override + public void removeSetsValue( final String keysPrefix, final Collection keys, final Object value ) { + SetOperations setOps = template.opsForSet(); + Iterator itrKeys = keys.iterator(); + while(itrKeys.hasNext()) { + Object nextKey = itrKeys.next(); + try { + setOps.remove(keysPrefix + (String) nextKey, value); + } catch(Exception e) { + log.warn("Invalid Key", e); + } + } + } + +// private static long keysCount = 0; +// public Set keys(String pattern) { +// Set result = template.keys(pattern); +// log.info("Keys: " + ++keysCount); +// +// return result; +// } + + /* (non-Javadoc) + * @see com.percero.agents.sync.datastore.IRedisCacheDataStore#hasKey(java.lang.String) + */ + @Override + public Boolean hasKey(String key) { + Boolean result = template.hasKey(key); + return result; + } + + /* (non-Javadoc) + * @see com.percero.agents.sync.datastore.IRedisCacheDataStore#hasHashKey(java.lang.String, java.lang.String) + */ + @Override + public Boolean hasHashKey(String key, String hashKey) { + Boolean result = template.opsForHash().hasKey(key, hashKey); + return result; + } + + /* (non-Javadoc) + * @see com.percero.agents.sync.datastore.IRedisCacheDataStore#swapHashKey(java.lang.String, java.lang.String, java.lang.String) + */ + @Override + @Transactional + public void swapHashKey( final String key, final String oldHashKey, final String newHashValue ) { + HashOperations hashOps = template.opsForHash(); + if (hashOps.hasKey(key, oldHashKey)) { + hashOps.put(key, newHashValue, hashOps.get(key, oldHashKey)); + hashOps.delete(key, oldHashKey); + } + } + + /* (non-Javadoc) + * @see com.percero.agents.sync.datastore.IRedisCacheDataStore#renameIfAbsent(java.lang.String, java.lang.String) + */ + @Override + public Boolean renameIfAbsent(String oldKey, String newKey) { + return template.renameIfAbsent(oldKey, newKey); + } + + /* (non-Javadoc) + * @see com.percero.agents.sync.datastore.IRedisCacheDataStore#rename(java.lang.String, java.lang.String) + */ + @Override + public void rename(String oldKey, String newKey) { + template.rename(oldKey, newKey); + } + + /* (non-Javadoc) + * @see com.percero.agents.sync.datastore.IRedisCacheDataStore#setUnionAndStore(java.lang.String, java.lang.String, java.lang.String) + */ + @Override + public Long setUnionAndStore(String key, String otherKey, String destKey) { + return template.opsForSet().unionAndStore(key, otherKey, destKey); + } + + /* (non-Javadoc) + * @see com.percero.agents.sync.datastore.IRedisCacheDataStore#setUnionAndStore(java.lang.String, java.util.Collection, java.lang.String) + */ + @Override + public Long setUnionAndStore(String key, Collection otherKeys, String destKey) { + return template.opsForSet().unionAndStore(key, otherKeys, destKey); + } + + /* (non-Javadoc) + * @see com.percero.agents.sync.datastore.IRedisDataStore#removeSetValues(java.lang.String, java.util.Collection) + */ + /* (non-Javadoc) + * @see com.percero.agents.sync.datastore.IRedisCacheDataStore#removeSetValues(java.lang.String, java.util.Collection) + */ + @Override + @Transactional + public void removeSetValues( final String key, final Collection values ) { + SetOperations setOps = template.opsForSet(); + + Iterator itrValues = values.iterator(); + while(itrValues.hasNext()) { + try { + setOps.remove(key, itrValues.next()); + } catch(Exception e) { + log.error("Unable to remove Set Value", e); + } + } + } + + /* (non-Javadoc) + * @see com.percero.agents.sync.datastore.IRedisDataStore#getSetValue(java.lang.String) + */ + /* (non-Javadoc) + * @see com.percero.agents.sync.datastore.IRedisCacheDataStore#getSetValue(java.lang.String) + */ + @Override + public Set getSetValue( final String key ) { + return template.opsForSet().members(key); + } + + /* (non-Javadoc) + * @see com.percero.agents.sync.datastore.IRedisCacheDataStore#replaceSet(java.lang.String, java.util.Collection) + */ + @Override + @Transactional + public Long replaceSet( final String key, Collection values ) { + template.delete(key); + if (values != null) { + return template.opsForSet().add(key, values.toArray()); + } + else { + return Long.valueOf(0); + } + } + + /* (non-Javadoc) + * @see com.percero.agents.sync.datastore.IRedisDataStore#swapSetValue(java.lang.String, java.lang.String, java.lang.String) + */ + /* (non-Javadoc) + * @see com.percero.agents.sync.datastore.IRedisCacheDataStore#swapSetValue(java.lang.String, java.lang.String, java.lang.String) + */ + @Override + @Transactional + public void swapSetValue( final String key, final String oldValue, final String newValue ) { + SetOperations setOps = template.opsForSet(); + if (setOps.isMember(key, oldValue)) { + setOps.remove(key, oldValue); + setOps.add(key, newValue); + } + } + + /* (non-Javadoc) + * @see com.percero.agents.sync.datastore.IRedisCacheDataStore#getSetIsMember(java.lang.String, java.lang.Object) + */ + @Override + public Boolean getSetIsMember( final String key, final Object object ) { + return template.opsForSet().isMember(key, object); + } + + /* (non-Javadoc) + * @see com.percero.agents.sync.datastore.IRedisCacheDataStore#getSetsContainsMembers(java.util.Collection, java.lang.Object[]) + */ + @Override + @Transactional + public Set getSetsContainsMembers( final Collection keys, final Object[] membersToCheck ) { + SetOperations setOps = template.opsForSet(); + String randomKey = "UserTokenCheck:" + System.currentTimeMillis(); + setOps.add(randomKey, membersToCheck); + + Set intersectingMembers = new HashSet(); + Iterator itrKeys = keys.iterator(); + while (itrKeys.hasNext()) { + String nextKey = itrKeys.next(); + intersectingMembers.addAll(setOps.intersect(randomKey, nextKey)); + } + template.delete(randomKey); + return intersectingMembers; + } + + /* (non-Javadoc) + * @see com.percero.agents.sync.datastore.IRedisCacheDataStore#getSetIntersect(java.lang.String, java.util.Collection) + */ + @Override + public Set getSetIntersect( final String key, Collection otherKeys ) { + return template.opsForSet().intersect(key, otherKeys); + } + + /* (non-Javadoc) + * @see com.percero.agents.sync.datastore.IRedisCacheDataStore#getSetIntersect(java.lang.String, java.lang.String) + */ + @Override + public Set getSetIntersect( final String key, String otherKey ) { + return template.opsForSet().intersect(key, otherKey); + } + + /* (non-Javadoc) + * @see com.percero.agents.sync.datastore.IRedisCacheDataStore#getSetUnion(java.lang.String, java.util.Collection) + */ + @Override + public Set getSetUnion( final String key, final Collection otherKeys ) { + return template.opsForSet().union(key, otherKeys); + } + + /* (non-Javadoc) + * @see com.percero.agents.sync.datastore.IRedisCacheDataStore#getSetUnion(java.lang.String, java.lang.String) + */ + @Override + public Set getSetUnion( final String key, final String otherKey ) { + return template.opsForSet().union(key, otherKey); + } + + /* (non-Javadoc) + * @see com.percero.agents.sync.datastore.IRedisCacheDataStore#deleteKey(java.lang.String) + */ + @Override + public void deleteKey( final String key ) { + template.delete(key); + } + + /* (non-Javadoc) + * @see com.percero.agents.sync.datastore.IRedisCacheDataStore#deleteKeys(java.util.Collection) + */ + @Override + public void deleteKeys( final Collection keys ) { + if (keys != null && !keys.isEmpty()) { + template.delete(keys); + } + } + + /* (non-Javadoc) + * @see com.percero.agents.sync.datastore.IRedisCacheDataStore#setSetValue(java.lang.String, java.lang.Object) + */ + @Override + @Transactional + public Long setSetValue(String key, Object value) { + return template.opsForSet().add(key, value);// ? Long.valueOf(1) : Long.valueOf(0); + } + + @Override + @Transactional + public Long setSetsValues(Map> values) { + Long result = new Long(0); + Iterator>> itr = values.entrySet().iterator(); + while (itr.hasNext()) { + Entry> nextEntry = itr.next(); + result += template.opsForSet().add(nextEntry.getKey(), nextEntry.getValue().toArray());// ? Long.valueOf(1) : Long.valueOf(0); + + } + return result; + } + + @Override + @Transactional + public Long removeSetsValues(Map> values) { + Long result = new Long(0); + Iterator>> itr = values.entrySet().iterator(); + while (itr.hasNext()) { + Entry> nextEntry = itr.next(); + result += template.opsForSet().remove(nextEntry.getKey(), nextEntry.getValue().toArray());// ? Long.valueOf(1) : Long.valueOf(0); + + } + return result; + } + + /* (non-Javadoc) + * @see com.percero.agents.sync.datastore.IRedisCacheDataStore#getSet(java.lang.String) + */ + @Override + public SetOperations getSet(String key) { + return template.opsForSet(); + } + + /* (non-Javadoc) + * @see com.percero.agents.sync.datastore.IRedisCacheDataStore#getTemplate() + */ + @Override + public RedisTemplate getTemplate() { + return template; + } + + /* (non-Javadoc) + * @see com.percero.agents.sync.datastore.IRedisCacheDataStore#keys(java.lang.String) + */ + @Override + public Collection keys(String pattern) { + return template.keys(pattern); + } + + /* (non-Javadoc) + * @see com.percero.agents.sync.datastore.IRedisCacheDataStore#removeKeysValue(java.lang.String, java.lang.Object) + */ + @Override + @Transactional + public void removeKeysValue(String pattern, Object value) { + Set keys = template.keys(pattern); + Iterator itrKeys = keys.iterator(); + + while(itrKeys.hasNext()) { + try { + template.opsForSet().remove(itrKeys.next(), value); + } catch(Exception e) { + log.error("Error removing KeysValue", e); + } + } + } + +} From fd9e5b1225a21a1d9aa62132fa93735902d2df60 Mon Sep 17 00:00:00 2001 From: Jay Eno Date: Mon, 18 Jan 2016 18:08:28 -0700 Subject: [PATCH 087/105] Removed unnecessary supress --- .../com/percero/agents/sync/datastore/RedisCacheDataStore.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/main/java/com/percero/agents/sync/datastore/RedisCacheDataStore.java b/src/main/java/com/percero/agents/sync/datastore/RedisCacheDataStore.java index 07ccd6b..a3ebacc 100644 --- a/src/main/java/com/percero/agents/sync/datastore/RedisCacheDataStore.java +++ b/src/main/java/com/percero/agents/sync/datastore/RedisCacheDataStore.java @@ -83,8 +83,6 @@ public Boolean expire( final Collection keys, final long timeout, final } } - - @SuppressWarnings("unchecked") private Map expiresToBeWritten = new ConcurrentHashMap(); /* (non-Javadoc) * @see com.percero.agents.sync.datastore.IRedisCacheDataStore#postExpires() From 3160d3562e08c6a7482335d470fb606676828a0a Mon Sep 17 00:00:00 2001 From: Jay Eno Date: Wed, 20 Jan 2016 17:22:32 -0700 Subject: [PATCH 088/105] Whoops. Missed the reason behind not acting on the return value from expires(). Abstraction prevents discovery of root cause on a failed call to expire(), so no way to determine what went wrong to do something about it. Best bet is just to assume the key is gone and just not try again, relying on the redis default key timeout to ultimately do it's job. This fix makes sure it tries once, then just moves on. Added logging to indicate success/failures as well, as it might prove useful someday. Maybe. --- .../agents/sync/datastore/RedisCacheDataStore.java | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/main/java/com/percero/agents/sync/datastore/RedisCacheDataStore.java b/src/main/java/com/percero/agents/sync/datastore/RedisCacheDataStore.java index a3ebacc..ade58c4 100644 --- a/src/main/java/com/percero/agents/sync/datastore/RedisCacheDataStore.java +++ b/src/main/java/com/percero/agents/sync/datastore/RedisCacheDataStore.java @@ -92,16 +92,18 @@ public Boolean expire( final Collection keys, final long timeout, final public void postExpires() { log.info("Posting expire(s).."); int expiredCounter = 0; + int failedCounter = 0; for(String key : expiresToBeWritten.keySet()){ PendingExpire nextPendingExpire = expiresToBeWritten.get(key); if (expire(nextPendingExpire.key, nextPendingExpire.timeout, nextPendingExpire.timeUnit, true)) { - expiresToBeWritten.remove(key); expiredCounter++; } else { - log.error("Failed to expire key: " + key); + failedCounter++; } + expiresToBeWritten.remove(key); } - log.info("Expired: " + expiredCounter); + log.info("Successully Expired: " + expiredCounter); + log.info("Failed to expire: " + failedCounter); } private class PendingExpire { From b1deaa862865ee7aa724a90b230e0f5883949fd7 Mon Sep 17 00:00:00 2001 From: Nikish Parikh Date: Wed, 3 Feb 2016 00:40:44 +0530 Subject: [PATCH 089/105] Data source property added --- pom.xml | 2 +- .../agents/sync/services/DAODataProvider.java | 10 ++++++++++ .../percero/agents/sync/vo/BaseDataObject.java | 16 ++++++++++++++++ 3 files changed, 27 insertions(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 819436f..19595b5 100644 --- a/pom.xml +++ b/pom.xml @@ -7,7 +7,7 @@ org.activestack syncengine - 1.1.57-SNAPSHOT + 1.1.58-SNAPSHOT 3.2.4.RELEASE diff --git a/src/main/java/com/percero/agents/sync/services/DAODataProvider.java b/src/main/java/com/percero/agents/sync/services/DAODataProvider.java index b7ef6f2..82464d8 100644 --- a/src/main/java/com/percero/agents/sync/services/DAODataProvider.java +++ b/src/main/java/com/percero/agents/sync/services/DAODataProvider.java @@ -405,6 +405,9 @@ private IPerceroObject retrieveFromRedisCache(ClassIDPair classIdPair) throws Ex } } + if (result instanceof BaseDataObject) { + ((BaseDataObject) result).setDataSource(BaseDataObject.DATA_SOURCE_CACHE); + } return result; } @@ -446,10 +449,17 @@ private Map retrieveFromRedisCache(ClassIDPairs classIdP if (IJsonObject.class.isAssignableFrom(theClass)) { IJsonObject jsonObject = (IJsonObject) theClass.newInstance(); jsonObject.fromJson(jsonObjectString); + if (jsonObject instanceof BaseDataObject) { + ((BaseDataObject) jsonObject).setDataSource(BaseDataObject.DATA_SOURCE_CACHE); + } result.put( (((IPerceroObject) jsonObject).getID()), (IPerceroObject) jsonObject ); } else { IPerceroObject nextPerceroObject = (IPerceroObject) safeObjectMapper.readValue(jsonObjectString, theClass); + + if (nextPerceroObject instanceof BaseDataObject) { + ((BaseDataObject) nextPerceroObject).setDataSource(BaseDataObject.DATA_SOURCE_CACHE); + } result.put( nextPerceroObject.getID(), nextPerceroObject); } diff --git a/src/main/java/com/percero/agents/sync/vo/BaseDataObject.java b/src/main/java/com/percero/agents/sync/vo/BaseDataObject.java index 111a281..a641f1f 100644 --- a/src/main/java/com/percero/agents/sync/vo/BaseDataObject.java +++ b/src/main/java/com/percero/agents/sync/vo/BaseDataObject.java @@ -34,6 +34,10 @@ public class BaseDataObject implements Externalizable, IPerceroObject, IJsonObject, Comparable { private static final Logger log = Logger.getLogger(BaseDataObject.class); + + public static final String DATA_SOURCE_CACHE="CACHE"; + + public static final String DATA_SOURCE_DATA_STORE="DATA_STORE"; public BaseDataObject() { super(); @@ -55,6 +59,16 @@ public final Boolean getIsClean() { public final void setIsClean(Boolean isClean) { this.isClean = isClean; } + + @Transient + transient private String dataSource = DATA_SOURCE_DATA_STORE; + public final String getDataSource() { + return dataSource; + } + public final void setDataSource(String value) { + this. dataSource = value; + } + public String classVersion() { return null; @@ -102,6 +116,7 @@ public String toEmbeddedJson(Boolean encloseString) { public String retrieveBaseJson() { String objectJson = "\"cn\":\"" + getClass().getCanonicalName() + "\"," + "\"ID\":\"" + getID() + "\""; + //+ "\"dataSource\":\"" + getDataSource() + "\"" ; return objectJson; } @@ -123,6 +138,7 @@ public void fromJson(String jsonString) { protected void fromJson(JsonObject jsonObject) { // ID this.setID(JsonUtils.getJsonString(jsonObject, "ID")); +// this.setDataSource(JsonUtils.getJsonString(jsonObject, "dataSource")); } public static ClassIDPair toClassIdPair(BaseDataObject object) { From 9d5b3a1471c323864f2cf132b5f14145b83effd4 Mon Sep 17 00:00:00 2001 From: Collin Brown Date: Thu, 4 Feb 2016 08:45:14 -0800 Subject: [PATCH 090/105] Adding Hikari Connection Pool option for UpdateTables --- pom.xml | 17 +- .../jobs/UpdateTableConnectionFactory.java | 81 +------ .../datasource/BaseConnectionFactory.java | 227 ++++++++++++++++++ .../datasource/IConnectionFactory.java | 13 + .../datasource/TestBaseConnectionFactory.java | 161 +++++++++++++ .../TestUpdateTablesConnectionFactory.java | 94 ++++++++ .../resources/baseconnectionfactories.yml | 11 + .../updatetablesconnectionfactories.yml | 27 +++ 8 files changed, 551 insertions(+), 80 deletions(-) create mode 100644 src/main/java/com/percero/datasource/BaseConnectionFactory.java create mode 100644 src/main/java/com/percero/datasource/IConnectionFactory.java create mode 100644 src/test/java/com/percero/datasource/TestBaseConnectionFactory.java create mode 100644 src/test/java/com/percero/datasource/TestUpdateTablesConnectionFactory.java create mode 100644 src/test/resources/baseconnectionfactories.yml create mode 100644 src/test/resources/updatetablesconnectionfactories.yml diff --git a/pom.xml b/pom.xml index 19595b5..8cb767e 100644 --- a/pom.xml +++ b/pom.xml @@ -7,7 +7,7 @@ org.activestack syncengine - 1.1.58-SNAPSHOT + 1.1.59-SNAPSHOT 3.2.4.RELEASE @@ -335,6 +335,21 @@ logback-ext-loggly 0.1.2 + + + + com.zaxxer + HikariCP + 2.4.1 + + + + + com.h2database + h2 + 1.3.167 + test + diff --git a/src/main/java/com/percero/agents/sync/jobs/UpdateTableConnectionFactory.java b/src/main/java/com/percero/agents/sync/jobs/UpdateTableConnectionFactory.java index 8576242..4e0192a 100644 --- a/src/main/java/com/percero/agents/sync/jobs/UpdateTableConnectionFactory.java +++ b/src/main/java/com/percero/agents/sync/jobs/UpdateTableConnectionFactory.java @@ -1,51 +1,12 @@ package com.percero.agents.sync.jobs; -import com.mchange.v2.c3p0.ComboPooledDataSource; -import org.apache.log4j.Logger; - -import java.beans.PropertyVetoException; -import java.sql.Connection; -import java.sql.SQLException; +import com.percero.datasource.BaseConnectionFactory; /** * Created by jonnysamps on 9/2/15. */ -//@Component -public class UpdateTableConnectionFactory { - - private static Logger logger = Logger.getLogger(UpdateTableConnectionFactory.class); - - private String driverClassName; - public void setDriverClassName(String val){ - this.driverClassName = val; - } - public String getDriverClassName(){ - return driverClassName; - } +public class UpdateTableConnectionFactory extends BaseConnectionFactory { - private String username; - public void setUsername(String val){ - this.username = val; - } - public String getUsername(){ - return username; - } - - private String password; - public void setPassword(String val){ - this.password = val; - } - public String getPassword(){ - return password; - } - - private String jdbcUrl; - public void setJdbcUrl(String val){ - this.jdbcUrl = val; - } - public String getJdbcUrl(){ - return jdbcUrl; - } private String[] tableNames; public void setTableNames(String[] val) { @@ -117,42 +78,4 @@ public int getWeight(){ return this.weight; } - private ComboPooledDataSource cpds; - - public void init() throws PropertyVetoException{ - try { - cpds = new ComboPooledDataSource(); - cpds.setDriverClass(driverClassName); //loads the jdbc driver - cpds.setJdbcUrl(jdbcUrl); - cpds.setUser(username); - cpds.setPassword(password); - - // the settings below are optional -- c3p0 can work with defaults - cpds.setMinPoolSize(10); - cpds.setAcquireIncrement(5); - cpds.setMaxPoolSize(this.weight); - cpds.setTestConnectionOnCheckout(true); - - }catch(PropertyVetoException pve){ - logger.error(pve.getMessage(), pve); - throw pve; - } - } - - public Connection getConnection() throws SQLException{ - try{ - if (cpds == null) { - init(); - } - return cpds.getConnection(); - } - catch(PropertyVetoException e){ - logger.error(e.getMessage(), e); - throw new SQLException(e); - } - catch(SQLException e){ - logger.error(e.getMessage(), e); - throw e; - } - } } diff --git a/src/main/java/com/percero/datasource/BaseConnectionFactory.java b/src/main/java/com/percero/datasource/BaseConnectionFactory.java new file mode 100644 index 0000000..440725e --- /dev/null +++ b/src/main/java/com/percero/datasource/BaseConnectionFactory.java @@ -0,0 +1,227 @@ +package com.percero.datasource; + +import java.beans.PropertyVetoException; +import java.sql.Connection; +import java.sql.SQLException; + +import javax.sql.DataSource; + +import org.apache.log4j.Logger; +import org.springframework.util.StringUtils; + +import com.mchange.v2.c3p0.ComboPooledDataSource; +import com.zaxxer.hikari.HikariConfig; +import com.zaxxer.hikari.HikariDataSource; + +public class BaseConnectionFactory implements IConnectionFactory { + + private static Logger logger = Logger.getLogger(BaseConnectionFactory.class); + + public static final String HIKARI_CONNECTION_POOL = "hikari"; + public static final String C3P0_CONNECTION_POOL = "c3p0"; + + private String name; + private String preferredConnectionPool = HIKARI_CONNECTION_POOL; + private Integer acquireIncrement = 4; + private Integer minPoolSize = 4; + private Integer maxPoolSize = 52; + private Integer maxIdleTime = 60 * 30; // 30 Minutes + private String testQuery = "SELECT 1 FROM dual"; + private Integer fetchSize = 100; + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getPreferredConnectionPool() { + return preferredConnectionPool; + } + + public void setPreferredConnectionPool(String preferredConnectionPool) { + this.preferredConnectionPool = preferredConnectionPool; + } + + public Integer getAcquireIncrement() { + return acquireIncrement; + } + + public void setAcquireIncrement(Integer acquireIncrement) { + this.acquireIncrement = acquireIncrement; + } + + public Integer getMinPoolSize() { + return minPoolSize; + } + + public void setMinPoolSize(Integer minPoolSize) { + this.minPoolSize = minPoolSize; + } + + public Integer getMaxPoolSize() { + return maxPoolSize; + } + + public void setMaxPoolSize(Integer maxPoolSize) { + this.maxPoolSize = maxPoolSize; + } + + public Integer getMaxIdleTime() { + return maxIdleTime; + } + + public void setMaxIdleTime(Integer maxIdleTime) { + this.maxIdleTime = maxIdleTime; + } + + public String getTestQuery() { + return testQuery; + } + + public void setTestQuery(String testQuery) { + this.testQuery = testQuery; + } + + public Integer getFetchSize() { + // Default to 100; + if (fetchSize == null || fetchSize <= 0) { + return 100; + } + return fetchSize; + } + + public void setFetchSize(Integer fetchSize) { + this.fetchSize = fetchSize; + } + + private String driverClassName; + public void setDriverClassName(String val){ + this.driverClassName = val; + } + public String getDriverClassName(){ + return driverClassName; + } + + private String username; + public void setUsername(String val){ + this.username = val; + } + public String getUsername(){ + return username; + } + + private String password; + public void setPassword(String val){ + this.password = val; + } + public String getPassword(){ + return password; + } + + private String jdbcUrl; + public void setJdbcUrl(String val){ + this.jdbcUrl = val; + } + public String getJdbcUrl(){ + return jdbcUrl; + } + + private DataSource ds; + public DataSource getDataSource() { + return ds; + } + + public void init() throws PropertyVetoException{ + try { + if (C3P0_CONNECTION_POOL.equalsIgnoreCase(preferredConnectionPool)) { + ComboPooledDataSource cpds = new ComboPooledDataSource(); + cpds.setDriverClass(driverClassName); // loads the jdbc driver + cpds.setJdbcUrl(jdbcUrl); + cpds.setUser(username); + cpds.setPassword(password); + + // the settings below are optional -- c3p0 can work with + // defaults + if (minPoolSize != null) { + cpds.setMinPoolSize(minPoolSize); + } + if (acquireIncrement != null) { + cpds.setAcquireIncrement(acquireIncrement); + } + if (maxPoolSize != null) { + cpds.setMaxPoolSize(maxPoolSize); + } + if (maxIdleTime != null) { + cpds.setMaxIdleTime(maxIdleTime); + cpds.setIdleConnectionTestPeriod(maxIdleTime); + } + cpds.setNumHelperThreads(30); + cpds.setTestConnectionOnCheckout(true); + if (StringUtils.hasText(testQuery)) { + cpds.setPreferredTestQuery(testQuery); + } + + ds = cpds; + } else { + // Default to Hikari Connection Pool. + HikariConfig config = new HikariConfig(); + config.setDriverClassName(driverClassName); + config.setJdbcUrl(jdbcUrl); + config.setUsername(username); + config.setPassword(password); + config.addDataSourceProperty("cachePrepStmts", "true"); + config.addDataSourceProperty("prepStmtCacheSize", "250"); + config.addDataSourceProperty("prepStmtCacheSqlLimit", "2048"); + + if (StringUtils.hasText(name)) { + config.setPoolName(name); + } + if (minPoolSize != null) { + config.setMinimumIdle(minPoolSize); + } + // if (acquireIncrement != null) { + // config.setAcquireIncrement(acquireIncrement); + // } + if (maxPoolSize != null) { + config.setMaximumPoolSize(maxPoolSize); + } + if (maxIdleTime != null) { + config.setIdleTimeout(maxIdleTime * 1000); // Convert to + // milliseconds + } + // config.setNumHelperThreads(30); + // config.setTestConnectionOnCheckout(true); + if (StringUtils.hasText(testQuery)) { + config.setConnectionTestQuery(testQuery); + } + + ds = new HikariDataSource(config); + } + + }catch(PropertyVetoException pve){ + logger.error(pve.getMessage(), pve); + throw pve; + } + } + + public Connection getConnection() throws SQLException{ + try{ + if (ds == null) { + init(); + } + return getDataSource().getConnection(); + } + catch(PropertyVetoException e){ + logger.error(e.getMessage(), e); + throw new SQLException(e); + } + catch(SQLException e){ + logger.error(e.getMessage(), e); + throw e; + } + } + +} diff --git a/src/main/java/com/percero/datasource/IConnectionFactory.java b/src/main/java/com/percero/datasource/IConnectionFactory.java new file mode 100644 index 0000000..62d71bd --- /dev/null +++ b/src/main/java/com/percero/datasource/IConnectionFactory.java @@ -0,0 +1,13 @@ +package com.percero.datasource; + +import java.sql.Connection; +import java.sql.SQLException; + +public interface IConnectionFactory { + + String getName(); + void setName(String name); + Connection getConnection() throws SQLException; + + Integer getFetchSize(); +} diff --git a/src/test/java/com/percero/datasource/TestBaseConnectionFactory.java b/src/test/java/com/percero/datasource/TestBaseConnectionFactory.java new file mode 100644 index 0000000..5bf0beb --- /dev/null +++ b/src/test/java/com/percero/datasource/TestBaseConnectionFactory.java @@ -0,0 +1,161 @@ +package com.percero.datasource; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import java.beans.PropertyVetoException; +import java.io.File; +import java.io.FileNotFoundException; +import java.io.FileReader; +import java.net.URL; +import java.sql.Connection; +import java.sql.SQLException; + +import javax.sql.DataSource; + +import org.junit.After; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; +import org.mockito.Mockito; + +import com.esotericsoftware.yamlbeans.YamlException; +import com.esotericsoftware.yamlbeans.YamlReader; +import com.mchange.v2.c3p0.ComboPooledDataSource; +import com.zaxxer.hikari.HikariDataSource; + +public class TestBaseConnectionFactory { + + @BeforeClass + public static void setUpBeforeClass() throws Exception { + } + + @AfterClass + public static void tearDownAfterClass() throws Exception { + } + + BaseConnectionFactory bcf = null; + @Before + public void setUp() throws Exception { + } + + @After + public void tearDown() throws Exception { + bcf = null; + } + + @Test + public void testInit() { + BaseConnectionFactory bcf = new BaseConnectionFactory(); + bcf.setJdbcUrl("jdbc:h2:~/test"); + bcf.setDriverClassName("org.h2.Driver"); + bcf.setName("TESTS"); + bcf.setUsername("sa"); + bcf.setPassword(""); + + String testQuery = "SELECT 1 FROM dual"; + bcf.setTestQuery(testQuery); + bcf.setMaxIdleTime(1000); + + try { + // Validate data source type. + bcf.init(); + assertTrue(bcf.getDataSource() instanceof HikariDataSource); + HikariDataSource hds = (HikariDataSource) bcf.getDataSource(); + assertEquals(bcf.getDriverClassName(), hds.getDriverClassName()); + assertEquals(bcf.getJdbcUrl(), hds.getJdbcUrl()); + assertEquals(bcf.getUsername(), hds.getUsername()); + assertEquals(bcf.getPassword(), hds.getPassword()); + assertEquals(bcf.getName(), hds.getPoolName()); + assertEquals(bcf.getMinPoolSize().intValue(), hds.getMinimumIdle()); + assertEquals(bcf.getMaxPoolSize().longValue(), hds.getMaximumPoolSize()); + assertEquals(bcf.getMaxIdleTime().longValue() * 1000, hds.getIdleTimeout()); + assertEquals(testQuery, hds.getConnectionTestQuery()); + + assertEquals("true", hds.getDataSourceProperties().getProperty("cachePrepStmts")); + assertEquals("250", hds.getDataSourceProperties().getProperty("prepStmtCacheSize")); + assertEquals("2048", hds.getDataSourceProperties().getProperty("prepStmtCacheSqlLimit")); + + // C3PO. + bcf.setPreferredConnectionPool(BaseConnectionFactory.C3P0_CONNECTION_POOL); + bcf.init(); + assertTrue(bcf.getDataSource() instanceof ComboPooledDataSource); + ComboPooledDataSource cpds = (ComboPooledDataSource) bcf.getDataSource(); + assertEquals(bcf.getDriverClassName(), cpds.getDriverClass()); + assertEquals(bcf.getJdbcUrl(), cpds.getJdbcUrl()); + assertEquals(bcf.getUsername(), cpds.getUser()); + assertEquals(bcf.getPassword(), cpds.getPassword()); + assertEquals(bcf.getMinPoolSize().intValue(), cpds.getMinPoolSize()); + assertEquals(bcf.getAcquireIncrement().intValue(), cpds.getAcquireIncrement()); + assertEquals(bcf.getMaxPoolSize().intValue(), cpds.getMaxPoolSize()); + assertEquals(bcf.getMaxIdleTime().intValue(), cpds.getMaxIdleTime()); + assertEquals(bcf.getMaxIdleTime().intValue(), cpds.getIdleConnectionTestPeriod()); + assertEquals(30, cpds.getNumHelperThreads()); + assertEquals(testQuery, cpds.getPreferredTestQuery()); + + // Hikari. + bcf.setPreferredConnectionPool(BaseConnectionFactory.HIKARI_CONNECTION_POOL); + bcf.init(); + assertTrue(bcf.getDataSource() instanceof HikariDataSource); + } catch (PropertyVetoException e) { + // TODO Auto-generated catch block + fail("Error with property: " + e.getMessage()); + } + } + + @Test + public void testGetConnection() { + + DataSource mockDataSource = Mockito.mock(DataSource.class); + Connection connection = Mockito.mock(Connection.class); + + try { + Mockito.when(mockDataSource.getConnection()).thenReturn(connection); + } catch (SQLException e) { + fail("Failed to Mock DataSource.getConnection(): " + e.getMessage()); + } + + bcf = Mockito.mock(BaseConnectionFactory.class); + Mockito.when(bcf.getDataSource()).thenReturn(mockDataSource); + + try { + Mockito.when(bcf.getConnection()).thenCallRealMethod(); + assertSame(connection, bcf.getConnection()); + } catch (SQLException e) { + fail("Failed to get Connection from DataSource: " + e.getMessage()); + } + } + + @Test + public void testYmlImport() { + try { + URL ymlUrl = TestBaseConnectionFactory.class.getClassLoader().getResource("baseconnectionfactories.yml"); + if (ymlUrl == null) { + fail("No configuration file found: baseconnectionFactories.yml"); + } + File configFile = new File(ymlUrl.getFile()); + YamlReader reader = new YamlReader(new FileReader(configFile)); + + BaseConnectionFactory connectionFactory = reader.read(BaseConnectionFactory.class); + assertNotNull(connectionFactory); + assertEquals("org.h2.Driver", connectionFactory.getDriverClassName()); + assertEquals("jdbc:h2:~/test", connectionFactory.getJdbcUrl()); + assertEquals("sa", connectionFactory.getUsername()); + assertEquals("pass", connectionFactory.getPassword()); + assertEquals("TEST", connectionFactory.getName()); + assertEquals(new Integer(5), connectionFactory.getMinPoolSize()); + assertEquals(new Integer(25), connectionFactory.getMaxPoolSize()); + assertEquals(new Integer(1000), connectionFactory.getMaxIdleTime()); + assertEquals("SELECT 1 FROM dual", connectionFactory.getTestQuery()); + } + catch (FileNotFoundException e) { + fail("Error processing config file baseconnectionfactories.yml: " + e.getMessage()); + } catch (YamlException e) { + fail("YAML Error processing config file baseconnectionfactories.yml: " + e.getMessage()); + } + } +} diff --git a/src/test/java/com/percero/datasource/TestUpdateTablesConnectionFactory.java b/src/test/java/com/percero/datasource/TestUpdateTablesConnectionFactory.java new file mode 100644 index 0000000..2a225b5 --- /dev/null +++ b/src/test/java/com/percero/datasource/TestUpdateTablesConnectionFactory.java @@ -0,0 +1,94 @@ +package com.percero.datasource; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.fail; + +import java.io.File; +import java.io.FileNotFoundException; +import java.io.FileReader; +import java.net.URL; + +import org.junit.After; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +import com.esotericsoftware.yamlbeans.YamlException; +import com.esotericsoftware.yamlbeans.YamlReader; +import com.percero.agents.sync.jobs.UpdateTableConnectionFactory; + +public class TestUpdateTablesConnectionFactory { + + @BeforeClass + public static void setUpBeforeClass() throws Exception { + } + + @AfterClass + public static void tearDownAfterClass() throws Exception { + } + + @Before + public void setUp() throws Exception { + } + + @After + public void tearDown() throws Exception { + } + + @Test + public void testYmlImport() { + try { + URL ymlUrl = TestUpdateTablesConnectionFactory.class.getClassLoader().getResource("updatetablesconnectionfactories.yml"); + if (ymlUrl == null) { + fail("No configuration file found: updatetablesconnectionfactories.yml"); + } + File configFile = new File(ymlUrl.getFile()); + YamlReader reader = new YamlReader(new FileReader(configFile)); + + // First UpdateTableConnectionFactory + UpdateTableConnectionFactory connectionFactory = reader.read(UpdateTableConnectionFactory.class); + assertNotNull(connectionFactory); + assertEquals("org.h2.Driver", connectionFactory.getDriverClassName()); + assertEquals("jdbc:h2:~/test", connectionFactory.getJdbcUrl()); + assertEquals("sa", connectionFactory.getUsername()); + assertEquals("pass", connectionFactory.getPassword()); + assertEquals("TEST", connectionFactory.getName()); + assertEquals(new Integer(5), connectionFactory.getMinPoolSize()); + assertEquals(new Integer(25), connectionFactory.getMaxPoolSize()); + assertEquals(new Integer(1000), connectionFactory.getMaxIdleTime()); + assertEquals("SELECT 1 FROM dual", connectionFactory.getTestQuery()); + + assertEquals(50, connectionFactory.getWeight()); + assertEquals("UPDATE_TABLE_LOCK_BATCH", connectionFactory.getStoredProcedureName()); + assertEquals("My Stored Proc", connectionFactory.getStoredProcedureDefinition()); + + assertNotNull(connectionFactory.getTableNames()); + assertEquals(2, connectionFactory.getTableNames().length); + assertEquals("update_table_1", connectionFactory.getTableNames()[0]); + assertEquals("update_table_2", connectionFactory.getTableNames()[1]); + + // Second UpdateTableConnectionFactory + connectionFactory = reader.read(UpdateTableConnectionFactory.class); + assertNotNull(connectionFactory); + assertEquals("org.h2.Driver", connectionFactory.getDriverClassName()); + assertEquals("jdbc:h2:~/test", connectionFactory.getJdbcUrl()); + assertEquals("sa", connectionFactory.getUsername()); + assertEquals("pass", connectionFactory.getPassword()); + + assertEquals(55, connectionFactory.getWeight()); + assertEquals("UPDATE_TABLE_LOCK_BATCH", connectionFactory.getStoredProcedureName()); + assertEquals("My Stored Proc", connectionFactory.getStoredProcedureDefinition()); + + assertNotNull(connectionFactory.getTableNames()); + assertEquals(1, connectionFactory.getTableNames().length); + assertEquals("update_table_1", connectionFactory.getTableNames()[0]); + } + catch (FileNotFoundException e) { + fail("Error processing config file updatetablesconnectionfactories.yml: " + e.getMessage()); + } catch (YamlException e) { + fail("YAML Error processing config file updatetablesconnectionfactories.yml: " + e.getMessage()); + } + } +} diff --git a/src/test/resources/baseconnectionfactories.yml b/src/test/resources/baseconnectionfactories.yml new file mode 100644 index 0000000..ea4edfa --- /dev/null +++ b/src/test/resources/baseconnectionfactories.yml @@ -0,0 +1,11 @@ +# --- +name: TEST +preferredConnectionPool: hikari +driverClassName: org.h2.Driver +jdbcUrl: jdbc:h2:~/test +username: sa +password: pass +testQuery: SELECT 1 FROM dual +maxIdleTime: 1000 +minPoolSize: 5 +maxPoolSize: 25 \ No newline at end of file diff --git a/src/test/resources/updatetablesconnectionfactories.yml b/src/test/resources/updatetablesconnectionfactories.yml new file mode 100644 index 0000000..6a5ec88 --- /dev/null +++ b/src/test/resources/updatetablesconnectionfactories.yml @@ -0,0 +1,27 @@ +# --- +name: TEST +preferredConnectionPool: hikari +driverClassName: org.h2.Driver +jdbcUrl: jdbc:h2:~/test +username: sa +password: pass +testQuery: SELECT 1 FROM dual +maxIdleTime: 1000 +minPoolSize: 5 +maxPoolSize: 25 +tableNames: + - update_table_1 + - update_table_2 +weight: 50 +storedProcedureName: UPDATE_TABLE_LOCK_BATCH +storedProcedureDefinition: My Stored Proc +--- +driverClassName: org.h2.Driver +jdbcUrl: jdbc:h2:~/test +username: sa +password: pass +tableNames: + - update_table_1 +weight: 55 +storedProcedureName: UPDATE_TABLE_LOCK_BATCH +storedProcedureDefinition: My Stored Proc From 31fdcb3a55d6a3fcd9823afd8d93e7949a19db96 Mon Sep 17 00:00:00 2001 From: Collin Brown Date: Wed, 10 Feb 2016 14:33:37 -0800 Subject: [PATCH 091/105] Added some comments around removing change watcher redis keys for a removed object --- .../sync/access/RedisAccessManager.java | 47 +++++++++++++++++-- 1 file changed, 42 insertions(+), 5 deletions(-) diff --git a/src/main/java/com/percero/agents/sync/access/RedisAccessManager.java b/src/main/java/com/percero/agents/sync/access/RedisAccessManager.java index 6cc03ba..1416826 100644 --- a/src/main/java/com/percero/agents/sync/access/RedisAccessManager.java +++ b/src/main/java/com/percero/agents/sync/access/RedisAccessManager.java @@ -1164,33 +1164,70 @@ public void updateWatcherFields(String category, String subCategory, String fiel public void removeChangeWatchersByObject(ClassIDPair classIdPair) { removeChangeWatchers(classIdPair.getClassName(), classIdPair.getID()); } + /** + * + * @param category + * @param subCategory + */ @SuppressWarnings("unchecked") public void removeChangeWatchers(String category, String subCategory) { String categoryKey = RedisKeyUtils.changeWatcherClass(category, subCategory); - // Get all change watcher values associated with this object. + // Get all change watcher values associated with this + // category/sub-category. + // For each stored change watcher property, a set of hash keys may exist + // that contain both a "RESULT" hash key and a "TIMESTAMP" hash key. The + // RESULT was calculated and stored at the TIMESTAMP. + // Example: + // "propertyName:r" + // "propertyName:t" Set changeWatcherValueKeys = cacheDataStore.getHashKeys(categoryKey); Iterator itrChangeWatcherValueKeys = changeWatcherValueKeys.iterator(); + while (itrChangeWatcherValueKeys.hasNext()) { String nextChangeWatcherValueKey = itrChangeWatcherValueKeys.next(); - // If this is a RESULT key, then add it to the list to check. + + // For each RESULT hash key, we need to remove the Change Watcher structure setup in the cache. if (nextChangeWatcherValueKey.endsWith(":" + RedisKeyUtils.RESULT)) { int resultIndex = nextChangeWatcherValueKey.lastIndexOf(":" + RedisKeyUtils.RESULT); + + // The changeWatcherKey for this property is of the form ":". + // Example: + // "cw:cf:com.namespace.mo.MyObject:ID:someProperty" String changeWatcherKey = categoryKey + ":" + nextChangeWatcherValueKey.substring(0, resultIndex); String key = changeWatcherKey; + + // The list of clients that are "watching" this change watcher + // property is in the cache by the key: + // client:". This set of clients should be + // notified when this value changes. + // Example: + // "client:cw:cf:com.namespace.mo.MyObject:ID:someProperty" String clientWatcherKey = RedisKeyUtils.clientWatcher(changeWatcherKey); - // Remove WatcherFields. - // For every watcherField, find the corresponding set and remove this changeWatcherKey - // from that set. + // For every watcherField, find the corresponding set and remove + // this changeWatcherKey from that set. The set of watchers + // fields is the set of properties that this ChangeWatcher is + // watching, meaning that if any of those properties change then + // this ChangeWatcher should be reprocessed. + // This takes the form: "cw:watcher:" + // Example: + // "cw:watcher:cw:cf:com.namespace.mo.MyObject:ID:someProperty" String watcherField = RedisKeyUtils.watcherField(changeWatcherKey); Set watcherFieldValues = (Set) cacheDataStore.getSetValue(watcherField); Iterator itrWatcherFieldValues = watcherFieldValues.iterator(); while (itrWatcherFieldValues.hasNext()) { String nextWatcherFieldValue = itrWatcherFieldValues.next(); + // This represents the reverse lookup: this is the set of + // ChangeWatchers to notify when nextWathcerFieldValue + // changes. So basically, we are saying "don't notify me" + // since this object is being removed. cacheDataStore.removeSetValue(nextWatcherFieldValue, changeWatcherKey); } + // This is the list of all properties that this ChangeWatcher is + // listening to. Since it is being deleted, we can simply delete + // this key from the cache as well. String fieldWatcher = RedisKeyUtils.fieldWatcher(changeWatcherKey); // Now remove all keys associated with this Change Watcher Value. From da71d0d168d1ae591825dfbc6b128fd535b72280 Mon Sep 17 00:00:00 2001 From: Collin Brown Date: Wed, 10 Feb 2016 14:34:00 -0800 Subject: [PATCH 092/105] Proposed fix for removing change watcher redis keys when removing an object --- .../java/com/percero/agents/sync/access/RedisAccessManager.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/percero/agents/sync/access/RedisAccessManager.java b/src/main/java/com/percero/agents/sync/access/RedisAccessManager.java index 1416826..67be19e 100644 --- a/src/main/java/com/percero/agents/sync/access/RedisAccessManager.java +++ b/src/main/java/com/percero/agents/sync/access/RedisAccessManager.java @@ -1194,7 +1194,7 @@ public void removeChangeWatchers(String category, String subCategory) { // The changeWatcherKey for this property is of the form ":". // Example: // "cw:cf:com.namespace.mo.MyObject:ID:someProperty" - String changeWatcherKey = categoryKey + ":" + nextChangeWatcherValueKey.substring(0, resultIndex); + String changeWatcherKey = RedisKeyUtils.changeWatcher(categoryKey, subCategory, nextChangeWatcherValueKey.substring(0, resultIndex)); String key = changeWatcherKey; // The list of clients that are "watching" this change watcher From 831a9f4f18bf657d6268ff6c80e29e634c215231 Mon Sep 17 00:00:00 2001 From: Collin Brown Date: Thu, 11 Feb 2016 15:17:25 -0800 Subject: [PATCH 093/105] Enable checkChangeWatchers to be threaded off so as to minimize the impact to clients on update/create/delete of objects --- .../agents/sync/access/AccessManager.java | 7 ++++ .../agents/sync/access/IAccessManager.java | 1 + .../sync/access/RedisAccessManager.java | 30 ++++++++++++-- .../access/RedisCheckChangeWatchersTask.java | 41 +++++++++++++++++++ .../agents/sync/helpers/PostDeleteHelper.java | 4 +- 5 files changed, 76 insertions(+), 7 deletions(-) create mode 100644 src/main/java/com/percero/agents/sync/access/RedisCheckChangeWatchersTask.java diff --git a/src/main/java/com/percero/agents/sync/access/AccessManager.java b/src/main/java/com/percero/agents/sync/access/AccessManager.java index bb4d880..7ebc7c9 100644 --- a/src/main/java/com/percero/agents/sync/access/AccessManager.java +++ b/src/main/java/com/percero/agents/sync/access/AccessManager.java @@ -1295,6 +1295,13 @@ public void updateWatcherFields(String category, String subCategory, } + @Override + public void checkAndRemoveChangeWatchers(ClassIDPair classIdPair, String[] fieldNames, String[] params, + IPerceroObject oldValue) { + // TODO Auto-generated method stub + + } + /* (non-Javadoc) * @see com.com.percero.agents.sync.services.IAccessManager#removeDeleteJournals(java.util.List) * diff --git a/src/main/java/com/percero/agents/sync/access/IAccessManager.java b/src/main/java/com/percero/agents/sync/access/IAccessManager.java index 9e8b1cd..506b6a0 100644 --- a/src/main/java/com/percero/agents/sync/access/IAccessManager.java +++ b/src/main/java/com/percero/agents/sync/access/IAccessManager.java @@ -216,6 +216,7 @@ public Set findClientByUserIdDeviceId(String deviceId, String userId) public Object getChangeWatcherResult(ClassIDPair classIdPair, String fieldName, String[] params); public void checkChangeWatchers(ClassIDPair classIdPair, String[] fieldNames, String[] params, IPerceroObject oldValue); public void removeChangeWatchersByObject(ClassIDPair classIdPair); + public void checkAndRemoveChangeWatchers(ClassIDPair classIdPair, String[] fieldNames, String[] params, IPerceroObject oldValue); public void recalculateChangeWatcher(String changeWatcherId, IPerceroObject oldValue); /** diff --git a/src/main/java/com/percero/agents/sync/access/RedisAccessManager.java b/src/main/java/com/percero/agents/sync/access/RedisAccessManager.java index 67be19e..26989b8 100644 --- a/src/main/java/com/percero/agents/sync/access/RedisAccessManager.java +++ b/src/main/java/com/percero/agents/sync/access/RedisAccessManager.java @@ -1164,7 +1164,10 @@ public void updateWatcherFields(String category, String subCategory, String fiel public void removeChangeWatchersByObject(ClassIDPair classIdPair) { removeChangeWatchers(classIdPair.getClassName(), classIdPair.getID()); } + /** + * Removes all Change Watcher data in the cache related to the + * `category`/`subcategory` * * @param category * @param subCategory @@ -1183,7 +1186,6 @@ public void removeChangeWatchers(String category, String subCategory) { // "propertyName:t" Set changeWatcherValueKeys = cacheDataStore.getHashKeys(categoryKey); Iterator itrChangeWatcherValueKeys = changeWatcherValueKeys.iterator(); - while (itrChangeWatcherValueKeys.hasNext()) { String nextChangeWatcherValueKey = itrChangeWatcherValueKeys.next(); @@ -1191,12 +1193,13 @@ public void removeChangeWatchers(String category, String subCategory) { if (nextChangeWatcherValueKey.endsWith(":" + RedisKeyUtils.RESULT)) { int resultIndex = nextChangeWatcherValueKey.lastIndexOf(":" + RedisKeyUtils.RESULT); - // The changeWatcherKey for this property is of the form ":". + // The changeWatcherKey for this property is of the form + // ":". // Example: // "cw:cf:com.namespace.mo.MyObject:ID:someProperty" - String changeWatcherKey = RedisKeyUtils.changeWatcher(categoryKey, subCategory, nextChangeWatcherValueKey.substring(0, resultIndex)); + String changeWatcherKey = categoryKey + ":" + nextChangeWatcherValueKey.substring(0, resultIndex); String key = changeWatcherKey; - + // The list of clients that are "watching" this change watcher // property is in the cache by the key: // client:". This set of clients should be @@ -1361,6 +1364,25 @@ public void checkChangeWatchers(ClassIDPair classIdPair) { } public void checkChangeWatchers(ClassIDPair classIdPair, String[] fieldNames, String[] params, IPerceroObject oldValue) { + if (useChangeWatcherQueue && taskExecutor != null) { + taskExecutor.execute(new RedisCheckChangeWatchersTask(this, classIdPair, fieldNames, params, false, oldValue)); + } + else { + internalCheckChangeWatchers(classIdPair, fieldNames, params, oldValue); + } + } + + public void checkAndRemoveChangeWatchers(ClassIDPair classIdPair, String[] fieldNames, String[] params, IPerceroObject oldValue) { + if (useChangeWatcherQueue && taskExecutor != null) { + taskExecutor.execute(new RedisCheckChangeWatchersTask(this, classIdPair, fieldNames, params, true, oldValue)); + } + else { + internalCheckChangeWatchers(classIdPair, fieldNames, params, oldValue); + removeChangeWatchersByObject(classIdPair); + } + } + + public void internalCheckChangeWatchers(ClassIDPair classIdPair, String[] fieldNames, String[] params, IPerceroObject oldValue) { Collection changeWatchers = getChangeWatchersForField(classIdPair, fieldNames, params); // If there are ChangeWatchers, then recalculate for each one. diff --git a/src/main/java/com/percero/agents/sync/access/RedisCheckChangeWatchersTask.java b/src/main/java/com/percero/agents/sync/access/RedisCheckChangeWatchersTask.java new file mode 100644 index 0000000..3a91a56 --- /dev/null +++ b/src/main/java/com/percero/agents/sync/access/RedisCheckChangeWatchersTask.java @@ -0,0 +1,41 @@ +package com.percero.agents.sync.access; + +import org.apache.log4j.Logger; + +import com.percero.agents.sync.vo.ClassIDPair; +import com.percero.framework.vo.IPerceroObject; + +public class RedisCheckChangeWatchersTask implements Runnable { + + private static final Logger log = Logger.getLogger(RedisCheckChangeWatchersTask.class); + + public RedisCheckChangeWatchersTask(RedisAccessManager redisAccessManager, + ClassIDPair classIdPair, String[] fieldNames, String[] params, boolean removeObjectChangeWatchers, IPerceroObject oldValue) { + this.redisAccessManager = redisAccessManager; + this.classIdPair = classIdPair; + this.fieldNames = fieldNames; + this.params = params; + this.removeObjectChangeWatchers = removeObjectChangeWatchers; + this.oldValue = oldValue; + } + + private RedisAccessManager redisAccessManager; + private ClassIDPair classIdPair; + private String[] fieldNames; + private String[] params; + private boolean removeObjectChangeWatchers = false; + private IPerceroObject oldValue; + + public void run() { + try { + redisAccessManager.internalCheckChangeWatchers(classIdPair, fieldNames, params, oldValue); + + if (removeObjectChangeWatchers) { + redisAccessManager.removeChangeWatchersByObject(classIdPair); + } + + } catch(Exception e) { + log.error("Error running process", e); + } + } +} diff --git a/src/main/java/com/percero/agents/sync/helpers/PostDeleteHelper.java b/src/main/java/com/percero/agents/sync/helpers/PostDeleteHelper.java index 31071c4..6915fcc 100644 --- a/src/main/java/com/percero/agents/sync/helpers/PostDeleteHelper.java +++ b/src/main/java/com/percero/agents/sync/helpers/PostDeleteHelper.java @@ -84,9 +84,7 @@ public void postDeleteObject(ClassIDPair pair, String pusherUserId, String pushe accessManager.removeHistoricalObjectsByObject(pair); // Now run past the ChangeWatcher. - accessManager.checkChangeWatchers(pair, null, null, null); - // Remove ChangeWatchers associated with this object. - accessManager.removeChangeWatchersByObject(pair); + accessManager.checkAndRemoveChangeWatchers(pair, null, null, null); /*Collection deleteJournals = */accessManager.saveDeleteJournalClients(pair, clientIds, guaranteeUpdateDelivery, pusherClientId, pushToUser); pushObjectDeleteJournals(clientIds, pair.getClassName(), pair.getID()); From f8cd675ae14e40ae1906a977987bc2e188f259d5 Mon Sep 17 00:00:00 2001 From: Collin Brown Date: Fri, 12 Feb 2016 09:05:39 -0800 Subject: [PATCH 094/105] Default rabbitHost to "localhost" --- src/main/java/com/percero/amqp/RabbitMQPushSyncHelper.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/percero/amqp/RabbitMQPushSyncHelper.java b/src/main/java/com/percero/amqp/RabbitMQPushSyncHelper.java index 6b56faa..e46394f 100644 --- a/src/main/java/com/percero/amqp/RabbitMQPushSyncHelper.java +++ b/src/main/java/com/percero/amqp/RabbitMQPushSyncHelper.java @@ -103,7 +103,7 @@ public void setRabbitMessageListenerContainer(AbstractMessageListenerContainer c String rabbitLogin = "guest"; @Autowired @Value("$pf{gateway.rabbitmq.password:guest}") String rabbitPassword = "guest"; - @Autowired @Value("$pf{gateway.rabbitmq.host:}") + @Autowired @Value("$pf{gateway.rabbitmq.host:localhost}") String rabbitHost = null; @Autowired @Value("$pf{gateway.rabbitmq.queue_timeout:43200000}") // 8 Hours long rabbitQueueTimeout = 43200000; From 1a1af6199b138489a1a954ae7e7422df66613ca5 Mon Sep 17 00:00:00 2001 From: Collin Brown Date: Fri, 12 Feb 2016 09:17:48 -0800 Subject: [PATCH 095/105] Restore EOL message to previous queue When removing an old queue, make sure the EOL message is maintained in that queue rather than moved over to the new one. --- .../percero/amqp/RabbitMQPushSyncHelper.java | 20 ++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/percero/amqp/RabbitMQPushSyncHelper.java b/src/main/java/com/percero/amqp/RabbitMQPushSyncHelper.java index e46394f..2d6a306 100644 --- a/src/main/java/com/percero/amqp/RabbitMQPushSyncHelper.java +++ b/src/main/java/com/percero/amqp/RabbitMQPushSyncHelper.java @@ -21,6 +21,7 @@ import org.apache.http.client.methods.HttpGet; import org.apache.http.impl.client.DefaultHttpClient; import org.apache.log4j.Logger; +import org.codehaus.jackson.JsonNode; import org.codehaus.jackson.map.ObjectMapper; import org.joda.time.DateTime; import org.joda.time.format.DateTimeFormat; @@ -334,8 +335,25 @@ else if (clientId.equalsIgnoreCase(thePreviousClientId)) { // Attempt to move messages from the previous client queue to the new one. try { Message nextExistingMessage = null; + + // If we find an EOL Message, then we want to make sure it stays in + // the previous queue. + Message eolMessage = null; while ((nextExistingMessage = template.receive(thePreviousClientId)) != null) { - template.send(clientId, nextExistingMessage); + JsonNode messageJsonNode = objectMapper.readTree(nextExistingMessage.getBody()); + if (messageJsonNode.has("EOL")) { + // We found an EOL message, keep hold of it so we can resent + // to the previous queue. + eolMessage = nextExistingMessage; + } + else { + template.send(clientId, nextExistingMessage); + } + } + + if (eolMessage != null) { + // Make sure the EOL message is left intact in the old queue. + template.send(thePreviousClientId, eolMessage); } } catch(AmqpIOException e) { // Most likely due to queue already being deleted. From bfdec01388447ebd70936bac780eb0a9e78d529a Mon Sep 17 00:00:00 2001 From: Nikish Parikh Date: Fri, 12 Feb 2016 23:32:07 +0530 Subject: [PATCH 096/105] Syncengine version change --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 8cb767e..e307456 100644 --- a/pom.xml +++ b/pom.xml @@ -7,7 +7,7 @@ org.activestack syncengine - 1.1.59-SNAPSHOT + 1.1.60-SNAPSHOT 3.2.4.RELEASE From 68cefb4b031fd0529f22798e3c7920cdb90a5b8d Mon Sep 17 00:00:00 2001 From: Nikish Parikh Date: Thu, 18 Feb 2016 21:39:23 +0530 Subject: [PATCH 097/105] Commenting log which is not necessary --- .../com/percero/agents/sync/jobs/UpdateTableProcessor.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/percero/agents/sync/jobs/UpdateTableProcessor.java b/src/main/java/com/percero/agents/sync/jobs/UpdateTableProcessor.java index 52111a0..c7a0ddd 100644 --- a/src/main/java/com/percero/agents/sync/jobs/UpdateTableProcessor.java +++ b/src/main/java/com/percero/agents/sync/jobs/UpdateTableProcessor.java @@ -800,7 +800,8 @@ protected void deleteRows(List rows){ int numUpdated = statement.executeUpdate(sql); if(numUpdated != rows.size()){ - logger.warn("Expected to delete "+rows.size()+", instead "+numUpdated); + //As discussed with Richard, commenting the log +// logger.warn("Expected to delete "+rows.size()+", instead "+numUpdated); } }catch(SQLException e){ logger.warn(e.getMessage(), e); @@ -817,7 +818,8 @@ protected void releaseRowLock(UpdateTableRow row){ int numUpdated = statement.executeUpdate(sql); if(numUpdated != 1){ - logger.warn("Expected to update 1, instead "+numUpdated); + //As discussed with Richard, commenting the log +// logger.warn("Expected to update 1, instead "+numUpdated); } }catch(SQLException e){ logger.warn(e.getMessage(), e); From 492eb01b065fb900afdde8368bb82dba6fba38e0 Mon Sep 17 00:00:00 2001 From: Nikish Parikh Date: Thu, 18 Feb 2016 21:46:08 +0530 Subject: [PATCH 098/105] Change version no --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index e307456..4f1a806 100644 --- a/pom.xml +++ b/pom.xml @@ -7,7 +7,7 @@ org.activestack syncengine - 1.1.60-SNAPSHOT + 1.1.61-SNAPSHOT 3.2.4.RELEASE From 78ac7fe7deacf3cace0f061234d09fb605c9b1a5 Mon Sep 17 00:00:00 2001 From: Jay Eno Date: Thu, 18 Feb 2016 12:42:22 -0700 Subject: [PATCH 099/105] Updating to tell Hikari to register it's MBeans for monitoring. --- src/main/java/com/percero/datasource/BaseConnectionFactory.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/java/com/percero/datasource/BaseConnectionFactory.java b/src/main/java/com/percero/datasource/BaseConnectionFactory.java index 440725e..e16664b 100644 --- a/src/main/java/com/percero/datasource/BaseConnectionFactory.java +++ b/src/main/java/com/percero/datasource/BaseConnectionFactory.java @@ -169,6 +169,7 @@ public void init() throws PropertyVetoException{ // Default to Hikari Connection Pool. HikariConfig config = new HikariConfig(); config.setDriverClassName(driverClassName); + config.setRegisterMbeans(true); config.setJdbcUrl(jdbcUrl); config.setUsername(username); config.setPassword(password); From ab6f0bde3baf789104b6e0a2c9df8d681cc8d409 Mon Sep 17 00:00:00 2001 From: Jay Eno Date: Thu, 18 Feb 2016 13:00:07 -0700 Subject: [PATCH 100/105] Incrementing Version --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 4f1a806..25b5758 100644 --- a/pom.xml +++ b/pom.xml @@ -7,7 +7,7 @@ org.activestack syncengine - 1.1.61-SNAPSHOT + 1.1.62-SNAPSHOT 3.2.4.RELEASE From f304a500b5b44ea66414be6be7037debd68cdbe2 Mon Sep 17 00:00:00 2001 From: Nikish Parikh Date: Mon, 29 Feb 2016 17:51:16 +0530 Subject: [PATCH 101/105] Update Table process log improved to debug null pointer exception --- .../agents/sync/jobs/UpdateTableProcessor.java | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/src/main/java/com/percero/agents/sync/jobs/UpdateTableProcessor.java b/src/main/java/com/percero/agents/sync/jobs/UpdateTableProcessor.java index c7a0ddd..866e636 100644 --- a/src/main/java/com/percero/agents/sync/jobs/UpdateTableProcessor.java +++ b/src/main/java/com/percero/agents/sync/jobs/UpdateTableProcessor.java @@ -564,6 +564,7 @@ protected void updateReferences(String className){ // Find all of this type and push down an update to all Set ids = accessManager.getClassAccessJournalIDs(mappedField.getMappedClass().className); + logger.warn("updateReferences - ids IS NULL : " + (ids == null) ); if (ids.contains("0")) { // If there is a 0 ID in the list, then we need to update ALL records of this type. Integer pageNumber = 0; @@ -572,6 +573,7 @@ protected void updateReferences(String className){ while (total < 0 || pageNumber * pageSize <= total) { PerceroList objectsToUpdate = mappedField.getMappedClass().getDataProvider().getAllByName(mappedField.getMappedClass().className, pageNumber, pageSize, true, null); + logger.warn("updateReferences - objectsToUpdate IS NULL : " + (objectsToUpdate == null) ); pageNumber++; total = objectsToUpdate.getTotalLength(); if (total <= 0) { @@ -581,13 +583,16 @@ protected void updateReferences(String className){ Iterator itrObjectsToUpdate = objectsToUpdate.iterator(); while (itrObjectsToUpdate.hasNext()) { IDataProvider dataProvider = dataProviderManager.getDefaultDataProvider(); + logger.warn("updateReferences - dataProvider IS NULL : " + (dataProvider == null) ); // We assume this is from the cache because some client has requested all of this class // and since we turned off cache expiration it must be in there. BaseDataObject nextObjectToUpdate = (BaseDataObject) itrObjectsToUpdate.next(); + logger.warn("updateReferences - BaseDataObject IS NULL : " + (nextObjectToUpdate == null) ); ClassIDPair pair = BaseDataObject.toClassIdPair(nextObjectToUpdate); + logger.warn("updateReferences - BaseDataObject pair IS NULL : " + (pair == null) ); // This call ignores the cache BaseDataObject nextObjectToUpdateFromDB = (BaseDataObject) dataProvider.findById(pair, null, true); - + logger.warn("updateReferences - nextObjectToUpdateFromDB IS NULL : " + (nextObjectToUpdateFromDB == null) ); Map> changedFields = dataProvider .getChangedMappedFields(nextObjectToUpdateFromDB, nextObjectToUpdate, nextObjectToUpdate != null); @@ -603,14 +608,17 @@ protected void updateReferences(String className){ String nextIdToUpdate = itrIdsToUpdate.next(); ClassIDPair pair = new ClassIDPair(nextIdToUpdate, mappedField.getMappedClass().className); - IDataProvider dataProvider = dataProviderManager.getDefaultDataProvider(); + logger.warn("updateReferences - ELSE pair IS NULL : " + (pair == null) ); + IDataProvider dataProvider = dataProviderManager.getDefaultDataProvider(); + logger.warn("updateReferences - ELSE dataProvider IS NULL : " + (dataProvider == null) ); // First call assumes we are getting it from the cache... because we can assume that // All objects that we care about are all cached (since we turned off cache expiration). BaseDataObject cachedObject = (BaseDataObject) dataProvider.findById(pair, null); + logger.warn("updateReferences - ELSE cachedObject IS NULL : " + (cachedObject == null) ); // This call ignores the cache BaseDataObject objectFromDB = (BaseDataObject) dataProvider.findById(pair, null, true); - + logger.warn("updateReferences - ELSE BaseDataObject objectFromDB IS NULL : " + (objectFromDB == null) ); Map> changedFields = dataProvider .getChangedMappedFields(objectFromDB, cachedObject, cachedObject != null); // Only push if different From 7a7e3eaee084a34c46934339abf3678bae024eaa Mon Sep 17 00:00:00 2001 From: Nikish Parikh Date: Mon, 29 Feb 2016 17:57:53 +0530 Subject: [PATCH 102/105] change of version --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 25b5758..f608724 100644 --- a/pom.xml +++ b/pom.xml @@ -7,7 +7,7 @@ org.activestack syncengine - 1.1.62-SNAPSHOT + 1.1.63-SNAPSHOT 3.2.4.RELEASE From da633ac5cd79c15782ff2ee0cce937236bc671e8 Mon Sep 17 00:00:00 2001 From: Nikish Parikh Date: Wed, 9 Mar 2016 12:29:58 +0530 Subject: [PATCH 103/105] move the warn log to debug and error respectively --- .../sync/jobs/UpdateTableProcessor.java | 54 +++++++++---------- 1 file changed, 27 insertions(+), 27 deletions(-) diff --git a/src/main/java/com/percero/agents/sync/jobs/UpdateTableProcessor.java b/src/main/java/com/percero/agents/sync/jobs/UpdateTableProcessor.java index 866e636..a9bfee0 100644 --- a/src/main/java/com/percero/agents/sync/jobs/UpdateTableProcessor.java +++ b/src/main/java/com/percero/agents/sync/jobs/UpdateTableProcessor.java @@ -133,7 +133,7 @@ public void run(){ result.addResult(row.getType().toString()); successfulRows.add(row); } catch (Exception e) { - logger.warn("Failed to process update: " + e.getMessage(), e); + logger.error("Failed to process update: " + e.getMessage(), e); result.addResult(row.getType().toString(), false, e.getMessage()); releaseRowLock(row); } @@ -150,8 +150,8 @@ public void run(){ } if (!result.isSuccess()) { - logger.warn("Update table processor (" + tableName + ") failed. Details:"); - logger.warn(result); + logger.debug("Update table processor (" + tableName + ") failed. Details:"); + logger.debug(result); } }finally{ try { @@ -207,7 +207,7 @@ protected void processUpdateSingle(UpdateTableRow row) throws Exception{ List classes = getClassesForTableName(row.getTableName()); if (classes == null || classes.isEmpty()) { - logger.warn("No Classes defined for UpdateTable table " + row.getTableName()); + logger.debug("No Classes defined for UpdateTable table " + row.getTableName()); } else { logger.debug("Processing " + classes.size() + " class(es) for UpdateTable UPDATE, table " + row.getTableName()); @@ -237,7 +237,7 @@ protected void processUpdateTable(UpdateTableRow row) throws Exception{ List classes = getClassesForTableName(row.getTableName()); if (classes == null || classes.isEmpty()) { - logger.warn("No Classes defined for UpdateTable table " + row.getTableName()); + logger.debug("No Classes defined for UpdateTable table " + row.getTableName()); } else { logger.debug("Processing " + classes.size() + " class(es) for UpdateTable UPDATE, table " + row.getTableName()); @@ -373,7 +373,7 @@ protected void processInsertSingle(UpdateTableRow row) throws Exception{ List classes = getClassesForTableName(row.getTableName()); if (classes == null || classes.isEmpty()) { - logger.warn("No Classes defined for UpdateTable table " + row.getTableName()); + logger.debug("No Classes defined for UpdateTable table " + row.getTableName()); } else { logger.debug("Processing " + classes.size() + " class(es) for UpdateTable INSERT, table " + row.getTableName()); @@ -399,7 +399,7 @@ protected void processInsertTable(UpdateTableRow row) throws Exception { List classes = getClassesForTableName(row.getTableName()); if (classes == null || classes.isEmpty()) { - logger.warn("No Classes defined for UpdateTable table " + row.getTableName()); + logger.debug("No Classes defined for UpdateTable table " + row.getTableName()); } else { logger.debug("Processing " + classes.size() + " class(es) for UpdateTable INSERT, table " + row.getTableName()); @@ -432,7 +432,7 @@ protected void processDeleteSingle(UpdateTableRow row) throws Exception{ List classes = getClassesForTableName(row.getTableName()); if (classes == null || classes.isEmpty()) { - logger.warn("No Classes defined for UpdateTable table " + row.getTableName()); + logger.debug("No Classes defined for UpdateTable table " + row.getTableName()); } else { logger.debug("Processing " + classes.size() + " class(es) for UpdateTable DELETE, table " + row.getTableName()); @@ -463,7 +463,7 @@ protected void processDeleteTable(UpdateTableRow row) throws Exception{ List classes = getClassesForTableName(row.getTableName()); if (classes == null || classes.isEmpty()) { - logger.warn("No Classes defined for UpdateTable table " + row.getTableName()); + logger.debug("No Classes defined for UpdateTable table " + row.getTableName()); } else { logger.debug("Processing " + classes.size() + " class(es) for UpdateTable DELETE, table " + row.getTableName()); @@ -564,7 +564,7 @@ protected void updateReferences(String className){ // Find all of this type and push down an update to all Set ids = accessManager.getClassAccessJournalIDs(mappedField.getMappedClass().className); - logger.warn("updateReferences - ids IS NULL : " + (ids == null) ); + logger.debug("updateReferences - ids IS NULL : " + (ids == null) ); if (ids.contains("0")) { // If there is a 0 ID in the list, then we need to update ALL records of this type. Integer pageNumber = 0; @@ -573,7 +573,7 @@ protected void updateReferences(String className){ while (total < 0 || pageNumber * pageSize <= total) { PerceroList objectsToUpdate = mappedField.getMappedClass().getDataProvider().getAllByName(mappedField.getMappedClass().className, pageNumber, pageSize, true, null); - logger.warn("updateReferences - objectsToUpdate IS NULL : " + (objectsToUpdate == null) ); + logger.debug("updateReferences - objectsToUpdate IS NULL : " + (objectsToUpdate == null) ); pageNumber++; total = objectsToUpdate.getTotalLength(); if (total <= 0) { @@ -583,16 +583,16 @@ protected void updateReferences(String className){ Iterator itrObjectsToUpdate = objectsToUpdate.iterator(); while (itrObjectsToUpdate.hasNext()) { IDataProvider dataProvider = dataProviderManager.getDefaultDataProvider(); - logger.warn("updateReferences - dataProvider IS NULL : " + (dataProvider == null) ); + logger.debug("updateReferences - dataProvider IS NULL : " + (dataProvider == null) ); // We assume this is from the cache because some client has requested all of this class // and since we turned off cache expiration it must be in there. BaseDataObject nextObjectToUpdate = (BaseDataObject) itrObjectsToUpdate.next(); - logger.warn("updateReferences - BaseDataObject IS NULL : " + (nextObjectToUpdate == null) ); + logger.debug("updateReferences - BaseDataObject IS NULL : " + (nextObjectToUpdate == null) ); ClassIDPair pair = BaseDataObject.toClassIdPair(nextObjectToUpdate); - logger.warn("updateReferences - BaseDataObject pair IS NULL : " + (pair == null) ); + logger.debug("updateReferences - BaseDataObject pair IS NULL : " + (pair == null) ); // This call ignores the cache BaseDataObject nextObjectToUpdateFromDB = (BaseDataObject) dataProvider.findById(pair, null, true); - logger.warn("updateReferences - nextObjectToUpdateFromDB IS NULL : " + (nextObjectToUpdateFromDB == null) ); + logger.debug("updateReferences - nextObjectToUpdateFromDB IS NULL : " + (nextObjectToUpdateFromDB == null) ); Map> changedFields = dataProvider .getChangedMappedFields(nextObjectToUpdateFromDB, nextObjectToUpdate, nextObjectToUpdate != null); @@ -608,17 +608,17 @@ protected void updateReferences(String className){ String nextIdToUpdate = itrIdsToUpdate.next(); ClassIDPair pair = new ClassIDPair(nextIdToUpdate, mappedField.getMappedClass().className); - logger.warn("updateReferences - ELSE pair IS NULL : " + (pair == null) ); + logger.debug("updateReferences - ELSE pair IS NULL : " + (pair == null) ); IDataProvider dataProvider = dataProviderManager.getDefaultDataProvider(); - logger.warn("updateReferences - ELSE dataProvider IS NULL : " + (dataProvider == null) ); + logger.debug("updateReferences - ELSE dataProvider IS NULL : " + (dataProvider == null) ); // First call assumes we are getting it from the cache... because we can assume that // All objects that we care about are all cached (since we turned off cache expiration). BaseDataObject cachedObject = (BaseDataObject) dataProvider.findById(pair, null); - logger.warn("updateReferences - ELSE cachedObject IS NULL : " + (cachedObject == null) ); + logger.debug("updateReferences - ELSE cachedObject IS NULL : " + (cachedObject == null) ); // This call ignores the cache BaseDataObject objectFromDB = (BaseDataObject) dataProvider.findById(pair, null, true); - logger.warn("updateReferences - ELSE BaseDataObject objectFromDB IS NULL : " + (objectFromDB == null) ); + logger.debug("updateReferences - ELSE BaseDataObject objectFromDB IS NULL : " + (objectFromDB == null) ); Map> changedFields = dataProvider .getChangedMappedFields(objectFromDB, cachedObject, cachedObject != null); // Only push if different @@ -723,7 +723,7 @@ private List getUpdateSelectRow(int lockId, DateTime expireThres } if(count != numUpdated) - logger.warn("Locked a "+numUpdated+" rows but found "+count); + logger.debug("Locked a "+numUpdated+" rows but found "+count); } } } @@ -749,7 +749,7 @@ private List getStoredProcRow(int lockId, DateTime expireThresho updateNum = cstmt.getInt(4); } catch(SQLException e){ // return null; - logger.warn(e.getMessage(), e); + logger.error(e.getMessage(), e); // If the stored proc doesn't exist, let's try and create it. if (StringUtils.hasText(connectionFactory.getStoredProcedureDefinition()) && @@ -762,11 +762,11 @@ private List getStoredProcRow(int lockId, DateTime expireThresho System.out.println(createResult); } catch(SQLSyntaxErrorException ssee) { - logger.warn("Unable to create UpdateTable stored procedure: " + ssee.getMessage()); + logger.error("Unable to create UpdateTable stored procedure: " + ssee.getMessage()); throw ssee; } catch(Exception e1) { - logger.warn("Unable to create UpdateTable stored procedure: " + e1.getMessage()); + logger.error("Unable to create UpdateTable stored procedure: " + e1.getMessage()); throw e1; } } @@ -787,7 +787,7 @@ private List getStoredProcRow(int lockId, DateTime expireThresho } if(count != updateNum) - logger.warn("Locked a "+updateNum+" rows but found "+count); + logger.error("Locked a "+updateNum+" rows but found "+count); } } @@ -812,7 +812,7 @@ protected void deleteRows(List rows){ // logger.warn("Expected to delete "+rows.size()+", instead "+numUpdated); } }catch(SQLException e){ - logger.warn(e.getMessage(), e); + logger.error(e.getMessage(), e); } } @@ -830,7 +830,7 @@ protected void releaseRowLock(UpdateTableRow row){ // logger.warn("Expected to update 1, instead "+numUpdated); } }catch(SQLException e){ - logger.warn(e.getMessage(), e); + logger.error(e.getMessage(), e); } } @@ -873,7 +873,7 @@ protected UpdateTableRow fromResultSet(ResultSet resultSet) throws SQLException{ try { row.type = UpdateTableRowType.valueOf(resultSet.getString("type")); } catch(IllegalArgumentException iae) { - logger.warn("Invalid UpdateTableRow TYPE, ignoring"); + logger.error("Invalid UpdateTableRow TYPE, ignoring"); row.type = UpdateTableRowType.NONE; } row.timestamp = resultSet.getDate(connectionFactory.getTimestampColumnName()); From 5b66e5c1c9940821e8fe61276a233d36141f0961 Mon Sep 17 00:00:00 2001 From: Nikish Parikh Date: Wed, 9 Mar 2016 16:53:25 +0530 Subject: [PATCH 104/105] Change the version num of synceng --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index f608724..6b59e1a 100644 --- a/pom.xml +++ b/pom.xml @@ -7,7 +7,7 @@ org.activestack syncengine - 1.1.63-SNAPSHOT + 1.1.64-SNAPSHOT 3.2.4.RELEASE From 156ea8927fba9681f8b547662904073a45f990b1 Mon Sep 17 00:00:00 2001 From: Nikish Parikh Date: Tue, 22 Mar 2016 11:55:15 +0530 Subject: [PATCH 105/105] Error log converted to debug --- pom.xml | 2 +- .../java/com/percero/agents/sync/jobs/UpdateTableProcessor.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index 6b59e1a..b588eef 100644 --- a/pom.xml +++ b/pom.xml @@ -7,7 +7,7 @@ org.activestack syncengine - 1.1.64-SNAPSHOT + 1.1.65-SNAPSHOT 3.2.4.RELEASE diff --git a/src/main/java/com/percero/agents/sync/jobs/UpdateTableProcessor.java b/src/main/java/com/percero/agents/sync/jobs/UpdateTableProcessor.java index a9bfee0..b5f01f8 100644 --- a/src/main/java/com/percero/agents/sync/jobs/UpdateTableProcessor.java +++ b/src/main/java/com/percero/agents/sync/jobs/UpdateTableProcessor.java @@ -787,7 +787,7 @@ private List getStoredProcRow(int lockId, DateTime expireThresho } if(count != updateNum) - logger.error("Locked a "+updateNum+" rows but found "+count); + logger.debug("Locked a "+updateNum+" rows but found "+count); } }