From 84bd79bd7e91bd549b11a8e59ec40e2bf0765d30 Mon Sep 17 00:00:00 2001 From: shub6691 Date: Tue, 29 Jun 2021 17:49:26 +0530 Subject: [PATCH 01/21] Added the new adpaters for dynamodb for search query and time stamp filteration --- adapters/dynamoDB_adapters/pom.xml | 90 +++++ .../adapter/DynamoDBFeedInformation.java | 21 ++ .../dynamodb/adapter/DynamoDBFeedSource.java | 216 ++++++++++++ .../adapter/DynamoFeedDBPublisher.java | 218 ++++++++++++ .../dynamodb/constant/DynamoDBConstant.java | 11 + .../dynamodb/model/PersistedEntry.java | 21 ++ .../query/CategoryStringGenerator.java | 160 +++++++++ .../dynamodb/query/DynamoDBQueryBuilder.java | 76 +++++ .../dynamodb/query/DynamoDBTextArray.java | 198 +++++++++++ .../dynamodb/query/SQLToNoSqlConverter.java | 311 ++++++++++++++++++ .../atomhopper/dynamodb/query/SearchType.java | 11 + .../src/main/resources/application.properties | 2 + .../src/main/resources/logback.xml | 0 .../test/java/DynamoDBFeedPublisherTest.java | 124 +++++++ .../src/test/java/DynamoDBFeedSourceTest.java | 172 ++++++++++ pom.xml | 1 + 16 files changed, 1632 insertions(+) create mode 100644 adapters/dynamoDB_adapters/pom.xml create mode 100644 adapters/dynamoDB_adapters/src/main/java/org/atomhopper/dynamodb/adapter/DynamoDBFeedInformation.java create mode 100644 adapters/dynamoDB_adapters/src/main/java/org/atomhopper/dynamodb/adapter/DynamoDBFeedSource.java create mode 100644 adapters/dynamoDB_adapters/src/main/java/org/atomhopper/dynamodb/adapter/DynamoFeedDBPublisher.java create mode 100644 adapters/dynamoDB_adapters/src/main/java/org/atomhopper/dynamodb/constant/DynamoDBConstant.java create mode 100644 adapters/dynamoDB_adapters/src/main/java/org/atomhopper/dynamodb/model/PersistedEntry.java create mode 100644 adapters/dynamoDB_adapters/src/main/java/org/atomhopper/dynamodb/query/CategoryStringGenerator.java create mode 100644 adapters/dynamoDB_adapters/src/main/java/org/atomhopper/dynamodb/query/DynamoDBQueryBuilder.java create mode 100644 adapters/dynamoDB_adapters/src/main/java/org/atomhopper/dynamodb/query/DynamoDBTextArray.java create mode 100644 adapters/dynamoDB_adapters/src/main/java/org/atomhopper/dynamodb/query/SQLToNoSqlConverter.java create mode 100644 adapters/dynamoDB_adapters/src/main/java/org/atomhopper/dynamodb/query/SearchType.java create mode 100644 adapters/dynamoDB_adapters/src/main/resources/application.properties create mode 100644 adapters/dynamoDB_adapters/src/main/resources/logback.xml create mode 100644 adapters/dynamoDB_adapters/src/test/java/DynamoDBFeedPublisherTest.java create mode 100644 adapters/dynamoDB_adapters/src/test/java/DynamoDBFeedSourceTest.java diff --git a/adapters/dynamoDB_adapters/pom.xml b/adapters/dynamoDB_adapters/pom.xml new file mode 100644 index 00000000..c0e21f5f --- /dev/null +++ b/adapters/dynamoDB_adapters/pom.xml @@ -0,0 +1,90 @@ + + + + parent + org.atomhopper + 1.2.35-SNAPSHOT + ../../pom.xml + + 4.0.0 + + + org.atomhopper.adapter + dynamoDB_adapters + jar + + + + org.apache.maven.plugins + maven-compiler-plugin + + 8 + 8 + + + + + + + + com.github.derjust + spring-data-dynamodb + 5.1.0 + + + + com.unboundid + unboundid-ldapsdk + + + + com.amazonaws + aws-java-sdk-dynamodb + 1.11.34 + + + + org.projectlombok + lombok + 1.18.20 + provided + + + + org.apache.abdera + abdera-core + + + + org.atomhopper + core + + + + com.yammer.metrics + metrics-core + + + junit + junit + + + org.mockito + mockito-all + + + org.atomhopper.adapter + jdbc-adapter + test + + + + + + 8 + 8 + + + \ No newline at end of file diff --git a/adapters/dynamoDB_adapters/src/main/java/org/atomhopper/dynamodb/adapter/DynamoDBFeedInformation.java b/adapters/dynamoDB_adapters/src/main/java/org/atomhopper/dynamodb/adapter/DynamoDBFeedInformation.java new file mode 100644 index 00000000..06defb4b --- /dev/null +++ b/adapters/dynamoDB_adapters/src/main/java/org/atomhopper/dynamodb/adapter/DynamoDBFeedInformation.java @@ -0,0 +1,21 @@ +package org.atomhopper.dynamodb.adapter; + +import org.apache.abdera.model.Categories; +import org.atomhopper.adapter.FeedInformation; +import org.atomhopper.adapter.NotImplemented; +import org.atomhopper.adapter.request.adapter.GetCategoriesRequest; +import org.atomhopper.adapter.request.feed.FeedRequest; + +public class DynamoDBFeedInformation implements FeedInformation { + + @Override + public String getId(FeedRequest feedRequest) { + throw new UnsupportedOperationException("Not supported yet."); + } + + @Override + @NotImplemented + public Categories getCategories(GetCategoriesRequest getCategoriesRequest) { + throw new UnsupportedOperationException("Not supported yet."); + } +} diff --git a/adapters/dynamoDB_adapters/src/main/java/org/atomhopper/dynamodb/adapter/DynamoDBFeedSource.java b/adapters/dynamoDB_adapters/src/main/java/org/atomhopper/dynamodb/adapter/DynamoDBFeedSource.java new file mode 100644 index 00000000..8743a802 --- /dev/null +++ b/adapters/dynamoDB_adapters/src/main/java/org/atomhopper/dynamodb/adapter/DynamoDBFeedSource.java @@ -0,0 +1,216 @@ +package org.atomhopper.dynamodb.adapter; + +import com.amazonaws.services.dynamodbv2.AmazonDynamoDBClient; +import com.amazonaws.services.dynamodbv2.datamodeling.DynamoDBMapper; +import com.amazonaws.services.dynamodbv2.datamodeling.DynamoDBQueryExpression; +import com.amazonaws.services.dynamodbv2.document.*; +import com.amazonaws.services.dynamodbv2.document.spec.QuerySpec; +import com.amazonaws.services.dynamodbv2.document.utils.ValueMap; +import com.amazonaws.services.dynamodbv2.model.AttributeValue; +import org.apache.commons.lang.StringUtils; +import org.atomhopper.adapter.*; +import org.atomhopper.adapter.request.adapter.GetFeedRequest; +import org.atomhopper.dbal.PageDirection; +import org.atomhopper.dynamodb.model.PersistedEntry; +import org.atomhopper.dynamodb.query.SQLToNoSqlConverter; +import org.atomhopper.dynamodb.query.SearchType; +import org.atomhopper.dynamodb.query.DynamoDBQueryBuilder; +import org.joda.time.DateTime; +import org.joda.time.format.DateTimeFormatter; +import org.joda.time.format.ISODateTimeFormat; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.net.URL; +import java.text.SimpleDateFormat; +import java.util.*; + +/** + * @author shub6691 + * Implements the DynamoDBFeedSource interface for retrieving feed entries from a datastore. This class implements + * the following: + * + * + *

+ * Mapping category prefixes to postgres columns is done through the following: + *

+ */ +public class DynamoDBFeedSource { + + static Logger LOG = LoggerFactory.getLogger(DynamoDBFeedSource.class); + private AmazonDynamoDBClient amazonDynamoDBClient; + private DynamoDB dynamoDB; + private DynamoDBMapper mapper; + private boolean enableTimers = false; + private boolean enableLoggingOnShortPage = false; + private int feedHeadDelayInSeconds = 2; + + private Map mapPrefix = new HashMap(); + private Map mapColumn = new HashMap(); + + private String split; + + private AdapterHelper helper = new AdapterHelper(); + + private SQLToNoSqlConverter getSearchToSqlConverter() { + + return new SQLToNoSqlConverter(mapPrefix, split); + } + public void setPrefixColumnMap(Map prefix) { + + mapPrefix = new HashMap(prefix); + + mapColumn = new HashMap(); + + for (String key : mapPrefix.keySet()) { + + mapColumn.put(mapPrefix.get(key), key); + } + } + + public void setDelimiter(String splitParam) { + + split = splitParam; + } + + @NotImplemented + public void setParameters(Map params) { + throw new UnsupportedOperationException("Not supported yet."); + } + + public void afterPropertiesSet() { + + if (split != null ^ !mapPrefix.isEmpty()) { + + throw new IllegalArgumentException("The 'delimiter' and 'prefixColumnMap' field must both be defined"); + } + } + + + public DynamoDBFeedSource(AmazonDynamoDBClient amazonDynamoDBClient, DynamoDBMapper mapper) { + this.amazonDynamoDBClient = amazonDynamoDBClient; + this.dynamoDB = new DynamoDB(this.amazonDynamoDBClient); + this.mapper = mapper; + setDynamoDB(dynamoDB); + } + + public void setDynamoDB(DynamoDB dynamoDB) { + this.dynamoDB = dynamoDB; + } + + /** + * This method is used to return the categories based search with the help of feed and entryId and search type is backward + * whch means it will search for date less the markerDate along with other params. + * @param feedName: Name of the feed for each entry . For ex: namespace/feed + * @param markerTimestamp: Timestamp for which the search is to be performed for category + * @param markerId: EntryID for every event. + * @param searchString: The string on which search is performed in db + * @param pageSize: default size is 25 + * @return List of PersistedEntry data found based on above params from db. + */ + public List getFeedBackward(String feedName, + Date markerTimestamp, + long markerId, + String searchString, + int pageSize) { + List feedPage; + DynamoDBQueryBuilder sqlBac = new DynamoDBQueryBuilder(getSearchToSqlConverter()).searchString(searchString); + sqlBac.searchType(SearchType.FEED_BACKWARD); + //Dynamodb query implementation + Map map = new HashMap(); + String filters = sqlBac.getFilters(map); + String feedNameFilter=" and feed= :feedName"; + SimpleDateFormat dateFormatter = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'"); + dateFormatter.setTimeZone(TimeZone.getTimeZone("UTC")); + String dateLastUpdated = dateFormatter.format(markerTimestamp); + Map valueMap = new HashMap(); + valueMap.put(":id", new AttributeValue().withS(String.valueOf(markerId))); + valueMap.put(":dateLastUpdated", new AttributeValue().withS(dateLastUpdated)); + valueMap.put(":feedName",new AttributeValue().withS(feedName)); + for (Map.Entry res : map.entrySet()) { + valueMap.put(res.getKey(), new AttributeValue().withS(res.getValue())); + } + DynamoDBQueryExpression querySpec = new DynamoDBQueryExpression() + .withKeyConditionExpression("entryId = :id and dateLastUpdated <= :dateLastUpdated") + .withScanIndexForward(false) + .withLimit(pageSize) + .withFilterExpression(filters + feedNameFilter) + .withExpressionAttributeValues(valueMap); + feedPage = mapper.query(PersistedEntry.class, querySpec); + return feedPage; + } + + /** + * This method is used to return list of entries from DynamoDb bases on feedName and timestamp. + * @param getFeedRequest: It contains the request object parameters + * @param startingAt: The timestamp from where we need to perform search + * @param pageSize: for pagination . Default page size is 25 + * @return List of entries found based on given timestamp ,feed name. + */ + //TODO LINK REF NEED TO BE IMPLEMENTED FOR HYDRATED FEED. + public List getFeedPageByTimestamp(GetFeedRequest getFeedRequest, String startingAt, int pageSize) throws Exception { + final String pageDirectionValue = getFeedRequest.getDirection(); + PageDirection pageDirection = PageDirection.FORWARD; + if (StringUtils.isNotEmpty(String.valueOf(pageDirection))) { + pageDirection = PageDirection.valueOf(pageDirectionValue.toUpperCase()); + } + final String searchString = getFeedRequest.getSearchQuery() != null ? getFeedRequest.getSearchQuery() : ""; + DateTimeFormatter isoDTF = ISODateTimeFormat.dateTime(); + DateTime startAt = isoDTF.parseDateTime(startingAt); + List entryMarker = getEntryByTimestamp(startAt, getFeedRequest.getFeedName(), pageDirection); + if (entryMarker.isEmpty()) { + throw new RuntimeException("No entry with specified startingAt timestamp found"); + } + return entryMarker; + } + + /** + * This method creates the query for fetching the data based on timestamp and the direction from DynamoDB + * @param + * @param markerDate: StartAt Must be in ISO 8601 Date and Time format, and must contain a time zone, + * for example: 2014-03-10T06:00:00.000Z. For more information, see ISO 8601 Date and Time format. + * @param direction: Specifies the direction from which to return entries, starting from the current marker or entry. + * Can be either forward or backward. + * @return List of data present based on the search query. + */ + protected List getEntryByTimestamp(final DateTime markerDate, final String feed, PageDirection direction) { + List feedPage = new ArrayList(); + Table table = dynamoDB.getTable("entries"); + Index index = table.getIndex("global-feed-index"); + QuerySpec spec = new QuerySpec() + .withKeyConditionExpression("feed = :feed and " + getTimeStampValueFilter(direction)) + .withValueMap(new ValueMap() + .withString(":feed", feed) + .withString(":markerDate", String.valueOf(markerDate))); + ItemCollection persistedEntryItems = index.query(spec); + Iterator itemsIterator = persistedEntryItems.iterator(); + while (itemsIterator.hasNext()) { + Item item = itemsIterator.next(); + feedPage.add(item.toJSONPretty()); + } + return feedPage; + } + + /** + *This method is used to create a query for dynamodb for timestamp search based on direction + * @param direction: If the startingAt parameter is used without a direction parameter, then the forward direction is assumed. + * If you want to fetch feeds from a time period before the time specified in the time stamp, + * you need to use the direction parameter and then the backward description, like the following: direction set to backward. + * @return : the formed query based upon the direction. + */ + private String getTimeStampValueFilter(PageDirection direction) { + if (direction.equals(PageDirection.BACKWARD)) { + return "dateLastUpdated <= :markerDate"; + } else { + return "dateLastUpdated >= :markerDate"; + } + } +} diff --git a/adapters/dynamoDB_adapters/src/main/java/org/atomhopper/dynamodb/adapter/DynamoFeedDBPublisher.java b/adapters/dynamoDB_adapters/src/main/java/org/atomhopper/dynamodb/adapter/DynamoFeedDBPublisher.java new file mode 100644 index 00000000..28184fc4 --- /dev/null +++ b/adapters/dynamoDB_adapters/src/main/java/org/atomhopper/dynamodb/adapter/DynamoFeedDBPublisher.java @@ -0,0 +1,218 @@ +package org.atomhopper.dynamodb.adapter; + +import com.amazonaws.services.dynamodbv2.AmazonDynamoDBClient; +import com.amazonaws.services.dynamodbv2.datamodeling.DynamoDBMapper; +import com.amazonaws.services.dynamodbv2.document.*; +import com.amazonaws.services.dynamodbv2.document.spec.QuerySpec; +import com.amazonaws.services.dynamodbv2.document.utils.ValueMap; +import com.yammer.metrics.Metrics; +import org.apache.abdera.model.Entry; +import org.apache.commons.lang.StringUtils; +import org.atomhopper.adapter.FeedPublisher; +import org.atomhopper.adapter.NotImplemented; +import org.atomhopper.adapter.PublicationException; +import org.atomhopper.adapter.ResponseBuilder; +import org.atomhopper.adapter.request.adapter.DeleteEntryRequest; +import org.atomhopper.adapter.request.adapter.PostEntryRequest; +import org.atomhopper.adapter.request.adapter.PutEntryRequest; +import org.atomhopper.dynamodb.constant.DynamoDBConstant; +import org.atomhopper.dynamodb.model.PersistedEntry; +import org.atomhopper.response.AdapterResponse; +import org.atomhopper.response.EmptyBody; +import org.atomhopper.util.uri.template.EnumKeyedTemplateParameters; +import org.atomhopper.util.uri.template.URITemplate; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import com.yammer.metrics.core.Counter; + +import java.io.IOException; +import java.io.StringWriter; +import java.text.Format; +import java.text.SimpleDateFormat; +import java.util.*; + +import static org.apache.abdera.i18n.text.UrlEncoding.decode; + +/** + * @Author: shub6691 + * This class is used to publish the records in dynamodb using EntryId,feedName . + * Index has been created on the feedName for fast search and getting the entry from the DynamoDb. + */ +public class DynamoFeedDBPublisher implements FeedPublisher { + private static final Logger LOG = LoggerFactory.getLogger(DynamoFeedDBPublisher.class); + private AmazonDynamoDBClient dynamoDBClient; + private DynamoDBMapper mapper; + private DynamoDB dynamoDB; + + + public void setDynamoDBClient(AmazonDynamoDBClient dynamoDBClient) { + this.dynamoDBClient = dynamoDBClient; + setDynamoMapper(new DynamoDBMapper(dynamoDBClient)); + this.dynamoDB = new DynamoDB(dynamoDBClient); + setDynamoDB(dynamoDB); + } + + public void setDynamoMapper(DynamoDBMapper mapper) { + this.mapper = mapper; + } + + public void setDynamoDB(DynamoDB dynamoDB) { + this.dynamoDB = dynamoDB; + } + + private boolean allowOverrideId = true; + private boolean allowOverrideDate = false; + + private Map counterMap = Collections.synchronizedMap(new HashMap()); + + @Override + @NotImplemented + public void setParameters(Map params) { + throw new UnsupportedOperationException("Not supported yet."); + + } + + public void setAllowOverrideId(boolean allowOverrideId) { + this.allowOverrideId = allowOverrideId; + } + + /** + * This method is used to post a new feed into DynamoDB as per the request by using the APACHE ABDERA LIBRARY + * @param postEntryRequest: This object has all the data for the feed to be published into DynamoDB which is + * parsed using the Abdera library. + * @return Return the response of the feed in format of atom format + */ + @Override + public AdapterResponse postEntry(PostEntryRequest postEntryRequest) { + final Entry abderaParsedEntry = postEntryRequest.getEntry(); + final PersistedEntry persistedEntry = new PersistedEntry(); + boolean entryIdSent = abderaParsedEntry.getId() != null; + + // Generate an ID for this entry + if (allowOverrideId && entryIdSent && StringUtils.isNotBlank(abderaParsedEntry.getId().toString().trim())) { + String entryId = abderaParsedEntry.getId().toString(); + // Check to see if entry with this id already exists + //Returns List of Object found with the entryId and feedName; + List exists = getEntry(entryId, postEntryRequest.getFeedName()); + if (!exists.isEmpty()) { + String errMsg = String.format("Unable to persist entry. Reason: entryId (%s) not unique.", entryId); + return ResponseBuilder.conflict(errMsg); + } + persistedEntry.setEntryId(abderaParsedEntry.getId().toString()); + } else { + persistedEntry.setEntryId(DynamoDBConstant.UUID_URI_SCHEME + UUID.randomUUID().toString()); + abderaParsedEntry.setId(persistedEntry.getEntryId()); + } + if (allowOverrideDate) { + Date updated = abderaParsedEntry.getUpdated(); + + if (updated != null) { + persistedEntry.setDateLastUpdated(getDateFormatInString(updated)); + persistedEntry.setCreationDate(getDateFormatInString(updated)); + } + } + + // Set the categories + persistedEntry.setCategories(processCategories(abderaParsedEntry.getCategories())); + + if (abderaParsedEntry.getSelfLink() == null) { + abderaParsedEntry.addLink(decode(postEntryRequest.urlFor(new EnumKeyedTemplateParameters(URITemplate.FEED))) + + "entries/" + persistedEntry.getEntryId()).setRel(DynamoDBConstant.LINK_REL_SELF); + } + + persistedEntry.setFeed(postEntryRequest.getFeedName()); + persistedEntry.setEntryBody(entryToString(abderaParsedEntry)); + abderaParsedEntry.setUpdated(persistedEntry.getDateLastUpdated()); + abderaParsedEntry.setPublished(persistedEntry.getCreationDate()); + mapper.save(persistedEntry);//dynamoDB save object + incrementCounterForFeed(postEntryRequest.getFeedName()); + return ResponseBuilder.created(abderaParsedEntry); + } + + private List processCategories(List abderaCategories) { + final List categoriesList = new ArrayList(); + + for (org.apache.abdera.model.Category abderaCat : abderaCategories) { + categoriesList.add(abderaCat.getTerm().toLowerCase()); + } + + return categoriesList; + } + + private String entryToString(Entry entry) { + final StringWriter writer = new StringWriter(); + + try { + entry.writeTo(writer); + } catch (IOException ioe) { + LOG.error("Unable to write entry to string. Unable to persist entry. Reason: " + ioe.getMessage(), ioe); + + throw new PublicationException(ioe.getMessage(), ioe); + } + + return writer.toString(); + } + + @Override + @NotImplemented + public AdapterResponse putEntry(PutEntryRequest putEntryRequest) { + throw new UnsupportedOperationException("Not supported."); + } + + @Override + @NotImplemented + public AdapterResponse deleteEntry(DeleteEntryRequest deleteEntryRequest) { + throw new UnsupportedOperationException("Not supported."); + } + + /** + * To get the entry from dynamodb based upon two params? + * + * @param entryId: It is the marker id for entry for every events + * @param feedName: feed name is used to search the records in dynamodb + * @return : list of entry found if that exits in dynamodb with the entryId and feedName. + */ + private List getEntry(String entryId, final String feedName) { + List persistedEntriesObject = new ArrayList(); + Table table = dynamoDB.getTable(DynamoDBConstant.ENTRIES); + Index index = table.getIndex(DynamoDBConstant.ENTRY_ID_FEED_INDEX); + QuerySpec spec = new QuerySpec() + .withKeyConditionExpression("entryId = :entryId and feed = :feed") + .withValueMap(new ValueMap() + .withString(":entryId", entryId) + .withString(":feed", feedName)); + ItemCollection persistedEntryItems = index.query(spec); + Iterator itemsIterator = persistedEntryItems.iterator(); + while (itemsIterator.hasNext()) { + Item item = itemsIterator.next(); + persistedEntriesObject.add(item.toJSONPretty()); + } + return persistedEntriesObject; + } + + private void incrementCounterForFeed(String feedName) { + + if (!counterMap.containsKey(feedName)) { + synchronized (counterMap) { + if (!counterMap.containsKey(feedName)) { + Counter counter = Metrics.newCounter(DynamoFeedDBPublisher + .class, "entries-created-for-" + feedName); + counterMap.put(feedName, counter); + } + } + } + + counterMap.get(feedName).inc(); + } + + /** + * Sets the date in String format as we save the date in string in dynamodb. + * + * @param date Date Object to be formatted in string. + * @return + */ + private String getDateFormatInString(Date date) { + Format formatter = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + return formatter.format(date); + } +} diff --git a/adapters/dynamoDB_adapters/src/main/java/org/atomhopper/dynamodb/constant/DynamoDBConstant.java b/adapters/dynamoDB_adapters/src/main/java/org/atomhopper/dynamodb/constant/DynamoDBConstant.java new file mode 100644 index 00000000..fd1dee96 --- /dev/null +++ b/adapters/dynamoDB_adapters/src/main/java/org/atomhopper/dynamodb/constant/DynamoDBConstant.java @@ -0,0 +1,11 @@ +package org.atomhopper.dynamodb.constant; + + +public final class DynamoDBConstant { + public static final String UUID_URI_SCHEME = "urn:uuid:"; + public static final String LINK_REL_SELF = "self"; + public static final String ENTRIES = "entries"; + public static final String ENTRY_ID_FEED_INDEX = "entryId-feed-index"; + public static final String ENTRY_ID = "entryId"; + public static final String FEED = "feed"; +} diff --git a/adapters/dynamoDB_adapters/src/main/java/org/atomhopper/dynamodb/model/PersistedEntry.java b/adapters/dynamoDB_adapters/src/main/java/org/atomhopper/dynamodb/model/PersistedEntry.java new file mode 100644 index 00000000..f1dcddb2 --- /dev/null +++ b/adapters/dynamoDB_adapters/src/main/java/org/atomhopper/dynamodb/model/PersistedEntry.java @@ -0,0 +1,21 @@ +package org.atomhopper.dynamodb.model; +import com.amazonaws.services.dynamodbv2.datamodeling.*; +import lombok.Data; +import java.util.*; +@Data +@DynamoDBTable(tableName = "entries") +public class PersistedEntry { + + @DynamoDBHashKey(attributeName = "entryId") + private String entryId; + @DynamoDBIndexRangeKey(attributeName = "feed", + localSecondaryIndexName = "entryId-feed-index") + @DynamoDBIndexHashKey(globalSecondaryIndexName="global-feed-index", attributeName = "feed") + private String feed; + private String entryBody; + @DynamoDBAutoGeneratedTimestamp(strategy = DynamoDBAutoGenerateStrategy.CREATE) + private String creationDate; + @DynamoDBRangeKey(attributeName = "dateLastUpdated") + private String dateLastUpdated; + private List categories; +} diff --git a/adapters/dynamoDB_adapters/src/main/java/org/atomhopper/dynamodb/query/CategoryStringGenerator.java b/adapters/dynamoDB_adapters/src/main/java/org/atomhopper/dynamodb/query/CategoryStringGenerator.java new file mode 100644 index 00000000..e979bf9e --- /dev/null +++ b/adapters/dynamoDB_adapters/src/main/java/org/atomhopper/dynamodb/query/CategoryStringGenerator.java @@ -0,0 +1,160 @@ +package org.atomhopper.dynamodb.query; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +public final class CategoryStringGenerator { + + private static final char INCLUSIVE_OPERATOR = '+', ESCAPE_OPERATOR = '\\'; + private static final char[] OPERATORS = {INCLUSIVE_OPERATOR, ESCAPE_OPERATOR}; + + private CategoryStringGenerator() { + throw new AssertionError(); + } + + /** + * Takes a search string, prefix map & prefix character & returns an array of strings which are arguments to the + * corresponding generated SQL statement. + * + * See SearchToSqlConverter for more details. + * + * @param searchString + * @param mapPrefix + * @param prefixSplit + * + * @return + */ + public static List getPostgresCategoryString(String searchString, Map mapPrefix, String prefixSplit ) { + + List finalList = new ArrayList(); + + + if (searchString == null || !(searchString.trim().length() > 0)) { + + finalList.add( "{}" ); + return finalList; + } + + List categories = parse(searchString.trim().toLowerCase()); + + List catHolder = new ArrayList(); + + // find if any categories are prefixed, if so, split them out. + for( String cat : categories ) { + + if (cat.matches( SQLToNoSqlConverter.BAD_SEARCH_REGEX ) ) { + + throw new IllegalArgumentException( SQLToNoSqlConverter.BAD_CHAR_MSG ); + } + + + if (prefixSplit != null ) { + + int index = cat.indexOf( prefixSplit ); + + if ( index != -1 ) { + + String prefix = cat.substring( 0, index ); + String value = cat.substring( index + prefixSplit.length() ); + + if ( mapPrefix.containsKey( prefix ) ) { + + addToFinalList( finalList, catHolder ); + + finalList.add( value ); + } + else { + + catHolder.add( cat ); + } + } + else { + + catHolder.add( cat ); + } + } + else { + catHolder.add( cat ); + } + } + + addToFinalList( finalList, catHolder ); + + return finalList; + } + + private static void addToFinalList( List finalList, List catHolder ) { + + if ( !catHolder.isEmpty() ) { + + String psArray = DynamoDBTextArray.stringArrayToPostgreSQLTextArray( catHolder.toArray( new String[ catHolder + .size() ] ) ); + finalList.add( psArray ); + + catHolder.clear(); + } + } + + private static List parse(String searchString) { + List categories = new ArrayList(); + + for (int charIndex = 0; charIndex < searchString.length(); charIndex++) { + final char nextOperator = searchString.charAt(charIndex); + final StringBuilder searchTermBuilder = new StringBuilder(); + + charIndex = readTerm(searchString, searchTermBuilder, charIndex + 1); + + switch (nextOperator) { + case INCLUSIVE_OPERATOR: + if( searchTermBuilder.toString().isEmpty() ) { + + throw new IllegalArgumentException( "Invalid Search Parameter: Parameter cannot be empty string." ); + } + + categories.add(searchTermBuilder.toString()); + break; + + default: + break; + } + } + + return categories; + } + + private static int readTerm(String searchString, StringBuilder builder, int currentCharIndex) { + int charIndex = currentCharIndex; + boolean isEscaped = false; + + + while( charIndex < searchString.length() ) { + final char nextChar = searchString.charAt(charIndex); + + if (isEscaped || !isOperator(nextChar)) { + builder.append( nextChar ); + isEscaped = false; + } else { + if (nextChar == ESCAPE_OPERATOR) { + isEscaped = true; + } else { + return charIndex - 1; + } + } + + charIndex++; + } + + return charIndex; + } + + private static boolean isOperator(char character) { + for (char operator : OPERATORS) { + if (operator == character) { + return true; + } + } + + return false; + } +} diff --git a/adapters/dynamoDB_adapters/src/main/java/org/atomhopper/dynamodb/query/DynamoDBQueryBuilder.java b/adapters/dynamoDB_adapters/src/main/java/org/atomhopper/dynamodb/query/DynamoDBQueryBuilder.java new file mode 100644 index 00000000..be2ad955 --- /dev/null +++ b/adapters/dynamoDB_adapters/src/main/java/org/atomhopper/dynamodb/query/DynamoDBQueryBuilder.java @@ -0,0 +1,76 @@ +package org.atomhopper.dynamodb.query; + +import org.joda.time.DateTime; + +import java.util.Map; + +public class DynamoDBQueryBuilder { + private String searchString; + private SearchType type; + private int feedHeadDelayInSeconds = -1; + private DateTime startingTimestamp; + + private static final String EQUALS = "="; + private static final String LESS_THAN = "<"; + private static final String GREATER_THAN = ">"; + + private static final String QUESTION_MARK = "?"; + + private static final String OPEN_PARENS = "("; + private static final String CLOSE_PARENS = ")"; + + private static final String SELECT = "SELECT * FROM entries WHERE feed = ?"; + private static final String AND = "AND"; + private static final String SPACE = " "; + private static final String DATELASTUPDATED = "datelastupdated %s ?"; + private static final String ID = "id %s ?"; + + private static final String UNION_ALL = "UNION ALL"; + + private static final String ORDER_BY_ASC = "ORDER BY datelastupdated ASC, id ASC LIMIT ?"; + private static final String ORDER_BY_ASC_LIMIT = "ORDER BY datelastupdated ASC, id ASC LIMIT %s"; + private static final String ORDER_BY_DATE_ASC_ID_DESC_LIMIT = "ORDER BY datelastupdated ASC, id DESC LIMIT %s"; + private static final String ORDER_BY_DESC_LIMIT = "ORDER BY datelastupdated DESC, id DESC LIMIT %s"; + private static final String ORDER_BY_DATE_DESC_ID_ASC_LIMIT = "ORDER BY datelastupdated DESC, id ASC LIMIT %s"; + private static final String DB_TIMESTAMP_PATTERN = "yyyy-MM-dd HH:mm:ss.SSS z"; + + private SQLToNoSqlConverter SQLToNoSqlConverter; + + public DynamoDBQueryBuilder(SQLToNoSqlConverter converter ) { + + SQLToNoSqlConverter = converter; + } + + public DynamoDBQueryBuilder searchString(String searchString) { + this.searchString = searchString; + return this; + } + + public DynamoDBQueryBuilder searchType(SearchType type) { + this.type = type; + return this; + } + + public DynamoDBQueryBuilder feedHeadDelayInSeconds(int delay) { + this.feedHeadDelayInSeconds = delay; + return this; + } + + public DynamoDBQueryBuilder startingTimestamp(DateTime timestamp) { + this.startingTimestamp = timestamp; + return this; + } + + public String getFilters(Map map) { + + String searchSql = SQLToNoSqlConverter.getSqlFromSearchString(searchString,map); + + switch (type) { + + case FEED_BACKWARD: + return searchSql; + } + return null; + } + +} diff --git a/adapters/dynamoDB_adapters/src/main/java/org/atomhopper/dynamodb/query/DynamoDBTextArray.java b/adapters/dynamoDB_adapters/src/main/java/org/atomhopper/dynamodb/query/DynamoDBTextArray.java new file mode 100644 index 00000000..4d1de435 --- /dev/null +++ b/adapters/dynamoDB_adapters/src/main/java/org/atomhopper/dynamodb/query/DynamoDBTextArray.java @@ -0,0 +1,198 @@ +package org.atomhopper.dynamodb.query; + +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.Arrays; +import java.util.Map; + +/** + * This is class provides {@link java.sql.Array} interface for PostgreSQL + * text array. + * + * + */ +public class DynamoDBTextArray implements java.sql.Array { + + private final String[] stringArray; + private final String stringValue; + + /** + * Initializing constructor + * + * @param stringArray + */ + public DynamoDBTextArray(String[] stringArray) { + if (stringArray == null) { + this.stringArray = null; + } else { + this.stringArray = Arrays.copyOf(stringArray, stringArray.length); + } + this.stringValue = stringArrayToPostgreSQLTextArray(this.stringArray); + } + + @Override + public String toString() { + return stringValue; + } + private static final String NULL = "NULL"; + + /** + * This static method can be used to convert an string array to string + * representation of PostgreSQL text array. + * + * @param stringArray source String array + * @return string representation of a given text array + */ + public static String stringArrayToPostgreSQLTextArray(String[] stringArray) { + final int arrayLength; + final int bufferAddition = 4; + if (stringArray == null) { + return NULL; + } + + arrayLength = stringArray.length; + + if (arrayLength == 0) { + return "{}"; + } + + // count the string length and if need to quote + int neededBufferLength = 2; + // count the beginning '{' and the ending '}' brackets + boolean[] shouldQuoteArray = new boolean[stringArray.length]; + for (int si = 0; si < arrayLength; si++) { + // count the comma after the first element + if (si > 0) { + neededBufferLength++; + } + + boolean shouldQuote; + final String s = stringArray[si]; + if (s == null) { + neededBufferLength += bufferAddition; + shouldQuote = false; + } else { + final int l = s.length(); + neededBufferLength += l; + if (l == 0 || s.equalsIgnoreCase(NULL)) { + shouldQuote = true; + } else { + shouldQuote = false; + // scan for commas and quotes + for (int i = 0; i < l; i++) { + final char ch = s.charAt(i); + switch (ch) { + case '"': + case '\\': + shouldQuote = true; + // we will escape these characters + neededBufferLength++; + break; + case ',': + case '\'': + case '{': + case '}': + shouldQuote = true; + break; + default: + if (Character.isWhitespace(ch)) { + shouldQuote = true; + } + break; + } + } + } + // count the quotes + if (shouldQuote) { + neededBufferLength += 2; + } + } + shouldQuoteArray[si] = shouldQuote; + } + + // construct the String + final StringBuilder sb = new StringBuilder(neededBufferLength); + sb.append('{'); + for (int si = 0; si < arrayLength; si++) { + final String s = stringArray[si]; + if (si > 0) { + sb.append(','); + } + if (s == null) { + sb.append(NULL); + } else { + final boolean shouldQuote = shouldQuoteArray[si]; + if (shouldQuote) { + sb.append('"'); + } + for (int i = 0, l = s.length(); i < l; i++) { + final char ch = s.charAt(i); + if (ch == '"' || ch == '\\') { + sb.append('\\'); + } + sb.append(ch); + } + if (shouldQuote) { + sb.append('"'); + } + } + } + sb.append('}'); + assert sb.length() == neededBufferLength; + return sb.toString(); + } + + @Override + public Object getArray() throws SQLException { + return stringArray == null ? null : Arrays.copyOf(stringArray, stringArray.length); + } + + @Override + public Object getArray(Map> map) throws SQLException { + return getArray(); + } + + @Override + public Object getArray(long index, int count) throws SQLException { + return stringArray == null ? null : Arrays.copyOfRange(stringArray, (int) index, (int) index + count); + } + + @Override + public Object getArray(long index, int count, Map> map) throws SQLException { + return getArray(index, count); + } + + @Override + public int getBaseType() throws SQLException { + return java.sql.Types.VARCHAR; + } + + @Override + public String getBaseTypeName() throws SQLException { + return "text"; + } + + @Override + public ResultSet getResultSet() throws SQLException { + throw new UnsupportedOperationException(); + } + + @Override + public ResultSet getResultSet(Map> map) throws SQLException { + throw new UnsupportedOperationException(); + } + + @Override + public ResultSet getResultSet(long index, int count) throws SQLException { + throw new UnsupportedOperationException(); + } + + @Override + public ResultSet getResultSet(long index, int count, Map> map) throws SQLException { + throw new UnsupportedOperationException(); + } + + @Override + public void free() throws SQLException { + } +} \ No newline at end of file diff --git a/adapters/dynamoDB_adapters/src/main/java/org/atomhopper/dynamodb/query/SQLToNoSqlConverter.java b/adapters/dynamoDB_adapters/src/main/java/org/atomhopper/dynamodb/query/SQLToNoSqlConverter.java new file mode 100644 index 00000000..9a5584e4 --- /dev/null +++ b/adapters/dynamoDB_adapters/src/main/java/org/atomhopper/dynamodb/query/SQLToNoSqlConverter.java @@ -0,0 +1,311 @@ +package org.atomhopper.dynamodb.query; + +import com.unboundid.ldap.sdk.Filter; +import com.unboundid.ldap.sdk.LDAPException; +import org.apache.commons.lang.StringUtils; + +import java.util.*; + +/** + * This class creates the search SQL and corresponding SQL parameters for a given set of query parameters. + * + * By default, this class assumes all categories are treated the same and stored in a single variable-length array. + * + * This class can also be customized to map specific categories to specific columns in the DB, allowing for higher + * performance when searching on these categories. + * + * To configure this functionality pass in the following: + * + *
    + *
  • Map[String, String] - a map of pairs where the key is the prefix found in the atom category and the + * value is the name of the SQL text column.
    • + *
  • String - the String which separates the prefix from the value within the atom category. E.g., + * for the category value of "tid:1234", if ':' is the mark, then "tid" is the prefix.
    • + *
+ * + * An example configuration might be: + * + * A map of { "tid" => "tenantid", "type" => "eventtype" } with a mark of ":". + * + * This maps the following atom categories: + * + *
    + *
  • "tid:1234" => enter "1234" into the "tenantid" column
    • + *
  • "type:lbaas.usage" => enter "lbaas.usage" into the "eventtype" column
    • + *
+ */ +public class SQLToNoSqlConverter { + + public static final String BAD_SEARCH_REGEX = ".*(\"|,).*"; + public static final String BAD_CHAR_MSG = "Invalid Search Parameter: '\"' ',' not allowed."; + + private static final String OPEN_PARENS = "("; + private static final String CLOSED_PARENS = ")"; + private static final String OPEN_CURLY_BRACKET = "{"; + private static final String CLOSED_CURLY_BRACKET = "}"; + private static final String PLUS_SIGN = "+"; + private static final String AND = " AND "; + private static final String OR = " OR "; + private static final String NOT = " not "; + + private static final String CATEGORY = "cat"; + private static final String CATEGORY_STRING = " (contains(categories,:replaceValue)) "; + + public static final String OLD_CATEGORY_STRING = " categories && ?::varchar[] "; + + private static final String COLUMN_STRING= " = ? "; + + private String prefixSplit = null; + + private Map mapPrefix = new HashMap(); + + public SQLToNoSqlConverter() { } + + public SQLToNoSqlConverter(Map mapper, String split ) { + + prefixSplit = split; + mapPrefix = new HashMap( mapper ); + } + + + public static void main(String[] args) { + //System.out.println(new SearchToSqlConverter().getSqlFromSearchString("(AND(cat=CAT1)(OR(cat=CAT2)(cat=CAT3))(NOT(cat=CAT4)))")); + } + + public String getSqlFromSearchString(String searchString,Map map) { + + if (StringUtils.isBlank(searchString)) { + return null; + } + + if (searchString.startsWith(PLUS_SIGN)) { + //return getSqlForClassicSearchFormat(searchString); + return null; + } else if (searchString.startsWith(OPEN_PARENS)) { + searchString = textToLDapSearch(searchString); + Filter filter; + try { + filter = Filter.create(searchString); + } catch (LDAPException ex) { + throw new IllegalArgumentException("Invalid LDAP Search Parameter"); + } + return getSqlFromLdapFilter(filter,map); + } else { + throw new IllegalArgumentException("Invalid Search Parameter: Search must begin with a '+' or a '(' character"); + } + } + + public List getParamsFromSearchString(String searchString) { + + if (StringUtils.isBlank(searchString)) { + return new ArrayList(); + } + + if (searchString.startsWith(PLUS_SIGN)) { + return getParametersForClassicSearchFormat((searchString)); + } else if (searchString.startsWith(OPEN_PARENS)) { + searchString = textToLDapSearch(searchString); + Filter filter; + try { + filter = Filter.create(searchString); + } catch (LDAPException ex) { + throw new IllegalArgumentException("Invalid LDAP Search Parameter"); + } + return getParametersFromLdapFilter(filter); + } else { + throw new IllegalArgumentException("Invalid Search Parameter: Search must begin with a '+' or a '(' character"); + } + } + + private String textToLDapSearch(String searchString) { + searchString = searchString.replace("(AND", "(&"); + searchString = searchString.replace("(OR", "(|"); + searchString = searchString.replace("(NOT", "(!"); + return searchString; + } + + private String getSqlForClassicSearchFormat(String searchString) { + + String[] params = searchString.split( "\\+" ); + + List sqlList = new ArrayList(); + String last = ""; + + // first item is an empty string, so we skip + for( int i = 1; i < params.length; i++ ) { + + String state = createSql( params[ i ], OLD_CATEGORY_STRING ); + + // if we have several generic categories, we only need 1 sql statement to handle them + if( !(state.equals( OLD_CATEGORY_STRING ) && last.equals( OLD_CATEGORY_STRING ) ) ) { + + sqlList.add( state ); + } + + last = state; + } + + StringBuilder sql = new StringBuilder(); + + sql.append(OPEN_PARENS); + + for( int i = 0; i < sqlList.size(); i++ ) { + + if ( i > 0 ) + sql.append( OR ); + + sql.append( sqlList.get( i ) ); + } + + sql.append(CLOSED_PARENS); + + return sql.toString(); + } + + private List getParametersForClassicSearchFormat(String searchString) { + List params = new ArrayList(); + params.addAll(CategoryStringGenerator.getPostgresCategoryString(searchString, mapPrefix, prefixSplit ) ); + return params; + } + + private String getSqlFromLdapFilter(Filter filter,Map a) { + + StringBuilder sql = new StringBuilder(); + + Filter[] filters = filter.getComponents(); + Filter notFilter = filter.getNOTComponent(); + + switch (filter.getFilterType()) { + + case Filter.FILTER_TYPE_AND: + for (int x=0 ; x < filters.length; x++) { + if (x == 0) { + sql.append(OPEN_PARENS); + } + if (x > 0) { + sql.append(AND); + } + sql.append(getSqlFromLdapFilter(filters[x],a)); + if (x == filters.length - 1) { + sql.append(CLOSED_PARENS); + } + } + break; + + case Filter.FILTER_TYPE_OR: + for (int x=0 ; x < filters.length; x++) { + if (x == 0) { + sql.append(OPEN_PARENS); + } + if (x > 0) { + sql.append(OR); + } + sql.append(getSqlFromLdapFilter(filters[x],a)); + if (x == filters.length - 1) { + sql.append(CLOSED_PARENS); + } + } + break; + + case Filter.FILTER_TYPE_NOT: + sql.append(OPEN_PARENS); + sql.append(NOT); + sql.append(getSqlFromLdapFilter(notFilter,a)); + sql.append(CLOSED_PARENS); + break; + + case Filter.FILTER_TYPE_EQUALITY: + if (!filter.getAttributeName().equals(CATEGORY)) { + throw new IllegalArgumentException("Invalid Search Parameter: LDAP attribute name must be 'cat'"); + } + //String key = UUID.randomUUID().toString(); + a.put(":"+filter.getAssertionValue(),filter.getAssertionValue()); + sql.append(CATEGORY_STRING.replaceAll("replaceValue",filter.getAssertionValue())); + break; + } + + return sql.toString(); + } + + private String createSql( String param, String defaultSql ) { + + if( prefixSplit != null ) { + + int index = param.indexOf( prefixSplit ); + + if ( index != -1 ) { + + String prefix = param.substring( 0, index ); + + // detect prefix in map + if ( mapPrefix.containsKey( prefix ) ) { + + String column = mapPrefix.get( prefix ); + + return " " + column + COLUMN_STRING; + } + } + } + + return defaultSql; + } + + private List getParametersFromLdapFilter(Filter filter) { + + List params = new ArrayList(); + + Filter[] filters = filter.getComponents(); + Filter notFilter = filter.getNOTComponent(); + + switch (filter.getFilterType()) { + + case Filter.FILTER_TYPE_AND: + case Filter.FILTER_TYPE_OR: + for (int x=0 ; x < filters.length; x++) { + params.addAll(getParametersFromLdapFilter(filters[x])); + } + break; + + case Filter.FILTER_TYPE_NOT: + params.addAll(getParametersFromLdapFilter(notFilter)); + break; + + case Filter.FILTER_TYPE_EQUALITY: + if (!filter.getAttributeName().equals(CATEGORY)) { + throw new IllegalArgumentException("Invalid Search Parameter: LDAP attribute name must be 'cat'"); + } + + // default + String param = OPEN_CURLY_BRACKET + filter.getAssertionValue().toLowerCase() + CLOSED_CURLY_BRACKET; + + if( prefixSplit != null ) { + + int index = filter.getAssertionValue().indexOf( prefixSplit ); + + if ( index != -1 ) { + + String prefix = filter.getAssertionValue().substring( 0, index ); + String value = filter.getAssertionValue().substring( index + prefixSplit.length() ); + + if ( mapPrefix.containsKey( prefix ) ) { + + param = value; + } + } + } + + if (param.matches( BAD_SEARCH_REGEX ) ) { + + throw new IllegalArgumentException( BAD_CHAR_MSG ); + } + + params.add( param ); + break; + default: + + throw new IllegalArgumentException( "Invalid Search Parameter" ); + } + + return params; + } +} diff --git a/adapters/dynamoDB_adapters/src/main/java/org/atomhopper/dynamodb/query/SearchType.java b/adapters/dynamoDB_adapters/src/main/java/org/atomhopper/dynamodb/query/SearchType.java new file mode 100644 index 00000000..c1d76f6c --- /dev/null +++ b/adapters/dynamoDB_adapters/src/main/java/org/atomhopper/dynamodb/query/SearchType.java @@ -0,0 +1,11 @@ +package org.atomhopper.dynamodb.query; + +public enum SearchType { + FEED_FORWARD, + FEED_BACKWARD, + FEED_HEAD, + LAST_PAGE, + NEXT_LINK, + BY_TIMESTAMP_FORWARD, + BY_TIMESTAMP_BACKWARD +} diff --git a/adapters/dynamoDB_adapters/src/main/resources/application.properties b/adapters/dynamoDB_adapters/src/main/resources/application.properties new file mode 100644 index 00000000..92f71008 --- /dev/null +++ b/adapters/dynamoDB_adapters/src/main/resources/application.properties @@ -0,0 +1,2 @@ +#server.port=8081 +logging.config=file:./src/main/resources/logback.xml \ No newline at end of file diff --git a/adapters/dynamoDB_adapters/src/main/resources/logback.xml b/adapters/dynamoDB_adapters/src/main/resources/logback.xml new file mode 100644 index 00000000..e69de29b diff --git a/adapters/dynamoDB_adapters/src/test/java/DynamoDBFeedPublisherTest.java b/adapters/dynamoDB_adapters/src/test/java/DynamoDBFeedPublisherTest.java new file mode 100644 index 00000000..771a109a --- /dev/null +++ b/adapters/dynamoDB_adapters/src/test/java/DynamoDBFeedPublisherTest.java @@ -0,0 +1,124 @@ +import com.amazonaws.services.dynamodbv2.AmazonDynamoDBClient; +import com.amazonaws.services.dynamodbv2.datamodeling.DynamoDBMapper; +import com.amazonaws.services.dynamodbv2.datamodeling.DynamoDBQueryExpression; +import com.amazonaws.services.dynamodbv2.document.*; +import com.amazonaws.services.dynamodbv2.document.internal.IteratorSupport; +import com.amazonaws.services.dynamodbv2.document.spec.QuerySpec; +import org.apache.abdera.model.Entry; +import org.apache.abdera.parser.stax.FOMEntry; +import org.atomhopper.adapter.request.adapter.DeleteEntryRequest; +import org.atomhopper.adapter.request.adapter.PostEntryRequest; +import org.atomhopper.adapter.request.adapter.PutEntryRequest; +import org.atomhopper.dynamodb.adapter.DynamoFeedDBPublisher; +import org.atomhopper.dynamodb.model.PersistedEntry; +import org.atomhopper.response.AdapterResponse; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.runners.MockitoJUnitRunner; +import org.springframework.http.HttpStatus; + +import java.text.Format; +import java.text.SimpleDateFormat; +import java.util.*; + +import static junit.framework.Assert.*; +import static org.mockito.Mockito.*; + +@RunWith(MockitoJUnitRunner.class) +public class DynamoDBFeedPublisherTest { + + @Rule + public ExpectedException expectedException = ExpectedException.none(); + @Mock + private DynamoDB dynamoDB; + @Mock + DynamoDBQueryExpression querySpec; + @Mock + private DynamoDBMapper dynamoDBMapper; + private DynamoFeedDBPublisher dynamoFeedDBPublisher = new DynamoFeedDBPublisher(); + @Mock + private AmazonDynamoDBClient amazonDynamoDBClient; + + + private final String MARKER_ID = UUID.randomUUID().toString(); + private final String ENTRY_BODY = ""; + private final String FEED_NAME = "namespace/feed"; + private PersistedEntry persistedEntry; + private List entryList; + private PostEntryRequest postEntryRequest; + private PutEntryRequest putEntryRequest; + private DeleteEntryRequest deleteEntryRequest; + + @Before + public void setUp() throws Exception { + dynamoFeedDBPublisher.setDynamoDBClient(amazonDynamoDBClient); + dynamoFeedDBPublisher.setDynamoMapper(dynamoDBMapper); + dynamoFeedDBPublisher.setDynamoDB(dynamoDB); + persistedEntry = new PersistedEntry(); + persistedEntry.setFeed(FEED_NAME); + persistedEntry.setEntryId(MARKER_ID); + persistedEntry.setEntryBody(ENTRY_BODY); + persistedEntry.setDateLastUpdated(getDateFormatInString(new Date())); + persistedEntry.setCreationDate(getDateFormatInString(new Date())); + entryList = new ArrayList(); + entryList.add(persistedEntry); + putEntryRequest = mock(PutEntryRequest.class); + deleteEntryRequest = mock(DeleteEntryRequest.class); + postEntryRequest = mock(PostEntryRequest.class); + when(postEntryRequest.getEntry()).thenReturn(entry()); + when(postEntryRequest.getFeedName()).thenReturn("namespace/feed"); + } + + @Test + public void showSaveTheObjectInDynamoDb() throws Exception { + dynamoFeedDBPublisher.setAllowOverrideId(false); + doNothing().when(dynamoDBMapper).save(persistedEntry); + AdapterResponse adapterResponse = dynamoFeedDBPublisher.postEntry(postEntryRequest); + assertEquals("Should return HTTP 201 (Created)", HttpStatus.CREATED, adapterResponse.getResponseStatus()); + } + + @Test + public void showErrorIfAlreadyExistsEntryIDInDynamoDb() throws Exception { + final Table mockTable = mock(Table.class); + when(dynamoDB.getTable(any(String.class))).thenReturn(mockTable); + final Index mockIndex = mock(Index.class); + when(mockTable.getIndex(anyString())).thenReturn(mockIndex); + final ItemCollection outcome = mock(ItemCollection.class); + when(mockIndex.query(any(QuerySpec.class))).thenReturn(outcome); + final IteratorSupport mockIterator = mock(IteratorSupport.class); + final Item mockItem = new Item(); + when(outcome.iterator()).thenReturn(mockIterator); + when(mockIterator.hasNext()).thenReturn(true, false); + when(mockIterator.next()).thenReturn(mockItem); + AdapterResponse adapterResponse = dynamoFeedDBPublisher.postEntry(postEntryRequest); + assertEquals("Should return HTTP 409 (Conflict)", HttpStatus.CONFLICT, adapterResponse.getResponseStatus()); + + } + + @Test(expected = UnsupportedOperationException.class) + public void shouldPutEntry() throws Exception { + dynamoFeedDBPublisher.putEntry(putEntryRequest); + } + + @Test(expected = UnsupportedOperationException.class) + public void shouldDeleteEntry() throws Exception { + dynamoFeedDBPublisher.deleteEntry(deleteEntryRequest); + } + + public Entry entry() { + final FOMEntry entry = new FOMEntry(); + entry.setId(UUID.randomUUID().toString()); + entry.setContent("testing"); + entry.addCategory("category"); + return entry; + } + + public String getDateFormatInString(Date date) { + Format formatter = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + return formatter.format(date); + } +} diff --git a/adapters/dynamoDB_adapters/src/test/java/DynamoDBFeedSourceTest.java b/adapters/dynamoDB_adapters/src/test/java/DynamoDBFeedSourceTest.java new file mode 100644 index 00000000..c9d1380d --- /dev/null +++ b/adapters/dynamoDB_adapters/src/test/java/DynamoDBFeedSourceTest.java @@ -0,0 +1,172 @@ +import com.amazonaws.services.dynamodbv2.AmazonDynamoDBClient; +import com.amazonaws.services.dynamodbv2.datamodeling.*; +import com.amazonaws.services.dynamodbv2.document.*; +import com.amazonaws.services.dynamodbv2.document.DynamoDB; +import com.amazonaws.services.dynamodbv2.document.internal.IteratorSupport; +import com.amazonaws.services.dynamodbv2.document.spec.QuerySpec; +import org.apache.abdera.Abdera; +import org.atomhopper.adapter.request.adapter.GetEntryRequest; +import org.atomhopper.adapter.request.adapter.GetFeedRequest; +import org.atomhopper.dynamodb.adapter.DynamoDBFeedSource; +import org.atomhopper.dynamodb.adapter.DynamoFeedDBPublisher; +import org.atomhopper.dynamodb.model.PersistedEntry; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.runners.MockitoJUnitRunner; +import java.util.*; +import static org.junit.Assert.assertEquals; +import static org.mockito.Matchers.any; +import static org.mockito.Mockito.*; + +@RunWith(MockitoJUnitRunner.class) +public class DynamoDBFeedSourceTest { + @Rule + public ExpectedException expectedException = ExpectedException.none(); + @Mock + private DynamoDBMapper dynamoDBMapper; + @Mock + private DynamoDB dynamoDB; + @Mock + private AmazonDynamoDBClient amazonDynamoDBClient; + @Mock + private PaginatedQueryList paginatedQueryList; + private DynamoFeedDBPublisher dynamoFeedDBPublisher = new DynamoFeedDBPublisher(); + private GetFeedRequest getFeedRequest; + private DynamoDBFeedSource dynamoDBFeedSource; + private GetEntryRequest getEntryRequest; + private PersistedEntry persistedEntry; + private List entryList; + private List emptyList; + private Abdera abdera; + private final String MARKER_ID = "101"; + private final String ENTRY_BODY = ""; + private final String FEED_NAME = "namespace/feed"; + private final String FORWARD = "forward"; + private final String BACKWARD = "backward"; + private final String SINGLE_CAT = "+Cat1"; + private final String MULTI_CAT = "+Cat1+Cat2"; + private final String AND_CAT = "(AND(cat=cat1)(cat=cat2))"; + private final String OR_CAT = "(OR(cat=cat1)(cat=cat2))"; + private final String NOT_CAT = "(NOT(cat=CAT1))"; + private final String MOCK_LAST_MARKER = "last"; + private final String NEXT_ARCHIVE = "next-archive"; + private final String ARCHIVE_LINK = "http://archive.com/namespace/feed/archive"; + private final String CURRENT = "current"; + + + @Before + public void setUp() throws Exception { + dynamoDBFeedSource = new DynamoDBFeedSource(amazonDynamoDBClient, dynamoDBMapper); + dynamoFeedDBPublisher.setDynamoDBClient(amazonDynamoDBClient); + dynamoFeedDBPublisher.setDynamoMapper(dynamoDBMapper); + dynamoDBFeedSource.setDynamoDB(dynamoDB); + persistedEntry = new PersistedEntry(); + persistedEntry.setFeed(FEED_NAME); + persistedEntry.setEntryId(MARKER_ID); + persistedEntry.setEntryBody(ENTRY_BODY); + + emptyList = new ArrayList(); + + entryList = new ArrayList(); + entryList.add(persistedEntry); + + // Mocks + abdera = mock(Abdera.class); + getFeedRequest = mock(GetFeedRequest.class); + getEntryRequest = mock(GetEntryRequest.class); + + // Mock GetEntryRequest + when(getEntryRequest.getFeedName()).thenReturn(FEED_NAME); + when(getEntryRequest.getEntryId()).thenReturn(MARKER_ID); + + //Mock GetFeedRequest + when(getFeedRequest.getFeedName()).thenReturn(FEED_NAME); + when(getFeedRequest.getPageSize()).thenReturn("25"); + when(getFeedRequest.getAbdera()).thenReturn(abdera); + when(getFeedRequest.getDirection()).thenReturn(FORWARD); + + } + + @Test(expected = IllegalArgumentException.class) + public void shouldThrowExceptionForPrefixColumnMap() throws Exception { + + dynamoDBFeedSource.setDelimiter(":"); + dynamoDBFeedSource.afterPropertiesSet(); + } + + @Test(expected = IllegalArgumentException.class) + public void shouldThrowExceptionForDelimiter() throws Exception { + + Map map = new HashMap(); + map.put("test1", "testA"); + + dynamoDBFeedSource.setPrefixColumnMap(map); + dynamoDBFeedSource.afterPropertiesSet(); + } + + @Test(expected = RuntimeException.class) + public void shouldReturnFeedWithCorrectTimeStampForForwardDirection() throws Exception { + final Table mockTable = mock(Table.class); + when(dynamoDB.getTable(any(String.class))).thenReturn(mockTable); + final Index mockIndex = mock(Index.class); + when(mockTable.getIndex(anyString())).thenReturn(mockIndex); + final ItemCollection outcome = mock(ItemCollection.class); + when(mockIndex.query(any(QuerySpec.class))).thenReturn(outcome); + final IteratorSupport mockIterator = mock(IteratorSupport.class); + final Item mockItem = new Item(); + when(outcome.iterator()).thenReturn(mockIterator); + when(mockIterator.hasNext()).thenReturn(false); + when(mockIterator.next()).thenReturn(mockItem); + when(getFeedRequest.getDirection()).thenReturn(BACKWARD); + dynamoDBFeedSource.getFeedPageByTimestamp(getFeedRequest, "2014-03-10T00:00:00.000Z", 25); + } + + @Test + public void shouldReturnFeedWithCorrectTimeStampForBackwardDirection() throws Exception { + final Table mockTable = mock(Table.class); + when(dynamoDB.getTable(any(String.class))).thenReturn(mockTable); + final Index mockIndex = mock(Index.class); + when(mockTable.getIndex(anyString())).thenReturn(mockIndex); + final ItemCollection outcome = mock(ItemCollection.class); + when(mockIndex.query(any(QuerySpec.class))).thenReturn(outcome); + final IteratorSupport mockIterator = mock(IteratorSupport.class); + final Item mockItem = new Item(); + when(outcome.iterator()).thenReturn(mockIterator); + when(mockIterator.hasNext()).thenReturn(true, false); + when(mockIterator.next()).thenReturn(mockItem); + assertEquals(1, + dynamoDBFeedSource.getFeedPageByTimestamp(getFeedRequest, "2014-03-10T00:00:00.000Z", 25).size()); + } + + @Test(expected = IllegalArgumentException.class) + public void shouldThrowExceptionForFeedWithInCorrectTimeStampForForwardDirection() throws Exception { + final Table mockTable = mock(Table.class); + when(dynamoDB.getTable(any(String.class))).thenReturn(mockTable); + final Index mockIndex = mock(Index.class); + when(mockTable.getIndex(anyString())).thenReturn(mockIndex); + final ItemCollection outcome = mock(ItemCollection.class); + when(mockIndex.query(any(QuerySpec.class))).thenReturn(outcome); + final IteratorSupport mockIterator = mock(IteratorSupport.class); + final Item mockItem = new Item(); + when(outcome.iterator()).thenReturn(mockIterator); + when(mockIterator.hasNext()).thenReturn(true, false); + when(mockIterator.next()).thenReturn(mockItem); + dynamoDBFeedSource.getFeedPageByTimestamp(getFeedRequest, "20140306T060000", 25).size(); + } + + + @Test(expected = UnsupportedOperationException.class) + public void shouldSetParameters() throws Exception { + Map map = new HashMap(); + map.put("test1", "test2"); + dynamoDBFeedSource.setParameters(map); + + + } + +} + diff --git a/pom.xml b/pom.xml index 630107c5..38b9ca98 100644 --- a/pom.xml +++ b/pom.xml @@ -40,6 +40,7 @@ atomhopper test-suite documentation + adapters/dynamoDB_adapters From e49050c6576acf5afcb312bf584150acbba67d54 Mon Sep 17 00:00:00 2001 From: shub6691 Date: Tue, 7 Sep 2021 13:48:48 +0530 Subject: [PATCH 02/21] Added the query implementation for pagination,marker --- adapters/dynamoDB_adapters/pom.xml | 5 - ...lisher.java => DynamoDBFeedPublisher.java} | 6 +- .../dynamodb/adapter/DynamoDBFeedSource.java | 816 +++++++++++++++++- .../dynamodb/constant/DynamoDBConstant.java | 17 +- .../dynamodb/model/PersistedEntry.java | 16 +- .../atomhopper/dynamodb/query/JsonUtil.java | 62 ++ .../dynamodb/query/SQLToNoSqlConverter.java | 5 - atomhopper/pom.xml | 13 + .../webapp/META-INF/application-context.xml | 194 +++-- pom.xml | 8 + 10 files changed, 1038 insertions(+), 104 deletions(-) rename adapters/dynamoDB_adapters/src/main/java/org/atomhopper/dynamodb/adapter/{DynamoFeedDBPublisher.java => DynamoDBFeedPublisher.java} (98%) create mode 100644 adapters/dynamoDB_adapters/src/main/java/org/atomhopper/dynamodb/query/JsonUtil.java diff --git a/adapters/dynamoDB_adapters/pom.xml b/adapters/dynamoDB_adapters/pom.xml index c0e21f5f..a5806f94 100644 --- a/adapters/dynamoDB_adapters/pom.xml +++ b/adapters/dynamoDB_adapters/pom.xml @@ -74,11 +74,6 @@ org.mockito mockito-all - - org.atomhopper.adapter - jdbc-adapter - test - diff --git a/adapters/dynamoDB_adapters/src/main/java/org/atomhopper/dynamodb/adapter/DynamoFeedDBPublisher.java b/adapters/dynamoDB_adapters/src/main/java/org/atomhopper/dynamodb/adapter/DynamoDBFeedPublisher.java similarity index 98% rename from adapters/dynamoDB_adapters/src/main/java/org/atomhopper/dynamodb/adapter/DynamoFeedDBPublisher.java rename to adapters/dynamoDB_adapters/src/main/java/org/atomhopper/dynamodb/adapter/DynamoDBFeedPublisher.java index 28184fc4..b283c3a2 100644 --- a/adapters/dynamoDB_adapters/src/main/java/org/atomhopper/dynamodb/adapter/DynamoFeedDBPublisher.java +++ b/adapters/dynamoDB_adapters/src/main/java/org/atomhopper/dynamodb/adapter/DynamoDBFeedPublisher.java @@ -38,8 +38,8 @@ * This class is used to publish the records in dynamodb using EntryId,feedName . * Index has been created on the feedName for fast search and getting the entry from the DynamoDb. */ -public class DynamoFeedDBPublisher implements FeedPublisher { - private static final Logger LOG = LoggerFactory.getLogger(DynamoFeedDBPublisher.class); +public class DynamoDBFeedPublisher implements FeedPublisher { + private static final Logger LOG = LoggerFactory.getLogger(DynamoDBFeedPublisher.class); private AmazonDynamoDBClient dynamoDBClient; private DynamoDBMapper mapper; private DynamoDB dynamoDB; @@ -195,7 +195,7 @@ private void incrementCounterForFeed(String feedName) { if (!counterMap.containsKey(feedName)) { synchronized (counterMap) { if (!counterMap.containsKey(feedName)) { - Counter counter = Metrics.newCounter(DynamoFeedDBPublisher + Counter counter = Metrics.newCounter(DynamoDBFeedPublisher .class, "entries-created-for-" + feedName); counterMap.put(feedName, counter); } diff --git a/adapters/dynamoDB_adapters/src/main/java/org/atomhopper/dynamodb/adapter/DynamoDBFeedSource.java b/adapters/dynamoDB_adapters/src/main/java/org/atomhopper/dynamodb/adapter/DynamoDBFeedSource.java index 8743a802..7738d927 100644 --- a/adapters/dynamoDB_adapters/src/main/java/org/atomhopper/dynamodb/adapter/DynamoDBFeedSource.java +++ b/adapters/dynamoDB_adapters/src/main/java/org/atomhopper/dynamodb/adapter/DynamoDBFeedSource.java @@ -7,23 +7,47 @@ import com.amazonaws.services.dynamodbv2.document.spec.QuerySpec; import com.amazonaws.services.dynamodbv2.document.utils.ValueMap; import com.amazonaws.services.dynamodbv2.model.AttributeValue; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.gson.Gson; +import com.yammer.metrics.Metrics; +import com.yammer.metrics.core.TimerContext; +import org.apache.abdera.Abdera; +import org.apache.abdera.model.Document; +import org.apache.abdera.model.Entry; +import org.apache.abdera.model.Feed; +import org.apache.abdera.model.Link; import org.apache.commons.lang.StringUtils; import org.atomhopper.adapter.*; +import org.atomhopper.adapter.request.adapter.GetEntryRequest; import org.atomhopper.adapter.request.adapter.GetFeedRequest; import org.atomhopper.dbal.PageDirection; +import org.atomhopper.dynamodb.constant.DynamoDBConstant; import org.atomhopper.dynamodb.model.PersistedEntry; +import org.atomhopper.dynamodb.query.JsonUtil; import org.atomhopper.dynamodb.query.SQLToNoSqlConverter; import org.atomhopper.dynamodb.query.SearchType; import org.atomhopper.dynamodb.query.DynamoDBQueryBuilder; +import org.atomhopper.response.AdapterResponse; +import org.atomhopper.util.uri.template.EnumKeyedTemplateParameters; +import org.atomhopper.util.uri.template.URITemplate; import org.joda.time.DateTime; import org.joda.time.format.DateTimeFormatter; import org.joda.time.format.ISODateTimeFormat; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.StringReader; +import java.io.UnsupportedEncodingException; import java.net.URL; +import java.net.URLEncoder; +import java.text.ParseException; import java.text.SimpleDateFormat; import java.util.*; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static org.apache.abdera.i18n.text.UrlEncoding.decode; /** * @author shub6691 @@ -43,7 +67,7 @@ * value would be 'tid:1234' * */ -public class DynamoDBFeedSource { +public class DynamoDBFeedSource implements FeedSource { static Logger LOG = LoggerFactory.getLogger(DynamoDBFeedSource.class); private AmazonDynamoDBClient amazonDynamoDBClient; @@ -64,6 +88,11 @@ private SQLToNoSqlConverter getSearchToSqlConverter() { return new SQLToNoSqlConverter(mapPrefix, split); } + + public Boolean getEnableLoggingOnShortPage() { + return enableLoggingOnShortPage; + } + public void setPrefixColumnMap(Map prefix) { mapPrefix = new HashMap(prefix); @@ -109,11 +138,12 @@ public void setDynamoDB(DynamoDB dynamoDB) { /** * This method is used to return the categories based search with the help of feed and entryId and search type is backward * whch means it will search for date less the markerDate along with other params. - * @param feedName: Name of the feed for each entry . For ex: namespace/feed + * + * @param feedName: Name of the feed for each entry . For ex: namespace/feed * @param markerTimestamp: Timestamp for which the search is to be performed for category - * @param markerId: EntryID for every event. - * @param searchString: The string on which search is performed in db - * @param pageSize: default size is 25 + * @param markerId: EntryID for every event. + * @param searchString: The string on which search is performed in db + * @param pageSize: default size is 25 * @return List of PersistedEntry data found based on above params from db. */ public List getFeedBackward(String feedName, @@ -127,14 +157,14 @@ public List getFeedBackward(String feedName, //Dynamodb query implementation Map map = new HashMap(); String filters = sqlBac.getFilters(map); - String feedNameFilter=" and feed= :feedName"; + String feedNameFilter = " and feed= :feedName"; SimpleDateFormat dateFormatter = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'"); dateFormatter.setTimeZone(TimeZone.getTimeZone("UTC")); String dateLastUpdated = dateFormatter.format(markerTimestamp); Map valueMap = new HashMap(); valueMap.put(":id", new AttributeValue().withS(String.valueOf(markerId))); valueMap.put(":dateLastUpdated", new AttributeValue().withS(dateLastUpdated)); - valueMap.put(":feedName",new AttributeValue().withS(feedName)); + valueMap.put(":feedName", new AttributeValue().withS(feedName)); for (Map.Entry res : map.entrySet()) { valueMap.put(res.getKey(), new AttributeValue().withS(res.getValue())); } @@ -148,16 +178,62 @@ public List getFeedBackward(String feedName, return feedPage; } + + /** + * This method is used to return the categories based search with the help of feed and entryId and search type is forward + * whch means it will search for date less the markerDate along with other params. + * + * @param feedName: Name of the feed for each entry . For ex: namespace/feed + * @param markerTimestamp: Timestamp for which the search is to be performed for category + * @param markerId: EntryID for every event. + * @param searchString: The string on which search is performed in db + * @param pageSize: default size is 25 + * @return List of PersistedEntry data found based on above params from db. + */ + public List getFeedForward(String feedName, + Date markerTimestamp, + long markerId, + String searchString, + int pageSize) { + List feedPage; + DynamoDBQueryBuilder sqlBac = new DynamoDBQueryBuilder(getSearchToSqlConverter()).searchString(searchString); + sqlBac.searchType(SearchType.FEED_BACKWARD); + //Dynamodb query implementation + Map map = new HashMap(); + String filters = sqlBac.getFilters(map); + String feedNameFilter = " and feed= :feedName"; + SimpleDateFormat dateFormatter = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'"); + dateFormatter.setTimeZone(TimeZone.getTimeZone("UTC")); + String dateLastUpdated = dateFormatter.format(markerTimestamp); + Map valueMap = new HashMap(); + valueMap.put(":id", new AttributeValue().withS(String.valueOf(markerId))); + valueMap.put(":dateLastUpdated", new AttributeValue().withS(dateLastUpdated)); + valueMap.put(":feedName", new AttributeValue().withS(feedName)); + for (Map.Entry res : map.entrySet()) { + valueMap.put(res.getKey(), new AttributeValue().withS(res.getValue())); + } + DynamoDBQueryExpression querySpec = new DynamoDBQueryExpression() + .withKeyConditionExpression("entryId = :id and dateLastUpdated >= :dateLastUpdated") + .withScanIndexForward(false) + .withLimit(pageSize) + .withFilterExpression(filters + feedNameFilter) + .withExpressionAttributeValues(valueMap); + feedPage = mapper.query(PersistedEntry.class, querySpec); + return feedPage; + } + /** * This method is used to return list of entries from DynamoDb bases on feedName and timestamp. - * @param getFeedRequest: It contains the request object parameters - * @param startingAt: The timestamp from where we need to perform search - * @param pageSize: for pagination . Default page size is 25 + * + * @param getFeedRequest : It contains the request object parameters + * @param startingAt : The timestamp from where we need to perform search + * @param pageSize : for pagination . Default page size is 25 * @return List of entries found based on given timestamp ,feed name. */ //TODO LINK REF NEED TO BE IMPLEMENTED FOR HYDRATED FEED. - public List getFeedPageByTimestamp(GetFeedRequest getFeedRequest, String startingAt, int pageSize) throws Exception { + public AdapterResponse getFeedPageByTimestamp(GetFeedRequest getFeedRequest, String startingAt, int pageSize) throws Exception { final String pageDirectionValue = getFeedRequest.getDirection(); + ObjectMapper mapper = new ObjectMapper(); PageDirection pageDirection = PageDirection.FORWARD; if (StringUtils.isNotEmpty(String.valueOf(pageDirection))) { pageDirection = PageDirection.valueOf(pageDirectionValue.toUpperCase()); @@ -169,27 +245,698 @@ public List getFeedPageByTimestamp(GetFeedRequest getFeedRequest, String if (entryMarker.isEmpty()) { throw new RuntimeException("No entry with specified startingAt timestamp found"); } - return entryMarker; + // This list contains the persistent object in json string format + PersistedEntry persistedEntry = mapper.readValue(entryMarker.get(0), PersistedEntry.class); + //Convert String to Date Format + Date lastDateUpdated = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.S").parse(persistedEntry.getDateLastUpdated()); + final Feed feed = hydrateFeed(getFeedRequest.getAbdera(), + enhancedGetFeedPage(getFeedRequest.getFeedName(), + lastDateUpdated, + Long.parseLong(persistedEntry.getEntryId()), + pageDirection, + searchString, pageSize), + getFeedRequest, pageSize); + return ResponseBuilder.found(feed); } + /** * This method creates the query for fetching the data based on timestamp and the direction from DynamoDB + * * @param * @param markerDate: StartAt Must be in ISO 8601 Date and Time format, and must contain a time zone, - * for example: 2014-03-10T06:00:00.000Z. For more information, see ISO 8601 Date and Time format. - * @param direction: Specifies the direction from which to return entries, starting from the current marker or entry. - * Can be either forward or backward. + * for example: 2014-03-10T06:00:00.000Z. For more information, see ISO 8601 Date and Time format. + * @param direction: Specifies the direction from which to return entries, starting from the current marker or entry. + * Can be either forward or backward. * @return List of data present based on the search query. */ protected List getEntryByTimestamp(final DateTime markerDate, final String feed, PageDirection direction) { - List feedPage = new ArrayList(); + ValueMap valueMap = new ValueMap(); + valueMap.withString(":feed", feed); + valueMap.withString(":markerDate", String.valueOf(markerDate)); + return getQueryBuilderMethod(dynamoDB, "feed = :feed and " + getTimeStampValueFilter(direction), valueMap); + } + + /** + * This method is used to create a query for dynamodb for timestamp search based on direction + * + * @param direction: If the startingAt parameter is used without a direction parameter, then the forward direction is assumed. + * If you want to fetch feeds from a time period before the time specified in the time stamp, + * you need to use the direction parameter and then the backward description, like the following: direction set to backward. + * @return : the formed query based upon the direction. + */ + private String getTimeStampValueFilter(PageDirection direction) { + if (direction.equals(PageDirection.BACKWARD)) { + return "dateLastUpdated <= :markerDate"; + } else { + return "dateLastUpdated >= :markerDate"; + } + } + + @Override + public FeedInformation getFeedInformation() { + throw new UnsupportedOperationException("Not supported yet."); + } + + /** + * This method returns the search result based on the type of operation like search getFeedHead,getFeedPageByTimestamp ect + * + * @param getFeedRequest: all the request type properties mentioned below ; + * List getCategories(); + * String getSearchQuery(); + * String getPageMarker(); + * String getPageSize(); + * String getDirection(); + * String getStartingAt(); + * @return + */ + @Override + public AdapterResponse getFeed(GetFeedRequest getFeedRequest) { + AdapterResponse response = null; + + TimerContext context = null; + int pageSize = DynamoDBConstant.PAGE_SIZE; + final String pageSizeString = getFeedRequest.getPageSize(); + if (StringUtils.isNotBlank(pageSizeString)) { + pageSize = Integer.parseInt(pageSizeString); + } + final String marker = getFeedRequest.getPageMarker(); + final String startingAt = getFeedRequest.getStartingAt(); + if (StringUtils.isNotBlank(marker) && StringUtils.isNotBlank(startingAt)) { + response = ResponseBuilder.badRequest("'marker' parameter can not be used together with the 'startingAt' parameter"); + return response; + } + + try { + + if (StringUtils.isBlank(marker) && StringUtils.isBlank(startingAt)) { + context = startTimer(String.format("get-feed-head-%s", getMetricBucketForPageSize(pageSize))); + response = getFeedHead(getFeedRequest, pageSize); + } else if (StringUtils.isNotBlank(marker) && marker.equals(DynamoDBConstant.MOCK_LAST_MARKER)) { + context = startTimer(String.format("get-last-page-%s", getMetricBucketForPageSize(pageSize))); + response = getLastPage(getFeedRequest, pageSize); + } else if (StringUtils.isNotBlank(marker)) { + context = startTimer(String.format("get-feed-page-%s", getMetricBucketForPageSize(pageSize))); + response = getFeedPage(getFeedRequest, marker, pageSize); + } else { + // we process 'startingAt' parameter here + context = startTimer(String.format("get-feed-page-startingAt-%s", getMetricBucketForPageSize(pageSize))); + response = getFeedPageByTimestamp(getFeedRequest, startingAt, pageSize); + } + } catch (IllegalArgumentException iae) { + response = ResponseBuilder.badRequest(iae.getMessage()); + } catch (Exception e) { + e.printStackTrace(); + } finally { + stopTimer(context); + } + + return response; + } + + @Override + public AdapterResponse getEntry(GetEntryRequest getEntryRequest) { + final List entry = getEntry(getEntryRequest.getEntryId(), getEntryRequest.getFeedName()); + + AdapterResponse response = ResponseBuilder.notFound(); + + if (!entry.isEmpty()) { + response = ResponseBuilder.found(hydrateEntry(entry.get(0), getEntryRequest.getAbdera())); + } + return response; + } + + /** + * This method returns the feed page based on the entryId for every event + * + * @param getFeedRequest + * @param marker: entry id fo every feed + * @param pageSize: default is 25,for pagination + * @throws ParseException + * @return: Response as a Http response + */ + + private AdapterResponse getFeedPage(GetFeedRequest getFeedRequest, String marker, int pageSize) throws ParseException { + + final String pageDirectionValue = getFeedRequest.getDirection(); + PageDirection pageDirection = PageDirection.FORWARD; + if (StringUtils.isNotEmpty(pageDirectionValue)) { + pageDirection = PageDirection.valueOf(pageDirectionValue.toUpperCase()); + } + + final String searchString = getFeedRequest.getSearchQuery() != null ? getFeedRequest.getSearchQuery() : ""; + + List entryMarker = getEntry(marker, getFeedRequest.getFeedName()); + if (entryMarker.isEmpty()) { + return ResponseBuilder.notFound("No entry with specified marker found"); + } + Date lastDateUpdated = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.S").parse(entryMarker.get(0).getDateLastUpdated()); + final Feed feed = hydrateFeed(getFeedRequest.getAbdera(), + enhancedGetFeedPage(getFeedRequest.getFeedName(), + lastDateUpdated, + Long.parseLong(entryMarker.get(0).getEntryId()), + pageDirection, + searchString, pageSize), + getFeedRequest, pageSize); + return ResponseBuilder.found(feed); + } + + /** + * This method is used to return the feed based on direction weather is a forward or backward + * + * @param feedName: name of feed + * @param markerTimestamp: for which time stamp it need to be searched + * @param markerId: entry id for every feed + * @param direction: it can be forward or backward + * @param searchString: category search string + * @param pageSize: default is 25 ,used for pagination + * @return: list of PersistedEntry object . + */ + private List enhancedGetFeedPage(final String feedName, final Date markerTimestamp, + final long markerId, + final PageDirection direction, final String searchString, + final int pageSize) { + + List feedPage = new LinkedList(); + + TimerContext context = null; + + boolean hasCats = !searchString.trim().isEmpty(); + + try { + switch (direction) { + case FORWARD: + + + if (hasCats) { + context = startTimer(String.format("db-get-feed-page-forward-with-cats-%s", + getMetricBucketForPageSize(pageSize))); + } else { + context = startTimer( + String.format("db-get-feed-page-forward-%s", getMetricBucketForPageSize(pageSize))); + } + feedPage = getFeedForward(feedName, + markerTimestamp, + markerId, + searchString, + pageSize); + + Collections.reverse(feedPage); + break; + + case BACKWARD: + + + if (hasCats) { + context = startTimer(String.format("db-get-feed-page-backward-with-cats-%s", + getMetricBucketForPageSize(pageSize))); + } else { + context = startTimer( + String.format("db-get-feed-page-backward-%s", getMetricBucketForPageSize(pageSize))); + } + feedPage = getFeedBackward(feedName, + markerTimestamp, + markerId, + searchString, + pageSize); + break; + } + } finally { + stopTimer(context); + } + + return feedPage; + } + + /** + * This method is used to return the last page of the particular feed based on entryID + * + * @param getFeedRequest: Its has all required feed request objects + * @param pageSize: Page size for pagination ,default size is 25 + * @return + */ + + private AdapterResponse getLastPage(GetFeedRequest getFeedRequest, int pageSize) { + + final String searchString = getFeedRequest.getSearchQuery() != null ? getFeedRequest.getSearchQuery() : ""; + AdapterResponse response; + + final Feed feed = hydrateFeed(getFeedRequest.getAbdera(), + enhancedGetLastPage(getFeedRequest.getFeedName(), pageSize, searchString), + getFeedRequest, pageSize); + response = ResponseBuilder.found(feed); + + return response; + } + + /** + * This method returns the Last Page of the feed by performing union of both the select statements output + * + * @param feedName: FeedName + * @param pageSize: for pagination + * @param searchString: Search Category to be passed + * @return List of persistent Object for union results + */ + private List enhancedGetLastPage(final String feedName, final int pageSize, + final String searchString) { + + List categoriesList = getSearchToSqlConverter().getParamsFromSearchString(searchString); + int numCats = categoriesList.size(); + + Object[] parms = null; + + if (numCats > 0) { + parms = new Object[numCats + 2]; + int index = 0; + parms[index++] = feedName; + for (String s : categoriesList) { + parms[index++] = s; + } + parms[index++] = pageSize; + + } else { + parms = new Object[]{feedName, pageSize}; + } + + TimerContext context = null; + List lastPersistedEntries; + try { + if (numCats > 0) { + context = startTimer( + String.format("db-get-last-page-with-cats-%s", getMetricBucketForPageSize(pageSize))); + } else { + context = startTimer(String.format("db-get-last-page-%s", getMetricBucketForPageSize(pageSize))); + } + + List feedPage; + ValueMap valueMap = new ValueMap(); + valueMap.withString(":feed", feedName); + valueMap.withString(":dateLastUpdated", JsonUtil.getCurrentDateWithMinusSecond(2)); + feedPage = getQueryBuilderMethod(dynamoDB, "feed = :feed and dateLastUpdated < :dateLastUpdated", pageSize, valueMap, true); + // comparator is written to perform sorting based on if two dates are equal then sort output based on entryId in desc order + List persistedEntryList = JsonUtil.getPersistenceEntity(feedPage); + Collections.sort(persistedEntryList, (a, b) -> { + SimpleDateFormat dateFormatter = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'"); + String dateLastUpdated = dateFormatter.format(a.getDateLastUpdated()); + String dateLastUpdatedNextDate = dateFormatter.format(b.getDateLastUpdated()); + if (dateLastUpdatedNextDate.equals(dateLastUpdated)) { + return b.getEntryId().compareTo(a.getEntryId()); + } + return -1; + }); + + } finally { + stopTimer(context); + } + return null; + } + + /** + * @param getFeedRequest + * @param pageSize + * @return + */ + private AdapterResponse getFeedHead(GetFeedRequest getFeedRequest, int pageSize) { + final Abdera abdera = getFeedRequest.getAbdera(); + + final String searchString = getFeedRequest.getSearchQuery() != null ? getFeedRequest.getSearchQuery() : ""; + + List persistedEntries = getFeedHead(getFeedRequest.getFeedName(), pageSize, searchString); + + Feed hydratedFeed = hydrateFeed(abdera, persistedEntries, getFeedRequest, pageSize); + + // Set the last link in the feed head + final String baseFeedUri = decode(getFeedRequest.urlFor( + new EnumKeyedTemplateParameters(URITemplate.FEED))); + + if (!helper.isArchived()) { + + hydratedFeed.addLink( + new StringBuilder().append(baseFeedUri) + .append(DynamoDBConstant.MARKER_EQ).append(DynamoDBConstant.MOCK_LAST_MARKER) + .append(DynamoDBConstant.AND_LIMIT_EQ).append(String.valueOf(pageSize)) + .append(DynamoDBConstant.AND_SEARCH_EQ).append(urlEncode(searchString)) + .append(DynamoDBConstant.AND_DIRECTION_EQ_BACKWARD).toString()) + .setRel(Link.REL_LAST); + } + + return ResponseBuilder.found(hydratedFeed); + + } + + private String getMetricBucketForPageSize(int pageSize) { + if (pageSize > 0 && pageSize <= 249) { + return "tiny"; + } else if (pageSize >= 250 && pageSize <= 499) { + return "small"; + } else if (pageSize >= 500 && pageSize <= 749) { + return "medium"; + } else { + // 750 - 1000 + return "large"; + } + } + + /** This method is used to return the feed head which means the starting feed for the particular entry + * @param feedName: name of the feed + * @param pageSize: default is 25, for pagination + * @param searchString: Search string to be used to filteration in the query + * @return List of all the output of PersistedEntry + */ + private List getFeedHead(final String feedName, final int pageSize, final String searchString) { + + List categoriesList = getSearchToSqlConverter().getParamsFromSearchString(searchString); + int numCats = categoriesList.size(); + + Object[] parms = null; + + if (numCats > 0) { + parms = new Object[numCats + 2]; + int index = 0; + parms[index++] = feedName; + for (String s : categoriesList) { + parms[index++] = s; + } + parms[index++] = pageSize; + } else { + parms = new Object[]{feedName, pageSize}; + } + + TimerContext context = null; + try { + if (numCats > 0) { + context = startTimer( + String.format("db-get-feed-head-with-cats-%s", getMetricBucketForPageSize(pageSize))); + } else { + context = startTimer(String.format("db-get-feed-head-%s", getMetricBucketForPageSize(pageSize))); + } + + List feedPage; + ValueMap valueMap = new ValueMap(); + valueMap.withString(":feed", feedName); + valueMap.withString(":dateLastUpdated", JsonUtil.getCurrentDateWithMinusSecond(2)); + feedPage = getQueryBuilderMethod(dynamoDB, "feed = :feed and dateLastUpdated < :dateLastUpdated", pageSize, valueMap, false); + List persistedEntryList = JsonUtil.getPersistenceEntity(feedPage); + Collections.sort(persistedEntryList, (a, b) -> { + SimpleDateFormat dateFormatter = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'"); + String dateLastUpdated = dateFormatter.format(a.getDateLastUpdated()); + String dateLastUpdatedNextDate = dateFormatter.format(b.getDateLastUpdated()); + if (dateLastUpdatedNextDate.equals(dateLastUpdated)) { + return b.getEntryId().compareTo(a.getEntryId()); + } + return -1; + }); + return persistedEntryList; + } finally { + stopTimer(context); + } + } + + @Override + public void setCurrentUrl(URL urlCurrent) { + helper.setCurrentUrl(urlCurrent); + } + + @Override + public void setArchiveUrl(URL url) { + helper.setArchiveUrl(url); + } + + private TimerContext startTimer(String name) { + if (enableTimers) { + final com.yammer.metrics.core.Timer timer = Metrics.newTimer(getClass(), name, TimeUnit.MILLISECONDS, + TimeUnit.SECONDS); + TimerContext context = timer.time(); + return context; + } else { + return null; + } + + + } + + private Feed hydrateFeed(Abdera abdera, List persistedEntries, + GetFeedRequest getFeedRequest, final int pageSize) { + + final Feed hydratedFeed = abdera.newFeed(); + final String baseFeedUri = decode(getFeedRequest.urlFor( + new EnumKeyedTemplateParameters(URITemplate.FEED))); + final String searchString = getFeedRequest.getSearchQuery() != null ? getFeedRequest.getSearchQuery() : ""; + + if (helper.isArchived()) { + + helper.addArchiveNode(hydratedFeed); + } + + // Set the feed links + addFeedCurrentLink(hydratedFeed, baseFeedUri); + addFeedSelfLink(hydratedFeed, baseFeedUri, getFeedRequest, pageSize, searchString); + + + PersistedEntry nextEntry = null; + + // TODO: We should have a link builder method for these + if (!(persistedEntries.isEmpty())) { + hydratedFeed.setId(DynamoDBConstant.UUID_URI_SCHEME + UUID.randomUUID().toString()); + hydratedFeed.setTitle(persistedEntries.get(0).getFeed()); + + // Set the previous link + hydratedFeed.addLink(new StringBuilder() + .append(baseFeedUri).append(DynamoDBConstant.MARKER_EQ) + .append(persistedEntries.get(0).getEntryId()) + .append(DynamoDBConstant.AND_LIMIT_EQ).append(String.valueOf(pageSize)) + .append(DynamoDBConstant.AND_SEARCH_EQ).append(urlEncode(searchString)) + .append(DynamoDBConstant.AND_DIRECTION_EQ_FORWARD).toString()) + .setRel(helper.getPrevLink()); + + final PersistedEntry lastEntryInCollection = persistedEntries.get(persistedEntries.size() - 1); + + nextEntry = getNextMarker(lastEntryInCollection, getFeedRequest.getFeedName(), searchString); + + if (nextEntry != null) { + // Set the next link + hydratedFeed.addLink(new StringBuilder().append(baseFeedUri) + .append(DynamoDBConstant.MARKER_EQ).append(nextEntry.getEntryId()) + .append(DynamoDBConstant.AND_LIMIT_EQ).append(String.valueOf(pageSize)) + .append(DynamoDBConstant.AND_SEARCH_EQ).append(urlEncode(searchString)) + .append(DynamoDBConstant.AND_DIRECTION_EQ_BACKWARD).toString()) + .setRel(helper.getNextLink()); + } + } + + if (nextEntry == null && helper.getArchiveUrl() != null) { + hydratedFeed.addLink(new StringBuilder().append(helper.getArchiveUrl()).append(DynamoDBConstant.LIMIT_EQ).append(String.valueOf(pageSize)) + .append(DynamoDBConstant.AND_DIRECTION_EQ_BACKWARD).toString()) + .setRel(FeedSource.REL_ARCHIVE_NEXT); + } + + for (PersistedEntry persistedFeedEntry : persistedEntries) { + hydratedFeed.addEntry(hydrateEntry(persistedFeedEntry, abdera)); + } + + if (getEnableLoggingOnShortPage()) { + if (hydratedFeed.getEntries() != null && hydratedFeed.getEntries().size() < pageSize) { + LOG.warn("User requested " + getFeedRequest.getFeedName() + " feed with limit " + pageSize + ", but returning only " + hydratedFeed.getEntries().size()); + List entries = hydratedFeed.getEntries(); + StringBuilder sb = new StringBuilder(); + for (int idx = 0; idx < entries.size(); idx++) { + Entry entry = entries.get(idx); + sb.append(entry.getId() + ", "); + } + LOG.warn("UUIDs: " + sb.toString()); + } else if (hydratedFeed.getEntries() == null) { + LOG.warn("User requested " + getFeedRequest.getFeedName() + " feed with limit " + pageSize + ", but no entries are available"); + } + } + + return hydratedFeed; + } + + /** + * This method is used to get the next marker based for the markerID and the feedName + * + * @param persistedEntry: PersistentEntryModel + * @param feedName: name of the feed + * @param searchString: Category search string for filteration + * @return Persistent model Object + */ + private PersistedEntry getNextMarker(final PersistedEntry persistedEntry, final String feedName, + final String searchString) { + + List categoriesList = getSearchToSqlConverter().getParamsFromSearchString(searchString); + int numCats = categoriesList.size(); + + Object[] parms = null; + + if (categoriesList.size() > 0) { + parms = new Object[numCats * 2 + 5]; + int index = 0; + parms[index++] = feedName; + parms[index++] = persistedEntry.getDateLastUpdated(); + parms[index++] = persistedEntry.getEntryId(); + for (String s : categoriesList) { + parms[index++] = s; + } + parms[index++] = feedName; + parms[index++] = persistedEntry.getDateLastUpdated(); + for (String s : categoriesList) { + parms[index++] = s; + } + + } else { + parms = new Object[]{feedName, persistedEntry.getDateLastUpdated(), persistedEntry.getEntryId(), + feedName, persistedEntry.getDateLastUpdated()}; + } + + List feedPage; + List firstUnionPersistentList; + ValueMap valueMap = new ValueMap(); + valueMap.withString(":feed", persistedEntry.getFeed()); + valueMap.withString(":feed", persistedEntry.getDateLastUpdated()); + valueMap.withString(":entryId", persistedEntry.getEntryId()); + firstUnionPersistentList = getQueryBuilderMethod(dynamoDB, "feed = :feed and dateLastUpdated =:dateLastUpdated and entryId <=:entryId", valueMap); + + + List nextUnionListOfPersistedItems; + ValueMap newValueMap = new ValueMap(); + newValueMap.withString(":feed", persistedEntry.getFeed()); + newValueMap.withString(":dateLastUpdated", persistedEntry.getDateLastUpdated()); + nextUnionListOfPersistedItems = getQueryBuilderMethod(dynamoDB, "feed = :feed and dateLastUpdated < :dateLastUpdated", valueMap); + feedPage = Stream.concat(firstUnionPersistentList.stream(), nextUnionListOfPersistedItems.stream()) + .collect(Collectors.toList()); + List persistedEntryList = JsonUtil.getPersistenceEntity(feedPage); + Collections.sort(persistedEntryList, (a, b) -> { + SimpleDateFormat dateFormatter = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'"); + String dateLastUpdated = dateFormatter.format(a.getDateLastUpdated()); + String dateLastUpdatedNextDate = dateFormatter.format(b.getDateLastUpdated()); + if (dateLastUpdatedNextDate.equals(dateLastUpdated)) { + return b.getEntryId().compareTo(a.getEntryId()); + } + return -1; + }); + return persistedEntryList.get(0); + } + + + private void addFeedSelfLink(Feed feed, String baseFeedUri, GetFeedRequest getFeedRequest, int pageSize, String searchString) { + StringBuilder queryParams = new StringBuilder(); + boolean markerIsSet = false; + + queryParams.append(baseFeedUri).append(DynamoDBConstant.LIMIT_EQ).append( + String.valueOf(pageSize)); + + if (searchString.length() > 0) { + queryParams.append(DynamoDBConstant.AND_SEARCH_EQ).append(urlEncode(searchString)); + } + if (getFeedRequest.getPageMarker() != null && getFeedRequest.getPageMarker().length() > 0) { + queryParams.append(DynamoDBConstant.AND_MARKER_EQ).append(getFeedRequest.getPageMarker()); + markerIsSet = true; + } + if (markerIsSet) { + queryParams.append(DynamoDBConstant.AND_DIRECTION_EQ).append(getFeedRequest.getDirection()); + } else { + queryParams.append(DynamoDBConstant.AND_DIRECTION_EQ_BACKWARD); + if (queryParams.toString().equalsIgnoreCase( + baseFeedUri + DynamoDBConstant.LIMIT_EQ + "25" + DynamoDBConstant.AND_DIRECTION_EQ_BACKWARD)) { + // They are calling the feedhead, just use the base feed uri + // This keeps the validator at http://validator.w3.org/ happy + queryParams.delete(0, queryParams.toString().length()).append( + baseFeedUri); + } + } + feed.addLink(queryParams.toString()).setRel(Link.REL_SELF); + } + + + private void addFeedCurrentLink(Feed hydratedFeed, String baseFeedUri) { + String url = helper.isArchived() ? helper.getCurrentUrl() : baseFeedUri; + + hydratedFeed.addLink(url, Link.REL_CURRENT); + } + + + private Entry hydrateEntry(PersistedEntry persistedEntry, Abdera abderaReference) { + + final Document hydratedEntryDocument = abderaReference.getParser().parse( + new StringReader(persistedEntry.getEntryBody())); + + Entry entry = null; + + if (hydratedEntryDocument != null) { + entry = hydratedEntryDocument.getRoot(); + entry.setUpdated(persistedEntry.getDateLastUpdated()); + entry.setPublished(persistedEntry.getCreationDate()); + } + + return entry; + } + + /** + * @param context + */ + private void stopTimer(TimerContext context) { + if (enableTimers && context != null) { + context.stop(); + } + } + + /** + * @param searchString + * @return + */ + private String urlEncode(String searchString) { + try { + return URLEncoder.encode(searchString, "UTF-8"); + } catch (UnsupportedEncodingException e) { + //noop - should never get here + return ""; + } + } + + /** + * To get the entry from dynamodb based upon two params? + * + * @param entryId: It is the marker id for entry for every events + * @param feedName: feed name is used to search the records in dynamodb + * @return : list of entry found if that exits in dynamodb with the entryId and feedName. + */ + private List getEntry(String entryId, final String feedName) { + Gson gson = new Gson(); + List persistedEntriesObject = new ArrayList(); + Table table = dynamoDB.getTable(DynamoDBConstant.ENTRIES); + Index index = table.getIndex(DynamoDBConstant.ENTRY_ID_FEED_INDEX); + QuerySpec spec = new QuerySpec() + .withKeyConditionExpression("entryId = :entryId and feed = :feed") + .withValueMap(new ValueMap() + .withString(":entryId", entryId) + .withString(":feed", feedName)); + ItemCollection persistedEntryItems = index.query(spec); + Iterator itemsIterator = persistedEntryItems.iterator(); + while (itemsIterator.hasNext()) { + Item item = itemsIterator.next(); + persistedEntriesObject.add(gson.fromJson(item.toJSONPretty(), PersistedEntry.class)); + } + return persistedEntriesObject; + } + + /** + * This method is used to build a query based on the condition expression(for ex with where clause)and other params + * + * @param dynamoDB: object of dynamoDb + * @param conditionExpression: Filtered Exppression based on which results are filtered(where clause conditions) + * @param pageSize: page size for pagination + * @param valueMap: map for keeping the mapped values passed in a condition expression + * @return List of String in json format. + */ + public List getQueryBuilderMethod(DynamoDB dynamoDB, String conditionExpression, int pageSize, ValueMap valueMap, boolean orderBy) { + List feedPage = new ArrayList<>(); Table table = dynamoDB.getTable("entries"); Index index = table.getIndex("global-feed-index"); QuerySpec spec = new QuerySpec() - .withKeyConditionExpression("feed = :feed and " + getTimeStampValueFilter(direction)) - .withValueMap(new ValueMap() - .withString(":feed", feed) - .withString(":markerDate", String.valueOf(markerDate))); + .withKeyConditionExpression(conditionExpression) + .withValueMap(valueMap) + .withMaxPageSize(pageSize)// for no of page limit to be displayed + .withScanIndexForward(orderBy);// for descending order sorting on lastDateUpdated ItemCollection persistedEntryItems = index.query(spec); Iterator itemsIterator = persistedEntryItems.iterator(); while (itemsIterator.hasNext()) { @@ -200,17 +947,26 @@ protected List getEntryByTimestamp(final DateTime markerDate, final Stri } /** - *This method is used to create a query for dynamodb for timestamp search based on direction - * @param direction: If the startingAt parameter is used without a direction parameter, then the forward direction is assumed. - * If you want to fetch feeds from a time period before the time specified in the time stamp, - * you need to use the direction parameter and then the backward description, like the following: direction set to backward. - * @return : the formed query based upon the direction. + * This is a overloaded method is used to build a query based on the condition expression(for ex with where clause)and other params + * + * @param dynamoDB: object of dynamoDb + * @param conditionExpression: Filtered Exppression based on which results are filtered(where clause conditions) + * @param valueMap: map for keeping the mapped values passed in a condition expression + * @return List of String in json format. */ - private String getTimeStampValueFilter(PageDirection direction) { - if (direction.equals(PageDirection.BACKWARD)) { - return "dateLastUpdated <= :markerDate"; - } else { - return "dateLastUpdated >= :markerDate"; + public List getQueryBuilderMethod(DynamoDB dynamoDB, String conditionExpression, ValueMap valueMap) { + List feedPage = new ArrayList<>(); + Table table = dynamoDB.getTable("entries"); + Index index = table.getIndex("global-feed-index"); + QuerySpec spec = new QuerySpec() + .withKeyConditionExpression(conditionExpression) + .withValueMap(valueMap); + ItemCollection persistedEntryItems = index.query(spec); + Iterator itemsIterator = persistedEntryItems.iterator(); + while (itemsIterator.hasNext()) { + Item item = itemsIterator.next(); + feedPage.add(item.toJSONPretty()); } + return feedPage; } } diff --git a/adapters/dynamoDB_adapters/src/main/java/org/atomhopper/dynamodb/constant/DynamoDBConstant.java b/adapters/dynamoDB_adapters/src/main/java/org/atomhopper/dynamodb/constant/DynamoDBConstant.java index fd1dee96..5dc674ee 100644 --- a/adapters/dynamoDB_adapters/src/main/java/org/atomhopper/dynamodb/constant/DynamoDBConstant.java +++ b/adapters/dynamoDB_adapters/src/main/java/org/atomhopper/dynamodb/constant/DynamoDBConstant.java @@ -1,6 +1,8 @@ package org.atomhopper.dynamodb.constant; - +/** + * Constant file for all the constant declared in the module. + */ public final class DynamoDBConstant { public static final String UUID_URI_SCHEME = "urn:uuid:"; public static final String LINK_REL_SELF = "self"; @@ -8,4 +10,17 @@ public final class DynamoDBConstant { public static final String ENTRY_ID_FEED_INDEX = "entryId-feed-index"; public static final String ENTRY_ID = "entryId"; public static final String FEED = "feed"; + public static final int PAGE_SIZE=25; + public static final String MOCK_LAST_MARKER = "last"; + public static final String MARKER_EQ = "?marker="; + public static final String LIMIT_EQ = "?limit="; + public static final String AND_SEARCH_EQ = "&search="; + public static final String AND_LIMIT_EQ = "&limit="; + public static final String AND_MARKER_EQ = "&marker="; + public static final String AND_DIRECTION_EQ = "&direction="; + public static final String AND_DIRECTION_EQ_BACKWARD = "&direction=backward"; + public static final String AND_DIRECTION_EQ_FORWARD = "&direction=forward"; + public static final String DATE_LAST_UPDATED="dateLastUpdated"; + private DynamoDBConstant() { + } } diff --git a/adapters/dynamoDB_adapters/src/main/java/org/atomhopper/dynamodb/model/PersistedEntry.java b/adapters/dynamoDB_adapters/src/main/java/org/atomhopper/dynamodb/model/PersistedEntry.java index f1dcddb2..065c99c6 100644 --- a/adapters/dynamoDB_adapters/src/main/java/org/atomhopper/dynamodb/model/PersistedEntry.java +++ b/adapters/dynamoDB_adapters/src/main/java/org/atomhopper/dynamodb/model/PersistedEntry.java @@ -1,21 +1,27 @@ package org.atomhopper.dynamodb.model; import com.amazonaws.services.dynamodbv2.datamodeling.*; import lombok.Data; +import lombok.NoArgsConstructor; +import org.atomhopper.dynamodb.constant.DynamoDBConstant; + import java.util.*; + @Data -@DynamoDBTable(tableName = "entries") +@NoArgsConstructor +@DynamoDBTable(tableName = DynamoDBConstant.ENTRIES) public class PersistedEntry { - @DynamoDBHashKey(attributeName = "entryId") + @DynamoDBHashKey(attributeName = DynamoDBConstant.ENTRY_ID) private String entryId; - @DynamoDBIndexRangeKey(attributeName = "feed", + @DynamoDBIndexRangeKey(attributeName = DynamoDBConstant.FEED, localSecondaryIndexName = "entryId-feed-index") - @DynamoDBIndexHashKey(globalSecondaryIndexName="global-feed-index", attributeName = "feed") + @DynamoDBIndexHashKey(globalSecondaryIndexName="global-feed-index", attributeName = DynamoDBConstant.FEED) private String feed; private String entryBody; @DynamoDBAutoGeneratedTimestamp(strategy = DynamoDBAutoGenerateStrategy.CREATE) private String creationDate; - @DynamoDBRangeKey(attributeName = "dateLastUpdated") + @DynamoDBRangeKey(attributeName = DynamoDBConstant.DATE_LAST_UPDATED) + @DynamoDBAutoGeneratedTimestamp(strategy=DynamoDBAutoGenerateStrategy.CREATE) private String dateLastUpdated; private List categories; } diff --git a/adapters/dynamoDB_adapters/src/main/java/org/atomhopper/dynamodb/query/JsonUtil.java b/adapters/dynamoDB_adapters/src/main/java/org/atomhopper/dynamodb/query/JsonUtil.java new file mode 100644 index 00000000..4e21a54a --- /dev/null +++ b/adapters/dynamoDB_adapters/src/main/java/org/atomhopper/dynamodb/query/JsonUtil.java @@ -0,0 +1,62 @@ +package org.atomhopper.dynamodb.query; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.atomhopper.dynamodb.model.PersistedEntry; + +import java.io.IOException; +import java.text.Format; +import java.text.SimpleDateFormat; +import java.util.*; + +/** + * Utility class for the methods declared in the dynamoDB feed Source. + */ +public class JsonUtil { + + /** + * This method is used to return the date from date interval of 2 seconds + * like now() - interval '2 seconds + * + * @param seconds: 2 seconds + * @return date format + */ + public static String getCurrentDateWithMinusSecond(int seconds) { + Format formatter = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + Calendar calendar = Calendar.getInstance(); + calendar.setTime(new Date()); + calendar.set(Calendar.SECOND, (calendar.get(Calendar.SECOND) - seconds)); + return formatter.format(calendar.getTime()); + } + + /** + * This method is used to parse list of string into PersistentEntry using object mapper + * + * @param feedPage: List of json object + * @return List of PersistentEntry + */ + + public static List getPersistenceEntity(List feedPage) { + ObjectMapper objectMapper = new ObjectMapper(); + String arrayToJson = null; + try { + arrayToJson = objectMapper.writeValueAsString(feedPage); + } catch (JsonProcessingException e) { + e.printStackTrace(); + } + //2. Convert JSON to List of Person objects + //Define Custom Type reference for List type + TypeReference> mapType = new TypeReference>() { + }; + List jsonToPersonList = null; + try { + jsonToPersonList = objectMapper.readValue(arrayToJson, mapType); + } catch (IOException e) { + e.printStackTrace(); + } + return jsonToPersonList; + } + + +} diff --git a/adapters/dynamoDB_adapters/src/main/java/org/atomhopper/dynamodb/query/SQLToNoSqlConverter.java b/adapters/dynamoDB_adapters/src/main/java/org/atomhopper/dynamodb/query/SQLToNoSqlConverter.java index 9a5584e4..aad25846 100644 --- a/adapters/dynamoDB_adapters/src/main/java/org/atomhopper/dynamodb/query/SQLToNoSqlConverter.java +++ b/adapters/dynamoDB_adapters/src/main/java/org/atomhopper/dynamodb/query/SQLToNoSqlConverter.java @@ -67,11 +67,6 @@ public SQLToNoSqlConverter(Map mapper, String split ) { mapPrefix = new HashMap( mapper ); } - - public static void main(String[] args) { - //System.out.println(new SearchToSqlConverter().getSqlFromSearchString("(AND(cat=CAT1)(OR(cat=CAT2)(cat=CAT3))(NOT(cat=CAT4)))")); - } - public String getSqlFromSearchString(String searchString,Map map) { if (StringUtils.isBlank(searchString)) { diff --git a/atomhopper/pom.xml b/atomhopper/pom.xml index f4af212b..ed4203e7 100644 --- a/atomhopper/pom.xml +++ b/atomhopper/pom.xml @@ -20,6 +20,13 @@ + + com.amazonaws + aws-java-sdk-dynamodb + 1.11.34 + + + org.atomhopper core @@ -55,6 +62,12 @@ jdbc-adapter + + org.atomhopper.adapter + dynamoDB_adapters + 1.2.35-SNAPSHOT + + postgresql postgresql diff --git a/atomhopper/src/main/webapp/META-INF/application-context.xml b/atomhopper/src/main/webapp/META-INF/application-context.xml index 255eebd4..ee37231d 100644 --- a/atomhopper/src/main/webapp/META-INF/application-context.xml +++ b/atomhopper/src/main/webapp/META-INF/application-context.xml @@ -4,7 +4,7 @@ xmlns:context="http://www.springframework.org/schema/context" xmlns:mongo="http://www.springframework.org/schema/data/mongo" xsi:schemaLocation= - "http://www.springframework.org/schema/context + "http://www.springframework.org/schema/context http://www.springframework.org/schema/context/spring-context-3.0.xsd http://www.springframework.org/schema/data/mongo http://www.springframework.org/schema/data/mongo/spring-mongo-1.0.xsd @@ -14,60 +14,60 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + com.amazonaws.regions.Regions + + + fromName + + + + ap-south-1 + + + + + + com.amazonaws.regions.Region + + + getRegion + + + + + + + + + + + + + setRegion + + + + + + + + + + + + + + setEndpoint + + + + https://dynamodb.ap-south-1.amazonaws.com + + + + + + + + + + + + + diff --git a/pom.xml b/pom.xml index 38b9ca98..814ea279 100644 --- a/pom.xml +++ b/pom.xml @@ -50,6 +50,14 @@ + + + org.springframework + spring-expression + 4.2.5.RELEASE + + + org.atomhopper core From def917cc5da59083b76435173350b3adb7556022 Mon Sep 17 00:00:00 2001 From: shub6691 Date: Tue, 28 Sep 2021 17:12:45 +0530 Subject: [PATCH 03/21] Done changes in Constrcutor for instation done in publisher --- .../org/atomhopper/dynamodb/adapter/DynamoDBFeedSource.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/adapters/dynamoDB_adapters/src/main/java/org/atomhopper/dynamodb/adapter/DynamoDBFeedSource.java b/adapters/dynamoDB_adapters/src/main/java/org/atomhopper/dynamodb/adapter/DynamoDBFeedSource.java index 7738d927..cb266b1c 100644 --- a/adapters/dynamoDB_adapters/src/main/java/org/atomhopper/dynamodb/adapter/DynamoDBFeedSource.java +++ b/adapters/dynamoDB_adapters/src/main/java/org/atomhopper/dynamodb/adapter/DynamoDBFeedSource.java @@ -124,10 +124,10 @@ public void afterPropertiesSet() { } - public DynamoDBFeedSource(AmazonDynamoDBClient amazonDynamoDBClient, DynamoDBMapper mapper) { + public DynamoDBFeedSource(AmazonDynamoDBClient amazonDynamoDBClient) { this.amazonDynamoDBClient = amazonDynamoDBClient; this.dynamoDB = new DynamoDB(this.amazonDynamoDBClient); - this.mapper = mapper; + this.mapper = new DynamoDBMapper(amazonDynamoDBClient); setDynamoDB(dynamoDB); } From fefb08c11a75734448e0e7817521259ef0c6a5af Mon Sep 17 00:00:00 2001 From: shub6691 Date: Thu, 14 Oct 2021 12:08:48 +0530 Subject: [PATCH 04/21] Fixed the issue to get a feed using feedName --- .../dynamodb/adapter/DynamoDBFeedSource.java | 78 ++++++++++--------- .../dynamodb/model/PersistedEntry.java | 60 ++++++++++++-- .../atomhopper/dynamodb/query/JsonUtil.java | 34 ++++---- 3 files changed, 114 insertions(+), 58 deletions(-) diff --git a/adapters/dynamoDB_adapters/src/main/java/org/atomhopper/dynamodb/adapter/DynamoDBFeedSource.java b/adapters/dynamoDB_adapters/src/main/java/org/atomhopper/dynamodb/adapter/DynamoDBFeedSource.java index cb266b1c..02eb4b3a 100644 --- a/adapters/dynamoDB_adapters/src/main/java/org/atomhopper/dynamodb/adapter/DynamoDBFeedSource.java +++ b/adapters/dynamoDB_adapters/src/main/java/org/atomhopper/dynamodb/adapter/DynamoDBFeedSource.java @@ -7,6 +7,7 @@ import com.amazonaws.services.dynamodbv2.document.spec.QuerySpec; import com.amazonaws.services.dynamodbv2.document.utils.ValueMap; import com.amazonaws.services.dynamodbv2.model.AttributeValue; +import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.gson.Gson; import com.yammer.metrics.Metrics; @@ -313,7 +314,8 @@ public FeedInformation getFeedInformation() { @Override public AdapterResponse getFeed(GetFeedRequest getFeedRequest) { AdapterResponse response = null; - + LOG.info("INSIDE GET FEED METHOD"); + LOG.info("PRINTING THE GET_FEED_REQUEST OBJECT :" + getFeedRequest); TimerContext context = null; int pageSize = DynamoDBConstant.PAGE_SIZE; final String pageSizeString = getFeedRequest.getPageSize(); @@ -321,8 +323,11 @@ public AdapterResponse getFeed(GetFeedRequest getFeedRequest) { pageSize = Integer.parseInt(pageSizeString); } final String marker = getFeedRequest.getPageMarker(); + LOG.info("PRINTING THE MARKER FOR THE REQUEST : " + marker); final String startingAt = getFeedRequest.getStartingAt(); + LOG.info("PRINTING THE STARTING_AT FOR THE REQUEST :" + startingAt); if (StringUtils.isNotBlank(marker) && StringUtils.isNotBlank(startingAt)) { + LOG.info("INSIDE THE IF BLOCK WHERE MARKER AND STARTING ID IS NOT BLANK :" + startingAt); response = ResponseBuilder.badRequest("'marker' parameter can not be used together with the 'startingAt' parameter"); return response; } @@ -330,15 +335,19 @@ public AdapterResponse getFeed(GetFeedRequest getFeedRequest) { try { if (StringUtils.isBlank(marker) && StringUtils.isBlank(startingAt)) { + LOG.info("INSIDE THE IF BLOCK WHERE MARKER AND STARTING ID BLANK :" + marker); context = startTimer(String.format("get-feed-head-%s", getMetricBucketForPageSize(pageSize))); response = getFeedHead(getFeedRequest, pageSize); } else if (StringUtils.isNotBlank(marker) && marker.equals(DynamoDBConstant.MOCK_LAST_MARKER)) { + LOG.info("INSIDE THE IF BLOCK WHERE MARKER IS NOT NULL AND WE HAVE MOCK_LAST_MARKER :" + marker); context = startTimer(String.format("get-last-page-%s", getMetricBucketForPageSize(pageSize))); response = getLastPage(getFeedRequest, pageSize); } else if (StringUtils.isNotBlank(marker)) { + LOG.info("INSIDE THE IF BLOCK WHERE MARKER ID IS NOT BLANK :" + marker); context = startTimer(String.format("get-feed-page-%s", getMetricBucketForPageSize(pageSize))); response = getFeedPage(getFeedRequest, marker, pageSize); } else { + LOG.info("INSIDE THE IF BLOCK WHERE MARKER AND STARTING ID IS NOT BLANK :" + startingAt); // we process 'startingAt' parameter here context = startTimer(String.format("get-feed-page-startingAt-%s", getMetricBucketForPageSize(pageSize))); response = getFeedPageByTimestamp(getFeedRequest, startingAt, pageSize); @@ -378,15 +387,18 @@ public AdapterResponse getEntry(GetEntryRequest getEntryRequest) { private AdapterResponse getFeedPage(GetFeedRequest getFeedRequest, String marker, int pageSize) throws ParseException { + LOG.info("INSIDE THE GET_FEED_PAGE METHOD :" + getFeedRequest); final String pageDirectionValue = getFeedRequest.getDirection(); PageDirection pageDirection = PageDirection.FORWARD; if (StringUtils.isNotEmpty(pageDirectionValue)) { pageDirection = PageDirection.valueOf(pageDirectionValue.toUpperCase()); } - + LOG.info("PAGE DIRECTION IN GET_FEED_PAGE:" + pageDirectionValue); final String searchString = getFeedRequest.getSearchQuery() != null ? getFeedRequest.getSearchQuery() : ""; + List entryMarker = getEntry(marker, getFeedRequest.getFeedName()); + LOG.info("ENTRY MARKER OF LIST :" + entryMarker.size()); if (entryMarker.isEmpty()) { return ResponseBuilder.notFound("No entry with specified marker found"); } @@ -531,10 +543,12 @@ private List enhancedGetLastPage(final String feedName, final in List feedPage; ValueMap valueMap = new ValueMap(); valueMap.withString(":feed", feedName); - valueMap.withString(":dateLastUpdated", JsonUtil.getCurrentDateWithMinusSecond(2)); + // valueMap.withString(":dateLastUpdated", JsonUtil.getCurrentDateWithMinusSecond(2)); feedPage = getQueryBuilderMethod(dynamoDB, "feed = :feed and dateLastUpdated < :dateLastUpdated", pageSize, valueMap, true); + LOG.info("GET QUERY_BUILDER_METHOD : " + feedPage); // comparator is written to perform sorting based on if two dates are equal then sort output based on entryId in desc order List persistedEntryList = JsonUtil.getPersistenceEntity(feedPage); + LOG.info("PERSISTEMT ENTRY LIST DATA:" + persistedEntryList.toString()); Collections.sort(persistedEntryList, (a, b) -> { SimpleDateFormat dateFormatter = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'"); String dateLastUpdated = dateFormatter.format(a.getDateLastUpdated()); @@ -557,6 +571,7 @@ private List enhancedGetLastPage(final String feedName, final in * @return */ private AdapterResponse getFeedHead(GetFeedRequest getFeedRequest, int pageSize) { + LOG.info("INSIDE THE GET_FEED_HEAD METHOD :" + getFeedRequest); final Abdera abdera = getFeedRequest.getAbdera(); final String searchString = getFeedRequest.getSearchQuery() != null ? getFeedRequest.getSearchQuery() : ""; @@ -565,6 +580,8 @@ private AdapterResponse getFeedHead(GetFeedRequest getFeedRequest, int pag Feed hydratedFeed = hydrateFeed(abdera, persistedEntries, getFeedRequest, pageSize); + LOG.info("HYDRATED FEED:" + hydratedFeed); + // Set the last link in the feed head final String baseFeedUri = decode(getFeedRequest.urlFor( new EnumKeyedTemplateParameters(URITemplate.FEED))); @@ -597,13 +614,16 @@ private String getMetricBucketForPageSize(int pageSize) { } } - /** This method is used to return the feed head which means the starting feed for the particular entry - * @param feedName: name of the feed - * @param pageSize: default is 25, for pagination + /** + * This method is used to return the feed head which means the starting feed for the particular entry + * + * @param feedName: name of the feed + * @param pageSize: default is 25, for pagination * @param searchString: Search string to be used to filteration in the query * @return List of all the output of PersistedEntry */ private List getFeedHead(final String feedName, final int pageSize, final String searchString) { + LOG.info("INSIDE THE GET_FEED_HEAD METHOD WITH CATEGORIES :" + feedName); List categoriesList = getSearchToSqlConverter().getParamsFromSearchString(searchString); int numCats = categoriesList.size(); @@ -634,18 +654,13 @@ private List getFeedHead(final String feedName, final int pageSi List feedPage; ValueMap valueMap = new ValueMap(); valueMap.withString(":feed", feedName); - valueMap.withString(":dateLastUpdated", JsonUtil.getCurrentDateWithMinusSecond(2)); - feedPage = getQueryBuilderMethod(dynamoDB, "feed = :feed and dateLastUpdated < :dateLastUpdated", pageSize, valueMap, false); + feedPage = getQueryBuilderMethod(dynamoDB, "feed = :feed", pageSize, valueMap, false); List persistedEntryList = JsonUtil.getPersistenceEntity(feedPage); - Collections.sort(persistedEntryList, (a, b) -> { - SimpleDateFormat dateFormatter = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'"); - String dateLastUpdated = dateFormatter.format(a.getDateLastUpdated()); - String dateLastUpdatedNextDate = dateFormatter.format(b.getDateLastUpdated()); - if (dateLastUpdatedNextDate.equals(dateLastUpdated)) { - return b.getEntryId().compareTo(a.getEntryId()); - } - return -1; - }); + try { + LOG.debug("PERSISTENT ENTRY LIST DATA OBJECT :" + new ObjectMapper().writeValueAsString(persistedEntryList)); + } catch (JsonProcessingException e) { + e.printStackTrace(); + } return persistedEntryList; } finally { stopTimer(context); @@ -756,8 +771,8 @@ private Feed hydrateFeed(Abdera abdera, List persistedEntries, * This method is used to get the next marker based for the markerID and the feedName * * @param persistedEntry: PersistentEntryModel - * @param feedName: name of the feed - * @param searchString: Category search string for filteration + * @param feedName: name of the feed + * @param searchString: Category search string for filteration * @return Persistent model Object */ private PersistedEntry getNextMarker(final PersistedEntry persistedEntry, final String feedName, @@ -792,28 +807,14 @@ private PersistedEntry getNextMarker(final PersistedEntry persistedEntry, final List firstUnionPersistentList; ValueMap valueMap = new ValueMap(); valueMap.withString(":feed", persistedEntry.getFeed()); - valueMap.withString(":feed", persistedEntry.getDateLastUpdated()); - valueMap.withString(":entryId", persistedEntry.getEntryId()); - firstUnionPersistentList = getQueryBuilderMethod(dynamoDB, "feed = :feed and dateLastUpdated =:dateLastUpdated and entryId <=:entryId", valueMap); - - + firstUnionPersistentList = getQueryBuilderMethod(dynamoDB, "feed = :feed ", valueMap); List nextUnionListOfPersistedItems; ValueMap newValueMap = new ValueMap(); newValueMap.withString(":feed", persistedEntry.getFeed()); - newValueMap.withString(":dateLastUpdated", persistedEntry.getDateLastUpdated()); - nextUnionListOfPersistedItems = getQueryBuilderMethod(dynamoDB, "feed = :feed and dateLastUpdated < :dateLastUpdated", valueMap); + nextUnionListOfPersistedItems = getQueryBuilderMethod(dynamoDB, "feed = :feed", valueMap); feedPage = Stream.concat(firstUnionPersistentList.stream(), nextUnionListOfPersistedItems.stream()) .collect(Collectors.toList()); List persistedEntryList = JsonUtil.getPersistenceEntity(feedPage); - Collections.sort(persistedEntryList, (a, b) -> { - SimpleDateFormat dateFormatter = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'"); - String dateLastUpdated = dateFormatter.format(a.getDateLastUpdated()); - String dateLastUpdatedNextDate = dateFormatter.format(b.getDateLastUpdated()); - if (dateLastUpdatedNextDate.equals(dateLastUpdated)) { - return b.getEntryId().compareTo(a.getEntryId()); - } - return -1; - }); return persistedEntryList.get(0); } @@ -901,6 +902,7 @@ private String urlEncode(String searchString) { * @return : list of entry found if that exits in dynamodb with the entryId and feedName. */ private List getEntry(String entryId, final String feedName) { + LOG.info("INSIDE GET_ENTRY METHOD :" + entryId + "feedName :" + feedName); Gson gson = new Gson(); List persistedEntriesObject = new ArrayList(); Table table = dynamoDB.getTable(DynamoDBConstant.ENTRIES); @@ -916,6 +918,7 @@ private List getEntry(String entryId, final String feedName) { Item item = itemsIterator.next(); persistedEntriesObject.add(gson.fromJson(item.toJSONPretty(), PersistedEntry.class)); } + LOG.info("PERSISTENT_ENTRIES OBJECT:" + persistedEntriesObject); return persistedEntriesObject; } @@ -929,20 +932,25 @@ private List getEntry(String entryId, final String feedName) { * @return List of String in json format. */ public List getQueryBuilderMethod(DynamoDB dynamoDB, String conditionExpression, int pageSize, ValueMap valueMap, boolean orderBy) { + LOG.info("INSIDE GET_QUERY_BUILDER_METHOD"); List feedPage = new ArrayList<>(); Table table = dynamoDB.getTable("entries"); Index index = table.getIndex("global-feed-index"); + LOG.info("CONDITION EXPRESSION :" + conditionExpression); QuerySpec spec = new QuerySpec() .withKeyConditionExpression(conditionExpression) .withValueMap(valueMap) .withMaxPageSize(pageSize)// for no of page limit to be displayed .withScanIndexForward(orderBy);// for descending order sorting on lastDateUpdated + LOG.info("AFTER QUERY SPEC"); ItemCollection persistedEntryItems = index.query(spec); + LOG.info("AFTER QUERY SPEC MAIN METHOD"); Iterator itemsIterator = persistedEntryItems.iterator(); while (itemsIterator.hasNext()) { Item item = itemsIterator.next(); feedPage.add(item.toJSONPretty()); } + LOG.info("FEED PAGE RESULT" + feedPage); return feedPage; } diff --git a/adapters/dynamoDB_adapters/src/main/java/org/atomhopper/dynamodb/model/PersistedEntry.java b/adapters/dynamoDB_adapters/src/main/java/org/atomhopper/dynamodb/model/PersistedEntry.java index 065c99c6..91135fda 100644 --- a/adapters/dynamoDB_adapters/src/main/java/org/atomhopper/dynamodb/model/PersistedEntry.java +++ b/adapters/dynamoDB_adapters/src/main/java/org/atomhopper/dynamodb/model/PersistedEntry.java @@ -1,13 +1,11 @@ package org.atomhopper.dynamodb.model; + import com.amazonaws.services.dynamodbv2.datamodeling.*; -import lombok.Data; -import lombok.NoArgsConstructor; import org.atomhopper.dynamodb.constant.DynamoDBConstant; import java.util.*; -@Data -@NoArgsConstructor + @DynamoDBTable(tableName = DynamoDBConstant.ENTRIES) public class PersistedEntry { @@ -15,13 +13,63 @@ public class PersistedEntry { private String entryId; @DynamoDBIndexRangeKey(attributeName = DynamoDBConstant.FEED, localSecondaryIndexName = "entryId-feed-index") - @DynamoDBIndexHashKey(globalSecondaryIndexName="global-feed-index", attributeName = DynamoDBConstant.FEED) + @DynamoDBIndexHashKey(globalSecondaryIndexName = "global-feed-index", attributeName = DynamoDBConstant.FEED) private String feed; private String entryBody; @DynamoDBAutoGeneratedTimestamp(strategy = DynamoDBAutoGenerateStrategy.CREATE) private String creationDate; @DynamoDBRangeKey(attributeName = DynamoDBConstant.DATE_LAST_UPDATED) - @DynamoDBAutoGeneratedTimestamp(strategy=DynamoDBAutoGenerateStrategy.CREATE) + @DynamoDBAutoGeneratedTimestamp(strategy = DynamoDBAutoGenerateStrategy.CREATE) private String dateLastUpdated; private List categories; + + public String getEntryId() { + return entryId; + } + + public void setEntryId(String entryId) { + this.entryId = entryId; + } + + public String getFeed() { + return feed; + } + + public void setFeed(String feed) { + this.feed = feed; + } + + public String getEntryBody() { + return entryBody; + } + + public void setEntryBody(String entryBody) { + this.entryBody = entryBody; + } + + public String getCreationDate() { + return creationDate; + } + + public void setCreationDate(String creationDate) { + this.creationDate = creationDate; + } + + public String getDateLastUpdated() { + return dateLastUpdated; + } + + public void setDateLastUpdated(String dateLastUpdated) { + this.dateLastUpdated = dateLastUpdated; + } + + public List getCategories() { + return categories; + } + + public void setCategories(List categories) { + this.categories = categories; + } + + } diff --git a/adapters/dynamoDB_adapters/src/main/java/org/atomhopper/dynamodb/query/JsonUtil.java b/adapters/dynamoDB_adapters/src/main/java/org/atomhopper/dynamodb/query/JsonUtil.java index 4e21a54a..26fe30a3 100644 --- a/adapters/dynamoDB_adapters/src/main/java/org/atomhopper/dynamodb/query/JsonUtil.java +++ b/adapters/dynamoDB_adapters/src/main/java/org/atomhopper/dynamodb/query/JsonUtil.java @@ -1,9 +1,10 @@ package org.atomhopper.dynamodb.query; -import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; import org.atomhopper.dynamodb.model.PersistedEntry; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.io.IOException; import java.text.Format; @@ -14,6 +15,7 @@ * Utility class for the methods declared in the dynamoDB feed Source. */ public class JsonUtil { + static Logger LOG = LoggerFactory.getLogger(JsonUtil.class); /** * This method is used to return the date from date interval of 2 seconds @@ -23,10 +25,12 @@ public class JsonUtil { * @return date format */ public static String getCurrentDateWithMinusSecond(int seconds) { + LOG.info("getCurrentDateWithMinusSecond"); Format formatter = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); Calendar calendar = Calendar.getInstance(); calendar.setTime(new Date()); calendar.set(Calendar.SECOND, (calendar.get(Calendar.SECOND) - seconds)); + LOG.info("RETURN RESULT VALUE:" + formatter.format(calendar.getTime())); return formatter.format(calendar.getTime()); } @@ -37,26 +41,22 @@ public static String getCurrentDateWithMinusSecond(int seconds) { * @return List of PersistentEntry */ - public static List getPersistenceEntity(List feedPage) { + public static List getPersistenceEntity(List feedPage) { ObjectMapper objectMapper = new ObjectMapper(); - String arrayToJson = null; - try { - arrayToJson = objectMapper.writeValueAsString(feedPage); - } catch (JsonProcessingException e) { - e.printStackTrace(); - } + List jsonToPersistentList = new ArrayList<>(); //2. Convert JSON to List of Person objects + //2. Convert JSON to List of PersistedEntry objectschanges //Define Custom Type reference for List type - TypeReference> mapType = new TypeReference>() { + TypeReference mapType = new TypeReference() { }; - List jsonToPersonList = null; - try { - jsonToPersonList = objectMapper.readValue(arrayToJson, mapType); - } catch (IOException e) { - e.printStackTrace(); + for (String s : feedPage) { + try { + jsonToPersistentList.add(objectMapper.readValue(s, mapType)); + LOG.info("PERSISTENT ENTRY DATA:" + jsonToPersistentList); + } catch (IOException e) { + e.printStackTrace(); + } } - return jsonToPersonList; + return jsonToPersistentList; } - - } From f14faee4c73b873d9f207d734d28328fc5f9c74e Mon Sep 17 00:00:00 2001 From: shub6691 Date: Thu, 14 Oct 2021 14:58:08 +0530 Subject: [PATCH 05/21] Added jar for unecape StringUtils --- adapters/dynamoDB_adapters/pom.xml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/adapters/dynamoDB_adapters/pom.xml b/adapters/dynamoDB_adapters/pom.xml index a5806f94..3b21318a 100644 --- a/adapters/dynamoDB_adapters/pom.xml +++ b/adapters/dynamoDB_adapters/pom.xml @@ -75,6 +75,14 @@ mockito-all + + + org.apache.commons + commons-text + 1.9 + + + From 160b80cc48e297ae4bb80a9322177b37f112824d Mon Sep 17 00:00:00 2001 From: shub6691 Date: Mon, 18 Oct 2021 16:42:40 +0530 Subject: [PATCH 06/21] Removed Logs abd fixed test cases --- .../dynamodb/adapter/DynamoDBFeedSource.java | 37 +------- .../atomhopper/dynamodb/query/JsonUtil.java | 1 - .../src/main/resources/application.properties | 2 - .../src/main/resources/logback.xml | 0 .../test/java/DynamoDBFeedPublisherTest.java | 20 ++--- .../src/test/java/DynamoDBFeedSourceTest.java | 84 +++++++++++++++---- 6 files changed, 79 insertions(+), 65 deletions(-) delete mode 100644 adapters/dynamoDB_adapters/src/main/resources/application.properties delete mode 100644 adapters/dynamoDB_adapters/src/main/resources/logback.xml diff --git a/adapters/dynamoDB_adapters/src/main/java/org/atomhopper/dynamodb/adapter/DynamoDBFeedSource.java b/adapters/dynamoDB_adapters/src/main/java/org/atomhopper/dynamodb/adapter/DynamoDBFeedSource.java index 02eb4b3a..b334bcc1 100644 --- a/adapters/dynamoDB_adapters/src/main/java/org/atomhopper/dynamodb/adapter/DynamoDBFeedSource.java +++ b/adapters/dynamoDB_adapters/src/main/java/org/atomhopper/dynamodb/adapter/DynamoDBFeedSource.java @@ -314,8 +314,6 @@ public FeedInformation getFeedInformation() { @Override public AdapterResponse getFeed(GetFeedRequest getFeedRequest) { AdapterResponse response = null; - LOG.info("INSIDE GET FEED METHOD"); - LOG.info("PRINTING THE GET_FEED_REQUEST OBJECT :" + getFeedRequest); TimerContext context = null; int pageSize = DynamoDBConstant.PAGE_SIZE; final String pageSizeString = getFeedRequest.getPageSize(); @@ -323,11 +321,8 @@ public AdapterResponse getFeed(GetFeedRequest getFeedRequest) { pageSize = Integer.parseInt(pageSizeString); } final String marker = getFeedRequest.getPageMarker(); - LOG.info("PRINTING THE MARKER FOR THE REQUEST : " + marker); final String startingAt = getFeedRequest.getStartingAt(); - LOG.info("PRINTING THE STARTING_AT FOR THE REQUEST :" + startingAt); if (StringUtils.isNotBlank(marker) && StringUtils.isNotBlank(startingAt)) { - LOG.info("INSIDE THE IF BLOCK WHERE MARKER AND STARTING ID IS NOT BLANK :" + startingAt); response = ResponseBuilder.badRequest("'marker' parameter can not be used together with the 'startingAt' parameter"); return response; } @@ -335,19 +330,17 @@ public AdapterResponse getFeed(GetFeedRequest getFeedRequest) { try { if (StringUtils.isBlank(marker) && StringUtils.isBlank(startingAt)) { - LOG.info("INSIDE THE IF BLOCK WHERE MARKER AND STARTING ID BLANK :" + marker); + context = startTimer(String.format("get-feed-head-%s", getMetricBucketForPageSize(pageSize))); response = getFeedHead(getFeedRequest, pageSize); } else if (StringUtils.isNotBlank(marker) && marker.equals(DynamoDBConstant.MOCK_LAST_MARKER)) { - LOG.info("INSIDE THE IF BLOCK WHERE MARKER IS NOT NULL AND WE HAVE MOCK_LAST_MARKER :" + marker); + context = startTimer(String.format("get-last-page-%s", getMetricBucketForPageSize(pageSize))); response = getLastPage(getFeedRequest, pageSize); } else if (StringUtils.isNotBlank(marker)) { - LOG.info("INSIDE THE IF BLOCK WHERE MARKER ID IS NOT BLANK :" + marker); context = startTimer(String.format("get-feed-page-%s", getMetricBucketForPageSize(pageSize))); response = getFeedPage(getFeedRequest, marker, pageSize); } else { - LOG.info("INSIDE THE IF BLOCK WHERE MARKER AND STARTING ID IS NOT BLANK :" + startingAt); // we process 'startingAt' parameter here context = startTimer(String.format("get-feed-page-startingAt-%s", getMetricBucketForPageSize(pageSize))); response = getFeedPageByTimestamp(getFeedRequest, startingAt, pageSize); @@ -386,19 +379,13 @@ public AdapterResponse getEntry(GetEntryRequest getEntryRequest) { */ private AdapterResponse getFeedPage(GetFeedRequest getFeedRequest, String marker, int pageSize) throws ParseException { - - LOG.info("INSIDE THE GET_FEED_PAGE METHOD :" + getFeedRequest); final String pageDirectionValue = getFeedRequest.getDirection(); PageDirection pageDirection = PageDirection.FORWARD; if (StringUtils.isNotEmpty(pageDirectionValue)) { pageDirection = PageDirection.valueOf(pageDirectionValue.toUpperCase()); } - LOG.info("PAGE DIRECTION IN GET_FEED_PAGE:" + pageDirectionValue); final String searchString = getFeedRequest.getSearchQuery() != null ? getFeedRequest.getSearchQuery() : ""; - - List entryMarker = getEntry(marker, getFeedRequest.getFeedName()); - LOG.info("ENTRY MARKER OF LIST :" + entryMarker.size()); if (entryMarker.isEmpty()) { return ResponseBuilder.notFound("No entry with specified marker found"); } @@ -543,12 +530,9 @@ private List enhancedGetLastPage(final String feedName, final in List feedPage; ValueMap valueMap = new ValueMap(); valueMap.withString(":feed", feedName); - // valueMap.withString(":dateLastUpdated", JsonUtil.getCurrentDateWithMinusSecond(2)); feedPage = getQueryBuilderMethod(dynamoDB, "feed = :feed and dateLastUpdated < :dateLastUpdated", pageSize, valueMap, true); - LOG.info("GET QUERY_BUILDER_METHOD : " + feedPage); // comparator is written to perform sorting based on if two dates are equal then sort output based on entryId in desc order List persistedEntryList = JsonUtil.getPersistenceEntity(feedPage); - LOG.info("PERSISTEMT ENTRY LIST DATA:" + persistedEntryList.toString()); Collections.sort(persistedEntryList, (a, b) -> { SimpleDateFormat dateFormatter = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'"); String dateLastUpdated = dateFormatter.format(a.getDateLastUpdated()); @@ -571,7 +555,6 @@ private List enhancedGetLastPage(final String feedName, final in * @return */ private AdapterResponse getFeedHead(GetFeedRequest getFeedRequest, int pageSize) { - LOG.info("INSIDE THE GET_FEED_HEAD METHOD :" + getFeedRequest); final Abdera abdera = getFeedRequest.getAbdera(); final String searchString = getFeedRequest.getSearchQuery() != null ? getFeedRequest.getSearchQuery() : ""; @@ -580,8 +563,6 @@ private AdapterResponse getFeedHead(GetFeedRequest getFeedRequest, int pag Feed hydratedFeed = hydrateFeed(abdera, persistedEntries, getFeedRequest, pageSize); - LOG.info("HYDRATED FEED:" + hydratedFeed); - // Set the last link in the feed head final String baseFeedUri = decode(getFeedRequest.urlFor( new EnumKeyedTemplateParameters(URITemplate.FEED))); @@ -623,8 +604,6 @@ private String getMetricBucketForPageSize(int pageSize) { * @return List of all the output of PersistedEntry */ private List getFeedHead(final String feedName, final int pageSize, final String searchString) { - LOG.info("INSIDE THE GET_FEED_HEAD METHOD WITH CATEGORIES :" + feedName); - List categoriesList = getSearchToSqlConverter().getParamsFromSearchString(searchString); int numCats = categoriesList.size(); @@ -656,11 +635,6 @@ private List getFeedHead(final String feedName, final int pageSi valueMap.withString(":feed", feedName); feedPage = getQueryBuilderMethod(dynamoDB, "feed = :feed", pageSize, valueMap, false); List persistedEntryList = JsonUtil.getPersistenceEntity(feedPage); - try { - LOG.debug("PERSISTENT ENTRY LIST DATA OBJECT :" + new ObjectMapper().writeValueAsString(persistedEntryList)); - } catch (JsonProcessingException e) { - e.printStackTrace(); - } return persistedEntryList; } finally { stopTimer(context); @@ -902,7 +876,6 @@ private String urlEncode(String searchString) { * @return : list of entry found if that exits in dynamodb with the entryId and feedName. */ private List getEntry(String entryId, final String feedName) { - LOG.info("INSIDE GET_ENTRY METHOD :" + entryId + "feedName :" + feedName); Gson gson = new Gson(); List persistedEntriesObject = new ArrayList(); Table table = dynamoDB.getTable(DynamoDBConstant.ENTRIES); @@ -918,7 +891,6 @@ private List getEntry(String entryId, final String feedName) { Item item = itemsIterator.next(); persistedEntriesObject.add(gson.fromJson(item.toJSONPretty(), PersistedEntry.class)); } - LOG.info("PERSISTENT_ENTRIES OBJECT:" + persistedEntriesObject); return persistedEntriesObject; } @@ -932,25 +904,20 @@ private List getEntry(String entryId, final String feedName) { * @return List of String in json format. */ public List getQueryBuilderMethod(DynamoDB dynamoDB, String conditionExpression, int pageSize, ValueMap valueMap, boolean orderBy) { - LOG.info("INSIDE GET_QUERY_BUILDER_METHOD"); List feedPage = new ArrayList<>(); Table table = dynamoDB.getTable("entries"); Index index = table.getIndex("global-feed-index"); - LOG.info("CONDITION EXPRESSION :" + conditionExpression); QuerySpec spec = new QuerySpec() .withKeyConditionExpression(conditionExpression) .withValueMap(valueMap) .withMaxPageSize(pageSize)// for no of page limit to be displayed .withScanIndexForward(orderBy);// for descending order sorting on lastDateUpdated - LOG.info("AFTER QUERY SPEC"); ItemCollection persistedEntryItems = index.query(spec); - LOG.info("AFTER QUERY SPEC MAIN METHOD"); Iterator itemsIterator = persistedEntryItems.iterator(); while (itemsIterator.hasNext()) { Item item = itemsIterator.next(); feedPage.add(item.toJSONPretty()); } - LOG.info("FEED PAGE RESULT" + feedPage); return feedPage; } diff --git a/adapters/dynamoDB_adapters/src/main/java/org/atomhopper/dynamodb/query/JsonUtil.java b/adapters/dynamoDB_adapters/src/main/java/org/atomhopper/dynamodb/query/JsonUtil.java index 26fe30a3..80234335 100644 --- a/adapters/dynamoDB_adapters/src/main/java/org/atomhopper/dynamodb/query/JsonUtil.java +++ b/adapters/dynamoDB_adapters/src/main/java/org/atomhopper/dynamodb/query/JsonUtil.java @@ -52,7 +52,6 @@ public static List getPersistenceEntity(List feedPage) { for (String s : feedPage) { try { jsonToPersistentList.add(objectMapper.readValue(s, mapType)); - LOG.info("PERSISTENT ENTRY DATA:" + jsonToPersistentList); } catch (IOException e) { e.printStackTrace(); } diff --git a/adapters/dynamoDB_adapters/src/main/resources/application.properties b/adapters/dynamoDB_adapters/src/main/resources/application.properties deleted file mode 100644 index 92f71008..00000000 --- a/adapters/dynamoDB_adapters/src/main/resources/application.properties +++ /dev/null @@ -1,2 +0,0 @@ -#server.port=8081 -logging.config=file:./src/main/resources/logback.xml \ No newline at end of file diff --git a/adapters/dynamoDB_adapters/src/main/resources/logback.xml b/adapters/dynamoDB_adapters/src/main/resources/logback.xml deleted file mode 100644 index e69de29b..00000000 diff --git a/adapters/dynamoDB_adapters/src/test/java/DynamoDBFeedPublisherTest.java b/adapters/dynamoDB_adapters/src/test/java/DynamoDBFeedPublisherTest.java index 771a109a..cfb0da7a 100644 --- a/adapters/dynamoDB_adapters/src/test/java/DynamoDBFeedPublisherTest.java +++ b/adapters/dynamoDB_adapters/src/test/java/DynamoDBFeedPublisherTest.java @@ -9,7 +9,7 @@ import org.atomhopper.adapter.request.adapter.DeleteEntryRequest; import org.atomhopper.adapter.request.adapter.PostEntryRequest; import org.atomhopper.adapter.request.adapter.PutEntryRequest; -import org.atomhopper.dynamodb.adapter.DynamoFeedDBPublisher; +import org.atomhopper.dynamodb.adapter.DynamoDBFeedPublisher; import org.atomhopper.dynamodb.model.PersistedEntry; import org.atomhopper.response.AdapterResponse; import org.junit.Before; @@ -39,7 +39,7 @@ public class DynamoDBFeedPublisherTest { DynamoDBQueryExpression querySpec; @Mock private DynamoDBMapper dynamoDBMapper; - private DynamoFeedDBPublisher dynamoFeedDBPublisher = new DynamoFeedDBPublisher(); + private DynamoDBFeedPublisher dynamoDBFeedPublisher = new DynamoDBFeedPublisher(); @Mock private AmazonDynamoDBClient amazonDynamoDBClient; @@ -55,9 +55,9 @@ public class DynamoDBFeedPublisherTest { @Before public void setUp() throws Exception { - dynamoFeedDBPublisher.setDynamoDBClient(amazonDynamoDBClient); - dynamoFeedDBPublisher.setDynamoMapper(dynamoDBMapper); - dynamoFeedDBPublisher.setDynamoDB(dynamoDB); + dynamoDBFeedPublisher.setDynamoDBClient(amazonDynamoDBClient); + dynamoDBFeedPublisher.setDynamoMapper(dynamoDBMapper); + dynamoDBFeedPublisher.setDynamoDB(dynamoDB); persistedEntry = new PersistedEntry(); persistedEntry.setFeed(FEED_NAME); persistedEntry.setEntryId(MARKER_ID); @@ -75,9 +75,9 @@ public void setUp() throws Exception { @Test public void showSaveTheObjectInDynamoDb() throws Exception { - dynamoFeedDBPublisher.setAllowOverrideId(false); + dynamoDBFeedPublisher.setAllowOverrideId(false); doNothing().when(dynamoDBMapper).save(persistedEntry); - AdapterResponse adapterResponse = dynamoFeedDBPublisher.postEntry(postEntryRequest); + AdapterResponse adapterResponse = dynamoDBFeedPublisher.postEntry(postEntryRequest); assertEquals("Should return HTTP 201 (Created)", HttpStatus.CREATED, adapterResponse.getResponseStatus()); } @@ -94,19 +94,19 @@ public void showErrorIfAlreadyExistsEntryIDInDynamoDb() throws Exception { when(outcome.iterator()).thenReturn(mockIterator); when(mockIterator.hasNext()).thenReturn(true, false); when(mockIterator.next()).thenReturn(mockItem); - AdapterResponse adapterResponse = dynamoFeedDBPublisher.postEntry(postEntryRequest); + AdapterResponse adapterResponse = dynamoDBFeedPublisher.postEntry(postEntryRequest); assertEquals("Should return HTTP 409 (Conflict)", HttpStatus.CONFLICT, adapterResponse.getResponseStatus()); } @Test(expected = UnsupportedOperationException.class) public void shouldPutEntry() throws Exception { - dynamoFeedDBPublisher.putEntry(putEntryRequest); + dynamoDBFeedPublisher.putEntry(putEntryRequest); } @Test(expected = UnsupportedOperationException.class) public void shouldDeleteEntry() throws Exception { - dynamoFeedDBPublisher.deleteEntry(deleteEntryRequest); + dynamoDBFeedPublisher.deleteEntry(deleteEntryRequest); } public Entry entry() { diff --git a/adapters/dynamoDB_adapters/src/test/java/DynamoDBFeedSourceTest.java b/adapters/dynamoDB_adapters/src/test/java/DynamoDBFeedSourceTest.java index c9d1380d..e88f3b38 100644 --- a/adapters/dynamoDB_adapters/src/test/java/DynamoDBFeedSourceTest.java +++ b/adapters/dynamoDB_adapters/src/test/java/DynamoDBFeedSourceTest.java @@ -4,20 +4,30 @@ import com.amazonaws.services.dynamodbv2.document.DynamoDB; import com.amazonaws.services.dynamodbv2.document.internal.IteratorSupport; import com.amazonaws.services.dynamodbv2.document.spec.QuerySpec; +import com.amazonaws.services.dynamodbv2.document.utils.ValueMap; import org.apache.abdera.Abdera; +import org.apache.abdera.model.Feed; import org.atomhopper.adapter.request.adapter.GetEntryRequest; import org.atomhopper.adapter.request.adapter.GetFeedRequest; import org.atomhopper.dynamodb.adapter.DynamoDBFeedSource; -import org.atomhopper.dynamodb.adapter.DynamoFeedDBPublisher; +import org.atomhopper.dynamodb.adapter.DynamoDBFeedPublisher; import org.atomhopper.dynamodb.model.PersistedEntry; +import org.atomhopper.dynamodb.query.JsonUtil; +import org.atomhopper.response.AdapterResponse; import org.junit.Before; import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; import org.junit.runner.RunWith; import org.mockito.Mock; +import org.mockito.Mockito; import org.mockito.runners.MockitoJUnitRunner; +import org.springframework.http.HttpStatus; + +import java.net.URL; import java.util.*; + +import static junit.framework.Assert.assertTrue; import static org.junit.Assert.assertEquals; import static org.mockito.Matchers.any; import static org.mockito.Mockito.*; @@ -34,7 +44,7 @@ public class DynamoDBFeedSourceTest { private AmazonDynamoDBClient amazonDynamoDBClient; @Mock private PaginatedQueryList paginatedQueryList; - private DynamoFeedDBPublisher dynamoFeedDBPublisher = new DynamoFeedDBPublisher(); + private DynamoDBFeedPublisher dynamoFeedDBPublisher = new DynamoDBFeedPublisher(); private GetFeedRequest getFeedRequest; private DynamoDBFeedSource dynamoDBFeedSource; private GetEntryRequest getEntryRequest; @@ -42,7 +52,7 @@ public class DynamoDBFeedSourceTest { private List entryList; private List emptyList; private Abdera abdera; - private final String MARKER_ID = "101"; + private final String MARKER_ID = UUID.randomUUID().toString(); private final String ENTRY_BODY = ""; private final String FEED_NAME = "namespace/feed"; private final String FORWARD = "forward"; @@ -60,7 +70,7 @@ public class DynamoDBFeedSourceTest { @Before public void setUp() throws Exception { - dynamoDBFeedSource = new DynamoDBFeedSource(amazonDynamoDBClient, dynamoDBMapper); + dynamoDBFeedSource = new DynamoDBFeedSource(amazonDynamoDBClient); dynamoFeedDBPublisher.setDynamoDBClient(amazonDynamoDBClient); dynamoFeedDBPublisher.setDynamoMapper(dynamoDBMapper); dynamoDBFeedSource.setDynamoDB(dynamoDB); @@ -79,6 +89,8 @@ public void setUp() throws Exception { getFeedRequest = mock(GetFeedRequest.class); getEntryRequest = mock(GetEntryRequest.class); + dynamoDBFeedSource.setArchiveUrl(new URL(ARCHIVE_LINK)); + // Mock GetEntryRequest when(getEntryRequest.getFeedName()).thenReturn(FEED_NAME); when(getEntryRequest.getEntryId()).thenReturn(MARKER_ID); @@ -87,7 +99,6 @@ public void setUp() throws Exception { when(getFeedRequest.getFeedName()).thenReturn(FEED_NAME); when(getFeedRequest.getPageSize()).thenReturn("25"); when(getFeedRequest.getAbdera()).thenReturn(abdera); - when(getFeedRequest.getDirection()).thenReturn(FORWARD); } @@ -108,8 +119,12 @@ public void shouldThrowExceptionForDelimiter() throws Exception { dynamoDBFeedSource.afterPropertiesSet(); } - @Test(expected = RuntimeException.class) - public void shouldReturnFeedWithCorrectTimeStampForForwardDirection() throws Exception { + @Test + public void shouldNotGetFeedWithMarkerDirectionForward() throws Exception { + Abdera localAbdera = new Abdera(); + when(getFeedRequest.getAbdera()).thenReturn(localAbdera); + when(getFeedRequest.getPageMarker()).thenReturn(MARKER_ID); + when(getFeedRequest.getDirection()).thenReturn("FORWARD"); final Table mockTable = mock(Table.class); when(dynamoDB.getTable(any(String.class))).thenReturn(mockTable); final Index mockIndex = mock(Index.class); @@ -121,12 +136,41 @@ public void shouldReturnFeedWithCorrectTimeStampForForwardDirection() throws Exc when(outcome.iterator()).thenReturn(mockIterator); when(mockIterator.hasNext()).thenReturn(false); when(mockIterator.next()).thenReturn(mockItem); - when(getFeedRequest.getDirection()).thenReturn(BACKWARD); - dynamoDBFeedSource.getFeedPageByTimestamp(getFeedRequest, "2014-03-10T00:00:00.000Z", 25); + assertEquals("Should get a 404 response", HttpStatus.NOT_FOUND, + dynamoDBFeedSource.getFeed(getFeedRequest).getResponseStatus()); + } + + @Test + public void shouldNotGetFeedWithMarkerDirectionBackward() throws Exception { + Abdera localAbdera = new Abdera(); + when(getFeedRequest.getAbdera()).thenReturn(localAbdera); + when(getFeedRequest.getPageMarker()).thenReturn(MARKER_ID); + when(getFeedRequest.getDirection()).thenReturn("BACKWARD"); + final Table mockTable = mock(Table.class); + when(dynamoDB.getTable(any(String.class))).thenReturn(mockTable); + final Index mockIndex = mock(Index.class); + when(mockTable.getIndex(anyString())).thenReturn(mockIndex); + final ItemCollection outcome = mock(ItemCollection.class); + when(mockIndex.query(any(QuerySpec.class))).thenReturn(outcome); + final IteratorSupport mockIterator = mock(IteratorSupport.class); + final Item mockItem = new Item(); + when(outcome.iterator()).thenReturn(mockIterator); + when(mockIterator.hasNext()).thenReturn(false); + when(getFeedRequest.getAbdera()).thenReturn(localAbdera); + when(getFeedRequest.getPageMarker()).thenReturn(MARKER_ID); + when(getFeedRequest.getDirection()).thenReturn("BACKWARD"); + assertEquals("Should get a 404 response", HttpStatus.NOT_FOUND, + dynamoDBFeedSource.getFeed(getFeedRequest).getResponseStatus()); } + @Test - public void shouldReturnFeedWithCorrectTimeStampForBackwardDirection() throws Exception { + public void shouldGetFeedHead() throws Exception { + Abdera localAbdera = new Abdera(); + List newList = new ArrayList<>(); + when(getFeedRequest.getDirection()).thenReturn("forward"); + when(getFeedRequest.getAbdera()).thenReturn(localAbdera); + when(getEntryRequest.getAbdera()).thenReturn(localAbdera); final Table mockTable = mock(Table.class); when(dynamoDB.getTable(any(String.class))).thenReturn(mockTable); final Index mockIndex = mock(Index.class); @@ -136,14 +180,19 @@ public void shouldReturnFeedWithCorrectTimeStampForBackwardDirection() throws Ex final IteratorSupport mockIterator = mock(IteratorSupport.class); final Item mockItem = new Item(); when(outcome.iterator()).thenReturn(mockIterator); - when(mockIterator.hasNext()).thenReturn(true, false); + when(mockIterator.hasNext()).thenReturn(false); when(mockIterator.next()).thenReturn(mockItem); - assertEquals(1, - dynamoDBFeedSource.getFeedPageByTimestamp(getFeedRequest, "2014-03-10T00:00:00.000Z", 25).size()); + newList.add("fe9eedc8-d10c-47a6-9209-f755ea8c35c3"); + newList.add(""); + DynamoDBFeedSource mainModel = Mockito.mock(DynamoDBFeedSource.class); + Mockito.when(mainModel.getQueryBuilderMethod(dynamoDB, "feed = :feed and dateLastUpdated < :dateLastUpdated", 25, new ValueMap(), true)).thenReturn(newList); + assertEquals("Should get a 200 response", HttpStatus.OK, + dynamoDBFeedSource.getFeed(getFeedRequest).getResponseStatus()); } - @Test(expected = IllegalArgumentException.class) - public void shouldThrowExceptionForFeedWithInCorrectTimeStampForForwardDirection() throws Exception { + + @Test(expected = RuntimeException.class) + public void shouldReturnFeedWithCorrectTimeStampForForwardDirection() throws Exception { final Table mockTable = mock(Table.class); when(dynamoDB.getTable(any(String.class))).thenReturn(mockTable); final Index mockIndex = mock(Index.class); @@ -153,9 +202,10 @@ public void shouldThrowExceptionForFeedWithInCorrectTimeStampForForwardDirection final IteratorSupport mockIterator = mock(IteratorSupport.class); final Item mockItem = new Item(); when(outcome.iterator()).thenReturn(mockIterator); - when(mockIterator.hasNext()).thenReturn(true, false); + when(mockIterator.hasNext()).thenReturn(false); when(mockIterator.next()).thenReturn(mockItem); - dynamoDBFeedSource.getFeedPageByTimestamp(getFeedRequest, "20140306T060000", 25).size(); + when(getFeedRequest.getDirection()).thenReturn(BACKWARD); + dynamoDBFeedSource.getFeedPageByTimestamp(getFeedRequest, "2014-03-10T00:00:00.000Z", 25); } From cd21ca5b0c2cbf0fb6ac18e185c6e2534ae5dd82 Mon Sep 17 00:00:00 2001 From: shub6691 Date: Mon, 18 Oct 2021 18:55:18 +0530 Subject: [PATCH 07/21] Fetched the spring version from properties --- pom.xml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 814ea279..3a81ca27 100644 --- a/pom.xml +++ b/pom.xml @@ -19,6 +19,7 @@ 2.2 3.0.1 9.4.17.v20190418 + 4.2.5.RELEASE @@ -54,7 +55,7 @@ org.springframework spring-expression - 4.2.5.RELEASE + ${spring.version} From 1fa408facfaab312682cf45c13ab9d506bbe4c3c Mon Sep 17 00:00:00 2001 From: shub6691 Date: Tue, 19 Oct 2021 17:35:31 +0530 Subject: [PATCH 08/21] Revert the changes back to previous application-context and web.xml file --- .../webapp/META-INF/application-context.xml | 194 +++++------------- atomhopper/src/main/webapp/WEB-INF/web.xml | 8 +- 2 files changed, 59 insertions(+), 143 deletions(-) diff --git a/atomhopper/src/main/webapp/META-INF/application-context.xml b/atomhopper/src/main/webapp/META-INF/application-context.xml index ee37231d..8b17dbfd 100644 --- a/atomhopper/src/main/webapp/META-INF/application-context.xml +++ b/atomhopper/src/main/webapp/META-INF/application-context.xml @@ -14,60 +14,60 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - com.amazonaws.regions.Regions - - - fromName - - - - ap-south-1 - - - - - - com.amazonaws.regions.Region - - - getRegion - - - - - - - - - - - - - setRegion - - - - - - - - - - - - - - setEndpoint - - - - https://dynamodb.ap-south-1.amazonaws.com - - - - - - - - - - - - - - + \ No newline at end of file diff --git a/atomhopper/src/main/webapp/WEB-INF/web.xml b/atomhopper/src/main/webapp/WEB-INF/web.xml index 1114d39f..8fb8d65c 100644 --- a/atomhopper/src/main/webapp/WEB-INF/web.xml +++ b/atomhopper/src/main/webapp/WEB-INF/web.xml @@ -14,7 +14,7 @@ --> file:///etc/atomhopper/application-context.xml - + org.atomhopper.ExternalConfigLoaderContextListener @@ -85,14 +85,14 @@ Atom-Hopper-Metrics /atommetrics/* - + logback/configuration-resource java.lang.String file:///etc/atomhopper/logback.xml - + ch.qos.logback.classic.selector.servlet.ContextDetachingSCL - + \ No newline at end of file From 4d69b05c8179674ccc55fcc816416c872e765ecb Mon Sep 17 00:00:00 2001 From: kavishkhanna Date: Thu, 22 Sep 2022 10:28:36 +0530 Subject: [PATCH 09/21] Temporary changes for war commit --- pom.xml | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/pom.xml b/pom.xml index 72fc2876..54ba6cba 100644 --- a/pom.xml +++ b/pom.xml @@ -384,21 +384,23 @@ Rackspace Research Repository https://maven.research.rackspacecloud.com/content/groups/public/ + + central + IAD Artifacts-releases + https://artifacts.rackspace.net:443/artifactory/cloudfeeds-maven-local + - releases.maven.research.rackspace.com - - Rackspace Research Releases - https://maven.research.rackspacecloud.com/content/repositories/releases + central + IAD Artifacts-releases + https://artifacts.rackspace.net:443/artifactory/cloudfeeds-maven-local - - snapshots.maven.research.rackspace.com - - Rackspace Research Snapshots - https://maven.research.rackspacecloud.com/content/repositories/snapshots + snapshots + IAD Artifacts-snapshots + https://artifacts.rackspace.net:443/artifactory/cloudfeeds-maven-local From 231cdc0896f1a9477174030dfad93f5e31c5d950 Mon Sep 17 00:00:00 2001 From: kavishkhanna Date: Thu, 22 Sep 2022 21:36:22 +0530 Subject: [PATCH 10/21] Changes for Supporting Dynamodb Adapter --- atomhopper/pom.xml | 31 +++ .../resources/META-INF/atom-server.cfg.xml | 14 +- .../webapp/META-INF/application-context.xml | 201 ++++++++++++------ 3 files changed, 174 insertions(+), 72 deletions(-) diff --git a/atomhopper/pom.xml b/atomhopper/pom.xml index c0c83b14..6688864d 100644 --- a/atomhopper/pom.xml +++ b/atomhopper/pom.xml @@ -98,6 +98,37 @@ logback-gelf + + com.amazonaws + aws-java-sdk-dynamodb + 1.11.34 + + + + org.springframework + spring-expression + 4.2.5.RELEASE + + + + org.atomhopper.adapter + dynamoDB_adapters + 1.2.35-SNAPSHOT + + + + jakarta.xml.bind + jakarta.xml.bind-api + 2.3.2 + + + + + org.glassfish.jaxb + jaxb-runtime + 2.3.2 + + com.yammer.metrics metrics-core diff --git a/atomhopper/src/main/resources/META-INF/atom-server.cfg.xml b/atomhopper/src/main/resources/META-INF/atom-server.cfg.xml index 49967182..85e19f12 100644 --- a/atomhopper/src/main/resources/META-INF/atom-server.cfg.xml +++ b/atomhopper/src/main/resources/META-INF/atom-server.cfg.xml @@ -1,8 +1,6 @@ @@ -15,23 +13,23 @@ NOTE: Place this file in the following folder: /etc/atomhopper/atom-server.cfg.x will be incorrect. Scheme should be either "http" or "https" --> - + - + - - + + - + \ No newline at end of file diff --git a/atomhopper/src/main/webapp/META-INF/application-context.xml b/atomhopper/src/main/webapp/META-INF/application-context.xml index 8b17dbfd..3baade67 100644 --- a/atomhopper/src/main/webapp/META-INF/application-context.xml +++ b/atomhopper/src/main/webapp/META-INF/application-context.xml @@ -14,60 +14,60 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + com.amazonaws.regions.Regions + + + fromName + + + + us-east-1 + + + + + + com.amazonaws.regions.Region + + + getRegion + + + + + + + + + + + + + setRegion + + + + + + + + + + + + + + setEndpoint + + + + https://dynamodb.us-east-1.amazonaws.com + + + + + + + + + + + + \ No newline at end of file From c5709931fa01771785f0a474edf7deceaa107340 Mon Sep 17 00:00:00 2001 From: kavishkhanna Date: Thu, 22 Sep 2022 22:12:14 +0530 Subject: [PATCH 11/21] [maven-release-plugin] prepare release parent-1.2.35 --- adapters/dynamoDB_adapters/pom.xml | 6 ++---- adapters/hibernate/pom.xml | 2 +- adapters/jdbc/pom.xml | 2 +- adapters/migration/pom.xml | 2 +- adapters/mongodb/pom.xml | 2 +- adapters/postgres-adapter/pom.xml | 2 +- atomhopper/pom.xml | 6 +++--- documentation/pom.xml | 2 +- hopper/pom.xml | 2 +- pom.xml | 4 ++-- server/pom.xml | 2 +- test-suite/pom.xml | 2 +- test-util/pom.xml | 2 +- 13 files changed, 17 insertions(+), 19 deletions(-) diff --git a/adapters/dynamoDB_adapters/pom.xml b/adapters/dynamoDB_adapters/pom.xml index 3b21318a..78b9310d 100644 --- a/adapters/dynamoDB_adapters/pom.xml +++ b/adapters/dynamoDB_adapters/pom.xml @@ -1,11 +1,9 @@ - + parent org.atomhopper - 1.2.35-SNAPSHOT + 1.2.35 ../../pom.xml 4.0.0 diff --git a/adapters/hibernate/pom.xml b/adapters/hibernate/pom.xml index eaf139ca..c272806e 100644 --- a/adapters/hibernate/pom.xml +++ b/adapters/hibernate/pom.xml @@ -5,7 +5,7 @@ org.atomhopper parent - 1.2.35-SNAPSHOT + 1.2.35 ./../../pom.xml diff --git a/adapters/jdbc/pom.xml b/adapters/jdbc/pom.xml index 3c00eb0e..dc04674b 100644 --- a/adapters/jdbc/pom.xml +++ b/adapters/jdbc/pom.xml @@ -5,7 +5,7 @@ org.atomhopper parent - 1.2.35-SNAPSHOT + 1.2.35 ./../../pom.xml diff --git a/adapters/migration/pom.xml b/adapters/migration/pom.xml index 83e2f4fd..291e1350 100644 --- a/adapters/migration/pom.xml +++ b/adapters/migration/pom.xml @@ -5,7 +5,7 @@ org.atomhopper parent - 1.2.35-SNAPSHOT + 1.2.35 ./../../pom.xml diff --git a/adapters/mongodb/pom.xml b/adapters/mongodb/pom.xml index 35436835..2eebe3f8 100644 --- a/adapters/mongodb/pom.xml +++ b/adapters/mongodb/pom.xml @@ -5,7 +5,7 @@ org.atomhopper parent - 1.2.35-SNAPSHOT + 1.2.35 ./../../pom.xml diff --git a/adapters/postgres-adapter/pom.xml b/adapters/postgres-adapter/pom.xml index a62cf3d2..b36a0f78 100644 --- a/adapters/postgres-adapter/pom.xml +++ b/adapters/postgres-adapter/pom.xml @@ -5,7 +5,7 @@ org.atomhopper parent - 1.2.35-SNAPSHOT + 1.2.35 ./../../pom.xml diff --git a/atomhopper/pom.xml b/atomhopper/pom.xml index 6688864d..3771e0bf 100644 --- a/atomhopper/pom.xml +++ b/atomhopper/pom.xml @@ -5,7 +5,7 @@ org.atomhopper parent - 1.2.35-SNAPSHOT + 1.2.35 org.atomhopper @@ -65,7 +65,7 @@ org.atomhopper.adapter dynamoDB_adapters - 1.2.35-SNAPSHOT + 1.2.35 @@ -113,7 +113,7 @@ org.atomhopper.adapter dynamoDB_adapters - 1.2.35-SNAPSHOT + 1.2.35 diff --git a/documentation/pom.xml b/documentation/pom.xml index 9a9da839..863fd974 100644 --- a/documentation/pom.xml +++ b/documentation/pom.xml @@ -5,7 +5,7 @@ org.atomhopper parent - 1.2.35-SNAPSHOT + 1.2.35 org.atomhopper diff --git a/hopper/pom.xml b/hopper/pom.xml index d17f416b..c65076ba 100644 --- a/hopper/pom.xml +++ b/hopper/pom.xml @@ -5,7 +5,7 @@ org.atomhopper parent - 1.2.35-SNAPSHOT + 1.2.35 org.atomhopper diff --git a/pom.xml b/pom.xml index 54ba6cba..79df92d9 100644 --- a/pom.xml +++ b/pom.xml @@ -5,7 +5,7 @@ org.atomhopper parent pom - 1.2.35-SNAPSHOT + 1.2.35 ATOM Hopper - ATOMpub Server Collection http://atomhopper.org/ @@ -46,7 +46,7 @@ scm:git:ssh://git@github.com/rackerlabs/atom-hopper.git - parent-1.2.32 + parent-1.2.35 diff --git a/server/pom.xml b/server/pom.xml index 5d19438e..87d4e58e 100644 --- a/server/pom.xml +++ b/server/pom.xml @@ -5,7 +5,7 @@ org.atomhopper parent - 1.2.35-SNAPSHOT + 1.2.35 org.atomhopper diff --git a/test-suite/pom.xml b/test-suite/pom.xml index 0576600f..47fc67dd 100644 --- a/test-suite/pom.xml +++ b/test-suite/pom.xml @@ -5,7 +5,7 @@ org.atomhopper parent - 1.2.35-SNAPSHOT + 1.2.35 org.atomhopper diff --git a/test-util/pom.xml b/test-util/pom.xml index 4526ead6..65fb42eb 100644 --- a/test-util/pom.xml +++ b/test-util/pom.xml @@ -5,7 +5,7 @@ org.atomhopper parent - 1.2.35-SNAPSHOT + 1.2.35 org.atomhopper From 02fd863d62b08f07a4c2aa71ccceebde52e80d9d Mon Sep 17 00:00:00 2001 From: kavishkhanna Date: Thu, 22 Sep 2022 22:12:24 +0530 Subject: [PATCH 12/21] [maven-release-plugin] prepare for next development iteration --- adapters/dynamoDB_adapters/pom.xml | 2 +- adapters/hibernate/pom.xml | 2 +- adapters/jdbc/pom.xml | 2 +- adapters/migration/pom.xml | 2 +- adapters/mongodb/pom.xml | 2 +- adapters/postgres-adapter/pom.xml | 2 +- atomhopper/pom.xml | 6 +++--- documentation/pom.xml | 2 +- hopper/pom.xml | 2 +- pom.xml | 4 ++-- server/pom.xml | 2 +- test-suite/pom.xml | 2 +- test-util/pom.xml | 2 +- 13 files changed, 16 insertions(+), 16 deletions(-) diff --git a/adapters/dynamoDB_adapters/pom.xml b/adapters/dynamoDB_adapters/pom.xml index 78b9310d..10193e40 100644 --- a/adapters/dynamoDB_adapters/pom.xml +++ b/adapters/dynamoDB_adapters/pom.xml @@ -3,7 +3,7 @@ parent org.atomhopper - 1.2.35 + 1.2.36-SNAPSHOT ../../pom.xml 4.0.0 diff --git a/adapters/hibernate/pom.xml b/adapters/hibernate/pom.xml index c272806e..993a73a5 100644 --- a/adapters/hibernate/pom.xml +++ b/adapters/hibernate/pom.xml @@ -5,7 +5,7 @@ org.atomhopper parent - 1.2.35 + 1.2.36-SNAPSHOT ./../../pom.xml diff --git a/adapters/jdbc/pom.xml b/adapters/jdbc/pom.xml index dc04674b..d39835d8 100644 --- a/adapters/jdbc/pom.xml +++ b/adapters/jdbc/pom.xml @@ -5,7 +5,7 @@ org.atomhopper parent - 1.2.35 + 1.2.36-SNAPSHOT ./../../pom.xml diff --git a/adapters/migration/pom.xml b/adapters/migration/pom.xml index 291e1350..686c59d7 100644 --- a/adapters/migration/pom.xml +++ b/adapters/migration/pom.xml @@ -5,7 +5,7 @@ org.atomhopper parent - 1.2.35 + 1.2.36-SNAPSHOT ./../../pom.xml diff --git a/adapters/mongodb/pom.xml b/adapters/mongodb/pom.xml index 2eebe3f8..8362e1a0 100644 --- a/adapters/mongodb/pom.xml +++ b/adapters/mongodb/pom.xml @@ -5,7 +5,7 @@ org.atomhopper parent - 1.2.35 + 1.2.36-SNAPSHOT ./../../pom.xml diff --git a/adapters/postgres-adapter/pom.xml b/adapters/postgres-adapter/pom.xml index b36a0f78..b3f67e4b 100644 --- a/adapters/postgres-adapter/pom.xml +++ b/adapters/postgres-adapter/pom.xml @@ -5,7 +5,7 @@ org.atomhopper parent - 1.2.35 + 1.2.36-SNAPSHOT ./../../pom.xml diff --git a/atomhopper/pom.xml b/atomhopper/pom.xml index 3771e0bf..1be30f92 100644 --- a/atomhopper/pom.xml +++ b/atomhopper/pom.xml @@ -5,7 +5,7 @@ org.atomhopper parent - 1.2.35 + 1.2.36-SNAPSHOT org.atomhopper @@ -65,7 +65,7 @@ org.atomhopper.adapter dynamoDB_adapters - 1.2.35 + 1.2.36-SNAPSHOT @@ -113,7 +113,7 @@ org.atomhopper.adapter dynamoDB_adapters - 1.2.35 + 1.2.36-SNAPSHOT diff --git a/documentation/pom.xml b/documentation/pom.xml index 863fd974..fcbd5c7d 100644 --- a/documentation/pom.xml +++ b/documentation/pom.xml @@ -5,7 +5,7 @@ org.atomhopper parent - 1.2.35 + 1.2.36-SNAPSHOT org.atomhopper diff --git a/hopper/pom.xml b/hopper/pom.xml index c65076ba..794a72cf 100644 --- a/hopper/pom.xml +++ b/hopper/pom.xml @@ -5,7 +5,7 @@ org.atomhopper parent - 1.2.35 + 1.2.36-SNAPSHOT org.atomhopper diff --git a/pom.xml b/pom.xml index 79df92d9..78204604 100644 --- a/pom.xml +++ b/pom.xml @@ -5,7 +5,7 @@ org.atomhopper parent pom - 1.2.35 + 1.2.36-SNAPSHOT ATOM Hopper - ATOMpub Server Collection http://atomhopper.org/ @@ -46,7 +46,7 @@ scm:git:ssh://git@github.com/rackerlabs/atom-hopper.git - parent-1.2.35 + parent-1.2.32 diff --git a/server/pom.xml b/server/pom.xml index 87d4e58e..832d1ecb 100644 --- a/server/pom.xml +++ b/server/pom.xml @@ -5,7 +5,7 @@ org.atomhopper parent - 1.2.35 + 1.2.36-SNAPSHOT org.atomhopper diff --git a/test-suite/pom.xml b/test-suite/pom.xml index 47fc67dd..7609a1b8 100644 --- a/test-suite/pom.xml +++ b/test-suite/pom.xml @@ -5,7 +5,7 @@ org.atomhopper parent - 1.2.35 + 1.2.36-SNAPSHOT org.atomhopper diff --git a/test-util/pom.xml b/test-util/pom.xml index 65fb42eb..63a05005 100644 --- a/test-util/pom.xml +++ b/test-util/pom.xml @@ -5,7 +5,7 @@ org.atomhopper parent - 1.2.35 + 1.2.36-SNAPSHOT org.atomhopper From 7939bdd1365946bc0219e05e85aba64868620999 Mon Sep 17 00:00:00 2001 From: kavishkhanna Date: Thu, 22 Sep 2022 22:22:59 +0530 Subject: [PATCH 13/21] Revert "[maven-release-plugin] prepare for next development iteration" This reverts commit 02fd863d62b08f07a4c2aa71ccceebde52e80d9d. --- adapters/dynamoDB_adapters/pom.xml | 2 +- adapters/hibernate/pom.xml | 2 +- adapters/jdbc/pom.xml | 2 +- adapters/migration/pom.xml | 2 +- adapters/mongodb/pom.xml | 2 +- adapters/postgres-adapter/pom.xml | 2 +- atomhopper/pom.xml | 6 +++--- documentation/pom.xml | 2 +- hopper/pom.xml | 2 +- pom.xml | 4 ++-- server/pom.xml | 2 +- test-suite/pom.xml | 2 +- test-util/pom.xml | 2 +- 13 files changed, 16 insertions(+), 16 deletions(-) diff --git a/adapters/dynamoDB_adapters/pom.xml b/adapters/dynamoDB_adapters/pom.xml index 10193e40..78b9310d 100644 --- a/adapters/dynamoDB_adapters/pom.xml +++ b/adapters/dynamoDB_adapters/pom.xml @@ -3,7 +3,7 @@ parent org.atomhopper - 1.2.36-SNAPSHOT + 1.2.35 ../../pom.xml 4.0.0 diff --git a/adapters/hibernate/pom.xml b/adapters/hibernate/pom.xml index 993a73a5..c272806e 100644 --- a/adapters/hibernate/pom.xml +++ b/adapters/hibernate/pom.xml @@ -5,7 +5,7 @@ org.atomhopper parent - 1.2.36-SNAPSHOT + 1.2.35 ./../../pom.xml diff --git a/adapters/jdbc/pom.xml b/adapters/jdbc/pom.xml index d39835d8..dc04674b 100644 --- a/adapters/jdbc/pom.xml +++ b/adapters/jdbc/pom.xml @@ -5,7 +5,7 @@ org.atomhopper parent - 1.2.36-SNAPSHOT + 1.2.35 ./../../pom.xml diff --git a/adapters/migration/pom.xml b/adapters/migration/pom.xml index 686c59d7..291e1350 100644 --- a/adapters/migration/pom.xml +++ b/adapters/migration/pom.xml @@ -5,7 +5,7 @@ org.atomhopper parent - 1.2.36-SNAPSHOT + 1.2.35 ./../../pom.xml diff --git a/adapters/mongodb/pom.xml b/adapters/mongodb/pom.xml index 8362e1a0..2eebe3f8 100644 --- a/adapters/mongodb/pom.xml +++ b/adapters/mongodb/pom.xml @@ -5,7 +5,7 @@ org.atomhopper parent - 1.2.36-SNAPSHOT + 1.2.35 ./../../pom.xml diff --git a/adapters/postgres-adapter/pom.xml b/adapters/postgres-adapter/pom.xml index b3f67e4b..b36a0f78 100644 --- a/adapters/postgres-adapter/pom.xml +++ b/adapters/postgres-adapter/pom.xml @@ -5,7 +5,7 @@ org.atomhopper parent - 1.2.36-SNAPSHOT + 1.2.35 ./../../pom.xml diff --git a/atomhopper/pom.xml b/atomhopper/pom.xml index 1be30f92..3771e0bf 100644 --- a/atomhopper/pom.xml +++ b/atomhopper/pom.xml @@ -5,7 +5,7 @@ org.atomhopper parent - 1.2.36-SNAPSHOT + 1.2.35 org.atomhopper @@ -65,7 +65,7 @@ org.atomhopper.adapter dynamoDB_adapters - 1.2.36-SNAPSHOT + 1.2.35 @@ -113,7 +113,7 @@ org.atomhopper.adapter dynamoDB_adapters - 1.2.36-SNAPSHOT + 1.2.35 diff --git a/documentation/pom.xml b/documentation/pom.xml index fcbd5c7d..863fd974 100644 --- a/documentation/pom.xml +++ b/documentation/pom.xml @@ -5,7 +5,7 @@ org.atomhopper parent - 1.2.36-SNAPSHOT + 1.2.35 org.atomhopper diff --git a/hopper/pom.xml b/hopper/pom.xml index 794a72cf..c65076ba 100644 --- a/hopper/pom.xml +++ b/hopper/pom.xml @@ -5,7 +5,7 @@ org.atomhopper parent - 1.2.36-SNAPSHOT + 1.2.35 org.atomhopper diff --git a/pom.xml b/pom.xml index 78204604..79df92d9 100644 --- a/pom.xml +++ b/pom.xml @@ -5,7 +5,7 @@ org.atomhopper parent pom - 1.2.36-SNAPSHOT + 1.2.35 ATOM Hopper - ATOMpub Server Collection http://atomhopper.org/ @@ -46,7 +46,7 @@ scm:git:ssh://git@github.com/rackerlabs/atom-hopper.git - parent-1.2.32 + parent-1.2.35 diff --git a/server/pom.xml b/server/pom.xml index 832d1ecb..87d4e58e 100644 --- a/server/pom.xml +++ b/server/pom.xml @@ -5,7 +5,7 @@ org.atomhopper parent - 1.2.36-SNAPSHOT + 1.2.35 org.atomhopper diff --git a/test-suite/pom.xml b/test-suite/pom.xml index 7609a1b8..47fc67dd 100644 --- a/test-suite/pom.xml +++ b/test-suite/pom.xml @@ -5,7 +5,7 @@ org.atomhopper parent - 1.2.36-SNAPSHOT + 1.2.35 org.atomhopper diff --git a/test-util/pom.xml b/test-util/pom.xml index 63a05005..65fb42eb 100644 --- a/test-util/pom.xml +++ b/test-util/pom.xml @@ -5,7 +5,7 @@ org.atomhopper parent - 1.2.36-SNAPSHOT + 1.2.35 org.atomhopper From afe11b6b92f63c1a554df83d4609643a672a0b5d Mon Sep 17 00:00:00 2001 From: kavishkhanna Date: Thu, 22 Sep 2022 22:23:02 +0530 Subject: [PATCH 14/21] Revert "[maven-release-plugin] prepare release parent-1.2.35" This reverts commit c5709931fa01771785f0a474edf7deceaa107340. --- adapters/dynamoDB_adapters/pom.xml | 6 ++++-- adapters/hibernate/pom.xml | 2 +- adapters/jdbc/pom.xml | 2 +- adapters/migration/pom.xml | 2 +- adapters/mongodb/pom.xml | 2 +- adapters/postgres-adapter/pom.xml | 2 +- atomhopper/pom.xml | 6 +++--- documentation/pom.xml | 2 +- hopper/pom.xml | 2 +- pom.xml | 4 ++-- server/pom.xml | 2 +- test-suite/pom.xml | 2 +- test-util/pom.xml | 2 +- 13 files changed, 19 insertions(+), 17 deletions(-) diff --git a/adapters/dynamoDB_adapters/pom.xml b/adapters/dynamoDB_adapters/pom.xml index 78b9310d..3b21318a 100644 --- a/adapters/dynamoDB_adapters/pom.xml +++ b/adapters/dynamoDB_adapters/pom.xml @@ -1,9 +1,11 @@ - + parent org.atomhopper - 1.2.35 + 1.2.35-SNAPSHOT ../../pom.xml 4.0.0 diff --git a/adapters/hibernate/pom.xml b/adapters/hibernate/pom.xml index c272806e..eaf139ca 100644 --- a/adapters/hibernate/pom.xml +++ b/adapters/hibernate/pom.xml @@ -5,7 +5,7 @@ org.atomhopper parent - 1.2.35 + 1.2.35-SNAPSHOT ./../../pom.xml diff --git a/adapters/jdbc/pom.xml b/adapters/jdbc/pom.xml index dc04674b..3c00eb0e 100644 --- a/adapters/jdbc/pom.xml +++ b/adapters/jdbc/pom.xml @@ -5,7 +5,7 @@ org.atomhopper parent - 1.2.35 + 1.2.35-SNAPSHOT ./../../pom.xml diff --git a/adapters/migration/pom.xml b/adapters/migration/pom.xml index 291e1350..83e2f4fd 100644 --- a/adapters/migration/pom.xml +++ b/adapters/migration/pom.xml @@ -5,7 +5,7 @@ org.atomhopper parent - 1.2.35 + 1.2.35-SNAPSHOT ./../../pom.xml diff --git a/adapters/mongodb/pom.xml b/adapters/mongodb/pom.xml index 2eebe3f8..35436835 100644 --- a/adapters/mongodb/pom.xml +++ b/adapters/mongodb/pom.xml @@ -5,7 +5,7 @@ org.atomhopper parent - 1.2.35 + 1.2.35-SNAPSHOT ./../../pom.xml diff --git a/adapters/postgres-adapter/pom.xml b/adapters/postgres-adapter/pom.xml index b36a0f78..a62cf3d2 100644 --- a/adapters/postgres-adapter/pom.xml +++ b/adapters/postgres-adapter/pom.xml @@ -5,7 +5,7 @@ org.atomhopper parent - 1.2.35 + 1.2.35-SNAPSHOT ./../../pom.xml diff --git a/atomhopper/pom.xml b/atomhopper/pom.xml index 3771e0bf..6688864d 100644 --- a/atomhopper/pom.xml +++ b/atomhopper/pom.xml @@ -5,7 +5,7 @@ org.atomhopper parent - 1.2.35 + 1.2.35-SNAPSHOT org.atomhopper @@ -65,7 +65,7 @@ org.atomhopper.adapter dynamoDB_adapters - 1.2.35 + 1.2.35-SNAPSHOT @@ -113,7 +113,7 @@ org.atomhopper.adapter dynamoDB_adapters - 1.2.35 + 1.2.35-SNAPSHOT diff --git a/documentation/pom.xml b/documentation/pom.xml index 863fd974..9a9da839 100644 --- a/documentation/pom.xml +++ b/documentation/pom.xml @@ -5,7 +5,7 @@ org.atomhopper parent - 1.2.35 + 1.2.35-SNAPSHOT org.atomhopper diff --git a/hopper/pom.xml b/hopper/pom.xml index c65076ba..d17f416b 100644 --- a/hopper/pom.xml +++ b/hopper/pom.xml @@ -5,7 +5,7 @@ org.atomhopper parent - 1.2.35 + 1.2.35-SNAPSHOT org.atomhopper diff --git a/pom.xml b/pom.xml index 79df92d9..54ba6cba 100644 --- a/pom.xml +++ b/pom.xml @@ -5,7 +5,7 @@ org.atomhopper parent pom - 1.2.35 + 1.2.35-SNAPSHOT ATOM Hopper - ATOMpub Server Collection http://atomhopper.org/ @@ -46,7 +46,7 @@ scm:git:ssh://git@github.com/rackerlabs/atom-hopper.git - parent-1.2.35 + parent-1.2.32 diff --git a/server/pom.xml b/server/pom.xml index 87d4e58e..5d19438e 100644 --- a/server/pom.xml +++ b/server/pom.xml @@ -5,7 +5,7 @@ org.atomhopper parent - 1.2.35 + 1.2.35-SNAPSHOT org.atomhopper diff --git a/test-suite/pom.xml b/test-suite/pom.xml index 47fc67dd..0576600f 100644 --- a/test-suite/pom.xml +++ b/test-suite/pom.xml @@ -5,7 +5,7 @@ org.atomhopper parent - 1.2.35 + 1.2.35-SNAPSHOT org.atomhopper diff --git a/test-util/pom.xml b/test-util/pom.xml index 65fb42eb..4526ead6 100644 --- a/test-util/pom.xml +++ b/test-util/pom.xml @@ -5,7 +5,7 @@ org.atomhopper parent - 1.2.35 + 1.2.35-SNAPSHOT org.atomhopper From a8798a120a9a4c0921a43456a06d04bcc79fd030 Mon Sep 17 00:00:00 2001 From: kavishkhanna Date: Mon, 26 Sep 2022 16:27:27 +0530 Subject: [PATCH 15/21] Revert Temporary changes for war commit --- pom.xml | 21 ++++++++------------- 1 file changed, 8 insertions(+), 13 deletions(-) diff --git a/pom.xml b/pom.xml index 54ba6cba..b2692840 100644 --- a/pom.xml +++ b/pom.xml @@ -384,24 +384,19 @@ Rackspace Research Repository https://maven.research.rackspacecloud.com/content/groups/public/ - - central - IAD Artifacts-releases - https://artifacts.rackspace.net:443/artifactory/cloudfeeds-maven-local - - central - IAD Artifacts-releases - https://artifacts.rackspace.net:443/artifactory/cloudfeeds-maven-local - + releases.maven.research.rackspace.com + + Rackspace Research Releases + https://maven.research.rackspacecloud.com/content/repositories/releases - snapshots - IAD Artifacts-snapshots - https://artifacts.rackspace.net:443/artifactory/cloudfeeds-maven-local - + snapshots.maven.research.rackspace.com + + Rackspace Research Snapshots + https://maven.research.rackspacecloud.com/content/repositories/snapshots From c1cea7a9bef73ec0c51f4449a9433a6af4be953e Mon Sep 17 00:00:00 2001 From: kavishkhanna Date: Tue, 11 Oct 2022 13:42:47 +0530 Subject: [PATCH 16/21] CF-3271 | Service Account changes. Removing the constructor arguments for Service Account changes. --- atomhopper/src/main/webapp/META-INF/application-context.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/atomhopper/src/main/webapp/META-INF/application-context.xml b/atomhopper/src/main/webapp/META-INF/application-context.xml index 3baade67..67d402a3 100644 --- a/atomhopper/src/main/webapp/META-INF/application-context.xml +++ b/atomhopper/src/main/webapp/META-INF/application-context.xml @@ -148,8 +148,8 @@ - - + Date: Fri, 2 Dec 2022 14:26:26 +0530 Subject: [PATCH 17/21] Latest Version of dynamo db sdk --- adapters/dynamoDB_adapters/pom.xml | 2 +- atomhopper/pom.xml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/adapters/dynamoDB_adapters/pom.xml b/adapters/dynamoDB_adapters/pom.xml index 3b21318a..af7b89b4 100644 --- a/adapters/dynamoDB_adapters/pom.xml +++ b/adapters/dynamoDB_adapters/pom.xml @@ -42,7 +42,7 @@ com.amazonaws aws-java-sdk-dynamodb - 1.11.34 + 1.12.352 diff --git a/atomhopper/pom.xml b/atomhopper/pom.xml index 6688864d..aba962d0 100644 --- a/atomhopper/pom.xml +++ b/atomhopper/pom.xml @@ -23,7 +23,7 @@ com.amazonaws aws-java-sdk-dynamodb - 1.11.34 + 1.12.352 @@ -101,7 +101,7 @@ com.amazonaws aws-java-sdk-dynamodb - 1.11.34 + 1.12.352 From bed7e22fcad71ffd50943251bc3ff9a29a598388 Mon Sep 17 00:00:00 2001 From: kavishkhanna Date: Thu, 22 Dec 2022 15:00:38 +0530 Subject: [PATCH 18/21] Removing duplicate dependency --- atomhopper/pom.xml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/atomhopper/pom.xml b/atomhopper/pom.xml index aba962d0..d8cf695f 100644 --- a/atomhopper/pom.xml +++ b/atomhopper/pom.xml @@ -97,12 +97,6 @@ me.moocar logback-gelf - - - com.amazonaws - aws-java-sdk-dynamodb - 1.12.352 - org.springframework From 4e27abd2fae58620f246fe19b045c832edae1ebf Mon Sep 17 00:00:00 2001 From: Kavish Khanna Date: Mon, 11 Mar 2024 15:59:50 +0530 Subject: [PATCH 19/21] CF-3853 | Dynamodb changes --- .../adapter/DynamoDBFeedPublisher.java | 25 ++- .../dynamodb/adapter/DynamoDBFeedSource.java | 144 +++++++++++------- .../dynamodb/model/PersistedEntry.java | 5 +- .../dynamodb/query/DynamoDBQueryBuilder.java | 3 + .../dynamodb/query/SQLToNoSqlConverter.java | 6 +- .../src/test/java/DynamoDBFeedSourceTest.java | 2 +- 6 files changed, 124 insertions(+), 61 deletions(-) diff --git a/adapters/dynamoDB_adapters/src/main/java/org/atomhopper/dynamodb/adapter/DynamoDBFeedPublisher.java b/adapters/dynamoDB_adapters/src/main/java/org/atomhopper/dynamodb/adapter/DynamoDBFeedPublisher.java index b283c3a2..4c039773 100644 --- a/adapters/dynamoDB_adapters/src/main/java/org/atomhopper/dynamodb/adapter/DynamoDBFeedPublisher.java +++ b/adapters/dynamoDB_adapters/src/main/java/org/atomhopper/dynamodb/adapter/DynamoDBFeedPublisher.java @@ -122,9 +122,23 @@ public AdapterResponse postEntry(PostEntryRequest postEntryRequest) { persistedEntry.setFeed(postEntryRequest.getFeedName()); persistedEntry.setEntryBody(entryToString(abderaParsedEntry)); - abderaParsedEntry.setUpdated(persistedEntry.getDateLastUpdated()); - abderaParsedEntry.setPublished(persistedEntry.getCreationDate()); - mapper.save(persistedEntry);//dynamoDB save object + + // abderaParsedEntry.setPublished(persistedEntry.getCreationDate()); + + try{ + if(null != abderaParsedEntry.getUpdated()){ + persistedEntry.setDateLastUpdated(getDateFormatInStringAWS(abderaParsedEntry.getUpdated())); + }else{ + persistedEntry.setDateLastUpdated(getDateFormatInStringAWS(new Date())); + } + + abderaParsedEntry.setUpdated(persistedEntry.getDateLastUpdated()); + abderaParsedEntry.setPublished(persistedEntry.getCreationDate()); + mapper.save(persistedEntry); + }catch(Exception e){ + } + + //dynamoDB save object incrementCounterForFeed(postEntryRequest.getFeedName()); return ResponseBuilder.created(abderaParsedEntry); } @@ -215,4 +229,9 @@ private String getDateFormatInString(Date date) { Format formatter = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); return formatter.format(date); } + + private static String getDateFormatInStringAWS(Date date) { + Format formatter = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'"); + return formatter.format(date); + } } diff --git a/adapters/dynamoDB_adapters/src/main/java/org/atomhopper/dynamodb/adapter/DynamoDBFeedSource.java b/adapters/dynamoDB_adapters/src/main/java/org/atomhopper/dynamodb/adapter/DynamoDBFeedSource.java index b334bcc1..77e46494 100644 --- a/adapters/dynamoDB_adapters/src/main/java/org/atomhopper/dynamodb/adapter/DynamoDBFeedSource.java +++ b/adapters/dynamoDB_adapters/src/main/java/org/atomhopper/dynamodb/adapter/DynamoDBFeedSource.java @@ -148,20 +148,20 @@ public void setDynamoDB(DynamoDB dynamoDB) { * @return List of PersistedEntry data found based on above params from db. */ public List getFeedBackward(String feedName, - Date markerTimestamp, - long markerId, + String markerTimestamp, + String markerId, String searchString, - int pageSize) { + int pageSize) { List feedPage; DynamoDBQueryBuilder sqlBac = new DynamoDBQueryBuilder(getSearchToSqlConverter()).searchString(searchString); sqlBac.searchType(SearchType.FEED_BACKWARD); //Dynamodb query implementation Map map = new HashMap(); String filters = sqlBac.getFilters(map); - String feedNameFilter = " and feed= :feedName"; - SimpleDateFormat dateFormatter = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'"); - dateFormatter.setTimeZone(TimeZone.getTimeZone("UTC")); - String dateLastUpdated = dateFormatter.format(markerTimestamp); + String feedNameFilter = "feed = :feedName and "; + // SimpleDateFormat dateFormatter = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'"); + // dateFormatter.setTimeZone(TimeZone.getTimeZone("UTC")); + String dateLastUpdated = markerTimestamp; Map valueMap = new HashMap(); valueMap.put(":id", new AttributeValue().withS(String.valueOf(markerId))); valueMap.put(":dateLastUpdated", new AttributeValue().withS(dateLastUpdated)); @@ -172,10 +172,10 @@ public List getFeedBackward(String feedName, DynamoDBQueryExpression querySpec = new DynamoDBQueryExpression() .withKeyConditionExpression("entryId = :id and dateLastUpdated <= :dateLastUpdated") .withScanIndexForward(false) - .withLimit(pageSize) - .withFilterExpression(filters + feedNameFilter) + .withLimit(pageSize).addExpressionAttributeNamesEntry(markerId, dateLastUpdated) + .withFilterExpression(feedNameFilter + filters) .withExpressionAttributeValues(valueMap); - feedPage = mapper.query(PersistedEntry.class, querySpec); + feedPage = mapper.query(PersistedEntry.class, querySpec); return feedPage; } @@ -192,20 +192,20 @@ public List getFeedBackward(String feedName, * @return List of PersistedEntry data found based on above params from db. */ public List getFeedForward(String feedName, - Date markerTimestamp, - long markerId, + String markerTimestamp, + String markerId, String searchString, int pageSize) { List feedPage; DynamoDBQueryBuilder sqlBac = new DynamoDBQueryBuilder(getSearchToSqlConverter()).searchString(searchString); - sqlBac.searchType(SearchType.FEED_BACKWARD); + sqlBac.searchType(SearchType.FEED_FORWARD); //Dynamodb query implementation Map map = new HashMap(); String filters = sqlBac.getFilters(map); - String feedNameFilter = " and feed= :feedName"; - SimpleDateFormat dateFormatter = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'"); - dateFormatter.setTimeZone(TimeZone.getTimeZone("UTC")); - String dateLastUpdated = dateFormatter.format(markerTimestamp); + + // SimpleDateFormat dateFormatter = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'"); + // dateFormatter.setTimeZone(TimeZone.getTimeZone("UTC")); + String dateLastUpdated = markerTimestamp; Map valueMap = new HashMap(); valueMap.put(":id", new AttributeValue().withS(String.valueOf(markerId))); valueMap.put(":dateLastUpdated", new AttributeValue().withS(dateLastUpdated)); @@ -213,13 +213,26 @@ public List getFeedForward(String feedName, for (Map.Entry res : map.entrySet()) { valueMap.put(res.getKey(), new AttributeValue().withS(res.getValue())); } - DynamoDBQueryExpression querySpec = new DynamoDBQueryExpression() - .withKeyConditionExpression("entryId = :id and dateLastUpdated >= :dateLastUpdated") - .withScanIndexForward(false) - .withLimit(pageSize) - .withFilterExpression(filters + feedNameFilter) - .withExpressionAttributeValues(valueMap); + DynamoDBQueryExpression querySpec; + if(null != filters){ + String feedNameFilter = "feed= :feedName and "; + querySpec = new DynamoDBQueryExpression() + .withKeyConditionExpression("entryId = :id and dateLastUpdated >= :dateLastUpdated") + .withScanIndexForward(true) + .withLimit(pageSize) + .withFilterExpression(feedNameFilter + filters) + .withExpressionAttributeValues(valueMap); + }else{ + String feedNameFilter = "feed= :feedName"; + querySpec = new DynamoDBQueryExpression() + .withKeyConditionExpression("entryId = :id and dateLastUpdated >= :dateLastUpdated") + .withScanIndexForward(true) + .withLimit(pageSize) + .withFilterExpression(feedNameFilter) + .withExpressionAttributeValues(valueMap); + } feedPage = mapper.query(PersistedEntry.class, querySpec); + int abc = feedPage.size(); return feedPage; } @@ -235,7 +248,7 @@ public List getFeedForward(String feedName, public AdapterResponse getFeedPageByTimestamp(GetFeedRequest getFeedRequest, String startingAt, int pageSize) throws Exception { final String pageDirectionValue = getFeedRequest.getDirection(); ObjectMapper mapper = new ObjectMapper(); - PageDirection pageDirection = PageDirection.FORWARD; + PageDirection pageDirection = PageDirection.FORWARD; if (StringUtils.isNotEmpty(String.valueOf(pageDirection))) { pageDirection = PageDirection.valueOf(pageDirectionValue.toUpperCase()); } @@ -249,14 +262,14 @@ public AdapterResponse getFeedPageByTimestamp(GetFeedRequest getFeedReques // This list contains the persistent object in json string format PersistedEntry persistedEntry = mapper.readValue(entryMarker.get(0), PersistedEntry.class); //Convert String to Date Format - Date lastDateUpdated = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.S").parse(persistedEntry.getDateLastUpdated()); + String lastDateUpdated = persistedEntry.getDateLastUpdated(); final Feed feed = hydrateFeed(getFeedRequest.getAbdera(), enhancedGetFeedPage(getFeedRequest.getFeedName(), lastDateUpdated, - Long.parseLong(persistedEntry.getEntryId()), + persistedEntry.getEntryId(), pageDirection, searchString, pageSize), - getFeedRequest, pageSize); + getFeedRequest, pageSize); return ResponseBuilder.found(feed); } @@ -274,8 +287,8 @@ public AdapterResponse getFeedPageByTimestamp(GetFeedRequest getFeedReques protected List getEntryByTimestamp(final DateTime markerDate, final String feed, PageDirection direction) { ValueMap valueMap = new ValueMap(); valueMap.withString(":feed", feed); - valueMap.withString(":markerDate", String.valueOf(markerDate)); - return getQueryBuilderMethod(dynamoDB, "feed = :feed and " + getTimeStampValueFilter(direction), valueMap); + valueMap.withString(":dateLastUpdated", String.valueOf(markerDate)); + return getQueryBuilderMethod(dynamoDB, "feed = :feed" , getTimeStampValueFilter(direction), valueMap); } /** @@ -288,9 +301,9 @@ protected List getEntryByTimestamp(final DateTime markerDate, final Stri */ private String getTimeStampValueFilter(PageDirection direction) { if (direction.equals(PageDirection.BACKWARD)) { - return "dateLastUpdated <= :markerDate"; + return "dateLastUpdated <= :dateLastUpdated"; } else { - return "dateLastUpdated >= :markerDate"; + return "dateLastUpdated > :dateLastUpdated"; } } @@ -389,11 +402,11 @@ private AdapterResponse getFeedPage(GetFeedRequest getFeedRequest, String if (entryMarker.isEmpty()) { return ResponseBuilder.notFound("No entry with specified marker found"); } - Date lastDateUpdated = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.S").parse(entryMarker.get(0).getDateLastUpdated()); + String lastDateUpdated = entryMarker.get(0).getDateLastUpdated(); final Feed feed = hydrateFeed(getFeedRequest.getAbdera(), enhancedGetFeedPage(getFeedRequest.getFeedName(), lastDateUpdated, - Long.parseLong(entryMarker.get(0).getEntryId()), + entryMarker.get(0).getEntryId(), pageDirection, searchString, pageSize), getFeedRequest, pageSize); @@ -411,11 +424,10 @@ private AdapterResponse getFeedPage(GetFeedRequest getFeedRequest, String * @param pageSize: default is 25 ,used for pagination * @return: list of PersistedEntry object . */ - private List enhancedGetFeedPage(final String feedName, final Date markerTimestamp, - final long markerId, + private List enhancedGetFeedPage(final String feedName, final String markerTimestamp, + final String markerId, final PageDirection direction, final String searchString, final int pageSize) { - List feedPage = new LinkedList(); TimerContext context = null; @@ -530,7 +542,7 @@ private List enhancedGetLastPage(final String feedName, final in List feedPage; ValueMap valueMap = new ValueMap(); valueMap.withString(":feed", feedName); - feedPage = getQueryBuilderMethod(dynamoDB, "feed = :feed and dateLastUpdated < :dateLastUpdated", pageSize, valueMap, true); + feedPage = getQueryBuilderMethod(dynamoDB, "feed = :feed" ,"dateLastUpdated < :dateLastUpdated", pageSize, valueMap, true); // comparator is written to perform sorting based on if two dates are equal then sort output based on entryId in desc order List persistedEntryList = JsonUtil.getPersistenceEntity(feedPage); Collections.sort(persistedEntryList, (a, b) -> { @@ -633,7 +645,7 @@ private List getFeedHead(final String feedName, final int pageSi List feedPage; ValueMap valueMap = new ValueMap(); valueMap.withString(":feed", feedName); - feedPage = getQueryBuilderMethod(dynamoDB, "feed = :feed", pageSize, valueMap, false); + feedPage = getQueryBuilderMethod(dynamoDB, "feed = :feed",null, pageSize, valueMap, false); List persistedEntryList = JsonUtil.getPersistenceEntity(feedPage); return persistedEntryList; } finally { @@ -665,13 +677,11 @@ private TimerContext startTimer(String name) { } private Feed hydrateFeed(Abdera abdera, List persistedEntries, - GetFeedRequest getFeedRequest, final int pageSize) { - + GetFeedRequest getFeedRequest, final int pageSize) { final Feed hydratedFeed = abdera.newFeed(); final String baseFeedUri = decode(getFeedRequest.urlFor( new EnumKeyedTemplateParameters(URITemplate.FEED))); final String searchString = getFeedRequest.getSearchQuery() != null ? getFeedRequest.getSearchQuery() : ""; - if (helper.isArchived()) { helper.addArchiveNode(hydratedFeed); @@ -781,11 +791,12 @@ private PersistedEntry getNextMarker(final PersistedEntry persistedEntry, final List firstUnionPersistentList; ValueMap valueMap = new ValueMap(); valueMap.withString(":feed", persistedEntry.getFeed()); - firstUnionPersistentList = getQueryBuilderMethod(dynamoDB, "feed = :feed ", valueMap); + firstUnionPersistentList = getQueryBuilderMethod(dynamoDB, "feed = :feed ",null, valueMap); List nextUnionListOfPersistedItems; ValueMap newValueMap = new ValueMap(); newValueMap.withString(":feed", persistedEntry.getFeed()); - nextUnionListOfPersistedItems = getQueryBuilderMethod(dynamoDB, "feed = :feed", valueMap); + + nextUnionListOfPersistedItems = getQueryBuilderMethod(dynamoDB, "feed = :feed",null, valueMap); feedPage = Stream.concat(firstUnionPersistentList.stream(), nextUnionListOfPersistedItems.stream()) .collect(Collectors.toList()); List persistedEntryList = JsonUtil.getPersistenceEntity(feedPage); @@ -903,15 +914,26 @@ private List getEntry(String entryId, final String feedName) { * @param valueMap: map for keeping the mapped values passed in a condition expression * @return List of String in json format. */ - public List getQueryBuilderMethod(DynamoDB dynamoDB, String conditionExpression, int pageSize, ValueMap valueMap, boolean orderBy) { + public List getQueryBuilderMethod(DynamoDB dynamoDB, String conditionExpression,String filterExpression, int pageSize, ValueMap valueMap, boolean orderBy) { List feedPage = new ArrayList<>(); Table table = dynamoDB.getTable("entries"); Index index = table.getIndex("global-feed-index"); - QuerySpec spec = new QuerySpec() + QuerySpec spec = null; + if(null!= filterExpression){ + spec = new QuerySpec() .withKeyConditionExpression(conditionExpression) + .withFilterExpression(filterExpression) .withValueMap(valueMap) .withMaxPageSize(pageSize)// for no of page limit to be displayed - .withScanIndexForward(orderBy);// for descending order sorting on lastDateUpdated + .withScanIndexForward(orderBy); + }else{ + spec = new QuerySpec() + .withKeyConditionExpression(conditionExpression) + .withValueMap(valueMap) + .withMaxPageSize(pageSize)// for no of page limit to be displayed + .withScanIndexForward(orderBy); + } + // for descending order sorting on lastDateUpdated ItemCollection persistedEntryItems = index.query(spec); Iterator itemsIterator = persistedEntryItems.iterator(); while (itemsIterator.hasNext()) { @@ -929,14 +951,34 @@ public List getQueryBuilderMethod(DynamoDB dynamoDB, String conditionExp * @param valueMap: map for keeping the mapped values passed in a condition expression * @return List of String in json format. */ - public List getQueryBuilderMethod(DynamoDB dynamoDB, String conditionExpression, ValueMap valueMap) { + public List getQueryBuilderMethod(DynamoDB dynamoDB, String conditionExpression,String filterExpression, ValueMap valueMap) { + LOG.error("getQueryBuilderMethod"); + LOG.error("conditionExpression " + conditionExpression); + LOG.error("valueMap" + valueMap.toString()); List feedPage = new ArrayList<>(); Table table = dynamoDB.getTable("entries"); Index index = table.getIndex("global-feed-index"); - QuerySpec spec = new QuerySpec() - .withKeyConditionExpression(conditionExpression) - .withValueMap(valueMap); - ItemCollection persistedEntryItems = index.query(spec); + QuerySpec spec; + if(null != filterExpression){ + spec = new QuerySpec() + .withKeyConditionExpression(conditionExpression + " and " + filterExpression) + // .withFilterExpression(filterExpression) + .withScanIndexForward(true) + .withValueMap(valueMap); + }else{ + spec = new QuerySpec() + .withKeyConditionExpression(conditionExpression) + .withScanIndexForward(true) + .withValueMap(valueMap); + } + + ItemCollection persistedEntryItems = null; + try{ + persistedEntryItems = index.query(spec); + }catch(Exception e){ + LOG.error("Exception : " + e + e.getMessage()) ; + } + Iterator itemsIterator = persistedEntryItems.iterator(); while (itemsIterator.hasNext()) { Item item = itemsIterator.next(); diff --git a/adapters/dynamoDB_adapters/src/main/java/org/atomhopper/dynamodb/model/PersistedEntry.java b/adapters/dynamoDB_adapters/src/main/java/org/atomhopper/dynamodb/model/PersistedEntry.java index 91135fda..413aaf1a 100644 --- a/adapters/dynamoDB_adapters/src/main/java/org/atomhopper/dynamodb/model/PersistedEntry.java +++ b/adapters/dynamoDB_adapters/src/main/java/org/atomhopper/dynamodb/model/PersistedEntry.java @@ -11,15 +11,14 @@ public class PersistedEntry { @DynamoDBHashKey(attributeName = DynamoDBConstant.ENTRY_ID) private String entryId; - @DynamoDBIndexRangeKey(attributeName = DynamoDBConstant.FEED, - localSecondaryIndexName = "entryId-feed-index") + @DynamoDBIndexRangeKey(attributeName = DynamoDBConstant.FEED, localSecondaryIndexName = "entryId-feed-index") @DynamoDBIndexHashKey(globalSecondaryIndexName = "global-feed-index", attributeName = DynamoDBConstant.FEED) private String feed; private String entryBody; @DynamoDBAutoGeneratedTimestamp(strategy = DynamoDBAutoGenerateStrategy.CREATE) private String creationDate; + @DynamoDBIndexRangeKey(globalSecondaryIndexName = "global-feed-index", attributeName = DynamoDBConstant.DATE_LAST_UPDATED) @DynamoDBRangeKey(attributeName = DynamoDBConstant.DATE_LAST_UPDATED) - @DynamoDBAutoGeneratedTimestamp(strategy = DynamoDBAutoGenerateStrategy.CREATE) private String dateLastUpdated; private List categories; diff --git a/adapters/dynamoDB_adapters/src/main/java/org/atomhopper/dynamodb/query/DynamoDBQueryBuilder.java b/adapters/dynamoDB_adapters/src/main/java/org/atomhopper/dynamodb/query/DynamoDBQueryBuilder.java index be2ad955..f7c901f7 100644 --- a/adapters/dynamoDB_adapters/src/main/java/org/atomhopper/dynamodb/query/DynamoDBQueryBuilder.java +++ b/adapters/dynamoDB_adapters/src/main/java/org/atomhopper/dynamodb/query/DynamoDBQueryBuilder.java @@ -69,6 +69,9 @@ public String getFilters(Map map) { case FEED_BACKWARD: return searchSql; + + case FEED_FORWARD: + return searchSql; } return null; } diff --git a/adapters/dynamoDB_adapters/src/main/java/org/atomhopper/dynamodb/query/SQLToNoSqlConverter.java b/adapters/dynamoDB_adapters/src/main/java/org/atomhopper/dynamodb/query/SQLToNoSqlConverter.java index aad25846..0653f75f 100644 --- a/adapters/dynamoDB_adapters/src/main/java/org/atomhopper/dynamodb/query/SQLToNoSqlConverter.java +++ b/adapters/dynamoDB_adapters/src/main/java/org/atomhopper/dynamodb/query/SQLToNoSqlConverter.java @@ -49,7 +49,7 @@ public class SQLToNoSqlConverter { private static final String NOT = " not "; private static final String CATEGORY = "cat"; - private static final String CATEGORY_STRING = " (contains(categories,:replaceValue)) "; + private static final String CATEGORY_STRING = " (contains(categories, :categories)) "; public static final String OLD_CATEGORY_STRING = " categories && ?::varchar[] "; @@ -214,8 +214,8 @@ private String getSqlFromLdapFilter(Filter filter,Map a) { throw new IllegalArgumentException("Invalid Search Parameter: LDAP attribute name must be 'cat'"); } //String key = UUID.randomUUID().toString(); - a.put(":"+filter.getAssertionValue(),filter.getAssertionValue()); - sql.append(CATEGORY_STRING.replaceAll("replaceValue",filter.getAssertionValue())); + a.put(":categories",filter.getAssertionValue()); + sql.append(CATEGORY_STRING); break; } diff --git a/adapters/dynamoDB_adapters/src/test/java/DynamoDBFeedSourceTest.java b/adapters/dynamoDB_adapters/src/test/java/DynamoDBFeedSourceTest.java index e88f3b38..7a37db9c 100644 --- a/adapters/dynamoDB_adapters/src/test/java/DynamoDBFeedSourceTest.java +++ b/adapters/dynamoDB_adapters/src/test/java/DynamoDBFeedSourceTest.java @@ -185,7 +185,7 @@ public void shouldGetFeedHead() throws Exception { newList.add("fe9eedc8-d10c-47a6-9209-f755ea8c35c3"); newList.add(""); DynamoDBFeedSource mainModel = Mockito.mock(DynamoDBFeedSource.class); - Mockito.when(mainModel.getQueryBuilderMethod(dynamoDB, "feed = :feed and dateLastUpdated < :dateLastUpdated", 25, new ValueMap(), true)).thenReturn(newList); + Mockito.when(mainModel.getQueryBuilderMethod(dynamoDB, "feed = :feed", "dateLastUpdated < :dateLastUpdated", 25, new ValueMap(), true)).thenReturn(newList); assertEquals("Should get a 200 response", HttpStatus.OK, dynamoDBFeedSource.getFeed(getFeedRequest).getResponseStatus()); } From db0481c68273779b8c8736fb619342ac41213175 Mon Sep 17 00:00:00 2001 From: Kavish Khanna Date: Tue, 19 Mar 2024 11:28:17 +0530 Subject: [PATCH 20/21] Tenant-Id-Issue All the feeds created by admin were coming in the other role calls. --- .../dynamodb/adapter/DynamoDBFeedSource.java | 94 +++++++------------ .../query/CategoryStringGenerator.java | 18 +--- .../dynamodb/query/SQLToNoSqlConverter.java | 6 +- 3 files changed, 36 insertions(+), 82 deletions(-) diff --git a/adapters/dynamoDB_adapters/src/main/java/org/atomhopper/dynamodb/adapter/DynamoDBFeedSource.java b/adapters/dynamoDB_adapters/src/main/java/org/atomhopper/dynamodb/adapter/DynamoDBFeedSource.java index 77e46494..7d0e68bd 100644 --- a/adapters/dynamoDB_adapters/src/main/java/org/atomhopper/dynamodb/adapter/DynamoDBFeedSource.java +++ b/adapters/dynamoDB_adapters/src/main/java/org/atomhopper/dynamodb/adapter/DynamoDBFeedSource.java @@ -288,7 +288,7 @@ protected List getEntryByTimestamp(final DateTime markerDate, final Stri ValueMap valueMap = new ValueMap(); valueMap.withString(":feed", feed); valueMap.withString(":dateLastUpdated", String.valueOf(markerDate)); - return getQueryBuilderMethod(dynamoDB, "feed = :feed" , getTimeStampValueFilter(direction), valueMap); + return getQueryBuilderMethod(dynamoDB, "feed = :feed and " + getTimeStampValueFilter(direction) , null, valueMap); } /** @@ -514,23 +514,13 @@ private List enhancedGetLastPage(final String feedName, final in List categoriesList = getSearchToSqlConverter().getParamsFromSearchString(searchString); int numCats = categoriesList.size(); - Object[] parms = null; + String filterExpression = null; if (numCats > 0) { - parms = new Object[numCats + 2]; - int index = 0; - parms[index++] = feedName; - for (String s : categoriesList) { - parms[index++] = s; - } - parms[index++] = pageSize; - - } else { - parms = new Object[]{feedName, pageSize}; + filterExpression = "(contains(categories, :categories))"; } TimerContext context = null; - List lastPersistedEntries; try { if (numCats > 0) { context = startTimer( @@ -542,7 +532,7 @@ private List enhancedGetLastPage(final String feedName, final in List feedPage; ValueMap valueMap = new ValueMap(); valueMap.withString(":feed", feedName); - feedPage = getQueryBuilderMethod(dynamoDB, "feed = :feed" ,"dateLastUpdated < :dateLastUpdated", pageSize, valueMap, true); + feedPage = getQueryBuilderMethod(dynamoDB, "feed = :feed and dateLastUpdated < :dateLastUpdated" ,filterExpression, pageSize, valueMap, true); // comparator is written to perform sorting based on if two dates are equal then sort output based on entryId in desc order List persistedEntryList = JsonUtil.getPersistenceEntity(feedPage); Collections.sort(persistedEntryList, (a, b) -> { @@ -619,18 +609,11 @@ private List getFeedHead(final String feedName, final int pageSi List categoriesList = getSearchToSqlConverter().getParamsFromSearchString(searchString); int numCats = categoriesList.size(); - Object[] parms = null; + + String filterExpression = null; if (numCats > 0) { - parms = new Object[numCats + 2]; - int index = 0; - parms[index++] = feedName; - for (String s : categoriesList) { - parms[index++] = s; - } - parms[index++] = pageSize; - } else { - parms = new Object[]{feedName, pageSize}; + filterExpression = "(contains(categories, :categories))"; } TimerContext context = null; @@ -645,7 +628,10 @@ private List getFeedHead(final String feedName, final int pageSi List feedPage; ValueMap valueMap = new ValueMap(); valueMap.withString(":feed", feedName); - feedPage = getQueryBuilderMethod(dynamoDB, "feed = :feed",null, pageSize, valueMap, false); + for(String s: categoriesList){ + valueMap.withString(":categories", s); + } + feedPage = getQueryBuilderMethod(dynamoDB, "feed = :feed",filterExpression, pageSize, valueMap, false); List persistedEntryList = JsonUtil.getPersistenceEntity(feedPage); return persistedEntryList; } finally { @@ -765,41 +751,25 @@ private PersistedEntry getNextMarker(final PersistedEntry persistedEntry, final List categoriesList = getSearchToSqlConverter().getParamsFromSearchString(searchString); int numCats = categoriesList.size(); - Object[] parms = null; + String filterExpression = null; - if (categoriesList.size() > 0) { - parms = new Object[numCats * 2 + 5]; - int index = 0; - parms[index++] = feedName; - parms[index++] = persistedEntry.getDateLastUpdated(); - parms[index++] = persistedEntry.getEntryId(); - for (String s : categoriesList) { - parms[index++] = s; - } - parms[index++] = feedName; - parms[index++] = persistedEntry.getDateLastUpdated(); - for (String s : categoriesList) { - parms[index++] = s; - } - - } else { - parms = new Object[]{feedName, persistedEntry.getDateLastUpdated(), persistedEntry.getEntryId(), - feedName, persistedEntry.getDateLastUpdated()}; + if (numCats > 0) { + filterExpression = "(contains(categories, :categories))"; } - List feedPage; + + List firstUnionPersistentList; ValueMap valueMap = new ValueMap(); valueMap.withString(":feed", persistedEntry.getFeed()); - firstUnionPersistentList = getQueryBuilderMethod(dynamoDB, "feed = :feed ",null, valueMap); - List nextUnionListOfPersistedItems; - ValueMap newValueMap = new ValueMap(); - newValueMap.withString(":feed", persistedEntry.getFeed()); - - nextUnionListOfPersistedItems = getQueryBuilderMethod(dynamoDB, "feed = :feed",null, valueMap); - feedPage = Stream.concat(firstUnionPersistentList.stream(), nextUnionListOfPersistedItems.stream()) - .collect(Collectors.toList()); - List persistedEntryList = JsonUtil.getPersistenceEntity(feedPage); + + for(String s: categoriesList){ + + valueMap.withString(":categories", s); + } + + firstUnionPersistentList = getQueryBuilderMethod(dynamoDB, "feed = :feed ",filterExpression, valueMap); + List persistedEntryList = JsonUtil.getPersistenceEntity(firstUnionPersistentList); return persistedEntryList.get(0); } @@ -934,7 +904,13 @@ public List getQueryBuilderMethod(DynamoDB dynamoDB, String conditionExp .withScanIndexForward(orderBy); } // for descending order sorting on lastDateUpdated - ItemCollection persistedEntryItems = index.query(spec); + ItemCollection persistedEntryItems = null; + try{ + persistedEntryItems = index.query(spec); + }catch(Exception e){ + LOG.error("Exception " + e + e.getMessage()); + } + Iterator itemsIterator = persistedEntryItems.iterator(); while (itemsIterator.hasNext()) { Item item = itemsIterator.next(); @@ -952,17 +928,15 @@ public List getQueryBuilderMethod(DynamoDB dynamoDB, String conditionExp * @return List of String in json format. */ public List getQueryBuilderMethod(DynamoDB dynamoDB, String conditionExpression,String filterExpression, ValueMap valueMap) { - LOG.error("getQueryBuilderMethod"); - LOG.error("conditionExpression " + conditionExpression); - LOG.error("valueMap" + valueMap.toString()); + List feedPage = new ArrayList<>(); Table table = dynamoDB.getTable("entries"); Index index = table.getIndex("global-feed-index"); QuerySpec spec; if(null != filterExpression){ spec = new QuerySpec() - .withKeyConditionExpression(conditionExpression + " and " + filterExpression) - // .withFilterExpression(filterExpression) + .withKeyConditionExpression(conditionExpression) + .withFilterExpression(filterExpression) .withScanIndexForward(true) .withValueMap(valueMap); }else{ diff --git a/adapters/dynamoDB_adapters/src/main/java/org/atomhopper/dynamodb/query/CategoryStringGenerator.java b/adapters/dynamoDB_adapters/src/main/java/org/atomhopper/dynamodb/query/CategoryStringGenerator.java index e979bf9e..65ed5699 100644 --- a/adapters/dynamoDB_adapters/src/main/java/org/atomhopper/dynamodb/query/CategoryStringGenerator.java +++ b/adapters/dynamoDB_adapters/src/main/java/org/atomhopper/dynamodb/query/CategoryStringGenerator.java @@ -28,49 +28,34 @@ private CategoryStringGenerator() { public static List getPostgresCategoryString(String searchString, Map mapPrefix, String prefixSplit ) { List finalList = new ArrayList(); - - if (searchString == null || !(searchString.trim().length() > 0)) { - finalList.add( "{}" ); return finalList; } List categories = parse(searchString.trim().toLowerCase()); - List catHolder = new ArrayList(); // find if any categories are prefixed, if so, split them out. for( String cat : categories ) { - if (cat.matches( SQLToNoSqlConverter.BAD_SEARCH_REGEX ) ) { - throw new IllegalArgumentException( SQLToNoSqlConverter.BAD_CHAR_MSG ); } - if (prefixSplit != null ) { - int index = cat.indexOf( prefixSplit ); - if ( index != -1 ) { - String prefix = cat.substring( 0, index ); - String value = cat.substring( index + prefixSplit.length() ); if ( mapPrefix.containsKey( prefix ) ) { - addToFinalList( finalList, catHolder ); - - finalList.add( value ); + finalList.add( cat ); } else { - catHolder.add( cat ); } } else { - catHolder.add( cat ); } } @@ -80,7 +65,6 @@ public static List getPostgresCategoryString(String searchString, Map getParametersFromLdapFilter(Filter filter) { if( prefixSplit != null ) { int index = filter.getAssertionValue().indexOf( prefixSplit ); - if ( index != -1 ) { String prefix = filter.getAssertionValue().substring( 0, index ); - String value = filter.getAssertionValue().substring( index + prefixSplit.length() ); - if ( mapPrefix.containsKey( prefix ) ) { - - param = value; + param = filter.getAssertionValue(); } } } From 996797c79b5f2edbfa8603abcd3aa90c9ace3574 Mon Sep 17 00:00:00 2001 From: Kavish Khanna Date: Mon, 1 Apr 2024 07:45:25 +0530 Subject: [PATCH 21/21] Categories issue. --- .../atomhopper/dynamodb/adapter/DynamoDBFeedSource.java | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/adapters/dynamoDB_adapters/src/main/java/org/atomhopper/dynamodb/adapter/DynamoDBFeedSource.java b/adapters/dynamoDB_adapters/src/main/java/org/atomhopper/dynamodb/adapter/DynamoDBFeedSource.java index 7d0e68bd..a45677b2 100644 --- a/adapters/dynamoDB_adapters/src/main/java/org/atomhopper/dynamodb/adapter/DynamoDBFeedSource.java +++ b/adapters/dynamoDB_adapters/src/main/java/org/atomhopper/dynamodb/adapter/DynamoDBFeedSource.java @@ -7,7 +7,6 @@ import com.amazonaws.services.dynamodbv2.document.spec.QuerySpec; import com.amazonaws.services.dynamodbv2.document.utils.ValueMap; import com.amazonaws.services.dynamodbv2.model.AttributeValue; -import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.gson.Gson; import com.yammer.metrics.Metrics; @@ -764,8 +763,12 @@ private PersistedEntry getNextMarker(final PersistedEntry persistedEntry, final valueMap.withString(":feed", persistedEntry.getFeed()); for(String s: categoriesList){ - - valueMap.withString(":categories", s); + if(s.charAt(0) == '{'){ + valueMap.withString(":categories", s.substring(1,s.length() -1)); + }else{ + valueMap.withString(":categories", s); + } + } firstUnionPersistentList = getQueryBuilderMethod(dynamoDB, "feed = :feed ",filterExpression, valueMap);