Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

closes #206 #211

Merged
merged 1 commit into from
Jan 2, 2016
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Ignore;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TestName;
Expand All @@ -24,17 +25,18 @@
import junitparams.JUnitParamsRunner;
import junitparams.Parameters;

@Ignore
@ContextConfiguration(locations = {"classpath*:META-INF/applicatonContext-loadtester.xml"})
@RunWith(JUnitParamsRunner.class)
public class LoadTesterIT {

private static final Logger LOGGER = LoggerFactory.getLogger(LoadTesterIT.class);

private TestContextManager testContextManager;
// private final Integer totalLoadCount = 5000000;
// private final Integer batchSize = 10000;
private final Integer totalLoadCount = 10000;
private final Integer batchSize = 1000;
private final Integer totalLoadCount = 5000000;
private final Integer batchSize = 10000;
// private final Integer totalLoadCount = 10000;
// private final Integer batchSize = 1000;
private final String endPointName = "tagClient";
private StopWatch stopWatch;

Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
package org.robbins.flashcards.cassandra.repository;

import org.robbins.flashcards.cassandra.repository.domain.BatchLoadingReceiptCassandraEntity;
import org.springframework.data.cassandra.repository.TypedIdCassandraRepository;
import org.springframework.stereotype.Repository;

@Repository
public interface BatchReceiptCassandraRepository extends TypedIdCassandraRepository<BatchLoadingReceiptCassandraEntity, Long> {

}
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@

package org.robbins.flashcards.cassandra.repository;

import javax.inject.Inject;

import org.robbins.flashcards.cassandra.repository.domain.BatchLoadingReceiptCassandraEntity;
import org.robbins.flashcards.repository.BatchLoadingReceiptRepository;
import org.springframework.stereotype.Repository;

@Repository
public class BatchReceiptRepositoryImpl extends AbstractCrudRepositoryImpl<BatchLoadingReceiptCassandraEntity, Long> implements
BatchLoadingReceiptRepository<BatchLoadingReceiptCassandraEntity, Long>
{
@Inject
private BatchReceiptCassandraRepository repository;

@Override
public BatchReceiptCassandraRepository getRepository() {
return repository;
}

}
Original file line number Diff line number Diff line change
@@ -1,10 +1,19 @@

package org.robbins.flashcards.cassandra.repository;

import com.datastax.driver.core.BatchStatement;
import com.datastax.driver.core.PreparedStatement;
import com.datastax.driver.core.RegularStatement;
import com.datastax.driver.core.Session;
import static com.datastax.driver.core.querybuilder.QueryBuilder.bindMarker;
import static com.datastax.driver.core.querybuilder.QueryBuilder.eq;
import static com.datastax.driver.core.querybuilder.QueryBuilder.insertInto;
import static com.datastax.driver.core.querybuilder.QueryBuilder.put;
import static com.datastax.driver.core.querybuilder.QueryBuilder.update;

import java.util.ArrayList;
import java.util.List;
import java.util.stream.Collectors;

import javax.annotation.PostConstruct;
import javax.inject.Inject;

import org.apache.commons.lang3.NotImplementedException;
import org.robbins.flashcards.cassandra.repository.domain.FlashCardCassandraEntity;
import org.robbins.flashcards.cassandra.repository.domain.TagCassandraEntity;
Expand All @@ -15,21 +24,26 @@
import org.springframework.data.cassandra.core.CassandraOperations;
import org.springframework.stereotype.Repository;

import javax.annotation.PostConstruct;
import javax.inject.Inject;
import java.util.List;
import java.util.UUID;

import static com.datastax.driver.core.querybuilder.QueryBuilder.*;
import com.datastax.driver.core.BatchStatement;
import com.datastax.driver.core.PreparedStatement;
import com.datastax.driver.core.RegularStatement;
import com.datastax.driver.core.Session;

@Repository
public class TagRepositoryImpl extends AbstractCrudRepositoryImpl<TagCassandraEntity, Long> implements
TagRepository<TagCassandraEntity, Long> {

private static final Logger LOGGER = LoggerFactory.getLogger(TagRepositoryImpl.class);

private static final String TAG_TABLE = "tag";
private static final String FLASHCARD_TABLE = "flashcard";
private static final String ID = "id";
private static final String NAME = "name";
private static final String TAGS = "tags";
private static final String BATCH_INSERT_CQL = "insert into tag (id, name) values (?, ?);";

@Inject
private CassandraOperations cassandraOperations;
private CassandraOperations cassandraTemplate;

@Inject
private TagCassandraRepository repository;
Expand All @@ -43,12 +57,6 @@ public class TagRepositoryImpl extends AbstractCrudRepositoryImpl<TagCassandraEn
private PreparedStatement tagStatement;
private PreparedStatement flashcardStatement;

private static final String TAG_TABLE = "tag";
private static final String FLASHCARD_TABLE = "flashcard";
private static final String ID = "id";
private static final String NAME = "name";
private static final String TAGS = "tags";

@Override
public TagCassandraRepository getRepository() {
return repository;
Expand All @@ -57,7 +65,7 @@ public TagCassandraRepository getRepository() {
@SuppressWarnings("unused")
@PostConstruct
private void initStatements() {
Session session = cassandraOperations.getSession();
final Session session = cassandraTemplate.getSession();
if (session == null) {
LOGGER.error("Cassandra not available");
} else {
Expand All @@ -69,21 +77,21 @@ private void initStatements() {

@Override
public TagCassandraEntity save(final TagCassandraEntity tag) {
cassandraOperations.execute(tagBatch(tag));
cassandraTemplate.execute(tagBatch(tag));

return tag;
}

private BatchStatement tagBatch(TagCassandraEntity tag) {
BatchStatement batch = new BatchStatement();
final BatchStatement batch = new BatchStatement();
batch.add(tagStatement.bind(
tag.getId(),
tag.getName()));

List<TagFlashCardCassandraEntity> tagFlashcards = tagFlashcardCassandraRepository.findByTagId(tag.getId());
final List<TagFlashCardCassandraEntity> tagFlashcards = tagFlashcardCassandraRepository.findByTagId(tag.getId());
if (tagFlashcards != null && tagFlashcards.size() > 0) {
for (TagFlashCardCassandraEntity tagFlashCard : tagFlashcards) {
FlashCardCassandraEntity flashcard = flashCardCassandraRepository.findOne(tagFlashCard.getId().getFlashCardId());
final FlashCardCassandraEntity flashcard = flashCardCassandraRepository.findOne(tagFlashCard.getId().getFlashCardId());
if (flashcard != null && flashcard.getTags() != null) {
batch.add(flashcardStatement.bind(
tag.getId(),
Expand Down Expand Up @@ -116,4 +124,27 @@ public TagCassandraEntity findByName(final String name) {
public List<TagCassandraEntity> findByFlashcards_Id(Long flashcardId) {
throw new NotImplementedException("method not yet implemented in Cassandra repository");
}

@Override
public int batchSave(final List<TagCassandraEntity> tags)
{
cassandraTemplate.ingest(BATCH_INSERT_CQL, convertTagsForIngestion(tags));
return tags.size();
}

private List<List<?>> convertTagsForIngestion(final List<TagCassandraEntity> tags)
{
return tags
.stream()
.map(this::convertTagForIngestion)
.collect(Collectors.toList());
}

private List<?> convertTagForIngestion(final TagCassandraEntity tag)
{
final List<Object> tagList = new ArrayList<>();
tagList.add(tag.getId());
tagList.add(tag.getName());
return tagList;
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,51 @@
package org.robbins.flashcards.cassandra.repository.conversion.impl;

import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;

import org.apache.commons.lang3.NotImplementedException;
import org.robbins.flashcards.cassandra.repository.domain.BatchLoadingReceiptCassandraEntity;
import org.robbins.flashcards.conversion.DtoConverter;
import org.robbins.flashcards.dto.BatchLoadingReceiptDto;
import org.robbins.flashcards.exceptions.RepositoryException;
import org.robbins.flashcards.repository.conversion.impl.AbstractDtoConverter;
import org.springframework.stereotype.Component;

@Component("batchReceiptDtoConverter")
public class DefaultBatchReceiptDtoConverter extends AbstractDtoConverter implements DtoConverter<BatchLoadingReceiptDto, BatchLoadingReceiptCassandraEntity> {

@Override
public BatchLoadingReceiptDto getDto(final BatchLoadingReceiptCassandraEntity entity) throws RepositoryException {
return getMapper().map(entity, BatchLoadingReceiptDto.class);
}

@Override
public BatchLoadingReceiptDto getDto(final BatchLoadingReceiptCassandraEntity entity, final Set<String> fields)
throws RepositoryException
{
throw new NotImplementedException("method not yet implemented in Cassandra repository");
}

@Override
public BatchLoadingReceiptCassandraEntity getEntity(final BatchLoadingReceiptDto dto) {
return getMapper().map(dto, BatchLoadingReceiptCassandraEntity.class);
}

@Override
public List<BatchLoadingReceiptDto> getDtos(List<BatchLoadingReceiptCassandraEntity> entities) throws RepositoryException {
return entities.stream().map(this::getDto).collect(Collectors.toList());
}

@Override
public List<BatchLoadingReceiptDto> getDtos(final List<BatchLoadingReceiptCassandraEntity> entities, final Set<String> fields)
throws RepositoryException {throw new NotImplementedException("method not yet implemented in Cassandra repository");
}

@Override
public List<BatchLoadingReceiptCassandraEntity> getEntities(final List<BatchLoadingReceiptDto> dtos) {
return dtos.stream()
.map(this::getEntity)
.collect(Collectors.toList());
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,84 @@

package org.robbins.flashcards.cassandra.repository.domain;

import java.io.Serializable;
import java.util.Date;

import org.springframework.data.cassandra.mapping.Column;
import org.springframework.data.cassandra.mapping.Table;

@Table(value = "batchreceipt")
public class BatchLoadingReceiptCassandraEntity extends AbstractPersistable implements Serializable {

@Column
private String type;

@Column
private int successCount;

@Column
private int failureCount;

@Column
private Date startTime;

@Column
private Date endTime;

public BatchLoadingReceiptCassandraEntity() {
}

public BatchLoadingReceiptCassandraEntity(final String type, final int successCount,
final int failureCount, final Date startTime,
final Date endTime) {
this.type = type;
this.successCount = successCount;
this.failureCount = failureCount;
this.startTime = startTime;
this.endTime = endTime;
}

public BatchLoadingReceiptCassandraEntity(final Long id) {
setId(id);
}

public String getType() {
return type;
}

public void setType(String type) {
this.type = type;
}

public int getSuccessCount() {
return successCount;
}

public void setSuccessCount(int successCount) {
this.successCount = successCount;
}

public int getFailureCount() {
return failureCount;
}

public void setFailureCount(int failureCount) {
this.failureCount = failureCount;
}

public Date getStartTime() {
return startTime;
}

public void setStartTime(Date startTime) {
this.startTime = startTime;
}

public Date getEndTime() {
return endTime;
}

public void setEndTime(Date endTime) {
this.endTime = endTime;
}
}
Original file line number Diff line number Diff line change
@@ -1,12 +1,14 @@
package org.robbins.flashcards.cassandra.repository.domain;

import org.springframework.data.cassandra.mapping.Column;
import org.springframework.data.cassandra.mapping.Table;

import java.io.Serializable;

@Table(value = "tag")
public class TagCassandraEntity extends AbstractPersistable implements Serializable {

@Column
private String name;

public String getName() {
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@

package org.robbins.flashcards.cassandra.repository.facade;

import javax.inject.Inject;

import org.robbins.flashcards.cassandra.repository.domain.BatchLoadingReceiptCassandraEntity;
import org.robbins.flashcards.cassandra.repository.facade.base.AbstractCrudRepositoryFacadeImpl;
import org.robbins.flashcards.conversion.DtoConverter;
import org.robbins.flashcards.dto.BatchLoadingReceiptDto;
import org.robbins.flashcards.facade.BatchReceiptFacade;
import org.robbins.flashcards.repository.BatchLoadingReceiptRepository;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.stereotype.Component;

@Component("batchReceiptRepositoryFacade")
public class DefaultBatchReceiptRepositoryFacade extends AbstractCrudRepositoryFacadeImpl<BatchLoadingReceiptDto, BatchLoadingReceiptCassandraEntity> implements
BatchReceiptFacade {

@Inject
private BatchLoadingReceiptRepository<BatchLoadingReceiptCassandraEntity, Long> repository;

@Inject
@Qualifier("batchReceiptDtoConverter")
private DtoConverter<BatchLoadingReceiptDto, BatchLoadingReceiptCassandraEntity> converter;

@Override
public DtoConverter<BatchLoadingReceiptDto, BatchLoadingReceiptCassandraEntity> getConverter()
{
return converter;
}

@Override
public BatchLoadingReceiptRepository<BatchLoadingReceiptCassandraEntity, Long> getRepository() {
return repository;
}

}
Loading