Skip to content
Permalink
Browse files

Introduce SQL based statistics service.

This introduces StatisticService, a SQL based service providing
statistics on how many km a user has been bicking etc.

It replaces the old, excessive Java 8 stream API usage. Back in 2014, it
seemed cool, doing it all on the client, but the right place to compute
is, is the database.

Even H2 supports all the analytic functions (as window functions) needed,
to compute the runnign sums from the recorded absolute milages.

Together with a handful of common table expressions, everything boils
down to 5 queries.

Those have been tested by creating a new empty database, inserting the
old test data, and then querying it.

With that setup, I want to be in complete control again over my schema,
so I also removed Hibernates automatic schema creation and replaced it
with Flyway and SQL scripts.

A lot of tests have been simplified in the process.
  • Loading branch information...
michael-simons committed Oct 28, 2019
1 parent e60e944 commit 2c7cb85d6038be5edc79d9d51e3594cffaec9081
Showing with 1,642 additions and 1,078 deletions.
  1. +10 −0 pom.xml
  2. +2 −2 src/jqassistant/structure.adoc
  3. +27 −150 src/main/java/ac/simons/biking2/bikes/BikeEntity.java
  4. +3 −4 src/main/java/ac/simons/biking2/bikes/BikeRepository.java
  5. +13 −1 src/main/java/ac/simons/biking2/bikes/BikesController.java
  6. +4 −3 src/main/java/ac/simons/biking2/{bikes → statistics}/AccumulatedPeriod.java
  7. +70 −118 src/main/java/ac/simons/biking2/{bikes → statistics}/ChartsController.java
  8. +83 −0 src/main/java/ac/simons/biking2/statistics/CurrentYear.java
  9. +64 −0 src/main/java/ac/simons/biking2/statistics/MonthlyAverage.java
  10. +51 −0 src/main/java/ac/simons/biking2/statistics/MonthlyStatistics.java
  11. +284 −0 src/main/java/ac/simons/biking2/statistics/StatisticService.java
  12. +7 −5 src/main/java/ac/simons/biking2/{summary → statistics}/Summary.java
  13. +39 −0 src/main/java/ac/simons/biking2/statistics/SummaryController.java
  14. +61 −0 src/main/java/ac/simons/biking2/statistics/YearlyStatistics.java
  15. +1 −1 src/main/java/ac/simons/biking2/{bikes → statistics}/highcharts/Axis.java
  16. +1 −1 src/main/java/ac/simons/biking2/{bikes → statistics}/highcharts/Chart.java
  17. +1 −1 src/main/java/ac/simons/biking2/{bikes → statistics}/highcharts/Column.java
  18. +1 −1 src/main/java/ac/simons/biking2/{bikes → statistics}/highcharts/Credits.java
  19. +5 −3 src/main/java/ac/simons/biking2/{bikes → statistics}/highcharts/HighchartsNgConfig.java
  20. +1 −1 src/main/java/ac/simons/biking2/{bikes → statistics}/highcharts/Marker.java
  21. +1 −1 src/main/java/ac/simons/biking2/{bikes → statistics}/highcharts/Options.java
  22. +1 −1 src/main/java/ac/simons/biking2/{bikes → statistics}/highcharts/PlotLine.java
  23. +1 −1 src/main/java/ac/simons/biking2/{bikes → statistics}/highcharts/PlotOptions.java
  24. +1 −1 src/main/java/ac/simons/biking2/{bikes → statistics}/highcharts/Series.java
  25. +1 −1 src/main/java/ac/simons/biking2/{bikes → statistics}/highcharts/SeriesOptions.java
  26. +1 −1 src/main/java/ac/simons/biking2/{bikes → statistics}/highcharts/Title.java
  27. +1 −1 src/main/java/ac/simons/biking2/{bikes → statistics}/highcharts/Tooltip.java
  28. +1 −1 src/main/java/ac/simons/biking2/{bikes → statistics}/highcharts/package-info.java
  29. +3 −3 src/main/java/ac/simons/biking2/{summary → statistics}/package-info.java
  30. +0 −59 src/main/java/ac/simons/biking2/summary/SummaryController.java
  31. +6 −11 src/main/java/ac/simons/biking2/trips/AssortedTripRepository.java
  32. +1 −3 src/main/resources/application-default.properties
  33. +21 −0 src/main/resources/db/migration/V0001__Create_table_assorted_trips.sql
  34. +27 −0 src/main/resources/db/migration/V0002__Create_table_bikes.sql
  35. +23 −0 src/main/resources/db/migration/V0003__Create_table_biking_pictures.sql
  36. +24 −0 src/main/resources/db/migration/V0004__Create_table_locations.sql
  37. +25 −0 src/main/resources/db/migration/V0005__Create_table_milages.sql
  38. +29 −0 src/main/resources/db/migration/V0006__Create_table_tracks.sql
  39. +24 −0 src/main/resources/db/migration/V0007__Create_table_gallery_pictures.sql
  40. +5 −4 src/main/resources/messages_en.properties
  41. +0 −48 src/test/java/ac/simons/biking2/bikes/BikeEntityTest.java
  42. +7 −26 src/test/java/ac/simons/biking2/bikes/BikeRepositoryTest.java
  43. +2 −2 src/test/java/ac/simons/biking2/bikes/BikesControllerTest.java
  44. +0 −363 src/test/java/ac/simons/biking2/bikes/ChartsControllerTest.java
  45. +2 −6 src/test/java/ac/simons/biking2/bikingpictures/BikingPictureRepositoryTest.java
  46. +234 −0 src/test/java/ac/simons/biking2/statistics/ChartsControllerTest.java
  47. +354 −0 src/test/java/ac/simons/biking2/statistics/StatisticServiceTest.java
  48. +100 −0 src/test/java/ac/simons/biking2/statistics/SummaryControllerTest.java
  49. +1 −1 src/test/java/ac/simons/biking2/{bikes → statistics}/highcharts/AxisTest.java
  50. +1 −1 src/test/java/ac/simons/biking2/{bikes → statistics}/highcharts/ChartTest.java
  51. +1 −1 src/test/java/ac/simons/biking2/{bikes → statistics}/highcharts/ColumnTest.java
  52. +1 −1 src/test/java/ac/simons/biking2/{bikes → statistics}/highcharts/CreditsTest.java
  53. +1 −1 src/test/java/ac/simons/biking2/{bikes → statistics}/highcharts/HighchartsNgConfigTest.java
  54. +1 −1 src/test/java/ac/simons/biking2/{bikes → statistics}/highcharts/MarkerTest.java
  55. +1 −1 src/test/java/ac/simons/biking2/{bikes → statistics}/highcharts/OptionsTest.java
  56. +1 −1 src/test/java/ac/simons/biking2/{bikes → statistics}/highcharts/PlotLineTest.java
  57. +1 −1 src/test/java/ac/simons/biking2/{bikes → statistics}/highcharts/PlotOptionsTest.java
  58. +1 −1 src/test/java/ac/simons/biking2/{bikes → statistics}/highcharts/SeriesOptionsTest.java
  59. +1 −1 src/test/java/ac/simons/biking2/{bikes → statistics}/highcharts/SeriesTest.java
  60. +1 −1 src/test/java/ac/simons/biking2/{bikes → statistics}/highcharts/TitleTest.java
  61. +1 −1 src/test/java/ac/simons/biking2/{bikes → statistics}/highcharts/TooltipTest.java
  62. +0 −122 src/test/java/ac/simons/biking2/summary/SummaryControllerTest.java
  63. +1 −1 src/test/java/ac/simons/biking2/support/BeanTester.java
  64. +3 −0 src/test/java/ac/simons/biking2/support/TestConfig.java
  65. +0 −47 src/test/java/ac/simons/biking2/trips/AssortedTripRepositoryTest.java
  66. +2 −4 src/test/resources/application-test.properties
  67. +0 −67 src/test/resources/schema.sql
10 pom.xml
@@ -51,6 +51,7 @@
<jacoco-maven-plugin.version>0.8.5</jacoco-maven-plugin.version>
<java.version>11</java.version>
<joor.version>0.9.12</joor.version>
<jool.version>0.9.14</jool.version>
<jqassistant.version>1.7.0</jqassistant.version>
<jqassistant.plugin.version>1.7.0</jqassistant.plugin.version>
<jquery.version>1.11.0</jquery.version>
@@ -116,6 +117,10 @@
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-reactor-netty</artifactId>
</dependency>
<dependency>
<groupId>org.flywaydb</groupId>
<artifactId>flyway-core</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-data-jpa</artifactId>
@@ -151,6 +156,11 @@
<groupId>com.fasterxml.jackson.module</groupId>
<artifactId>jackson-module-jaxb-annotations</artifactId>
</dependency>
<dependency>
<groupId>org.jooq</groupId>
<artifactId>jool</artifactId>
<version>${jool.version}</version>
</dependency>

<dependency>
<groupId>com.h2database</groupId>
@@ -1,7 +1,8 @@
[[structure:Default]]
[role=group,includesConstraints="structure:packagesShouldConformToTheMainBuildingBlocks"]

All the blackboxes above should correspond to Java packages. Those packages should have no dependencies to other packages outside themselves but for the support or shared package:
All the blackboxes above should correspond to Java packages.
Those packages should have no dependencies to other packages outside themselves but for the support or shared package:

[[structure:packagesShouldConformToTheMainBuildingBlocks]]
[source,cypher,role=constraint,requiresConcepts="structure:configPackages,structure:supportingPackages"]
@@ -12,6 +13,5 @@ MATCH (a) -[:CONTAINS]-> (p1:Package) -[:DEPENDS_ON]-> (p2:Package) <-[:CONTAINS
WHERE not p1:Config
and not (p1) -[:CONTAINS]-> (p2)
and not p2:Support
and not p1.fqn = 'ac.simons.biking2.summary'
RETURN p1, p2
----
@@ -15,24 +15,12 @@
*/
package ac.simons.biking2.bikes;

import com.fasterxml.jackson.annotation.JsonAutoDetect;
import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.annotation.JsonProperty;
import java.io.Serializable;
import java.time.LocalDate;
import java.time.Month;
import java.time.OffsetDateTime;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Comparator;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.TreeMap;
import java.util.function.Predicate;
import java.util.stream.Collectors;
import java.util.stream.IntStream;

import javax.persistence.CascadeType;
import javax.persistence.Column;
import javax.persistence.Embeddable;
@@ -45,23 +33,25 @@
import javax.persistence.OneToMany;
import javax.persistence.OrderBy;
import javax.persistence.Table;
import javax.validation.constraints.NotBlank;
import javax.validation.constraints.NotNull;
import javax.validation.constraints.Size;

import org.hibernate.validator.constraints.URL;

import static java.util.stream.IntStream.rangeClosed;
import static java.util.stream.Collectors.reducing;
import javax.validation.constraints.NotBlank;
import com.fasterxml.jackson.annotation.JsonAutoDetect;
import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.annotation.JsonProperty;
import lombok.AccessLevel;
import lombok.EqualsAndHashCode;
import lombok.Getter;
import lombok.NoArgsConstructor;
import lombok.RequiredArgsConstructor;
import lombok.Setter;

/**
* @author Michael J. Simons, 2014-02-08
* @author Michael J. Simons
* @since 2014-02-08
*/
@Entity
@Table(name = "bikes")
@@ -110,7 +100,8 @@ public Link(final String url, final String label) {
@Column(length = 6, nullable = false)
@NotBlank
@Size(max = 6)
@Getter @Setter
@Getter
@Setter
private String color = "CCCCCC";

@Column(name = "bought_on", nullable = false)
@@ -134,16 +125,12 @@ public Link(final String url, final String label) {
private OffsetDateTime createdAt;

@Embedded
@Getter @Setter
@Getter
@Setter
private Link story;

/**
* Contains all monthly periods that bike has been used
*/
@JsonIgnore
private transient Map<LocalDate, Integer> periods;

public BikeEntity(final String name, final LocalDate boughtOn) {

this.name = name;
this.boughtOn = boughtOn;
this.createdAt = OffsetDateTime.now();
@@ -154,13 +141,15 @@ public BikeEntity(final String name, final LocalDate boughtOn) {
* @return true if the bike was decommissioned
*/
public boolean decommission(final LocalDate decommissionDate) {

if (decommissionDate != null) {
this.decommissionedOn = decommissionDate;
}
return this.decommissionedOn != null;
}

public synchronized MilageEntity addMilage(final LocalDate recordedOn, final double amount) {

if (!this.milages.isEmpty()) {
final MilageEntity lastMilage = this.milages.get(this.milages.size() - 1);
LocalDate nextValidDate = lastMilage.getRecordedOn().plusMonths(1);
@@ -173,150 +162,38 @@ public synchronized MilageEntity addMilage(final LocalDate recordedOn, final dou
}
final MilageEntity milage = new MilageEntity(this, recordedOn.withDayOfMonth(1), amount);
this.milages.add(milage);
this.periods = null;
return milage;
}

/**
* An IntStream is used to create indizes, which are collected in a HashMap
* supplied by the supplier HashMap::new, a method reference capture for the
* constructor.
*
* @return
*/
public synchronized Map<LocalDate, Integer> getPeriods() {
if (this.periods == null) {
this.periods = IntStream.range(1, this.milages.size()).collect(TreeMap::new, (map, i) -> {
final MilageEntity left = milages.get(i - 1);
map.put(
left.getRecordedOn(),
milages.get(i).getAmount().subtract(left.getAmount()).intValue()
);
}, TreeMap::putAll);
}

return this.periods;
}

/**
* Example of reducing a stream to a skalar value
*
* @return
* @return The total milage that has been recorded here.
*/
@JsonProperty
public int getMilage() {
return this.getPeriods().values().parallelStream().collect(reducing(Integer::sum)).orElse(0);
}

@JsonProperty
public int getLastMilage() {
return this.milages.isEmpty() ? 0 : this.milages.get(this.milages.size() - 1).getAmount().intValue();
}
if (this.milages.isEmpty()) {
return 0;
}

/**
* An example of a nullable Optional
*
* @param period
* @return
*/
public int getMilageInPeriod(final LocalDate period) {
return Optional.ofNullable(this.getPeriods().get(period)).orElse(0);
return this.getLastMilage() - this.getFirstMilage();
}

/**
* Returns an array with 12 elements, containing the milages for each month
* of the given year
*
* @param year The year in which the milages should be computed
* @return 12 values of milages for the month of the given year
* @return The first milage recorded with this app.
*/
public int[] getMilagesInYear(final int year) {
final LocalDate january1st = LocalDate.of(year, Month.JANUARY, 1);
// The limit is necessary because the range contains 13 elements for
// computing the correct periods, the last element is January 1st of year +1
return rangeClosed(0, 12).map(i -> getMilageInPeriod(january1st.plusMonths(i))).limit(12).toArray();
int getFirstMilage() {
return this.milages.isEmpty() ? 0 : this.milages.get(0).getAmount().intValue();
}

/**
* Returns the sum of all milages in the given year
*
* @param year The year for which the sum of milages should be computed
* @return The sum of all milages in the given year
* @return The last milage recorded with this app.
*/
public int getMilageInYear(final int year) {
return Arrays.stream(getMilagesInYear(year)).sum();
@JsonProperty
public int getLastMilage() {
return this.milages.isEmpty() ? 0 : this.milages.get(this.milages.size() - 1).getAmount().intValue();
}

public boolean hasMilages() {
return !this.milages.isEmpty();
}

public static int comparePeriodsByValue(final Map.Entry<LocalDate, Integer> period1, final Map.Entry<LocalDate, Integer> period2) {
return Integer.compare(period1.getValue(), period2.getValue());
}

@RequiredArgsConstructor
public static class BikeByMilageInYearComparator implements Comparator<BikeEntity> {

private final int year;

@Override
public int compare(final BikeEntity o1, final BikeEntity o2) {
return Integer.compare(o1.getMilageInYear(year), o2.getMilageInYear(year));
}
}

/**
* This method groups all periods of the given bikes by their period start
* and summarizes the value
*
* @param bikes A likst of bikes whose milage periods should be grouped
* together
* @param entryFilter An optional filter for the entries
* @return A map of grouped periods
*/
public static Map<LocalDate, Integer> summarizePeriods(final List<BikeEntity> bikes, final Predicate<Map.Entry<LocalDate, Integer>> entryFilter) {
return bikes.stream()
.filter(BikeEntity::hasMilages)
.flatMap(bike -> bike.getPeriods().entrySet().stream())
.filter(Optional.ofNullable(entryFilter).orElse(entry -> true))
.collect(
Collectors.groupingBy(
Map.Entry::getKey,
Collectors.reducing(0, Map.Entry::getValue, Integer::sum)
)
);
}

/**
* Returns the worst performing period in the list of summarized (grouped)
* periods
*
* @param summarizedPeriods A list of grouped periods
* @return The worst (with the lowest value) period
*/
public static AccumulatedPeriod getWorstPeriod(final Map<LocalDate, Integer> summarizedPeriods) {
return summarizedPeriods
.entrySet()
.stream()
.min(BikeEntity::comparePeriodsByValue)
.map(entry -> new AccumulatedPeriod(entry.getKey(), entry.getValue()))
.orElse(null);
}

/**
* Returns the best performing period in the list of summarized (grouped)
* periods
*
* @param summarizedPeriods A list of grouped periods
* @return The best (with the highest value) period
*/
public static AccumulatedPeriod getBestPeriod(final Map<LocalDate, Integer> summarizedPeriods) {
return summarizedPeriods
.entrySet()
.stream()
.max(BikeEntity::comparePeriodsByValue)
.map(entry -> new AccumulatedPeriod(entry.getKey(), entry.getValue()))
.orElse(null);
}
}
@@ -24,7 +24,9 @@
import org.springframework.data.repository.Repository;

/**
* @author Michael J. Simons, 2014-02-08
* @author Michael J. Simons
*
* @since 2014-02-08
*/
public interface BikeRepository extends Repository<BikeEntity, Integer> {

@@ -40,9 +42,6 @@

List<BikeEntity> findByDecommissionedOnIsNull(Sort sort);

@Query("Select coalesce(min(m.recordedOn), current_date()) as dateOfFirstRecord from MilageEntity m")
LocalDate getDateOfFirstRecord();

List<BikeEntity> findAll(Sort sort);

List<BikeEntity> findAll();
@@ -41,12 +41,24 @@
import static org.springframework.web.bind.annotation.RequestMethod.PUT;

/**
* @author Michael J. Simons, 2014-02-19
* @author Michael J. Simons
* @since 2014-02-19
*/
@RestController
@RequestMapping("/api")
class BikesController {

enum Messages {

ALREADY_DECOMMISSIONED("alreadyDecommissioned");

public final String key;

Messages(final String key) {
this.key = "bikes." + key;
}
}

private final BikeRepository bikeRepository;

private final MessageSourceAccessor i18n;
@@ -13,7 +13,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package ac.simons.biking2.bikes;
package ac.simons.biking2.statistics;

import java.time.LocalDate;
import lombok.Getter;
@@ -22,11 +22,12 @@
/**
* Represents a accumulated period value
*
* @author Michael J. Simons, 2014-05-05
* @author Michael J. Simons
* @since 2014-05-05
*/
@RequiredArgsConstructor
@Getter
public final class AccumulatedPeriod {
final class AccumulatedPeriod {
private final LocalDate startOfPeriod;

private final int value;

0 comments on commit 2c7cb85

Please sign in to comment.
You can’t perform that action at this time.