Merge pull request #93 from radibnia77/master

Performance-Forecasting Application Suite

IMService/README.md

@@ -0,0 +1,19 @@
+### Requirements
+
+1. Java 8
+2. Gradle 6.6.1
+
+### Run
+This project uses gradle as a build tool.
+'gradle tasks' shows different tasks that are handle by jar and war plugins.
+'gradle war' to compile and build the project.
+Default value for webAppDirName is src/main/webapp
+
+### War
+
+The default behavior of the War task is to copy the content of src/main/webapp to the root of the archive. 
+Your webapp directory may of course contain a WEB-INF sub-directory, which may contain a web.xml file. 
+Your compiled classes are compiled to WEB-INF/classes. 
+All the dependencies of the runtime [1] configuration are copied to WEB-INF/lib.
+
+

IMService/VERSION.md

@@ -0,0 +1,11 @@
+# BM Version 1.1
+
+1. Update build.gradle to version 6.6.1
+2. Add 'es.predictions.inventory.path' to db.properties which indicates the path of h0,h1,h2,h3 in predictions index
+3. Add new service to return daily inventory "/api/inventory/daily/count" .
+4. Make Day objects immutable
+
+# BM Version 1.7
+1. Add new API to return daily impressions. The document is in doc/api-daily-count.md
+
+

IMService/imsservice/doc/api-daily-count.md

@@ -0,0 +1,88 @@
+### Count Inventory
+##### Description
+> This end-point returns amount of daily inventory for a targeting channel considering time period and price.
+> The inventory calculation process considers all the bookings.
+> Targeting Channel is a set of attributes that specify a group of users (audience) plus the media.
+##### Method: POST
+##### URL: http://service-domain/imsservice/api/inventory/daily/count
+#####POST Body:
+```sh
+{
+    targeting_channel:<ucdocument>,
+    ranges: [ {
+        sd:<start-date date in yyyy-mm-dd>,
+        ed:<end-date date in yyyy-mm-dd>,
+        sh:<start-hour in hh>,
+        eh:<end-hour in hh>} …]
+    price: <CPC price> numeric format.
+}
+
+The following attributes are used in targeting channel:
+1)	pm: Price Model – CPM/CPC/CPD/CPT
+2)	ai: Ad slot - list
+3)	ipl: Geo-location – list of city code
+4)	r: Resident-location – list of city code
+5)	g: Gender - list
+6)	a: Age – list
+-	0 - (0,18)
+-	1 - [18,24]
+-	2 - (24,34]
+-	3 - (34,44]
+-	4 - (44,54]
+-	5 - (55,+)
+7)	aus: App usage – list of {category?_appusage_state}, state is one of following
+a.	active_30days
+b.	activated
+c.	installed_not_activated
+d.	not_instatlled
+8)	ais: App interest – list
+9)	t: Connection type - list
+10)	pdas: Pre-defined audiences – list
+11)	exclude_pdas: Excluded pre-defined audiences - list
+12)	dms: Device model – list
+13)	dpc: Device price category range of [0,4] - list
+-	(0-1500] – 0_1500
+-	(1500-2500] – 1500_2500
+-	(2500-3500] – 2500_3500
+-	(3500-4500] – 3500_4500
+-	(4500-+) – 4500
+
+
+Excample:
+{
+	"targetingChannel": {
+		"g":["g_f"]
+	},
+	"price":0,
+	"days": [{
+		"st": "2020-07-02",
+		"ed": "2020-07-04",
+		"sh": 0,
+		"eh": 23
+	}]
+	
+}
+```
+##### Response:
+```sh
+{
+    "2020-07-03": {
+        "h0": 0,
+        "h1": 38739678,
+        "h2": 419115,
+        "h3": 505916
+    },
+    "2020-07-04": {
+        "h0": 0,
+        "h1": 10249965,
+        "h2": 429323,
+        "h3": 255991
+    },
+    "2020-07-02": {
+        "h0": 0,
+        "h1": 24837972,
+        "h2": 298850,
+        "h3": 287530
+    }
+}
+```

IMService/imsservice/doc/build-unix.md

@@ -4,12 +4,12 @@ Gradle runs on all major operating systems and requires only a Java JDK or JRE v
 ###  Download and Install
 Download code using following git command:
 
-git clone -b opensource-1.0.1 http://10.124.206.248:30080/device-cloud-service/common-pps-ims-tmp.git
+git clone -b master https://github.com/Futurewei-io/blue-marlin.git
 
-go to directory ../PPS-IMS/FIBased/IMService and run gradle war to build war file.
+go to directory ../IMService and run gradle war to build war file.
 
 The war file will be created in 
-.../PPS-IMS/FIBased/IMService/imsservice/build
+.../IMService/imsservice/build
 
 Deploy the war file into a web server such as tomcat, weblogic, jboss or etc.
 
@@ -18,7 +18,7 @@ http://<server-ip>/imsservice/ping
 
 ###  Data Preparation
 
-Three scripts are provided in .../PPS-IMS/FIBased/IMService/imsservice/src/main/resources/data to generate data documents for Elasticsearch.
+Three scripts are provided in .../IMService/imsservice/src/main/resources/data to generate data documents for Elasticsearch.
 More information is provided by each script.
 
 ###  Testing

IMService/imsservice/src/main/java/com/bluemarlin/ims/imsservice/controller/IMSController.java

@@ -21,12 +21,7 @@
 
 package com.bluemarlin.ims.imsservice.controller;
 
-import com.bluemarlin.ims.imsservice.model.BookResult;
-import com.bluemarlin.ims.imsservice.model.Booking;
-import com.bluemarlin.ims.imsservice.model.DayImpression;
-import com.bluemarlin.ims.imsservice.model.IMSBookingRequest;
-import com.bluemarlin.ims.imsservice.model.IMSRequestQuery;
-import com.bluemarlin.ims.imsservice.model.InventoryResult;
+import com.bluemarlin.ims.imsservice.model.*;
 import com.bluemarlin.ims.imsservice.service.BookingService;
 import com.bluemarlin.ims.imsservice.service.InventoryEstimateService;
 import com.bluemarlin.ims.imsservice.service.UserEstimateService;
@@ -111,6 +106,37 @@ public ResponseEntity getInventory(@RequestBody IMSRequestQuery payload)
         return ResponseBuilder.build(result, HTTP_OK_CODE);
     }
 
+    /**
+     * This end-point returns daily inventories for a targeting channel considering day period (no price).
+     * The inventory calculation process considers all the bookings.
+     * Targeting Channel is a set of attributes that specify a group of users (audience) plus the media.
+     *
+     * @param payload
+     * @return
+     */
+    @RequestMapping(value = "/api/inventory/daily/count", method = RequestMethod.POST)
+    @ResponseBody
+    public ResponseEntity getDailyInventory(@RequestBody IMSRequestQuery payload)
+    {
+        Map<Day, Impression.ImpressionShort> result;
+        try
+        {
+            result = inventoryEstimateService.aggregateDailyInventory(payload.getTargetingChannel(), payload.getDays(),
+                    payload.getPrice());
+
+            String logMessage = String.format("Targeting channel: %s - daily count",
+                    payload.getTargetingChannel().getQueryKey());
+            LOGGER.info(logMessage);
+        }
+        catch (Exception e)
+        {
+            LOGGER.error(e.getMessage());
+            return ResponseBuilder.buildError(e);
+        }
+
+        return ResponseBuilder.build(result, HTTP_OK_CODE);
+    }
+
     /**
      * This end-point returns number of unique users for a targeting channel in a time period.
      *

IMService/imsservice/src/main/java/com/bluemarlin/ims/imsservice/dao/BaseDaoESImp.java

@@ -60,6 +60,7 @@ public class BaseDaoESImp implements BaseDao
     protected String bookingsIndex = "";
     protected String userPredictionsIndex = "";
     protected String bookingBucketsIndex = "";
+    protected String esPredictionsInventoryPath = "";
 
     protected static final IMSLogger LOGGER = IMSLogger.instance();
     protected Properties properties;
@@ -105,6 +106,7 @@ public BaseDaoESImp(Properties properties)
         bookingsIndex = properties.getProperty("es.bookings.index");
         bookingBucketsIndex = properties.getProperty("es.booking_buckets.index");
         userPredictionsIndex = properties.getProperty("es.user_predictions.index");
+        esPredictionsInventoryPath = properties.getProperty("es.predictions.inventory.path");
     }
 
     public void setESClient(ESClient esclient)

IMService/imsservice/src/main/java/com/bluemarlin/ims/imsservice/dao/inventory/InventoryEstimateDaoESImp.java

@@ -90,7 +90,7 @@ private Map<Day, Impression> getImpressionCountForFullDays(BoolQueryBuilder bool
             for (int i = 0; i < 4; i++)
             {
                 SumAggregationBuilder aggregationBuilders =
-                        new SumAggregationBuilder("h" + i + "_" + day).field(ES_PREDICTION_DOC_PREFIX + "predictions." + day + ".h" + i);
+                        new SumAggregationBuilder("h" + i + "_" + day).field(String.format(esPredictionsInventoryPath, day) + "h" + i);
                 sourceBuilder = sourceBuilder.aggregation(aggregationBuilders);
             }
         }
@@ -199,7 +199,7 @@ private Map<Day, Impression> applyRegionRatio(Map<Day, Impression> totalMap, Map
         {
             Day day = entry.getKey();
             Impression impression = entry.getValue();
-            Day newDay = new Day(day.getDayStr());
+            Day newDay = new Day(day.toString());
             Impression newImpression = Impression.multiply(impression, ratio);
             if (totalMap != null && totalMap.containsKey(day))
             {
@@ -213,7 +213,7 @@ private Map<Day, Impression> applyRegionRatio(Map<Day, Impression> totalMap, Map
 
     private Impression getPredictionsForQuery(Day day, BoolQueryBuilder boolQueryBuilder) throws IOException
     {
-        Map<Day, Impression> countMap = getImpressionCountForFullDays(boolQueryBuilder, new HashSet<>(Arrays.asList(day.getDayStr())));
+        Map<Day, Impression> countMap = getImpressionCountForFullDays(boolQueryBuilder, new HashSet<>(Arrays.asList(day.toString())));
         Impression result = new Impression();
         if (countMap.size() > 0)
         {

IMService/imsservice/src/main/java/com/bluemarlin/ims/imsservice/model/Day.java

@@ -29,10 +29,10 @@
 import java.util.Map;
 import java.util.Set;
 
-public class Day implements Comparable
+final public class Day implements Comparable
 {
-    private String dayStr;
-    private List<Integer> hours = new ArrayList<>();
+    private final String dayStr;
+    private final List<Integer> hours = new ArrayList<>();
 
     /**
      * Day format is YYYY-MM-DD
@@ -52,10 +52,12 @@ public Day(String dayStr)
     public Day(String dayStr, List<Integer> hours)
     {
         this.dayStr = dayStr;
-        this.hours = hours;
+        for (Integer i : hours){
+            this.hours.add(i.intValue());
+        }
     }
 
-    public String getDayStr()
+    public String toString()
     {
         return dayStr;
     }

IMService/imsservice/src/main/java/com/bluemarlin/ims/imsservice/model/Impression.java

@@ -24,15 +24,39 @@
 import com.bluemarlin.ims.imsservice.service.BookingService;
 import com.bluemarlin.ims.imsservice.util.CommonUtil;
 import com.bluemarlin.ims.imsservice.util.IMSLogger;
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
 import com.fasterxml.jackson.annotation.JsonProperty;
 
 import java.io.Serializable;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
 
+import static java.lang.Math.min;
+
 public class Impression implements Serializable
 {
+    static public class ImpressionShort
+    {
+        @JsonProperty("h0")
+        private long h0;
+
+        @JsonProperty("h1")
+        private long h1;
+
+        @JsonProperty("h2")
+        private long h2;
+
+        @JsonProperty("h3")
+        private long h3;
+
+        public ImpressionShort(Impression impression){
+            this.h0 = impression.h0.t;
+            this.h1 = impression.h1.t;
+            this.h2 = impression.h2.t;
+            this.h3 = impression.h3.t;
+        }
+    }
     private static final IMSLogger LOGGER = IMSLogger.instance();
 
     public static final int TOTAL_NUM_OF_PRICE_CATEGORIES = 4;
@@ -124,8 +148,7 @@ public long countImpressions(double price, TargetingChannel.PriceModel priceMode
         if (price == BookingService.HIGH_PRICE)
         {
             result = this.getTotal();
-        }
-        else
+        } else
         {
             int priceCategory = CommonUtil.determinePriceCat(price, priceModel);
             result = getCountByPriceCategory(priceCategory);
@@ -244,6 +267,25 @@ public static Impression multiply(Impression i1, double r)
         return item;
     }
 
+    public static Impression subtractBookedValue(Impression i1, long v)
+    {
+        Impression item = new Impression();
+        long[] impValues = new long[]{i1.getH0().getT(), i1.getH1().getT(), i1.getH2().getT(), i1.getH3().getT()};
+
+        long[] hBooked = new long[4];
+        for (int i = 0; i < 4; i++)
+        {
+            hBooked[0] = min(impValues[i], v);
+            v -= hBooked[0];
+        }
+
+        item.h0 = new H0(impValues[0] - hBooked[0]);
+        item.h1 = new H1(impValues[1] - hBooked[1]);
+        item.h2 = new H2(impValues[2] - hBooked[2]);
+        item.h3 = new H3(impValues[3] - hBooked[3]);
+        return item;
+    }
+
     public long getCountByPriceCategory(int priceCategory)
     {
         long t = 0;

IMService/imsservice/src/main/java/com/bluemarlin/ims/imsservice/service/BookingBucketAllocation.java

@@ -218,6 +218,7 @@ public Object call() throws Exception
          */
         if (insideAmount <= 0 || utilVariables.atomicRequestAmount.get() <= 0)
         {
+            //System.out.println("booking bucket allocation "+insideAmount+" , "+utilVariables.atomicRequestAmount.get());
             return 0;
         }
 

IMService/imsservice/src/main/java/com/bluemarlin/ims/imsservice/service/BookingService.java

@@ -246,7 +246,7 @@ public BookResult book(TargetingChannel targetingChannel, List<Range> ranges, do
             {
                 Future bookedForDayFuture = executor.submit((Callable<Object>) () ->
                 {
-                    BookingBucket maxPriorityBB = bookingDao.getMaxBookingBucketPriority(day.getDayStr());
+                    BookingBucket maxPriorityBB = bookingDao.getMaxBookingBucketPriority(day.toString());
                     BookingBucket bookingBucket;
 
                     List<Booking> bookings = bookingDayMap.get(day);
@@ -260,7 +260,7 @@ public BookResult book(TargetingChannel targetingChannel, List<Range> ranges, do
                         priority = maxPriorityBB.getPriority() + 1;
                     }
 
-                    bookingBucket = new BookingBucket(day.getDayStr(), booking.getBookingId(), previousBookingIds, priority + 1);
+                    bookingBucket = new BookingBucket(day.toString(), booking.getBookingId(), previousBookingIds, priority + 1);
                     bookingDao.createBookingBucket(bookingBucket);
 
                     /**