Audit task flags for fine grained label handling implemented and controllable through Admin dashboard.

app/controllers/AdminController.scala

@@ -619,4 +619,54 @@ class AdminController @Inject() (implicit val env: Environment[User, SessionAuth
       Future.failed(new AuthenticationException("User is not an administrator"))
     }
   }
+
+  /**
+   * Updates the flags of all tasks before the given date for the given user.
+   */
+  def setTaskFlagsByDate() = UserAwareAction.async(BodyParsers.parse.json) { implicit request =>
+    val submission = request.body.validate[TaskFlagsByDateSubmission]
+
+    submission.fold(
+      errors => {
+        Future.successful(BadRequest(Json.obj("status" -> "Error", "message" -> JsError.toFlatJson(errors))))
+      },
+      submission => {
+        if (isAdmin(request.identity)) {
+          UserTable.find(submission.username) match {
+            case Some(user) =>
+              val userId: UUID = UUID.fromString(user.userId)
+              val date: Timestamp = new Timestamp(submission.date)
+              AuditTaskTable.updateTaskFlagByDate(userId, date, submission.flag, submission.state)
+              Future.successful(Ok(Json.obj("userId" -> userId, "date" -> date, "flag" -> submission.flag, "state" -> submission.state)))
+            case None =>
+              Future.successful(BadRequest("No user has this user ID"))
+          }
+        } else {
+          Future.failed(new AuthenticationException("User is not an administrator"))
+        }
+      }
+    )
+  }
+
+  /**
+   * Updates a single flag for a single audit task specified by the audit task id.
+   * @return
+   */
+  def setTaskFlag() = UserAwareAction.async(BodyParsers.parse.json) { implicit request =>
+    val submission = request.body.validate[TaskFlagSubmission]
+
+    submission.fold(
+      errors => {
+        Future.successful(BadRequest(Json.obj("status" -> "Error", "message" -> JsError.toFlatJson(errors))))
+      },
+      submission => {
+        if (isAdmin(request.identity)) {
+          AuditTaskTable.updateTaskFlags(submission.auditTaskId, submission.flag, submission.state)
+          Future.successful(Ok(Json.obj("auditTaskId" -> submission.auditTaskId, "flag" -> submission.flag, "state" -> submission.state)))
+        } else {
+          Future.failed(new AuthenticationException("User is not an administrator"))
+        }
+      }
+    )
+  }
 }

app/controllers/TaskController.scala

@@ -123,12 +123,13 @@ class TaskController @Inject() (implicit val env: Environment[User, SessionAuthe
       val auditTaskObj: AuditTask = user match {
         case Some(user) => AuditTask(0, amtAssignmentId, user.userId.toString, auditTask.streetEdgeId,
           new Timestamp(auditTask.taskStart), timestamp, completed=false, auditTask.currentLat, auditTask.currentLng,
-          auditTask.startPointReversed, Some(missionId), auditTask.currentMissionStart)
+          auditTask.startPointReversed, Some(missionId), auditTask.currentMissionStart, lowQuality=false,
+          incomplete=false, stale=false)
         case None =>
           val user: Option[DBUser] = UserTable.find("anonymous")
           AuditTask(0, amtAssignmentId, user.get.userId, auditTask.streetEdgeId, new Timestamp(auditTask.taskStart),
             timestamp, completed=false, auditTask.currentLat, auditTask.currentLng, auditTask.startPointReversed,
-            Some(missionId), auditTask.currentMissionStart)
+            Some(missionId), auditTask.currentMissionStart, lowQuality=false, incomplete=false, stale=false)
       }
       AuditTaskTable.save(auditTaskObj)
     }

app/formats/AdminUpdateSubmissionFormats.scala

@@ -6,6 +6,10 @@ import play.api.libs.functional.syntax._
 object AdminUpdateSubmissionFormats {
   case class UserRoleSubmission(userId: String, roleId: String)
   case class UserOrgSubmission(userId: String, orgId: Int)
+  case class TaskFlagsByDateSubmission(username: String, date: Long, flag: String, state: Boolean)
+  case class TaskFlagSubmission(auditTaskId: Int, flag: String, state: Boolean) {
+    require(flag == "low_quality" || flag == "incomplete" || flag == "stale")
+  }
 
   implicit val userRoleSubmissionReads: Reads[UserRoleSubmission] = (
     (JsPath \ "user_id").read[String] and
@@ -16,4 +20,17 @@ object AdminUpdateSubmissionFormats {
     (JsPath \ "user_id").read[String] and
       (JsPath \ "org_id").read[Int]
     )(UserOrgSubmission.apply _)
+
+  implicit val taskFlagsByDateSubmissionReads: Reads[TaskFlagsByDateSubmission] = (
+    (JsPath \ "username").read[String] and
+      (JsPath \ "date").read[Long] and
+      (JsPath \ "flag").read[String] and
+      (JsPath \ "state").read[Boolean]
+    )(TaskFlagsByDateSubmission.apply _)
+
+  implicit val taskFlagSubmissionReads: Reads[TaskFlagSubmission] = (
+    (JsPath \ "auditTaskId").read[Int] and
+      (JsPath \ "flag").read[String] and
+      (JsPath \ "state").read[Boolean]
+    )(TaskFlagSubmission.apply _)
 }

app/formats/json/LabelFormat.scala

@@ -121,6 +121,9 @@ object LabelFormat {
       "num_disagree" -> labelMetadata.validations("disagree"),
       "num_notsure" -> labelMetadata.validations("notsure"),
       "tags" -> labelMetadata.tags,
+      "low_quality" -> labelMetadata.lowQualityIncompleteStaleFlags._1,
+      "incomplete" -> labelMetadata.lowQualityIncompleteStaleFlags._2,
+      "stale" -> labelMetadata.lowQualityIncompleteStaleFlags._3,
       // The part below is just lifted straight from Admin Validate without much care.
       "admin_data" -> Json.obj(
         "username" -> adminData.username,

app/formats/json/TaskFormats.scala

@@ -29,7 +29,10 @@ object TaskFormats {
       (__ \ "current_lng").write[Float] and
       (__ \ "start_point_reversed").write[Boolean] and
       (__ \ "current_mission_id").writeNullable[Int] and
-      (__ \ "current_mission_start").writeNullable[Point]
+      (__ \ "current_mission_start").writeNullable[Point] and
+      (__ \ "low_quality").write[Boolean] and
+      (__ \ "incomplete").write[Boolean] and
+      (__ \ "stale").write[Boolean]
     )(unlift(AuditTask.unapply _))
 
   implicit val auditTaskInteractionWrites: Writes[AuditTaskInteraction] = (

app/models/attribute/UserClusteringSessionTable.scala

@@ -5,6 +5,7 @@ import models.label.{LabelTable, LabelTypeTable}
 import models.mission.MissionTable
 import models.region.RegionTable
 import models.user.UserStatTable
+import models.audit.AuditTaskTable
 import models.utils.MyPostgresDriver.simple._
 import play.api.Play.current
 import play.api.db.slick
@@ -77,11 +78,13 @@ object UserClusteringSessionTable {
     _region <- RegionTable.regions if _mission.regionId === _region.regionId
     _userStat <- UserStatTable.userStats if _mission.userId === _userStat.userId
     _lab <- LabelTable.labels if _lab.missionId === _mission.missionId
+    _task <- AuditTaskTable.auditTasks if _lab.auditTaskId === _task.auditTaskId
     _latlng <- LabelTable.labelPoints if _lab.labelId === _latlng.labelId
     _type <- LabelTable.labelTypes if _lab.labelTypeId === _type.labelTypeId
     if _region.deleted === false
     if _lab.correct || (_userStat.highQuality && _lab.correct.isEmpty)
     if _latlng.lat.isDefined && _latlng.lng.isDefined
+    if !_task.lowQuality && !_task.stale
   } yield (_mission.userId, _lab.labelId, _type.labelType, _latlng.lat.get, _latlng.lng.get, _lab.severity, _lab.temporary)
 
   /**

app/models/audit/AuditTaskTable.scala

@@ -20,7 +20,8 @@ import scala.slick.jdbc.{GetResult, StaticQuery => Q}
 
 case class AuditTask(auditTaskId: Int, amtAssignmentId: Option[Int], userId: String, streetEdgeId: Int,
                      taskStart: Timestamp, taskEnd: Timestamp, completed: Boolean, currentLat: Float, currentLng: Float,
-                     startPointReversed: Boolean, currentMissionId: Option[Int], currentMissionStart: Option[Point])
+                     startPointReversed: Boolean, currentMissionId: Option[Int], currentMissionStart: Option[Point],
+                     lowQuality: Boolean, incomplete: Boolean, stale: Boolean)
 case class NewTask(edgeId: Int, geom: LineString,
                    currentLng: Float, currentLat: Float,
                    wayType: String, // OSM road type (residential, trunk, etc.).
@@ -96,8 +97,11 @@ class AuditTaskTable(tag: slick.lifted.Tag) extends Table[AuditTask](tag, "audit
   def startPointReversed = column[Boolean]("start_point_reversed", O.NotNull)
   def currentMissionId = column[Option[Int]]("current_mission_id", O.Nullable)
   def currentMissionStart = column[Option[Point]]("current_mission_start", O.Nullable)
+  def lowQuality = column[Boolean]("low_quality", O.NotNull)
+  def incomplete = column[Boolean]("incomplete", O.NotNull)
+  def stale = column[Boolean]("stale", O.NotNull)
 
-  def * = (auditTaskId, amtAssignmentId, userId, streetEdgeId, taskStart, taskEnd, completed, currentLat, currentLng, startPointReversed, currentMissionId, currentMissionStart) <> ((AuditTask.apply _).tupled, AuditTask.unapply)
+  def * = (auditTaskId, amtAssignmentId, userId, streetEdgeId, taskStart, taskEnd, completed, currentLat, currentLng, startPointReversed, currentMissionId, currentMissionStart, lowQuality, incomplete, stale) <> ((AuditTask.apply _).tupled, AuditTask.unapply)
 
   def streetEdge: ForeignKeyQuery[StreetEdgeTable, StreetEdge] =
     foreignKey("audit_task_street_edge_id_fkey", streetEdgeId, TableQuery[StreetEdgeTable])(_.streetEdgeId)
@@ -117,7 +121,8 @@ object AuditTaskTable {
 
   implicit val auditTaskConverter = GetResult[AuditTask](r => {
     AuditTask(r.nextInt, r.nextIntOption, r.nextString, r.nextInt, r.nextTimestamp, r.nextTimestamp, r.nextBoolean,
-      r.nextFloat, r.nextFloat, r.nextBoolean, r.nextIntOption, r.nextGeometryOption[Point])
+      r.nextFloat, r.nextFloat, r.nextBoolean, r.nextIntOption, r.nextGeometryOption[Point], r.nextBoolean,
+      r.nextBoolean, r.nextBoolean)
   })
 
   implicit val newTaskConverter = GetResult[NewTask](r => {
@@ -506,4 +511,45 @@ object AuditTaskTable {
     val q = for { t <- auditTasks if t.auditTaskId === auditTaskId } yield (t.taskEnd, t.currentLat, t.currentLng, t.currentMissionId, t.currentMissionStart)
     q.update((timestamp, lat, lng, Some(missionId), currMissionStart))
   }
+
+  /**
+   * Update a single task's flag given the flag type and the status to change to.
+   * @param auditTaskId
+   * @param flag
+   * @param state
+   * @return
+   */
+  def updateTaskFlags(auditTaskId: Int, flag: String, state: Boolean): Int = db.withSession { implicit session =>
+    val q = for {
+      t <- auditTasks if t.auditTaskId === auditTaskId
+    } yield (flag match {
+      case "low_quality" => t.lowQuality
+      case "incomplete" => t.incomplete
+      case "stale" => t.stale
+    })
+
+    q.update(state)
+  }
+
+  /**
+   * Update all flags of a single type for tasks starting before a specified date.
+   * @param userId
+   * @param date
+   * @param flag
+   * @param state
+   * @return
+   */
+  def updateTaskFlagByDate(userId: UUID, date: Timestamp, flag: String, state: Boolean): Int = db.withSession { implicit session =>
+    val q = for {
+      t <- auditTasks if t.userId === userId.toString && t.taskStart < date
+    } yield (flag match {
+      case "low_quality" => t.lowQuality
+      case "incomplete" => t.incomplete
+      case "stale" => t.stale
+    })
+
+    q.update(state)
+  }
+
+
 }

app/models/label/LabelTable.scala

@@ -151,7 +151,8 @@ object LabelTable {
                            streetEdgeId: Int, regionId: Int, userId: String, username: String,
                            timestamp: java.sql.Timestamp, labelTypeKey: String, labelTypeValue: String,
                            severity: Option[Int], temporary: Boolean, description: Option[String],
-                           userValidation: Option[Int], validations: Map[String, Int], tags: List[String])
+                           userValidation: Option[Int], validations: Map[String, Int], tags: List[String],
+                           lowQualityIncompleteStaleFlags: (Boolean, Boolean, Boolean))
 
   case class LabelMetadataUserDash(labelId: Int, gsvPanoramaId: String, heading: Float, pitch: Float, zoom: Int,
                                    canvasX: Int, canvasY: Int, labelType: String,
@@ -191,7 +192,8 @@ object LabelTable {
       (r.nextInt, r.nextInt), r.nextInt, r.nextInt, r.nextInt, r.nextString, r.nextString,
       r.nextTimestamp, r.nextString, r.nextString, r.nextIntOption, r.nextBoolean, r.nextStringOption, r.nextIntOption,
       r.nextString.split(',').map(x => x.split(':')).map { y => (y(0), y(1).toInt) }.toMap,
-      r.nextStringOption.map(tags => tags.split(",").toList).getOrElse(List())
+      r.nextStringOption.map(tags => tags.split(",").toList).getOrElse(List()),
+      (r.nextBoolean, r.nextBoolean, r.nextBoolean)
     )
   )
 
@@ -447,7 +449,10 @@ object LabelTable {
          |       lb_big.description,
          |       lb_big.validation_result,
          |       val.val_counts,
-         |       lb_big.tag_list
+         |       lb_big.tag_list,
+         |       at.low_quality,
+         |       at.incomplete,
+         |       at.stale
          |FROM label AS lb1,
          |     gsv_data,
          |     audit_task AS at,
@@ -619,13 +624,13 @@ object LabelTable {
            |-- marked as high quality. Up to 100 more points (100 / (1 + abs(agree_count - disagree_count))) depending
            |-- on how far we are from consensus. Another 25 points if the label was added in the past week. Then add a
            |-- random number so that the max score for each label is 276.
-           |ORDER BY CASE WHEN COALESCE(needs_validations, TRUE) AND label.correct IS NULL THEN 100 ELSE 0 END +
-           |    CASE WHEN user_stat.high_quality THEN 50 ELSE 0 END +
+           |ORDER BY CASE WHEN COALESCE(needs_validations, TRUE) AND label.correct IS NULL AND NOT audit_task.low_quality AND NOT audit_task.stale THEN 100 ELSE 0 END +
+           |    CASE WHEN user_stat.high_quality AND NOT audit_task.low_quality AND NOT audit_task.stale THEN 50 ELSE 0 END +
            |    100.0 / (1 + abs(label.agree_count - label.disagree_count)) +
            |    CASE WHEN label.time_created > now() - INTERVAL '1 WEEK' THEN 25 ELSE 0 END +
            |    RANDOM() * (276 - (
-           |        CASE WHEN COALESCE(needs_validations,  TRUE) AND label.correct IS NULL THEN 100 ELSE 0 END +
-           |            CASE WHEN user_stat.high_quality THEN 50 ELSE 0 END +
+           |        CASE WHEN COALESCE(needs_validations,  TRUE) AND label.correct IS NULL AND NOT audit_task.low_quality AND NOT audit_task.stale THEN 100 ELSE 0 END +
+           |            CASE WHEN user_stat.high_quality AND NOT audit_task.low_quality AND NOT audit_task.stale THEN 50 ELSE 0 END +
            |            100.0 / (1 + abs(label.agree_count - label.disagree_count)) +
            |            CASE WHEN label.time_created > now() - INTERVAL '1 WEEK' THEN 25 ELSE 0 END
            |        )) DESC

app/models/street/StreetEdgePriorityTable.scala

@@ -203,10 +203,12 @@ object StreetEdgePriorityTable {
     // frequency above our threshold.
     // NOTE We are calling the getQualityOfUsers function below, which does the heavy lifting.
     val completions = AuditTaskTable.completedTasks
-      .groupBy(task => (task.streetEdgeId, task.userId)).map(_._1)  // select distinct on street edge id and user id
+      // Select distinct street edge ids, and keep user id and street flags
+      .groupBy(task => (task.streetEdgeId, task.userId, task.lowQuality, task.incomplete, task.stale)).map(_._1)
       .innerJoin(UserStatTable.getQualityOfUsers).on(_._2 === _._1)  // join on user_id
       .filterNot(_._2._3) // filter out users marked with excluded = TRUE
-      .map { case (_task, _qual) => (_task._1, _qual._2) }  // SELECT street_edge_id, is_good_user
+      // SELECT street_edge_id, (is_good_user or low_quality or incomplete or stale)
+      .map { case (_task, _qual) => (_task._1, (_qual._2 || _task._3 || _task._4 || _task._5)) }
 
     /********** Compute Audit Counts **********/
 

app/models/user/UserStatTable.scala

@@ -341,12 +341,15 @@ object UserStatTable {
         |    INNER JOIN user_stat ON sidewalk_user.user_id = user_stat.user_id
         |    INNER JOIN mission ON sidewalk_user.user_id = mission.user_id
         |    INNER JOIN label ON mission.mission_id = label.mission_id
+        |    INNER JOIN audit_task ON label.audit_task_id = audit_task.audit_task_id
         |    $joinUserOrgTable
         |    WHERE label.deleted = FALSE
         |        AND label.tutorial = FALSE
         |        AND role.role IN ('Registered', 'Administrator', 'Researcher')
         |        AND user_stat.excluded = FALSE
         |        AND (label.time_created AT TIME ZONE 'US/Pacific') > $statStartTime
+        |        AND low_quality = FALSE
+        |        AND incomplete = FALSE
         |        $orgFilter
         |    GROUP BY $groupingCol
         |    ORDER BY label_count DESC