Added JSON writer

atcoder-problems-backend/src/main/scala/com/kenkoooo/atcoder/Main.scala

@@ -2,22 +2,21 @@ package com.kenkoooo.atcoder
 
 import java.util.concurrent.TimeUnit.{HOURS, MILLISECONDS, MINUTES}
 import java.util.concurrent.{Executors, ScheduledExecutorService}
-
 import akka.actor.ActorSystem
 import akka.http.scaladsl.Http
 import akka.stream.ActorMaterializer
 import com.kenkoooo.atcoder.api.JsonApi
 import com.kenkoooo.atcoder.common.Configure
 import com.kenkoooo.atcoder.common.ScheduledExecutorServiceExtension._
 import com.kenkoooo.atcoder.db.SqlClient
-import com.kenkoooo.atcoder.model.{Contest, Problem}
+import com.kenkoooo.atcoder.model.{ApiJsonSupport, Contest, Problem}
 import com.kenkoooo.atcoder.runner.{AllSubmissionScrapingRunner, NewerSubmissionScrapingRunner}
 import com.kenkoooo.atcoder.scraper.{ContestScraper, ProblemScraper, SubmissionScraper}
 import org.apache.logging.log4j.scala.Logging
-
 import scala.util.{Failure, Success}
+import com.kenkoooo.atcoder.common.JsonWriter._
 
-object Main extends Logging {
+object Main extends Logging with ApiJsonSupport {
   implicit val system: ActorSystem = ActorSystem()
   implicit val materializer: ActorMaterializer = ActorMaterializer()
 
@@ -55,7 +54,19 @@ object Main extends Logging {
         }
 
         // reload records per minute
-        service.tryAtFixedDelay(0, 1, MINUTES)(sql.reloadRecords())
+        service.tryAtFixedDelay(0, 1, MINUTES)({
+          sql.reloadRecords()
+
+          sql.contests.values.toList.toJsonFile(s"${config.files.path}/contests.json")
+          sql.problems.values.toList.toJsonFile(s"${config.files.path}/problems.json")
+          sql.acceptedCounts.toJsonFile(s"${config.files.path}/ac.json")
+          sql.fastestSubmissionCounts.toJsonFile(s"${config.files.path}/fast.json")
+          sql.firstSubmissionCounts.toJsonFile(s"${config.files.path}/first.json")
+          sql.shortestSubmissionCounts.toJsonFile(s"${config.files.path}/short.json")
+          sql.mergedProblems.toJsonFile(s"${config.files.path}/merged-problems.json")
+          sql.ratedPointSums.toJsonFile(s"${config.files.path}/sums.json")
+          sql.languageCounts.toJsonFile(s"${config.files.path}/lang.json")
+        })
 
         // scrape contests per hour
         service.tryAtFixedDelay(0, 1, HOURS) {

atcoder-problems-backend/src/main/scala/com/kenkoooo/atcoder/common/Configure.scala

@@ -7,14 +7,19 @@ import pureconfig.loadConfig
 
 import scala.util.Try
 
-case class Configure(scraper: ScraperConfig, sql: SQLConfig, server: ServerConfig)
+case class Configure(scraper: ScraperConfig,
+                     sql: SQLConfig,
+                     server: ServerConfig,
+                     files: FileConfig)
 
 case class ScraperConfig(threads: Int)
 
 case class SQLConfig(url: String, user: String, password: String)
 
 case class ServerConfig(port: Int)
 
+case class FileConfig(path: String)
+
 object Configure {
   def apply(filepath: String): Try[Configure] = Try {
     loadConfig[Configure](parseFile(new File(filepath))).right.get

atcoder-problems-backend/src/main/scala/com/kenkoooo/atcoder/common/JsonWriter.scala

@@ -0,0 +1,17 @@
+package com.kenkoooo.atcoder.common
+
+import java.io.PrintWriter
+import org.apache.logging.log4j.scala.Logging
+import spray.json._
+
+object JsonWriter extends Logging {
+  implicit class JsonFileWriter[T](any: T) {
+    def toJsonFile(filepath: String)(implicit writer: JsonWriter[T]): Unit = {
+      logger.info(s"writing to $filepath ...")
+      val file = new PrintWriter(filepath)
+      file.write(any.toJson(writer).toString())
+      file.close()
+      logger.info(s"finished writing to $filepath")
+    }
+  }
+}

atcoder-problems-backend/src/test/resources/test-conf.json

@@ -9,5 +9,8 @@
   },
   "server": {
     "port": 9090
+  },
+  "files": {
+    "path": "/home/kenkoooo/"
   }
 }

atcoder-problems-backend/src/test/scala/com/kenkoooo/atcoder/common/ConfigureTest.scala

@@ -9,5 +9,6 @@ class ConfigureTest extends FunSuite with Matchers {
     config.scraper.threads shouldBe 11
     config.sql.url shouldBe "sql-url"
     config.server.port shouldBe 9090
+    config.files.path shouldBe "/home/kenkoooo/"
   }
 }

atcoder-problems-backend/src/test/scala/com/kenkoooo/atcoder/common/JsonWriterTest.scala

@@ -0,0 +1,13 @@
+package com.kenkoooo.atcoder.common
+import com.kenkoooo.atcoder.model.{AcceptedCount, ApiJsonSupport}
+import org.scalatest.{FunSuite, Matchers}
+import scala.io.Source
+import JsonWriter._
+
+class JsonWriterTest extends FunSuite with Matchers with ApiJsonSupport {
+  test("write to temp json file") {
+    val filepath = "/tmp/a.json"
+    List(AcceptedCount("kenkoooo", 11)).toJsonFile(filepath)
+    Source.fromFile(filepath).mkString shouldBe "[{\"user_id\":\"kenkoooo\",\"problem_count\":11}]"
+  }
+}