-
Notifications
You must be signed in to change notification settings - Fork 92
/
MagdaApp.scala
60 lines (47 loc) · 1.91 KB
/
MagdaApp.scala
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
package au.csiro.data61.magda
import scala.collection.JavaConversions._
import scala.util.Failure
import scala.util.Success
import com.typesafe.config.ConfigObject
import com.typesafe.config.ConfigValue
import akka.actor.Actor
import akka.actor.ActorLogging
import akka.actor.ActorSystem
import akka.actor.DeadLetter
import akka.actor.Props
import akka.event.Logging
import akka.stream.ActorMaterializer
import au.csiro.data61.magda.crawler.Crawler
import au.csiro.data61.magda.external.InterfaceConfig
import au.csiro.data61.magda.search.SearchIndexer
import au.csiro.data61.magda.search.elasticsearch.DefaultClientProvider
object MagdaApp extends App {
implicit val system = ActorSystem()
implicit val executor = system.dispatcher
implicit val materializer = ActorMaterializer()
implicit val config = AppConfig.conf
val logger = Logging(system, getClass)
logger.info("Starting Indexer in env {}", AppConfig.env)
val listener = system.actorOf(Props(classOf[Listener]))
system.eventStream.subscribe(listener, classOf[DeadLetter])
val interfaceConfigs = config.getConfig("indexedServices").root().map {
case (name: String, serviceConfig: ConfigValue) =>
InterfaceConfig(serviceConfig.asInstanceOf[ConfigObject].toConfig)
}.toSeq
// Index erryday
// system.scheduler.schedule(0 millis, 1 days, supervisor, Start(List((ExternalInterfaceType.CKAN, new URL(config.getString("services.dga-api.baseUrl"))))))
logger.debug("Starting Crawler")
val indexer = SearchIndexer(new DefaultClientProvider, config)
val crawler = new Crawler(system, config.getConfig("indexer"), interfaceConfigs, materializer, indexer)
crawler.crawl() onComplete {
case Success(_) =>
logger.info("Successfully completed crawl")
case Failure(e) =>
logger.error(e, "Crawl failed")
}
}
class Listener extends Actor with ActorLogging {
def receive = {
case d: DeadLetter => log.debug(d.message.toString())
}
}