Skip to content
This repository was archived by the owner on Dec 15, 2025. It is now read-only.

Change LDA Optimization Method #503

Merged
mpjlu merged 3 commits into from
Oct 23, 2017
Merged

Change LDA Optimization Method #503

Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
36aae3b
Keep only ml.linear
jtengyp Oct 17, 2017
02b73dc
Merge branch 'linear' of https://github.com/jtengyp/HiBench into linear
jtengyp Oct 17, 2017
b661a1c
Change the Optimizer of LDA to Online method
jtengyp Oct 18, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions sparkbench/ml/src/main/scala/com/intel/sparkbench/ml/LDAExample.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ package com.intel.hibench.sparkbench.ml

import org.apache.spark.{SparkConf, SparkContext}

import org.apache.spark.mllib.clustering.{DistributedLDAModel, LDA}
import org.apache.spark.mllib.clustering.{LDA, DistributedLDAModel, LocalLDAModel}
import org.apache.spark.mllib.linalg.{Vector, Vectors}
import org.apache.spark.rdd.RDD

Expand Down Expand Up @@ -51,11 +51,11 @@ object LDAExample {
val corpus: RDD[(Long, Vector)] = sc.objectFile(inputPath)

// Cluster the documents into numTopics topics using LDA
val ldaModel = new LDA().setK(numTopics).run(corpus)
val ldaModel = new LDA().setK(numTopics).setOptimizer("online").run(corpus)

// Save and load model.
ldaModel.save(sc, outputPath)
val sameModel = DistributedLDAModel.load(sc, outputPath)
val savedModel = LocalLDAModel.load(sc, outputPath)

sc.stop()
}
Expand Down