word2vec java版本的一个实现
Switch branches/tags
Nothing to show
Clone or download
Latest commit 76ff20f Nov 21, 2017
Permalink
Failed to load latest commit information.
src 修改compareTo方法的错误 Feb 28, 2017
.gitignore 去掉外部jar包依赖。 May 11, 2015
README.md Update README.md Nov 21, 2017
pom.xml maven project May 9, 2016

README.md

Word2VEC_java

word2vec java版本的一个实现

有人抱怨没有测试代码。我工作中用到。写了个例子正好发这里。大家领会下精神把

有人抱怨没有语料 https://pan.baidu.com/s/1jIy3YSY 大家用这个吧

package com.kuyun.document_class;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.util.List;

import org.ansj.domain.Term;
import org.ansj.splitWord.analysis.ToAnalysis;

import com.alibaba.fastjson.JSONObject;
import com.ansj.vec.Learn;
import com.ansj.vec.Word2VEC;

import love.cq.util.IOUtil;
import love.cq.util.StringUtil;

public class Word2VecTest {
    private static final File sportCorpusFile = new File("corpus/result.txt");

    public static void main(String[] args) throws IOException {
        File[] files = new File("corpus/sport/").listFiles();
        
        //构建语料
        try (FileOutputStream fos = new FileOutputStream(sportCorpusFile)) {
            for (File file : files) {
                if (file.canRead() && file.getName().endsWith(".txt")) {
                    parserFile(fos, file);
                }
            }
        }
        
        //进行分词训练
        
        Learn lean = new Learn() ;
        
        lean.learnFile(sportCorpusFile) ;
        
        lean.saveModel(new File("model/vector.mod")) ;
        
        
        
        //加载测试
        
        Word2VEC w2v = new Word2VEC() ;
        
        w2v.loadJavaModel("model/vector.mod") ;
        
        System.out.println(w2v.distance("姚明")); ;

    }

    private static void parserFile(FileOutputStream fos, File file) throws FileNotFoundException,
                                                                   IOException {
        // TODO Auto-generated method stub
        try (BufferedReader br = IOUtil.getReader(file.getAbsolutePath(), IOUtil.UTF8)) {
            String temp = null;
            JSONObject parse = null;
            while ((temp = br.readLine()) != null) {
                parse = JSONObject.parseObject(temp);
                paserStr(fos, parse.getString("title"));
                paserStr(fos, StringUtil.rmHtmlTag(parse.getString("content")));
            }
        }
    }

    private static void paserStr(FileOutputStream fos, String title) throws IOException {
        List<Term> parse2 = ToAnalysis.parse(title) ;
        StringBuilder sb = new StringBuilder() ;
        for (Term term : parse2) {
            sb.append(term.getName()) ;
            sb.append(" ");
        }
        fos.write(sb.toString().getBytes()) ;
        fos.write("\n".getBytes()) ;
    }
}