Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP

Comparing changes

Choose two branches to see what's changed or to start a new pull request. If you need to, you can also compare across forks.

Open a pull request

Create a new pull request by comparing changes across two branches. If you need to, you can also compare across forks.
base fork: genericDataCompany/importConfig
base: c5a26313d3
...
head fork: genericDataCompany/importConfig
compare: b18f35ed6a
Checking mergeability… Don't worry, you can still create the pull request.
  • 2 commits
  • 24 files changed
  • 0 commit comments
  • 1 contributor
Commits on Sep 18, 2013
wangcb Init commit f4a11cc
wangcb add gitignore b18f35e
Showing with 1,721 additions and 1 deletion.
  1. +1 −1  .gitignore
  2. +1 −0  file1.txt
  3. +23 −0 importConfig.xml
  4. +74 −0 importIndex.iml
  5. +6 −0 mapping/qq.xml
  6. +20 −0 mapping/transportation.xml
  7. +176 −0 pom.xml
  8. +367 −0 src/main/java/com/GenItemID.java
  9. +150 −0 src/main/java/com/ImportCSV.java
  10. +25 −0 src/main/java/com/yao/core/Import.java
  11. +159 −0 src/main/java/com/yao/dao/impl/BaseDaoImpl.java
  12. +134 −0 src/main/java/com/yao/dao/impl/ImportIndex.java
  13. +179 −0 src/main/java/com/yao/entity/AllConfig.java
  14. +92 −0 src/main/java/com/yao/io/LoadConfig.java
  15. +38 −0 src/test/java/com/yao/importIndex/AppTest.java
  16. +5 −0 target/classes/META-INF/MANIFEST.MF
  17. +7 −0 target/classes/META-INF/maven/com.yao/importIndex/pom.properties
  18. +176 −0 target/classes/META-INF/maven/com.yao/importIndex/pom.xml
  19. +5 −0 target/maven-archiver/pom.properties
  20. 0  target/maven-status/maven-compiler-plugin/compile/default-compile/createdFiles.lst
  21. +7 −0 target/maven-status/maven-compiler-plugin/compile/default-compile/inputFiles.lst
  22. 0  target/maven-status/maven-compiler-plugin/testCompile/default-testCompile/createdFiles.lst
  23. +72 −0 target/surefire-reports/TEST-com.yao.importIndex.AppTest.xml
  24. +4 −0 target/surefire-reports/com.yao.importIndex.AppTest.txt
View
2  .gitignore
@@ -1,5 +1,5 @@
*.class
-
+*.iml
# Package Files #
*.jar
*.war
View
1  file1.txt
@@ -0,0 +1 @@
+499fc421577045fb9a35bd6ae8ebc388,61,京O3542,2013-09-18 03:57:44,888888888888,北京市西直门2号桥,9999999999,82,54,1111111111111111,36,289,ftp://pic4/0039/20120505/114,ftp://pic4/0039/20120505/114,ftp://pic4/0039/20120505/114,5,11,BMW,null,0,a0cc2b573d844342af77743da4bac847
View
23 importConfig.xml
@@ -0,0 +1,23 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<config>
+<!-- 数据库信息 -->
+ <database-url>jdbc:mysql://10.201.20.228:3306</database-url>
+ <database-username>root</database-username>
+ <database-password>12345</database-password>
+
+ <database-name>test</database-name>
+ <database-table>wf_mds_chn_book</database-table>
+
+<!-- 集群信息 -->
+ <cluster-name>transportation</cluster-name>
+ <cluster-host>10.201.20.228</cluster-host>
+ <cluster-port>9300</cluster-port>
+<!-- 索引信息 -->
+ <index-name>transportation</index-name>
+ <index-type>transportation</index-type>
+
+<!-- 运行信息 -->
+ <batchId>id</batchId><!-- f_id -->
+ <identify>nid</identify> <!-- 唯一标识 -->
+ <batch-size>100</batch-size>
+</config>
View
74 importIndex.iml
@@ -0,0 +1,74 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<module org.jetbrains.idea.maven.project.MavenProjectsManager.isMavenModule="true" type="JAVA_MODULE" version="4">
+ <component name="NewModuleRootManager" LANGUAGE_LEVEL="JDK_1_6" inherit-compiler-output="false">
+ <output url="file://$MODULE_DIR$/target/classes" />
+ <output-test url="file://$MODULE_DIR$/target/test-classes" />
+ <content url="file://$MODULE_DIR$">
+ <sourceFolder url="file://$MODULE_DIR$/src/main/java" isTestSource="false" />
+ <sourceFolder url="file://$MODULE_DIR$/src/test/java" isTestSource="true" />
+ <excludeFolder url="file://$MODULE_DIR$/target" />
+ </content>
+ <orderEntry type="inheritedJdk" />
+ <orderEntry type="sourceFolder" forTests="false" />
+ <orderEntry type="library" scope="TEST" name="Maven: junit:junit:3.8.1" level="project" />
+ <orderEntry type="library" name="Maven: org.elasticsearch:elasticsearch:0.90.0" level="project" />
+ <orderEntry type="library" name="Maven: org.apache.lucene:lucene-core:4.2.1" level="project" />
+ <orderEntry type="library" name="Maven: org.apache.lucene:lucene-analyzers-common:4.2.1" level="project" />
+ <orderEntry type="library" name="Maven: org.apache.lucene:lucene-codecs:4.2.1" level="project" />
+ <orderEntry type="library" name="Maven: org.apache.lucene:lucene-queries:4.2.1" level="project" />
+ <orderEntry type="library" name="Maven: org.apache.lucene:lucene-memory:4.2.1" level="project" />
+ <orderEntry type="library" name="Maven: org.apache.lucene:lucene-highlighter:4.2.1" level="project" />
+ <orderEntry type="library" name="Maven: org.apache.lucene:lucene-queryparser:4.2.1" level="project" />
+ <orderEntry type="library" name="Maven: org.apache.lucene:lucene-sandbox:4.2.1" level="project" />
+ <orderEntry type="library" name="Maven: org.apache.lucene:lucene-suggest:4.2.1" level="project" />
+ <orderEntry type="library" name="Maven: org.apache.lucene:lucene-join:4.2.1" level="project" />
+ <orderEntry type="library" name="Maven: org.apache.lucene:lucene-grouping:4.2.1" level="project" />
+ <orderEntry type="library" name="Maven: org.apache.lucene:lucene-spatial:4.2.1" level="project" />
+ <orderEntry type="library" name="Maven: com.spatial4j:spatial4j:0.3" level="project" />
+ <orderEntry type="library" name="Maven: org.elasticsearch:elasticsearch-mapper-attachments:1.8.0" level="project" />
+ <orderEntry type="library" name="Maven: org.apache.tika:tika-parsers:1.2" level="project" />
+ <orderEntry type="library" name="Maven: org.apache.tika:tika-core:1.2" level="project" />
+ <orderEntry type="library" name="Maven: org.gagravarr:vorbis-java-tika:0.1" level="project" />
+ <orderEntry type="library" scope="RUNTIME" name="Maven: org.gagravarr:vorbis-java-core:tests:0.1" level="project" />
+ <orderEntry type="library" name="Maven: edu.ucar:netcdf:4.2-min" level="project" />
+ <orderEntry type="library" name="Maven: org.slf4j:slf4j-api:1.7.5" level="project" />
+ <orderEntry type="library" name="Maven: org.apache.james:apache-mime4j-core:0.7.2" level="project" />
+ <orderEntry type="library" name="Maven: org.apache.james:apache-mime4j-dom:0.7.2" level="project" />
+ <orderEntry type="library" name="Maven: org.apache.commons:commons-compress:1.4.1" level="project" />
+ <orderEntry type="library" name="Maven: org.tukaani:xz:1.0" level="project" />
+ <orderEntry type="library" name="Maven: commons-codec:commons-codec:1.5" level="project" />
+ <orderEntry type="library" name="Maven: org.apache.pdfbox:pdfbox:1.7.0" level="project" />
+ <orderEntry type="library" name="Maven: org.apache.pdfbox:fontbox:1.7.0" level="project" />
+ <orderEntry type="library" name="Maven: org.apache.pdfbox:jempbox:1.7.0" level="project" />
+ <orderEntry type="library" name="Maven: commons-logging:commons-logging:1.1.1" level="project" />
+ <orderEntry type="library" name="Maven: org.bouncycastle:bcmail-jdk15:1.45" level="project" />
+ <orderEntry type="library" name="Maven: org.bouncycastle:bcprov-jdk15:1.45" level="project" />
+ <orderEntry type="library" name="Maven: org.apache.poi:poi:3.8" level="project" />
+ <orderEntry type="library" name="Maven: org.apache.poi:poi-scratchpad:3.8" level="project" />
+ <orderEntry type="library" name="Maven: org.apache.poi:poi-ooxml:3.8" level="project" />
+ <orderEntry type="library" name="Maven: org.apache.poi:poi-ooxml-schemas:3.8" level="project" />
+ <orderEntry type="library" name="Maven: org.apache.xmlbeans:xmlbeans:2.3.0" level="project" />
+ <orderEntry type="library" name="Maven: dom4j:dom4j:1.6.1" level="project" />
+ <orderEntry type="library" name="Maven: org.apache.geronimo.specs:geronimo-stax-api_1.0_spec:1.0.1" level="project" />
+ <orderEntry type="library" name="Maven: org.ccil.cowan.tagsoup:tagsoup:1.2.1" level="project" />
+ <orderEntry type="library" name="Maven: asm:asm:3.1" level="project" />
+ <orderEntry type="library" name="Maven: com.googlecode.mp4parser:isoparser:1.0-RC-1" level="project" />
+ <orderEntry type="library" name="Maven: org.aspectj:aspectjrt:1.6.11" level="project" />
+ <orderEntry type="library" name="Maven: com.drewnoakes:metadata-extractor:2.4.0-beta-1" level="project" />
+ <orderEntry type="library" name="Maven: de.l3s.boilerpipe:boilerpipe:1.1.0" level="project" />
+ <orderEntry type="library" name="Maven: rome:rome:0.9" level="project" />
+ <orderEntry type="library" name="Maven: jdom:jdom:1.0" level="project" />
+ <orderEntry type="library" name="Maven: org.gagravarr:vorbis-java-core:0.1" level="project" />
+ <orderEntry type="library" name="Maven: com.googlecode.juniversalchardet:juniversalchardet:1.0.3" level="project" />
+ <orderEntry type="library" scope="RUNTIME" name="Maven: log4j:log4j:1.2.17" level="project" />
+ <orderEntry type="library" name="Maven: com.alibaba:fastjson:1.1.35" level="project" />
+ <orderEntry type="library" name="Maven: org.json:json:20090211" level="project" />
+ <orderEntry type="library" scope="TEST" name="Maven: org.apache.lucene:lucene-test-framework:4.2.1" level="project" />
+ <orderEntry type="library" scope="TEST" name="Maven: com.carrotsearch.randomizedtesting:randomizedtesting-runner:2.0.8" level="project" />
+ <orderEntry type="library" name="Maven: org.slf4j:slf4j-simple:1.7.5" level="project" />
+ <orderEntry type="library" name="Maven: commons-io:commons-io:2.4" level="project" />
+ <orderEntry type="library" name="Maven: com.googlecode.json-simple:json-simple:1.1" level="project" />
+ <orderEntry type="library" name="Maven: mysql:mysql-connector-java:5.1.9" level="project" />
+ </component>
+</module>
+
View
6 mapping/qq.xml
@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<config>
+ <mapping column="id" filed="id"/>
+ <mapping column="ABSTRACT" filed="name"/>
+ <mapping column="TI" filed="AU"/>
+</config>
View
20 mapping/transportation.xml
@@ -0,0 +1,20 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<config>
+ <mapping column="nid" filed="NID"/>
+ <mapping column="clicensetype" filed="CLICENSETYPE"/>
+ <mapping column="ccarnumber" filed="CCARNUMBER"/>
+ <mapping column="ccarnumber" filed="CCARNUMBER_NOTANALYZE"/>
+ <mapping column="dcollectiondate" filed="DCOLLECTIONDATE"/>
+ <mapping column="caddresscode" filed="CADDRESSCODE"/>
+ <mapping column="ccollectionaddress" filed="CCOLLECTIONADDRESS"/>
+ <mapping column="ccollectionagencies" filed="CCOLLECTIONAGENCIES"/>
+ <mapping column="cdatasource" filed="CDATASOURCE"/>
+ <mapping column="csnaptype" filed="CSNAPTYPE"/>
+ <mapping column="cdevicecode" filed="CDEVICECODE"/>
+ <mapping column="clanenumber" filed="CLANENUMBER"/>
+ <mapping column="nvehiclespeed" filed="NVEHICLESPEED"/>
+ <mapping column="nderictrion" filed="NDERICTRION"/>
+ <mapping column="carcolor" filed="CARCOLOR"/>
+ <mapping column="carbrand" filed="CARBRAND"/>
+ <mapping column="chiscarnumber" filed="CHISCARNUMBER"/>
+</config>
View
176 pom.xml
@@ -0,0 +1,176 @@
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+
+ <groupId>com.yao</groupId>
+ <artifactId>importIndex</artifactId>
+ <version>0.0.1-SNAPSHOT</version>
+ <packaging>jar</packaging>
+
+ <name>importIndex</name>
+ <url>http://maven.apache.org</url>
+
+ <properties>
+ <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+ <junit.version>4.11</junit.version>
+ <jdk.version>1.6</jdk.version>
+ <es.version>0.90.0</es.version>
+ <lucene.version>4.2.1</lucene.version>
+ <mysql.version>5.1.9</mysql.version>
+ </properties>
+
+ <dependencies>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <version>3.8.1</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.elasticsearch</groupId>
+ <artifactId>elasticsearch</artifactId>
+ <version>${es.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.elasticsearch</groupId>
+ <artifactId>elasticsearch-mapper-attachments</artifactId>
+ <version>1.8.0</version>
+ </dependency>
+ <dependency>
+ <groupId>com.alibaba</groupId>
+ <artifactId>fastjson</artifactId>
+ <version>1.1.35</version>
+ </dependency>
+ <dependency>
+ <groupId>org.json</groupId>
+ <artifactId>json</artifactId>
+ <version>20090211</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.lucene</groupId>
+ <artifactId>lucene-test-framework</artifactId>
+ <version>${lucene.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.lucene</groupId>
+ <artifactId>lucene-core</artifactId>
+ <version>${lucene.version}</version>
+ <scope>compile</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.lucene</groupId>
+ <artifactId>lucene-analyzers-common</artifactId>
+ <version>${lucene.version}</version>
+ <scope>compile</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.lucene</groupId>
+ <artifactId>lucene-codecs</artifactId>
+ <version>${lucene.version}</version>
+ <scope>compile</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.lucene</groupId>
+ <artifactId>lucene-queries</artifactId>
+ <version>${lucene.version}</version>
+ <scope>compile</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>jakarta-regexp</groupId>
+ <artifactId>jakarta-regexp</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.lucene</groupId>
+ <artifactId>lucene-memory</artifactId>
+ <version>${lucene.version}</version>
+ <scope>compile</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.lucene</groupId>
+ <artifactId>lucene-highlighter</artifactId>
+ <version>${lucene.version}</version>
+ <scope>compile</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.lucene</groupId>
+ <artifactId>lucene-queryparser</artifactId>
+ <version>${lucene.version}</version>
+ <scope>compile</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>jakarta-regexp</groupId>
+ <artifactId>jakarta-regexp</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.lucene</groupId>
+ <artifactId>lucene-suggest</artifactId>
+ <version>${lucene.version}</version>
+ <scope>compile</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.lucene</groupId>
+ <artifactId>lucene-join</artifactId>
+ <version>${lucene.version}</version>
+ <scope>compile</scope>
+ </dependency>
+ <!-- Lucene spatial, make sure when upgrading to work with latest version
+ of jts/spatial4j dependencies -->
+ <dependency>
+ <groupId>org.apache.lucene</groupId>
+ <artifactId>lucene-spatial</artifactId>
+ <version>${lucene.version}</version>
+ <scope>compile</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ <version>1.7.5</version>
+ </dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-simple</artifactId>
+ <version>1.7.5</version>
+ </dependency>
+
+ <dependency>
+ <groupId>commons-io</groupId>
+ <artifactId>commons-io</artifactId>
+ <version>2.4</version>
+ </dependency>
+
+ <dependency>
+ <groupId>com.googlecode.json-simple</groupId>
+ <artifactId>json-simple</artifactId>
+ <version>1.1</version>
+ </dependency>
+ <!-- mysql jdbc dependencies -->
+ <dependency>
+ <groupId>mysql</groupId>
+ <artifactId>mysql-connector-java</artifactId>
+ <version>5.1.9</version>
+ </dependency>
+ </dependencies>
+
+
+
+ <build>
+ <finalName>rmscloud</finalName>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-compiler-plugin</artifactId>
+ <version>3.0</version>
+ <configuration>
+ <source>${jdk.version}</source>
+ <target>${jdk.version}</target>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
+</project>
View
367 src/main/java/com/GenItemID.java
@@ -0,0 +1,367 @@
+package com; /**
+ * Created with IntelliJ IDEA.
+ * User: yanhui
+ * Date: 13-8-2
+ * Time: 下午5:30
+ * To change this template use File | Settings | File Templates.
+ */
+
+import java.io.*;
+import java.util.*;
+
+public class GenItemID {
+ public static Map<String, String> map = new HashMap<String, String>();
+ public static Set<Integer> lines = new HashSet<Integer>();
+
+ public static void main(String[] a) {
+
+ readFileByLines("/Users/yanhui/Downloads/sale-cn/201206.txt");
+ readFileByLines("/Users/yanhui/Downloads/sale-cn/201207.txt");
+ readFileByLines("/Users/yanhui/Downloads/sale-cn/201208.txt");
+ readFileByLines("/Users/yanhui/Downloads/sale-cn/201209.txt");
+ readFileByLines("/Users/yanhui/Downloads/sale-cn/201210.txt");
+// readFileByLines("/Users/yanhui/Downloads/sale-cn/201211.txt");
+ readFileByLines("/Users/yanhui/Downloads/sale-cn/201212.txt");
+
+ readFileByLines("/Users/yanhui/Downloads/sale-cn/201301.txt");
+ readFileByLines("/Users/yanhui/Downloads/sale-cn/201302.txt");
+ readFileByLines("/Users/yanhui/Downloads/sale-cn/201303.txt");
+ readFileByLines("/Users/yanhui/Downloads/sale-cn/201304.txt");
+ readFileByLines("/Users/yanhui/Downloads/sale-cn/201305.txt");
+ int num1 = 0;
+ genFpg();
+ //fpg();
+ Set<String> set = map.keySet();
+ generateItemFile("/Users/yanhui/Downloads/sale-cn/item.txt", null, list);
+ for (String s : set) {
+
+ //System.out.println(s + "-" + map.get(s).split(",").length + " " + list.get(num1++).size());
+
+ //System.out.println(map.get(s));
+ //System.out.println(list.get(num1 - 1));
+ }
+ generate("/Users/yanhui/Downloads/sale-cn/text.txt");
+ System.out.println(map.size());
+ for (Integer in : lines) {
+ //System.out.println(in + "===");
+ }
+
+ }
+
+ public static void fpg() {
+ for (Set<String> s : list) {
+ Object[] obj = null;
+ obj = s.toArray();
+ //System.out.println(obj);
+ for (int i = 0; i < obj.length; i++) {
+ for (int j = i + 1; j < obj.length; j++) {
+ String str = obj[i] + "," + obj[j];
+ if (fpg.get(str) != null) {
+ fpg.put(str, fpg.get(str) + 1);
+ } else {
+ fpg.put(str, 1);
+ }
+ }
+ }
+ for (String str : s) {
+ //System.out.print(s + ",");
+ }
+ //System.out.println();
+ }
+ Set<String> mSet = fpg.keySet();
+ for (String s : mSet) {
+ System.out.println(s + " " + fpg.get(s));
+ }
+ }
+
+ public static Map<String, Integer> fpg = new HashMap<String, Integer>();
+
+
+ public static HashMap<String, TreeSet> mapTree = new HashMap<String, TreeSet>();
+ public static List<Set> list = new ArrayList<Set>();
+
+ public static void genFpg() {
+ Set<String> set = map.keySet();
+ String value = null;
+ Integer number = 0;
+ String[] line = null;
+ TreeSet<String> treeSet = null;
+ int num = 0;
+ for (String s : set) {
+ treeSet = new TreeSet<String>();
+ line = map.get(s).split(",");
+ for (String str : line) {
+ treeSet.add(str);
+ }
+ if (num++ == 0) {
+
+ //System.out.println(s + "-" + map.get(s));
+
+ }
+ mapTree.put(s, treeSet);
+ list.add(treeSet);
+ }
+ Set<String> sets = mapTree.keySet();
+ int num1 = 0;
+ for (String s : sets) {
+ //System.out.println(s + "-" + mapTree.get(s).size()+" "+ list.get(num1++).size());
+ //System.out.println(mapTree.get(s));
+ //System.out.println(list.get(num1-1));
+ }
+
+ for (Set<String> s : list) {
+ System.out.println(s);
+ for (String str : s) {
+ //System.out.print(s+",");
+ }
+ System.out.println();
+ }
+ System.out.println(list.size());
+ }
+
+ public static String generate(String fileName) {
+ StringBuffer out = new StringBuffer();
+ BufferedWriter bw = null;
+ FileWriter fw = null;
+ File file = null;
+ PrintWriter pw = null;
+ OutputStreamWriter osw = null;
+
+ FileOutputStream fo = null;
+
+ try {
+// fo=new FileOutputStream("/Users/yanhui/test.txt");
+// osw=new OutputStreamWriter(fo,true)
+// pw = new PrintWriter(new OutputStreamWriter(),true);
+
+ file = new File(fileName);
+ fw = new FileWriter(file);
+ bw = new BufferedWriter(fw);
+
+ Set<String> set = map.keySet();
+ for (String s : set) {
+ //System.out.println(s + "-" + map.get(s));
+
+
+ bw.write(map.get(s));
+ bw.newLine();
+
+ }
+ bw.flush();
+
+ } catch (IOException e) {
+ e.printStackTrace(); //To change body of catch statement use File | Settings | File Templates.
+ } finally {
+ if (bw != null) {
+ try {
+ bw.close();
+ } catch (IOException e) {
+ e.printStackTrace(); //To change body of catch statement use File | Settings | File Templates.
+ }
+ }
+ if (fw != null) {
+ try {
+ fw.close();
+ } catch (IOException e) {
+ e.printStackTrace(); //To change body of catch statement use File | Settings | File Templates.
+ }
+ }
+
+ }
+ return out.toString();
+ }
+
+ public static Set<String> itemIds = new HashSet<String>();
+ public static int itemNumber = 8;
+ public static Set<String> userIds = new HashSet<String>();
+ public static int userNumber = 5;
+
+ /**
+ * 以行为单位读取文件,常用于读面向行的格式化文件
+ */
+ public static void readFileByLines(String fileName) {
+ File file = new File(fileName);
+ BufferedReader reader = null;
+
+ FileInputStream fi = null;
+ InputStreamReader isr = null;
+
+ BufferedReader br = null;
+
+ try {
+ System.out.println("以行为单位读取文件内容,一次读一整行:" + fileName);
+ fi = new FileInputStream(fileName);
+ isr = new InputStreamReader(fi, code);
+ reader = new BufferedReader(isr);
+ String tempString = null;
+ int line = 1;
+ String str[] = null;
+ String uid = "";
+ String iid = "";
+ // 一次读入一行,直到读入null为文件结束
+ while ((tempString = reader.readLine()) != null) {
+ // 显示行号
+ //System.out.println(tempString);
+ if (line > 1) {
+ tempString = tempString.replaceAll("\"", "");
+ //System.out.println(tempString);
+ str = tempString.split("\t");
+ if (str.length != 10) {
+
+ continue;
+ }
+ uid = str[userNumber];
+ iid = str[itemNumber];
+ uid = uid.replaceAll("'", "");
+ uid = uid.replaceAll(" ", "");
+ iid = iid.replaceAll("'", "");
+ iid = iid.replaceAll(" ", "");
+
+ itemIds.add(iid);
+ userIds.add(uid);
+ if (map.get(uid) != null) {
+ //System.out.println(iid);
+ if (map.get(uid).indexOf(iid + ",") >= 0 || map.get(uid).indexOf("," + iid) >= 0 || map.get(uid).equals(iid)) {
+
+ } else {
+ map.put(uid, map.get(uid) + "," + iid);
+ }
+ } else {
+ map.put(uid, iid);
+ }
+ }
+ line++;
+ }
+ lines.add(line);
+ reader.close();
+ } catch (IOException e) {
+ e.printStackTrace();
+ } finally {
+ if (reader != null) {
+ try {
+ reader.close();
+ } catch (IOException e1) {
+ }
+ }
+ }
+ }
+
+ public static String code = "Unicode";
+
+ public static boolean generateItemFile(String fileName, Set<String> set, List<Set> listTree) {
+ boolean ret = false;
+ BufferedWriter bw = null;
+ FileWriter fw = null;
+ File file = null;
+ PrintWriter pw = null;
+ OutputStreamWriter osw = null;
+
+ FileOutputStream fo = null;
+
+ try {
+
+ file = new File(fileName);
+ fw = new FileWriter(file);
+ fo = new FileOutputStream(file);
+ osw = new OutputStreamWriter(fo, code);
+ bw = new BufferedWriter(osw);
+ //fi=new FileInputStream(fileName);
+ //isr=new InputStreamReader(fi,"UTF-8");
+ //reader = new BufferedReader(isr);
+ int number = 1;
+
+ String str=null;
+ if (listTree != null) {
+ for (Set s : listTree) {
+ str=s.toString().substring(1,s.toString().length()-1);
+ str=str.replaceAll(", "," ");
+
+ bw.write(str);
+ bw.newLine();
+ }
+ }
+ if (set != null) {
+
+
+ for (String s : set) {
+ //System.out.println(s + "-" + map.get(s));
+ bw.write(number++ + ",");
+ bw.write(s);
+ bw.newLine();
+
+ }
+ }
+ bw.flush();
+ ret = true;
+ } catch (IOException e) {
+ e.printStackTrace(); //To change body of catch statement use File | Settings | File Templates.
+ } finally {
+ if (bw != null) {
+ try {
+ bw.close();
+ } catch (IOException e) {
+ e.printStackTrace(); //To change body of catch statement use File | Settings | File Templates.
+ }
+ }
+ if (fw != null) {
+ try {
+ fw.close();
+ } catch (IOException e) {
+ e.printStackTrace(); //To change body of catch statement use File | Settings | File Templates.
+ }
+ }
+
+ }
+ return ret;
+ }
+
+ public static boolean generateUserFile(String fileName) {
+ boolean ret = false;
+ BufferedWriter bw = null;
+ FileWriter fw = null;
+ File file = null;
+ PrintWriter pw = null;
+ OutputStreamWriter osw = null;
+
+ FileOutputStream fo = null;
+
+ try {
+
+ file = new File(fileName);
+ fw = new FileWriter(file);
+ bw = new BufferedWriter(fw);
+
+ int number = 1;
+ for (String s : itemIds) {
+ //System.out.println(s + "-" + map.get(s));
+ bw.write(number++);
+ bw.write(",");
+ bw.write(s);
+ bw.newLine();
+
+ }
+ bw.flush();
+ ret = true;
+ } catch (IOException e) {
+ e.printStackTrace(); //To change body of catch statement use File | Settings | File Templates.
+ } finally {
+ if (bw != null) {
+ try {
+ bw.close();
+ } catch (IOException e) {
+ e.printStackTrace(); //To change body of catch statement use File | Settings | File Templates.
+ }
+ }
+ if (fw != null) {
+ try {
+ fw.close();
+ } catch (IOException e) {
+ e.printStackTrace(); //To change body of catch statement use File | Settings | File Templates.
+ }
+ }
+
+ }
+ return ret;
+ }
+
+}
View
150 src/main/java/com/ImportCSV.java
@@ -0,0 +1,150 @@
+package com;
+
+import com.yao.dao.impl.ImportIndex;
+import com.yao.entity.AllConfig;
+import com.yao.io.LoadConfig;
+
+import java.io.*;
+
+/**
+ * Generte Data!
+ */
+public class ImportCSV extends Thread {
+
+
+ public static void main(String[] args) {
+
+ if (args.length != 3) {
+ System.out.println("Plase Input : <filePath> <fileCount> <fileLine>");
+ System.exit(0);
+ }
+ String filePath = args[0];
+ if (filePath.length() <= 0) {
+ System.out.println("Plase Input : <filePath> <fileCount> <fileLine>");
+ System.exit(0);
+ }
+ int fileCount = Integer.parseInt(args[1]);
+ if (fileCount <= 0) {
+ System.out.println("Plase Input : <filePath> <fileCount> <fileLine>");
+ System.exit(0);
+ }
+ double fileLine = Double.parseDouble(args[2]);
+ if (fileLine <= 0) {
+ System.out.println("Plase Input : <filePath> <fileCount> <fileLine>");
+ System.exit(0);
+ }
+ System.out.println(filePath + fileCount + fileLine);
+ long startTime = System.currentTimeMillis();
+ ImportCSV thread = null;
+ for (int i = 1; i <= fileCount; i++) {
+ thread = new ImportCSV();
+ if (filePath.endsWith("/")) {
+ thread.generteFileName = filePath + "file" + i + ".txt";
+ } else {
+ thread.generteFileName = filePath + File.separator + "file" + i + ".txt";
+ }
+
+ //thread.generteFileName = "/Users/yanhui/file"+i+"";
+ thread.generteNumber = fileLine;
+ thread.start();
+ }
+
+
+ System.out.println(" done " + (System.currentTimeMillis() - startTime) / 1000 + "");
+ //new GenerteData().generate(3,"/Users/yanhui/test.txt");
+ }
+
+ private double generteNumber;
+ private String generteFileName;
+
+ public void GenerteData(double number, String fileName) {
+ this.generteNumber = number;
+ this.generteFileName = fileName;
+ }
+
+ @Override
+ public void run() {
+ super.run(); //To change body of overridden methods use File | Settings | File Templates.
+ //this.generate(generteNumber, generteFileName);
+ readFile(generteFileName);
+ System.out.println(generteFileName);
+
+ }
+
+ public static String code = "Unicode";
+
+
+ public String readFile(String fileName) {
+ File file = new File(fileName);
+ if (!file.exists()){
+ System.exit(0);
+ }
+
+
+ AllConfig config=null;
+ System.out.println("sss");
+ try {
+ config= LoadConfig.load();
+ } catch (Exception e) {
+ e.printStackTrace();
+ System.out.println("读取配置文件错误");
+ }
+
+ ImportIndex importIndex = new ImportIndex();
+ importIndex.initClusterClient(config);
+
+ System.out.println("sss");
+
+ BufferedReader reader = null;
+
+ FileInputStream fi = null;
+ InputStreamReader isr = null;
+
+ BufferedReader br = null;
+
+ try {
+ System.out.println("以行为单位读取文件内容,一次读一整行:" + fileName);
+ fi = new FileInputStream(fileName);
+ isr = new InputStreamReader(fi);
+ reader = new BufferedReader(isr);
+ String tempString = null;
+ int line = 1;
+ String str[] = null;
+ String uid = "";
+ String iid = "";
+ // 一次读入一行,直到读入null为文件结束
+ while ((tempString = reader.readLine()) != null) {
+
+
+ //tempString = tempString.replaceAll("\"", "");
+ System.out.println(tempString);
+ str = tempString.split(",");
+ System.out.println(str.length);
+ try {
+ importIndex.importIndex(config, str);
+ } catch (Exception e) {
+ e.printStackTrace(); //To change body of catch statement use File | Settings | File Templates.
+ }
+
+ }
+
+ System.out.println("以行为单位读取文件内容,一次读一整行:" + fileName);
+ reader.close();
+ importIndex.close();
+ } catch (IOException e) {
+ e.printStackTrace();
+ } finally {
+ if (reader != null) {
+ try {
+ reader.close();
+ } catch (IOException e1) {
+ }
+ }
+ }
+ return "";
+ }
+
+
+
+
+}
View
25 src/main/java/com/yao/core/Import.java
@@ -0,0 +1,25 @@
+package com.yao.core;
+
+
+
+import com.yao.dao.impl.BaseDaoImpl;
+import com.yao.entity.AllConfig;
+import com.yao.io.LoadConfig;
+
+public class Import {
+ public static void main(String[] args) {
+ System.out.println("we are running...");
+ AllConfig config=null;
+ try {
+ config=LoadConfig.load();
+ } catch (Exception e) {
+ e.printStackTrace();
+ System.out.println("读取配置文件错误");
+ }
+
+ BaseDaoImpl baseDao=new BaseDaoImpl();
+ baseDao.index(config);
+
+
+ }
+}
View
159 src/main/java/com/yao/dao/impl/BaseDaoImpl.java
@@ -0,0 +1,159 @@
+package com.yao.dao.impl;
+
+import com.yao.entity.AllConfig;
+
+import java.io.IOException;
+import java.sql.*;
+
+
+public class BaseDaoImpl {
+
+
+ public boolean index(AllConfig indexObj) {
+
+
+ boolean flag = true;
+ //getConn(indexObj);
+ getMinAndMax_f_id(indexObj);
+
+ System.out.println("min batch-id\t-->" + minFlag);
+
+ System.out.println("max batch-id\t-->" + maxFlg);
+
+
+ ResultSet rs;
+
+ long from = 0;
+ long to = 0;
+
+ from = minFlag;
+
+
+ importIndex = new ImportIndex();
+ importIndex.initClusterClient(indexObj);
+ while (to < maxFlg) {
+ to = from + Long.valueOf(indexObj.getBatchSize());
+ System.out.println("from\t" + from + "\tto\t" + to);
+ rs = getRs(indexObj, from, to);
+
+ try {
+ importIndex.importIndex(indexObj, rs);
+ } catch (SQLException e) {
+ flag = false;
+ e.printStackTrace();
+ } catch (IOException e) {
+ flag = false;
+ // TODO Auto-generated catch block
+ e.printStackTrace();
+ }
+ ;
+ from = to + 1;
+ }
+
+ close();
+ importIndex.close();
+
+ return flag;
+ }
+
+
+ private void getMinAndMax_f_id(AllConfig indexObj) {
+ getConn(indexObj);
+ Statement stmt;
+ try {
+ stmt = conn.createStatement();
+ String sql = "SELECT MIN(" + indexObj.getBatch_id() + ") FROM "
+ + indexObj.getDb_table();
+
+
+ rs = stmt.executeQuery(sql);
+ rs.next();
+ String s = rs.getString(1);
+
+ minFlag = Long.valueOf(s);
+
+
+ sql = "select max(" + indexObj.getBatch_id() + ") FROM " + indexObj.getDb_table();
+
+
+ rs = stmt.executeQuery(sql);
+ rs.next();
+ s = rs.getString(1);
+ maxFlg = Long.valueOf(s);
+
+ } catch (SQLException e) {
+ e.printStackTrace();
+ }
+ close();
+ }
+
+
+ private ResultSet getRs(AllConfig indexObj, long from, long to) {
+ getConn(indexObj);
+
+ String selectSql = "select * from " + indexObj.getDb_table() + " where " + indexObj.getBatch_id() + " between ? and ? order by " + indexObj.getBatch_id();
+ try {
+ pstmt = conn.prepareStatement(selectSql, ResultSet.TYPE_SCROLL_INSENSITIVE, ResultSet.CONCUR_READ_ONLY);
+ pstmt.setLong(1, from);
+ pstmt.setLong(2, to);
+ rs = pstmt.executeQuery();
+ } catch (SQLException e) {
+ e.printStackTrace();
+ }
+ return rs;
+ }
+
+
+ public void getConn(AllConfig obj) {
+ String url = obj.getDb_url() + "/" + obj.getDb_name();
+
+ try {
+ Class.forName(JDBCCLASS);
+ } catch (ClassNotFoundException e) {
+ e.printStackTrace();
+ }
+ try {
+ conn = DriverManager.getConnection(url, obj.getDb_username(),
+ obj.getDb_password());
+ } catch (SQLException e) {
+ System.out.println("init database connection fail");
+ e.printStackTrace();
+ }
+
+ }
+
+
+ public void close() {
+ try {
+ if (rs != null && !rs.isClosed()) {
+ rs.close();
+ }
+ } catch (SQLException e) {
+ e.printStackTrace();
+ }
+ try {
+ if (pstmt != null && !pstmt.isClosed()) {
+ pstmt.close();
+ }
+ } catch (SQLException e) {
+ e.printStackTrace();
+ }
+ try {
+ if (conn != null && !conn.isClosed()) {
+ conn.close();
+ }
+ } catch (SQLException e) {
+ // TODO Auto-generated catch block
+ e.printStackTrace();
+ }
+ }
+
+
+ private Connection conn;
+ private PreparedStatement pstmt;
+ private ResultSet rs;
+ private long minFlag;
+ private long maxFlg;
+ private ImportIndex importIndex;
+ private static final String JDBCCLASS = "com.mysql.jdbc.Driver";
+}
View
134 src/main/java/com/yao/dao/impl/ImportIndex.java
@@ -0,0 +1,134 @@
+package com.yao.dao.impl;
+
+import com.yao.entity.AllConfig;
+import org.elasticsearch.action.bulk.BulkRequestBuilder;
+import org.elasticsearch.action.bulk.BulkResponse;
+import org.elasticsearch.client.transport.TransportClient;
+import org.elasticsearch.common.settings.ImmutableSettings;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.transport.InetSocketTransportAddress;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentFactory;
+
+import java.io.IOException;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Set;
+
+
+public class ImportIndex {
+
+ public void importIndex(AllConfig indexObj, String[] rs)
+ throws Exception {
+
+ bulkRequest = client.prepareBulk();
+
+
+ bulkRequest.add(client.prepareIndex(indexObj.getIndex_name(),
+ indexObj.getIndex_type()
+ // rs.getString(indexConfig.getTableId())// this is for es index id
+ // ,remove
+ , rs[0]
+ ).setSource(getJsonObject(indexObj, rs)));
+
+
+ bulkResponse = bulkRequest.execute().actionGet();
+
+ if (bulkResponse != null && bulkResponse.hasFailures()) {
+ String s = bulkResponse.buildFailureMessage();
+ System.out.println(s);
+ }
+
+ System.gc();
+ }
+
+ public void importIndex(AllConfig indexObj, ResultSet rs)
+ throws SQLException, IOException {
+
+ bulkRequest = client.prepareBulk();
+
+ while (rs.next()) {
+
+ bulkRequest.add(client.prepareIndex(indexObj.getIndex_name(),
+ indexObj.getIndex_type()
+ // rs.getString(indexConfig.getTableId())// this is for es index id
+ // ,remove
+ , rs.getString(indexObj.getIdentify())
+ ).setSource(getJsonObject(indexObj, rs)));
+ }
+
+ bulkResponse = bulkRequest.execute().actionGet();
+
+ if (bulkResponse != null && bulkResponse.hasFailures()) {
+ String s = bulkResponse.buildFailureMessage();
+ System.out.println(s);
+ }
+
+ System.gc();
+ }
+
+ public void initClusterClient(AllConfig cluster) {
+ settings = ImmutableSettings.settingsBuilder()
+ .put("cluster.name", cluster.getCluster_name()).build();
+ client = new TransportClient(settings);
+ ista = new InetSocketTransportAddress(cluster.getCluster_host(),
+ Integer.valueOf(cluster.getCluster_port()));
+ client = client.addTransportAddress(ista);
+ }
+
+ public void close() {
+ if (client != null) {
+ client.close();
+ }
+ }
+
+ private XContentBuilder getJsonObject(AllConfig indexObj, ResultSet results)
+ throws IOException, SQLException {
+ final XContentBuilder jsonBuilder = XContentFactory.jsonBuilder();
+
+ jsonBuilder.startObject();
+
+ Map<String, String> map = indexObj.getMap();
+ Set<String> keySet = map.keySet();
+ Iterator<String> it = keySet.iterator();
+ String temp = null;
+ while (it.hasNext()) {
+ temp = it.next();
+ jsonBuilder.field(map.get(temp), results.getString(temp));
+
+ }
+
+ jsonBuilder.endObject();
+ return jsonBuilder;
+ }
+
+ private XContentBuilder getJsonObject(AllConfig indexObj, String[] results)
+ throws IOException, SQLException {
+ final XContentBuilder jsonBuilder = XContentFactory.jsonBuilder();
+
+ jsonBuilder.startObject();
+
+ Map<String, String> map = indexObj.getMap();
+ Set<String> keySet = map.keySet();
+ Iterator<String> it = keySet.iterator();
+ String temp = null;
+ int number = 0;
+ while (it.hasNext()) {
+ temp = it.next();
+ jsonBuilder.field(map.get(temp), results[number++]);
+
+ }
+
+ jsonBuilder.endObject();
+ return jsonBuilder;
+ }
+
+ private Settings settings;
+ private TransportClient client;
+ private InetSocketTransportAddress ista;
+ private BulkRequestBuilder bulkRequest = null;
+ private BulkResponse bulkResponse = null;
+
+}
View
179 src/main/java/com/yao/entity/AllConfig.java
@@ -0,0 +1,179 @@
+package com.yao.entity;
+
+import java.util.Map;
+
+public class AllConfig {
+ private String db_url;
+ private String db_username;
+ private String db_password;
+ private String db_name;
+ private String db_table;
+ private String cluster_name;
+ private String cluster_host;
+ private String cluster_port;
+ private String batch_id;
+ private String identify;
+ private String index_name;
+ private String index_type;
+ private String batchSize;
+ public AllConfig(String db_url, String db_username, String db_password,
+ String db_name, String db_table, String cluster_name,
+ String cluster_host, String cluster_port, String batch_id,
+ String identify, String index_name, String index_type,
+ String batchSize, Map<String, String> map) {
+ super();
+ this.db_url = db_url;
+ this.db_username = db_username;
+ this.db_password = db_password;
+ this.db_name = db_name;
+ this.db_table = db_table;
+ this.cluster_name = cluster_name;
+ this.cluster_host = cluster_host;
+ this.cluster_port = cluster_port;
+ this.batch_id = batch_id;
+ this.identify = identify;
+ this.index_name = index_name;
+ this.index_type = index_type;
+ this.batchSize = batchSize;
+ this.map = map;
+ }
+
+ public String getBatchSize() {
+ return batchSize;
+ }
+
+ public void setBatchSize(String batchSize) {
+ this.batchSize = batchSize;
+ }
+ private Map<String, String> map;
+
+
+ public Map<String, String> getMap() {
+ return map;
+ }
+
+ public void setMap(Map<String, String> map) {
+ this.map = map;
+ }
+
+
+
+
+ public String getIndex_name() {
+ return index_name;
+ }
+
+ public void setIndex_name(String index_name) {
+ this.index_name = index_name;
+ }
+
+ public String getIndex_type() {
+ return index_type;
+ }
+
+ @Override
+ public String toString() {
+ return "AllConfig [db_url=" + db_url + ", db_username=" + db_username
+ + ", db_password=" + db_password + ", db_name=" + db_name
+ + ", db_table=" + db_table + ", cluster_name=" + cluster_name
+ + ", cluster_host=" + cluster_host + ", cluster_port="
+ + cluster_port + ", batch_id=" + batch_id + ", identify="
+ + identify + ", index_name=" + index_name + ", index_type="
+ + index_type + ", batchSize=" + batchSize + ", map=" + map
+ + "]";
+ }
+
+ public void setIndex_type(String index_type) {
+ this.index_type = index_type;
+ }
+
+ public AllConfig(String db_url, String db_username, String db_password,
+ String db_name, String db_table, String cluster_name,
+ String cluster_host, String cluster_port, String batch_id,
+ String identify, String index_name, String index_type,
+ Map<String, String> map) {
+ super();
+ this.db_url = db_url;
+ this.db_username = db_username;
+ this.db_password = db_password;
+ this.db_name = db_name;
+ this.db_table = db_table;
+ this.cluster_name = cluster_name;
+ this.cluster_host = cluster_host;
+ this.cluster_port = cluster_port;
+ this.batch_id = batch_id;
+ this.identify = identify;
+ this.index_name = index_name;
+ this.index_type = index_type;
+ this.map = map;
+ }
+
+ public AllConfig() {
+ super();
+ // TODO Auto-generated constructor stub
+ }
+
+ public String getDb_url() {
+ return db_url;
+ }
+ public void setDb_url(String db_url) {
+ this.db_url = db_url;
+ }
+ public String getDb_username() {
+ return db_username;
+ }
+ public void setDb_username(String db_username) {
+ this.db_username = db_username;
+ }
+ public String getDb_password() {
+ return db_password;
+ }
+ public void setDb_password(String db_password) {
+ this.db_password = db_password;
+ }
+ public String getDb_name() {
+ return db_name;
+ }
+ public void setDb_name(String db_name) {
+ this.db_name = db_name;
+ }
+ public String getDb_table() {
+ return db_table;
+ }
+ public void setDb_table(String db_table) {
+ this.db_table = db_table;
+ }
+ public String getCluster_name() {
+ return cluster_name;
+ }
+ public void setCluster_name(String cluster_name) {
+ this.cluster_name = cluster_name;
+ }
+ public String getCluster_host() {
+ return cluster_host;
+ }
+ public void setCluster_host(String cluster_host) {
+ this.cluster_host = cluster_host;
+ }
+ public String getCluster_port() {
+ return cluster_port;
+ }
+ public void setCluster_port(String cluster_port) {
+ this.cluster_port = cluster_port;
+ }
+ public String getBatch_id() {
+ return batch_id;
+ }
+ public void setBatch_id(String batch_id) {
+ this.batch_id = batch_id;
+ }
+ public String getIdentify() {
+ return identify;
+ }
+ public void setIdentify(String identify) {
+ this.identify = identify;
+ }
+
+
+
+}
View
92 src/main/java/com/yao/io/LoadConfig.java
@@ -0,0 +1,92 @@
+package com.yao.io;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+import javax.xml.parsers.DocumentBuilder;
+import javax.xml.parsers.DocumentBuilderFactory;
+import javax.xml.parsers.ParserConfigurationException;
+import org.w3c.dom.Document;
+import org.w3c.dom.NamedNodeMap;
+import org.w3c.dom.NodeList;
+import org.xml.sax.SAXException;
+import com.yao.entity.AllConfig;
+
+
+public class LoadConfig {
+
+ public static AllConfig load() throws ParserConfigurationException, SAXException, IOException {
+ DocumentBuilderFactory dbf=DocumentBuilderFactory.newInstance();
+ DocumentBuilder db=null;
+ db=dbf.newDocumentBuilder();
+ doc=db.parse(CONFIGFILE);
+
+ String db_url=getNodeValueByTagName("database-url");
+ String db_username=getNodeValueByTagName("database-username");
+ String db_password=getNodeValueByTagName("database-password");
+ String db_name=getNodeValueByTagName("database-name");
+ String db_table=getNodeValueByTagName("database-table");
+ String cluster_name=getNodeValueByTagName("cluster-name");
+ String cluster_host=getNodeValueByTagName("cluster-host");
+ String cluster_port=getNodeValueByTagName("cluster-port");
+ String batch_id=getNodeValueByTagName("batchId");
+ String identify=getNodeValueByTagName("identify");
+ String index_name=getNodeValueByTagName("index-name");
+ String index_type=getNodeValueByTagName("index-type");
+ String batchSize=getNodeValueByTagName("batch-size");
+
+
+
+ Map<String, String> map=null;
+ try {
+ map = getMap(index_name);
+ } catch (Exception e) {
+ e.printStackTrace();
+ System.out.println("读取 map 错误");
+ }
+ AllConfig config=new AllConfig(db_url, db_username, db_password, db_name, db_table, cluster_name, cluster_host, cluster_port, batch_id, identify, index_name, index_type, batchSize, map);
+ return config;
+ };
+
+
+
+
+ private static Map<String, String> getMap(String indexName) throws Exception {
+ DocumentBuilderFactory dbf=DocumentBuilderFactory.newInstance();
+ DocumentBuilder db=null;
+
+ db=dbf.newDocumentBuilder();
+ doc=db.parse(new File(MAPPINGDIR,indexName+".xml"));
+
+ NodeList nodeList=doc.getElementsByTagName("mapping");
+ Map<String, String> map=new HashMap<String, String>();
+
+ NamedNodeMap namedNodeMap=null;
+ for (int i = 0; i < nodeList.getLength(); i++) {
+ namedNodeMap=nodeList.item(i).getAttributes();
+ map.put(namedNodeMap.item(0).getNodeValue(), namedNodeMap.item(1).getNodeValue());
+ }
+
+
+ return map;
+ }
+
+
+
+
+
+ private static String getNodeValueByTagName(String tagName){
+ return doc.getElementsByTagName(tagName).item(0).getTextContent();
+ }
+
+ private static final String MAPPINGDIR="mapping";
+ private static final String CONFIGFILE="importConfig.xml";
+ private static Document doc;
+
+
+
+
+
+
+}
View
38 src/test/java/com/yao/importIndex/AppTest.java
@@ -0,0 +1,38 @@
+package com.yao.importIndex;
+
+import junit.framework.Test;
+import junit.framework.TestCase;
+import junit.framework.TestSuite;
+
+/**
+ * Unit test for simple App.
+ */
+public class AppTest
+ extends TestCase
+{
+ /**
+ * Create the test case
+ *
+ * @param testName name of the test case
+ */
+ public AppTest( String testName )
+ {
+ super( testName );
+ }
+
+ /**
+ * @return the suite of tests being tested
+ */
+ public static Test suite()
+ {
+ return new TestSuite( AppTest.class );
+ }
+
+ /**
+ * Rigourous Test :-)
+ */
+ public void testApp()
+ {
+ assertTrue( true );
+ }
+}
View
5 target/classes/META-INF/MANIFEST.MF
@@ -0,0 +1,5 @@
+Manifest-Version: 1.0
+Built-By: root
+Build-Jdk: 1.7.0_25
+Created-By: Maven Integration for Eclipse
+
View
7 target/classes/META-INF/maven/com.yao/importIndex/pom.properties
@@ -0,0 +1,7 @@
+#Generated by Maven Integration for Eclipse
+#Sat Sep 07 11:34:49 EDT 2013
+version=0.0.1-SNAPSHOT
+groupId=com.yao
+m2e.projectName=importIndex
+m2e.projectLocation=/root/workspace/importIndex
+artifactId=importIndex
View
176 target/classes/META-INF/maven/com.yao/importIndex/pom.xml
@@ -0,0 +1,176 @@
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+
+ <groupId>com.yao</groupId>
+ <artifactId>importIndex</artifactId>
+ <version>0.0.1-SNAPSHOT</version>
+ <packaging>jar</packaging>
+
+ <name>importIndex</name>
+ <url>http://maven.apache.org</url>
+
+ <properties>
+ <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+ <junit.version>4.11</junit.version>
+ <jdk.version>1.6</jdk.version>
+ <es.version>0.90.0</es.version>
+ <lucene.version>4.2.1</lucene.version>
+ <mysql.version>5.1.9</mysql.version>
+ </properties>
+
+ <dependencies>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <version>3.8.1</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.elasticsearch</groupId>
+ <artifactId>elasticsearch</artifactId>
+ <version>${es.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.elasticsearch</groupId>
+ <artifactId>elasticsearch-mapper-attachments</artifactId>
+ <version>1.8.0</version>
+ </dependency>
+ <dependency>
+ <groupId>com.alibaba</groupId>
+ <artifactId>fastjson</artifactId>
+ <version>1.1.35</version>
+ </dependency>
+ <dependency>
+ <groupId>org.json</groupId>
+ <artifactId>json</artifactId>
+ <version>20090211</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.lucene</groupId>
+ <artifactId>lucene-test-framework</artifactId>
+ <version>${lucene.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.lucene</groupId>
+ <artifactId>lucene-core</artifactId>
+ <version>${lucene.version}</version>
+ <scope>compile</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.lucene</groupId>
+ <artifactId>lucene-analyzers-common</artifactId>
+ <version>${lucene.version}</version>
+ <scope>compile</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.lucene</groupId>
+ <artifactId>lucene-codecs</artifactId>
+ <version>${lucene.version}</version>
+ <scope>compile</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.lucene</groupId>
+ <artifactId>lucene-queries</artifactId>
+ <version>${lucene.version}</version>
+ <scope>compile</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>jakarta-regexp</groupId>
+ <artifactId>jakarta-regexp</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.lucene</groupId>
+ <artifactId>lucene-memory</artifactId>
+ <version>${lucene.version}</version>
+ <scope>compile</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.lucene</groupId>
+ <artifactId>lucene-highlighter</artifactId>
+ <version>${lucene.version}</version>
+ <scope>compile</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.lucene</groupId>
+ <artifactId>lucene-queryparser</artifactId>
+ <version>${lucene.version}</version>
+ <scope>compile</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>jakarta-regexp</groupId>
+ <artifactId>jakarta-regexp</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.lucene</groupId>
+ <artifactId>lucene-suggest</artifactId>
+ <version>${lucene.version}</version>
+ <scope>compile</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.lucene</groupId>
+ <artifactId>lucene-join</artifactId>
+ <version>${lucene.version}</version>
+ <scope>compile</scope>
+ </dependency>
+ <!-- Lucene spatial, make sure when upgrading to work with latest version
+ of jts/spatial4j dependencies -->
+ <dependency>
+ <groupId>org.apache.lucene</groupId>
+ <artifactId>lucene-spatial</artifactId>
+ <version>${lucene.version}</version>
+ <scope>compile</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ <version>1.7.5</version>
+ </dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-simple</artifactId>
+ <version>1.7.5</version>
+ </dependency>
+
+ <dependency>
+ <groupId>commons-io</groupId>
+ <artifactId>commons-io</artifactId>
+ <version>2.4</version>
+ </dependency>
+
+ <dependency>
+ <groupId>com.googlecode.json-simple</groupId>
+ <artifactId>json-simple</artifactId>
+ <version>1.1</version>
+ </dependency>
+ <!-- mysql jdbc dependencies -->
+ <dependency>
+ <groupId>mysql</groupId>
+ <artifactId>mysql-connector-java</artifactId>
+ <version>5.1.9</version>
+ </dependency>
+ </dependencies>
+
+
+
+ <build>
+ <finalName>rmscloud</finalName>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-compiler-plugin</artifactId>
+ <version>3.0</version>
+ <configuration>
+ <source>${jdk.version}</source>
+ <target>${jdk.version}</target>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
+</project>
View
5 target/maven-archiver/pom.properties
@@ -0,0 +1,5 @@
+#Generated by Maven
+#Wed Sep 18 17:12:19 CST 2013
+version=0.0.1-SNAPSHOT
+groupId=com.yao
+artifactId=importIndex
View
0  target/maven-status/maven-compiler-plugin/compile/default-compile/createdFiles.lst
No changes.
View
7 target/maven-status/maven-compiler-plugin/compile/default-compile/inputFiles.lst
@@ -0,0 +1,7 @@
+/Users/owen2785/Coding/importIndex/src/main/java/com/yao/dao/impl/ImportIndex.java
+/Users/owen2785/Coding/importIndex/src/main/java/com/yao/entity/AllConfig.java
+/Users/owen2785/Coding/importIndex/src/main/java/com/yao/dao/impl/BaseDaoImpl.java
+/Users/owen2785/Coding/importIndex/src/main/java/com/yao/io/LoadConfig.java
+/Users/owen2785/Coding/importIndex/src/main/java/com/GenItemID.java
+/Users/owen2785/Coding/importIndex/src/main/java/com/yao/core/Import.java
+/Users/owen2785/Coding/importIndex/src/main/java/com/ImportCSV.java
View
0  target/maven-status/maven-compiler-plugin/testCompile/default-testCompile/createdFiles.lst
No changes.
View
72 target/surefire-reports/TEST-com.yao.importIndex.AppTest.xml
@@ -0,0 +1,72 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<testsuite failures="0" time="0.014" errors="0" skipped="0" tests="1" name="com.yao.importIndex.AppTest">
+ <properties>
+ <property name="java.runtime.name" value="Java(TM) SE Runtime Environment"/>
+ <property name="sun.boot.library.path" value="/System/Library/Java/JavaVirtualMachines/1.6.0.jdk/Contents/Libraries"/>
+ <property name="java.vm.version" value="20.51-b01-456"/>
+ <property name="awt.nativeDoubleBuffering" value="true"/>
+ <property name="gopherProxySet" value="false"/>
+ <property name="mrj.build" value="11M4508"/>
+ <property name="java.vm.vendor" value="Apple Inc."/>
+ <property name="java.vendor.url" value="http://www.apple.com/"/>
+ <property name="path.separator" value=":"/>
+ <property name="guice.disable.misplaced.annotation.check" value="true"/>
+ <property name="java.vm.name" value="Java HotSpot(TM) 64-Bit Server VM"/>
+ <property name="file.encoding.pkg" value="sun.io"/>
+ <property name="user.country" value="CN"/>
+ <property name="sun.java.launcher" value="SUN_STANDARD"/>
+ <property name="sun.os.patch.level" value="unknown"/>
+ <property name="java.vm.specification.name" value="Java Virtual Machine Specification"/>
+ <property name="user.dir" value="/Users/owen2785/Coding/importIndex"/>
+ <property name="java.runtime.version" value="1.6.0_51-b11-456-11M4508"/>
+ <property name="java.awt.graphicsenv" value="apple.awt.CGraphicsEnvironment"/>
+ <property name="java.endorsed.dirs" value="/System/Library/Java/JavaVirtualMachines/1.6.0.jdk/Contents/Home/lib/endorsed"/>
+ <property name="os.arch" value="x86_64"/>
+ <property name="java.io.tmpdir" value="/var/folders/_c/2jxhxrq500q_q3kmm9k2vfmc0000gn/T/"/>
+ <property name="line.separator" value="
+"/>
+ <property name="java.vm.specification.vendor" value="Sun Microsystems Inc."/>
+ <property name="https.proxyHost" value="127.0.0.1"/>
+ <property name="os.name" value="Mac OS X"/>
+ <property name="classworlds.conf" value="/usr/share/maven/bin/m2.conf"/>
+ <property name="sun.jnu.encoding" value="EUC_CN"/>
+ <property name="java.library.path" value="/usr/local/mysql/lib/:.:/Library/Java/Extensions:/System/Library/Java/Extensions:/usr/lib/java"/>
+ <property name="http.proxyPort" value="8086"/>
+ <property name="java.specification.name" value="Java Platform API Specification"/>
+ <property name="java.class.version" value="50.0"/>
+ <property name="sun.management.compiler" value="HotSpot 64-Bit Tiered Compilers"/>
+ <property name="os.version" value="10.8.5"/>
+ <property name="http.nonProxyHosts" value="local„ÄÅ169.254/16|*.local„ÄÅ169.254/16"/>
+ <property name="user.home" value="/Users/owen2785"/>
+ <property name="user.timezone" value="Asia/Shanghai"/>
+ <property name="java.awt.printerjob" value="apple.awt.CPrinterJob"/>
+ <property name="java.specification.version" value="1.6"/>
+ <property name="file.encoding" value="EUC_CN"/>
+ <property name="user.name" value="owen2785"/>
+ <property name="java.class.path" value="/usr/share/maven/boot/plexus-classworlds-2.4.jar"/>
+ <property name="java.vm.specification.version" value="1.0"/>
+ <property name="sun.arch.data.model" value="64"/>
+ <property name="java.home" value="/System/Library/Java/JavaVirtualMachines/1.6.0.jdk/Contents/Home"/>
+ <property name="sun.java.command" value="org.codehaus.plexus.classworlds.launcher.Launcher install"/>
+ <property name="java.specification.vendor" value="Sun Microsystems Inc."/>
+ <property name="user.language" value="zh"/>
+ <property name="awt.toolkit" value="apple.awt.CToolkit"/>
+ <property name="java.vm.info" value="mixed mode"/>
+ <property name="java.version" value="1.6.0_51"/>
+ <property name="java.ext.dirs" value="/Library/Java/Extensions:/System/Library/Java/Extensions:/System/Library/Java/JavaVirtualMachines/1.6.0.jdk/Contents/Home/lib/ext"/>
+ <property name="http.proxyHost" value="127.0.0.1"/>
+ <property name="sun.boot.class.path" value="/System/Library/Java/JavaVirtualMachines/1.6.0.jdk/Contents/Classes/jsfd.jar:/System/Library/Java/JavaVirtualMachines/1.6.0.jdk/Contents/Classes/classes.jar:/System/Library/Frameworks/JavaVM.framework/Frameworks/JavaRuntimeSupport.framework/Resources/Java/JavaRuntimeSupport.jar:/System/Library/Java/JavaVirtualMachines/1.6.0.jdk/Contents/Classes/ui.jar:/System/Library/Java/JavaVirtualMachines/1.6.0.jdk/Contents/Classes/laf.jar:/System/Library/Java/JavaVirtualMachines/1.6.0.jdk/Contents/Classes/sunrsasign.jar:/System/Library/Java/JavaVirtualMachines/1.6.0.jdk/Contents/Classes/jsse.jar:/System/Library/Java/JavaVirtualMachines/1.6.0.jdk/Contents/Classes/jce.jar:/System/Library/Java/JavaVirtualMachines/1.6.0.jdk/Contents/Classes/charsets.jar"/>
+ <property name="java.vendor" value="Apple Inc."/>
+ <property name="maven.home" value="/usr/share/maven"/>
+ <property name="file.separator" value="/"/>
+ <property name="java.vendor.url.bug" value="http://bugreport.apple.com/"/>
+ <property name="sun.cpu.endian" value="little"/>
+ <property name="sun.io.unicode.encoding" value="UnicodeLittle"/>
+ <property name="mrj.version" value="1070.1.6.0_51-456"/>
+ <property name="https.proxyPort" value="8086"/>
+ <property name="socksNonProxyHosts" value="local„ÄÅ169.254/16|*.local„ÄÅ169.254/16"/>
+ <property name="ftp.nonProxyHosts" value="local„ÄÅ169.254/16|*.local„ÄÅ169.254/16"/>
+ <property name="sun.cpu.isalist" value=""/>
+ </properties>
+ <testcase time="0.002" classname="com.yao.importIndex.AppTest" name="testApp"/>
+</testsuite>
View
4 target/surefire-reports/com.yao.importIndex.AppTest.txt
@@ -0,0 +1,4 @@
+-------------------------------------------------------------------------------
+Test set: com.yao.importIndex.AppTest
+-------------------------------------------------------------------------------
+Tests run: 1, Failures: 0, Errors: 0, Skipped: 0, Time elapsed: 0.014 sec

No commit comments for this range

Something went wrong with that request. Please try again.