初次提交

chj9 · Nov 19, 2018 · cd57ab0 · cd57ab0
1 parent 5f957cd
commit cd57ab0
Show file tree

Hide file tree

Showing 20 changed files with 1,684 additions and 0 deletions.
diff --git a/build.bat b/build.bat
@@ -0,0 +1 @@
+ant
diff --git a/build.xml b/build.xml
@@ -0,0 +1,51 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project name="es_data_export" default="zip" basedir=".">
+	<property name="build" value="build" />
+	<property name="target" value="target" />
+	<property name="word" value="docs/V1.0"/>
+	<property name="jar_version" value="ES数据导出工具V1.0(build 1)" />
+	<property name="routine" value="build/${jar_version}/程序包"/>
+	<property name="doc" value="build/${jar_version}/文档" />
+
+	<target name="clean">
+		<delete dir="${target}" />
+		<delete dir="${build}" />
+		<delete dir="${routine}" />
+		<delete dir="${doc}" />
+		<delete dir="${script}" />
+	</target>
+
+	<target name="create-path" depends="clean">
+		<mkdir dir="${build}" />
+		<mkdir dir="${build}/${jar_version}" />
+	</target>
+
+	<target name="mvn_package" depends="create-path">
+		<exec executable="cmd" failonerror="true">
+			<arg line="/c mvn clean package" />
+		</exec>
+	</target>
+
+	<target name="copyAll" depends="mvn_package">
+		<copy todir="${routine}" file="${target}/es_data_export.jar"></copy> 
+		<copy todir="${routine}" file="global.properties"></copy>
+		<copy todir="${routine}" file="logback.xml"></copy>
+		<copy todir="${routine}" file="run.sh"></copy>
+		<copy todir="${routine}" file="stop.sh"></copy>
+		<copy todir="${doc}" file="${word}/ES数据导出工具-改动说明.docx"></copy>
+		<copy todir="${doc}" file="${word}/ES数据导出工具-升级指导.docx"></copy>
+		<copy todir="${doc}" file="${word}/ES数据导出工具-验证方法.docx"></copy>
+
+
+	</target>
+
+	<!--文件打包-->
+	 <target name="zip" depends="copyAll">
+        <zip destfile="build/${jar_version}.zip" basedir="${build}" excludes=".svn"/>
+    </target>
+
+	<target name="cleanFiles" depends="zip">
+		<delete dir="${build}/${jar_version}" />
+	</target>
+
+</project>
diff --git a/docs/V1.0/ES数据导出工具-升级指导.docx b/docs/V1.0/ES数据导出工具-升级指导.docx
diff --git a/docs/V1.0/ES数据导出工具-改动说明.docx b/docs/V1.0/ES数据导出工具-改动说明.docx
diff --git a/docs/V1.0/ES数据导出工具-验证方法.docx b/docs/V1.0/ES数据导出工具-验证方法.docx
diff --git a/global.properties b/global.properties
@@ -0,0 +1,31 @@
+####常规设置######
+#数据索引
+index=pb_sa_phone
+#索引type
+type=pb_sa_phone
+#查询条件,默认:
+query={"size": 1000,"query": {"term":{"mftid": {"value": 180000}}},"sort": [{"creatm": {"order": "desc"}}]}
+#取哪些字段的数据,逗号隔开,如果全部取则设为空
+includes=phone
+#导出数据ES地址
+es.servers=192.169.2.98:9200
+####导出文件的设置####
+#是否需要换行,默认:true
+isLineFeed=true
+#输出源数据形式:分别为json与txt,如果为txt字段间是用逗号隔开,默认:json
+dataLayout=txt
+#数据输出文件路径,默认：当前程序路径
+filePath=F:\\pb_sa_phone
+#文件名,无则取默认:index
+fileName=pb_sa_phone
+#每个文件多少条数据分割,需要则设置该项,单位:条
+fileSize=
+#自定义字段名,将库里该字段取出来后换为该字段名,原字段名:替换后的字段名,多个逗号隔开,如phone:telphone
+customFieldName=
+#####以下是dataLayout=txt,为json时以下配置都无效的时候的自定义设置######
+#字段以什么分割,不设置则默认英文逗号隔开
+fieldSplit=,
+#字段输出顺序(必设),必须和索引表字段名一样,逗号隔开
+fieldSort=phone
+#输出为txt的时候需要字段名字,默认:false,需要的时候以此形式输出类似:fieldName1=fieldValue1,fieldName2=fieldValue2
+needFieldName=false
diff --git a/logback.xml b/logback.xml
@@ -0,0 +1,109 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<configuration  scan="true" scanPeriod="30 seconds" >
+	<property name="LOG_HOME" value="logs"/>
+	<appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
+		<filter class="ch.qos.logback.classic.filter.LevelFilter">  
+            <level>DEBUG</level>  
+            <onMatch>DENY</onMatch>  
+            <onMismatch>ACCEPT</onMismatch>  
+        </filter>
+		<layout class="ch.qos.logback.classic.PatternLayout">
+			<Pattern>
+				%d{yyyy-MM-dd HH:mm:ss} %c %-5level - %msg%n
+			</Pattern>
+		</layout>
+	</appender>
+	<!-- DEBUG日志配置 -->
+	<appender name="DEBUG" class="ch.qos.logback.core.rolling.RollingFileAppender">
+		<filter class="ch.qos.logback.classic.filter.LevelFilter">  
+            <level>DEBUG</level>  
+            <onMatch>ACCEPT</onMatch>  
+            <onMismatch>DENY</onMismatch>  
+        </filter>
+		<rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy">
+			<!-- rollover daily -->
+			<fileNamePattern>${LOG_HOME}/debug-%d{yyyy-MM-dd}.%i.txt</fileNamePattern>
+			<!-- each file should be at most 10MB, keep 31 days worth of history, but at most 10GB -->
+			<maxFileSize>10MB</maxFileSize>
+			<maxHistory>7</maxHistory>
+			<totalSizeCap>10GB</totalSizeCap>
+		</rollingPolicy>
+		<layout class="ch.qos.logback.classic.PatternLayout">
+			<Pattern>
+				%d{yyyy-MM-dd HH:mm:ss} %c %-5level - %msg%n
+			</Pattern>
+		</layout>
+	</appender>
+	<!-- INFO日志配置 -->
+	<appender name="INFO" class="ch.qos.logback.core.rolling.RollingFileAppender">
+		<filter class="ch.qos.logback.classic.filter.LevelFilter">  
+            <level>INFO</level>  
+            <onMatch>ACCEPT</onMatch>  
+            <onMismatch>DENY</onMismatch>  
+        </filter>
+		<rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy">
+			<!-- rollover daily -->
+			<fileNamePattern>${LOG_HOME}/info-%d{yyyy-MM-dd}.%i.txt</fileNamePattern>
+			<!-- each file should be at most 10MB, keep 31 days worth of history, but at most 10GB -->
+			<maxFileSize>10MB</maxFileSize>
+			<maxHistory>7</maxHistory>
+			<totalSizeCap>10GB</totalSizeCap>
+		</rollingPolicy>
+		<layout class="ch.qos.logback.classic.PatternLayout">
+			<Pattern>
+				%d{yyyy-MM-dd HH:mm:ss} %c %-5level - %msg%n
+			</Pattern>
+		</layout>
+	</appender>
+	<!-- WARN日志配置 -->
+	<appender name="WARN" class="ch.qos.logback.core.rolling.RollingFileAppender">
+		<filter class="ch.qos.logback.classic.filter.LevelFilter">  
+            <level>WARN</level>  
+            <onMatch>ACCEPT</onMatch>  
+            <onMismatch>DENY</onMismatch>  
+        </filter>
+		<rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy">
+			<!-- rollover daily -->
+			<fileNamePattern>${LOG_HOME}/warn-%d{yyyy-MM-dd}.%i.txt</fileNamePattern>
+			<!-- each file should be at most 10MB, keep 31 days worth of history, but at most 10GB -->
+			<maxFileSize>10MB</maxFileSize>
+			<maxHistory>7</maxHistory>
+			<totalSizeCap>10GB</totalSizeCap>
+		</rollingPolicy>
+		<layout class="ch.qos.logback.classic.PatternLayout">
+			<Pattern>
+				%d{yyyy-MM-dd HH:mm:ss} %c %-5level - %msg%n
+			</Pattern>
+		</layout>
+	</appender>
+	<!-- ERROR日志配置 -->
+	<appender name="ERROR" class="ch.qos.logback.core.rolling.RollingFileAppender">
+		<filter class="ch.qos.logback.classic.filter.LevelFilter">  
+            <level>ERROR</level>
+            <onMatch>ACCEPT</onMatch>
+            <onMismatch>DENY</onMismatch>  
+        </filter>
+		<rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy">
+			<!-- rollover daily -->
+			<fileNamePattern>${LOG_HOME}/error-%d{yyyy-MM-dd}.%i.txt</fileNamePattern>
+			<!-- each file should be at most 10MB, keep 31 days worth of history, but at most 10GB -->
+			<maxFileSize>10MB</maxFileSize>
+			<maxHistory>7</maxHistory>
+			<totalSizeCap>10GB</totalSizeCap>
+		</rollingPolicy>
+		<layout class="ch.qos.logback.classic.PatternLayout">
+			<Pattern>
+				%d{yyyy-MM-dd HH:mm:ss} %c %-5level - %msg%n
+			</Pattern>
+		</layout>
+	</appender>
+
+	<root level="DEBUG">
+		<appender-ref ref="STDOUT"/>
+		<appender-ref ref="DEBUG" />
+		<appender-ref ref="INFO" />
+		<appender-ref ref="WARN" />
+		<appender-ref ref="ERROR" />
+	</root>
+
+</configuration>
diff --git a/pom.xml b/pom.xml
@@ -0,0 +1,119 @@
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+  xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+
+  <groupId>com.chenhj</groupId>
+  <artifactId>es_data_export</artifactId>
+  <version>0.0.1-SNAPSHOT</version>
+  <packaging>jar</packaging>
+
+  <name>es_data_export</name>
+  <url>http://maven.apache.org</url>
+
+  <properties>
+  		<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+        <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
+  		<fastjson>1.2.41</fastjson>
+  		<logback>1.2.3</logback>
+		<slf4j>1.7.25</slf4j>
+    	<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+  </properties>
+
+  <dependencies>
+  		<!--fastjson 相关jar -->
+		<dependency>
+			<groupId>com.alibaba</groupId>
+			<artifactId>fastjson</artifactId>
+			<version>${fastjson}</version>
+		</dependency>
+        <dependency>
+			<groupId>org.montnets.elastic</groupId>
+			<artifactId>es-serve-pool</artifactId>
+			<version>1.1.12</version>
+		</dependency>
+    		<!--httpclient依赖 -->
+		<dependency>
+		    <groupId>org.apache.httpcomponents</groupId>
+		    <artifactId>httpclient</artifactId>
+		    <version>4.5.6</version>
+		</dependency>
+		<dependency>
+	        <groupId>org.apache.commons</groupId>
+	    	<artifactId>commons-lang3</artifactId>
+	   		<version>3.8.1</version>
+		</dependency>
+				<!--日志 -->
+		<dependency>
+			<groupId>org.slf4j</groupId>
+			<artifactId>slf4j-api</artifactId>
+			<version>${slf4j}</version>
+		</dependency>
+		<dependency>
+			<groupId>ch.qos.logback</groupId>
+			<artifactId>logback-classic</artifactId>
+			<version>${logback}</version>
+		</dependency>
+		<dependency>
+			<groupId>ch.qos.logback</groupId>
+			<artifactId>logback-core</artifactId>
+			<version>${logback}</version>
+	</dependency>
+    <dependency>
+      <groupId>junit</groupId>
+      <artifactId>junit</artifactId>
+      <version>3.8.1</version>
+      <scope>test</scope>
+    </dependency>
+  </dependencies>
+  	<build>
+		<resources>
+			<resource>
+				<directory>src/main/resource/mapper</directory>
+				<includes>
+					<include>*.xml</include>
+				</includes>
+				<targetPath>${project.build.directory}/classes/mapper</targetPath>
+				<filtering>false</filtering>
+			</resource>
+			<resource>
+				<directory>src/main/resource</directory>
+				<includes>
+					<include>*.properties</include>
+					<include>*.xml</include>
+					<include>*.tld</include>
+				</includes>
+				<filtering>false</filtering>
+			</resource>
+		</resources>
+		<defaultGoal>compile</defaultGoal>
+		<sourceDirectory>src</sourceDirectory>
+		<finalName>es_data_export</finalName>
+    <plugins>  
+        <plugin>  
+                <groupId>org.apache.maven.plugins</groupId>  
+                <artifactId>maven-assembly-plugin</artifactId>  
+                <version>2.5.5</version>  
+                <configuration>  
+                    <appendAssemblyId>false</appendAssemblyId>  
+                    <descriptorRefs>  
+                        <descriptorRef>jar-with-dependencies</descriptorRef>  
+                    </descriptorRefs>  
+                    <archive>  
+                        <manifest>  
+                            <mainClass>com.montnets.App</mainClass>  
+                        </manifest>  
+                    </archive>  
+                </configuration>  
+                <executions>  
+                    <execution>  
+                        <id>make-assembly</id>  
+                        <phase>package</phase>  
+                        <goals>  
+                            <goal>assembly</goal>  
+                        </goals>  
+                    </execution>  
+                </executions>  
+            </plugin>   
+    </plugins>  
+	</build>
+</project>
diff --git a/run.sh b/run.sh
@@ -0,0 +1,11 @@
+APPDIR=`pwd`
+PIDFILE=$APPDIR/es_data_export.pid
+if [ -f "$PIDFILE" ] && kill -0 $(cat "$PIDFILE"); then
+echo "es_data_export is already running..."
+exit 1
+fi
+nohup java -jar $APPDIR/es_data_export.jar >/dev/null 2>&1 &
+echo $! > $PIDFILE
+echo "start es_data_export..."
+
+