Skip to content
Permalink
Browse files

Clean up

  • Loading branch information
lwj5 committed Oct 10, 2019
1 parent 06d7a8e commit a89ba854662e4df3d6cc0bde920899d2c75a1e5f
156 pom.xml
@@ -2,89 +2,89 @@
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<modelVersion>4.0.0</modelVersion>

<groupId>ai.preferred</groupId>
<artifactId>venom-examples</artifactId>
<version>1.0.1</version>
<groupId>ai.preferred</groupId>
<artifactId>venom-examples</artifactId>
<version>1.0.1</version>

<properties>
<maven.compiler.source>11</maven.compiler.source>
<maven.compiler.target>11</maven.compiler.target>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
</properties>
<properties>
<maven.compiler.source>11</maven.compiler.source>
<maven.compiler.target>11</maven.compiler.target>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
</properties>

<build>
<plugins>
<plugin>
<artifactId>maven-surefire-plugin</artifactId>
<version>2.22.2</version>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
<version>3.1.0</version>
<executions>
<execution>
<phase>package</phase>
<goals>
<goal>shade</goal>
</goals>
<configuration>
<minimizeJar>false</minimizeJar>
<finalName>${project.artifactId}-${project.version}-jar-with-dependencies</finalName>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
<build>
<plugins>
<plugin>
<artifactId>maven-surefire-plugin</artifactId>
<version>2.22.2</version>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
<version>3.1.0</version>
<executions>
<execution>
<phase>package</phase>
<goals>
<goal>shade</goal>
</goals>
<configuration>
<minimizeJar>false</minimizeJar>
<finalName>${project.artifactId}-${project.version}-jar-with-dependencies</finalName>
</configuration>
</execution>
</executions>
</plugin>
</plugins>

<resources>
<resource>
<directory>src/main/resources</directory>
</resource>
</resources>
<resources>
<resource>
<directory>src/main/resources</directory>
</resource>
</resources>

<testResources>
<testResource>
<directory>src/test/resources</directory>
</testResource>
</testResources>
</build>
<testResources>
<testResource>
<directory>src/test/resources</directory>
</testResource>
</testResources>
</build>

<dependencies>
<dependency>
<groupId>ai.preferred</groupId>
<artifactId>venom</artifactId>
<version>[4.2,4.3)</version>
</dependency>
<dependency>
<groupId>org.json</groupId>
<artifactId>json</artifactId>
<version>[20170000,)</version>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
<version>[1.7,1.8)</version>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-csv</artifactId>
<version>1.6</version>
</dependency>
<dependency>
<groupId>org.junit.jupiter</groupId>
<artifactId>junit-jupiter-api</artifactId>
<version>5.5.2</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.junit.jupiter</groupId>
<artifactId>junit-jupiter-engine</artifactId>
<version>5.5.2</version>
<scope>test</scope>
</dependency>
</dependencies>
<dependencies>
<dependency>
<groupId>ai.preferred</groupId>
<artifactId>venom</artifactId>
<version>[4.2,4.3)</version>
</dependency>
<dependency>
<groupId>org.json</groupId>
<artifactId>json</artifactId>
<version>[20170000,)</version>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
<version>[1.7,1.8)</version>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-csv</artifactId>
<version>1.6</version>
</dependency>
<dependency>
<groupId>org.junit.jupiter</groupId>
<artifactId>junit-jupiter-api</artifactId>
<version>5.5.2</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.junit.jupiter</groupId>
<artifactId>junit-jupiter-engine</artifactId>
<version>5.5.2</version>
<scope>test</scope>
</dependency>
</dependencies>

</project>
@@ -29,15 +29,15 @@ public static void main(String[] args) {
final String filename = "data/iproperty.csv";

// Start CSV printer
try (final EntityCSVStorage<Property> storage = new EntityCSVStorage<>(filename)) {
try (EntityCSVStorage<Property> storage = new EntityCSVStorage<>(filename)) {

// Let's init the session, this allows us to retrieve the array list in the handler
final Session session = Session.builder()
.put(STORAGE_KEY, storage)
.build();

// Start crawler
try (final Crawler crawler = createCrawler(createFetcher(), session).start()) {
try (Crawler crawler = createCrawler(createFetcher(), session).start()) {
LOGGER.info("starting crawler...");

final String startUrl = "https://www.iproperty.com.sg/rent/list/";
@@ -62,7 +62,7 @@ public static void main(String[] args) {
.build();

// Start crawler and FileManager
try (final Crawler crawler = createCrawler(createFetcher(fileManager), session).start()) {
try (Crawler crawler = createCrawler(createFetcher(fileManager), session).start()) {
LOGGER.info("Starting crawler...");

final String startUrl = "https://stackoverflow.com/jobs?l=Singapore&d=20&u=Km";

0 comments on commit a89ba85

Please sign in to comment.
You can’t perform that action at this time.