Skip to content

Commit

Permalink
fix build and test failures on windows
Browse files Browse the repository at this point in the history
  • Loading branch information
@senchi @mfussenegger
senchi authored and mfussenegger committed Feb 5, 2015
1 parent b062d05 commit 8ef3513
Show file tree
Hide file tree
Showing 16 changed files with 144 additions and 80 deletions.
13 changes: 4 additions & 9 deletions DEVELOP.rst
View file
Original file line number Diff line number Diff line change
Expand Up @@ -156,15 +156,6 @@ First create the folders for the configuration and data::

for i in {1..2}; do mkdir -p sandbox/crate_$i/{config,data,plugins}; done

In order for the admin interface to work an additional folder and symbolic
links are necessary::

for i in {1..2}; do mkdir -p sandbox/crate_$i/plugins/crate-admin/; done

ln -s ../../../../app/plugins/crate-admin/_site sandbox/crate_1/plugins/crate-admin/_site
ln -s ../../../../app/plugins/crate-admin/_site sandbox/crate_2/plugins/crate-admin/_site


Then create the configuration files for both nodes::

touch sandbox/crate_1/config/crate.yml
Expand Down Expand Up @@ -206,6 +197,10 @@ A minimal example for the logging configuration looks like this::
layout:
type: consolePattern
conversionPattern: "[%d{ISO8601}][%-5p][%-25c] %m%n"
In order for the admin interface to work please check out the crate admin repository::

https://github.com/crate/crate-admin

After that the Run/Debug Configurations can be added within IntelliJ. Go to the
`Run/Debug Configurations` window and add a new `Application` configuration
Expand Down
12 changes: 7 additions & 5 deletions blob/src/test/java/io/crate/BlobEnvironmentTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,23 +24,23 @@
import io.crate.blob.BlobEnvironment;
import org.elasticsearch.cluster.ClusterName;
import org.elasticsearch.common.io.FileSystemUtils;
import org.elasticsearch.common.os.OsUtils;
import org.elasticsearch.common.settings.ImmutableSettings;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.settings.SettingsException;
import org.elasticsearch.env.Environment;
import org.elasticsearch.env.NodeEnvironment;
import org.elasticsearch.index.Index;
import org.elasticsearch.index.shard.ShardId;
import org.junit.After;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.*;
import org.junit.rules.ExpectedException;
import org.junit.rules.TemporaryFolder;

import java.io.File;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.attribute.PosixFilePermissions;

import static org.hamcrest.MatcherAssert.assertThat;
import static org.hamcrest.Matchers.is;
Expand Down Expand Up @@ -91,7 +91,7 @@ public void testShardLocationWithoutCustomPath() throws Exception {
File shardLocation = blobEnvironment.shardLocation(new ShardId(".blob_test", 0));
File nodeShardLocation = nodeEnvironment.shardLocations(new ShardId(".blob_test", 0))[0];
assertThat(shardLocation.getAbsolutePath().substring(nodeShardLocation.getAbsolutePath().length()),
is("/"+BlobEnvironment.BLOBS_SUB_PATH));
is(File.separator + BlobEnvironment.BLOBS_SUB_PATH));
}

@Test
Expand All @@ -114,6 +114,8 @@ public void testValidateIsFile() throws Exception {

@Test
public void testValidateNotCreatable() throws Exception {
Assume.assumeFalse(OsUtils.WINDOWS);

File tmpDir = folder.newFolder();
assertThat(tmpDir.setReadable(false), is(true));
assertThat(tmpDir.setWritable(false), is(true));
Expand Down
1 change: 1 addition & 0 deletions blob/src/test/java/io/crate/DigestBlobTests.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -117,6 +117,7 @@ public void testDigestBlobResumeHeadAndAddContent() throws IOException {
stream.close();

File file = digestBlob.commit();
stream.close();

This comment has been minimized.

Sign in to view

Copy link
@senchi

senchi Feb 6, 2015

Author Contributor

This stream.close() call is redundant and has been fixed in #1576, but has been closed and squashed with this one.
During that merge this small issue got overlooked!
Should I create new pull request for this small fix only or squash (eventho the PR is already merged and closed?) what is the procedure you guys do in such cases?

Thanks

This comment has been minimized.

Sign in to view

Copy link
@mfussenegger

mfussenegger Feb 6, 2015

Member

please create a new PR for it
assertTrue(file.exists());
// just in case any references to file left
assertTrue(file.delete());
Expand Down
11 changes: 9 additions & 2 deletions sql-parser/build.gradle
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,8 @@ apply plugin: 'java'

archivesBaseName = 'crate-sql-parser'

import org.apache.tools.ant.taskdefs.condition.Os

repositories {
mavenCentral()
}
Expand All @@ -25,8 +27,13 @@ dependencies {
testCompile 'junit:junit:4.11'
}

task generateWithANTLR3(type:Exec) {
commandLine = ['java', '-cp', configurations.antlr3.getAsPath(), 'org.antlr.Tool', '-o', '.', 'src/main/java/io/crate/sql/parser/Statement.g', 'src/main/java/io/crate/sql/parser/StatementBuilder.g']
task generateWithANTLR3(type:Exec) {
String outputPath = '.';
if (Os.isFamily(Os.FAMILY_WINDOWS)) {
outputPath = 'src/main/java/io/crate/sql/parser';
}

commandLine = ['java', '-cp', configurations.antlr3.getAsPath(), 'org.antlr.Tool', '-o', outputPath, 'src/main/java/io/crate/sql/parser/Statement.g', 'src/main/java/io/crate/sql/parser/StatementBuilder.g']
}

compileJava {
Expand Down
3 changes: 2 additions & 1 deletion sql-parser/src/test/java/io/crate/sql/parser/TestStatementBuilder.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,7 @@

public class TestStatementBuilder
{

@Test
public void testStatementBuilder()
throws Exception
Expand Down Expand Up @@ -515,7 +516,7 @@ private static void printStatement(String sql)
private static void println(String s)
{
if (Boolean.parseBoolean(System.getProperty("printParse"))) {
System.out.println(s);
System.out.print(s + "\n");
}
}

Expand Down
6 changes: 4 additions & 2 deletions sql/src/test/java/io/crate/executor/transport/TransportExecutorTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,7 @@
import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.common.collect.ImmutableOpenMap;
import org.elasticsearch.common.collect.MapBuilder;
import org.elasticsearch.common.os.OsUtils;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.search.SearchHits;
Expand Down Expand Up @@ -132,8 +133,9 @@ public void testRemoteCollectTask() throws Exception {
assertThat(result.size(), is(2));
for (ListenableFuture<TaskResult> nodeResult : result) {
assertEquals(1, nodeResult.get().rows().length);
assertThat((Double) nodeResult.get().rows()[0][0], is(greaterThan(0.0)));

if(!OsUtils.WINDOWS) {
assertThat((Double) nodeResult.get().rows()[0][0], is(greaterThan(0.0)));
}
}
}

Expand Down
5 changes: 2 additions & 3 deletions sql/src/test/java/io/crate/integrationtests/ColumnPolicyIntegrationTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -121,9 +121,8 @@ public void testCopyFromFileStrictTable() throws Exception {
execute("create table quotes (id int primary key) with (column_policy='strict', number_of_replicas = 0)");
ensureGreen();

String filePath = Joiner.on(File.separator).join(copyFilePath, "test_copy_from.json");

execute("copy quotes from ?", new Object[]{filePath});
String uriPath = Joiner.on("/").join(copyFilePath, "test_copy_from.json");
execute("copy quotes from ?", new Object[]{uriPath});
assertThat(response.rowCount(), is(0L));
}

Expand Down
29 changes: 14 additions & 15 deletions sql/src/test/java/io/crate/integrationtests/CopyIntegrationTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,6 @@
import org.junit.rules.ExpectedException;
import org.junit.rules.TemporaryFolder;

import java.io.File;
import java.nio.charset.StandardCharsets;
import java.nio.file.DirectoryStream;
import java.nio.file.Files;
Expand Down Expand Up @@ -68,8 +67,8 @@ public void testCopyFromFile() throws Exception {
"quote string index using fulltext) with (number_of_replicas = 0)");
ensureGreen();

String filePath = Joiner.on(File.separator).join(copyFilePath, "test_copy_from.json");
execute("copy quotes from ?", new Object[]{filePath});
String uriPath = Joiner.on("/").join(copyFilePath, "test_copy_from.json");
execute("copy quotes from ?", new Object[]{uriPath});
// 2 nodes on same machine resulting in double affected rows
assertEquals(6L, response.rowCount());
assertThat(response.duration(), greaterThanOrEqualTo(0L));
Expand All @@ -80,7 +79,7 @@ public void testCopyFromFile() throws Exception {
assertThat(response.rows()[0].length, is(2));

execute("select quote from quotes where id = 1");
assertThat((String) response.rows()[0][0], is("Don't pañic."));
assertThat((String) response.rows()[0][0], is("Don't pa\u00f1ic."));
}

@Test
Expand All @@ -89,8 +88,8 @@ public void testCopyFromFileWithoutPK() throws Exception {
"quote string index using fulltext) with (number_of_replicas=0)");
ensureGreen();

String filePath = Joiner.on(File.separator).join(copyFilePath, "test_copy_from.json");
execute("copy quotes from ?", new Object[]{filePath});
String uriPath = Joiner.on("/").join(copyFilePath, "test_copy_from.json");
execute("copy quotes from ?", new Object[]{uriPath});
// 2 nodes on same machine resulting in double affected rows
assertEquals(6L, response.rowCount());
assertThat(response.duration(), greaterThanOrEqualTo(0L));
Expand Down Expand Up @@ -121,8 +120,8 @@ public void testCopyFromFilePattern() throws Exception {
"quote string index using fulltext) with (number_of_replicas=0)");
ensureGreen();

String filePath = Joiner.on(File.separator).join(copyFilePath, "*.json");
execute("copy quotes from ?", new Object[]{filePath});
String uriPath = Joiner.on("/").join(copyFilePath, "*.json");
execute("copy quotes from ?", new Object[]{uriPath});
// 2 nodes on same machine resulting in double affected rows
assertEquals(6L, response.rowCount());
refresh();
Expand All @@ -138,7 +137,7 @@ public void testCopyToFile() throws Exception {
execute("insert into singleshard (name) values ('foo')");
execute("refresh table singleshard");

String uri = Paths.get(folder.getRoot().toURI()).resolve("testsingleshard.json").toAbsolutePath().toString();
String uri = Paths.get(folder.getRoot().toURI()).resolve("testsingleshard.json").toUri().toString();
SQLResponse response = execute("copy singleshard to ?", new Object[] { uri });
assertThat(response.rowCount(), is(1L));
List<String> lines = Files.readAllLines(
Expand All @@ -155,7 +154,7 @@ public void testCopyToFile() throws Exception {
public void testCopyColumnsToDirectory() throws Exception {
this.setup.groupBySetup();

String uriTemplate = Paths.get(folder.getRoot().toURI()).toAbsolutePath().toString();
String uriTemplate = Paths.get(folder.getRoot().toURI()).toUri().toString();
SQLResponse response = execute("copy characters (name, details['job']) to DIRECTORY ?", new Object[]{uriTemplate});
assertThat(response.cols().length, is(0));
assertThat(response.rowCount(), is(7L));
Expand Down Expand Up @@ -185,7 +184,7 @@ public void testCopyColumnsToDirectory() throws Exception {
public void testCopyToDirectory() throws Exception {
this.setup.groupBySetup();

String uriTemplate = Paths.get(folder.getRoot().toURI()).toAbsolutePath().toString();
String uriTemplate = Paths.get(folder.getRoot().toURI()).toUri().toString();
SQLResponse response = execute("copy characters to DIRECTORY ?", new Object[]{uriTemplate});
assertThat(response.rowCount(), is(7L));
List<String> lines = new ArrayList<>(7);
Expand All @@ -209,8 +208,8 @@ public void testCopyFromNestedArrayRow() throws Exception {
execute("create table users (id int, " +
"name string) with (number_of_replicas=0)");
ensureGreen();
String filePath = Joiner.on(File.separator).join(nestedArrayCopyFilePath, "nested_array_copy_from.json");
execute("copy users from ? with (shared=true)", new Object[]{filePath});
String uriPath = Joiner.on("/").join(nestedArrayCopyFilePath, "nested_array_copy_from.json");
execute("copy users from ? with (shared=true)", new Object[]{uriPath});
// 2 nodes on same machine resulting in double affected rows
assertEquals(1L, response.rowCount()); // only 1 document got inserted
refresh();
Expand All @@ -235,7 +234,7 @@ public void testCopyToDirectoryPath() throws Exception {
") with (number_of_replicas=0)");
ensureGreen();

String directory = folder.newFolder().getCanonicalPath();
execute("COPY characters TO ?", new Object[]{directory});
String directoryUri = Paths.get(folder.newFolder().toURI()).toUri().toString();
execute("COPY characters TO ?", new Object[]{directoryUri});
}
}
13 changes: 7 additions & 6 deletions sql/src/test/java/io/crate/integrationtests/PartitionedTableIntegrationTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -74,19 +74,19 @@ public void testCopyFromIntoPartitionedTableWithPARTITIONKeyword() throws Except
"quote string index using fulltext" +
") partitioned by (date) with (number_of_replicas=0)");
ensureGreen();
String filePath = Joiner.on(File.separator).join(copyFilePath, "test_copy_from.json");
execute("copy quotes partition (date=1400507539938) from ?", new Object[]{filePath});
String uriPath = Joiner.on("/").join(copyFilePath, "test_copy_from.json");
execute("copy quotes partition (date=1400507539938) from ?", new Object[]{uriPath});
refresh();
execute("select id, date, quote from quotes order by id asc");
assertEquals(3L, response.rowCount());
assertThat((Integer) response.rows()[0][0], is(1));
assertThat((Long) response.rows()[0][1], is(1400507539938L));
assertThat((String) response.rows()[0][2], is("Don't pañic."));
assertThat((String) response.rows()[0][2], is("Don't pa\u00f1ic."));

execute("select count(*) from information_schema.table_partitions where table_name = 'quotes'");
assertThat((Long) response.rows()[0][0], is(1L));

execute("copy quotes partition (date=1800507539938) from ?", new Object[]{filePath});
execute("copy quotes partition (date=1800507539938) from ?", new Object[]{uriPath});
refresh();

execute("select partition_ident from information_schema.table_partitions " +
Expand All @@ -105,8 +105,8 @@ public void testCopyFromIntoPartitionedTable() throws Exception {
") partitioned by (id)");
ensureGreen();

String filePath = Joiner.on(File.separator).join(copyFilePath, "test_copy_from.json");
execute("copy quotes from ?", new Object[]{filePath});
String uriPath = Joiner.on("/").join(copyFilePath, "test_copy_from.json");
execute("copy quotes from ?", new Object[]{uriPath});
// 2 nodes on same machine resulting in double affected rows
assertEquals(6L, response.rowCount());
assertThat(response.duration(), greaterThanOrEqualTo(0L));
Expand Down Expand Up @@ -957,6 +957,7 @@ public void testInsertDynamicToPartitionedTable() throws Exception {
put("name", "Ford");
}}});
ensureGreen();
waitNoPendingTasksOnAll();
refresh();

execute("select * from information_schema.columns where table_name = 'quotes'");
Expand Down
12 changes: 11 additions & 1 deletion sql/src/test/java/io/crate/integrationtests/RestSQLActionIntegrationTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,7 @@ public void testWithInvalidPayload() throws IOException {
CloseableHttpResponse response = post("{\"foo\": \"bar\"}");
assertEquals(400, response.getStatusLine().getStatusCode());
String bodyAsString = EntityUtils.toString(response.getEntity());
assertThat(bodyAsString, startsWith("{\"error\":{\"message\":\"SQLActionException[Failed to parse source [{\\\"foo\\\": \\\"bar\\\"}]]\",\"code\":4000},\"error_trace\":\"io.crate.exceptions.SQLParseException: Failed to parse source [{\\\"foo\\\": \\\"bar\\\"}]\\n\\tat "));
assertThat(bodyAsString, startsWith("{\"error\":{\"message\":\"SQLActionException[Failed to parse source [{\\\"foo\\\": \\\"bar\\\"}]]\",\"code\":4000},\"error_trace\":\"io.crate.exceptions.SQLParseException: Failed to parse source [{\\\"foo\\\": \\\"bar\\\"}]" + resolveEscapedNL() + "\\tat "));
}

@Test
Expand All @@ -58,4 +58,14 @@ public void testWithArgsAndBulkArgs() throws IOException {
String bodyAsString = EntityUtils.toString(response.getEntity());
assertEquals("{\"error\":{\"message\":\"SQLActionException[request body contains args and bulk_args. It's forbidden to provide both]\",\"code\":4000},\"error_trace\":null}", bodyAsString);
}

private String resolveEscapedNL(){
String LN = System.getProperty("line.separator");
// http://en.wikipedia.org/wiki/Newline#Representations
switch (LN){
case "\r": return "\\r";
case "\r\n": return "\\r\\n";
default: return "\\n";
}
}
}
11 changes: 7 additions & 4 deletions sql/src/test/java/io/crate/integrationtests/TransportSQLActionClassLifecycleTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@
import io.crate.test.integration.ClassLifecycleIntegrationTest;
import io.crate.testing.SQLTransportExecutor;
import io.crate.testing.TestingHelpers;
import org.elasticsearch.common.os.OsUtils;
import org.hamcrest.Matchers;
import org.junit.*;
import org.junit.rules.ExpectedException;
Expand Down Expand Up @@ -401,7 +402,7 @@ public void testOrderByNullsFirstAndLast() throws Exception {

@Test
public void testCopyToDirectoryOnPartitionedTableWithPartitionClause() throws Exception {
String uriTemplate = Paths.get(folder.getRoot().toURI()).toAbsolutePath().toString();
String uriTemplate = Paths.get(folder.getRoot().toURI()).toUri().toString();
SQLResponse response = executor.exec("copy parted partition (date='2014-01-01') to DIRECTORY ?", uriTemplate);
assertThat(response.rowCount(), is(2L));

Expand All @@ -421,7 +422,7 @@ public void testCopyToDirectoryOnPartitionedTableWithPartitionClause() throws Ex

@Test
public void testCopyToDirectoryOnPartitionedTableWithoutPartitionClause() throws Exception {
String uriTemplate = Paths.get(folder.getRoot().toURI()).toAbsolutePath().toString();
String uriTemplate = Paths.get(folder.getRoot().toURI()).toUri().toString();
SQLResponse response = executor.exec("copy parted to DIRECTORY ?", uriTemplate);
assertThat(response.rowCount(), is(5L));

Expand Down Expand Up @@ -453,8 +454,10 @@ public void testArithmeticFunctions() throws Exception {
response = executor.exec("select ? + 2 from sys.cluster", 1);
assertThat((Long)response.rows()[0][0], is(3L));

response = executor.exec("select load['1'] + load['5'], load['1'], load['5'] from sys.nodes limit 1");
assertEquals(response.rows()[0][0], (Double) response.rows()[0][1] + (Double) response.rows()[0][2]);
if(!OsUtils.WINDOWS) {
response = executor.exec("select load['1'] + load['5'], load['1'], load['5'] from sys.nodes limit 1");
assertEquals(response.rows()[0][0], (Double) response.rows()[0][1] + (Double) response.rows()[0][2]);
}
}

@Test
Expand Down
3 changes: 2 additions & 1 deletion sql/src/test/java/io/crate/operation/collect/MapSideDataCollectOperationTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@

import java.io.File;
import java.io.FileWriter;
import java.nio.file.Paths;
import java.util.Arrays;
import java.util.HashMap;
import java.util.Map;
Expand Down Expand Up @@ -105,7 +106,7 @@ public ReferenceImplementation getImplementation(ReferenceIdent ident) {
FileUriCollectNode collectNode = new FileUriCollectNode(
"test",
routing,
Literal.newLiteral(tmpFile.getAbsolutePath()),
Literal.newLiteral(Paths.get(tmpFile.toURI()).toUri().toString()),
Arrays.<Symbol>asList(
createReference("name", DataTypes.STRING),
createReference(new ColumnIdent("details", "age"), DataTypes.INTEGER)
Expand Down
Loading

0 comments on commit 8ef3513

Please sign in to comment.