Fix some jUnit tests on Linux (#7)

* Fix some jUnit tests on Linux

Fix many path related and other issues with jUnit tests on Linux.

Note that these changes don't yet make every test pass, there
are still issues with font differences in image generation,
non-portable performance checks, and a handful of other differences.

* Fix watch directory tests on windows

While linux may not send the modify events on deletion or the directory event, windows does

* Disable the display in browser, it causes tests to pause

* Have the path regex work for both windows and linux

* Fix the forwardSlashDir utility function

This is only used in one spot, so should be safe to modify.
Also it was previously checking if the string started with a / and if not, adding a slash to the end of it.

* Fix some of the EDDGridFromNcFilesTests on Linux

The wording of FileNotFoundException differs, so just make sure its that kind of exception. Mark testNcml flaky (for now), and fix the capitilzation of testGridNThreads for systems that are sensitive to that.

* Disable the performance part of persistentTableTests for now

* Change to using a temp dir for eddtablefromhttpgettests

---------

Co-authored-by: Chris John <chris.john@noaa.gov>

WEB-INF/classes/com/cohort/util/File2.java

@@ -1951,7 +1951,7 @@ public static String getSystemTempDirectory() throws Exception {
     public static String forwardSlashDir(String tDir) {
         StringBuilder sb = new StringBuilder(tDir);
         String2.replaceAll(sb, '\\', '/');
-        if (sb.length() == 0 || sb.charAt(0) != '/' )
+        if (sb.length() == 0 || sb.charAt(tDir.length() - 1) != '/' )
             sb.append('/');
         return sb.toString();
     }

src/test/java/com/cohort/util/TestUtil.java

@@ -17,6 +17,7 @@
 import java.util.concurrent.locks.Lock;
 
 import org.junit.jupiter.api.RepeatedTest;
+import org.junit.jupiter.api.condition.DisabledIf;
 import org.junit.jupiter.api.io.TempDir;
 import org.junit.jupiter.api.parallel.Isolated;
 
@@ -1250,11 +1251,6 @@ void testString2() throws Throwable {
         int n, iar[];
         String s, results, expected;
         long time;
-        /* */
-        // clipboard
-        String2.log("Clipboard was: " + String2.getClipboardString());
-        String2.setClipboardString("Test String2.setClipboardString.");
-        Test.ensureEqual(String2.getClipboardString(), "Test String2.setClipboardString.", "");
 
         // md5Hex
         Test.ensureEqual(String2.md5Hex("This is a test01234.ÀÑ"), "b62023b8dffda52f4b1ea48f2cee739e", "");
@@ -3083,6 +3079,15 @@ void testString2() throws Throwable {
 
     }
 
+    @org.junit.jupiter.api.Test
+    @DisabledIf(value = "java.awt.GraphicsEnvironment#isHeadless", disabledReason = "headless environment")
+    void testString2Clipboard() throws Throwable {
+        // clipboard
+        String2.log("Clipboard was: " + String2.getClipboardString());
+        String2.setClipboardString("Test String2.setClipboardString.");
+        Test.ensureEqual(String2.getClipboardString(), "Test String2.setClipboardString.", "");
+    }
+
     private static double nextEpochSecond() {
         return Math.ceil(System.currentTimeMillis() / 1000.0);
     }

src/test/java/gov/noaa/pfel/coastwatch/util/FileVisitorSubdirTests.java

@@ -31,6 +31,7 @@ void testLocal() throws Throwable {
 
     // test forward slashes 
     alps = FileVisitorSubdir.oneStep(contextDir + "WEB-INF/classes/com/cohort", null); // without trailing slash
+    alps.sort(); // sort is required before test comparison in Linux
     String results = alps.toNewlineString();
     String expected = contextDir + "WEB-INF/classes/com/cohort/\n" +
         contextDir + "WEB-INF/classes/com/cohort/array/\n" +

src/test/java/gov/noaa/pfel/coastwatch/util/PersistentTableTests.java

@@ -290,10 +290,11 @@ void basicTest() throws Throwable {
       int expected[] = { 203, 744, 1819, 2063 }; // java 17 203,744,1819,2063 java 1.6 140,693,1491,1615
       modeTime = System.currentTimeMillis() - modeTime;
       String2.log("mode=" + mode + " time=" + modeTime + "ms");
-      Test.ensureTrue(modeTime < 2 * expected[mode],
-          modes[mode] + " TOTAL time to read " + n + " items=" +
-              modeTime + "ms  (" + expected[mode] + "ms)\n" +
-              "That is too slow! But it is usually fast enough when I run the test by itself.");
+      // TODO set up a better way to do performance testing
+      // Test.ensureTrue(modeTime < 2 * expected[mode],
+      //     modes[mode] + " TOTAL time to read " + n + " items=" +
+      //         modeTime + "ms  (" + expected[mode] + "ms)\n" +
+      //         "That is too slow! But it is usually fast enough when I run the test by itself.");
 
       pt.close();
     }

src/test/java/gov/noaa/pfel/coastwatch/util/WatchDirectoryTests.java

@@ -22,12 +22,12 @@ void basicTest() throws Throwable {
     // verbose = true;
     ArrayList<WatchEvent.Kind> eventKinds = new ArrayList();
     StringArray contexts = new StringArray();
-    String testDataDir = Path.of(WatchDirectoryTests.class.getResource("/data/").toURI()).toString();
+    String testDataDir = Path.of(WatchDirectoryTests.class.getResource("/data/").toURI()).toString().replace('\\', '/');
     String sourceDir = testDataDir;
-    String watchDir = testDataDir + "\\watchService";
-    String subDirNS = testDataDir + "\\watchService\\watchSub";
-    String file1 = "\\columnarAsciiWithComments.txt";
-    String file2 = "\\csvAscii.txt";
+    String watchDir = testDataDir + "/watchService";
+    String subDirNS = testDataDir + "/watchService/watchSub";
+    String file1 = "/columnarAsciiWithComments.txt";
+    String file2 = "/csvAscii.txt";
     String results;
     int n;
     // On Bob's M4700, even 2000 isn't sufficient to reliably catch all events
@@ -127,7 +127,12 @@ void basicTest() throws Throwable {
               results.equals(WatchDirectory.MODIFY + " " + subDirNS),
           "");
     }
-    Test.ensureBetween(n, 3, 5, ""); // sometimes the dir event isn't caught
+    // on linux modify events are not created during this delete
+    // also a subdir deletion was previously checked for here,
+    // but RegexFilenameFilter.regexDelete explicitly excludes directories
+    // from matching (directoriesToo is false), so only two events are possible here
+    // (the two file DELETEs)
+    Test.ensureBetween(n, 2, 5, "");
 
     // *** test creating a huge number
     // This is allowed on Windows. It doesn't appear to have max number.

src/test/java/gov/noaa/pfel/erddap/dataset/EDDGridFromAudioFilesTests.java

@@ -64,7 +64,7 @@ void testGenerateDatasetsXml() throws Throwable {
         "    <defaultDataQuery>&amp;time=min(time)</defaultDataQuery>\n" +
         "    <reloadEveryNMinutes>1440</reloadEveryNMinutes>\n" +
         "    <updateEveryNMillis>10000</updateEveryNMillis>\n" +
-        "    <fileDir>" + dir + "\\</fileDir>\n" +
+        "    <fileDir>" + File2.addSlash(dir) + "</fileDir>\n" +
         "    <fileNameRegex>aco_acoustic\\.[0-9]{8}_[0-9]{6}\\.wav</fileNameRegex>\n" +
         "    <recursive>true</recursive>\n" +
         "    <pathRegex>.*</pathRegex>\n" +
@@ -195,7 +195,7 @@ void testGenerateDatasetsXml2() throws Throwable {
         "    <defaultDataQuery>&amp;time=min(time)</defaultDataQuery>\n" +
         "    <reloadEveryNMinutes>1440</reloadEveryNMinutes>\n" +
         "    <updateEveryNMillis>10000</updateEveryNMillis>\n" +
-        "    <fileDir>" + dir + "\\</fileDir>\n" +
+        "    <fileDir>" + File2.addSlash(dir) + "</fileDir>\n" +
         "    <fileNameRegex>aco_acoustic\\.[0-9]{8}_[0-9]{6}\\.wav</fileNameRegex>\n" +
         "    <recursive>true</recursive>\n" +
         "    <pathRegex>.*</pathRegex>\n" +
@@ -471,7 +471,7 @@ void testBasic(boolean deleteCachedDatasetInfo) throws Throwable {
         "9.375E-5,-7458\n" +
         "...\n";
     Test.ensureEqual(results, expected, "\nresults=\n" + results);
-    Test.displayInBrowser("file://" + dir + tName);
+    // Test.displayInBrowser("file://" + dir + tName);
     // String2.pressEnterToContinue("Close audio player when done.");
     File2.delete(dir + tName);
 

src/test/java/gov/noaa/pfel/erddap/dataset/EDDGridFromMergeIRFilesTests.java

@@ -43,7 +43,7 @@ void testGenerateDatasetsXml() throws Throwable {
     String expected = "<dataset type=\"EDDGridFromMergeIRFiles\" datasetID=\"mergeIR\" active=\"true\">\n" +
         "    <reloadEveryNMinutes>1440</reloadEveryNMinutes>\n" +
         "    <updateEveryNMillis>10000</updateEveryNMillis>\n" +
-        "    <fileDir>" + dataDir + "\\</fileDir>\n" +
+        "    <fileDir>" + File2.addSlash(dataDir) + "</fileDir>\n" +
         "    <fileNameRegex>merg_[0-9]{10}_4km-pixel\\.gz</fileNameRegex>\n" +
         "    <recursive>true</recursive>\n" +
         "    <pathRegex>.*</pathRegex>\n" +

src/test/java/gov/noaa/pfel/erddap/dataset/EDDGridFromNcFilesTests.java

@@ -94,6 +94,7 @@ private static String dumpTimeLatLon(String ncmlName) throws Exception {
 
   /** test reading an .ncml file */
   @org.junit.jupiter.api.Test
+  @TagFlaky // https://github.com/ERDDAP/erddap/issues/148
   void testNcml() throws Throwable {
 
     // String2.log("\n*** EDDGridFromNcFiles.testNcml");
@@ -165,6 +166,8 @@ void testNcml() throws Throwable {
     } catch (Exception e) {
       Test.knownProblem("2022-07-07 This fails with switch to netcdf v5.5.3 and modules (not netcdfAll). " +
           "I reported to netcdf-java people.", e);
+        // Bob's 2022 email to netcdf-java mailing list here (seems to be unanswered):
+        // https://www.unidata.ucar.edu/mailing_lists/archives/netcdf-java/2022/msg00026.html
     }
   }
 
@@ -14117,8 +14120,8 @@ void testQuickRestart() throws Throwable {
       expected = "There was a (temporary?) problem.  Wait a minute, then try again.  (In a browser, click the Reload button.)\n"
           +
           "(Cause: java.io.FileNotFoundException: " + dataDir
-          + "erdQSwind1day_20080101_03.nc.gz (The system cannot find the file specified))";
-      Test.ensureEqual(results, expected, "\nresults=\n" + results);
+          + "erdQSwind1day_20080101_03.nc.gz";
+      Test.ensureEqual(results.substring(0, expected.length()), expected, "\nresults=\n" + results);
 
     } finally {
       // rename file back to original

src/test/java/gov/noaa/pfel/erddap/dataset/EDDTableFromAsciiFilesTests.java

@@ -2538,7 +2538,7 @@ void testGenerateDatasetsXmlFromInPort2() throws Throwable {
         + "\" active=\"true\">\n"
         +
         "    <defaultGraphQuery>&amp;.marker=1|5</defaultGraphQuery>\n" +
-        "    <fileDir>" + dataDir + "/26938/</fileDir>\n" +
+        "    <fileDir>" + File2.addSlash(dataDir) + "26938/</fileDir>\n" +
         "    <fileNameRegex>???</fileNameRegex>\n" +
         "    <charset>ISO-8859-1</charset>\n" +
         "    <columnNamesRow>1</columnNamesRow>\n" +
@@ -2700,7 +2700,7 @@ void testGenerateDatasetsXmlFromInPort2() throws Throwable {
     expected = "<dataset type=\"EDDTableFromAsciiFiles\" datasetID=\"" + suggDatasetID + "\" active=\"true\">\n"
         +
         "    <defaultGraphQuery>&amp;.marker=1|5</defaultGraphQuery>\n" +
-        "    <fileDir>" + dataDir + "/26938/</fileDir>\n" +
+        "    <fileDir>" + File2.addSlash(dataDir) + "26938/</fileDir>\n" +
         "    <fileNameRegex>AFSC_RACE_FBEP_Hurst__Distributional_patterns_of_0-group_Pacific_cod__Gadus_macrocephalus__in_the_eastern_Bering_Sea_under_variable_recruitment_and_thermal_conditions\\.csv</fileNameRegex>\n"
         +
         "    <charset>ISO-8859-1</charset>\n" +
@@ -3696,7 +3696,7 @@ void testGenerateDatasetsXmlFromInPort() throws Throwable {
         + "\" active=\"true\">\n"
         +
         "    <defaultGraphQuery>&amp;.marker=1|5</defaultGraphQuery>\n" +
-        "    <fileDir>" + dataDir + "/27377/</fileDir>\n" +
+        "    <fileDir>" + File2.addSlash(dataDir) + "27377/</fileDir>\n" +
         "    <fileNameRegex>???</fileNameRegex>\n" +
         "    <charset>ISO-8859-1</charset>\n" +
         "    <columnNamesRow>1</columnNamesRow>\n" +
@@ -3908,7 +3908,7 @@ void testGenerateDatasetsXmlFromInPort() throws Throwable {
     expected = "<dataset type=\"EDDTableFromAsciiFiles\" datasetID=\"" + suggDatasetID + "\" active=\"true\">\n"
         +
         "    <defaultGraphQuery>&amp;.marker=1|5</defaultGraphQuery>\n" +
-        "    <fileDir>" + dataDir + "/27377/</fileDir>\n" +
+        "    <fileDir>" + File2.addSlash(dataDir) + "27377/</fileDir>\n" +
         "    <fileNameRegex>dummy\\.csv</fileNameRegex>\n" +
         "    <charset>ISO-8859-1</charset>\n" +
         "    <columnNamesRow>1</columnNamesRow>\n" +

src/test/java/gov/noaa/pfel/erddap/dataset/EDDTableFromAudioFilesTests.java

@@ -34,12 +34,12 @@ void testGenerateDatasetsXml() throws Throwable {
     boolean oEDDDebugMode = EDD.debugMode;
     // EDD.debugMode = true;
 
-    String dataDir = Path.of(EDDTableFromAudioFilesTests.class.getResource("/largeFiles/audio/wav/").toURI())
-        .toString();
-
+    String dataDir = File2.addSlash(Path.of(
+        EDDTableFromAudioFilesTests.class.getResource("/largeFiles/audio/wav/").toURI()).toString());
+    String fileNameRegex = ".*\\.wav";
     String results = EDDTableFromAudioFiles.generateDatasetsXml(
         dataDir, // test no trailing /
-        ".*\\.wav",
+        fileNameRegex,
         "",
         1440,
         "aco_acoustic\\.", "\\.wav", ".*", "time", "yyyyMMdd'_'HHmmss",
@@ -53,7 +53,7 @@ void testGenerateDatasetsXml() throws Throwable {
     String gdxResults = (new GenerateDatasetsXml()).doIt(new String[] { "-verbose",
         "EDDTableFromAudioFiles",
         dataDir,
-        ".*\\.wav",
+        fileNameRegex,
         "",
         "1440",
         "aco_acoustic\\.", "\\.wav", ".*", "time", "yyyyMMdd'_'HHmmss",
@@ -62,15 +62,15 @@ void testGenerateDatasetsXml() throws Throwable {
         false); // doIt loop?
     Test.ensureEqual(gdxResults, results, "Unexpected results from GenerateDatasetsXml.doIt.");
     String suggDatasetID = EDDTableFromAudioFiles.suggestDatasetID(
-        dataDir + "\\.*\\.wav");
+        dataDir + fileNameRegex);
     String expected = "<dataset type=\"EDDTableFromAudioFiles\" datasetID=\"" + suggDatasetID + "\" active=\"true\">\n" +
         "    <reloadEveryNMinutes>1440</reloadEveryNMinutes>\n" +
         "    <updateEveryNMillis>10000</updateEveryNMillis>\n" +
         "    <defaultGraphQuery>elapsedTime,channel_1&amp;time=min(time)&amp;elapsedTime&gt;=0&amp;elapsedTime&lt;=1&amp;.draw=lines</defaultGraphQuery>\n"
         +
         "    <defaultDataQuery>&amp;time=min(time)</defaultDataQuery>\n" +
-        "    <fileDir>" + dataDir + "\\</fileDir>\n" +
-        "    <fileNameRegex>.*\\.wav</fileNameRegex>\n" +
+        "    <fileDir>" + dataDir + "</fileDir>\n" +
+        "    <fileNameRegex>" + fileNameRegex + "</fileNameRegex>\n" +
         "    <recursive>true</recursive>\n" +
         "    <pathRegex>.*</pathRegex>\n" +
         "    <metadataFrom>last</metadataFrom>\n" +

src/test/java/gov/noaa/pfel/erddap/dataset/EDDTableFromAwsXmlFilesTests.java

@@ -32,9 +32,11 @@ void testGenerateDatasetsXml() throws Throwable {
     // testVerboseOn();
     Attributes externalAddAttributes = new Attributes();
     externalAddAttributes.add("title", "New Title!");
-    String dataDir = Path.of(EDDTableFromAwsXmlFilesTests.class.getResource("/data/aws/xml/").toURI()).toString();
+    String dataDir = File2.addSlash(Path.of(
+        EDDTableFromAwsXmlFilesTests.class.getResource("/data/aws/xml/").toURI()).toString());
+    String fileNameRegex = ".*\\.xml";
     String results = EDDTableFromAwsXmlFiles.generateDatasetsXml(
-        dataDir, ".*\\.xml", "",
+        dataDir, fileNameRegex, "",
         1, 2, 1440,
         "", "-.*$", ".*", "fileName", // just for test purposes; station is already a column in the file
         "ob-date", "station-id ob-date",
@@ -45,7 +47,7 @@ void testGenerateDatasetsXml() throws Throwable {
     // GenerateDatasetsXml
     String gdxResults = (new GenerateDatasetsXml()).doIt(new String[] { "-verbose",
         "EDDTableFromAwsXmlFiles",
-        dataDir, ".*\\.xml", "",
+        dataDir, fileNameRegex, "",
         "1", "2", "1440",
         "", "-.*$", ".*", "fileName", // just for test purposes; station is already a column in the file
         "ob-date", "station-id ob-date",
@@ -54,14 +56,14 @@ void testGenerateDatasetsXml() throws Throwable {
         false); // doIt loop?
     Test.ensureEqual(gdxResults, results, "Unexpected results from GenerateDatasetsXml.doIt.");
     String suggDatasetID = EDDTableFromAwsXmlFiles.suggestDatasetID(
-        dataDir + "\\.*\\.xml");
+        dataDir + fileNameRegex);
     String expected = "<!-- NOTE! Since the source files don't have any metadata, you must add metadata\n" +
         "  below, notably 'units' for each of the dataVariables. -->\n" +
         "<dataset type=\"EDDTableFromAwsXmlFiles\" datasetID=\"" + suggDatasetID + "\" active=\"true\">\n" +
         "    <reloadEveryNMinutes>1440</reloadEveryNMinutes>\n" +
         "    <updateEveryNMillis>10000</updateEveryNMillis>\n" +
-        "    <fileDir>" + dataDir + "\\</fileDir>\n" +
-        "    <fileNameRegex>.*\\.xml</fileNameRegex>\n" +
+        "    <fileDir>" + dataDir + "</fileDir>\n" +
+        "    <fileNameRegex>" + fileNameRegex + "</fileNameRegex>\n" +
         "    <recursive>true</recursive>\n" +
         "    <pathRegex>.*</pathRegex>\n" +
         "    <metadataFrom>last</metadataFrom>\n" +

src/test/java/gov/noaa/pfel/erddap/dataset/EDDTableFromHttpGetTests.java

@@ -5,6 +5,7 @@
 import java.util.HashSet;
 
 import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.io.TempDir;
 
 import com.cohort.array.Attributes;
 import com.cohort.array.IntArray;
@@ -27,6 +28,10 @@
 import testDataset.Initialization;
 
 class EDDTableFromHttpGetTests {
+
+  @TempDir
+  private static Path TEMP_DIR;
+
   @BeforeAll
   static void init() {
     Initialization.edStatic();
@@ -82,7 +87,7 @@ private static void testStatic(int hammer) throws Throwable {
         "");
 
     // set up
-    String startDir = "/data/httpGet/";
+    String startDir = TEMP_DIR.toAbsolutePath().toString() + "/";
     if (hammer < 0)
       File2.deleteAllFiles(startDir, true, true); // recursive, deleteEmptySubdirectories
     EDDTableFromHttpGet.parseHttpGetDirectoryStructure("stationID/2months", dsColumnName, dsN, dsCalendar);

src/test/java/gov/noaa/pfel/erddap/dataset/EDDTableFromInvalidCRAFilesTests.java

@@ -595,12 +595,13 @@ void testGenerateDatasetsXml() throws Throwable {
     // String tInfoUrl, String tInstitution, String tSummary, String tTitle,
     // Attributes externalAddGlobalAttributes) throws Throwable {
 
-    String dataDir = Path.of(EDDTableFromInvalidCRAFilesTests.class.getResource("/largeFiles/nccf/wod/").toURI())
-        .toString();
+    String dataDir = File2.addSlash(Path.of(
+        EDDTableFromInvalidCRAFilesTests.class.getResource("/largeFiles/nccf/wod/").toURI()).toString());
+    String fileNameRegex = "wod_drb_.*\\.nc";
 
     String results = EDDTableFromInvalidCRAFiles.generateDatasetsXml(
         dataDir,
-        "wod_drb_.*\\.nc",
+        fileNameRegex,
         "",
         1440,
         "", "", "", "", // just for test purposes; station is already a column in the file
@@ -613,7 +614,7 @@ void testGenerateDatasetsXml() throws Throwable {
     String gdxResults = (new GenerateDatasetsXml()).doIt(new String[] { "-verbose",
         "EDDTableFromInvalidCRAFiles",
         dataDir,
-        "wod_drb_.*\\.nc",
+        fileNameRegex,
         "",
         "1440",
         "", "", "", "", // just for test purposes; station is already a column in the file
@@ -623,16 +624,14 @@ void testGenerateDatasetsXml() throws Throwable {
         false); // doIt loop?
     Test.ensureEqual(gdxResults, results, "Unexpected results from GenerateDatasetsXml.doIt.");
 
-    String tDatasetID = EDDTableFromInvalidCRAFiles.suggestDatasetID(dataDir + "\\wod_drb_.*\\.nc");
+    String tDatasetID = EDDTableFromInvalidCRAFiles.suggestDatasetID(dataDir + fileNameRegex);
     String expected = "<dataset type=\"EDDTableFromInvalidCRAFiles\" datasetID=\"" + tDatasetID
         + "\" active=\"true\">\n"
         +
         "    <reloadEveryNMinutes>1440</reloadEveryNMinutes>\n" +
         "    <updateEveryNMillis>10000</updateEveryNMillis>\n" +
-        "    <fileDir>"
-        + dataDir
-        + "\\</fileDir>\n" +
-        "    <fileNameRegex>wod_drb_.*\\.nc</fileNameRegex>\n" +
+        "    <fileDir>" + dataDir + "</fileDir>\n" +
+        "    <fileNameRegex>" + fileNameRegex + "</fileNameRegex>\n" +
         "    <recursive>true</recursive>\n" +
         "    <pathRegex>.*</pathRegex>\n" +
         "    <metadataFrom>last</metadataFrom>\n" +