From f1f7a6adf096ad688b181fdc95b62d1a0283189f Mon Sep 17 00:00:00 2001 From: Gavin Date: Wed, 10 Apr 2024 16:41:00 -0700 Subject: [PATCH 1/5] Excise cruft Tests pass locally --- .codecov.yml | 30 --- COMMANDS | 0 DEPENDENCIES | 1 - LICENSE.md | 2 +- RELEASE_NOTES.txt | 254 ++++++++---------- deploy.cfg | 16 -- jenkins_instructions.txt | 11 - src/AllCommonTestsRunner.java | 76 ------ src/us/kbase/common/mongo/GetMongoDB.java | 236 ---------------- .../exceptions/InvalidHostException.java | 16 -- .../mongo/exceptions/MongoAuthException.java | 16 -- src/us/kbase/common/test/TestCommon.java | 1 - .../test/service/JsonTokenStreamTest.java | 1 - 13 files changed, 117 insertions(+), 543 deletions(-) delete mode 100644 .codecov.yml delete mode 100644 COMMANDS delete mode 100644 DEPENDENCIES delete mode 100644 deploy.cfg delete mode 100644 jenkins_instructions.txt delete mode 100644 src/AllCommonTestsRunner.java delete mode 100644 src/us/kbase/common/mongo/GetMongoDB.java delete mode 100644 src/us/kbase/common/mongo/exceptions/InvalidHostException.java delete mode 100644 src/us/kbase/common/mongo/exceptions/MongoAuthException.java diff --git a/.codecov.yml b/.codecov.yml deleted file mode 100644 index 42c0d62..0000000 --- a/.codecov.yml +++ /dev/null @@ -1,30 +0,0 @@ -codecov: - notify: - require_ci_to_pass: yes - -coverage: - precision: 2 - round: down - range: "70...100" - - status: - project: no - patch: yes - changes: no - -parsers: - gcov: - branch_detection: - conditional: yes - loop: yes - method: no - macro: no - -comment: - layout: "header, diff" - behavior: default - require_changes: no - -ignore: - - "build" - - "deployment" diff --git a/COMMANDS b/COMMANDS deleted file mode 100644 index e69de29..0000000 diff --git a/DEPENDENCIES b/DEPENDENCIES deleted file mode 100644 index c44896b..0000000 --- a/DEPENDENCIES +++ /dev/null @@ -1 +0,0 @@ -jars \ No newline at end of file diff --git a/LICENSE.md b/LICENSE.md index fb2bc8a..e5b3fb9 100644 --- a/LICENSE.md +++ b/LICENSE.md @@ -1,4 +1,4 @@ -Copyright (c) 2014 The KBase Project and its Contributors +Copyright (c) 2014-present The KBase Project and its Contributors Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: diff --git a/RELEASE_NOTES.txt b/RELEASE_NOTES.txt index d39f6c8..2016bfe 100644 --- a/RELEASE_NOTES.txt +++ b/RELEASE_NOTES.txt @@ -1,22 +1,23 @@ -JavaCommon repo +# JavaCommon repo + +## OVERVIEW -OVERVIEW ------------------------------------------ Repo for code shared between Java services. -VERSION: 0.2.0 (Release 2/7/24) ---------------------------------- +## VERSION: 0.3.0 (Release TBD) + +* The GetMongoDB class has been removed. + +## VERSION: 0.2.0 (Release 2/7/24) * MongoController is now compatible with Mongo versions 2 through 7. * The ShockController was removed. -VERSION: 0.1.1 (Release 12/21/22) ---------------------------------- +## VERSION: 0.1.1 (Release 12/21/22) * MongoController will retry once if it encounters an IOError when deleting test files. -VERSION: 0.1.0 (Release 11/12/19) ---------------------------------- +## VERSION: 0.1.0 (Release 11/12/19) * JsonServerServlet method dispatch to the Narrative Job Service (via *_async and *_check) methods has been removed. @@ -30,321 +31,298 @@ VERSION: 0.1.0 (Release 11/12/19) * Fixed several bugs with token stream processing in JsonTokenStream.java. * Updated Jackson jars to v2.9. -VERSION: 0.0.25 (Release 11/15/2018) ------------------------------------- -- The SDK logger now reads an environment variable, `KB_SDK_LOGGER_TARGET_PACKAGE`, +## VERSION: 0.0.25 (Release 11/15/2018) + +* The SDK logger now reads an environment variable, `KB_SDK_LOGGER_TARGET_PACKAGE`, that determines what classes are included in logging and how error traces are trimmed. This variable must be set to a parent package of the generated SDK code packages or undefined behavior may result. It defaults to `us.kbase`. -- JsonServerSyslog.findCaller() is now private. -- It is now permissible to pass null for the `configFileParam` in the JsonServerSyslog +* JsonServerSyslog.findCaller() is now private. +* It is now permissible to pass null for the `configFileParam` in the JsonServerSyslog constructor. In this case, no logging configuration will be loaded from an external file. -VERSION: 0.0.24 (Release 5/25/2017) ------------------------------------ +## VERSION: 0.0.24 (Release 5/25/2017) UPDATED FEATURES / MAJOR BUG FIXES: -- The shock controller now accepts and requires an auth url to be used for Globus lookups. This is +* The shock controller now accepts and requires an auth url to be used for Globus lookups. This is expected to be https://nexus.api.globusonline.org/ (soon to be deprecated) or https://(ci.|next.|appdev.|)kbase.us/services/auth/api/legacy/globus. -VERSION: 0.0.23 (Released 2/23/2017) --------------------------------------- +## VERSION: 0.0.23 (Released 2/23/2017) UPDATED FEATURES / MAJOR BUG FIXES: -- Some private methods in JsonServerServlet are made protected (necessary +* Some private methods in JsonServerServlet are made protected (necessary for coming switch to auth2 in kb-sdk and njs_wrapper). -- Old code was deleted (custom task queue support, AWE API client). +* Old code was deleted (custom task queue support, AWE API client). -VERSION: 0.0.22 (Released 9/15/2016) --------------------------------------- +## VERSION: 0.0.22 (Released 9/15/2016) UPDATED FEATURES / MAJOR BUG FIXES: -- Fixed a bug that would throw an IllegalStateError when configuration of the +* Fixed a bug that would throw an IllegalStateError when configuration of the authentication client in JsonServerServlet failed rather than logging the error. -VERSION: 0.0.21 (Released 8/26/2016) --------------------------------------- +## VERSION: 0.0.21 (Released 8/26/2016) NEW FEATURES: -- The Java servers now accept an 'auth-service-url-allow-insecure' parameter in +* The Java servers now accept an 'auth-service-url-allow-insecure' parameter in the deployment configuration file. Set to 'true' if using an http url rather than https for the auth service. -VERSION: 0.0.20 (Released 8/5/2016) --------------------------------------- +## VERSION: 0.0.20 (Released 8/5/2016) NEW FEATURES: -- None +* None UPDATED FEATURES / MAJOR BUG FIXES: -- Java servers now instantiate an auth client once on startup rather than +* Java servers now instantiate an auth client once on startup rather than every call (if an auth url is provided in the config). This prevents an extra call to the auth service for every server call. ANTICIPATED FUTURE DEVELOPMENTS: -- None +* None -VERSION: 0.0.19 (Released 8/3/2016) --------------------------------------- +## VERSION: 0.0.19 (Released 8/3/2016) NEW FEATURES: -- None +* None UPDATED FEATURES / MAJOR BUG FIXES: -- Java clients no longer validate a token before making a call. +* Java clients no longer validate a token before making a call. ANTICIPATED FUTURE DEVELOPMENTS: -- None +* None -VERSION: 0.0.18 (Released 7/31/2016) --------------------------------------- +## VERSION: 0.0.18 (Released 7/31/2016) NEW FEATURES: -- None +* None UPDATED FEATURES / MAJOR BUG FIXES: -- Updated java servers and clients to the latest auth library. +* Updated java servers and clients to the latest auth library. ANTICIPATED FUTURE DEVELOPMENTS: -- None +* None -VERSION: 0.0.17 (Released 6/14/2016) --------------------------------------- +## VERSION: 0.0.17 (Released 6/14/2016) NEW FEATURES: -- None +* None UPDATED FEATURES / MAJOR BUG FIXES: -- Support for dynamic Java client URL lookup is added to JsonClientCaller +* Support for dynamic Java client URL lookup is added to JsonClientCaller ANTICIPATED FUTURE DEVELOPMENTS: -- None +* None -VERSION: 0.0.16 (Released 6/13/2016) --------------------------------------- +## VERSION: 0.0.16 (Released 6/13/2016) NEW FEATURES: -- The JsonServerSyslog class now logs SLF4J messages. +* The JsonServerSyslog class now logs SLF4J messages. UPDATED FEATURES / MAJOR BUG FIXES: -- Fixed a bug where the server would stop logging after a syslog daemon +* Fixed a bug where the server would stop logging after a syslog daemon restart. -- removed the UTCDateFormat class (use joda library classes instead). -- removed UTF8Utils, unused. +* removed the UTCDateFormat class (use joda library classes instead). +* removed UTF8Utils, unused. ANTICIPATED FUTURE DEVELOPMENTS: -- None +* None -VERSION: 0.0.15 (Released 1/24/2015) --------------------------------------- +## VERSION: 0.0.15 (Released 1/24/2015) NEW FEATURES: -- None +* None UPDATED FEATURES / MAJOR BUG FIXES: -- Added workarounds to the ShockController for starting the server without +* Added workarounds to the ShockController for starting the server without human intervention and adding an admin that works on the first startup ANTICIPATED FUTURE DEVELOPMENTS: -- None +* None -VERSION: 0.0.14 (Released 1/4/2015) --------------------------------------- +## VERSION: 0.0.14 (Released 1/4/2015) NEW FEATURES: -- None +* None -- Made public static methods in JsonServerServlet for getting the UPDATED FEATURES / MAJOR BUG FIXES: + +* Made public static methods in JsonServerServlet for getting the server configuration, getting the IP address to log, and setting up response headers. ANTICIPATED FUTURE DEVELOPMENTS: -- None +* None -VERSION: 0.0.13 (Released 12/14/2015) --------------------------------------- +## VERSION: 0.0.13 (Released 12/14/2015) NEW FEATURES: -- Support for auto-generated provenance was added in JsonServerServlet. +* Support for auto-generated provenance was added in JsonServerServlet. UPDATED FEATURES / MAJOR BUG FIXES: -- None +* None ANTICIPATED FUTURE DEVELOPMENTS: -- None +* None -VERSION: 0.0.12 (Released 10/26/2015) --------------------------------------- +## VERSION: 0.0.12 (Released 10/26/2015) NEW FEATURES: -- N/A +* N/A UPDATED FEATURES / MAJOR BUG FIXES: -- Bug was fixed in Java Client Caller related to recent backward-incompatible changes +* Bug was fixed in Java Client Caller related to recent backward-incompatible changes in signatures of public methods used by old java generated JSON RPC clients. ANTICIPATED FUTURE DEVELOPMENTS: -- None +* None -VERSION: 0.0.11 (Released 10/21/2015) --------------------------------------- +## VERSION: 0.0.11 (Released 10/21/2015) NEW FEATURES: -- N/A +* N/A UPDATED FEATURES / MAJOR BUG FIXES: -- Java Server Servlet was redesigned in order to support async server calls +* Java Server Servlet was redesigned in order to support async server calls ANTICIPATED FUTURE DEVELOPMENTS: -- None +* None -VERSION: 0.0.10 (Released 4/10/2015) --------------------------------------- +## VERSION: 0.0.10 (Released 4/10/2015) NEW FEATURES: -- N/A +* N/A UPDATED FEATURES / MAJOR BUG FIXES: -- Add method to close all mongo connections to GetMongoDB class. +* Add method to close all mongo connections to GetMongoDB class. ANTICIPATED FUTURE DEVELOPMENTS: -- None +* None -VERSION: 0.0.9 (Released 3/27/2015) --------------------------------------- +## VERSION: 0.0.9 (Released 3/27/2015) NEW FEATURES: -- N/A +* N/A UPDATED FEATURES / MAJOR BUG FIXES: -- Changes was added into makefile in order to be compatible with AWE deployment process. -- Reference to logger was removed from GetMongoDB (possible cause of permgen errors with glassfish). -- Fix for a bug in tuple deserializer related to treating non-array data as null instead of throwing an exception. -- Fix for a bug in JsonServerServlet related to wrapping returned null by array in case of void methods. +* Changes was added into makefile in order to be compatible with AWE deployment process. +* Reference to logger was removed from GetMongoDB (possible cause of permgen errors with glassfish). +* Fix for a bug in tuple deserializer related to treating non-array data as null instead of throwing an exception. +* Fix for a bug in JsonServerServlet related to wrapping returned null by array in case of void methods. ANTICIPATED FUTURE DEVELOPMENTS: -- None +* None -VERSION: 0.0.8 (Released 11/10/2014) --------------------------------------- +## VERSION: 0.0.8 (Released 11/10/2014) NEW FEATURES: -- AWE task holder was added. +* AWE task holder was added. UPDATED FEATURES / MAJOR BUG FIXES: -- N/A +* N/A ANTICIPATED FUTURE DEVELOPMENTS: -- None +* None -VERSION: 0.0.7 (Released 10/20/2014) --------------------------------------- +## VERSION: 0.0.7 (Released 10/20/2014) NEW FEATURES: -- JsonClientCaller now allows configuring whether data should be streamed to +* JsonClientCaller now allows configuring whether data should be streamed to the server or not. Default is off, as many servers do not support this mode. UPDATED FEATURES / MAJOR BUG FIXES: -- N/A +* N/A ANTICIPATED FUTURE DEVELOPMENTS: -- None +* None -VERSION: 0.0.6 (Released 8/31/2014) --------------------------------------- +## VERSION: 0.0.6 (Released 8/31/2014) NEW FEATURES: -- The Java server parent class now handles X-Forwarded-For and X-Real-IP +* The Java server parent class now handles X-Forwarded-For and X-Real-IP headers. Add dont_trust_x_ip_headers=true to deploy.cfg to ignore them. UPDATED FEATURES / MAJOR BUG FIXES: -- N/A +* N/A ANTICIPATED FUTURE DEVELOPMENTS: -- None +* None -VERSION: 0.0.5 (Released 8/13/2014) --------------------------------------- +## VERSION: 0.0.5 (Released 8/13/2014) NEW FEATURES: -- test controllers for mongo, mysql, shock -- print out RPC error when writing to socket fails +* test controllers for mongo, mysql, shock +* print out RPC error when writing to socket fails UPDATED FEATURES / MAJOR BUG FIXES: -- N/A +* N/A ANTICIPATED FUTURE DEVELOPMENTS: -- None +* None -VERSION: 0.0.4 (Released 7/18/2014) --------------------------------------- +## VERSION: 0.0.4 (Released 7/18/2014) NEW FEATURES: -- minor string checking fn +* minor string checking fn UPDATED FEATURES / MAJOR BUG FIXES: -- N/A +* N/A ANTICIPATED FUTURE DEVELOPMENTS: -- None +* None -VERSION: 0.0.3 (Released 7/7/2014) --------------------------------------- +## VERSION: 0.0.3 (Released 7/7/2014) NEW FEATURES: -- Client code allows trusting unsigned certs. -- Configurable retries when establishing a connection to MongoDB. +* Client code allows trusting unsigned certs. +* Configurable retries when establishing a connection to MongoDB. UPDATED FEATURES / MAJOR BUG FIXES: -- N/A +* N/A ANTICIPATED FUTURE DEVELOPMENTS: -- None +* None -VERSION: 0.0.2 (Released 5/16/2014) --------------------------------------- +## VERSION: 0.0.2 (Released 5/16/2014) NEW FEATURES: -- Common files from the java type generator were moved here -- Added two JSON sorter implementations. Generally either faster than Jackson +* Common files from the java type generator were moved here +* Added two JSON sorter implementations. Generally either faster than Jackson or uses far less memory -- Various compbio utilities -- Add a simple AWE client (experimental) -- Add a task queue system (alpha, may be removed) +* Various compbio utilities +* Add a simple AWE client (experimental) +* Add a task queue system (alpha, may be removed) UPDATED FEATURES / MAJOR BUG FIXES: -- N/A +* N/A ANTICIPATED FUTURE DEVELOPMENTS: -- None +* None -VERSION: 0.0.1 (Released 2/24/2014) --------------------------------------- +## VERSION: 0.0.1 (Released 2/24/2014) NEW FEATURES: -- First release. +* First release. UPDATED FEATURES / MAJOR BUG FIXES: -- N/A +* N/A ANTICIPATED FUTURE DEVELOPMENTS: -- None \ No newline at end of file +* None \ No newline at end of file diff --git a/deploy.cfg b/deploy.cfg deleted file mode 100644 index 871ff9c..0000000 --- a/deploy.cfg +++ /dev/null @@ -1,16 +0,0 @@ -# -# The deploy.cfg file is located in the module top dir is a necessary file -# needed for deployments. This file contains config variables, such as -# mongodb-host, mysqldb-host. Standard variable names are listed on the trac -# wiki at https://trac.kbase.us/projects/kbase/wiki/DeployCfgStandards -# -# The variables in the deploy.cfg file will get "injected" into the module -# makefile. In short, the variable is uppercased and dashes become underscores, -# so mongodb-host in the deploy.cfg file becomes MONGODB_HOST in the module -# makefile when the module makefile is invoked by the system make file at the -# top level of the dev container. -# -# -# -# Deployment configuration for the java_common module -# diff --git a/jenkins_instructions.txt b/jenkins_instructions.txt deleted file mode 100644 index 1efd60c..0000000 --- a/jenkins_instructions.txt +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/bash -projdir=$(pwd) -cd /kb/dev_container -source user-env.sh -cd modules/jars -git pull -cd "$projdir" -export JENKINS_JARS_MODULE_PATH=/kb/dev_container/modules/jars -export JENKINS_JAVA_COMMON_JARFILE=java_commons_jenkins -export JENKINS_REPORT_DIR=junitreports -make jenkins test diff --git a/src/AllCommonTestsRunner.java b/src/AllCommonTestsRunner.java deleted file mode 100644 index 08dbf0b..0000000 --- a/src/AllCommonTestsRunner.java +++ /dev/null @@ -1,76 +0,0 @@ -import java.io.File; -import java.lang.reflect.Method; -import java.util.ArrayList; -import java.util.List; -import java.util.regex.Pattern; - -import junit.framework.JUnit4TestAdapter; -import junit.framework.TestSuite; - -import org.junit.Test; -import org.junit.runner.RunWith; -import org.junit.runners.AllTests; - -@RunWith(AllTests.class) -public class AllCommonTestsRunner { - private static boolean skipLongTests = false; - - public static TestSuite suite() throws Exception { - TestSuite suite = new TestSuite(); - for (Class classObj : collectTestClasses()) { - suite.addTest(new JUnit4TestAdapter(classObj)); - } - return suite; - } - - private static List> collectTestClasses() throws Exception { - List> ret = new ArrayList>(); - collectTestClasses(new File("src"), "", ret); - return ret; - } - - private static void collectTestClasses(File packageDir, String packagePrefix, List> ret) throws Exception { - for (File f : packageDir.listFiles()) { - if (f.isDirectory()) { - String subPackage = addSuffixFromFile(packagePrefix, f.getName()); - collectTestClasses(f, subPackage, ret); - } else if (f.isFile() && f.getName().endsWith(".java")) { - String nameWithoutExt = f.getName().substring(0, f.getName().length() - 5); - String className = addSuffixFromFile(packagePrefix, nameWithoutExt); - Class classObj = Class.forName(className); - if (isTestPackage(packagePrefix)) { - if (!isJUnitTestClass(classObj)) - continue; - if (nameWithoutExt.endsWith("Test")) { - if (skipLongTests && nameWithoutExt.endsWith("LongTest")) - continue; - ret.add(classObj); - } - } - } - } - } - - private static String addSuffixFromFile(String packagePrefix, String fileName) { - String subPackage = packagePrefix; - if (subPackage.length() > 0) - subPackage += "."; - subPackage += fileName; - return subPackage; - } - - public static boolean isJUnitTestClass(Class classObj) { - for (Method m : classObj.getMethods()) - if (m.getAnnotation(Test.class) != null) - return true; - return false; - } - - private static boolean isTestPackage(String packageName) throws Exception { - String[] parts = packageName.split(Pattern.quote(".")); - for (String part : parts) - if (part.equals("test")) - return true; - return false; - } -} diff --git a/src/us/kbase/common/mongo/GetMongoDB.java b/src/us/kbase/common/mongo/GetMongoDB.java deleted file mode 100644 index aedce10..0000000 --- a/src/us/kbase/common/mongo/GetMongoDB.java +++ /dev/null @@ -1,236 +0,0 @@ -package us.kbase.common.mongo; - -import java.io.IOException; -import java.net.UnknownHostException; -import java.util.HashMap; -import java.util.Map; -import java.util.logging.Level; - -import us.kbase.common.mongo.exceptions.MongoAuthException; -import us.kbase.common.mongo.exceptions.InvalidHostException; - -import com.mongodb.DB; -import com.mongodb.MongoClient; -import com.mongodb.MongoClientOptions; -import com.mongodb.MongoException; -import com.mongodb.MongoTimeoutException; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * Convenience methods for getting Mongo DB instances. - * - * Does the following: - * 1) ensures there's only one MongoClient per host. Per the Mongo docs, - * there should only be one MongoClient per process (presumably per host - * [multiple host applications would be very weird, but the capability is - * available]). - * 2) turns off the very annoying com.mongodb logger. - * 3) sets autoConnectRetry to true. - * 4) makes the exceptions thrown by the client initialization easier to deal - * with. - * - * @author gaprice@lbl.gov - * @deprecated - * - */ -public class GetMongoDB { - - //TODO remove. Uses incredibly old mongo client. - - private static Logger getLogger() { - return LoggerFactory.getLogger(GetMongoDB.class); - } - - private static final Map HOST_TO_CLIENT = - new HashMap(); - - /** @deprecated */ - public synchronized static void closeAllConnections() { - for (final String host: HOST_TO_CLIENT.keySet()) { - HOST_TO_CLIENT.get(host).close(); - } - HOST_TO_CLIENT.clear(); - } - - private synchronized static MongoClient getMongoClient(final String host) - throws UnknownHostException, InvalidHostException { - //Only make one instance of MongoClient per JVM per mongo docs - final MongoClient client; - if (!HOST_TO_CLIENT.containsKey(host)) { - // Don't print to stderr - java.util.logging.Logger.getLogger("com.mongodb") - .setLevel(Level.OFF); - final MongoClientOptions opts = MongoClientOptions.builder() - .autoConnectRetry(true).build(); - try { - client = new MongoClient(host, opts); - } catch (NumberFormatException nfe) { - //throw a better exception if 10gen ever fixes this - throw new InvalidHostException(host - + " is not a valid mongodb host"); - } - HOST_TO_CLIENT.put(host, client); - } else { - client = HOST_TO_CLIENT.get(host); - } - return client; - } - - /** - * Gets a database from a MongoDB instance without authentication. - * Retries 600s times (e.g. 10m), logs every 10 retries. - * @param host the MongoDB host address. - * @param database the database to get. - * @return the MongoDB database instance. - * @throws UnknownHostException if the host is unknown - * @throws InvalidHostException if the host is an invalid Mongo address - * @throws IOException if an IO exception occurs - * @throws InterruptedException if the thread is interrupted while it's - * sleeping between connection attempts - * @deprecated - */ - public static DB getDB(final String host, final String database) - throws UnknownHostException, InvalidHostException, IOException, - InterruptedException { - return getDB(host, database, 600, 10); - } - - /** - * Gets a database from a MongoDB instance without authentication. - * @param host the MongoDB host address. - * @param database the database to get. - * @param retryCount - the number of times to retry the MongoDB - * connection, 1 retry / sec, not including the time retrying takes. - * @param logIntervalCount - how often to log the retries. Logs occur when - * retries % logIntervalCount = 0. - * @return the MongoDB database instance. - * @throws UnknownHostException if the host is unknown - * @throws InvalidHostException if the host is an invalid Mongo address - * @throws IOException if an IO exception occurs - * @throws InterruptedException if the thread is interrupted while it's - * sleeping between connection attempts - * @deprecated - */ - public static DB getDB(final String host, final String database, - final int retryCount, final int logIntervalCount) - throws UnknownHostException, InvalidHostException, IOException, - InterruptedException { - if (database == null || database.isEmpty()) { - throw new IllegalArgumentException( - "database may not be null or the empty string"); - } - final DB db = getMongoClient(host).getDB(database); - int retries = 0; - while (true) { - try { - db.getCollectionNames(); - break; - } catch (MongoException.Network | MongoTimeoutException e ) { - if (retries >= retryCount) { - if (e instanceof MongoException.Network) { - throw (IOException) e.getCause(); - } - throw e; - } - if (retries % logIntervalCount == 0) { - getLogger().info( - "Retrying MongoDB connection {}/{}, attempt {}/{}", - host, database, retries, retryCount); - } - Thread.sleep(1000); - } - retries++; - } - return db; - } - - /** - * Gets a database from a MongoDB instance with authentication. - * Retries 600s times (e.g. 10m), logs every 10 retries. - * @param host the MongoDB host address. - * @param database the database to get. - * @param user the MongoDB user with access to the database. - * @param pwd the MongoDB user's password. - * @return the MongoDB database instance. - * @throws UnknownHostException if the host is unknown - * @throws InvalidHostException if the host is an invalid Mongo address - * @throws IOException if an IO exception occurs - * @throws MongoAuthException if the provided credentials are incorrect. - * @throws InterruptedException if the thread is interrupted while it's - * sleeping between connection attempts - * @deprecated - */ - public static DB getDB(final String host, final String database, - final String user, final String pwd) - throws UnknownHostException, InvalidHostException, IOException, - MongoAuthException, InterruptedException { - return getDB(host, database, user, pwd, 600, 10); - } - - /** - * Gets a database from a MongoDB instance with authentication. - * @param host the MongoDB host address. - * @param database the database to get. - * @param user the MongoDB user with access to the database. - * @param pwd the MongoDB user's password. - * @param retryCount - the number of times to retry the MongoDB - * connection, 1 retry / sec, not including the time retrying takes. - * @param logIntervalCount - how often to log the retries. Logs occur when - * retries % logIntervalCount = 0. - * @return the MongoDB database instance. - * @throws UnknownHostException if the host is unknown - * @throws InvalidHostException if the host is an invalid Mongo address - * @throws IOException if an IO exception occurs - * @throws MongoAuthException if the provided credentials are incorrect. - * @throws InterruptedException if the thread is interrupted while it's - * sleeping between connection attempts - * @deprecated - */ - public static DB getDB(final String host, final String database, - final String user, final String pwd, - final int retryCount, final int logIntervalCount) - throws UnknownHostException, InvalidHostException, IOException, - MongoAuthException, InterruptedException { - if (database == null || database.isEmpty()) { - throw new IllegalArgumentException( - "database may not be null or the empty string"); - } - final DB db = getMongoClient(host).getDB(database); - int retries = 0; - while (true) { - try { - db.authenticate(user, pwd.toCharArray()); - break; - } catch (MongoException.Network | MongoTimeoutException e) { - if (retries >= retryCount) { - if (e instanceof MongoException.Network) { - throw (IOException) e.getCause(); - } - throw e; - } - if (retries % logIntervalCount == 0) { - getLogger().info( - "Retrying MongoDB connection {}/{}, attempt {}/{}", - host, database, retries, retryCount); - } - Thread.sleep(1000); - } - retries++; - } - try { - db.getCollectionNames(); - } catch (MongoException me) { - throw new MongoAuthException("Not authorized for database " - + database, me); - } - return db; - } - - public static void main(String[] args) throws Exception { - //TODO require mongo for tests or change to a different module or something - System.out.println(getDB("localhost", "ws_test", "ws", "foo")); - } - -} diff --git a/src/us/kbase/common/mongo/exceptions/InvalidHostException.java b/src/us/kbase/common/mongo/exceptions/InvalidHostException.java deleted file mode 100644 index 912821f..0000000 --- a/src/us/kbase/common/mongo/exceptions/InvalidHostException.java +++ /dev/null @@ -1,16 +0,0 @@ -package us.kbase.common.mongo.exceptions; - -/** - * Thrown when the host provided for a mongo database is not valid. - * @author gaprice@lbl.gov - * - */ -public class InvalidHostException extends Exception { - - private static final long serialVersionUID = 1L; - - public InvalidHostException() { super(); } - public InvalidHostException(String message) { super(message); } - public InvalidHostException(String message, Throwable cause) { super(message, cause); } - public InvalidHostException(Throwable cause) { super(cause); } -} diff --git a/src/us/kbase/common/mongo/exceptions/MongoAuthException.java b/src/us/kbase/common/mongo/exceptions/MongoAuthException.java deleted file mode 100644 index 7116cb8..0000000 --- a/src/us/kbase/common/mongo/exceptions/MongoAuthException.java +++ /dev/null @@ -1,16 +0,0 @@ -package us.kbase.common.mongo.exceptions; - -/** - * Thrown when authorization to a mongo database is denied. - * @author gaprice@lbl.gov - * - */ -public class MongoAuthException extends Exception { - - private static final long serialVersionUID = 1L; - - public MongoAuthException() { super(); } - public MongoAuthException(String message) { super(message); } - public MongoAuthException(String message, Throwable cause) { super(message, cause); } - public MongoAuthException(Throwable cause) { super(cause); } -} diff --git a/src/us/kbase/common/test/TestCommon.java b/src/us/kbase/common/test/TestCommon.java index becc513..1bb5334 100644 --- a/src/us/kbase/common/test/TestCommon.java +++ b/src/us/kbase/common/test/TestCommon.java @@ -31,7 +31,6 @@ import ch.qos.logback.classic.spi.ILoggingEvent; import ch.qos.logback.classic.spi.IThrowableProxy; import ch.qos.logback.core.AppenderBase; -import us.kbase.common.test.TestException; public class TestCommon { diff --git a/src/us/kbase/common/test/service/JsonTokenStreamTest.java b/src/us/kbase/common/test/service/JsonTokenStreamTest.java index 438ee9c..e6b4b53 100644 --- a/src/us/kbase/common/test/service/JsonTokenStreamTest.java +++ b/src/us/kbase/common/test/service/JsonTokenStreamTest.java @@ -559,7 +559,6 @@ private void checkStreamingWithMultiByteChars(Object data, int buffersize, Strin public void detectEncoding() throws Exception { for (String data: basicJsonData) { for (Charset enc: encodings) { - @SuppressWarnings("resource") JsonTokenStream jts = new JsonTokenStream(data.getBytes(enc)); assertThat("encoding correctly detected", jts.getEncoding(), is(enc)); From 656e36b224c1981aee14d20900389d9144ee3e12 Mon Sep 17 00:00:00 2001 From: Gavin Date: Wed, 10 Apr 2024 17:10:20 -0700 Subject: [PATCH 2/5] Add GHA Also move performance code out of main code area. Ant ignores it anyway --- .github/codeql.yml | 62 +++++++++++++++++++ .github/dependabot.yml | 17 +++++ .github/workflows/test.yml | 58 +++++++++++++++++ .../performance/PerformanceMeasurement.java | 0 .../common/performance/sortjson/83333.2.txt | 0 .../sortjson/MeasureSortJsonMem.java | 0 .../sortjson/MeasureSortJsonSpeed.java | 0 .../sortjson/MeasureSortRunner.java | 0 8 files changed, 137 insertions(+) create mode 100644 .github/codeql.yml create mode 100644 .github/dependabot.yml create mode 100644 .github/workflows/test.yml rename {src => performance}/us/kbase/common/performance/PerformanceMeasurement.java (100%) rename {src => performance}/us/kbase/common/performance/sortjson/83333.2.txt (100%) rename {src => performance}/us/kbase/common/performance/sortjson/MeasureSortJsonMem.java (100%) rename {src => performance}/us/kbase/common/performance/sortjson/MeasureSortJsonSpeed.java (100%) rename {src => performance}/us/kbase/common/performance/sortjson/MeasureSortRunner.java (100%) diff --git a/.github/codeql.yml b/.github/codeql.yml new file mode 100644 index 0000000..b4ae59c --- /dev/null +++ b/.github/codeql.yml @@ -0,0 +1,62 @@ +name: "Code scanning - action" + +on: + pull_request: + types: + - opened + - reopened + - synchronize + - ready_for_review + push: + # run workflow when merging to main/master or develop + branches: + - main + - master + - develop + schedule: + - cron: '0 19 * * 0' + +jobs: + CodeQL-Build: + + # CodeQL runs on ubuntu-latest and windows-latest + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v2 + with: + # We must fetch at least the immediate parents so that if this is + # a pull request then we can checkout the head. + fetch-depth: 2 + + # If this run was triggered by a pull request event, then checkout + # the head of the pull request instead of the merge commit. + - run: git checkout HEAD^2 + if: ${{ github.event_name == 'pull_request' }} + + # Initializes the CodeQL tools for scanning. + - name: Initialize CodeQL + uses: github/codeql-action/init@v1 + # Override language selection by uncommenting this and choosing your languages + # with: + # languages: go, javascript, csharp, python, cpp, java + + # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). + # If this step fails, then you should remove it and run the build manually (see below) + - name: Autobuild + uses: github/codeql-action/autobuild@v1 + + # ℹī¸ Command-line programs to run using the OS shell. + # 📚 https://git.io/JvXDl + + # ✏ī¸ If the Autobuild fails above, remove it and uncomment the following three lines + # and modify them (or add more) to build your code if your project + # uses a compiled language + + #- run: | + # make bootstrap + # make release + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v1 diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..f506fa4 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,17 @@ +version: 2 +updates: + + # Docker + - package-ecosystem: docker + directory: "/" + schedule: + interval: weekly + time: '11:00' + open-pull-requests-limit: 25 + + # GitHub Actions + - package-ecosystem: "github-actions" + directory: ".github/workflows" + schedule: + interval: "monthly" + open-pull-requests-limit: 25 diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..d822752 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,58 @@ +name: KBase Java Common tests + +on: + pull_request: + types: + - opened + - reopened + - synchronize + - ready_for_review + push: + # run workflow when merging to main / master / develop + branches: + - main + - master + - develop + +jobs: + + java_common_tests: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + include: + - java: '8' + - java: '11' + + steps: + - uses: actions/checkout@v4 + + - name: Setup java + uses: actions/setup-java@v4 + with: + distribution: 'temurin' + java-version: ${{matrix.java}} + + - name: Install dependencies and set up test config + shell: bash + run: | + export HOMEDIR=`pwd` + + # move to parent dir of homedir to install binaries etc + cd .. + + # set up jars + git clone https://github.com/kbase/jars + + cd $HOMEDIR + + - name: Run tests + run: | + make test + + - name: Upload coverage to Codecov + uses: codecov/codecov-action@v3 + with: + token: ${{ secrets.CODECOV_TOKEN } + fail_ci_if_error: true diff --git a/src/us/kbase/common/performance/PerformanceMeasurement.java b/performance/us/kbase/common/performance/PerformanceMeasurement.java similarity index 100% rename from src/us/kbase/common/performance/PerformanceMeasurement.java rename to performance/us/kbase/common/performance/PerformanceMeasurement.java diff --git a/src/us/kbase/common/performance/sortjson/83333.2.txt b/performance/us/kbase/common/performance/sortjson/83333.2.txt similarity index 100% rename from src/us/kbase/common/performance/sortjson/83333.2.txt rename to performance/us/kbase/common/performance/sortjson/83333.2.txt diff --git a/src/us/kbase/common/performance/sortjson/MeasureSortJsonMem.java b/performance/us/kbase/common/performance/sortjson/MeasureSortJsonMem.java similarity index 100% rename from src/us/kbase/common/performance/sortjson/MeasureSortJsonMem.java rename to performance/us/kbase/common/performance/sortjson/MeasureSortJsonMem.java diff --git a/src/us/kbase/common/performance/sortjson/MeasureSortJsonSpeed.java b/performance/us/kbase/common/performance/sortjson/MeasureSortJsonSpeed.java similarity index 100% rename from src/us/kbase/common/performance/sortjson/MeasureSortJsonSpeed.java rename to performance/us/kbase/common/performance/sortjson/MeasureSortJsonSpeed.java diff --git a/src/us/kbase/common/performance/sortjson/MeasureSortRunner.java b/performance/us/kbase/common/performance/sortjson/MeasureSortRunner.java similarity index 100% rename from src/us/kbase/common/performance/sortjson/MeasureSortRunner.java rename to performance/us/kbase/common/performance/sortjson/MeasureSortRunner.java From 8773e2a5d98c1bf80e65f35630c571dca041c69d Mon Sep 17 00:00:00 2001 From: Gavin Date: Wed, 10 Apr 2024 17:12:27 -0700 Subject: [PATCH 3/5] Fix test.yml --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index d822752..a39f7a3 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -54,5 +54,5 @@ jobs: - name: Upload coverage to Codecov uses: codecov/codecov-action@v3 with: - token: ${{ secrets.CODECOV_TOKEN } + token: ${{ secrets.CODECOV_TOKEN }} fail_ci_if_error: true From 4e6e68039f3ef7014a187a6cc0ee7ed4991ec77d Mon Sep 17 00:00:00 2001 From: Gavin Date: Wed, 10 Apr 2024 17:17:30 -0700 Subject: [PATCH 4/5] Fix coverage --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index a39f7a3..f31ba61 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -49,7 +49,7 @@ jobs: - name: Run tests run: | - make test + make test-coverage - name: Upload coverage to Codecov uses: codecov/codecov-action@v3 From 5b0cae2ad99d3faae6193d62a891066eec05a668 Mon Sep 17 00:00:00 2001 From: Gavin Date: Wed, 10 Apr 2024 17:32:08 -0700 Subject: [PATCH 5/5] Delete travis.yml --- .travis.yml | 21 --------------------- 1 file changed, 21 deletions(-) delete mode 100644 .travis.yml diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index f7b6ac2..0000000 --- a/.travis.yml +++ /dev/null @@ -1,21 +0,0 @@ -dist: trusty -sudo: required -language: java -jdk: - - openjdk8 - - openjdk11 - -install: - - cd .. - - git clone https://github.com/kbase/jars - - export JARSDIR=`pwd`/jars/lib/jars/ - - cd - - -script: - # - ant javadoc javadoc is a mess. Pointless testing for now. - - make test-coverage - -after_success: - - ls test-reports - - bash <(curl -s https://codecov.io/bash) -f test-reports/coverage-report.xml -