From 0f1c0fa16fa94f3f04fe5d0f98a2a0e010fa93ff Mon Sep 17 00:00:00 2001 From: Sam Stouffer Date: Wed, 25 Sep 2019 11:41:23 -0400 Subject: [PATCH] Upgrade apache commons libraries. Add commons-text, the new home for text based functions from commons-lang. Update/replace usages of deprecated methods. --- pom.xml | 15 ++++++++--- .../client/view/bill/BillStatusView.java | 2 +- .../view/committee/CommitteeIdView.java | 2 +- .../client/view/updates/UpdateDigestView.java | 6 +---- .../openleg/config/WebSocketsConfig.java | 5 ++-- .../api/admin/AdminAccountCtrl.java | 12 +++------ .../controller/api/admin/EnvironmentCtrl.java | 2 +- .../openleg/JsonOpenlegAgendaDao.java | 6 ++--- .../nysenate/openleg/dao/base/SqlBaseDao.java | 4 +-- .../openleg/dao/base/SqlQueryUtils.java | 6 ++--- .../openleg/dao/bill/data/SqlBillDao.java | 4 +-- .../reference/openleg/JsonOpenlegBillDao.java | 4 +-- .../reference/senatesite/FsSenateSiteDao.java | 8 +++--- .../scrape/SqlFsBillScrapeReferenceDao.java | 4 +-- .../openleg/JsonOpenlegCalendarDao.java | 4 +-- .../openleg/dao/scraping/LRSScraper.java | 3 ++- .../dao/updates/SqlAggregateUpdatesQuery.java | 4 +-- .../openleg/model/base/PublishStatus.java | 4 +-- .../spotcheck/billscrape/BillScrapeVote.java | 2 +- .../spotcheck/daybreak/DaybreakFile.java | 3 ++- .../hearing/PublicHearingParser.java | 4 ++- .../openleg/processor/law/LawTitleParser.java | 2 +- .../CalendarAlertActiveListParser.java | 5 ++-- .../CalendarAlertSupplementalParser.java | 11 +++----- .../processor/transcript/TranscriptLine.java | 2 +- .../nysenate/openleg/script/BaseScript.java | 2 +- .../nysenate/openleg/script/LRSScraper.java | 15 ++++++----- .../service/auth/CachedSqlApiUserService.java | 15 +++++------ .../bill/BillScrapeReferenceFactory.java | 3 ++- .../service/scraping/bill/BillScraper.java | 7 ++--- .../openleg/service/slack/SlackApi.java | 27 +++++++++++-------- .../spotcheck/base/BaseCheckMailService.java | 3 ++- .../daybreak/DaybreakCheckService.java | 2 +- .../nysenate/openleg/util/FileIOUtils.java | 6 ++--- .../processor/BaseXmlProcessorTest.java | 3 ++- .../hearing/PublicHearingTestHelper.java | 4 +-- .../openleg/processor/legdata/XmlJsonIT.java | 3 ++- .../transcript/TranscriptLineTest.java | 2 +- .../openleg/script/MemberProcessorTest.java | 15 ++++++----- 39 files changed, 120 insertions(+), 111 deletions(-) diff --git a/pom.xml b/pom.xml index 875a53952..13092192f 100644 --- a/pom.xml +++ b/pom.xml @@ -104,9 +104,11 @@ 0.9.5.3 - 1.2 - 2.4 - 3.3.2 + 1.4 + 2.6 + 3.9 + 1.8 + 1.6 2.8.3 7.0.0 9.1.16 @@ -339,9 +341,14 @@ commons-validator commons-validator - 1.6 + ${commons-validator.version} + + org.apache.commons + commons-text + ${commons-text.version} + diff --git a/src/main/java/gov/nysenate/openleg/client/view/bill/BillStatusView.java b/src/main/java/gov/nysenate/openleg/client/view/bill/BillStatusView.java index 66f416197..7f381bbb3 100644 --- a/src/main/java/gov/nysenate/openleg/client/view/bill/BillStatusView.java +++ b/src/main/java/gov/nysenate/openleg/client/view/bill/BillStatusView.java @@ -4,7 +4,7 @@ import com.google.common.base.Objects; import gov.nysenate.openleg.client.view.base.ViewObject; import gov.nysenate.openleg.model.bill.BillStatus; -import org.apache.commons.lang3.text.WordUtils; +import org.apache.commons.text.WordUtils; import java.time.LocalDate; public class BillStatusView implements ViewObject diff --git a/src/main/java/gov/nysenate/openleg/client/view/committee/CommitteeIdView.java b/src/main/java/gov/nysenate/openleg/client/view/committee/CommitteeIdView.java index f90fe5a56..eba6433b7 100644 --- a/src/main/java/gov/nysenate/openleg/client/view/committee/CommitteeIdView.java +++ b/src/main/java/gov/nysenate/openleg/client/view/committee/CommitteeIdView.java @@ -2,7 +2,7 @@ import gov.nysenate.openleg.client.view.base.ViewObject; import gov.nysenate.openleg.model.entity.CommitteeId; -import org.apache.commons.lang3.text.WordUtils; +import org.apache.commons.text.WordUtils; public class CommitteeIdView implements ViewObject { diff --git a/src/main/java/gov/nysenate/openleg/client/view/updates/UpdateDigestView.java b/src/main/java/gov/nysenate/openleg/client/view/updates/UpdateDigestView.java index b5a8b6a4e..71d042471 100644 --- a/src/main/java/gov/nysenate/openleg/client/view/updates/UpdateDigestView.java +++ b/src/main/java/gov/nysenate/openleg/client/view/updates/UpdateDigestView.java @@ -2,12 +2,8 @@ import gov.nysenate.openleg.client.view.base.ViewObject; import gov.nysenate.openleg.model.updates.UpdateDigest; -import gov.nysenate.openleg.util.DateUtils; -import org.apache.commons.lang3.text.WordUtils; +import org.apache.commons.text.WordUtils; -import java.time.LocalDateTime; -import java.time.format.DateTimeFormatter; -import java.time.format.DateTimeParseException; import java.util.HashMap; import java.util.Map; diff --git a/src/main/java/gov/nysenate/openleg/config/WebSocketsConfig.java b/src/main/java/gov/nysenate/openleg/config/WebSocketsConfig.java index 71265a37a..f2fb2d6a2 100644 --- a/src/main/java/gov/nysenate/openleg/config/WebSocketsConfig.java +++ b/src/main/java/gov/nysenate/openleg/config/WebSocketsConfig.java @@ -3,17 +3,16 @@ import org.springframework.context.annotation.Configuration; import org.springframework.messaging.simp.config.MessageBrokerRegistry; -import org.springframework.web.socket.config.annotation.AbstractWebSocketMessageBrokerConfigurer; import org.springframework.web.socket.config.annotation.EnableWebSocketMessageBroker; import org.springframework.web.socket.config.annotation.StompEndpointRegistry; +import org.springframework.web.socket.config.annotation.WebSocketMessageBrokerConfigurer; @Configuration @EnableWebSocketMessageBroker -public class WebSocketsConfig extends AbstractWebSocketMessageBrokerConfigurer +public class WebSocketsConfig implements WebSocketMessageBrokerConfigurer { @Override public void configureMessageBroker(MessageBrokerRegistry registry) { - super.configureMessageBroker(registry); registry.enableSimpleBroker("/event"); registry.setApplicationDestinationPrefixes("/app"); } diff --git a/src/main/java/gov/nysenate/openleg/controller/api/admin/AdminAccountCtrl.java b/src/main/java/gov/nysenate/openleg/controller/api/admin/AdminAccountCtrl.java index 3cfe737a8..57ef93e76 100644 --- a/src/main/java/gov/nysenate/openleg/controller/api/admin/AdminAccountCtrl.java +++ b/src/main/java/gov/nysenate/openleg/controller/api/admin/AdminAccountCtrl.java @@ -2,7 +2,6 @@ import com.google.common.collect.ImmutableMap; import gov.nysenate.openleg.client.response.base.BaseResponse; -import gov.nysenate.openleg.client.response.base.ListViewResponse; import gov.nysenate.openleg.client.response.base.SimpleResponse; import gov.nysenate.openleg.client.response.base.ViewObjectResponse; import gov.nysenate.openleg.client.response.error.ErrorCode; @@ -16,17 +15,12 @@ import gov.nysenate.openleg.model.auth.AdminUser; import gov.nysenate.openleg.service.auth.AdminUserService; import gov.nysenate.openleg.service.auth.InvalidUsernameException; -import gov.nysenate.openleg.service.auth.OpenLegRole; import gov.nysenate.openleg.service.mail.SendMailService; import gov.nysenate.openleg.util.RandomUtils; import org.apache.commons.lang3.StringUtils; -import org.apache.commons.lang3.text.StrSubstitutor; +import org.apache.commons.text.StringSubstitutor; import org.apache.shiro.SecurityUtils; -import org.apache.shiro.authc.UnknownAccountException; -import org.apache.shiro.authz.UnauthenticatedException; -import org.apache.shiro.authz.annotation.RequiresAuthentication; import org.apache.shiro.authz.annotation.RequiresPermissions; -import org.apache.shiro.authz.annotation.RequiresRoles; import org.apache.shiro.authz.annotation.RequiresUser; import org.mindrot.jbcrypt.BCrypt; import org.slf4j.Logger; @@ -216,8 +210,8 @@ private String getSubjectUsername() { * @param password the password of the new user */ @Async - private void sendNewUserEmail(String username, String password) { - String message = StrSubstitutor.replace(registrationEmailTemplate, + protected void sendNewUserEmail(String username, String password) { + String message = StringSubstitutor.replace(registrationEmailTemplate, ImmutableMap.of("username", username, "password", password, "base_url", environment.getUrl())); sendMailService.sendMessage(username, registrationEmailSubject, message); } diff --git a/src/main/java/gov/nysenate/openleg/controller/api/admin/EnvironmentCtrl.java b/src/main/java/gov/nysenate/openleg/controller/api/admin/EnvironmentCtrl.java index cf916ca97..6bf666d61 100644 --- a/src/main/java/gov/nysenate/openleg/controller/api/admin/EnvironmentCtrl.java +++ b/src/main/java/gov/nysenate/openleg/controller/api/admin/EnvironmentCtrl.java @@ -119,7 +119,7 @@ public BaseResponse getVariables(@RequestParam(defaultValue = "") String[] varNa variables.stream() .filter(var -> !mutableOnly || var.isMutable()) .sorted((a, b) -> ComparisonChain.start() - .compare(a.isMutable(), b.isMutable()) + .compareFalseFirst(a.isMutable(), b.isMutable()) .compare(a.getName(), b.getName()) .result()) .collect(Collectors.toList()))); diff --git a/src/main/java/gov/nysenate/openleg/dao/agenda/reference/openleg/JsonOpenlegAgendaDao.java b/src/main/java/gov/nysenate/openleg/dao/agenda/reference/openleg/JsonOpenlegAgendaDao.java index bc4ba3369..fbb944d18 100644 --- a/src/main/java/gov/nysenate/openleg/dao/agenda/reference/openleg/JsonOpenlegAgendaDao.java +++ b/src/main/java/gov/nysenate/openleg/dao/agenda/reference/openleg/JsonOpenlegAgendaDao.java @@ -8,7 +8,7 @@ import gov.nysenate.openleg.dao.base.PaginatedList; import gov.nysenate.openleg.model.agenda.AgendaId; import gov.nysenate.openleg.service.spotcheck.openleg.JsonOpenlegDaoUtils; -import org.apache.commons.lang3.text.StrSubstitutor; +import org.apache.commons.text.StringSubstitutor; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; @@ -37,7 +37,7 @@ public JsonOpenlegAgendaDao(JsonOpenlegDaoUtils jsonOpenlegDaoUtils) { public List getAgendaViews(int year) { final String agendasForYearUri = - StrSubstitutor.replace(agendasForYearUriTemplate, ImmutableMap.of("year", year)); + StringSubstitutor.replace(agendasForYearUriTemplate, ImmutableMap.of("year", year)); PaginatedList agendaSummaries = jsonOpenlegDaoUtils.queryForViewObjects(AgendaSummaryView.class, agendasForYearUri, LimitOffset.ALL); @@ -54,7 +54,7 @@ public AgendaView getAgendaView(AgendaId agendaId) { "year", agendaId.getYear(), "agendaNo", agendaId.getNumber() ); - final String agendaUri = StrSubstitutor.replace(getAgendaUriTemplate, subMap); + final String agendaUri = StringSubstitutor.replace(getAgendaUriTemplate, subMap); return jsonOpenlegDaoUtils.queryForViewObject(AgendaView.class, agendaUri); } } diff --git a/src/main/java/gov/nysenate/openleg/dao/base/SqlBaseDao.java b/src/main/java/gov/nysenate/openleg/dao/base/SqlBaseDao.java index ca6e0535d..d3fe20814 100644 --- a/src/main/java/gov/nysenate/openleg/dao/base/SqlBaseDao.java +++ b/src/main/java/gov/nysenate/openleg/dao/base/SqlBaseDao.java @@ -8,7 +8,7 @@ import gov.nysenate.openleg.model.sourcefiles.LegDataFragment; import gov.nysenate.openleg.model.updates.UpdateType; import gov.nysenate.openleg.util.DateUtils; -import org.apache.commons.lang3.text.StrSubstitutor; +import org.apache.commons.text.StringSubstitutor; import org.postgresql.util.PGInterval; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -114,7 +114,7 @@ protected static void setModPubDatesFromResultSet(BaseLegislativeContent obj, Re protected static String queryReplace(String originalQuery, String key, String replacement) { Map replaceMap = new HashMap<>(); replaceMap.put(key, replacement); - return new StrSubstitutor(replaceMap).replace(originalQuery); + return new StringSubstitutor(replaceMap).replace(originalQuery); } /** --- PostgreSQL Hstore handling methods --- */ diff --git a/src/main/java/gov/nysenate/openleg/dao/base/SqlQueryUtils.java b/src/main/java/gov/nysenate/openleg/dao/base/SqlQueryUtils.java index 2b1e92ea1..6ae5fdfe3 100644 --- a/src/main/java/gov/nysenate/openleg/dao/base/SqlQueryUtils.java +++ b/src/main/java/gov/nysenate/openleg/dao/base/SqlQueryUtils.java @@ -2,7 +2,7 @@ import com.google.common.collect.ImmutableMap; import org.apache.commons.lang3.StringUtils; -import org.apache.commons.lang3.text.StrSubstitutor; +import org.apache.commons.text.StringSubstitutor; import java.util.HashMap; import java.util.List; @@ -25,7 +25,7 @@ public abstract class SqlQueryUtils public static String getSqlWithSchema(String sql, String schema) { Map replaceMap = new HashMap<>(); replaceMap.put("schema", schema); - return new StrSubstitutor(replaceMap).replace(sql); + return new StringSubstitutor(replaceMap).replace(sql); } /** @@ -35,7 +35,7 @@ public static String getSqlWithSchema(String sql, String dataSchema, String sear Map replaceMap = new HashMap<>(); replaceMap.put("schema", dataSchema); replaceMap.put("search_schema", searchSchema); - return new StrSubstitutor(replaceMap).replace(sql); + return new StringSubstitutor(replaceMap).replace(sql); } /** diff --git a/src/main/java/gov/nysenate/openleg/dao/bill/data/SqlBillDao.java b/src/main/java/gov/nysenate/openleg/dao/bill/data/SqlBillDao.java index 4032bf414..dcfa3dd1c 100644 --- a/src/main/java/gov/nysenate/openleg/dao/bill/data/SqlBillDao.java +++ b/src/main/java/gov/nysenate/openleg/dao/bill/data/SqlBillDao.java @@ -20,7 +20,7 @@ import gov.nysenate.openleg.service.bill.data.VetoDataService; import gov.nysenate.openleg.service.bill.data.VetoNotFoundException; import gov.nysenate.openleg.service.entity.member.data.MemberService; -import org.apache.commons.lang3.text.StrSubstitutor; +import org.apache.commons.text.StringSubstitutor; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; @@ -1058,6 +1058,6 @@ private String applyFullTextFields(String queryTemplate, Set ful .map(field -> ", " + field) .collect(Collectors.joining()); final ImmutableMap subMap = ImmutableMap.of("fullTextFields", fullTextFields); - return StrSubstitutor.replace(queryTemplate, subMap); + return StringSubstitutor.replace(queryTemplate, subMap); } } \ No newline at end of file diff --git a/src/main/java/gov/nysenate/openleg/dao/bill/reference/openleg/JsonOpenlegBillDao.java b/src/main/java/gov/nysenate/openleg/dao/bill/reference/openleg/JsonOpenlegBillDao.java index e094a9459..79daaa0ae 100644 --- a/src/main/java/gov/nysenate/openleg/dao/bill/reference/openleg/JsonOpenlegBillDao.java +++ b/src/main/java/gov/nysenate/openleg/dao/bill/reference/openleg/JsonOpenlegBillDao.java @@ -7,7 +7,7 @@ import gov.nysenate.openleg.dao.base.PaginatedList; import gov.nysenate.openleg.model.base.SessionYear; import gov.nysenate.openleg.service.spotcheck.openleg.JsonOpenlegDaoUtils; -import org.apache.commons.lang3.text.StrSubstitutor; +import org.apache.commons.text.StringSubstitutor; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; @@ -38,6 +38,6 @@ public PaginatedList getBillViews(SessionYear sessionYear, LimitOffset } private String getSessionUri(SessionYear sessionYear) { - return StrSubstitutor.replace(getBillsForSessionUriTemplate, ImmutableMap.of("session", sessionYear)); + return StringSubstitutor.replace(getBillsForSessionUriTemplate, ImmutableMap.of("session", sessionYear)); } } diff --git a/src/main/java/gov/nysenate/openleg/dao/bill/reference/senatesite/FsSenateSiteDao.java b/src/main/java/gov/nysenate/openleg/dao/bill/reference/senatesite/FsSenateSiteDao.java index 58c1ec919..1d9bf79b0 100644 --- a/src/main/java/gov/nysenate/openleg/dao/bill/reference/senatesite/FsSenateSiteDao.java +++ b/src/main/java/gov/nysenate/openleg/dao/bill/reference/senatesite/FsSenateSiteDao.java @@ -11,7 +11,7 @@ import gov.nysenate.openleg.util.SenateSiteDumpFragParser; import org.apache.commons.io.FileUtils; import org.apache.commons.io.filefilter.RegexFileFilter; -import org.apache.commons.lang3.text.StrSubstitutor; +import org.apache.commons.text.StringSubstitutor; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; @@ -112,7 +112,7 @@ private SenateSiteDumpFragment getFragmentFromFile(File fragFile) throws IOExcep * @return String - the prefix that is used for all dump fragments of the designated dump */ private static String getDumpFragFilenamePrefix(SenateSiteDumpFragment fragment) { - return StrSubstitutor.replace(dumpFragFilenamePrefix(fragment.getDumpId().getRefType()), getDumpIdSubMap(fragment.getDumpId())); + return StringSubstitutor.replace(dumpFragFilenamePrefix(fragment.getDumpId().getRefType()), getDumpIdSubMap(fragment.getDumpId())); } /** @@ -120,7 +120,7 @@ private static String getDumpFragFilenamePrefix(SenateSiteDumpFragment fragment) * @return String - the filename that is used for the designated dump fragment */ private static String getDumpFragFilename(SenateSiteDumpFragment fragment) { - return StrSubstitutor.replace(dumpFragFilename(fragment.getDumpId().getRefType()), getDumpFragSubMap(fragment)); + return StringSubstitutor.replace(dumpFragFilename(fragment.getDumpId().getRefType()), getDumpFragSubMap(fragment)); } private static ImmutableMap getDumpIdSubMap(SenateSiteDumpId dumpId) { @@ -159,7 +159,7 @@ private static String dumpFragFilename(SpotCheckRefType refType) { /** Regex to match SenateSiteDumpFragments of the given refType. */ private static Pattern dumpFragFilenameRegex(SpotCheckRefType refType) { String reTemplate = "^" + dumpFragFilename(refType); - String regex = StrSubstitutor.replace(reTemplate, ImmutableMap.of( + String regex = StringSubstitutor.replace(reTemplate, ImmutableMap.of( "year", "(\\d{4})", "refDateTime", "(" + BASIC_ISO_DATE_TIME_REGEX.toString() + ")", "seqNo", "(\\d+)")); diff --git a/src/main/java/gov/nysenate/openleg/dao/bill/scrape/SqlFsBillScrapeReferenceDao.java b/src/main/java/gov/nysenate/openleg/dao/bill/scrape/SqlFsBillScrapeReferenceDao.java index 734d798b8..df65e2541 100644 --- a/src/main/java/gov/nysenate/openleg/dao/bill/scrape/SqlFsBillScrapeReferenceDao.java +++ b/src/main/java/gov/nysenate/openleg/dao/bill/scrape/SqlFsBillScrapeReferenceDao.java @@ -12,7 +12,7 @@ import org.apache.commons.io.FileUtils; import org.apache.commons.io.FilenameUtils; import org.apache.commons.io.filefilter.FileFilterUtils; -import org.apache.commons.lang3.text.StrSubstitutor; +import org.apache.commons.text.StringSubstitutor; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; @@ -71,7 +71,7 @@ public void saveScrapedBillContent(String content, BaseBillId scrapedBill) throw } private File createScrapeFile(File stagingDir, BaseBillId baseBillId) { - String file = StrSubstitutor.replace(FILE_TEMPLATE, ImmutableMap.builder() + String file = StringSubstitutor.replace(FILE_TEMPLATE, ImmutableMap.builder() .put("sessionYear", Integer.toString(baseBillId.getSession().getYear())) .put("printNo", baseBillId.getPrintNo()) .put("scrapedTime", LocalDateTime.now().format(DateUtils.BASIC_ISO_DATE_TIME)) diff --git a/src/main/java/gov/nysenate/openleg/dao/calendar/reference/openleg/JsonOpenlegCalendarDao.java b/src/main/java/gov/nysenate/openleg/dao/calendar/reference/openleg/JsonOpenlegCalendarDao.java index 773e5312f..14f8f9ea3 100644 --- a/src/main/java/gov/nysenate/openleg/dao/calendar/reference/openleg/JsonOpenlegCalendarDao.java +++ b/src/main/java/gov/nysenate/openleg/dao/calendar/reference/openleg/JsonOpenlegCalendarDao.java @@ -5,7 +5,7 @@ import gov.nysenate.openleg.controller.api.base.BaseCtrl; import gov.nysenate.openleg.dao.base.LimitOffset; import gov.nysenate.openleg.service.spotcheck.openleg.JsonOpenlegDaoUtils; -import org.apache.commons.lang3.text.StrSubstitutor; +import org.apache.commons.text.StringSubstitutor; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; @@ -29,7 +29,7 @@ public JsonOpenlegCalendarDao(JsonOpenlegDaoUtils jsonOpenlegDaoUtils) { @Override public List getCalendarViews(int year) { - final String getCalsUri = StrSubstitutor.replace(getCalendarsUriTemplate, ImmutableMap.of("year", year)); + final String getCalsUri = StringSubstitutor.replace(getCalendarsUriTemplate, ImmutableMap.of("year", year)); return jsonOpenlegDaoUtils.queryForViewObjects(CalendarView.class, getCalsUri, LimitOffset.ALL).getResults(); } } diff --git a/src/main/java/gov/nysenate/openleg/dao/scraping/LRSScraper.java b/src/main/java/gov/nysenate/openleg/dao/scraping/LRSScraper.java index 1a9206496..516924272 100644 --- a/src/main/java/gov/nysenate/openleg/dao/scraping/LRSScraper.java +++ b/src/main/java/gov/nysenate/openleg/dao/scraping/LRSScraper.java @@ -18,6 +18,7 @@ import java.io.File; import java.io.IOException; import java.net.URL; +import java.nio.charset.Charset; import java.time.Duration; import java.time.LocalDateTime; import java.time.format.DateTimeFormatter; @@ -115,7 +116,7 @@ protected Document getJsoupDocument(String url) { protected String getUrlContents(URL url) { try { - return IOUtils.toString(url); + return IOUtils.toString(url, Charset.defaultCharset()); } catch (IOException ex) { throw new ScrapingException(url, ex); } diff --git a/src/main/java/gov/nysenate/openleg/dao/updates/SqlAggregateUpdatesQuery.java b/src/main/java/gov/nysenate/openleg/dao/updates/SqlAggregateUpdatesQuery.java index 43bcc56a7..62831fadf 100644 --- a/src/main/java/gov/nysenate/openleg/dao/updates/SqlAggregateUpdatesQuery.java +++ b/src/main/java/gov/nysenate/openleg/dao/updates/SqlAggregateUpdatesQuery.java @@ -5,7 +5,7 @@ import gov.nysenate.openleg.model.updates.UpdateContentType; import gov.nysenate.openleg.model.updates.UpdateReturnType; import gov.nysenate.openleg.model.updates.UpdateType; -import org.apache.commons.lang3.text.StrSubstitutor; +import org.apache.commons.text.StringSubstitutor; import java.util.Map; import java.util.Set; @@ -135,7 +135,7 @@ public static String buildQuery(String schema, LimitOffset limOff, SortOrder ord .putAll(getColumnReplaceMap(returnType)) .put("dateColumn", updateType == UpdateType.PROCESSED_DATE ? "action_date_time" : "published_date_time") .build(); - aggregateQuery = StrSubstitutor.replace(aggregateQuery, replaceMap); + aggregateQuery = StringSubstitutor.replace(aggregateQuery, replaceMap); return aggregateQuery; } diff --git a/src/main/java/gov/nysenate/openleg/model/base/PublishStatus.java b/src/main/java/gov/nysenate/openleg/model/base/PublishStatus.java index a3bc47fed..d16485432 100644 --- a/src/main/java/gov/nysenate/openleg/model/base/PublishStatus.java +++ b/src/main/java/gov/nysenate/openleg/model/base/PublishStatus.java @@ -70,8 +70,8 @@ public String toString() { public int compareTo(PublishStatus o) { return ComparisonChain.start() .compare(this.effectDateTime, o.effectDateTime) - .compare(this.override, o.override) - .compare(this.published, o.published) + .compareFalseFirst(this.override, o.override) + .compareFalseFirst(this.published, o.published) .result(); } diff --git a/src/main/java/gov/nysenate/openleg/model/spotcheck/billscrape/BillScrapeVote.java b/src/main/java/gov/nysenate/openleg/model/spotcheck/billscrape/BillScrapeVote.java index f1c9d685e..820f26863 100644 --- a/src/main/java/gov/nysenate/openleg/model/spotcheck/billscrape/BillScrapeVote.java +++ b/src/main/java/gov/nysenate/openleg/model/spotcheck/billscrape/BillScrapeVote.java @@ -2,7 +2,7 @@ import com.google.common.collect.SortedSetMultimap; import gov.nysenate.openleg.model.bill.BillVoteCode; -import org.apache.commons.lang3.text.WordUtils; +import org.apache.commons.text.WordUtils; import java.time.LocalDate; import java.util.Objects; diff --git a/src/main/java/gov/nysenate/openleg/model/spotcheck/daybreak/DaybreakFile.java b/src/main/java/gov/nysenate/openleg/model/spotcheck/daybreak/DaybreakFile.java index ae3ede7f4..e57cb6ae6 100644 --- a/src/main/java/gov/nysenate/openleg/model/spotcheck/daybreak/DaybreakFile.java +++ b/src/main/java/gov/nysenate/openleg/model/spotcheck/daybreak/DaybreakFile.java @@ -8,6 +8,7 @@ import java.io.File; import java.io.FileNotFoundException; import java.io.IOException; +import java.nio.charset.Charset; import java.text.ParseException; import java.time.LocalDate; import java.time.LocalDateTime; @@ -85,7 +86,7 @@ private LocalDate getReportDateFromFileName() { @JsonIgnore public String getText() { try { - return FileUtils.readFileToString(file); + return FileUtils.readFileToString(file, Charset.defaultCharset()); } catch (IOException e) { throw new RuntimeException("Failed to read text from Daybreak file:" + this.toString()); diff --git a/src/main/java/gov/nysenate/openleg/processor/hearing/PublicHearingParser.java b/src/main/java/gov/nysenate/openleg/processor/hearing/PublicHearingParser.java index c3539efc8..f35a8d53d 100644 --- a/src/main/java/gov/nysenate/openleg/processor/hearing/PublicHearingParser.java +++ b/src/main/java/gov/nysenate/openleg/processor/hearing/PublicHearingParser.java @@ -11,6 +11,7 @@ import org.springframework.stereotype.Service; import java.io.IOException; +import java.nio.charset.Charset; import java.time.LocalDate; import java.time.LocalDateTime; import java.time.LocalTime; @@ -44,7 +45,8 @@ public class PublicHearingParser * @throws IOException */ public void process(PublicHearingFile publicHearingFile) throws IOException { - final List> pages = PublicHearingTextUtils.getPages(FileUtils.readFileToString(publicHearingFile.getFile())); + final List> pages = PublicHearingTextUtils.getPages( + FileUtils.readFileToString(publicHearingFile.getFile(), Charset.defaultCharset())); final List firstPage = pages.get(0); String title = titleParser.parse(firstPage); diff --git a/src/main/java/gov/nysenate/openleg/processor/law/LawTitleParser.java b/src/main/java/gov/nysenate/openleg/processor/law/LawTitleParser.java index c417e84fa..63e04bc93 100644 --- a/src/main/java/gov/nysenate/openleg/processor/law/LawTitleParser.java +++ b/src/main/java/gov/nysenate/openleg/processor/law/LawTitleParser.java @@ -3,7 +3,7 @@ import gov.nysenate.openleg.model.law.LawChapterCode; import gov.nysenate.openleg.model.law.LawDocInfo; import org.apache.commons.lang3.StringUtils; -import org.apache.commons.lang3.text.WordUtils; +import org.apache.commons.text.WordUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/src/main/java/gov/nysenate/openleg/processor/spotcheck/calendar/CalendarAlertActiveListParser.java b/src/main/java/gov/nysenate/openleg/processor/spotcheck/calendar/CalendarAlertActiveListParser.java index bf7e905c1..b273175c7 100644 --- a/src/main/java/gov/nysenate/openleg/processor/spotcheck/calendar/CalendarAlertActiveListParser.java +++ b/src/main/java/gov/nysenate/openleg/processor/spotcheck/calendar/CalendarAlertActiveListParser.java @@ -16,6 +16,7 @@ import java.io.File; import java.io.IOException; +import java.nio.charset.Charset; import java.time.LocalDate; import java.time.format.DateTimeFormatter; import java.util.regex.Matcher; @@ -47,7 +48,7 @@ protected CalendarActiveList parseActiveList(CalendarAlertFile calFile) throws I * @throws IOException */ private void parseActiveListEntries(File file, CalendarActiveList activeList) throws IOException { - String html = FileUtils.readFileToString(file); + String html = FileUtils.readFileToString(file, Charset.defaultCharset()); Document doc = Jsoup.parse(html); Elements entries = doc.select("body table tr"); @@ -71,7 +72,7 @@ private int extractBillCalNo(Element entry) { private LocalDate parseCalendarDate(File file) { try { - String html = FileUtils.readFileToString(file); + String html = FileUtils.readFileToString(file, Charset.defaultCharset()); Document doc = Jsoup.parse(html); Element title = doc.select("h3[align]").get(0); String calTitle = title.text(); diff --git a/src/main/java/gov/nysenate/openleg/processor/spotcheck/calendar/CalendarAlertSupplementalParser.java b/src/main/java/gov/nysenate/openleg/processor/spotcheck/calendar/CalendarAlertSupplementalParser.java index d977c51ac..9d1a49aba 100644 --- a/src/main/java/gov/nysenate/openleg/processor/spotcheck/calendar/CalendarAlertSupplementalParser.java +++ b/src/main/java/gov/nysenate/openleg/processor/spotcheck/calendar/CalendarAlertSupplementalParser.java @@ -1,6 +1,5 @@ package gov.nysenate.openleg.processor.spotcheck.calendar; -import gov.nysenate.openleg.dao.base.LimitOffset; import gov.nysenate.openleg.model.base.SessionYear; import gov.nysenate.openleg.model.bill.BillId; import gov.nysenate.openleg.model.calendar.CalendarSectionType; @@ -8,12 +7,7 @@ import gov.nysenate.openleg.model.calendar.CalendarSupplementalEntry; import gov.nysenate.openleg.model.calendar.alert.CalendarAlertFile; import gov.nysenate.openleg.model.entity.Chamber; -import gov.nysenate.openleg.model.entity.Member; -import gov.nysenate.openleg.model.entity.SessionMember; -import gov.nysenate.openleg.model.search.SearchException; -import gov.nysenate.openleg.model.search.SearchResults; import gov.nysenate.openleg.service.entity.member.data.MemberService; -import gov.nysenate.openleg.service.entity.member.search.MemberSearchService; import org.apache.commons.io.FileUtils; import org.apache.commons.lang3.StringUtils; import org.jsoup.Jsoup; @@ -27,6 +21,7 @@ import java.io.File; import java.io.IOException; +import java.nio.charset.Charset; import java.time.LocalDate; import java.time.format.DateTimeFormatter; import java.util.HashMap; @@ -51,7 +46,7 @@ protected CalendarSupplemental parseSupplemental(CalendarAlertFile calFile) thro } private void parseSupplementalEntries(File file, CalendarSupplemental supplemental) throws IOException { - String html = FileUtils.readFileToString(file); + String html = FileUtils.readFileToString(file, Charset.defaultCharset()); Document doc = Jsoup.parse(html); Elements sectionTypes = doc.select("a[name]"); Elements entryTables = doc.select("table"); @@ -112,7 +107,7 @@ private Chamber getChamber(SessionYear sessionYear, String sponsor) { private LocalDate parseCalendarDate(File file) { try { - String html = FileUtils.readFileToString(file); + String html = FileUtils.readFileToString(file, Charset.defaultCharset()); Document doc = Jsoup.parse(html); Element title = doc.select("div.center").get(0); String calTitle = title.text(); diff --git a/src/main/java/gov/nysenate/openleg/processor/transcript/TranscriptLine.java b/src/main/java/gov/nysenate/openleg/processor/transcript/TranscriptLine.java index 82027bd42..d998de270 100644 --- a/src/main/java/gov/nysenate/openleg/processor/transcript/TranscriptLine.java +++ b/src/main/java/gov/nysenate/openleg/processor/transcript/TranscriptLine.java @@ -1,6 +1,6 @@ package gov.nysenate.openleg.processor.transcript; -import org.apache.commons.lang3.text.WordUtils; +import org.apache.commons.text.WordUtils; import java.time.LocalDate; import java.time.LocalTime; diff --git a/src/main/java/gov/nysenate/openleg/script/BaseScript.java b/src/main/java/gov/nysenate/openleg/script/BaseScript.java index 696b36165..12dcd50e6 100644 --- a/src/main/java/gov/nysenate/openleg/script/BaseScript.java +++ b/src/main/java/gov/nysenate/openleg/script/BaseScript.java @@ -52,7 +52,7 @@ public static void shutdown(AnnotationConfigApplicationContext ctx) { * @throws ParseException */ public static CommandLine getCommandLine(Options opts, String[] args) throws ParseException { - return new PosixParser().parse(opts, args); + return new DefaultParser().parse(opts, args); } /** diff --git a/src/main/java/gov/nysenate/openleg/script/LRSScraper.java b/src/main/java/gov/nysenate/openleg/script/LRSScraper.java index 8742f89f0..14b3c7a5b 100644 --- a/src/main/java/gov/nysenate/openleg/script/LRSScraper.java +++ b/src/main/java/gov/nysenate/openleg/script/LRSScraper.java @@ -11,6 +11,7 @@ import java.io.IOException; import java.net.MalformedURLException; import java.net.URL; +import java.nio.charset.Charset; import java.text.SimpleDateFormat; import java.util.Date; import java.util.regex.Matcher; @@ -40,14 +41,15 @@ public static void main(String[] args) throws Exception public void scrapeCalendars(URL landingURL, File directory, Date currentTime) throws IOException { logger.info("Fetching landing page."); - String landingPage = IOUtils.toString(landingURL.openStream()); + String landingPage = IOUtils.toString(landingURL.openStream(), Charset.defaultCharset()); Matcher tokenMatcher = bottomPattern.matcher(landingPage); logger.info("Searching for link to bottom half"); if (tokenMatcher.find()) { String link = tokenMatcher.group(1); URL contentURL = resolveLink(landingURL, link); logger.info("Fetching bottom half"); - String contentPage = IOUtils.toString(contentURL.openStream()).replace("\r\n", " "); + String contentPage = IOUtils.toString(contentURL.openStream(), Charset.defaultCharset()) + .replace("\r\n", " "); Matcher linkMatcher = linkPattern.matcher(contentPage); while(linkMatcher.find()) { URL linkURL = resolveLink(contentURL, linkMatcher.group(1)); @@ -55,7 +57,7 @@ public void scrapeCalendars(URL landingURL, File directory, Date currentTime) th .replace(" ", "_").toLowerCase()+".html"; // add 0 for last active list and increment File outfile = new File(directory, filename); logger.info("Fetching "+linkMatcher.group(2).trim()); - String contents = IOUtils.toString(linkURL); + String contents = IOUtils.toString(linkURL, Charset.defaultCharset()); logger.info("Writing content to "+filename); FileIOUtils.write(outfile, contents); } @@ -65,7 +67,7 @@ public void scrapeCalendars(URL landingURL, File directory, Date currentTime) th public void scrapeAgendas(URL landingURL, File directory, Date currentTime) throws IOException { logger.info("Fetching landing page."); - String landingPage = IOUtils.toString(landingURL.openStream()); + String landingPage = IOUtils.toString(landingURL.openStream(), Charset.defaultCharset()); Matcher tokenMatcher = bottomPattern.matcher(landingPage); logger.info("Searching for link to bottom half"); if (tokenMatcher.find()) { @@ -73,7 +75,8 @@ public void scrapeAgendas(URL landingURL, File directory, Date currentTime) thro URL contentURL = resolveLink(landingURL, link); //calen logger.info("Fetching bottom half"); - String contentPage = IOUtils.toString(contentURL.openStream()).replace("\r\n", " "); + String contentPage = IOUtils.toString(contentURL.openStream(), Charset.defaultCharset()) + .replace("\r\n", " "); Matcher linkMatcher = linkPattern.matcher(contentPage); logger.info("Searching for all committee agendas link"); while (linkMatcher.find()) { @@ -82,7 +85,7 @@ public void scrapeAgendas(URL landingURL, File directory, Date currentTime) thro String filename = dateFormat.format(currentTime)+".all_agendas.html"; File outfile = new File(directory, filename); logger.info("Fetching all committee agendas"); - String contents = IOUtils.toString(linkURL); + String contents = IOUtils.toString(linkURL, Charset.defaultCharset()); logger.info("Writing content to "+filename); FileIOUtils.write(outfile, contents); } diff --git a/src/main/java/gov/nysenate/openleg/service/auth/CachedSqlApiUserService.java b/src/main/java/gov/nysenate/openleg/service/auth/CachedSqlApiUserService.java index 43183222e..3c603e40b 100644 --- a/src/main/java/gov/nysenate/openleg/service/auth/CachedSqlApiUserService.java +++ b/src/main/java/gov/nysenate/openleg/service/auth/CachedSqlApiUserService.java @@ -5,7 +5,6 @@ import com.google.common.eventbus.EventBus; import com.google.common.eventbus.Subscribe; import gov.nysenate.openleg.config.Environment; -import gov.nysenate.openleg.controller.api.base.BaseCtrl; import gov.nysenate.openleg.dao.auth.ApiUserDao; import gov.nysenate.openleg.model.auth.ApiUser; import gov.nysenate.openleg.model.auth.ApiUserAuthEvictEvent; @@ -17,26 +16,26 @@ import gov.nysenate.openleg.model.notification.Notification; import gov.nysenate.openleg.model.notification.NotificationType; import gov.nysenate.openleg.service.base.data.CachingService; -import gov.nysenate.openleg.service.mail.MimeSendMailService; import gov.nysenate.openleg.service.mail.SendMailService; import net.sf.ehcache.Cache; import net.sf.ehcache.CacheManager; import net.sf.ehcache.Ehcache; import net.sf.ehcache.config.CacheConfiguration; import org.apache.commons.lang3.RandomStringUtils; -import org.apache.commons.lang3.text.StrSubstitutor; +import org.apache.commons.text.StringSubstitutor; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.cache.ehcache.EhCacheCache; import org.springframework.dao.EmptyResultDataAccessException; import org.springframework.stereotype.Service; -import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; import java.time.LocalDateTime; -import java.util.*; +import java.util.Arrays; +import java.util.List; +import java.util.Optional; +import java.util.Set; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -309,7 +308,7 @@ private void sendRegistrationEmail(ApiUser user) { userPlaceholder, user.getName(), domainPlaceholder, environment.getUrl(), tokenPlaceholder, user.getRegistrationToken()); - final String message = StrSubstitutor.replace(regEmailTemplate, subMap); + final String message = StringSubstitutor.replace(regEmailTemplate, subMap); sendMailService.sendMessage(user.getEmail(), "Open Legislation API Account Registration", message); } @@ -336,7 +335,7 @@ private void sendApikeyEmail(ApiUser user) { keyPlaceholder, user.getApiKey(), domainPlaceholder, environment.getUrl(), fromPlaceholder, environment.getEmailFromAddress()); - final String message = StrSubstitutor.replace(keyEmailTemplate, subMap); + final String message = StringSubstitutor.replace(keyEmailTemplate, subMap); sendMailService.sendMessage(user.getEmail(), "Your Open Legislation API Key", message); } diff --git a/src/main/java/gov/nysenate/openleg/service/scraping/bill/BillScrapeReferenceFactory.java b/src/main/java/gov/nysenate/openleg/service/scraping/bill/BillScrapeReferenceFactory.java index 09f9783c2..f2740d591 100644 --- a/src/main/java/gov/nysenate/openleg/service/scraping/bill/BillScrapeReferenceFactory.java +++ b/src/main/java/gov/nysenate/openleg/service/scraping/bill/BillScrapeReferenceFactory.java @@ -12,6 +12,7 @@ import org.springframework.stereotype.Service; import java.io.IOException; +import java.nio.charset.Charset; @Service public class BillScrapeReferenceFactory { @@ -48,6 +49,6 @@ private String formatText(String text, BillType billType) { private BillScrapeReference errorBillScrapeReference(BillScrapeFile btrFile) throws IOException { return BillScrapeReference.getErrorBtr(btrFile.getBaseBillId(), btrFile.getReferenceDateTime(), - FileUtils.readFileToString(btrFile.getFile())); + FileUtils.readFileToString(btrFile.getFile(), Charset.defaultCharset())); } } diff --git a/src/main/java/gov/nysenate/openleg/service/scraping/bill/BillScraper.java b/src/main/java/gov/nysenate/openleg/service/scraping/bill/BillScraper.java index df249cab5..a4e3ab569 100644 --- a/src/main/java/gov/nysenate/openleg/service/scraping/bill/BillScraper.java +++ b/src/main/java/gov/nysenate/openleg/service/scraping/bill/BillScraper.java @@ -6,7 +6,7 @@ import gov.nysenate.openleg.dao.scraping.ScrapingException; import gov.nysenate.openleg.model.bill.BaseBillId; import org.apache.commons.io.IOUtils; -import org.apache.commons.lang3.text.StrSubstitutor; +import org.apache.commons.text.StringSubstitutor; import org.apache.http.HttpResponse; import org.apache.http.client.HttpClient; import org.apache.http.client.methods.HttpGet; @@ -18,6 +18,7 @@ import org.springframework.stereotype.Repository; import java.io.IOException; +import java.nio.charset.Charset; /** * Created by kyle on 1/29/15. @@ -45,7 +46,7 @@ protected int doScrape() throws IOException, ScrapingException { BaseBillId baseBillId = scrapeDao.getScrapeQueueHead(); HttpResponse response = makeRequest(constructUrl(baseBillId)); // Verify bill data is in the file before saving. - String content = IOUtils.toString(response.getEntity().getContent()); + String content = IOUtils.toString(response.getEntity().getContent(), Charset.defaultCharset()); if (!htmlParser.isLrsOutage(content)) { scrapeDao.saveScrapedBillContent(content, baseBillId); scrapeDao.deleteBillFromScrapeQueue(baseBillId); @@ -79,7 +80,7 @@ public HttpResponse makeRequest(String url) { } public String constructUrl(BaseBillId billId) { - return StrSubstitutor.replace(URL_TEMPLATE, + return StringSubstitutor.replace(URL_TEMPLATE, ImmutableMap.of("printNo", billId.getPrintNo(), "sessionYear", Integer.toString(billId.getSession().getYear()))); } diff --git a/src/main/java/gov/nysenate/openleg/service/slack/SlackApi.java b/src/main/java/gov/nysenate/openleg/service/slack/SlackApi.java index e7355a05f..bedae3c05 100644 --- a/src/main/java/gov/nysenate/openleg/service/slack/SlackApi.java +++ b/src/main/java/gov/nysenate/openleg/service/slack/SlackApi.java @@ -1,14 +1,17 @@ package gov.nysenate.openleg.service.slack; -import java.io.*; -import java.net.HttpURLConnection; -import java.net.URL; -import java.net.URLEncoder; - import com.google.gson.JsonObject; import gov.nysenate.openleg.model.slack.SlackMessage; import org.apache.commons.io.IOUtils; +import java.io.DataOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.net.HttpURLConnection; +import java.net.URL; +import java.net.URLEncoder; +import java.nio.charset.Charset; + /** * Copied from https://github.com/gpedro/slack-webhook with some customizations * (This done before that api was released via maven) @@ -62,9 +65,10 @@ private String send(JsonObject message) { //Get Response int responseCode = connection.getResponseCode(); if (responseCode != 200) { - InputStream es = connection.getErrorStream(); - String errorMessage = IOUtils.toString(es); - IOUtils.closeQuietly(es); + String errorMessage; + try (InputStream es = connection.getErrorStream()) { + errorMessage = IOUtils.toString(es, Charset.defaultCharset()); + } if (responseCode == 404 && channelNotFoundMessage.equals(errorMessage)) { throw new SlackChannelNotFoundException(message.get("channel").getAsString(), errorMessage); @@ -72,9 +76,10 @@ private String send(JsonObject message) { throw new SlackApiException(errorMessage, responseCode); } - InputStream is = connection.getInputStream(); - String responseMessage = IOUtils.toString(is); - IOUtils.closeQuietly(is); + String responseMessage; + try (InputStream is = connection.getInputStream()) { + responseMessage = IOUtils.toString(is, Charset.defaultCharset()); + } return responseMessage; diff --git a/src/main/java/gov/nysenate/openleg/service/spotcheck/base/BaseCheckMailService.java b/src/main/java/gov/nysenate/openleg/service/spotcheck/base/BaseCheckMailService.java index fa6d8f2d7..e6cd7a873 100644 --- a/src/main/java/gov/nysenate/openleg/service/spotcheck/base/BaseCheckMailService.java +++ b/src/main/java/gov/nysenate/openleg/service/spotcheck/base/BaseCheckMailService.java @@ -11,6 +11,7 @@ import javax.mail.*; import java.io.File; import java.io.IOException; +import java.nio.charset.Charset; public abstract class BaseCheckMailService { @@ -52,7 +53,7 @@ protected void saveMessageAttachment(Message message, File file) throws Messagin Part part = content.getBodyPart(i); if (Part.ATTACHMENT.equalsIgnoreCase(part.getDisposition())) { logger.info("\tSaving " + part.getFileName() + " to " + file.getAbsolutePath()); - String attachment = IOUtils.toString(part.getInputStream()); + String attachment = IOUtils.toString(part.getInputStream(), Charset.defaultCharset()); FileIOUtils.write(file, attachment); } } diff --git a/src/main/java/gov/nysenate/openleg/service/spotcheck/daybreak/DaybreakCheckService.java b/src/main/java/gov/nysenate/openleg/service/spotcheck/daybreak/DaybreakCheckService.java index ecbe0e640..e7352001b 100644 --- a/src/main/java/gov/nysenate/openleg/service/spotcheck/daybreak/DaybreakCheckService.java +++ b/src/main/java/gov/nysenate/openleg/service/spotcheck/daybreak/DaybreakCheckService.java @@ -13,7 +13,7 @@ import gov.nysenate.openleg.service.spotcheck.base.SpotCheckUtils; import gov.nysenate.openleg.util.BillTextUtils; import gov.nysenate.openleg.util.DateUtils; -import org.apache.commons.lang3.StringEscapeUtils; +import org.apache.commons.text.StringEscapeUtils; import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/src/main/java/gov/nysenate/openleg/util/FileIOUtils.java b/src/main/java/gov/nysenate/openleg/util/FileIOUtils.java index e6865e9aa..69017a577 100644 --- a/src/main/java/gov/nysenate/openleg/util/FileIOUtils.java +++ b/src/main/java/gov/nysenate/openleg/util/FileIOUtils.java @@ -175,7 +175,7 @@ public static boolean isFileClosed(File file) throws IOException { * @throws IOException */ public static void write(File file, CharSequence data) throws IOException { - FileUtils.write(file, data); + FileUtils.write(file, data, Charset.defaultCharset()); setCommonFilePermissions(file); } @@ -202,7 +202,7 @@ public static void write(File file, CharSequence data, Charset encoding) throws * @throws IOException */ public static void writeStringToFile(File file, String data) throws IOException { - FileUtils.writeStringToFile(file, data); + FileUtils.writeStringToFile(file, data, Charset.defaultCharset()); setCommonFilePermissions(file); } @@ -282,7 +282,7 @@ public static File getResourceFile(String relativePath) { * @throws IOException */ public static String getResourceFileContents(String relativePath) throws IOException { - return FileUtils.readFileToString(getResourceFile(relativePath)); + return FileUtils.readFileToString(getResourceFile(relativePath), Charset.defaultCharset()); } /* --- Internal Methods --- */ diff --git a/src/test/java/gov/nysenate/openleg/processor/BaseXmlProcessorTest.java b/src/test/java/gov/nysenate/openleg/processor/BaseXmlProcessorTest.java index 16fe0841d..150cf5507 100644 --- a/src/test/java/gov/nysenate/openleg/processor/BaseXmlProcessorTest.java +++ b/src/test/java/gov/nysenate/openleg/processor/BaseXmlProcessorTest.java @@ -25,6 +25,7 @@ import javax.annotation.PostConstruct; import java.io.File; import java.io.IOException; +import java.nio.charset.Charset; import java.util.Collections; import java.util.EnumSet; import java.util.List; @@ -90,7 +91,7 @@ protected LegDataFragment generateXmlSobiFragment(String xmlFilePath) { try { File file = FileIOUtils.getResourceFile(xmlFilePath); - String contents = FileUtils.readFileToString(file); + String contents = FileUtils.readFileToString(file, Charset.defaultCharset()); XmlFile xmlFile = new XmlFile(file); diff --git a/src/test/java/gov/nysenate/openleg/processor/hearing/PublicHearingTestHelper.java b/src/test/java/gov/nysenate/openleg/processor/hearing/PublicHearingTestHelper.java index cf32a7226..be4b417ac 100644 --- a/src/test/java/gov/nysenate/openleg/processor/hearing/PublicHearingTestHelper.java +++ b/src/test/java/gov/nysenate/openleg/processor/hearing/PublicHearingTestHelper.java @@ -1,19 +1,19 @@ package gov.nysenate.openleg.processor.hearing; import gov.nysenate.openleg.TestUtils; -import gov.nysenate.openleg.util.FileIOUtils; import gov.nysenate.openleg.util.PublicHearingTextUtils; import org.apache.commons.io.FileUtils; import java.io.File; import java.io.IOException; import java.net.URISyntaxException; +import java.nio.charset.Charset; import java.util.List; public class PublicHearingTestHelper { public static List> getPagesFromFileName(String fileName) throws URISyntaxException, IOException { File file = TestUtils.openTestResource("hearing/" + fileName); - return PublicHearingTextUtils.getPages(FileUtils.readFileToString(file)); + return PublicHearingTextUtils.getPages(FileUtils.readFileToString(file, Charset.defaultCharset())); } } diff --git a/src/test/java/gov/nysenate/openleg/processor/legdata/XmlJsonIT.java b/src/test/java/gov/nysenate/openleg/processor/legdata/XmlJsonIT.java index f4edf3d33..a757ab59c 100644 --- a/src/test/java/gov/nysenate/openleg/processor/legdata/XmlJsonIT.java +++ b/src/test/java/gov/nysenate/openleg/processor/legdata/XmlJsonIT.java @@ -12,6 +12,7 @@ import javax.annotation.PostConstruct; import java.io.File; import java.io.IOException; +import java.nio.charset.Charset; import java.util.Iterator; import java.util.LinkedList; import java.util.List; @@ -51,7 +52,7 @@ private void verifyJSONParse() { private String readInFileToString(File file) { try { - return FileUtils.readFileToString(file); + return FileUtils.readFileToString(file, Charset.defaultCharset()); } catch (IOException e) { e.printStackTrace(); fail("Could not convert the file to a string"); diff --git a/src/test/java/gov/nysenate/openleg/processor/transcript/TranscriptLineTest.java b/src/test/java/gov/nysenate/openleg/processor/transcript/TranscriptLineTest.java index e7afbf570..b2389efbe 100644 --- a/src/test/java/gov/nysenate/openleg/processor/transcript/TranscriptLineTest.java +++ b/src/test/java/gov/nysenate/openleg/processor/transcript/TranscriptLineTest.java @@ -1,7 +1,7 @@ package gov.nysenate.openleg.processor.transcript; import gov.nysenate.openleg.annotation.UnitTest; -import org.apache.commons.lang3.text.WordUtils; +import org.apache.commons.text.WordUtils; import org.junit.Test; import org.junit.experimental.categories.Category; diff --git a/src/test/java/gov/nysenate/openleg/script/MemberProcessorTest.java b/src/test/java/gov/nysenate/openleg/script/MemberProcessorTest.java index cf4f7a163..55b9235c1 100644 --- a/src/test/java/gov/nysenate/openleg/script/MemberProcessorTest.java +++ b/src/test/java/gov/nysenate/openleg/script/MemberProcessorTest.java @@ -16,6 +16,7 @@ import org.springframework.jdbc.core.SingleColumnRowMapper; import java.io.File; +import java.nio.charset.Charset; import java.util.*; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -80,7 +81,7 @@ public void testMemberInsertion() throws Exception { public void testInsertMember() throws Exception { File f = new File("/data/2013_lbdc_sn.txt"); HashSet shortNames = new HashSet<>(); - List lines = FileUtils.readLines(f); + List lines = FileUtils.readLines(f, Charset.defaultCharset()); for (String l : lines) { Matcher m = Pattern.compile(".*\\:(.{11})6([^0-9]+)").matcher(l); if (m.find() && !l.contains("DELETE")) { @@ -129,7 +130,7 @@ public void testInsertMember() throws Exception { public void testMissingSenators() throws Exception { File f = new File("/data/2013.cosponsors.sn.txt"); HashSet shortNames = new HashSet<>(); - List lines = FileUtils.readLines(f); + List lines = FileUtils.readLines(f, Charset.defaultCharset()); for (String l : lines) { Matcher m = Pattern.compile(".*\\:(.{11})7([^0-9]+)").matcher(l); if (m.find() && !l.contains("DELETE")) { @@ -154,7 +155,7 @@ public void testMissingSenators() throws Exception { @Test public void testAssemblyCSV() throws Exception { Resource f = new FileSystemResource("/data/2013_assembly_members.csv"); - List lines = FileUtils.readLines(f.getFile()); + List lines = FileUtils.readLines(f.getFile(), Charset.defaultCharset()); for (String s : lines) { String[] parts = s.split(","); int district = Integer.parseInt(parts[0]); @@ -185,7 +186,7 @@ public void testAssemblyShortNames() throws Exception { File cf = new File("/data/2013_assembly_cosponsors_sn.txt"); File csvF = new File("/data/openleg_member_data/2013_assembly_members.csv"); HashSet shortNames = new HashSet<>(); - List lines = FileUtils.readLines(f); + List lines = FileUtils.readLines(f, Charset.defaultCharset()); for (String l : lines) { Matcher m = Pattern.compile(".*\\:(.{11})6([^0-9]+)").matcher(l); if (m.find() && !l.contains("DELETE")) { @@ -197,7 +198,7 @@ public void testAssemblyShortNames() throws Exception { } // Add co sponsors to set - List csLines = FileUtils.readLines(cf); + List csLines = FileUtils.readLines(cf, Charset.defaultCharset()); for (String l : csLines) { Matcher m = Pattern.compile(".*\\:(.{11})7([^0-9]+)").matcher(l); if (m.find() && !l.contains("DELETE") && !l.contains("RULES")) { @@ -210,7 +211,7 @@ public void testAssemblyShortNames() throws Exception { } HashMap distAss = new HashMap<>(); - List csvLines = FileUtils.readLines(csvF); + List csvLines = FileUtils.readLines(csvF, Charset.defaultCharset()); for (String s : csvLines) { String[] parts = s.split(","); int district = Integer.parseInt(parts[0]); @@ -249,7 +250,7 @@ public void testAssemblyShortNames() throws Exception { @Test public void testInsert2013AssemblyExtra() throws Exception { File f = new File("/data/openleg_member_data/2013_assembly_extra.csv"); - List lines = FileUtils.readLines(f); + List lines = FileUtils.readLines(f, Charset.defaultCharset()); for (String line : lines) { String[] parts = line.split(","); String fullName = (parts[0] + ((!parts[1].isEmpty()) ? (" " + parts[1]) : "") + " " + parts[2] + " " + parts[3]).trim();