Browse files

Merge pull request #464 from mwoodiupui/DS-1531

[DS-1531] bug in current DSpace (3.1) with log importing
  • Loading branch information...
2 parents 0bc01a8 + 5ef301c commit 8c4862fc12ff741e00a130c9f954f3b9349cc279 @mwoodiupui mwoodiupui committed Feb 5, 2014
View
20 dspace-api/src/main/java/org/dspace/statistics/SolrLogger.java
@@ -7,7 +7,6 @@
*/
package org.dspace.statistics;
-import au.com.bytecode.opencsv.CSVParser;
import au.com.bytecode.opencsv.CSVReader;
import au.com.bytecode.opencsv.CSVWriter;
import com.maxmind.geoip.Location;
@@ -78,8 +77,6 @@
private static final LookupService locationService;
- private static Map<String, String> metadataStorageInfo;
-
private static final boolean useProxies;
private static List<String> statisticYearCores = new ArrayList<String>();
@@ -563,11 +560,6 @@ public static void postWorkflow(UsageWorkflowEvent usageWorkflowEvent) throws SQ
}
- public static Map<String, String> getMetadataStorageInfo()
- {
- return metadataStorageInfo;
- }
-
/**
* Method just used to log the parents.
* <ul>
@@ -676,18 +668,6 @@ public static void removeIndex(String query) throws IOException,
// We have at least one document good
SolrDocument document = response.getResults().get(0);
- for (Object storedField : metadataStorageInfo.keySet())
- {
- // For each of these fields that are stored we are to create a
- // list of the values it holds now
- java.util.Collection collection = document
- .getFieldValues((String) storedField);
- List<String> storedVals = new ArrayList<String>();
- storedVals.addAll(collection);
- // Now add it to our hashmap
- currentValsStored.put((String) storedField, storedVals);
- }
-
// System.out.println("HERE");
// Get the info we need
}
View
40 dspace-api/src/main/java/org/dspace/statistics/StatisticsLoggingConsumer.java
@@ -37,6 +37,7 @@
private Set<String> toRemoveQueries = null;
+ @Override
public void initialize() throws Exception
{
@@ -45,6 +46,7 @@ public void initialize() throws Exception
// TODO: checkout whether moving of collections, communities and bitstreams works
// TODO: use async threaded consumer as this might require some processing time
// TODO: we might be able to improve the performance: changing the collection will trigger 4 update commands
+ @Override
public void consume(Context ctx, Event event) throws Exception
{
if (toRemoveQueries == null)
@@ -78,45 +80,9 @@ else if (eventType == Event.MODIFY_METADATA
updateQuery, null, null);
// Get all the metadata
- Map<String, String> metadataStorageInfo = SolrLogger.getMetadataStorageInfo();
List<String> storageFieldList = new ArrayList<String>();
List<List<Object>> storageValuesList = new ArrayList<List<Object>>();
- for (Map.Entry<String, String> entry : metadataStorageInfo.entrySet())
- {
- String[] metadataFieldInfo = entry.getValue().split("\\.");
-
- List<Object> values = new ArrayList<Object>();
- List<Object> valuesLow = new ArrayList<Object>();
- for (int i = 0; i < item.getMetadata(metadataFieldInfo[0],
- metadataFieldInfo[1], metadataFieldInfo[2], Item.ANY).length; i++)
- {
- values.add(item.getMetadata(metadataFieldInfo[0],
- metadataFieldInfo[1], metadataFieldInfo[2],
- Item.ANY)[i].value);
-
- valuesLow.add(item.getMetadata(metadataFieldInfo[0],
- metadataFieldInfo[1], metadataFieldInfo[2],
- Item.ANY)[i].value.toLowerCase());
- }
-
- List<String> indexedVals = indexedValues.get(entry.getKey());
-
- boolean update = true;
- if (values.size() == indexedVals.size() && values.containsAll(indexedVals))
- {
- update = false;
- }
-
- if (update)
- {
- storageFieldList.add(entry.getKey());
- storageFieldList.add(entry.getKey() + "_search");
- storageValuesList.add(values);
- storageValuesList.add(valuesLow);
- }
- }
-
SolrLogger.update(updateQuery, "replace", storageFieldList,
storageValuesList);
@@ -202,6 +168,7 @@ private void findComms(Community comm, List<Object> parentComms)
findComms(comm.getParentCommunity(), parentComms);
}
+ @Override
public void end(Context ctx) throws Exception
{
if (toRemoveQueries != null)
@@ -215,6 +182,7 @@ public void end(Context ctx) throws Exception
toRemoveQueries = null;
}
+ @Override
public void finish(Context ctx) throws Exception
{
}
View
23 dspace-api/src/main/java/org/dspace/statistics/util/StatisticsDataGenerator.java
@@ -16,13 +16,10 @@
import org.dspace.core.ConfigurationManager;
import org.dspace.content.DSpaceObject;
import org.dspace.content.Bitstream;
-import org.dspace.content.DCValue;
-import org.dspace.content.Item;
import org.dspace.eperson.EPerson;
import org.dspace.statistics.SolrLogger;
import java.util.Date;
-import java.util.Map;
import java.text.SimpleDateFormat;
import com.maxmind.geoip.LookupService;
@@ -196,8 +193,6 @@ public static void main(String[] args) throws Exception {
solr.deleteByQuery("*:*");
solr.commit();
- Map<String, String> metadataStorageInfo = SolrLogger.getMetadataStorageInfo();
-
String prevIp = null;
String dbfile = ConfigurationManager.getProperty("usage-statistics", "dbfile");
LookupService cl = new LookupService(dbfile,
@@ -366,24 +361,6 @@ public static void main(String[] args) throws Exception {
doc1.addField("dns", dns.toLowerCase());
}
- if (dso instanceof Item) {
- Item item = (Item) dso;
- // Store the metadata
- for (Map.Entry<String, String> entry : metadataStorageInfo.entrySet())
- {
- String dcField = entry.getValue();
-
- DCValue[] vals = item.getMetadata(dcField.split("\\.")[0],
- dcField.split("\\.")[1], dcField.split("\\.")[2],
- Item.ANY);
- for (DCValue val1 : vals) {
- String val = val1.value;
- doc1.addField(entry.getKey(), val);
- doc1.addField(entry.getKey() + "_search", val.toLowerCase());
- }
- }
- }
-
SolrLogger.storeParents(doc1, dso);
solr.add(doc1);
View
31 dspace-api/src/main/java/org/dspace/statistics/util/StatisticsImporter.java
@@ -29,7 +29,7 @@
import com.maxmind.geoip.Location;
/**
- * Class to load intermediate statistics files (produced from log files by <code>ClassicDSpaceLogConverter</code>) into Solr
+ * Class to load intermediate statistics files (produced from log files by {@link ClassicDSpaceLogConverter}) into Solr.
*
* @see ClassicDSpaceLogConverter
*
@@ -40,17 +40,14 @@
private static final Logger log = Logger.getLogger(StatisticsImporter.class);
/** Date format (for solr) */
- private static SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss");
+ private static final SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss");
/** Solr server connection */
private static HttpSolrServer solr;
/** GEOIP lookup service */
private static LookupService geoipLookup;
- /** Metadata storage information */
- private static Map<String, String> metadataStorageInfo;
-
/** Whether to skip the DNS reverse lookup or not */
private static boolean skipReverseDNS = false;
@@ -67,7 +64,7 @@
private List<Integer> localBitstreams;
/** Whether or not to replace item IDs with local values (for testing) */
- private boolean useLocal;
+ private final boolean useLocal;
/**
* Constructor. Optionally loads local data to replace foreign data
@@ -339,24 +336,6 @@ else if ("view_community".equals(action))
sid.addField("dns", dns.toLowerCase());
}
- if (dso instanceof Item) {
- Item item = (Item) dso;
- // Store the metadata
- for (String storedField : metadataStorageInfo.keySet()) {
- String dcField = metadataStorageInfo.get(storedField);
-
- DCValue[] vals = item.getMetadata(dcField.split("\\.")[0],
- dcField.split("\\.")[1], dcField.split("\\.")[2],
- Item.ANY);
- for (DCValue val1 : vals) {
- String val = val1.value;
- sid.addField(String.valueOf(storedField), val);
- sid.addField(String.valueOf(storedField + "_search"),
- val.toLowerCase());
- }
- }
- }
-
SolrLogger.storeParents(sid, dso);
solr.add(sid);
errors--;
@@ -471,7 +450,6 @@ public static void main(String[] args) throws Exception
}
solr = new HttpSolrServer(sserver);
- metadataStorageInfo = SolrLogger.getMetadataStorageInfo();
String dbfile = ConfigurationManager.getProperty("usage-statistics", "dbfile");
try
{
@@ -495,6 +473,7 @@ public static void main(String[] args) throws Exception
File dir = sample.getParentFile();
FilenameFilter filter = new FilenameFilter()
{
+ @Override
public boolean accept(File dir, String name)
{
return name.startsWith(sample.getName());
@@ -522,7 +501,7 @@ public boolean accept(File dir, String name)
*/
static class DNSCache<K,V> extends LinkedHashMap<K,V>
{
- private int maxCapacity;
+ private final int maxCapacity;
public DNSCache(int initialCapacity, float loadFactor, int maxCapacity)
{

0 comments on commit 8c4862f

Please sign in to comment.