Skip to content

Commit

Permalink
Merge pull request #464 from mwoodiupui/DS-1531
Browse files Browse the repository at this point in the history 
[DS-1531] bug in current DSpace (3.1) with log importing
  • Loading branch information
@mwoodiupui
mwoodiupui committed Feb 5, 2014
2 parents 0bc01a8 + 5ef301c commit 8c4862f
Show file tree
Hide file tree
Showing 4 changed files with 9 additions and 105 deletions.
20 changes: 0 additions & 20 deletions dspace-api/src/main/java/org/dspace/statistics/SolrLogger.java
View file
Expand Up @@ -7,7 +7,6 @@
*/ */
package org.dspace.statistics; package org.dspace.statistics;


import au.com.bytecode.opencsv.CSVParser;
import au.com.bytecode.opencsv.CSVReader; import au.com.bytecode.opencsv.CSVReader;
import au.com.bytecode.opencsv.CSVWriter; import au.com.bytecode.opencsv.CSVWriter;
import com.maxmind.geoip.Location; import com.maxmind.geoip.Location;
Expand Down Expand Up @@ -78,8 +77,6 @@ public class SolrLogger


private static final LookupService locationService; private static final LookupService locationService;


private static Map<String, String> metadataStorageInfo;

private static final boolean useProxies; private static final boolean useProxies;


private static List<String> statisticYearCores = new ArrayList<String>(); private static List<String> statisticYearCores = new ArrayList<String>();
Expand Down Expand Up @@ -563,11 +560,6 @@ public static void postWorkflow(UsageWorkflowEvent usageWorkflowEvent) throws SQ


} }


public static Map<String, String> getMetadataStorageInfo()
{
return metadataStorageInfo;
}

/** /**
* Method just used to log the parents. * Method just used to log the parents.
* <ul> * <ul>
Expand Down Expand Up @@ -676,18 +668,6 @@ public static Map<String, List<String>> queryField(String query,


// We have at least one document good // We have at least one document good
SolrDocument document = response.getResults().get(0); SolrDocument document = response.getResults().get(0);
for (Object storedField : metadataStorageInfo.keySet())
{
// For each of these fields that are stored we are to create a
// list of the values it holds now
java.util.Collection collection = document
.getFieldValues((String) storedField);
List<String> storedVals = new ArrayList<String>();
storedVals.addAll(collection);
// Now add it to our hashmap
currentValsStored.put((String) storedField, storedVals);
}

// System.out.println("HERE"); // System.out.println("HERE");
// Get the info we need // Get the info we need
} }
Expand Down
40 changes: 4 additions & 36 deletions dspace-api/src/main/java/org/dspace/statistics/StatisticsLoggingConsumer.java
View file
Expand Up @@ -37,6 +37,7 @@ public class StatisticsLoggingConsumer implements Consumer


private Set<String> toRemoveQueries = null; private Set<String> toRemoveQueries = null;


@Override
public void initialize() throws Exception public void initialize() throws Exception
{ {


Expand All @@ -45,6 +46,7 @@ public void initialize() throws Exception
// TODO: checkout whether moving of collections, communities and bitstreams works // TODO: checkout whether moving of collections, communities and bitstreams works
// TODO: use async threaded consumer as this might require some processing time // TODO: use async threaded consumer as this might require some processing time
// TODO: we might be able to improve the performance: changing the collection will trigger 4 update commands // TODO: we might be able to improve the performance: changing the collection will trigger 4 update commands
@Override
public void consume(Context ctx, Event event) throws Exception public void consume(Context ctx, Event event) throws Exception
{ {
if (toRemoveQueries == null) if (toRemoveQueries == null)
Expand Down Expand Up @@ -78,45 +80,9 @@ else if (eventType == Event.MODIFY_METADATA
updateQuery, null, null); updateQuery, null, null);


// Get all the metadata // Get all the metadata
Map<String, String> metadataStorageInfo = SolrLogger.getMetadataStorageInfo();
List<String> storageFieldList = new ArrayList<String>(); List<String> storageFieldList = new ArrayList<String>();
List<List<Object>> storageValuesList = new ArrayList<List<Object>>(); List<List<Object>> storageValuesList = new ArrayList<List<Object>>();


for (Map.Entry<String, String> entry : metadataStorageInfo.entrySet())
{
String[] metadataFieldInfo = entry.getValue().split("\\.");

List<Object> values = new ArrayList<Object>();
List<Object> valuesLow = new ArrayList<Object>();
for (int i = 0; i < item.getMetadata(metadataFieldInfo[0],
metadataFieldInfo[1], metadataFieldInfo[2], Item.ANY).length; i++)
{
values.add(item.getMetadata(metadataFieldInfo[0],
metadataFieldInfo[1], metadataFieldInfo[2],
Item.ANY)[i].value);

valuesLow.add(item.getMetadata(metadataFieldInfo[0],
metadataFieldInfo[1], metadataFieldInfo[2],
Item.ANY)[i].value.toLowerCase());
}

List<String> indexedVals = indexedValues.get(entry.getKey());

boolean update = true;
if (values.size() == indexedVals.size() && values.containsAll(indexedVals))
{
update = false;
}

if (update)
{
storageFieldList.add(entry.getKey());
storageFieldList.add(entry.getKey() + "_search");
storageValuesList.add(values);
storageValuesList.add(valuesLow);
}
}

SolrLogger.update(updateQuery, "replace", storageFieldList, SolrLogger.update(updateQuery, "replace", storageFieldList,
storageValuesList); storageValuesList);


Expand Down Expand Up @@ -202,6 +168,7 @@ private void findComms(Community comm, List<Object> parentComms)
findComms(comm.getParentCommunity(), parentComms); findComms(comm.getParentCommunity(), parentComms);
} }


@Override
public void end(Context ctx) throws Exception public void end(Context ctx) throws Exception
{ {
if (toRemoveQueries != null) if (toRemoveQueries != null)
Expand All @@ -215,6 +182,7 @@ public void end(Context ctx) throws Exception
toRemoveQueries = null; toRemoveQueries = null;
} }


@Override
public void finish(Context ctx) throws Exception public void finish(Context ctx) throws Exception
{ {
} }
Expand Down
23 changes: 0 additions & 23 deletions dspace-api/src/main/java/org/dspace/statistics/util/StatisticsDataGenerator.java
View file
Expand Up @@ -16,13 +16,10 @@
import org.dspace.core.ConfigurationManager; import org.dspace.core.ConfigurationManager;
import org.dspace.content.DSpaceObject; import org.dspace.content.DSpaceObject;
import org.dspace.content.Bitstream; import org.dspace.content.Bitstream;
import org.dspace.content.DCValue;
import org.dspace.content.Item;
import org.dspace.eperson.EPerson; import org.dspace.eperson.EPerson;
import org.dspace.statistics.SolrLogger; import org.dspace.statistics.SolrLogger;


import java.util.Date; import java.util.Date;
import java.util.Map;
import java.text.SimpleDateFormat; import java.text.SimpleDateFormat;


import com.maxmind.geoip.LookupService; import com.maxmind.geoip.LookupService;
Expand Down Expand Up @@ -196,8 +193,6 @@ public static void main(String[] args) throws Exception {
solr.deleteByQuery("*:*"); solr.deleteByQuery("*:*");
solr.commit(); solr.commit();


Map<String, String> metadataStorageInfo = SolrLogger.getMetadataStorageInfo();

String prevIp = null; String prevIp = null;
String dbfile = ConfigurationManager.getProperty("usage-statistics", "dbfile"); String dbfile = ConfigurationManager.getProperty("usage-statistics", "dbfile");
LookupService cl = new LookupService(dbfile, LookupService cl = new LookupService(dbfile,
Expand Down Expand Up @@ -366,24 +361,6 @@ public static void main(String[] args) throws Exception {
doc1.addField("dns", dns.toLowerCase()); doc1.addField("dns", dns.toLowerCase());
} }


if (dso instanceof Item) {
Item item = (Item) dso;
// Store the metadata
for (Map.Entry<String, String> entry : metadataStorageInfo.entrySet())
{
String dcField = entry.getValue();

DCValue[] vals = item.getMetadata(dcField.split("\\.")[0],
dcField.split("\\.")[1], dcField.split("\\.")[2],
Item.ANY);
for (DCValue val1 : vals) {
String val = val1.value;
doc1.addField(entry.getKey(), val);
doc1.addField(entry.getKey() + "_search", val.toLowerCase());
}
}
}

SolrLogger.storeParents(doc1, dso); SolrLogger.storeParents(doc1, dso);


solr.add(doc1); solr.add(doc1);
Expand Down
31 changes: 5 additions & 26 deletions dspace-api/src/main/java/org/dspace/statistics/util/StatisticsImporter.java
View file
Expand Up @@ -29,7 +29,7 @@
import com.maxmind.geoip.Location; import com.maxmind.geoip.Location;


/** /**
* Class to load intermediate statistics files (produced from log files by <code>ClassicDSpaceLogConverter</code>) into Solr * Class to load intermediate statistics files (produced from log files by {@link ClassicDSpaceLogConverter}) into Solr.
* *
* @see ClassicDSpaceLogConverter * @see ClassicDSpaceLogConverter
* *
Expand All @@ -40,17 +40,14 @@ public class StatisticsImporter
private static final Logger log = Logger.getLogger(StatisticsImporter.class); private static final Logger log = Logger.getLogger(StatisticsImporter.class);


/** Date format (for solr) */ /** Date format (for solr) */
private static SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss"); private static final SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss");


/** Solr server connection */ /** Solr server connection */
private static HttpSolrServer solr; private static HttpSolrServer solr;


/** GEOIP lookup service */ /** GEOIP lookup service */
private static LookupService geoipLookup; private static LookupService geoipLookup;


/** Metadata storage information */
private static Map<String, String> metadataStorageInfo;

/** Whether to skip the DNS reverse lookup or not */ /** Whether to skip the DNS reverse lookup or not */
private static boolean skipReverseDNS = false; private static boolean skipReverseDNS = false;


Expand All @@ -67,7 +64,7 @@ public class StatisticsImporter
private List<Integer> localBitstreams; private List<Integer> localBitstreams;


/** Whether or not to replace item IDs with local values (for testing) */ /** Whether or not to replace item IDs with local values (for testing) */
private boolean useLocal; private final boolean useLocal;


/** /**
* Constructor. Optionally loads local data to replace foreign data * Constructor. Optionally loads local data to replace foreign data
Expand Down Expand Up @@ -339,24 +336,6 @@ else if ("view_community".equals(action))
sid.addField("dns", dns.toLowerCase()); sid.addField("dns", dns.toLowerCase());
} }


if (dso instanceof Item) {
Item item = (Item) dso;
// Store the metadata
for (String storedField : metadataStorageInfo.keySet()) {
String dcField = metadataStorageInfo.get(storedField);

DCValue[] vals = item.getMetadata(dcField.split("\\.")[0],
dcField.split("\\.")[1], dcField.split("\\.")[2],
Item.ANY);
for (DCValue val1 : vals) {
String val = val1.value;
sid.addField(String.valueOf(storedField), val);
sid.addField(String.valueOf(storedField + "_search"),
val.toLowerCase());
}
}
}

SolrLogger.storeParents(sid, dso); SolrLogger.storeParents(sid, dso);
solr.add(sid); solr.add(sid);
errors--; errors--;
Expand Down Expand Up @@ -471,7 +450,6 @@ public static void main(String[] args) throws Exception
} }
solr = new HttpSolrServer(sserver); solr = new HttpSolrServer(sserver);


metadataStorageInfo = SolrLogger.getMetadataStorageInfo();
String dbfile = ConfigurationManager.getProperty("usage-statistics", "dbfile"); String dbfile = ConfigurationManager.getProperty("usage-statistics", "dbfile");
try try
{ {
Expand All @@ -495,6 +473,7 @@ public static void main(String[] args) throws Exception
File dir = sample.getParentFile(); File dir = sample.getParentFile();
FilenameFilter filter = new FilenameFilter() FilenameFilter filter = new FilenameFilter()
{ {
@Override
public boolean accept(File dir, String name) public boolean accept(File dir, String name)
{ {
return name.startsWith(sample.getName()); return name.startsWith(sample.getName());
Expand Down Expand Up @@ -522,7 +501,7 @@ public boolean accept(File dir, String name)
*/ */
static class DNSCache<K,V> extends LinkedHashMap<K,V> static class DNSCache<K,V> extends LinkedHashMap<K,V>
{ {
private int maxCapacity; private final int maxCapacity;


public DNSCache(int initialCapacity, float loadFactor, int maxCapacity) public DNSCache(int initialCapacity, float loadFactor, int maxCapacity)
{ {
Expand Down

0 comments on commit 8c4862f

Please sign in to comment.