Skip to content

Commit

Permalink
Merge pull request #464 from mwoodiupui/DS-1531
Browse files Browse the repository at this point in the history 
[DS-1531] bug in current DSpace (3.1) with log importing
  • Loading branch information
@mwoodiupui
mwoodiupui committed Feb 5, 2014
2 parents 0bc01a8 + 5ef301c commit 8c4862f
Show file tree
Hide file tree
Showing 4 changed files with 9 additions and 105 deletions.
20 changes: 0 additions & 20 deletions dspace-api/src/main/java/org/dspace/statistics/SolrLogger.java
View file
Expand Up @@ -7,7 +7,6 @@
*/
package org.dspace.statistics;

import au.com.bytecode.opencsv.CSVParser;
import au.com.bytecode.opencsv.CSVReader;
import au.com.bytecode.opencsv.CSVWriter;
import com.maxmind.geoip.Location;
Expand Down Expand Up @@ -78,8 +77,6 @@ public class SolrLogger

private static final LookupService locationService;

private static Map<String, String> metadataStorageInfo;

private static final boolean useProxies;

private static List<String> statisticYearCores = new ArrayList<String>();
Expand Down Expand Up @@ -563,11 +560,6 @@ public static void postWorkflow(UsageWorkflowEvent usageWorkflowEvent) throws SQ

}

public static Map<String, String> getMetadataStorageInfo()
{
return metadataStorageInfo;
}

/**
* Method just used to log the parents.
* <ul>
Expand Down Expand Up @@ -676,18 +668,6 @@ public static Map<String, List<String>> queryField(String query,

// We have at least one document good
SolrDocument document = response.getResults().get(0);
for (Object storedField : metadataStorageInfo.keySet())
{
// For each of these fields that are stored we are to create a
// list of the values it holds now
java.util.Collection collection = document
.getFieldValues((String) storedField);
List<String> storedVals = new ArrayList<String>();
storedVals.addAll(collection);
// Now add it to our hashmap
currentValsStored.put((String) storedField, storedVals);
}

// System.out.println("HERE");
// Get the info we need
}
Expand Down
40 changes: 4 additions & 36 deletions dspace-api/src/main/java/org/dspace/statistics/StatisticsLoggingConsumer.java
View file
Expand Up @@ -37,6 +37,7 @@ public class StatisticsLoggingConsumer implements Consumer

private Set<String> toRemoveQueries = null;

@Override
public void initialize() throws Exception
{

Expand All @@ -45,6 +46,7 @@ public void initialize() throws Exception
// TODO: checkout whether moving of collections, communities and bitstreams works
// TODO: use async threaded consumer as this might require some processing time
// TODO: we might be able to improve the performance: changing the collection will trigger 4 update commands
@Override
public void consume(Context ctx, Event event) throws Exception
{
if (toRemoveQueries == null)
Expand Down Expand Up @@ -78,45 +80,9 @@ else if (eventType == Event.MODIFY_METADATA
updateQuery, null, null);

// Get all the metadata
Map<String, String> metadataStorageInfo = SolrLogger.getMetadataStorageInfo();
List<String> storageFieldList = new ArrayList<String>();
List<List<Object>> storageValuesList = new ArrayList<List<Object>>();

for (Map.Entry<String, String> entry : metadataStorageInfo.entrySet())
{
String[] metadataFieldInfo = entry.getValue().split("\\.");

List<Object> values = new ArrayList<Object>();
List<Object> valuesLow = new ArrayList<Object>();
for (int i = 0; i < item.getMetadata(metadataFieldInfo[0],
metadataFieldInfo[1], metadataFieldInfo[2], Item.ANY).length; i++)
{
values.add(item.getMetadata(metadataFieldInfo[0],
metadataFieldInfo[1], metadataFieldInfo[2],
Item.ANY)[i].value);

valuesLow.add(item.getMetadata(metadataFieldInfo[0],
metadataFieldInfo[1], metadataFieldInfo[2],
Item.ANY)[i].value.toLowerCase());
}

List<String> indexedVals = indexedValues.get(entry.getKey());

boolean update = true;
if (values.size() == indexedVals.size() && values.containsAll(indexedVals))
{
update = false;
}

if (update)
{
storageFieldList.add(entry.getKey());
storageFieldList.add(entry.getKey() + "_search");
storageValuesList.add(values);
storageValuesList.add(valuesLow);
}
}

SolrLogger.update(updateQuery, "replace", storageFieldList,
storageValuesList);

Expand Down Expand Up @@ -202,6 +168,7 @@ private void findComms(Community comm, List<Object> parentComms)
findComms(comm.getParentCommunity(), parentComms);
}

@Override
public void end(Context ctx) throws Exception
{
if (toRemoveQueries != null)
Expand All @@ -215,6 +182,7 @@ public void end(Context ctx) throws Exception
toRemoveQueries = null;
}

@Override
public void finish(Context ctx) throws Exception
{
}
Expand Down
23 changes: 0 additions & 23 deletions dspace-api/src/main/java/org/dspace/statistics/util/StatisticsDataGenerator.java
View file
Expand Up @@ -16,13 +16,10 @@
import org.dspace.core.ConfigurationManager;
import org.dspace.content.DSpaceObject;
import org.dspace.content.Bitstream;
import org.dspace.content.DCValue;
import org.dspace.content.Item;
import org.dspace.eperson.EPerson;
import org.dspace.statistics.SolrLogger;

import java.util.Date;
import java.util.Map;
import java.text.SimpleDateFormat;

import com.maxmind.geoip.LookupService;
Expand Down Expand Up @@ -196,8 +193,6 @@ public static void main(String[] args) throws Exception {
solr.deleteByQuery("*:*");
solr.commit();

Map<String, String> metadataStorageInfo = SolrLogger.getMetadataStorageInfo();

String prevIp = null;
String dbfile = ConfigurationManager.getProperty("usage-statistics", "dbfile");
LookupService cl = new LookupService(dbfile,
Expand Down Expand Up @@ -366,24 +361,6 @@ public static void main(String[] args) throws Exception {
doc1.addField("dns", dns.toLowerCase());
}

if (dso instanceof Item) {
Item item = (Item) dso;
// Store the metadata
for (Map.Entry<String, String> entry : metadataStorageInfo.entrySet())
{
String dcField = entry.getValue();

DCValue[] vals = item.getMetadata(dcField.split("\\.")[0],
dcField.split("\\.")[1], dcField.split("\\.")[2],
Item.ANY);
for (DCValue val1 : vals) {
String val = val1.value;
doc1.addField(entry.getKey(), val);
doc1.addField(entry.getKey() + "_search", val.toLowerCase());
}
}
}

SolrLogger.storeParents(doc1, dso);

solr.add(doc1);
Expand Down
31 changes: 5 additions & 26 deletions dspace-api/src/main/java/org/dspace/statistics/util/StatisticsImporter.java
View file
Expand Up @@ -29,7 +29,7 @@
import com.maxmind.geoip.Location;

/**
* Class to load intermediate statistics files (produced from log files by <code>ClassicDSpaceLogConverter</code>) into Solr
* Class to load intermediate statistics files (produced from log files by {@link ClassicDSpaceLogConverter}) into Solr.
*
* @see ClassicDSpaceLogConverter
*
Expand All @@ -40,17 +40,14 @@ public class StatisticsImporter
private static final Logger log = Logger.getLogger(StatisticsImporter.class);

/** Date format (for solr) */
private static SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss");
private static final SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss");

/** Solr server connection */
private static HttpSolrServer solr;

/** GEOIP lookup service */
private static LookupService geoipLookup;

/** Metadata storage information */
private static Map<String, String> metadataStorageInfo;

/** Whether to skip the DNS reverse lookup or not */
private static boolean skipReverseDNS = false;

Expand All @@ -67,7 +64,7 @@ public class StatisticsImporter
private List<Integer> localBitstreams;

/** Whether or not to replace item IDs with local values (for testing) */
private boolean useLocal;
private final boolean useLocal;

/**
* Constructor. Optionally loads local data to replace foreign data
Expand Down Expand Up @@ -339,24 +336,6 @@ else if ("view_community".equals(action))
sid.addField("dns", dns.toLowerCase());
}

if (dso instanceof Item) {
Item item = (Item) dso;
// Store the metadata
for (String storedField : metadataStorageInfo.keySet()) {
String dcField = metadataStorageInfo.get(storedField);

DCValue[] vals = item.getMetadata(dcField.split("\\.")[0],
dcField.split("\\.")[1], dcField.split("\\.")[2],
Item.ANY);
for (DCValue val1 : vals) {
String val = val1.value;
sid.addField(String.valueOf(storedField), val);
sid.addField(String.valueOf(storedField + "_search"),
val.toLowerCase());
}
}
}

SolrLogger.storeParents(sid, dso);
solr.add(sid);
errors--;
Expand Down Expand Up @@ -471,7 +450,6 @@ public static void main(String[] args) throws Exception
}
solr = new HttpSolrServer(sserver);

metadataStorageInfo = SolrLogger.getMetadataStorageInfo();
String dbfile = ConfigurationManager.getProperty("usage-statistics", "dbfile");
try
{
Expand All @@ -495,6 +473,7 @@ public static void main(String[] args) throws Exception
File dir = sample.getParentFile();
FilenameFilter filter = new FilenameFilter()
{
@Override
public boolean accept(File dir, String name)
{
return name.startsWith(sample.getName());
Expand Down Expand Up @@ -522,7 +501,7 @@ public boolean accept(File dir, String name)
*/
static class DNSCache<K,V> extends LinkedHashMap<K,V>
{
private int maxCapacity;
private final int maxCapacity;

public DNSCache(int initialCapacity, float loadFactor, int maxCapacity)
{
Expand Down

0 comments on commit 8c4862f

Please sign in to comment.