Skip to content
This repository has been archived by the owner on Dec 24, 2019. It is now read-only.

Commit

Permalink
Fix issue 439
Browse files Browse the repository at this point in the history
Now it creates a local collection that stores the last article processed
  • Loading branch information
creat89 committed Dec 2, 2019
1 parent 79e1029 commit e7477e2
Show file tree
Hide file tree
Showing 7 changed files with 249 additions and 21 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -8,4 +8,8 @@
</communicationChannel>
</extension>

<extension point="com.googlecode.pongo.runtime.osgi">
<class name="org.eclipse.scava.platform.communicationchannel.nntp.model.NntpDataManager"/>
<class name="org.eclipse.scava.platform.communicationchannel.nntp.model.NntpDatum"/>
</extension>
</plugin>
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/*******************************************************************************
* Copyright (c) 2018 University of York
* Copyright (c) 2019 Edge Hill University
*
* This program and the accompanying materials are made
* available under the terms of the Eclipse Public License 2.0
Expand All @@ -16,6 +16,8 @@
import org.apache.commons.net.nntp.NNTPClient;
import org.apache.commons.net.nntp.NewsgroupInfo;
import org.eclipse.scava.platform.Date;
import org.eclipse.scava.platform.communicationchannel.nntp.model.NntpDataManager;
import org.eclipse.scava.platform.communicationchannel.nntp.model.NntpDatum;
import org.eclipse.scava.platform.delta.communicationchannel.CommunicationChannelArticle;
import org.eclipse.scava.platform.delta.communicationchannel.CommunicationChannelDelta;
import org.eclipse.scava.platform.delta.communicationchannel.ICommunicationChannelManager;
Expand All @@ -42,20 +44,39 @@ public CommunicationChannelDelta getDelta(DB db, NntpNewsGroup newsgroup, Date d

NewsgroupInfo newsgroupInfo = NntpUtil.selectNewsgroup(nntpClient, newsgroup);
long lastArticle = newsgroupInfo.getLastArticleLong();
NntpDataManager nntpDataManager = new NntpDataManager(db);

NntpDatum data=null;
Iterable<NntpDatum> dataIt = nntpDataManager.getNntpData().find(NntpDatum.OSSMERTERID.eq(newsgroup.getOSSMeterId()));
long lastArticleChecked=-1;
for(NntpDatum nd : dataIt)
{
data=nd;
}
if(data==null)
{
data = new NntpDatum();
data.setDate(date.toString());
data.setOssmerterId(newsgroup.getOSSMeterId());
data.setLastArticleChecked(-1);
nntpDataManager.getNntpData().add(data);
nntpDataManager.sync();
}
else
{
if(date.compareTo(new Date(data.getDate()).addDays(1))<0)
{
data.setDate(date.toString());
data.setOssmerterId(newsgroup.getOSSMeterId());
data.setLastArticleChecked(-1);
nntpDataManager.sync();
}
else
{
lastArticleChecked=data.getLastArticleChecked();
}
}

// The following statement is not really needed, but I added it to speed up running,
// in the date is far latter than the first day of the newsgroup.
// if (Integer.parseInt(newsgroup.getLastArticleChecked())<134500)
// newsgroup.setLastArticleChecked("134500"); //137500");

/*FIXME: Edge Hill. This will be always the original value
* Readers cannot change values in the MongoDB
* And therefore the reader will always start from the first article*/
//TODO: This means that it is necessary to create a variable that stores the last article checked
String lac = newsgroup.getLastArticleChecked();
if (lac == null || lac.equals("") || lac.equals("null"))
lac = "-1";
long lastArticleChecked = Long.parseLong(lac);
if (lastArticleChecked<0) lastArticleChecked = newsgroupInfo.getFirstArticleLong();

CommunicationChannelDelta delta = new CommunicationChannelDelta();
Expand Down Expand Up @@ -149,7 +170,7 @@ else if (date.compareTo(articleDate) == 0)
// System.out.println("dayNOTCompleted");
}
else {
logger.warn("It has been found an article that could mean previous deltas incomplete " + article.getDate());
logger.warn("It has been found an article that could mean previous deltas are incomplete " + article.getDate());
}
}
else
Expand All @@ -162,19 +183,19 @@ else if (date.compareTo(articleDate) == 0)
String mainMessage="Article Date has been found to be null and it is impossible to recover. ";
if(delta.getArticles().size()==0)
{
logger.error(mainMessage+"Returning empty delta.");
nntpClient.disconnect();
return delta;
logger.error(mainMessage+"Returning empty delta.");
}
else
{
dayCompleted=true;
logger.error(mainMessage+"Returning partial delta.");
}
dayCompleted=true;
}
}
nntpClient.disconnect();
newsgroup.setLastArticleChecked(lastArticleChecked+"");
nntpClient.disconnect();
data.setLastArticleChecked(lastArticleChecked);
data.setDate(date.toString());
nntpDataManager.sync();
logger.info(newsgroup.getNewsGroupName() + " on " + date.toString()+") contains:\t"+ delta.getArticles().size() + " nntp articles");
return delta;
}
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
/*******************************************************************************
* Copyright (c) 2019 Edge Hill University
*
* This program and the accompanying materials are made
* available under the terms of the Eclipse Public License 2.0
* which is available at https://www.eclipse.org/legal/epl-2.0/
*
* SPDX-License-Identifier: EPL-2.0
******************************************************************************/
package org.eclipse.scava.platform.communicationchannel.nntp.model;

import com.googlecode.pongo.runtime.*;
import com.mongodb.*;
// protected region custom-imports on begin
// protected region custom-imports end

public class NntpDataManager extends PongoDB {

public NntpDataManager() {}

public NntpDataManager(DB db) {
setDb(db);
}

protected NntpDatumCollection nntpData = null;

// protected region custom-fields-and-methods on begin
// protected region custom-fields-and-methods end


public NntpDatumCollection getNntpData() {
return nntpData;
}


@Override
public void setDb(DB db) {
super.setDb(db);
nntpData = new NntpDatumCollection(db.getCollection("NntpDataManager.nntpData"));
pongoCollections.add(nntpData);
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,64 @@
/*******************************************************************************
* Copyright (c) 2019 Edge Hill University
*
* This program and the accompanying materials are made
* available under the terms of the Eclipse Public License 2.0
* which is available at https://www.eclipse.org/legal/epl-2.0/
*
* SPDX-License-Identifier: EPL-2.0
******************************************************************************/
package org.eclipse.scava.platform.communicationchannel.nntp.model;

import com.googlecode.pongo.runtime.Pongo;
import com.googlecode.pongo.runtime.querying.NumericalQueryProducer;
import com.googlecode.pongo.runtime.querying.StringQueryProducer;


public class NntpDatum extends Pongo {



public NntpDatum() {
super();
OSSMERTERID.setOwningType("org.eclipse.scava.platform.communicationchannel.nntp.model.NntpDatum");
LASTARTICLECHECKED.setOwningType("org.eclipse.scava.platform.communicationchannel.nntp.model.NntpDatum");
DATE.setOwningType("org.eclipse.scava.platform.communicationchannel.nntp.model.NntpDatum");
}

public static StringQueryProducer OSSMERTERID = new StringQueryProducer("ossmerterId");
public static NumericalQueryProducer LASTARTICLECHECKED = new NumericalQueryProducer("lastArticleChecked");
public static StringQueryProducer DATE = new StringQueryProducer("date");


public String getOssmerterId() {
return parseString(dbObject.get("ossmerterId")+"", "");
}

public NntpDatum setOssmerterId(String ossmerterId) {
dbObject.put("ossmerterId", ossmerterId);
notifyChanged();
return this;
}
public long getLastArticleChecked() {
return parseLong(dbObject.get("lastArticleChecked")+"", 0);
}

public NntpDatum setLastArticleChecked(long lastArticleChecked) {
dbObject.put("lastArticleChecked", lastArticleChecked);
notifyChanged();
return this;
}
public String getDate() {
return parseString(dbObject.get("date")+"", "");
}

public NntpDatum setDate(String date) {
dbObject.put("date", date);
notifyChanged();
return this;
}




}
Original file line number Diff line number Diff line change
@@ -0,0 +1,57 @@
/*******************************************************************************
* Copyright (c) 2019 Edge Hill University
*
* This program and the accompanying materials are made
* available under the terms of the Eclipse Public License 2.0
* which is available at https://www.eclipse.org/legal/epl-2.0/
*
* SPDX-License-Identifier: EPL-2.0
******************************************************************************/
package org.eclipse.scava.platform.communicationchannel.nntp.model;

import com.googlecode.pongo.runtime.*;
import java.util.*;
import com.mongodb.*;

public class NntpDatumCollection extends PongoCollection<NntpDatum> {

public NntpDatumCollection(DBCollection dbCollection) {
super(dbCollection);
createIndex("ossmerterId");
}

public Iterable<NntpDatum> findById(String id) {
return new IteratorIterable<NntpDatum>(new PongoCursorIterator<NntpDatum>(this, dbCollection.find(new BasicDBObject("_id", id))));
}

public Iterable<NntpDatum> findByOssmerterId(String q) {
return new IteratorIterable<NntpDatum>(new PongoCursorIterator<NntpDatum>(this, dbCollection.find(new BasicDBObject("ossmerterId", q + ""))));
}

public NntpDatum findOneByOssmerterId(String q) {
NntpDatum nntpDatum = (NntpDatum) PongoFactory.getInstance().createPongo(dbCollection.findOne(new BasicDBObject("ossmerterId", q + "")));
if (nntpDatum != null) {
nntpDatum.setPongoCollection(this);
}
return nntpDatum;
}


public long countByOssmerterId(String q) {
return dbCollection.count(new BasicDBObject("ossmerterId", q + ""));
}

@Override
public Iterator<NntpDatum> iterator() {
return new PongoCursorIterator<NntpDatum>(this, dbCollection.find());
}

public void add(NntpDatum nntpDatum) {
super.add(nntpDatum);
}

public void remove(NntpDatum nntpDatum) {
super.remove(nntpDatum);
}

}
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
<?xml version="1.0" encoding="ASCII"?>
<ecore:EPackage xmi:version="2.0" xmlns:xmi="http://www.omg.org/XMI" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:ecore="http://www.eclipse.org/emf/2002/Ecore" name="org.eclipse.scava.platform.communicationchannel.nntp.model">
<eClassifiers xsi:type="ecore:EClass" name="NntpDataManager">
<eAnnotations source="db">
<details key="qualifiedCollectionNames" value="true"/>
</eAnnotations>
<eAnnotations source="customize"/>
<eStructuralFeatures xsi:type="ecore:EReference" name="nntpData" upperBound="-1" eType="//NntpDatum" containment="true">
<eAnnotations source="series"/>
</eStructuralFeatures>
</eClassifiers>
<eClassifiers xsi:type="ecore:EClass" name="NntpDatum">
<eStructuralFeatures xsi:type="ecore:EAttribute" name="ossmerterId">
<eAnnotations source="searchable"/>
<eType xsi:type="ecore:EDataType" href="http://www.eclipse.org/emf/2002/Ecore#//EString"/>
</eStructuralFeatures>
<eStructuralFeatures xsi:type="ecore:EAttribute" name="lastArticleChecked">
<eType xsi:type="ecore:EDataType" href="http://www.eclipse.org/emf/2002/Ecore#//ELong"/>
</eStructuralFeatures>
<eStructuralFeatures xsi:type="ecore:EAttribute" name="date">
<eType xsi:type="ecore:EDataType" href="http://www.eclipse.org/emf/2002/Ecore#//EString"/>
</eStructuralFeatures>
</eClassifiers>
</ecore:EPackage>
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
package org.eclipse.scava.platform.communicationchannel.nntp.model;

@db(qualifiedCollectionNames="true")
@customize
class NntpDataManager {
@series
val NntpDatum[*] nntpData;
}

class NntpDatum {
@searchable
attr String ossmerterId;
attr long lastArticleChecked;
attr String date;
}

0 comments on commit e7477e2

Please sign in to comment.