34 changes: 24 additions & 10 deletions pan/tasks/task-xover.cc
Expand Up @@ -172,8 +172,8 @@ void
TaskXOver :: on_nntp_group (NNTP * nntp,
const Quark & group,
unsigned long qty,
unsigned long low,
unsigned long high)
uint64_t low,
uint64_t high)
{
const Quark& servername (nntp->_server);

Expand All @@ -188,7 +188,7 @@ TaskXOver :: on_nntp_group (NNTP * nntp,
<< " low " << low
<< " high " << high);

unsigned long l(low), h(high);
uint64_t l(low), h(high);
_data.set_xover_low (group, nntp->_server, low);
//std::cerr << LINE_ID << " This group's range is [" << low << "..." << high << ']' << std::endl;

Expand All @@ -200,7 +200,7 @@ TaskXOver :: on_nntp_group (NNTP * nntp,
l = std::max (low, high+1-_sample_size);
}
else { // NEW
unsigned long xh (_data.get_xover_high (group, nntp->_server));
uint64_t xh (_data.get_xover_high (group, nntp->_server));
//std::cerr << LINE_ID << " current xover high is " << xh << std::endl;
l = std::max (xh+1, low);
}
Expand All @@ -211,7 +211,7 @@ TaskXOver :: on_nntp_group (NNTP * nntp,
add_steps (h-l);
const int INCREMENT (1000);
MiniTasks_t& minitasks (_server_to_minitasks[servername]);
for (unsigned long m=l; m<=h; m+=INCREMENT) {
for (uint64_t m=l; m<=h; m+=INCREMENT) {
MiniTask mt (MiniTask::XOVER, m, m+INCREMENT);
debug ("adding MiniTask for " << servername << ": xover [" << mt._low << '-' << mt._high << ']');
minitasks.push_front (mt);
Expand Down Expand Up @@ -240,6 +240,19 @@ namespace

return ul;
}
uint64_t view_to_ull (const StringView& view)
{
uint64_t ul = 0ul;

if (!view.empty()) {
errno = 0;
ul = g_ascii_strtoull (view.str, 0, 10);
if (errno)
ul = 0ul;
}

return ul;
}

bool header_is_nonencoded_utf8 (const StringView& in)
{
Expand All @@ -260,10 +273,11 @@ TaskXOver :: on_nntp_line (NNTP * nntp,
_bytes_so_far += line.len;

unsigned int lines=0u;
unsigned long number=0ul, bytes=0ul;
unsigned long bytes=0ul;
uint64_t number=0;
StringView subj, author, date, mid, ref, tmp, xref, l(line);
bool ok = !l.empty();
ok = ok && l.pop_token (tmp, '\t'); if (ok) number = view_to_ul (tmp);
ok = ok && l.pop_token (tmp, '\t'); if (ok) number = view_to_ull (tmp);
ok = ok && l.pop_token (subj, '\t'); if (ok) subj.trim ();
ok = ok && l.pop_token (author, '\t'); if (ok) author.trim ();
ok = ok && l.pop_token (date, '\t'); if (ok) date.trim ();
Expand Down Expand Up @@ -291,10 +305,10 @@ TaskXOver :: on_nntp_line (NNTP * nntp,
// if news server doesn't provide an xref, fake one
char * buf (0);
if (xref.empty())
xref = buf = g_strdup_printf ("%s %s:%lu",
xref = buf = g_strdup_printf ("%s %s:%llu",
nntp->_server.c_str(), nntp->_group.c_str(), number);

unsigned long& h (_high[nntp->_server]);
uint64_t& h (_high[nntp->_server]);
h = std::max (h, number);

const char * fallback_charset = NULL; // FIXME
Expand All @@ -318,7 +332,7 @@ TaskXOver :: on_nntp_line (NNTP * nntp,
++_articles_so_far;

// emit a status update
int& prev = _last_xover_number[nntp];
uint64_t& prev = _last_xover_number[nntp];
increment_step (number - prev);
prev = number;
if (!(_parts_so_far % 500))
Expand Down
12 changes: 6 additions & 6 deletions pan/tasks/task-xover.h
Expand Up @@ -49,14 +49,14 @@ namespace pan
private: // NNTP::Listener
virtual void on_nntp_line (NNTP*, const StringView&);
virtual void on_nntp_done (NNTP*, Health, const StringView&);
virtual void on_nntp_group (NNTP*, const Quark&, unsigned long, unsigned long, unsigned long);
virtual void on_nntp_group (NNTP*, const Quark&, unsigned long, uint64_t, uint64_t);

private: // implementation - minitasks
struct MiniTask {
enum Type { GROUP, XOVER };
Type _type;
unsigned long _low, _high;
MiniTask (Type type, unsigned long low=0ul, unsigned long high=0ul):
uint64_t _low, _high;
MiniTask (Type type, uint64_t low=0ul, uint64_t high=0ul):
_type(type), _low(low), _high(high) {}
};
typedef std::deque<MiniTask> MiniTasks_t;
Expand All @@ -68,14 +68,14 @@ namespace pan
const Quark _group;
std::string _short_group_name;
Mode _mode;
unsigned long _sample_size;
uint64_t _sample_size;
time_t _days_cutoff;
bool _group_xover_is_reffed;
typedef std::map<Quark,unsigned long> server_to_high_t;
typedef std::map<Quark,uint64_t> server_to_high_t;
server_to_high_t _high;
void update_work (bool subtract_one_from_nntp_count=false);
std::set<Quark> _servers_that_got_xover_minitasks;
std::map<NNTP*,int> _last_xover_number;
std::map<NNTP*,uint64_t> _last_xover_number;
unsigned long _bytes_so_far;
unsigned long _parts_so_far;
unsigned long _articles_so_far;
Expand Down
44 changes: 22 additions & 22 deletions pan/usenet-utils/numbers.cc
Expand Up @@ -19,9 +19,9 @@

#include <config.h>
#include <cctype>
#include <algorithm>
#include <pan/general/string-view.h>
#include "numbers.h"
#include <algorithm>

using namespace pan;

Expand All @@ -36,7 +36,7 @@ namespace
typedef Numbers::ranges_t::const_iterator r_cit;

bool
maybe_merge_ranges (Numbers::ranges_t & ranges, int low)
maybe_merge_ranges (Numbers::ranges_t & ranges, uint64_t low)
{
bool merged = false;

Expand All @@ -59,15 +59,15 @@ namespace
/**
* @return the number of articles newly-marked as read
*/
unsigned long
uint64_t
Numbers :: mark_range (const Range& rr)
{
int i;
unsigned long retval = 0;
uint64_t i;
uint64_t retval = 0;
bool range_found = false;
int low_index (std::lower_bound (_marked.begin(), _marked.end(), rr.low)
uint64_t low_index (std::lower_bound (_marked.begin(), _marked.end(), rr.low)
- _marked.begin());
int high_index (std::lower_bound (_marked.begin(), _marked.end(), rr.high)
uint64_t high_index (std::lower_bound (_marked.begin(), _marked.end(), rr.high)
- _marked.begin());

retval = rr.high + 1 - rr.low;
Expand Down Expand Up @@ -122,14 +122,14 @@ Numbers :: mark_range (const Range& rr)
return retval;
}

unsigned long
uint64_t
Numbers :: unmark_range (const Range& ur)
{
unsigned long retval = 0;
int i;
int low_index (std::lower_bound (_marked.begin(), _marked.end(), ur.low)
uint64_t retval = 0;
uint64_t i;
uint64_t low_index (std::lower_bound (_marked.begin(), _marked.end(), ur.low)
- _marked.begin());
int high_index (std::lower_bound (_marked.begin(), _marked.end(), ur.high)
uint64_t high_index (std::lower_bound (_marked.begin(), _marked.end(), ur.high)
- _marked.begin ());

for (i=low_index; i<=high_index && i<(int)_marked.size(); )
Expand Down Expand Up @@ -172,17 +172,17 @@ Numbers :: unmark_range (const Range& ur)
****** MARK (PUBLIC)
*****/

unsigned long
Numbers :: mark_range (unsigned long low, unsigned long high, bool add)
uint64_t
Numbers :: mark_range (uint64_t low, uint64_t high, bool add)
{
const Range r (low, high);
return add ? mark_range(r) : unmark_range(r);
}

bool
Numbers :: mark_one (unsigned long number, bool add)
Numbers :: mark_one (uint64_t number, bool add)
{
const unsigned long changed_qty (mark_range (number, number, add));
const uint64_t changed_qty (mark_range (number, number, add));

if (add)
return changed_qty==0;
Expand All @@ -201,8 +201,8 @@ Numbers :: mark_str (const StringView& str, bool add)
phigh.pop_token (plow, '-');
plow.trim ();
phigh.trim ();
const unsigned long low (plow.empty() ? 0 : strtoul (plow.str, NULL, 10));
const unsigned long high (phigh.empty() ? low : strtoul (phigh.str, NULL, 10));
const uint64_t low (plow.empty() ? 0 : strtoul (plow.str, NULL, 10));
const uint64_t high (phigh.empty() ? low : strtoul (phigh.str, NULL, 10));
mark_range (low, high, add);
}
}
Expand All @@ -218,7 +218,7 @@ Numbers :: clear ()
*****/

void
Numbers :: clip (unsigned long low, unsigned long high)
Numbers :: clip (uint64_t low, uint64_t high)
{
r_it it = std::lower_bound (_marked.begin(), _marked.end(), low);
_marked.erase (_marked.begin(), it);
Expand All @@ -237,7 +237,7 @@ Numbers :: clip (unsigned long low, unsigned long high)
*****/

bool
Numbers :: is_marked (unsigned long number) const
Numbers :: is_marked (uint64_t number) const
{
r_cit it = std::lower_bound (_marked.begin(), _marked.end(), number);

Expand All @@ -262,11 +262,11 @@ Numbers :: to_string (std::string & str) const
{
Range r (*it);

snprintf (buf, sizeof(buf), "%lu", r.low);
snprintf (buf, sizeof(buf), "%llu", r.low);
str += buf;

if (r.low != r.high) {
snprintf (buf, sizeof(buf), "-%lu", r.high);
snprintf (buf, sizeof(buf), "-%llu", r.high);
str += buf;
}

Expand Down
25 changes: 13 additions & 12 deletions pan/usenet-utils/numbers.h
Expand Up @@ -20,6 +20,7 @@
#ifndef __Numbers_h__
#define __Numbers_h__

#include <stdint.h>
#include <vector>
#include <pan/general/string-view.h>

Expand All @@ -38,25 +39,25 @@ namespace pan

/** Simple two-field struct defining a range. */
struct Range {
unsigned long low;
unsigned long high;
uint64_t low;
uint64_t high;
bool operator ==(const Range& that) const {
return low==that.low && high==that.high;
}
Range (): low(0), high(0) {}
Range (unsigned long l, unsigned long h): low(l<h?l:h), high(h>l?h:l) {}
bool contains (unsigned long point) const {
Range (uint64_t l, uint64_t h): low(l<h?l:h), high(h>l?h:l) {}
bool contains (uint64_t point) const {
return low<=point && point<=high;
}
bool contains (const Range& r) const {
return low<=r.low && r.high<=high;
}
int compare (unsigned long point) const {
int compare (uint64_t point) const {
if (point < low) return -1;
if (point > high) return 1;
return 0;
}
bool operator< (unsigned long point) const {
bool operator< (uint64_t point) const {
return high < point;
}
};
Expand All @@ -65,8 +66,8 @@ namespace pan

private:
ranges_t _marked;
unsigned long mark_range (const Range&);
unsigned long unmark_range (const Range&);
uint64_t mark_range (const Range&);
uint64_t unmark_range (const Range&);

public:
bool operator== (const Numbers& that) const {
Expand All @@ -85,7 +86,7 @@ namespace pan
* @param add true if we're adding the number, false if removing
* @return the number's previous state in the set.
*/
bool mark_one (unsigned long number, bool add=true);
bool mark_one (uint64_t number, bool add=true);

/**
* Add or remove the specified range of numbers from the set.
Expand All @@ -96,7 +97,7 @@ namespace pan
* @param add true if we're adding the numbers, false if removing
* @return the quantity of numbers whose presence in the set changed.
*/
unsigned long mark_range (unsigned long low, unsigned long high, bool add=true);
uint64_t mark_range (uint64_t low, uint64_t high, bool add=true);

/**
* Mark numbers from a text string in to_str() and from_str() fromat.
Expand All @@ -108,7 +109,7 @@ namespace pan
/**
* @return true if the number is in this set, false otherwise
*/
bool is_marked (const unsigned long) const;
bool is_marked (const uint64_t) const;

void clear ();

Expand All @@ -122,7 +123,7 @@ namespace pan

public:

void clip (const unsigned long low, const unsigned long high);
void clip (const uint64_t low, const uint64_t high);
};
}

Expand Down