Skip to content
Permalink
master
Switch branches/tags

Name already in use

A tag already exists with the provided branch name. Many Git commands accept both tag and branch names, so creating this branch may cause unexpected behavior. Are you sure you want to create this branch?
Go to file
 
 
Cannot retrieve contributors at this time
// Package nntp implements a client for the news protocol NNTP,
// as defined in RFC 3977.
package nntp
import (
"bufio"
"bytes"
"compress/zlib"
"crypto/tls"
"fmt"
"io"
"io/ioutil"
"net/textproto"
"sort"
"strconv"
"strings"
"time"
log "github.com/Sirupsen/logrus"
)
// timeFormatNew is the NNTP time format string for NEWNEWS / NEWGROUPS
const timeFormatNew = "20060102 150405"
// timeFormatDate is the NNTP time format string for responses to the DATE command
const timeFormatDate = "20060102150405"
// A ProtocolError represents responses from an NNTP server
// that seem incorrect for NNTP.
type ProtocolError string
func (p ProtocolError) Error() string {
return string(p)
}
// A Conn represents a connection to an NNTP server. The connection with
// an NNTP server is stateful; it keeps track of what group you have
// selected, if any, and (if you have a group selected) which article is
// current, next, or previous.
//
// Some methods that return information about a specific message take
// either a message-id, which is global across all NNTP servers, groups,
// and messages, or a message-number, which is an integer number that is
// local to the NNTP session and currently selected group.
//
// For all methods that return an io.Reader (or an *Article, which contains
// an io.Reader), that io.Reader is only valid until the next call to a
// method of Conn.
type Conn struct {
conn *textproto.Conn
Banner string
compress bool
}
// New connects to an NNTP server.
// The network and addr are passed to net.Dial to
// make the connection.
//
// Example:
// conn, err := nntp.Dial("tcp", "my.news:nntp")
//
func New(network, addr string) (*Conn, error) {
c, err := textproto.Dial(network, addr)
if err != nil {
return nil, err
}
return newClient(c)
}
// NewTLS connects with TLS
func NewTLS(net, addr string, cfg *tls.Config) (*Conn, error) {
c, err := tls.Dial(net, addr, cfg)
if err != nil {
return nil, err
}
return newClient(textproto.NewConn(c))
}
func newClient(conn *textproto.Conn) (*Conn, error) {
_, msg, err := conn.ReadCodeLine(200)
if err != nil {
return nil, err
}
return &Conn{
conn: conn,
Banner: msg,
}, nil
}
// Command sends a low-level command and get a response.
//
// This will return an error if the code doesn't match the expectCode
// prefix. For example, if you specify "200", the response code MUST
// be 200 or you'll get an error. If you specify "2", any code from
// 200 (inclusive) to 300 (exclusive) will be success. An expectCode
// of -1 disables this behavior.
func (c *Conn) Command(cmd string, expectCode int) (int, string, error) {
log.Infof("client: %s", cmd)
err := c.conn.PrintfLine(cmd)
if err != nil {
return 0, "", err
}
code, msg, err := c.conn.ReadCodeLine(expectCode)
log.Infof("server code: %d, msg: %s, err: %v", code, msg, err)
return code, msg, err
}
// MultilineCommand wraps the functionality to
func (c *Conn) MultilineCommand(cmd string, expectCode int) (int, []string, error) {
log.Infof("client: %s", cmd)
err := c.conn.PrintfLine(cmd)
if err != nil {
return 0, nil, err
}
rc, l, err := c.conn.ReadCodeLine(expectCode)
log.Infof("server code: %d, msg: %s, err: %v", rc, l, err)
if err != nil {
return rc, nil, err
}
lines := []string{l}
ls, err := c.conn.ReadDotLines()
if err != nil {
return rc, nil, err
}
for _, line := range ls {
log.Debugf("server: %v", line)
}
lines = append(lines, ls...)
return rc, lines, err
}
// A Group gives information about a single news group on the server.
type Group struct {
Name string
// High and low message-numbers
High int64
Low int64
// Estimated count of articles in the group
Count int64
// Status indicates if general posting is allowed --
// typical values are "y", "n", or "m".
Status string
}
// An Article represents an NNTP article.
type Article struct {
Header map[string][]string
Body []string
}
func (a *Article) String() string {
res := []string{}
for k, v := range a.Header {
res = append(res, fmt.Sprintf("%s: %s", k, strings.Join(v, ",")))
}
res = append(res, "")
res = append(res, a.Body...)
return strings.Join(res, "\n")
}
func maybeID(cmd, id string) string {
if len(id) > 0 {
return cmd + " " + id
}
return cmd
}
// Authenticate logs in to the NNTP server.
// It only sends the password if the server requires one.
func (c *Conn) Authenticate(username, password string) error {
// Spec says you might not need a password and a username is it. This needs
// to change to support that. Status code 381 means to send a password
code, _, err := c.Command(fmt.Sprintf("AUTHINFO USER %s", username), 381)
if code/100 == 3 {
_, _, err = c.Command(fmt.Sprintf("AUTHINFO PASS %s", password), 281)
}
return err
}
// SetCompression turns on compression for this connection
func (c *Conn) SetCompression() error {
_, _, err := c.Command("XFEATURE COMPRESS GZIP", 290)
if err == nil {
c.compress = true
}
return err
}
// ModeReader switches the NNTP server to "reader" mode, if it
// is a mode-switching server.
func (c *Conn) ModeReader() error {
_, _, err := c.Command("MODE READER", 20)
return err
}
// NewGroups returns a list of groups added since the given time.
func (c *Conn) NewGroups(since time.Time) ([]*Group, error) {
_, _, err := c.Command(fmt.Sprintf("NEWGROUPS %s GMT", since.Format(timeFormatNew)), 231)
if err != nil {
return nil, err
}
lines, err := c.conn.ReadDotLines()
if err != nil {
return nil, err
}
return parseNewGroups(lines)
}
// NewNews returns a list of the IDs of articles posted
// to the given group since the given time.
func (c *Conn) NewNews(group string, since time.Time) ([]string, error) {
_, lines, err := c.MultilineCommand(fmt.Sprintf("NEWNEWS %s %s GMT", group, since.Format(timeFormatNew)), 230)
if err != nil {
return nil, err
}
sort.Strings(lines)
w := 0
for r, s := range lines {
if r == 0 || lines[r-1] != s {
lines[w] = s
w++
}
}
lines = lines[0:w]
return lines, nil
}
// MessageOverview of a message returned by OVER/XOVER command.
type MessageOverview struct {
MessageNumber int64 // Message number in the group
Subject string // Subject header value. Empty if the header is missing.
From string // From header value. Empty is the header is missing.
Date time.Time // Parsed Date header value. Zero if the header is missing or unparseable.
MessageID string // Message-Id header value. Empty is the header is missing.
References []string // Message-Id's of referenced messages (References header value, split on spaces). Empty if the header is missing.
Bytes int // Message size in bytes, called :bytes metadata item in RFC3977.
Lines int // Message size in lines, called :lines metadata item in RFC3977.
Extra []string // Any additional fields returned by the server.
}
// Xref returns the Xref header if set otherwise the empty string.
func (m *MessageOverview) Xref() string {
for _, line := range m.Extra {
if strings.HasPrefix(line, "Xref") && strings.Contains(line, ":") {
xref := strings.SplitN(line, ":", 2)
return strings.TrimSpace(xref[1])
}
}
return ""
}
// Overview returns overviews of all messages in the current group with message number between
// begin and end, inclusive.
func (c *Conn) Overview(begin, end int64) ([]MessageOverview, error) {
_, _, err := c.Command(fmt.Sprintf("XOVER %d-%d", begin, end), 224)
if err != nil {
return nil, err
}
result := []MessageOverview{}
var lines []string
if c.compress {
log.Debugf("Reading compressed data")
zr, err := zlib.NewReader(c.conn.R)
if err != nil {
return nil, err
}
defer zr.Close()
scanner := bufio.NewScanner(zr)
for scanner.Scan() {
l := scanner.Text()
if "." == l {
break
}
lines = append(lines, l)
}
//Read last dot out of buffer
c.conn.ReadLine()
} else {
lines, err = c.conn.ReadDotLines()
log.Debugf("Read %d lines from connection", len(lines))
if err != nil {
return nil, err
}
}
for _, line := range lines {
if "" == line {
return result, nil
}
overview := MessageOverview{}
ss := strings.SplitN(strings.TrimSpace(line), "\t", 9)
if len(ss) < 8 {
return nil, ProtocolError("short header listing line: " + line + strconv.Itoa(len(ss)))
}
overview.MessageNumber, err = strconv.ParseInt(ss[0], 10, 64)
if err != nil {
return nil, ProtocolError("bad message number '" + ss[0] + "' in line: " + line)
}
overview.Subject = ss[1]
overview.From = ss[2]
overview.Date, err = parseDate(ss[3])
if err != nil {
// Inability to parse date is not fatal: the field in the message may be broken or missing.
overview.Date = time.Time{}
}
overview.MessageID = ss[4]
overview.References = strings.Split(ss[5], " ") // Message-Id's contain no spaces, so this is safe.
overview.Bytes, err = strconv.Atoi(ss[6])
if err != nil {
return nil, ProtocolError("bad byte count '" + ss[6] + "'in line:" + line)
}
overview.Lines, err = strconv.Atoi(ss[7])
if err != nil {
return nil, ProtocolError("bad line count '" + ss[7] + "'in line:" + line)
}
overview.Extra = append([]string{}, ss[8:]...)
result = append(result, overview)
}
return result, nil
}
func parseGroup(line string) (*Group, error) {
ss := strings.SplitN(strings.TrimSpace(line), " ", 4)
if len(ss) < 4 {
return nil, ProtocolError("short group info line: " + line)
}
low, err := strconv.ParseInt(ss[1], 10, 64)
if err != nil {
return nil, ProtocolError("bad high article number in line: " + line)
}
high, err := strconv.ParseInt(ss[2], 10, 64)
if err != nil {
return nil, ProtocolError("bad low article number in line: " + line)
}
count, err := strconv.ParseInt(ss[0], 10, 64)
if err != nil {
return nil, ProtocolError("bad count in line: " + line)
}
return &Group{
Name: ss[3],
High: high,
Low: low,
Count: count,
}, nil
}
// parseNewGroups is used to parse a list of group states.
func parseNewGroups(lines []string) ([]*Group, error) {
res := make([]*Group, len(lines))
for i, line := range lines {
ss := strings.SplitN(strings.TrimSpace(line), " ", 4)
if len(ss) < 4 {
return nil, ProtocolError("short group info line: " + line)
}
high, err := strconv.ParseInt(ss[1], 10, 64)
if err != nil {
return nil, ProtocolError("bad number in line: " + line)
}
low, err := strconv.ParseInt(ss[2], 10, 64)
if err != nil {
return nil, ProtocolError("bad number in line: " + line)
}
res[i] = &Group{
Name: ss[0],
High: high,
Low: low,
Status: ss[3],
}
}
return res, nil
}
// Capabilities returns a list of features this server performs.
// Not all servers support capabilities.
func (c *Conn) Capabilities() ([]string, error) {
_, lines, err := c.MultilineCommand("CAPABILITIES", 101)
if err != nil {
return nil, err
}
return lines, nil
}
// Date returns the current time on the server.
// Typically the time is later passed to NewGroups or NewNews.
func (c *Conn) Date() (time.Time, error) {
_, line, err := c.Command("DATE", 111)
if err != nil {
return time.Time{}, err
}
t, err := time.Parse(timeFormatDate, line)
if err != nil {
return time.Time{}, ProtocolError("invalid time: " + line)
}
return t, nil
}
// List returns a list of groups present on the server.
// Valid forms are:
//
// List() - return active groups
// List(keyword) - return different kinds of information about groups
// List(keyword, pattern) - filter groups against a glob-like pattern called a wildmat
//
func (c *Conn) List(a ...string) ([]string, error) {
if len(a) > 2 {
return nil, ProtocolError("List only takes up to 2 arguments")
}
cmd := "LIST"
if len(a) > 0 {
cmd += " " + a[0]
if len(a) > 1 {
cmd += " " + a[1]
}
}
_, lines, err := c.MultilineCommand(cmd, 215)
if err != nil {
return nil, err
}
return lines, nil
}
// Group changes the current group.
func (c *Conn) Group(group string) (*Group, error) {
_, line, err := c.Command(fmt.Sprintf("GROUP %s", group), 211)
if err != nil {
return nil, err
}
return parseGroup(line)
}
// Help returns the server's help text.
func (c *Conn) Help() ([]string, error) {
_, lines, err := c.MultilineCommand("HELP", 100)
if err != nil {
return nil, err
}
return lines, nil
}
// nextLastStat performs the work for NEXT, LAST, and STAT.
func (c *Conn) nextLastStat(cmd, id string) (string, string, error) {
_, line, err := c.Command(maybeID(cmd, id), 223)
if err != nil {
return "", "", err
}
ss := strings.SplitN(line, " ", 3) // optional comment ignored
if len(ss) < 2 {
return "", "", ProtocolError("Bad response to " + cmd + ": " + line)
}
return ss[0], ss[1], nil
}
// Stat looks up the message with the given id and returns its
// message number in the current group, and vice versa.
// The returned message number can be "0" if the current group
// isn't one of the groups the message was posted to.
func (c *Conn) Stat(id string) (number, msgid string, err error) {
return c.nextLastStat("STAT", id)
}
// Last selects the previous article, returning its message number and id.
func (c *Conn) Last() (number, msgid string, err error) {
return c.nextLastStat("LAST", "")
}
// Next selects the next article, returning its message number and id.
func (c *Conn) Next() (number, msgid string, err error) {
return c.nextLastStat("NEXT", "")
}
// ArticleText returns the article named by id as a []string.
// The article is in plain text format, not NNTP wire format.
func (c *Conn) ArticleText(id string) ([]string, error) {
_, lines, err := c.MultilineCommand(maybeID("ARTICLE", id), 220)
if err != nil {
return nil, err
}
return lines, nil
}
// Article returns the article named by id as an *Article.
func (c *Conn) Article(id string) (*Article, error) {
_, _, err := c.Command(maybeID("ARTICLE", id), 220)
if err != nil {
return nil, err
}
h, err := c.conn.ReadMIMEHeader()
if err != nil {
return nil, err
}
a := &Article{}
a.Header = h
a.Body, err = c.conn.ReadDotLines()
if err != nil {
return nil, err
}
return a, nil
}
// HeadText returns the header for the article named by id as []string.
// The article is in plain text format, not NNTP wire format.
func (c *Conn) HeadText(id string) ([]string, error) {
_, lines, err := c.MultilineCommand(maybeID("HEAD", id), 221)
if err != nil {
return nil, err
}
return lines, nil
}
// Head returns the header for the article named by id as an *Article.
// The Body field in the Article is nil.
func (c *Conn) Head(id string) (*Article, error) {
_, _, err := c.Command(maybeID("HEAD", id), 221)
if err != nil {
return nil, err
}
r := c.conn.DotReader()
header, err := ioutil.ReadAll(r)
if err != nil {
return nil, err
}
header = append(header, []byte("\n")...)
tp := textproto.NewReader(bufio.NewReader(bytes.NewReader(header)))
headerStruct, err := tp.ReadMIMEHeader()
if err != nil {
return nil, err
}
a := &Article{
Header: headerStruct,
}
return a, nil
}
// Body returns the body for the article named by id as an io.Reader.
func (c *Conn) Body(id string) ([]string, error) {
_, _, err := c.Command(maybeID("BODY", id), 222)
if err != nil {
return nil, err
}
lines, err := c.conn.ReadDotLines()
if err != nil {
return nil, err
}
return lines, nil
}
// RawPost reads a text-formatted article from r and posts it to the server.
func (c *Conn) RawPost(r io.Reader) error {
_, _, err := c.Command("POST", 3)
if err != nil {
return err
}
br := bufio.NewReader(r)
eof := false
for {
line, err := br.ReadString('\n')
if err == io.EOF {
eof = true
} else if err != nil {
return err
}
if eof && len(line) == 0 {
break
}
if strings.HasSuffix(line, "\n") {
line = line[0 : len(line)-1]
}
var prefix string
if strings.HasPrefix(line, ".") {
prefix = "."
}
_, err = fmt.Fprintf(c.conn.W, "%s%s\r\n", prefix, line)
if err != nil {
return err
}
if eof {
break
}
}
_, _, err = c.Command(".", 240)
if err != nil {
return err
}
return nil
}
// Quit sends the QUIT command and closes the connection to the server.
func (c *Conn) Quit() error {
_, _, err := c.Command("QUIT", 0)
c.conn.Close()
return err
}