converter/converter.go

// Copyright 2013 The Agostle Authors. All rights reserved.
// Use of this source code is governed by an Apache 2.0
// license that can be found in the LICENSE file.

package converter

import (
	"bufio"
	"errors"
	"fmt"
	"io"
	"mime"
	"mime/multipart"
	"os"
	"os/exec"
	"path/filepath"
	"runtime"
	"strings"
	"sync"

	"bitbucket.org/taruti/mimemagic"
	"github.com/tgulacsi/go/text"
)

// Converter converts to Pdf (destination filename, source reader and source content-type)
type Converter func(string, io.Reader, string, *Context) error

// TextToPdf converts text (text/plain) to PDF
func TextToPdf(destfn string, r io.Reader, contentType string, ctx *Context) error {
	logger.Info("msg", "converting", "ct", contentType, "dest", destfn)
	if strings.HasSuffix(destfn, ".pdf") {
		destfn = destfn[:len(destfn)-4]
	}
	inpfn := destfn + ".html"
	fh, err := os.Create(inpfn)
	if err != nil {
		return err
	}
	defer func() { _ = unlink(inpfn, "TextToPdf") }()
	if _, err = io.WriteString(fh, `<!DOCTYPE html><html>
	<meta http-equiv="Content-Type" content="text/html; charset=utf-8">
	</head><body><pre>
`); err != nil {
		return err
	}
	bw := bufio.NewWriter(htmlEscaper{fh})
	if _, err = io.Copy(bw, text.NewStringReader(r)); err != nil {
		return err
	}
	if err = bw.Flush(); err != nil {
		return err
	}
	if _, err = io.WriteString(fh, "\n</pre></body></html>"); err != nil {
		return err
	}
	return lofficeConvert(filepath.Dir(destfn), inpfn)
}

// ImageToPdf convert image (image/...) to PDF
func ImageToPdf(destfn string, r io.Reader, contentType string, ctx *Context) error {
	logger.Info("msg", "converting image", "ct", contentType, "dest", destfn)
	imgtyp := contentType[strings.Index(contentType, "/")+1:]
	if strings.HasSuffix(destfn, ".pdf") {
		destfn = destfn[:len(destfn)-4]
	}
	ifh, ok := r.(*os.File)
	if !ok {
		var n int64
		var err error
		inpfn := destfn + "." + imgtyp
		ifh, err = os.Create(inpfn)
		if err != nil {
			return fmt.Errorf("cannot create temp image file %s: %s", inpfn, err)
		}
		if n, err = io.Copy(ifh, r); err != nil {
			logger.Error("msg", "ImageToPdf reading", "file", ifh.Name(), "error", err)
		}
		logger.Debug("msg", "written temp image file", "file", inpfn, "length", n)
		if err = ifh.Close(); err != nil {
			logger.Error("msg", "ImageToPdf writing", "file", ifh.Name(), "error", err)
		}
		if ifh, err = os.Open(inpfn); err != nil {
			return fmt.Errorf("cannot open inp %s: %s", inpfn, err)
		}
		defer func() { _ = ifh.Close() }()
		if !LeaveTempFiles {
			defer func() { _ = unlink(inpfn, "ImageToPdf") }()
		}
	}
	destfn = destfn + ".pdf"
	if _, err := ifh.Seek(0, 0); err != nil {
		return err
	}
	if !fileExists(ifh.Name()) {
		logger.Crit("msg", "input file not exists!", "file", ifh.Name())
		return fmt.Errorf("input file %q not exists", ifh.Name())
	}
	w, err := os.Create(destfn)
	if err != nil {
		return err
	}
	if err = ImageToPdfGm(w, ifh, contentType); err != nil {
		logger.Error("msg", "ImageToPdfGm", "error", err)
	}
	closeErr := w.Close()
	if err != nil {
		return err
	}
	return closeErr
}

// OfficeToPdf converts other to PDF with LibreOffice
func OfficeToPdf(destfn string, r io.Reader, contentType string, ctx *Context) error {
	logger.Info("msg", "converting", "ct", contentType, "dest", destfn)
	if strings.HasSuffix(destfn, ".pdf") {
		destfn = destfn[:len(destfn)-4]
	}
	inpfn := destfn + ".raw"
	fh, err := os.Create(inpfn)
	if err != nil {
		return err
	}
	defer func() { _ = unlink(inpfn, "OtherToPdf") }()
	if _, err = io.Copy(fh, r); err != nil {
		return err
	}
	return lofficeConvert(filepath.Dir(destfn), inpfn)
}

// OtherToPdf is the default converter
var OtherToPdf = OfficeToPdf

// PdfToPdf "converts" PDF (application/pdf) to PDF (just copies)
func PdfToPdf(destfn string, r io.Reader, _ string, ctx *Context) error {
	logger.Info("msg", `"Converting" pdf`, "dest", destfn)
	fh, err := os.Create(destfn)
	if err != nil {
		return err
	}
	_, err = io.Copy(fh, r)
	closeErr := fh.Close()
	if err != nil {
		return err
	}
	return closeErr
}

// MPRelatedToPdf converts multipart/related to PDF
func MPRelatedToPdf(destfn string, r io.Reader, contentType string, ctx *Context) error {
	var (
		err    error
		params map[string]string
	)
	contentType, params, err = mime.ParseMediaType(contentType)
	if err != nil {
		err = fmt.Errorf("cannot parse Content-Type %s: %s", contentType, err)
		return err
	}

	parts := multipart.NewReader(r, params["boundary"])
	part, e := parts.NextPart()
	for e == nil {
		fmt.Printf("part=%#v", part)
		part, e = parts.NextPart()
	}
	if e != nil && e != io.EOF {
		return e
	}
	return nil
}

// HTMLToPdf converts HTML (text/html) to PDF
func HTMLToPdf(destfn string, r io.Reader, contentType string, ctx *Context) error {
	var inpfn string
	if fh, ok := r.(*os.File); ok && fileExists(fh.Name()) {
		inpfn = fh.Name()
	}
	if inpfn == "" {
		inpfn = nakeFilename(destfn) + ".html"
		fh, err := os.Create(inpfn)
		if err != nil {
			return err
		}
		if !LeaveTempFiles {
			defer func() { _ = unlink(inpfn, "HtmlToPdf") }()
		}
		if _, err = io.Copy(fh, r); err != nil {
			return err
		}
	}
	dn := filepath.Dir(destfn)
	outfn := filepath.Join(dn, filepath.Base(nakeFilename(inpfn))+".pdf")

	if err := lofficeConvert(dn, inpfn); err != nil {
		return err
	}
	if outfn != destfn {
		//log.Printf("moving %s to %s", outfn, destfn)
		return moveFile(outfn, destfn)
	}
	return nil
}

var (
	lofficeMtx      = sync.Mutex{}
	lofficePortLock = NewPortLock(LofficeLockPort)
)

// calls loffice converter with only one instance at a time,
// in the input file's directory
func lofficeConvert(outDir, inpfn string) error {
	if outDir == "" {
		return errors.New("outDir is required!")
	}
	args := []string{"--headless", "--convert-to", "pdf", "--outdir",
		outDir, inpfn}
	lofficeMtx.Lock()
	defer lofficeMtx.Unlock()
	lofficePortLock.Lock()
	defer lofficePortLock.Unlock()
	logger.Debug("msg", "lofficeConvert calls", "command", *ConfLoffice, "args", args)
	cmd := exec.Command(*ConfLoffice, args...)
	cmd.Dir = filepath.Dir(inpfn)
	cmd.Stderr = os.Stderr
	cmd.Stdout = cmd.Stderr
	if runtime.GOOS != "windows" {
		// This induces "soffice.exe: The parameter is incorrect." error under Windows!
		cmd.Env = make([]string, 1, len(os.Environ())+1)
		lcAll := os.Getenv("LC_ALL")
		if i := strings.IndexByte(lcAll, '.'); i > 0 && strings.HasPrefix(lcAll, "en_") {
			lcAll = lcAll[:i+1] + "UTF-8"
		} else {
			lcAll = "en_US.UTF-8"
		}
		cmd.Env[0] = lcAll
		logger.Info("msg", "env", "LC_ALL", lcAll)
		// delete LC_* LANG* env vars.
		for _, s := range os.Environ() {
			if strings.HasPrefix(s, "LC_") || s == "LANG" || s == "LANGUAGE" {
				continue
			}
			cmd.Env = append(cmd.Env, s)
		}
	}

	err := runWithTimeout(cmd)
	logger.Debug("msg", "result of", "command", *ConfLoffice, "args", args, "error", err)
	if err != nil {
		return err
	}
	outfn := filepath.Join(outDir, filepath.Base(nakeFilename(inpfn))+".pdf")
	if _, err = os.Stat(outfn); err != nil {
		return errors.New("loffice no output for " + filepath.Base(inpfn))
	}
	return nil
}

// file extension -> content-type map
var ExtContentType = map[string]string{
	"doc":  "application/vnd.ms-word",
	"docx": "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
	"dotx": "application/vnd.openxmlformats-officedocument.wordprocessingml.template",
	"xls":  "application/vnd.ms-excel",
	"xlsx": "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
	"xltx": "application/vnd.openxmlformats-officedocument.spreadsheetml.template",
	"ppt":  "application/vnd.ms-powerpoint",
	"pptx": "application/vnd.openxmlformats-officedocument.presentationml.presentation",
	"ppsx": "application/vnd.openxmlformats-officedocument.presentationml.slideshow",
	"potx": "application/vnd.openxmlformats-officedocument.presentationml.template",

	"odg": "application/vnd.oasis.opendocument.graphics",
	"otg": "application/vnd.oasis.opendocument.graphics-template",
	"otp": "application/vnd.oasis.opendocument.presentation-template",
	"odp": "application/vnd.oasis.opendocument.presentation",
	"odm": "application/vnd.oasis.opendocument.text-master",
	"odt": "application/vnd.oasis.opendocument.text",
	"oth": "application/vnd.oasis.opendocument.text-web",
	"ott": "application/vnd.oasis.opendocument.text-template",
	"ods": "application/vnd.oasis.spreadsheet",
	"ots": "application/vnd.oasis.spreadsheet-template",
	"odc": "application/vnd.oasis.chart",
	"odf": "application/vnd.oasis.formula",
	"odb": "application/vnd.oasis.database",
	"odi": "application/vnd.oasis.image",

	"txt": "text/plain",
	"msg": "application/x-ole-storage",

	"jpg":  "image/jpeg",
	"jpeg": "image/jpeg",
	"gif":  "image/gif",
	"png":  "image/png",
}

// FixContentType ensures proper content-type
// (uses mimemagic for "" and application/octet-stream)
func FixContentType(body []byte, contentType, fileName string) string {
	switch contentType {
	case "application/x-zip-compressed":
		return "application/zip"
	case "application/x-rar-compressed":
		return "application/rar"
	case "", "application/octet-stream", "application/pdf", "application/x-as400attachment", "application/save-as":
		//log.Printf("body=%s", body)
		if nct := mimemagic.Match(contentType, body); nct != "" {
			return nct
		}
	}
	if GetConverter(contentType, nil) == nil { // no converter for this
		if nct := mimemagic.Match(contentType, body); nct != "" {
			return nct
		}
	}
	if fileName != "" &&
		(contentType == "" || contentType == "application/octet-stream" ||
			GetConverter(contentType, nil) == nil) {
		if i := strings.LastIndex(fileName, "."); i >= 0 {
			if nct, ok := ExtContentType[fileName[i+1:]]; ok {
				return nct
			}
			if nct := mime.TypeByExtension(fileName[i:]); nct != "" {
				return nct
			}
		}
	}
	//log.Printf("ct=%s ==> %s", ct, contentType)
	return contentType
}

// GetConverter gets converter for the content-type
func GetConverter(contentType string, mediaType map[string]string) (converter Converter) {
	converter = nil
	switch contentType {
	case "application/pdf":
		converter = PdfToPdf
	case "application/rtf":
		converter = OfficeToPdf
	case "text/plain":
		if mediaType != nil {
			if cs, ok := mediaType["charset"]; ok && cs != "" {
				converter = NewTextConverter(cs)
			}
		}
		if converter == nil {
			converter = TextToPdf
		}
	case "text/html":
		converter = HTMLToPdf
	case "message/rfc822":
		converter = MailToPdfZip
	case "multipart/related":
		converter = MPRelatedToPdf
	default:
		// from http://www.openoffice.org/framework/documentation/mimetypes/mimetypes.html
		if strings.HasPrefix(contentType, "application/vnd.oasis.") ||
			//ODF
			strings.HasPrefix(contentType, "application/vnd.openxmlformats-officedocument.") ||
			//MS Office
			strings.HasPrefix(contentType, "application/vnd.ms-word") ||
			strings.HasPrefix(contentType, "application/vnd.ms-excel") ||
			strings.HasPrefix(contentType, "application/vnd.ms-powerpoint") ||
			contentType == "application/x-ole-storage" ||
			//StarOffice
			strings.HasPrefix(contentType, "application/vnd.sun.xml.") ||
			strings.HasPrefix(contentType, "application/vnd.stardivision.") ||
			strings.HasPrefix(contentType, "application/x-star.") ||
			//Word
			contentType == "application/msword" {
			converter = OfficeToPdf
			break
		}
		i := strings.Index(contentType, "/")
		if i > 0 {
			switch contentType[:i] {
			case "image":
				converter = ImageToPdf
			case "text":
				converter = TextToPdf
			case "audio", "video":
				converter = nil
			}
		}
	}
	if converter == nil {
		logger.Warn("msg", "cannot find converter for", "ct", contentType)
	} else {
		logger.Debug("msg", "converter for", "ct", contentType, "converter", converter)
	}
	return
}