writer file

.gitignore

@@ -33,3 +33,4 @@ _testmain.go
 /Gemfile.lock
 /public/
 /requirements.txt
+/tmp

stm/adapter_file.go

@@ -22,16 +22,17 @@ func (adp *FileAdapter) Write(loc *Location, data []byte) {
 	if err != nil {
 		_ = os.MkdirAll(dir, 0755)
 	} else if !fi.IsDir() {
-		log.Fatal("%s should be a directory", dir)
+		log.Fatalf("%s should be a directory", dir)
 	}
 
-	file, _ := os.Open(loc.Path())
+	file, _ := os.OpenFile(loc.Path(), os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666)
 	fi, err = file.Stat()
 	if err != nil {
-		log.Fatal("%s file not exists", loc.Path())
+		log.Fatalf("%s file not exists", loc.Path())
 	} else if !fi.Mode().IsRegular() {
-		log.Fatal("%s should be a filename", loc.Path())
+		log.Fatalf("%s should be a filename", loc.Path())
 	}
+	defer file.Close()
 
 	if gzipPtn.MatchString(loc.Path()) {
 		adp.gzip(file, data)

stm/builder.go

@@ -12,7 +12,7 @@ type Builder interface {
 	// AddWithErr(url interface{}) (Builder, error)
 	// location() *Location
 
-	finalize()
-	write()
+	Finalize()
+	Write()
 	run()
 }

stm/builder_file.go

@@ -2,31 +2,63 @@ package stm
 
 import "log"
 
+type builderFileError struct {
+	error
+	full      bool
+	finalized bool
+}
+
+func (e *builderFileError) FullError() bool {
+	return e.full
+}
+
+func (e *builderFileError) FinalizedError() bool {
+	return e.finalized
+}
+
 func NewBuilderFile(loc *Location) *BuilderFile {
-	return &BuilderFile{
-		xmlContent: make([]byte, 50000, 52428800),  // Number of URLs = 50,000 File size ( uncompressed ) = 50MB
-		build:      make(chan sitemapURL),
-		loc:        loc,
+	b := &BuilderFile{
+		build: make(chan sitemapURL),
+		loc:   loc,
 	}
+	b.clear()
+	return b
 }
 
 type BuilderFile struct {
-	xmlContent []byte // We can use this later
-	build      chan sitemapURL
-	loc        *Location
+	content      []byte
+	build        chan sitemapURL
+	loc          *Location
+	frozen       bool
+	linkcnt      int
+	newscnt      int
+	written      bool
+	reservedName string
 
 	urls []interface{} // XXX: For debug
 }
 
 func (b *BuilderFile) Add(url interface{}) BuilderError {
 	smu, err := NewSitemapURL(url)
 	if err != nil {
-		// panic(fmt.Sprintf("[F] Sitemap: %s", err))
-		log.Println("[F] Sitemap: ", err)
-		return &builderFileError{err, true, false}
+		log.Fatalln("[F] Sitemap: %s", err)
 	}
 
-	b.xmlContent = append(b.xmlContent, smu.Xml()...) // TODO: Sitemap xml have limit length
+	bytes := smu.Xml()
+
+	if b.isFinalized() {
+		return &builderFileError{error: err, finalized: true}
+	} else if !b.isFileCanFit(bytes) {
+		return &builderFileError{error: err, full: true}
+	}
+
+	// TODO: News sitemap xml
+	// if smu.isNews() {
+	// b.newscnt += 1
+	// }
+
+	b.content = append(b.content, bytes...) // TODO: Sitemap xml have limit length
+	b.linkcnt += 1
 	// b.build <- smu; b.urls = append(b.urls, url) // XXX: For debug
 	return nil
 }
@@ -36,49 +68,70 @@ func (b *BuilderFile) Add(url interface{}) BuilderError {
 // if err != nil {
 // log.Println("[E] Sitemap: ", err)
 // }
-// b.xmlContent += smu.Xml() // TODO: Sitemap xml have limit length
+// b.content += smu.Xml() // TODO: Sitemap xml have limit length
 // // b.build <- smu; b.urls = append(b.urls, url) // XXX: For debug
 // return b, nil
 // }
 
 func (b *BuilderFile) Content() []byte {
-	return b.xmlContent
+	return b.content
+}
+
+func (b *BuilderFile) Finalize() {
+	b.frozen = true
+}
+
+func (b *BuilderFile) isFinalized() bool {
+	return b.frozen
+}
+
+func (b *BuilderFile) isWritten() bool {
+	return b.written
+}
+
+func (b *BuilderFile) isFileCanFit(bytes []byte) bool {
+	r := len(append(b.content, bytes...)) < MaxSitemapFilesize
+	r = r && b.linkcnt < MaxSitemapLinks
+	return r && b.newscnt < MaxSitemapNews
 }
 
 // func (b *BuilderFile) location() *Location {
 // return b.loc
 // }
 
-func (b *BuilderFile) finalize() {}
-func (b *BuilderFile) write() {
+func (b *BuilderFile) setReverseName() {
+	if b.reservedName == "" {
+		b.reservedName = b.loc.ReserveName()
+	}
+}
+
+func (b *BuilderFile) clear() {
+	b.content = make([]byte, MaxSitemapLinks, MaxSitemapFilesize)
+}
+
+func (b *BuilderFile) Write() {
+	if b.isWritten() {
+		log.Fatalln("[F] Sitemap already written!")
+	}
+
+	if !b.isFinalized() {
+		b.Finalize()
+	}
+
+	b.setReverseName()
+
+	// TODO: header and footer
+	b.loc.Write(b.content, b.linkcnt) // @location.write(@xml_wrapper_start + @xml_content + @xml_wrapper_end, link_count)
 
-	// raise SitemapGenerator::SitemapError.new("Sitemap already written!") if written?
-	// finalize! unless finalized?
-	// reserve_name
-	// @location.write(@xml_wrapper_start + @xml_content + @xml_wrapper_end, link_count)
-	// @xml_content = @xml_wrapper_start = @xml_wrapper_end = ''
-	// @written = true
+	b.clear() // @xml_content = @xml_wrapper_start = @xml_wrapper_end = ''
+	b.written = true
 }
 
 func (b *BuilderFile) run() {
 	for {
 		select {
 		case smu := <-b.build:
-			b.xmlContent = append(b.xmlContent, smu.Xml()...) // TODO: Sitemap xml have limit length
+			b.content = append(b.content, smu.Xml()...) // TODO: Sitemap xml have limit length
 		}
 	}
 }
-
-type builderFileError struct {
-	error
-	isFull      bool
-	isFinalized bool
-}
-
-func (e *builderFileError) FullError() bool {
-	return e.isFull
-}
-
-func (e *builderFileError) FinalizedError() bool {
-	return e.isFinalized
-}

stm/builder_indexfile.go

@@ -1,27 +1,42 @@
 package stm
 
+import "github.com/k0kubun/pp"
+
 func NewBuilderIndexfile(loc *Location) *BuilderIndexfile {
 	return &BuilderIndexfile{
 		loc: loc,
 	}
 }
 
 type BuilderIndexfile struct {
-	loc           *Location
-	linkCount     int
-	bldrLinkCount int
+	loc      *Location
+	linkcnt  int
+	totalcnt int
 }
 
 func (b *BuilderIndexfile) Add(link interface{}) BuilderError {
-	bldr := link.(Builder)
-	bldr.write()
+	bldr := link.(*BuilderFile)
+
+	b.totalcnt += bldr.linkcnt
+
+	if !bldr.isFinalized() {
+		bldr.Finalize()
+	}
+
+	// TODO: first sitemap
+	// if b.linkcnt == 0 { }
+
+	bldr.Write()
 	return nil
 }
 
 // func (b *BuilderIndexfile) AddWithErr(url interface{}) (Builder, error) {
-	// return b, nil
+// return b, nil
 // }
 
-func (b *BuilderIndexfile) finalize() {}
-func (b *BuilderIndexfile) write()    {}
-func (b *BuilderIndexfile) run()      {}
+func (b *BuilderIndexfile) Finalize() {}
+func (b *BuilderIndexfile) Write() {
+	pp.Println("write indexfile")
+}
+
+func (b *BuilderIndexfile) run() {}

stm/builder_url.go

@@ -96,7 +96,7 @@ func (su *sitemapURL) Xml() []byte {
 	}
 
 	buf := &bytes.Buffer{}
-	doc.Indent(2)
+	// doc.Indent(2)
 	doc.WriteTo(buf)
 
 	return buf.Bytes()

stm/consts.go

@@ -0,0 +1,21 @@
+// General sitemap guidelines: https://support.google.com/webmasters/answer/183668
+// Number of URLs = 50,000
+// File size ( uncompressed ) = 50MB
+package stm
+
+const (
+	MaxSitemapFiles    = 50000    // max sitemap links per index file
+	MaxSitemapLinks    = 50000    // max links per sitemap
+	MaxSitemapImages   = 1000     // max images per url
+	MaxSitemapNews     = 1000     // max news sitemap per index_file
+	MaxSitemapFilesize = 10485760 // bytes
+)
+
+const (
+	SchemaGeo     = "http://www.google.com/geo/schemas/sitemap/1.0"
+	SchemaImage   = "http://www.google.com/schemas/sitemap-image/1.1"
+	SchemaMobile  = "http://www.google.com/schemas/sitemap-mobile/1.0"
+	SchemaNews    = "http://www.google.com/schemas/sitemap-news/0.9"
+	SchemaPagemap = "http://www.google.com/schemas/sitemap-pagemap/1.0"
+	SchemaVideo   = "http://www.google.com/schemas/sitemap-video/1.1"
+)

stm/location.go

@@ -18,7 +18,7 @@ func NewLocation(opts *Options) *Location {
 }
 
 type Location struct {
-	adp  Adapter
+	// adp  Adapter
 	opts *Options
 }
 
@@ -109,7 +109,7 @@ func (loc *Location) IsVerbose() bool {
 }
 
 func (loc *Location) Write(data []byte, linkCount int) {
-	loc.adp.Write(loc, data)
+	loc.opts.adp.Write(loc, data)
 	if loc.IsVerbose() {
 		pp.Println(loc.Summary(linkCount))
 	}

stm/sitemap.go

@@ -1,8 +1,12 @@
 package stm
 
-import "runtime"
+import (
+	"log"
+	"runtime"
+)
 
 func NewSitemap() *Sitemap {
+	log.SetFlags(log.LstdFlags | log.Llongfile)
 	runtime.GOMAXPROCS(runtime.NumCPU())
 
 	sm := &Sitemap{
@@ -61,11 +65,11 @@ func (sm *Sitemap) finalize() {
 }
 
 func (sm *Sitemap) finalizeFile() {
-	sm.bldr.finalize()
+	sm.bldr.Finalize()
 	sm.bldrs.Add(sm.bldr)
 }
 
 func (sm *Sitemap) finalizeIndexfile() {
-	sm.bldrs.finalize()
-	sm.bldrs.write()
+	sm.bldrs.Finalize()
+	sm.bldrs.Write()
 }