Skip to content
Permalink
Browse files

writer file

  • Loading branch information...
ikeikeikeike committed Nov 7, 2015
1 parent 3ce7022 commit 2dbf51ac731f9602529ab7e497bbfe3def03f01e
Showing with 153 additions and 58 deletions.
  1. +1 −0 .gitignore
  2. +5 −4 stm/adapter_file.go
  3. +2 −2 stm/builder.go
  4. +89 −36 stm/builder_file.go
  5. +24 −9 stm/builder_indexfile.go
  6. +1 −1 stm/builder_url.go
  7. +21 −0 stm/consts.go
  8. +2 −2 stm/location.go
  9. +8 −4 stm/sitemap.go
@@ -33,3 +33,4 @@ _testmain.go
/Gemfile.lock
/public/
/requirements.txt
/tmp
@@ -22,16 +22,17 @@ func (adp *FileAdapter) Write(loc *Location, data []byte) {
if err != nil {
_ = os.MkdirAll(dir, 0755)
} else if !fi.IsDir() {
log.Fatal("%s should be a directory", dir)
log.Fatalf("%s should be a directory", dir)
}

file, _ := os.Open(loc.Path())
file, _ := os.OpenFile(loc.Path(), os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666)
fi, err = file.Stat()
if err != nil {
log.Fatal("%s file not exists", loc.Path())
log.Fatalf("%s file not exists", loc.Path())
} else if !fi.Mode().IsRegular() {
log.Fatal("%s should be a filename", loc.Path())
log.Fatalf("%s should be a filename", loc.Path())
}
defer file.Close()

if gzipPtn.MatchString(loc.Path()) {
adp.gzip(file, data)
@@ -12,7 +12,7 @@ type Builder interface {
// AddWithErr(url interface{}) (Builder, error)
// location() *Location

finalize()
write()
Finalize()
Write()
run()
}
@@ -2,31 +2,63 @@ package stm

import "log"

type builderFileError struct {
error
full bool
finalized bool
}

func (e *builderFileError) FullError() bool {
return e.full
}

func (e *builderFileError) FinalizedError() bool {
return e.finalized
}

func NewBuilderFile(loc *Location) *BuilderFile {
return &BuilderFile{
xmlContent: make([]byte, 50000, 52428800), // Number of URLs = 50,000 File size ( uncompressed ) = 50MB
build: make(chan sitemapURL),
loc: loc,
b := &BuilderFile{
build: make(chan sitemapURL),
loc: loc,
}
b.clear()
return b
}

type BuilderFile struct {
xmlContent []byte // We can use this later
build chan sitemapURL
loc *Location
content []byte
build chan sitemapURL
loc *Location
frozen bool
linkcnt int
newscnt int
written bool
reservedName string

urls []interface{} // XXX: For debug
}

func (b *BuilderFile) Add(url interface{}) BuilderError {
smu, err := NewSitemapURL(url)
if err != nil {
// panic(fmt.Sprintf("[F] Sitemap: %s", err))
log.Println("[F] Sitemap: ", err)
return &builderFileError{err, true, false}
log.Fatalln("[F] Sitemap: %s", err)
}

b.xmlContent = append(b.xmlContent, smu.Xml()...) // TODO: Sitemap xml have limit length
bytes := smu.Xml()

if b.isFinalized() {
return &builderFileError{error: err, finalized: true}
} else if !b.isFileCanFit(bytes) {
return &builderFileError{error: err, full: true}
}

// TODO: News sitemap xml
// if smu.isNews() {
// b.newscnt += 1
// }

b.content = append(b.content, bytes...) // TODO: Sitemap xml have limit length
b.linkcnt += 1
// b.build <- smu; b.urls = append(b.urls, url) // XXX: For debug
return nil
}
@@ -36,49 +68,70 @@ func (b *BuilderFile) Add(url interface{}) BuilderError {
// if err != nil {
// log.Println("[E] Sitemap: ", err)
// }
// b.xmlContent += smu.Xml() // TODO: Sitemap xml have limit length
// b.content += smu.Xml() // TODO: Sitemap xml have limit length
// // b.build <- smu; b.urls = append(b.urls, url) // XXX: For debug
// return b, nil
// }

func (b *BuilderFile) Content() []byte {
return b.xmlContent
return b.content
}

func (b *BuilderFile) Finalize() {
b.frozen = true
}

func (b *BuilderFile) isFinalized() bool {
return b.frozen
}

func (b *BuilderFile) isWritten() bool {
return b.written
}

func (b *BuilderFile) isFileCanFit(bytes []byte) bool {
r := len(append(b.content, bytes...)) < MaxSitemapFilesize
r = r && b.linkcnt < MaxSitemapLinks
return r && b.newscnt < MaxSitemapNews
}

// func (b *BuilderFile) location() *Location {
// return b.loc
// }

func (b *BuilderFile) finalize() {}
func (b *BuilderFile) write() {
func (b *BuilderFile) setReverseName() {
if b.reservedName == "" {
b.reservedName = b.loc.ReserveName()
}
}

func (b *BuilderFile) clear() {
b.content = make([]byte, MaxSitemapLinks, MaxSitemapFilesize)
}

func (b *BuilderFile) Write() {
if b.isWritten() {
log.Fatalln("[F] Sitemap already written!")
}

if !b.isFinalized() {
b.Finalize()
}

b.setReverseName()

// TODO: header and footer
b.loc.Write(b.content, b.linkcnt) // @location.write(@xml_wrapper_start + @xml_content + @xml_wrapper_end, link_count)

// raise SitemapGenerator::SitemapError.new("Sitemap already written!") if written?
// finalize! unless finalized?
// reserve_name
// @location.write(@xml_wrapper_start + @xml_content + @xml_wrapper_end, link_count)
// @xml_content = @xml_wrapper_start = @xml_wrapper_end = ''
// @written = true
b.clear() // @xml_content = @xml_wrapper_start = @xml_wrapper_end = ''
b.written = true
}

func (b *BuilderFile) run() {
for {
select {
case smu := <-b.build:
b.xmlContent = append(b.xmlContent, smu.Xml()...) // TODO: Sitemap xml have limit length
b.content = append(b.content, smu.Xml()...) // TODO: Sitemap xml have limit length
}
}
}

type builderFileError struct {
error
isFull bool
isFinalized bool
}

func (e *builderFileError) FullError() bool {
return e.isFull
}

func (e *builderFileError) FinalizedError() bool {
return e.isFinalized
}
@@ -1,27 +1,42 @@
package stm

import "github.com/k0kubun/pp"

func NewBuilderIndexfile(loc *Location) *BuilderIndexfile {
return &BuilderIndexfile{
loc: loc,
}
}

type BuilderIndexfile struct {
loc *Location
linkCount int
bldrLinkCount int
loc *Location
linkcnt int
totalcnt int
}

func (b *BuilderIndexfile) Add(link interface{}) BuilderError {
bldr := link.(Builder)
bldr.write()
bldr := link.(*BuilderFile)

b.totalcnt += bldr.linkcnt

if !bldr.isFinalized() {
bldr.Finalize()
}

// TODO: first sitemap
// if b.linkcnt == 0 { }

bldr.Write()
return nil
}

// func (b *BuilderIndexfile) AddWithErr(url interface{}) (Builder, error) {
// return b, nil
// return b, nil
// }

func (b *BuilderIndexfile) finalize() {}
func (b *BuilderIndexfile) write() {}
func (b *BuilderIndexfile) run() {}
func (b *BuilderIndexfile) Finalize() {}
func (b *BuilderIndexfile) Write() {
pp.Println("write indexfile")
}

func (b *BuilderIndexfile) run() {}
@@ -96,7 +96,7 @@ func (su *sitemapURL) Xml() []byte {
}

buf := &bytes.Buffer{}
doc.Indent(2)
// doc.Indent(2)
doc.WriteTo(buf)

return buf.Bytes()
@@ -0,0 +1,21 @@
// General sitemap guidelines: https://support.google.com/webmasters/answer/183668
// Number of URLs = 50,000
// File size ( uncompressed ) = 50MB
package stm

const (
MaxSitemapFiles = 50000 // max sitemap links per index file
MaxSitemapLinks = 50000 // max links per sitemap
MaxSitemapImages = 1000 // max images per url
MaxSitemapNews = 1000 // max news sitemap per index_file
MaxSitemapFilesize = 10485760 // bytes
)

const (
SchemaGeo = "http://www.google.com/geo/schemas/sitemap/1.0"
SchemaImage = "http://www.google.com/schemas/sitemap-image/1.1"
SchemaMobile = "http://www.google.com/schemas/sitemap-mobile/1.0"
SchemaNews = "http://www.google.com/schemas/sitemap-news/0.9"
SchemaPagemap = "http://www.google.com/schemas/sitemap-pagemap/1.0"
SchemaVideo = "http://www.google.com/schemas/sitemap-video/1.1"
)
@@ -18,7 +18,7 @@ func NewLocation(opts *Options) *Location {
}

type Location struct {
adp Adapter
// adp Adapter
opts *Options
}

@@ -109,7 +109,7 @@ func (loc *Location) IsVerbose() bool {
}

func (loc *Location) Write(data []byte, linkCount int) {
loc.adp.Write(loc, data)
loc.opts.adp.Write(loc, data)
if loc.IsVerbose() {
pp.Println(loc.Summary(linkCount))
}
@@ -1,8 +1,12 @@
package stm

import "runtime"
import (
"log"
"runtime"
)

func NewSitemap() *Sitemap {
log.SetFlags(log.LstdFlags | log.Llongfile)
runtime.GOMAXPROCS(runtime.NumCPU())

sm := &Sitemap{
@@ -61,11 +65,11 @@ func (sm *Sitemap) finalize() {
}

func (sm *Sitemap) finalizeFile() {
sm.bldr.finalize()
sm.bldr.Finalize()
sm.bldrs.Add(sm.bldr)
}

func (sm *Sitemap) finalizeIndexfile() {
sm.bldrs.finalize()
sm.bldrs.write()
sm.bldrs.Finalize()
sm.bldrs.Write()
}

0 comments on commit 2dbf51a

Please sign in to comment.
You can’t perform that action at this time.