Skip to content

chore: rewrite for pure utf-8 content handling #511

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 5 commits into from
Sep 23, 2025
Merged

chore: rewrite for pure utf-8 content handling #511

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
042e3c9
chore: update testdata with utf8-header
tobiasehlert Sep 22, 2025
ededcee
chore: update testdata for world/premia
tobiasehlert Sep 23, 2025
001a467
chore: add world/antica testcase for umlaut name
tobiasehlert Sep 23, 2025
ee003f1
chore: remove TibiaDataConvertEncoding-functions
tobiasehlert Sep 23, 2025
c485542
test: commenting out two fake-tests
tobiasehlert Sep 23, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 0 additions & 12 deletions src/TibiaDataUtils.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,6 @@ package main

import (
"html"
"io"
"log"
"net/url"
"os"
Expand All @@ -13,7 +12,6 @@ import (
"unicode/utf8"

"golang.org/x/text/cases"
"golang.org/x/text/encoding/charmap"
"golang.org/x/text/language"
"golang.org/x/text/unicode/norm"
)
Expand Down Expand Up @@ -186,16 +184,6 @@ func RemoveHtmlTag(s string) string {
return s
}

// TibiaDataConvertEncodingtoISO88591 func - convert string from UTF-8 to latin1 (ISO 8859-1)
func TibiaDataConvertEncodingtoISO88591(data string) (string, error) {
return charmap.ISO8859_1.NewEncoder().String(data)
}

// TibiaDataConvertEncodingtoUTF8 func - convert string from latin1 (ISO 8859-1) to UTF-8
func TibiaDataConvertEncodingtoUTF8(data io.Reader) io.Reader {
return norm.NFKC.Reader(charmap.ISO8859_1.NewDecoder().Reader(data))
}

// TibiaDataSanitizeEscapedString func - run unescape string on string
func TibiaDataSanitizeEscapedString(data string) string {
return html.UnescapeString(data)
Expand Down
43 changes: 36 additions & 7 deletions src/TibiaWorldsWorld_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,35 @@ import (
"github.com/tibiadata/tibiadata-api-go/src/static"
)

func TestWorldAntica(t *testing.T) {
file, err := static.TestFiles.Open("testdata/worlds/world/Antica.html")
if err != nil {
t.Fatalf("file opening error: %s", err)
}
defer file.Close()

data, err := io.ReadAll(file)
if err != nil {
t.Fatalf("File reading error: %s", err)
}

worldJson, err := TibiaWorldsWorldImpl("Antica", string(data), "https://www.tibia.com/community/?subtopic=worlds&world=Antica")
if err != nil {
t.Fatal(err)
}

assert := assert.New(t)
world := worldJson.World
information := worldJson.Information

assert.Equal("https://www.tibia.com/community/?subtopic=worlds&world=Antica", information.TibiaURLs[0])

umlautPlayer := world.OnlinePlayers[171]
assert.Equal("Nöber", umlautPlayer.Name)
assert.Equal(455, umlautPlayer.Level)
assert.Equal("Elder Druid", umlautPlayer.Vocation)
}

func TestWorldEndebra(t *testing.T) {
file, err := static.TestFiles.Open("testdata/worlds/world/Endebra.html")
if err != nil {
Expand Down Expand Up @@ -70,25 +99,25 @@ func TestWorldPremia(t *testing.T) {
world := worldJson.World

assert.Equal("Premia", world.Name)
assert.Equal("offline", world.Status)
assert.Equal(0, world.PlayersOnline)
assert.Equal("online", world.Status)
assert.Equal(53, world.PlayersOnline)
assert.Equal(531, world.RecordPlayers)
assert.Equal("2013-08-08T15:30:30Z", world.RecordDate)
assert.Equal("2002-04", world.CreationDate)
assert.Equal("Europe", world.Location)
assert.Equal("Open PvP", world.PvpType)
assert.True(world.PremiumOnly)
assert.Equal("regular", world.TransferType)
assert.Equal(4, len(world.WorldsQuestTitles))
assert.Equal("Rise of Devovorga", world.WorldsQuestTitles[0])
assert.Equal("Bewitched", world.WorldsQuestTitles[1])
assert.Equal("The Colours of Magic", world.WorldsQuestTitles[2])
assert.Equal(7, len(world.WorldsQuestTitles))
assert.Equal("The Lightbearer", world.WorldsQuestTitles[0])
assert.Equal("Rise of Devovorga", world.WorldsQuestTitles[1])
assert.Equal("Annual Autumn Vintage", world.WorldsQuestTitles[2])
assert.Equal("A Piece of Cake", world.WorldsQuestTitles[3])
assert.True(world.BattleyeProtected)
assert.Equal("2017-09-05", world.BattleyeDate)
assert.Equal("regular", world.GameWorldType)
assert.Empty(world.TournamentWorldType)
assert.Equal(0, len(world.OnlinePlayers))
assert.Equal(53, len(world.OnlinePlayers))
}

func TestWorldWintera(t *testing.T) {
Expand Down
4 changes: 2 additions & 2 deletions src/static/testdata/boostablebosses/boostablebosses.html
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@

<!DOCTYPE html>
<html>
<html lang="en">
<head>
<title>Tibia - Free Multiplayer Online Role Playing Game - Library</title>
<meta http-equiv="content-type" content="text/html; charset=ISO-8859-1" /> <meta name="description" content="Tibia is a free massively multiplayer online role-playing game (MMORPG). Join this fascinating game that has thousands of fans from all over the world! - http://www.tibia.com" />
<meta http-equiv="content-type" content="text/html; charset=UTF-8" /> <meta name="description" content="Tibia is a free massively multiplayer online role-playing game (MMORPG). Join this fascinating game that has thousands of fans from all over the world! - http://www.tibia.com" />
<meta name="author" content="CipSoft GmbH" />
<meta http-equiv="content-language" content="en" />
<meta name="keywords" content="free online game, free multiplayer game, free online rpg, free mmorpg, mmorpg, mmog, online role playing game, online multiplayer game, internet game, online rpg, rpg" />
Expand Down
2 changes: 1 addition & 1 deletion src/static/testdata/characters/Akayev.html
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
<html lang="en">
<head>
<title>Tibia - Free Multiplayer Online Role Playing Game - Community</title>
<meta http-equiv="content-type" content="text/html; charset=ISO-8859-1" /> <meta name="description" content="Tibia is a free massively multiplayer online role-playing game (MMORPG). Join this fascinating game that has thousands of fans from all over the world! - http://www.tibia.com" />
<meta http-equiv="content-type" content="text/html; charset=UTF-8" /> <meta name="description" content="Tibia is a free massively multiplayer online role-playing game (MMORPG). Join this fascinating game that has thousands of fans from all over the world! - http://www.tibia.com" />
<meta name="author" content="CipSoft GmbH" />
<meta http-equiv="content-language" content="en" />
<meta name="keywords" content="free online game, free multiplayer game, free online rpg, free mmorpg, mmorpg, mmog, online role playing game, online multiplayer game, internet game, online rpg, rpg" />
Expand Down
4 changes: 2 additions & 2 deletions src/static/testdata/characters/Akura Aleus.html
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,10 +1,10 @@


<!DOCTYPE html>
<html>
<html lang="en">
<head>
<title>Tibia - Free Multiplayer Online Role Playing Game - Community</title>
<meta http-equiv="content-type" content="text/html; charset=ISO-8859-1" /> <meta name="description" content="Tibia is a free massively multiplayer online role-playing game (MMORPG). Join this fascinating game that has thousands of fans from all over the world! - http://www.tibia.com" />
<meta http-equiv="content-type" content="text/html; charset=UTF-8" /> <meta name="description" content="Tibia is a free massively multiplayer online role-playing game (MMORPG). Join this fascinating game that has thousands of fans from all over the world! - http://www.tibia.com" />
<meta name="author" content="CipSoft GmbH" />
<meta http-equiv="content-language" content="en" />
<meta name="keywords" content="free online game, free multiplayer game, free online rpg, free mmorpg, mmorpg, mmog, online role playing game, online multiplayer game, internet game, online rpg, rpg" />
Expand Down
4 changes: 2 additions & 2 deletions src/static/testdata/characters/Borttagna Gubben.html
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@

<!DOCTYPE html>
<html>
<html lang="en">
<head>
<title>Tibia - Free Multiplayer Online Role Playing Game - Community</title>
<meta http-equiv="content-type" content="text/html; charset=ISO-8859-1" /> <meta name="description" content="Tibia is a free massively multiplayer online role-playing game (MMORPG). Join this fascinating game that has thousands of fans from all over the world! - http://www.tibia.com" />
<meta http-equiv="content-type" content="text/html; charset=UTF-8" /> <meta name="description" content="Tibia is a free massively multiplayer online role-playing game (MMORPG). Join this fascinating game that has thousands of fans from all over the world! - http://www.tibia.com" />
<meta name="author" content="CipSoft GmbH" />
<meta http-equiv="content-language" content="en" />
<meta name="keywords" content="free online game, free multiplayer game, free online rpg, free mmorpg, mmorpg, mmog, online role playing game, online multiplayer game, internet game, online rpg, rpg" />
Expand Down
4 changes: 2 additions & 2 deletions src/static/testdata/characters/Darkside Rafa.html
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@

<!DOCTYPE html>
<html>
<html lang="en">
<head>
<title>Tibia - Free Multiplayer Online Role Playing Game - Community</title>
<meta http-equiv="content-type" content="text/html; charset=ISO-8859-1" /> <meta name="description" content="Tibia is a free massively multiplayer online role-playing game (MMORPG). Join this fascinating game that has thousands of fans from all over the world! - http://www.tibia.com" />
<meta http-equiv="content-type" content="text/html; charset=UTF-8" /> <meta name="description" content="Tibia is a free massively multiplayer online role-playing game (MMORPG). Join this fascinating game that has thousands of fans from all over the world! - http://www.tibia.com" />
<meta name="author" content="CipSoft GmbH" />
<meta http-equiv="content-language" content="en" />
<meta name="keywords" content="free online game, free multiplayer game, free online rpg, free mmorpg, mmorpg, mmog, online role playing game, online multiplayer game, internet game, online rpg, rpg" />
Expand Down
4 changes: 2 additions & 2 deletions src/static/testdata/characters/Jowjow Invencivel.html
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,10 +1,10 @@


<!DOCTYPE html>
<html>
<html lang="en">
<head>
<title>Tibia - Free Multiplayer Online Role Playing Game - Community</title>
<meta http-equiv="content-type" content="text/html; charset=ISO-8859-1" /> <meta name="description" content="Tibia is a free massively multiplayer online role-playing game (MMORPG). Join this fascinating game that has thousands of fans from all over the world! - http://www.tibia.com" />
<meta http-equiv="content-type" content="text/html; charset=UTF-8" /> <meta name="description" content="Tibia is a free massively multiplayer online role-playing game (MMORPG). Join this fascinating game that has thousands of fans from all over the world! - http://www.tibia.com" />
<meta name="author" content="CipSoft GmbH" />
<meta http-equiv="content-language" content="en" />
<meta name="keywords" content="free online game, free multiplayer game, free online rpg, free mmorpg, mmorpg, mmog, online role playing game, online multiplayer game, internet game, online rpg, rpg" />
Expand Down
4 changes: 2 additions & 2 deletions src/static/testdata/characters/Luminals.html
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@

<!DOCTYPE html>
<html>
<html lang="en">
<head>
<title>Tibia - Free Multiplayer Online Role Playing Game - Community</title>
<meta http-equiv="content-type" content="text/html; charset=ISO-8859-1" /> <meta name="description" content="Tibia is a free massively multiplayer online role-playing game (MMORPG). Join this fascinating game that has thousands of fans from all over the world! - http://www.tibia.com" />
<meta http-equiv="content-type" content="text/html; charset=UTF-8" /> <meta name="description" content="Tibia is a free massively multiplayer online role-playing game (MMORPG). Join this fascinating game that has thousands of fans from all over the world! - http://www.tibia.com" />
<meta name="author" content="CipSoft GmbH" />
<meta http-equiv="content-language" content="en" />
<meta name="keywords" content="free online game, free multiplayer game, free online rpg, free mmorpg, mmorpg, mmog, online role playing game, online multiplayer game, internet game, online rpg, rpg" />
Expand Down
4 changes: 2 additions & 2 deletions src/static/testdata/characters/Mieluffy.html
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@

<!DOCTYPE html>
<html>
<html lang="en">
<head>
<title>Tibia - Free Multiplayer Online Role Playing Game - Community</title>
<meta http-equiv="content-type" content="text/html; charset=ISO-8859-1" /> <meta name="description" content="Tibia is a free massively multiplayer online role-playing game (MMORPG). Join this fascinating game that has thousands of fans from all over the world! - http://www.tibia.com" />
<meta http-equiv="content-type" content="text/html; charset=UTF-8" /> <meta name="description" content="Tibia is a free massively multiplayer online role-playing game (MMORPG). Join this fascinating game that has thousands of fans from all over the world! - http://www.tibia.com" />
<meta name="author" content="CipSoft GmbH" />
<meta http-equiv="content-language" content="en" />
<meta name="keywords" content="free online game, free multiplayer game, free online rpg, free mmorpg, mmorpg, mmog, online role playing game, online multiplayer game, internet game, online rpg, rpg" />
Expand Down
2 changes: 1 addition & 1 deletion src/static/testdata/characters/Nannorka.html
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
<html lang="en">
<head>
<title>Tibia - Free Multiplayer Online Role Playing Game - Community</title>
<meta http-equiv="content-type" content="text/html; charset=ISO-8859-1" /> <meta name="description" content="Tibia is a free massively multiplayer online role-playing game (MMORPG). Join this fascinating game that has thousands of fans from all over the world! - http://www.tibia.com" />
<meta http-equiv="content-type" content="text/html; charset=UTF-8" /> <meta name="description" content="Tibia is a free massively multiplayer online role-playing game (MMORPG). Join this fascinating game that has thousands of fans from all over the world! - http://www.tibia.com" />
<meta name="author" content="CipSoft GmbH" />
<meta http-equiv="content-language" content="en" />
<meta name="keywords" content="free online game, free multiplayer game, free online rpg, free mmorpg, mmorpg, mmog, online role playing game, online multiplayer game, internet game, online rpg, rpg" />
Expand Down
4 changes: 2 additions & 2 deletions src/static/testdata/characters/Ninth Dimension.html
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@

<!DOCTYPE html>
<html>
<html lang="en">
<head>
<title>Tibia - Free Multiplayer Online Role Playing Game - Community</title>
<meta http-equiv="content-type" content="text/html; charset=ISO-8859-1" /> <meta name="description" content="Tibia is a free massively multiplayer online role-playing game (MMORPG). Join this fascinating game that has thousands of fans from all over the world! - http://www.tibia.com" />
<meta http-equiv="content-type" content="text/html; charset=UTF-8" /> <meta name="description" content="Tibia is a free massively multiplayer online role-playing game (MMORPG). Join this fascinating game that has thousands of fans from all over the world! - http://www.tibia.com" />
<meta name="author" content="CipSoft GmbH" />
<meta http-equiv="content-language" content="en" />
<meta name="keywords" content="free online game, free multiplayer game, free online rpg, free mmorpg, mmorpg, mmog, online role playing game, online multiplayer game, internet game, online rpg, rpg" />
Expand Down
4 changes: 2 additions & 2 deletions src/static/testdata/characters/Nocna Furia.html
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@

<!DOCTYPE html>
<html>
<html lang="en">
<head>
<title>Tibia - Free Multiplayer Online Role Playing Game - Community</title>
<meta http-equiv="content-type" content="text/html; charset=ISO-8859-1" /> <meta name="description" content="Tibia is a free massively multiplayer online role-playing game (MMORPG). Join this fascinating game that has thousands of fans from all over the world! - http://www.tibia.com" />
<meta http-equiv="content-type" content="text/html; charset=UTF-8" /> <meta name="description" content="Tibia is a free massively multiplayer online role-playing game (MMORPG). Join this fascinating game that has thousands of fans from all over the world! - http://www.tibia.com" />
<meta name="author" content="CipSoft GmbH" />
<meta http-equiv="content-language" content="en" />
<meta name="keywords" content="free online game, free multiplayer game, free online rpg, free mmorpg, mmorpg, mmog, online role playing game, online multiplayer game, internet game, online rpg, rpg" />
Expand Down
4 changes: 2 additions & 2 deletions src/static/testdata/characters/Orca Kaoksh.html
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@

<!DOCTYPE html>
<html>
<html lang="en">
<head>
<title>Tibia - Free Multiplayer Online Role Playing Game - Community</title>
<meta http-equiv="content-type" content="text/html; charset=ISO-8859-1" /> <meta name="description" content="Tibia is a free massively multiplayer online role-playing game (MMORPG). Join this fascinating game that has thousands of fans from all over the world! - http://www.tibia.com" />
<meta http-equiv="content-type" content="text/html; charset=UTF-8" /> <meta name="description" content="Tibia is a free massively multiplayer online role-playing game (MMORPG). Join this fascinating game that has thousands of fans from all over the world! - http://www.tibia.com" />
<meta name="author" content="CipSoft GmbH" />
<meta http-equiv="content-language" content="en" />
<meta name="keywords" content="free online game, free multiplayer game, free online rpg, free mmorpg, mmorpg, mmog, online role playing game, online multiplayer game, internet game, online rpg, rpg" />
Expand Down
4 changes: 2 additions & 2 deletions src/static/testdata/characters/Rejana on Fera.html
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@

<!DOCTYPE html>
<html>
<html lang="en">
<head>
<title>Tibia - Free Multiplayer Online Role Playing Game - Community</title>
<meta http-equiv="content-type" content="text/html; charset=ISO-8859-1" /> <meta name="description" content="Tibia is a free massively multiplayer online role-playing game (MMORPG). Join this fascinating game that has thousands of fans from all over the world! - http://www.tibia.com" />
<meta http-equiv="content-type" content="text/html; charset=UTF-8" /> <meta name="description" content="Tibia is a free massively multiplayer online role-playing game (MMORPG). Join this fascinating game that has thousands of fans from all over the world! - http://www.tibia.com" />
<meta name="author" content="CipSoft GmbH" />
<meta http-equiv="content-language" content="en" />
<meta name="keywords" content="free online game, free multiplayer game, free online rpg, free mmorpg, mmorpg, mmog, online role playing game, online multiplayer game, internet game, online rpg, rpg" />
Expand Down
4 changes: 2 additions & 2 deletions src/static/testdata/characters/Riley No Hands.html
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@

<!DOCTYPE html>
<html>
<html lang="en">
<head>
<title>Tibia - Free Multiplayer Online Role Playing Game - Community</title>
<meta http-equiv="content-type" content="text/html; charset=ISO-8859-1" /> <meta name="description" content="Tibia is a free massively multiplayer online role-playing game (MMORPG). Join this fascinating game that has thousands of fans from all over the world! - http://www.tibia.com" />
<meta http-equiv="content-type" content="text/html; charset=UTF-8" /> <meta name="description" content="Tibia is a free massively multiplayer online role-playing game (MMORPG). Join this fascinating game that has thousands of fans from all over the world! - http://www.tibia.com" />
<meta name="author" content="CipSoft GmbH" />
<meta http-equiv="content-language" content="en" />
<meta name="keywords" content="free online game, free multiplayer game, free online rpg, free mmorpg, mmorpg, mmog, online role playing game, online multiplayer game, internet game, online rpg, rpg" />
Expand Down
4 changes: 2 additions & 2 deletions src/static/testdata/characters/Sergiozk.html
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@

<!DOCTYPE html>
<html>
<html lang="en">
<head>
<title>Tibia - Free Multiplayer Online Role Playing Game - Community</title>
<meta http-equiv="content-type" content="text/html; charset=ISO-8859-1" /> <meta name="description" content="Tibia is a free massively multiplayer online role-playing game (MMORPG). Join this fascinating game that has thousands of fans from all over the world! - http://www.tibia.com" />
<meta http-equiv="content-type" content="text/html; charset=UTF-8" /> <meta name="description" content="Tibia is a free massively multiplayer online role-playing game (MMORPG). Join this fascinating game that has thousands of fans from all over the world! - http://www.tibia.com" />
<meta name="author" content="CipSoft GmbH" />
<meta http-equiv="content-language" content="en" />
<meta name="keywords" content="free online game, free multiplayer game, free online rpg, free mmorpg, mmorpg, mmog, online role playing game, online multiplayer game, internet game, online rpg, rpg" />
Expand Down
2 changes: 1 addition & 1 deletion src/static/testdata/characters/Sir Dumbas.html
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
<html lang="en">
<head>
<title>Tibia - Free Multiplayer Online Role Playing Game - Community</title>
<meta http-equiv="content-type" content="text/html; charset=ISO-8859-1" /> <meta name="description" content="Tibia is a free massively multiplayer online role-playing game (MMORPG). Join this fascinating game that has thousands of fans from all over the world! - http://www.tibia.com" />
<meta http-equiv="content-type" content="text/html; charset=UTF-8" /> <meta name="description" content="Tibia is a free massively multiplayer online role-playing game (MMORPG). Join this fascinating game that has thousands of fans from all over the world! - http://www.tibia.com" />
<meta name="author" content="CipSoft GmbH" />
<meta http-equiv="content-language" content="en" />
<meta name="keywords" content="free online game, free multiplayer game, free online rpg, free mmorpg, mmorpg, mmog, online role playing game, online multiplayer game, internet game, online rpg, rpg" />
Expand Down
Loading