csv2bin.go

package main

import (
	"bufio"
	"encoding/binary"
	"encoding/csv"
	"fmt"
	"io"
	"math/big"
	"net"
	"os"
	"strconv"
	"strings"
)

var countryPosition = [27]uint8{0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2}
var regionPosition = [27]uint8{0, 0, 0, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3}
var cityPosition = [27]uint8{0, 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4}
var latitudePosition = [27]uint8{0, 0, 0, 0, 0, 5, 5, 0, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5}
var longitudePosition = [27]uint8{0, 0, 0, 0, 0, 6, 6, 0, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6}
var zipCodePosition = [27]uint8{0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 7, 7, 7, 0, 7, 7, 7, 0, 7, 0, 7, 7, 7, 0, 7, 7, 7}
var timeZonePosition = [27]uint8{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 8, 7, 8, 8, 8, 7, 8, 0, 8, 8, 8, 0, 8, 8, 8}
var ispPosition = [27]uint8{0, 0, 3, 0, 5, 0, 7, 5, 7, 0, 8, 0, 9, 0, 9, 0, 9, 0, 9, 7, 9, 0, 9, 7, 9, 9, 9}
var domainPosition = [27]uint8{0, 0, 0, 0, 0, 0, 0, 6, 8, 0, 9, 0, 10, 0, 10, 0, 10, 0, 10, 8, 10, 0, 10, 8, 10, 10, 10}
var netSpeedPosition = [27]uint8{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 11, 0, 11, 8, 11, 0, 11, 0, 11, 0, 11, 11, 11}
var iddCodePosition = [27]uint8{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 12, 0, 12, 0, 12, 9, 12, 0, 12, 12, 12}
var areaCodePosition = [27]uint8{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 13, 0, 13, 0, 13, 10, 13, 0, 13, 13, 13}
var weatherStationCodePosition = [27]uint8{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 14, 0, 14, 0, 14, 0, 14, 14, 14}
var weatherStationNamePosition = [27]uint8{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 15, 0, 15, 0, 15, 0, 15, 15, 15}
var mccPosition = [27]uint8{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 16, 0, 16, 9, 16, 16, 16}
var mncPosition = [27]uint8{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 17, 0, 17, 10, 17, 17, 17}
var mobileBrandPosition = [27]uint8{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 18, 0, 18, 11, 18, 18, 18}
var elevationPosition = [27]uint8{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 19, 0, 19, 19, 19}
var usageTypePosition = [27]uint8{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 20, 20, 20}
var addressTypePosition = [27]uint8{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 21}
var categoryPosition = [27]uint8{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 22}
var districtPosition = [27]uint8{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23}
var asnPosition = [27]uint8{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24}
var asPosition = [27]uint8{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25}
var columnSize = [27]uint8{0, 2, 3, 4, 5, 6, 7, 6, 8, 7, 9, 8, 10, 8, 11, 10, 13, 10, 15, 11, 18, 11, 19, 12, 20, 22, 25}

type countryType struct {
	addr uint32
	long string
}

func WriteBIN(input string, output string, dbPackage string) {
	var err error
	var ispCase uint8 = 0 // need to perform some data manipulation if CSV is IPv6 and contains ISP field

	var dbYear uint8 = 21
	var dbMonth uint8 = 1
	var dbDay uint8 = 20
	var dbProductCode uint8 = 1 // 1 for IP2Location, 2 for IP2Proxy
	var dbProductType uint8 = 3 // 1 for commercial, 2 for LITE, 3 for generated by this script
	var dbFileSize uint32 = 0   // dummy, calculate later after file written

	var dbType64 uint64

	if dbType64, err = strconv.ParseUint(dbPackage, 10, 8); err != nil {
		fmt.Println("Invalid DB package.")
		return
	}
	dbType := uint8(dbType64)
	var ipv4IndexBase uint32 = 64
	var ipv6IndexBase uint32 = ipv4IndexBase + (256 * 256 * 8)
	ipv4IndexRowMin := make(map[uint32]uint32, 65535)
	ipv4IndexRowMax := make(map[uint32]uint32, 65535)
	ipv6IndexRowMin := make(map[uint32]uint32, 65535)
	ipv6IndexRowMax := make(map[uint32]uint32, 65535)
	var ipv4Count uint32 = 0
	var ipv4Base uint32 = ipv6IndexBase + (256 * 256 * 8)
	var longSize uint32 = 4
	dbColl := columnSize[dbType]
	var ipv6Count uint32 = 0
	var ipv6Base uint32 = 0

	// cannot have initial size as we won't know the total elements for each until we read the CSV
	country := map[string]*countryType{}
	region := map[string]uint32{}
	city := map[string]uint32{}
	zipCode := map[string]uint32{}
	timeZone := map[string]uint32{}
	isp := map[string]uint32{}
	domain := map[string]uint32{}
	netSpeed := map[string]uint32{}
	iddCode := map[string]uint32{}
	areaCode := map[string]uint32{}
	weatherStationCode := map[string]uint32{}
	weatherStationName := map[string]uint32{}
	mcc := map[string]uint32{}
	mnc := map[string]uint32{}
	mobileBrand := map[string]uint32{}
	elevation := map[string]uint32{}
	usageType := map[string]uint32{}
	addressType := map[string]uint32{}
	category := map[string]uint32{}
	district := map[string]uint32{}
	asn := map[string]uint32{}
	as := map[string]uint32{}

	countryEnabled := (countryPosition[dbType] > 0)
	regionEnabled := (regionPosition[dbType] > 0)
	cityEnabled := (cityPosition[dbType] > 0)
	latitudeEnabled := (latitudePosition[dbType] > 0)
	longitudeEnabled := (longitudePosition[dbType] > 0)
	zipCodeEnabled := (zipCodePosition[dbType] > 0)
	timeZoneEnabled := (timeZonePosition[dbType] > 0)
	ispEnabled := (ispPosition[dbType] > 0)
	domainEnabled := (domainPosition[dbType] > 0)
	netSpeedEnabled := (netSpeedPosition[dbType] > 0)
	iddCodeEnabled := (iddCodePosition[dbType] > 0)
	areaCodeEnabled := (areaCodePosition[dbType] > 0)
	weatherStationCodeEnabled := (weatherStationCodePosition[dbType] > 0)
	weatherStationNameEnabled := (weatherStationNamePosition[dbType] > 0)
	mccEnabled := (mccPosition[dbType] > 0)
	mncEnabled := (mncPosition[dbType] > 0)
	mobileBrandEnabled := (mobileBrandPosition[dbType] > 0)
	elevationEnabled := (elevationPosition[dbType] > 0)
	usageTypeEnabled := (usageTypePosition[dbType] > 0)
	addressTypeEnabled := (addressTypePosition[dbType] > 0)
	categoryEnabled := (categoryPosition[dbType] > 0)
	districtEnabled := (districtPosition[dbType] > 0)
	asnEnabled := (asnPosition[dbType] > 0)
	asEnabled := (asPosition[dbType] > 0)

	var countrySorted []string
	var regionSorted []string
	var citySorted []string
	var zipCodeSorted []string
	var timeZoneSorted []string
	var ispSorted []string
	var domainSorted []string
	var netSpeedSorted []string
	var iddCodeSorted []string
	var areaCodeSorted []string
	var weatherStationCodeSorted []string
	var weatherStationNameSorted []string
	var mccSorted []string
	var mncSorted []string
	var mobileBrandSorted []string
	var elevationSorted []string
	var usageTypeSorted []string
	var addressTypeSorted []string
	var categorySorted []string
	var districtSorted []string
	var asnSorted []string
	var asSorted []string

	lastIPv4To := ""
	lastIPv6To := ""

	var inFile *os.File
	if inFile, err = os.Open(input); err != nil {
		fmt.Printf("Invalid input file %v.\n", input)
		return
	}
	defer inFile.Close()

	delim := ','
	var rdr *csv.Reader
	inFileBuffered := bufio.NewReaderSize(inFile, 65536)

	csvRdr := csv.NewReader(inFileBuffered)
	csvRdr.Comma = delim
	csvRdr.LazyQuotes = false

	rdr = csvRdr

	lines := 0
	for {
		parts, err := rdr.Read()
		if err == io.EOF {
			break
		} else if err != nil {
			fmt.Println("Unable to read input file.")
			return
		}

		lines++

		if parts[2] == "UK" {
			parts[2] = "GB"
		}
		if countryEnabled {
			if _, ok := country[parts[2]]; !ok { // if map key not exist
				country[parts[2]] = &countryType{addr: 0, long: parts[3]}
			}
		}
		if regionEnabled {
			region[parts[regionPosition[dbType]+1]] = 1
		}
		if cityEnabled {
			city[parts[cityPosition[dbType]+1]] = 1
		}
		if zipCodeEnabled {
			zipCode[parts[zipCodePosition[dbType]+1]] = 1
		}
		if timeZoneEnabled {
			timeZone[parts[timeZonePosition[dbType]+1]] = 1
		}
		if ispEnabled {
			if lines == 1 {
				if strings.Contains(parts[ispPosition[dbType]+1], "Broadcast RFC1700") {
					ispCase = 4 // IPv4 case: no special handling required for the moment
				} else {
					ispCase = 6 // IPv6 case: need to manipulate some rows at the start due to differences in the original input CSV files (pure IPv4 & pure IPv6 vs. merged IP files)
				}
			}
			isp[parts[ispPosition[dbType]+1]] = 1
		}
		if domainEnabled {
			domain[parts[domainPosition[dbType]+1]] = 1
		}
		if netSpeedEnabled {
			netSpeed[parts[netSpeedPosition[dbType]+1]] = 1
		}
		if iddCodeEnabled {
			iddCode[parts[iddCodePosition[dbType]+1]] = 1
		}
		if areaCodeEnabled {
			areaCode[parts[areaCodePosition[dbType]+1]] = 1
		}
		if weatherStationCodeEnabled {
			weatherStationCode[parts[weatherStationCodePosition[dbType]+1]] = 1
		}
		if weatherStationNameEnabled {
			weatherStationName[parts[weatherStationNamePosition[dbType]+1]] = 1
		}
		if mccEnabled {
			mcc[parts[mccPosition[dbType]+1]] = 1
		}
		if mncEnabled {
			mnc[parts[mncPosition[dbType]+1]] = 1
		}
		if mobileBrandEnabled {
			mobileBrand[parts[mobileBrandPosition[dbType]+1]] = 1
		}
		if elevationEnabled {
			elevation[parts[elevationPosition[dbType]+1]] = 1
		}
		if usageTypeEnabled {
			usageType[parts[usageTypePosition[dbType]+1]] = 1
		}
		if addressTypeEnabled {
			addressType[parts[addressTypePosition[dbType]+1]] = 1
		}
		if categoryEnabled {
			category[parts[categoryPosition[dbType]+1]] = 1
		}
		if districtEnabled {
			district[parts[districtPosition[dbType]+1]] = 1
		}
		if asnEnabled {
			asn[parts[asnPosition[dbType]+1]] = 1
		}
		if asEnabled {
			as[parts[asPosition[dbType]+1]] = 1
		}

		startNum := new(big.Int)
		startNum, _ = startNum.SetString(parts[0], 10)

		endNum := new(big.Int)
		endNum, _ = endNum.SetString(parts[1], 10)

		var startIP net.IP
		var endIP net.IP

		if ispCase == 6 && ((lines >= 1 && lines <= 4) || strings.Contains(parts[ispPosition[dbType]+1], "Broadcast RFC1700")) { // special case when ISP field is present in IPv6 CSV
			// First 4 lines treat as IPv6 (16 bytes)
			// After 4th line, insert special 5th line for IPv4Map (hardcode this line for insertion)
			// Special case: Broadcast RFC1700 line convert to Ipv4 and insert under IPv4 section
			if lines >= 1 && lines <= 4 {
				// first 4 lines must treat as IPv6 to insert under IPv6 section
				no2From, err := GetIPv6First2Octet(parts[0])
				if err != nil {
					fmt.Println("Unable to get first 2 octets.")
					return
				}
				no2To, err := GetIPv6First2Octet(parts[1])
				if err != nil {
					fmt.Println("Unable to get first 2 octets.")
					return
				}
				for no2 := no2From; no2 <= no2To; no2++ {
					if _, ok := ipv6IndexRowMin[no2]; !ok { // if map key not exist
						ipv6IndexRowMin[no2] = ipv6Count
					}
					ipv6IndexRowMax[no2] = ipv6Count
				}
				lastIPv6To = parts[1]
				ipv6Count++

				if lines == 4 {
					// need to manually insert another line for IPv4Map
					isp["IPv4Map"] = 1

					no2From, err := GetIPv6First2Octet("281470681743360")
					if err != nil {
						fmt.Println("Unable to get first 2 octets.")
						return
					}
					no2To, err := GetIPv6First2Octet("281474976710655")
					if err != nil {
						fmt.Println("Unable to get first 2 octets.")
						return
					}
					for no2 := no2From; no2 <= no2To; no2++ {
						if _, ok := ipv6IndexRowMin[no2]; !ok { // if map key not exist
							ipv6IndexRowMin[no2] = ipv6Count
						}
						ipv6IndexRowMax[no2] = ipv6Count
					}
					lastIPv6To = "281474976710655"
					ipv6Count++
				}
			} else {
				// Broadcast RFC1700 case where we need to insert into IPv4 section with different IP numbers
				no2From, err := GetIPv4First2Octet("0")
				if err != nil {
					fmt.Println("Unable to get first 2 octets.")
					return
				}
				no2To, err := GetIPv4First2Octet("16777215")
				if err != nil {
					fmt.Println("Unable to get first 2 octets.")
					return
				}
				for no2 := no2From; no2 <= no2To; no2++ {
					if _, ok := ipv4IndexRowMin[no2]; !ok { // if map key not exist
						ipv4IndexRowMin[no2] = ipv4Count
					}
					ipv4IndexRowMax[no2] = ipv4Count
				}
				lastIPv4To = "16777215"
				ipv4Count++
			}

		} else { // normal case where ISP field not present or is IPv4 CSV or rows not covered by the above criteria
			if startIP, err = DecimalToIPv4(startNum); err != nil {
				if startIP, err = DecimalToIPv6(startNum); err != nil {
					fmt.Println("Decimal to IP conversion failed.")
					return
				}
			}
			startIPStr := startIP.String() // this will return plain IPv4 from IPv4-mapped IPv6 since we only want plain IPv4

			if endIP, err = DecimalToIPv4(endNum); err != nil {
				if endIP, err = DecimalToIPv6(endNum); err != nil {
					fmt.Println("Decimal to IP conversion failed.")
					return
				}
			}
			endIPStr := endIP.String() // this will return plain IPv4 from IPv4-mapped IPv6 since we only want plain IPv4

			var newStartNum *big.Int
			var newEndNum *big.Int
			startIPType := 0
			if IsIPv4(startIPStr) {
				startIPType = 4
				if newStartNum, err = IPv4ToDecimal(startIPStr); err != nil {
					fmt.Println("IP to decimal conversion failed.")
					return
				}
				parts[0] = newStartNum.String()
			} else if IsIPv6(startIPStr) {
				startIPType = 6
			}
			endIPType := 0
			if IsIPv4(endIPStr) {
				endIPType = 4
				if newEndNum, err = IPv4ToDecimal(endIPStr); err != nil {
					fmt.Println("IP to decimal conversion failed.")
					return
				}
				parts[1] = newEndNum.String()
			} else if IsIPv6(endIPStr) {
				endIPType = 6
			}

			if startIPType == 4 && endIPType == 4 {
				no2From, err := GetIPv4First2Octet(parts[0])
				if err != nil {
					fmt.Println("Unable to get first 2 octets.")
					return
				}
				no2To, err := GetIPv4First2Octet(parts[1])
				if err != nil {
					fmt.Println("Unable to get first 2 octets.")
					return
				}
				for no2 := no2From; no2 <= no2To; no2++ {
					if _, ok := ipv4IndexRowMin[no2]; !ok { // if map key not exist
						ipv4IndexRowMin[no2] = ipv4Count
					}
					ipv4IndexRowMax[no2] = ipv4Count
				}
				lastIPv4To = parts[1]
				ipv4Count++
			} else if startIPType == 6 && endIPType == 6 {
				no2From, err := GetIPv6First2Octet(parts[0])
				if err != nil {
					fmt.Println("Unable to get first 2 octets.")
					return
				}
				no2To, err := GetIPv6First2Octet(parts[1])
				if err != nil {
					fmt.Println("Unable to get first 2 octets.")
					return
				}
				for no2 := no2From; no2 <= no2To; no2++ {
					if _, ok := ipv6IndexRowMin[no2]; !ok { // if map key not exist
						ipv6IndexRowMin[no2] = ipv6Count
					}
					ipv6IndexRowMax[no2] = ipv6Count
				}
				lastIPv6To = parts[1]
				ipv6Count++
			} else if startIPType == 4 && endIPType == 6 { // special boundary case where we need to split the range into IPv4 only and IPv6 only
				// IPv4 range
				no2From, err := GetIPv4First2Octet(parts[0])
				if err != nil {
					fmt.Println("Unable to get first 2 octets.")
					return
				}
				no2To, err := GetIPv4First2Octet(maxIPv4Range.String()) // 255.255.255.255
				if err != nil {
					fmt.Println("Unable to get first 2 octets.")
					return
				}
				for no2 := no2From; no2 <= no2To; no2++ {
					if _, ok := ipv4IndexRowMin[no2]; !ok { // if map key not exist
						ipv4IndexRowMin[no2] = ipv4Count
					}
					ipv4IndexRowMax[no2] = ipv4Count
				}
				lastIPv4To = maxIPv4Range.String()
				ipv4Count++

				// IPv6 range
				no2From2, err := GetIPv6First2Octet(maxIPv4RangePlusOne.String())
				if err != nil {
					fmt.Println("Unable to get first 2 octets.")
					return
				}
				no2To2, err := GetIPv6First2Octet(parts[1])
				if err != nil {
					fmt.Println("Unable to get first 2 octets.")
					return
				}
				for no2 := no2From2; no2 <= no2To2; no2++ {
					if _, ok := ipv6IndexRowMin[no2]; !ok { // if map key not exist
						ipv6IndexRowMin[no2] = ipv6Count
					}
					ipv6IndexRowMax[no2] = ipv6Count
				}
				lastIPv6To = parts[1]
				ipv6Count++
			}
		}
	}

	if lastIPv4To != "4294967295" {
		fmt.Printf("The last IP address in the CSV file is %s not 4294967295.\n", lastIPv4To)
		return
	}
	if lastIPv6To != "" && lastIPv6To != "340282366920938463463374607431768211455" { // blank means IPv4-only CSV
		fmt.Printf("The last IP address in the CSV file is %s not 340282366920938463463374607431768211455.\n", lastIPv6To)
		return
	}

	ipv4Count++
	if lastIPv6To != "" { // if is IPv6 CSV
		ipv6Count++
	}

	ipv6Base = ipv4Base + ipv4Count*longSize*uint32(dbColl)
	addr := ipv6Base + ipv6Count*longSize*(uint32(dbColl)+3) // IPv6 address range is 4 bytes vs 1 byte in IPv4

	if countryEnabled {
		countrySorted = GetSortedKeysCountry(country)
		for _, v := range countrySorted {
			country[v].addr = addr
			addr = addr + 1 + 2 + 1 + uint32(len(country[v].long))
		}
	}

	if regionEnabled {
		regionSorted = GetSortedKeys(region)
		for _, v := range regionSorted {
			region[v] = addr
			addr = addr + 1 + uint32(len(v))
		}
	}

	if cityEnabled {
		citySorted = GetSortedKeys(city)
		for _, v := range citySorted {
			city[v] = addr
			addr = addr + 1 + uint32(len(v))
		}
	}

	if zipCodeEnabled {
		zipCodeSorted = GetSortedKeys(zipCode)
		for _, v := range zipCodeSorted {
			zipCode[v] = addr
			addr = addr + 1 + uint32(len(v))
		}
	}

	if timeZoneEnabled {
		timeZoneSorted = GetSortedKeys(timeZone)
		for _, v := range timeZoneSorted {
			timeZone[v] = addr
			addr = addr + 1 + uint32(len(v))
		}
	}

	if ispEnabled {
		ispSorted = GetSortedKeys(isp)
		for _, v := range ispSorted {
			isp[v] = addr
			addr = addr + 1 + uint32(len(v))
		}
	}

	if domainEnabled {
		domainSorted = GetSortedKeys(domain)
		for _, v := range domainSorted {
			domain[v] = addr
			addr = addr + 1 + uint32(len(v))
		}
	}

	if netSpeedEnabled {
		netSpeedSorted = GetSortedKeys(netSpeed)
		for _, v := range netSpeedSorted {
			netSpeed[v] = addr
			addr = addr + 1 + uint32(len(v))
		}
	}

	if iddCodeEnabled {
		iddCodeSorted = GetSortedKeys(iddCode)
		for _, v := range iddCodeSorted {
			iddCode[v] = addr
			addr = addr + 1 + uint32(len(v))
		}
	}

	if areaCodeEnabled {
		areaCodeSorted = GetSortedKeys(areaCode)
		for _, v := range areaCodeSorted {
			areaCode[v] = addr
			addr = addr + 1 + uint32(len(v))
		}
	}

	if weatherStationCodeEnabled {
		weatherStationCodeSorted = GetSortedKeys(weatherStationCode)
		for _, v := range weatherStationCodeSorted {
			weatherStationCode[v] = addr
			addr = addr + 1 + uint32(len(v))
		}
	}

	if weatherStationNameEnabled {
		weatherStationNameSorted = GetSortedKeys(weatherStationName)
		for _, v := range weatherStationNameSorted {
			weatherStationName[v] = addr
			addr = addr + 1 + uint32(len(v))
		}
	}

	if mccEnabled {
		mccSorted = GetSortedKeys(mcc)
		for _, v := range mccSorted {
			mcc[v] = addr
			addr = addr + 1 + uint32(len(v))
		}
	}

	if mncEnabled {
		mncSorted = GetSortedKeys(mnc)
		for _, v := range mncSorted {
			mnc[v] = addr
			addr = addr + 1 + uint32(len(v))
		}
	}

	if mobileBrandEnabled {
		mobileBrandSorted = GetSortedKeys(mobileBrand)
		for _, v := range mobileBrandSorted {
			mobileBrand[v] = addr
			addr = addr + 1 + uint32(len(v))
		}
	}

	if elevationEnabled {
		elevationSorted = GetSortedKeys(elevation)
		for _, v := range elevationSorted {
			elevation[v] = addr
			addr = addr + 1 + uint32(len(v))
		}
	}

	if usageTypeEnabled {
		usageTypeSorted = GetSortedKeys(usageType)
		for _, v := range usageTypeSorted {
			usageType[v] = addr
			addr = addr + 1 + uint32(len(v))
		}
	}

	if addressTypeEnabled {
		addressTypeSorted = GetSortedKeys(addressType)
		for _, v := range addressTypeSorted {
			addressType[v] = addr
			addr = addr + 1 + uint32(len(v))
		}
	}

	if categoryEnabled {
		categorySorted = GetSortedKeys(category)
		for _, v := range categorySorted {
			category[v] = addr
			addr = addr + 1 + uint32(len(v))
		}
	}

	if districtEnabled {
		districtSorted = GetSortedKeys(district)
		for _, v := range districtSorted {
			district[v] = addr
			addr = addr + 1 + uint32(len(v))
		}
	}

	if asnEnabled {
		asnSorted = GetSortedKeys(asn)
		for _, v := range asnSorted {
			asn[v] = addr
			addr = addr + 1 + uint32(len(v))
		}
	}

	if asEnabled {
		asSorted = GetSortedKeys(as)
		for _, v := range asSorted {
			as[v] = addr
			addr = addr + 1 + uint32(len(v))
		}
	}

	var inFile2 *os.File
	if inFile2, err = os.Open(input); err != nil {
		fmt.Printf("Invalid input file %v.\n", input)
		return
	}
	defer inFile2.Close()

	var outFile *os.File
	if outFile, err = os.Create(output); err != nil {
		fmt.Printf("Could not create output file %v.\n", output)
		return
	}
	defer outFile.Close()

	var header = []any{
		dbType,
		dbColl,
		dbYear,
		dbMonth,
		dbDay,
		ipv4Count,
		ipv4Base + 1,
		ipv6Count,
		ipv6Base + 1,
		ipv4IndexBase + 1,
		ipv6IndexBase + 1,
		dbProductCode,
		dbProductType,
		dbFileSize,
	}

	for _, v := range header {
		// due to different data types, we loop and let the Write perform the conversion to bytes
		WriteMe(outFile, v)
	}

	bytes := make([]byte, 63-35+1) // bunch of zero bytes
	WriteMe(outFile, bytes)

	var sortedUint []uint32
	sortedUint = GetSortedKeysUint(ipv4IndexRowMin)
	for _, v := range sortedUint {
		WriteMe(outFile, ipv4IndexRowMin[v])
		WriteMe(outFile, ipv4IndexRowMax[v])
	}

	sortedUint = GetSortedKeysUint(ipv6IndexRowMin)
	for _, v := range sortedUint {
		WriteMe(outFile, ipv6IndexRowMin[v])
		WriteMe(outFile, ipv6IndexRowMax[v])
	}

	p := Tell(outFile)
	if p != 1048640 {
		fmt.Println("Index out of range.")
		return
	}

	var rdr2 *csv.Reader
	inFileBuffered2 := bufio.NewReaderSize(inFile2, 65536)

	csvRdr2 := csv.NewReader(inFileBuffered2)
	csvRdr2.Comma = delim
	csvRdr2.LazyQuotes = false

	rdr2 = csvRdr2

	lines = 0

	// these 2 are for ISP IPv6 special cases to rearrange some rows
	var row4 = []any{}
	var row6 = []any{}
	var outputV4Ending = 0 // force output the ending row for IPv4 in ISP IPv6 case

	for {
		parts, err := rdr2.Read()

		if err == io.EOF {
			break
		} else if err != nil {
			fmt.Println("Unable to read input file.")
			return
		}

		lines++

		startNum := new(big.Int)
		startNum, _ = startNum.SetString(parts[0], 10)

		endNum := new(big.Int)
		endNum, _ = endNum.SetString(parts[1], 10)

		var startIP net.IP
		var endIP net.IP

		if parts[2] == "UK" {
			parts[2] = "GB"
		}

		var row = []any{}

		if ispCase == 6 && ((lines >= 1 && lines <= 4) || strings.Contains(parts[ispPosition[dbType]+1], "Broadcast RFC1700")) { // special case when ISP field is present in IPv6 CSV
			outputV4Ending = 1
			if lines >= 1 && lines <= 4 {
				// These 4 lines should be IPv6 even though first 3 lines are showing IPv4
				v6Bytes, err := ForceAsIPv6(parts[0])
				if err != nil {
					fmt.Println("IP to bytes conversion failed.")
					return
				}
				ReverseBytes(v6Bytes)
				row6 = append(row6, v6Bytes)
				if countryEnabled {
					row6 = append(row6, country[parts[2]].addr)
				}
				if regionEnabled {
					row6 = append(row6, region[parts[regionPosition[dbType]+1]])
				}
				if cityEnabled {
					row6 = append(row6, city[parts[cityPosition[dbType]+1]])
				}
				if latitudeEnabled {
					lat, err := strconv.ParseFloat(parts[latitudePosition[dbType]+1], 64)
					if err != nil {
						fmt.Println("String to float conversion failed.")
						return
					}
					row6 = append(row6, float32(lat))
				}
				if longitudeEnabled {
					long, err := strconv.ParseFloat(parts[longitudePosition[dbType]+1], 64)
					if err != nil {
						fmt.Println("String to float conversion failed.")
						return
					}
					row6 = append(row6, float32(long))
				}
				if zipCodeEnabled {
					row6 = append(row6, zipCode[parts[zipCodePosition[dbType]+1]])
				}
				if timeZoneEnabled {
					row6 = append(row6, timeZone[parts[timeZonePosition[dbType]+1]])
				}
				if ispEnabled {
					row6 = append(row6, isp[parts[ispPosition[dbType]+1]])
				}
				if domainEnabled {
					row6 = append(row6, domain[parts[domainPosition[dbType]+1]])
				}
				if netSpeedEnabled {
					row6 = append(row6, netSpeed[parts[netSpeedPosition[dbType]+1]])
				}
				if iddCodeEnabled {
					row6 = append(row6, iddCode[parts[iddCodePosition[dbType]+1]])
				}
				if areaCodeEnabled {
					row6 = append(row6, areaCode[parts[areaCodePosition[dbType]+1]])
				}
				if weatherStationCodeEnabled {
					row6 = append(row6, weatherStationCode[parts[weatherStationCodePosition[dbType]+1]])
				}
				if weatherStationNameEnabled {
					row6 = append(row6, weatherStationName[parts[weatherStationNamePosition[dbType]+1]])
				}
				if mccEnabled {
					row6 = append(row6, mcc[parts[mccPosition[dbType]+1]])
				}
				if mncEnabled {
					row6 = append(row6, mnc[parts[mncPosition[dbType]+1]])
				}
				if mobileBrandEnabled {
					row6 = append(row6, mobileBrand[parts[mobileBrandPosition[dbType]+1]])
				}
				if elevationEnabled {
					row6 = append(row6, elevation[parts[elevationPosition[dbType]+1]])
				}
				if usageTypeEnabled {
					row6 = append(row6, usageType[parts[usageTypePosition[dbType]+1]])
				}
				if addressTypeEnabled {
					row6 = append(row6, addressType[parts[addressTypePosition[dbType]+1]])
				}
				if categoryEnabled {
					row6 = append(row6, category[parts[categoryPosition[dbType]+1]])
				}
				if districtEnabled {
					row6 = append(row6, district[parts[districtPosition[dbType]+1]])
				}
				if asnEnabled {
					row6 = append(row6, asn[parts[asnPosition[dbType]+1]])
				}
				if asEnabled {
					row6 = append(row6, as[parts[asPosition[dbType]+1]])
				}

				if lines == 4 {
					// need to manually insert another line for IPv4Map (should be IPv6)
					v6Bytes, err := ForceAsIPv6("281470681743360")
					if err != nil {
						fmt.Println("IP to bytes conversion failed.")
						return
					}
					ReverseBytes(v6Bytes)
					row6 = append(row6, v6Bytes)
					if countryEnabled {
						row6 = append(row6, country["-"].addr)
					}
					if regionEnabled {
						row6 = append(row6, region["-"])
					}
					if cityEnabled {
						row6 = append(row6, city["-"])
					}
					if latitudeEnabled {
						lat, err := strconv.ParseFloat("0.000000", 64)
						if err != nil {
							fmt.Println("String to float conversion failed.")
							return
						}
						row6 = append(row6, float32(lat))
					}
					if longitudeEnabled {
						long, err := strconv.ParseFloat("0.000000", 64)
						if err != nil {
							fmt.Println("String to float conversion failed.")
							return
						}
						row6 = append(row6, float32(long))
					}
					if zipCodeEnabled {
						row6 = append(row6, zipCode["-"])
					}
					if timeZoneEnabled {
						row6 = append(row6, timeZone["-"])
					}
					if ispEnabled {
						row6 = append(row6, isp["IPv4Map"])
					}
					if domainEnabled {
						row6 = append(row6, domain["-"])
					}
					if netSpeedEnabled {
						row6 = append(row6, netSpeed["-"])
					}
					if iddCodeEnabled {
						row6 = append(row6, iddCode["-"])
					}
					if areaCodeEnabled {
						row6 = append(row6, areaCode["-"])
					}
					if weatherStationCodeEnabled {
						row6 = append(row6, weatherStationCode["-"])
					}
					if weatherStationNameEnabled {
						row6 = append(row6, weatherStationName["-"])
					}
					if mccEnabled {
						row6 = append(row6, mcc["-"])
					}
					if mncEnabled {
						row6 = append(row6, mnc["-"])
					}
					if mobileBrandEnabled {
						row6 = append(row6, mobileBrand["-"])
					}
					if elevationEnabled {
						row6 = append(row6, elevation["0"])
					}
					if usageTypeEnabled {
						row6 = append(row6, usageType["RSV"])
					}
					if addressTypeEnabled {
						row6 = append(row6, addressType["U"])
					}
					if categoryEnabled {
						row6 = append(row6, category["IAB24"])
					}
					if districtEnabled {
						row6 = append(row6, district["-"])
					}
					if asnEnabled {
						row6 = append(row6, asn["-"])
					}
					if asEnabled {
						row6 = append(row6, as["-"])
					}
				}
			} else {
				// Broadcast RFC1700 case where we need to insert into IPv4 section with different IPv4 IP numbers
				v4Bytes, err := ForceAsIPv4("0")
				if err != nil {
					fmt.Println("IP to bytes conversion failed.")
					return
				}
				ReverseBytes(v4Bytes)
				row = append(row, v4Bytes)
				if countryEnabled {
					row = append(row, country[parts[2]].addr)
				}
				if regionEnabled {
					row = append(row, region[parts[regionPosition[dbType]+1]])
				}
				if cityEnabled {
					row = append(row, city[parts[cityPosition[dbType]+1]])
				}
				if latitudeEnabled {
					lat, err := strconv.ParseFloat(parts[latitudePosition[dbType]+1], 64)
					if err != nil {
						fmt.Println("String to float conversion failed.")
						return
					}
					row = append(row, float32(lat))
				}
				if longitudeEnabled {
					long, err := strconv.ParseFloat(parts[longitudePosition[dbType]+1], 64)
					if err != nil {
						fmt.Println("String to float conversion failed.")
						return
					}
					row = append(row, float32(long))
				}
				if zipCodeEnabled {
					row = append(row, zipCode[parts[zipCodePosition[dbType]+1]])
				}
				if timeZoneEnabled {
					row = append(row, timeZone[parts[timeZonePosition[dbType]+1]])
				}
				if ispEnabled {
					row = append(row, isp[parts[ispPosition[dbType]+1]])
				}
				if domainEnabled {
					row = append(row, domain[parts[domainPosition[dbType]+1]])
				}
				if netSpeedEnabled {
					row = append(row, netSpeed[parts[netSpeedPosition[dbType]+1]])
				}
				if iddCodeEnabled {
					row = append(row, iddCode[parts[iddCodePosition[dbType]+1]])
				}
				if areaCodeEnabled {
					row = append(row, areaCode[parts[areaCodePosition[dbType]+1]])
				}
				if weatherStationCodeEnabled {
					row = append(row, weatherStationCode[parts[weatherStationCodePosition[dbType]+1]])
				}
				if weatherStationNameEnabled {
					row = append(row, weatherStationName[parts[weatherStationNamePosition[dbType]+1]])
				}
				if mccEnabled {
					row = append(row, mcc[parts[mccPosition[dbType]+1]])
				}
				if mncEnabled {
					row = append(row, mnc[parts[mncPosition[dbType]+1]])
				}
				if mobileBrandEnabled {
					row = append(row, mobileBrand[parts[mobileBrandPosition[dbType]+1]])
				}
				if elevationEnabled {
					row = append(row, elevation[parts[elevationPosition[dbType]+1]])
				}
				if usageTypeEnabled {
					row = append(row, usageType[parts[usageTypePosition[dbType]+1]])
				}
				if addressTypeEnabled {
					row = append(row, addressType[parts[addressTypePosition[dbType]+1]])
				}
				if categoryEnabled {
					row = append(row, category[parts[categoryPosition[dbType]+1]])
				}
				if districtEnabled {
					row = append(row, district[parts[districtPosition[dbType]+1]])
				}
				if asnEnabled {
					row = append(row, asn[parts[asnPosition[dbType]+1]])
				}
				if asEnabled {
					row = append(row, as[parts[asPosition[dbType]+1]])
				}
			}
		} else { // normal case where ISP field not present or is IPv4 CSV or rows not covered by the above criteria
			if startIP, err = DecimalToIPv4(startNum); err != nil {
				if startIP, err = DecimalToIPv6(startNum); err != nil {
					fmt.Println("Decimal to IP conversion failed.")
					return
				}
			}
			startIPStr := startIP.String() // this will return plain IPv4 from IPv4-mapped IPv6 since we only want plain IPv4

			if endIP, err = DecimalToIPv4(endNum); err != nil {
				if endIP, err = DecimalToIPv6(endNum); err != nil {
					fmt.Println("Decimal to IP conversion failed.")
					return
				}
			}
			endIPStr := endIP.String() // this will return plain IPv4 from IPv4-mapped IPv6 since we only want plain IPv4

			var newStartNum *big.Int
			var newEndNum *big.Int
			startIPType := 0
			if IsIPv4(startIPStr) {
				startIPType = 4
				if newStartNum, err = IPv4ToDecimal(startIPStr); err != nil {
					fmt.Println("IP to decimal conversion failed.")
					return
				}
				parts[0] = newStartNum.String()
			} else if IsIPv6(startIPStr) {
				startIPType = 6
			}
			endIPType := 0
			if IsIPv4(endIPStr) {
				endIPType = 4
				if newEndNum, err = IPv4ToDecimal(endIPStr); err != nil {
					fmt.Println("IP to decimal conversion failed.")
					return
				}
				parts[1] = newEndNum.String()
			} else if IsIPv6(endIPStr) {
				endIPType = 6
			}

			ipv4boundary := false

			if startIPType == 4 && endIPType == 4 {
				v4Bytes, err := IPv4ToBytes(startIPStr)
				if err != nil {
					fmt.Println("IP to bytes conversion failed.")
					return
				}
				ReverseBytes(v4Bytes)
				row = append(row, v4Bytes)
			} else if startIPType == 6 && endIPType == 6 {
				// assume the ISP cases only hit this case
				if outputV4Ending == 1 {
					outputV4Ending = 0

					v4Bytes, err := IPv4ToBytes("255.255.255.255")
					if err != nil {
						fmt.Println("IP to bytes conversion failed.")
						return
					}
					ReverseBytes(v4Bytes)
					row4 = append(row4, v4Bytes)

					if countryEnabled {
						row4 = append(row4, country["-"].addr)
					}
					if regionEnabled {
						row4 = append(row4, region["-"])
					}
					if cityEnabled {
						row4 = append(row4, city["-"])
					}
					if latitudeEnabled {
						row4 = append(row4, float32(0.0))
					}
					if longitudeEnabled {
						row4 = append(row4, float32(0.0))
					}
					if zipCodeEnabled {
						row4 = append(row4, zipCode["-"])
					}
					if timeZoneEnabled {
						row4 = append(row4, timeZone["-"])
					}
					if ispEnabled {
						row4 = append(row4, isp["-"])
					}
					if domainEnabled {
						row4 = append(row4, domain["-"])
					}
					if netSpeedEnabled {
						row4 = append(row4, netSpeed["-"])
					}
					if iddCodeEnabled {
						row4 = append(row4, iddCode["-"])
					}
					if areaCodeEnabled {
						row4 = append(row4, areaCode["-"])
					}
					if weatherStationCodeEnabled {
						row4 = append(row4, weatherStationCode["-"])
					}
					if weatherStationNameEnabled {
						row4 = append(row4, weatherStationName["-"])
					}
					if mccEnabled {
						row4 = append(row4, mcc["-"])
					}
					if mncEnabled {
						row4 = append(row4, mnc["-"])
					}
					if mobileBrandEnabled {
						row4 = append(row4, mobileBrand["-"])
					}
					if elevationEnabled {
						row4 = append(row4, elevation["-"])
					}
					if usageTypeEnabled {
						row4 = append(row4, usageType["-"])
					}
					if addressTypeEnabled {
						row4 = append(row4, addressType["-"])
					}
					if categoryEnabled {
						row4 = append(row4, category["-"])
					}
					if districtEnabled {
						row4 = append(row4, district["-"])
					}
					if asnEnabled {
						row4 = append(row4, asn["-"])
					}
					if asEnabled {
						row4 = append(row4, as["-"])
					}

					for _, v := range row4 {
						// due to different data types, we loop and let the Write perform the conversion to bytes
						WriteMe(outFile, v)
					}
					row4 = row4[:0] // reset row4
				}

				if len(row6) > 0 {
					for _, v := range row6 {
						// due to different data types, we loop and let the Write perform the conversion to bytes
						WriteMe(outFile, v)
					}
					row6 = row6[:0] // reset row6
				}

				v6Bytes, err := IPv6ToBytes(startIPStr)
				if err != nil {
					fmt.Println("IP to bytes conversion failed.")
					return
				}
				ReverseBytes(v6Bytes)
				row = append(row, v6Bytes)
			} else if startIPType == 4 && endIPType == 6 { // special boundary case where we need to split the range into IPv4 only and IPv6 only
				ipv4boundary = true
				v4Bytes, err := IPv4ToBytes(startIPStr)
				if err != nil {
					fmt.Println("IP to bytes conversion failed.")
					return
				}
				ReverseBytes(v4Bytes)
				row = append(row, v4Bytes)
			}

			if countryEnabled {
				row = append(row, country[parts[2]].addr)
			}
			if regionEnabled {
				row = append(row, region[parts[regionPosition[dbType]+1]])
			}
			if cityEnabled {
				row = append(row, city[parts[cityPosition[dbType]+1]])
			}
			if latitudeEnabled {
				lat, err := strconv.ParseFloat(parts[latitudePosition[dbType]+1], 64)
				if err != nil {
					fmt.Println("String to float conversion failed.")
					return
				}
				row = append(row, float32(lat))
			}
			if longitudeEnabled {
				long, err := strconv.ParseFloat(parts[longitudePosition[dbType]+1], 64)
				if err != nil {
					fmt.Println("String to float conversion failed.")
					return
				}
				row = append(row, float32(long))
			}
			if zipCodeEnabled {
				row = append(row, zipCode[parts[zipCodePosition[dbType]+1]])
			}
			if timeZoneEnabled {
				row = append(row, timeZone[parts[timeZonePosition[dbType]+1]])
			}
			if ispEnabled {
				row = append(row, isp[parts[ispPosition[dbType]+1]])
			}
			if domainEnabled {
				row = append(row, domain[parts[domainPosition[dbType]+1]])
			}
			if netSpeedEnabled {
				row = append(row, netSpeed[parts[netSpeedPosition[dbType]+1]])
			}
			if iddCodeEnabled {
				row = append(row, iddCode[parts[iddCodePosition[dbType]+1]])
			}
			if areaCodeEnabled {
				row = append(row, areaCode[parts[areaCodePosition[dbType]+1]])
			}
			if weatherStationCodeEnabled {
				row = append(row, weatherStationCode[parts[weatherStationCodePosition[dbType]+1]])
			}
			if weatherStationNameEnabled {
				row = append(row, weatherStationName[parts[weatherStationNamePosition[dbType]+1]])
			}
			if mccEnabled {
				row = append(row, mcc[parts[mccPosition[dbType]+1]])
			}
			if mncEnabled {
				row = append(row, mnc[parts[mncPosition[dbType]+1]])
			}
			if mobileBrandEnabled {
				row = append(row, mobileBrand[parts[mobileBrandPosition[dbType]+1]])
			}
			if elevationEnabled {
				row = append(row, elevation[parts[elevationPosition[dbType]+1]])
			}
			if usageTypeEnabled {
				row = append(row, usageType[parts[usageTypePosition[dbType]+1]])
			}
			if addressTypeEnabled {
				row = append(row, addressType[parts[addressTypePosition[dbType]+1]])
			}
			if categoryEnabled {
				row = append(row, category[parts[categoryPosition[dbType]+1]])
			}
			if districtEnabled {
				row = append(row, district[parts[districtPosition[dbType]+1]])
			}
			if asnEnabled {
				row = append(row, asn[parts[asnPosition[dbType]+1]])
			}
			if asEnabled {
				row = append(row, as[parts[asPosition[dbType]+1]])
			}

			if ipv4boundary {
				ipv4boundary = false

				var err error
				var v4Bytes []byte
				var v6Bytes []byte

				// IPv4 part
				v4Bytes, err = IPv4ToBytes("255.255.255.255")
				if err != nil {
					fmt.Println("IP to bytes conversion failed.")
					return
				}
				ReverseBytes(v4Bytes)
				row = append(row, v4Bytes)

				if countryEnabled {
					row = append(row, country["-"].addr)
				}
				if regionEnabled {
					row = append(row, region["-"])
				}
				if cityEnabled {
					row = append(row, city["-"])
				}
				if latitudeEnabled {
					row = append(row, float32(0.0))
				}
				if longitudeEnabled {
					row = append(row, float32(0.0))
				}
				if zipCodeEnabled {
					row = append(row, zipCode["-"])
				}
				if timeZoneEnabled {
					row = append(row, timeZone["-"])
				}
				if ispEnabled {
					row = append(row, isp["-"])
				}
				if domainEnabled {
					row = append(row, domain["-"])
				}
				if netSpeedEnabled {
					row = append(row, netSpeed["-"])
				}
				if iddCodeEnabled {
					row = append(row, iddCode["-"])
				}
				if areaCodeEnabled {
					row = append(row, areaCode["-"])
				}
				if weatherStationCodeEnabled {
					row = append(row, weatherStationCode["-"])
				}
				if weatherStationNameEnabled {
					row = append(row, weatherStationName["-"])
				}
				if mccEnabled {
					row = append(row, mcc["-"])
				}
				if mncEnabled {
					row = append(row, mnc["-"])
				}
				if mobileBrandEnabled {
					row = append(row, mobileBrand["-"])
				}
				if elevationEnabled {
					row = append(row, elevation["-"])
				}
				if usageTypeEnabled {
					row = append(row, usageType["-"])
				}
				if addressTypeEnabled {
					row = append(row, addressType["-"])
				}
				if categoryEnabled {
					row = append(row, category["-"])
				}
				if districtEnabled {
					row = append(row, district["-"])
				}
				if asnEnabled {
					row = append(row, asn["-"])
				}
				if asEnabled {
					row = append(row, as["-"])
				}

				// IPv6 part
				v6Bytes, err = IPv6ToBytes("::1:0:0")
				if err != nil {
					fmt.Println("IP to bytes conversion failed.")
					return
				}
				ReverseBytes(v6Bytes)
				row = append(row, v6Bytes)

				if countryEnabled {
					row = append(row, country["-"].addr)
				}
				if regionEnabled {
					row = append(row, region["-"])
				}
				if cityEnabled {
					row = append(row, city["-"])
				}
				if latitudeEnabled {
					row = append(row, float32(0.0))
				}
				if longitudeEnabled {
					row = append(row, float32(0.0))
				}
				if zipCodeEnabled {
					row = append(row, zipCode["-"])
				}
				if timeZoneEnabled {
					row = append(row, timeZone["-"])
				}
				if ispEnabled {
					row = append(row, isp["-"])
				}
				if domainEnabled {
					row = append(row, domain["-"])
				}
				if netSpeedEnabled {
					row = append(row, netSpeed["-"])
				}
				if iddCodeEnabled {
					row = append(row, iddCode["-"])
				}
				if areaCodeEnabled {
					row = append(row, areaCode["-"])
				}
				if weatherStationCodeEnabled {
					row = append(row, weatherStationCode["-"])
				}
				if weatherStationNameEnabled {
					row = append(row, weatherStationName["-"])
				}
				if mccEnabled {
					row = append(row, mcc["-"])
				}
				if mncEnabled {
					row = append(row, mnc["-"])
				}
				if mobileBrandEnabled {
					row = append(row, mobileBrand["-"])
				}
				if elevationEnabled {
					row = append(row, elevation["-"])
				}
				if usageTypeEnabled {
					row = append(row, usageType["-"])
				}
				if addressTypeEnabled {
					row = append(row, addressType["-"])
				}
				if categoryEnabled {
					row = append(row, category["-"])
				}
				if districtEnabled {
					row = append(row, district["-"])
				}
				if asnEnabled {
					row = append(row, asn["-"])
				}
				if asEnabled {
					row = append(row, as["-"])
				}
			}
		}

		for _, v := range row {
			// due to different data types, we loop and let the Write perform the conversion to bytes
			WriteMe(outFile, v)
		}
	}

	// dealing with ending record
	var row = []any{}
	if lastIPv6To != "" { // output ending record for IPv6 range if is IPv6 CSV
		v6Bytes, err := IPv6ToBytes("ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff")
		if err != nil {
			fmt.Println("IP to bytes conversion failed.")
			return
		}
		ReverseBytes(v6Bytes)
		row = append(row, v6Bytes)
	} else { // only IPv4 CSV so need to output ending record for IPv4 range
		v4Bytes, err := IPv4ToBytes("255.255.255.255")
		if err != nil {
			fmt.Println("IP to bytes conversion failed.")
			return
		}
		ReverseBytes(v4Bytes)
		row = append(row, v4Bytes)
	}

	if countryEnabled {
		row = append(row, country["-"].addr)
	}
	if regionEnabled {
		row = append(row, region["-"])
	}
	if cityEnabled {
		row = append(row, city["-"])
	}
	if latitudeEnabled {
		row = append(row, float32(0.0))
	}
	if longitudeEnabled {
		row = append(row, float32(0.0))
	}
	if zipCodeEnabled {
		row = append(row, zipCode["-"])
	}
	if timeZoneEnabled {
		row = append(row, timeZone["-"])
	}
	if ispEnabled {
		row = append(row, isp["-"])
	}
	if domainEnabled {
		row = append(row, domain["-"])
	}
	if netSpeedEnabled {
		row = append(row, netSpeed["-"])
	}
	if iddCodeEnabled {
		row = append(row, iddCode["-"])
	}
	if areaCodeEnabled {
		row = append(row, areaCode["-"])
	}
	if weatherStationCodeEnabled {
		row = append(row, weatherStationCode["-"])
	}
	if weatherStationNameEnabled {
		row = append(row, weatherStationName["-"])
	}
	if mccEnabled {
		row = append(row, mcc["-"])
	}
	if mncEnabled {
		row = append(row, mnc["-"])
	}
	if mobileBrandEnabled {
		row = append(row, mobileBrand["-"])
	}
	if elevationEnabled {
		row = append(row, elevation["-"])
	}
	if usageTypeEnabled {
		row = append(row, usageType["-"])
	}
	if addressTypeEnabled {
		row = append(row, addressType["-"])
	}
	if categoryEnabled {
		row = append(row, category["-"])
	}
	if districtEnabled {
		row = append(row, district["-"])
	}
	if asnEnabled {
		row = append(row, asn["-"])
	}
	if asEnabled {
		row = append(row, as["-"])
	}

	for _, v := range row {
		// due to different data types, we loop and let the Write perform the conversion to bytes
		WriteMe(outFile, v)
	}

	if countryEnabled {
		for _, v := range countrySorted {
			WriteMe(outFile, uint8(len(v)))
			WriteMe(outFile, v)
			if v == "-" {
				WriteMe(outFile, " ") // to make it 2 bytes
			}
			WriteMe(outFile, uint8(len(country[v].long)))
			WriteMe(outFile, country[v].long)
		}
	}
	if regionEnabled {
		for _, v := range regionSorted {
			WriteMe(outFile, uint8(len(v)))
			WriteMe(outFile, v)
		}
	}
	if cityEnabled {
		for _, v := range citySorted {
			WriteMe(outFile, uint8(len(v)))
			WriteMe(outFile, v)
		}
	}
	if zipCodeEnabled {
		for _, v := range zipCodeSorted {
			WriteMe(outFile, uint8(len(v)))
			WriteMe(outFile, v)
		}
	}
	if timeZoneEnabled {
		for _, v := range timeZoneSorted {
			WriteMe(outFile, uint8(len(v)))
			WriteMe(outFile, v)
		}
	}
	if ispEnabled {
		for _, v := range ispSorted {
			WriteMe(outFile, uint8(len(v)))
			WriteMe(outFile, v)
		}
	}
	if domainEnabled {
		for _, v := range domainSorted {
			WriteMe(outFile, uint8(len(v)))
			WriteMe(outFile, v)
		}
	}
	if netSpeedEnabled {
		for _, v := range netSpeedSorted {
			WriteMe(outFile, uint8(len(v)))
			WriteMe(outFile, v)
		}
	}
	if iddCodeEnabled {
		for _, v := range iddCodeSorted {
			WriteMe(outFile, uint8(len(v)))
			WriteMe(outFile, v)
		}
	}
	if areaCodeEnabled {
		for _, v := range areaCodeSorted {
			WriteMe(outFile, uint8(len(v)))
			WriteMe(outFile, v)
		}
	}
	if weatherStationCodeEnabled {
		for _, v := range weatherStationCodeSorted {
			WriteMe(outFile, uint8(len(v)))
			WriteMe(outFile, v)
		}
	}
	if weatherStationNameEnabled {
		for _, v := range weatherStationNameSorted {
			WriteMe(outFile, uint8(len(v)))
			WriteMe(outFile, v)
		}
	}
	if mccEnabled {
		for _, v := range mccSorted {
			WriteMe(outFile, uint8(len(v)))
			WriteMe(outFile, v)
		}
	}
	if mncEnabled {
		for _, v := range mncSorted {
			WriteMe(outFile, uint8(len(v)))
			WriteMe(outFile, v)
		}
	}
	if mobileBrandEnabled {
		for _, v := range mobileBrandSorted {
			WriteMe(outFile, uint8(len(v)))
			WriteMe(outFile, v)
		}
	}
	if elevationEnabled {
		for _, v := range elevationSorted {
			WriteMe(outFile, uint8(len(v)))
			WriteMe(outFile, v)
		}
	}
	if usageTypeEnabled {
		for _, v := range usageTypeSorted {
			WriteMe(outFile, uint8(len(v)))
			WriteMe(outFile, v)
		}
	}
	if addressTypeEnabled {
		for _, v := range addressTypeSorted {
			WriteMe(outFile, uint8(len(v)))
			WriteMe(outFile, v)
		}
	}
	if categoryEnabled {
		for _, v := range categorySorted {
			WriteMe(outFile, uint8(len(v)))
			WriteMe(outFile, v)
		}
	}
	if districtEnabled {
		for _, v := range districtSorted {
			WriteMe(outFile, uint8(len(v)))
			WriteMe(outFile, v)
		}
	}
	if asnEnabled {
		for _, v := range asnSorted {
			WriteMe(outFile, uint8(len(v)))
			WriteMe(outFile, v)
		}
	}
	if asEnabled {
		for _, v := range asSorted {
			WriteMe(outFile, uint8(len(v)))
			WriteMe(outFile, v)
		}
	}

	if err = outFile.Sync(); err != nil {
		fmt.Println("Error flushing to disk.")
		return
	}
	var fi os.FileInfo
	if fi, err = outFile.Stat(); err != nil {
		fmt.Println("Unable to get file size.")
		return
	}
	fileSize := fi.Size() // int64
	dbFileSize = uint32(fileSize)

	if _, err = outFile.Seek(31, os.SEEK_SET); err != nil {
		fmt.Println("Unable to seek.")
		return
	}

	WriteMe(outFile, dbFileSize)
}

func WriteMe(out *os.File, data any) {
	if str, ok := data.(string); ok { // check that is string type
		if err := binary.Write(out, binary.LittleEndian, []byte(str)); err != nil {
			fmt.Printf("Write failed %v\n", err)
		}
	} else if err := binary.Write(out, binary.LittleEndian, data); err != nil {
		fmt.Printf("Write failed %v\n", err)
	}
}

func Tell(out *os.File) uint32 {
	if pos, err := out.Seek(0, os.SEEK_CUR); err != nil {
		fmt.Println("Tell failed.")
		return 0
	} else {
		return uint32(pos)
	}
}