Skip to content

Commit

Permalink
Reduced file I/O
Browse files Browse the repository at this point in the history
  • Loading branch information
@ip2location
ip2location committed Jun 28, 2022
1 parent 6ee4468 commit f0e78c9
Show file tree
Hide file tree
Showing 2 changed files with 158 additions and 154 deletions.
2 changes: 1 addition & 1 deletion LICENSE.TXT
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
MIT License

Copyright (c) 2021 IP2Location.com
Copyright (c) 2022 IP2Location.com

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
Expand Down
310 changes: 157 additions & 153 deletions ip2location.erl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@
-define(IF(Cond), (case (Cond) of true -> (0); false -> (1) end)).

apiversion() ->
"8.4.0".
"8.5.0".

getapiversion() ->
io:format("API Version: ~p~n", [apiversion()]).
Expand All @@ -36,42 +36,27 @@ round(Number, Precision) ->
P = math:pow(10, Precision),
round(Number * P) / P.

readuint(S, StartPos, Len) ->
case file:pread(S, StartPos - 1, Len) of
eof ->
ok;
{ok, Data} ->
binary:decode_unsigned(Data, little)
end.

readuintrow(R, StartPos, Len) ->
Data = binary:part(R, StartPos, Len),
binary:decode_unsigned(Data, little).

readuint8(S, StartPos) ->
readuint(S, StartPos, 1).

readuint32(S, StartPos) ->
readuint(S, StartPos, 4).
readuint8row(R, StartPos) ->
readuintrow(R, StartPos, 1).

readuint32row(R, StartPos) ->
readuintrow(R, StartPos, 4).

readuint128(S, StartPos) ->
readuint(S, StartPos, 16).
readuint128row(R, StartPos) ->
readuintrow(R, StartPos, 16).

readstr(S, StartPos) ->
case file:pread(S, StartPos, 1) of
case file:pread(S, StartPos, 256) of % max size of string field + 1 byte for the length
eof ->
ok;
{ok, LenRaw} ->
Len = binary:decode_unsigned(LenRaw, little),
case file:pread(S, StartPos + 1, Len) of
eof ->
ok;
{ok, Data} ->
binary_to_list(Data)
end
{ok, R} ->
Len = readuint8row(R, 0),
Data = binary:part(R, 1, Len),
binary_to_list(Data)
end.

readfloatrow(R, StartPos) ->
Expand All @@ -90,49 +75,55 @@ input(InputFile) ->

new(InputFile) ->
S = input(InputFile),
Databasetype = readuint8(S, 1),
Databasecolumn = readuint8(S, 2),
Databaseyear = readuint8(S, 3),
% Databasemonth = readuint8(S, 4),
% Databaseday = readuint8(S, 5),
Ipv4databasecount = readuint32(S, 6),
Ipv4databaseaddr = readuint32(S, 10),
Ipv6databasecount = readuint32(S, 14),
Ipv6databaseaddr = readuint32(S, 18),
Ipv4indexbaseaddr = readuint32(S, 22),
Ipv6indexbaseaddr = readuint32(S, 26),
Productcode = readuint8(S, 30),
Ipv4columnsize = Databasecolumn bsl 2, % 4 bytes each column
Ipv6columnsize = 16 + ((Databasecolumn - 1) bsl 2), % 4 bytes each column, except IPFrom column which is 16 bytes
% Producttype = readuint8(S, 31),
% Filesize = readuint32(S, 32),
file:close(S),

if
% check if is correct BIN (should be 1 for IP2Location BIN file), also checking for zipped file (PK being the first 2 chars)
(Productcode /= 1 andalso Databaseyear >= 21) orelse (Databasetype == 80 andalso Databasecolumn == 75) ->
io:format("Incorrect IP2Location BIN file format. Please make sure that you are using the latest IP2Location BIN file.~n", []),
halt();
true ->
case ets:info(mymeta) of
undefined ->
ets:new(mymeta, [set, named_table]),
ets:insert(mymeta, {inputfile, InputFile}),
ets:insert(mymeta, {databasetype, Databasetype}),
ets:insert(mymeta, {databasetype, Databasetype}),
ets:insert(mymeta, {databasecolumn, Databasecolumn}),
ets:insert(mymeta, {ipv4databasecount, Ipv4databasecount}),
ets:insert(mymeta, {ipv4databaseaddr, Ipv4databaseaddr}),
ets:insert(mymeta, {ipv6databasecount, Ipv6databasecount}),
ets:insert(mymeta, {ipv6databaseaddr, Ipv6databaseaddr}),
ets:insert(mymeta, {ipv4indexbaseaddr, Ipv4indexbaseaddr}),
ets:insert(mymeta, {ipv6indexbaseaddr, Ipv6indexbaseaddr}),
ets:insert(mymeta, {ipv4columnsize, Ipv4columnsize}),
ets:insert(mymeta, {ipv6columnsize, Ipv6columnsize});
_ ->
ok % do nothing
end
end.
case file:pread(S, 0, 64) of % 64-byte header
eof ->
halt();
{ok, Data} ->
R = Data,
Databasetype = readuint8row(R, 0),
Databasecolumn = readuint8row(R, 1),
Databaseyear = readuint8row(R, 2),
% Databasemonth = readuint8row(R, 3),
% Databaseday = readuint8row(R, 4),
Ipv4databasecount = readuint32row(R, 5),
Ipv4databaseaddr = readuint32row(R, 9),
Ipv6databasecount = readuint32row(R, 13),
Ipv6databaseaddr = readuint32row(R, 17),
Ipv4indexbaseaddr = readuint32row(R, 21),
Ipv6indexbaseaddr = readuint32row(R, 25),
Productcode = readuint8row(R, 29),
Ipv4columnsize = Databasecolumn bsl 2, % 4 bytes each column
Ipv6columnsize = 16 + ((Databasecolumn - 1) bsl 2), % 4 bytes each column, except IPFrom column which is 16 bytes
% Producttype = readuint8row(R, 30),
% Filesize = readuint32row(R, 31),

if
% check if is correct BIN (should be 1 for IP2Location BIN file), also checking for zipped file (PK being the first 2 chars)
(Productcode /= 1 andalso Databaseyear >= 21) orelse (Databasetype == 80 andalso Databasecolumn == 75) ->
io:format("Incorrect IP2Location BIN file format. Please make sure that you are using the latest IP2Location BIN file.~n", []),
halt();
true ->
case ets:info(mymeta) of
undefined ->
ets:new(mymeta, [set, named_table]),
ets:insert(mymeta, {inputfile, InputFile}),
ets:insert(mymeta, {databasetype, Databasetype}),
ets:insert(mymeta, {databasetype, Databasetype}),
ets:insert(mymeta, {databasecolumn, Databasecolumn}),
ets:insert(mymeta, {ipv4databasecount, Ipv4databasecount}),
ets:insert(mymeta, {ipv4databaseaddr, Ipv4databaseaddr}),
ets:insert(mymeta, {ipv6databasecount, Ipv6databasecount}),
ets:insert(mymeta, {ipv6databaseaddr, Ipv6databaseaddr}),
ets:insert(mymeta, {ipv4indexbaseaddr, Ipv4indexbaseaddr}),
ets:insert(mymeta, {ipv6indexbaseaddr, Ipv6indexbaseaddr}),
ets:insert(mymeta, {ipv4columnsize, Ipv4columnsize}),
ets:insert(mymeta, {ipv6columnsize, Ipv6columnsize});
_ ->
ok % do nothing
end
end
end,
file:close(S).

readcolcountryrow(S, R, Dbtype, Col) ->
X = "This parameter is unavailable for selected data file. Please upgrade the data file.",
Expand Down Expand Up @@ -180,7 +171,7 @@ readcolfloatstringrow(S, R, Dbtype, Col) ->
end
end.

readrecord(S, Dbtype, Rowoffset) ->
readrecord(S, R, Dbtype) ->
Country_position = [0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2],
Region_position = [0, 0, 0, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3],
City_position = [0, 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4],
Expand All @@ -203,94 +194,95 @@ readrecord(S, Dbtype, Rowoffset) ->
Addresstype_position = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21],
Category_position = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22],

Cols = ?IF(lists:nth(Dbtype, Country_position) == 0) + ?IF(lists:nth(Dbtype, Region_position) == 0) + ?IF(lists:nth(Dbtype, City_position) == 0) + ?IF(lists:nth(Dbtype, Isp_position) == 0) + ?IF(lists:nth(Dbtype, Latitude_position) == 0) + ?IF(lists:nth(Dbtype, Longitude_position) == 0) + ?IF(lists:nth(Dbtype, Domain_position) == 0) + ?IF(lists:nth(Dbtype, Zipcode_position) == 0) + ?IF(lists:nth(Dbtype, Timezone_position) == 0) + ?IF(lists:nth(Dbtype, Netspeed_position) == 0) + ?IF(lists:nth(Dbtype, Iddcode_position) == 0) + ?IF(lists:nth(Dbtype, Areacode_position) == 0) + ?IF(lists:nth(Dbtype, Weatherstationcode_position) == 0) + ?IF(lists:nth(Dbtype, Weatherstationname_position) == 0) + ?IF(lists:nth(Dbtype, Mcc_position) == 0) + ?IF(lists:nth(Dbtype, Mnc_position) == 0) + ?IF(lists:nth(Dbtype, Mobilebrand_position) == 0) + ?IF(lists:nth(Dbtype, Elevation_position) == 0) + ?IF(lists:nth(Dbtype, Usagetype_position) == 0) + ?IF(lists:nth(Dbtype, Addresstype_position) == 0) + ?IF(lists:nth(Dbtype, Category_position) == 0),
Rowlength = Cols bsl 2,
{Country_short, Country_long} = readcolcountryrow(S, R, Dbtype, Country_position),
Region = readcolstringrow(S, R, Dbtype, Region_position),
City = readcolstringrow(S, R, Dbtype, City_position),
Isp = readcolstringrow(S, R, Dbtype, Isp_position),
Latitude = readcolfloatrow(R, Dbtype, Latitude_position),
Longitude = readcolfloatrow(R, Dbtype, Longitude_position),
Domain = readcolstringrow(S, R, Dbtype, Domain_position),
Zipcode = readcolstringrow(S, R, Dbtype, Zipcode_position),
Timezone = readcolstringrow(S, R, Dbtype, Timezone_position),
Netspeed = readcolstringrow(S, R, Dbtype, Netspeed_position),
Iddcode = readcolstringrow(S, R, Dbtype, Iddcode_position),
Areacode = readcolstringrow(S, R, Dbtype, Areacode_position),
Weatherstationcode = readcolstringrow(S, R, Dbtype, Weatherstationcode_position),
Weatherstationname = readcolstringrow(S, R, Dbtype, Weatherstationname_position),
Mcc = readcolstringrow(S, R, Dbtype, Mcc_position),
Mnc = readcolstringrow(S, R, Dbtype, Mnc_position),
Mobilebrand = readcolstringrow(S, R, Dbtype, Mobilebrand_position),
Elevation = readcolfloatstringrow(S, R, Dbtype, Elevation_position),
Usagetype = readcolstringrow(S, R, Dbtype, Usagetype_position),
Addresstype = readcolstringrow(S, R, Dbtype, Addresstype_position),
Category = readcolstringrow(S, R, Dbtype, Category_position),

case file:pread(S, Rowoffset - 1, Rowlength) of
eof ->
#ip2locationrecord{};
{ok, Data} ->
R = Data,

{Country_short, Country_long} = readcolcountryrow(S, R, Dbtype, Country_position),
Region = readcolstringrow(S, R, Dbtype, Region_position),
City = readcolstringrow(S, R, Dbtype, City_position),
Isp = readcolstringrow(S, R, Dbtype, Isp_position),
Latitude = readcolfloatrow(R, Dbtype, Latitude_position),
Longitude = readcolfloatrow(R, Dbtype, Longitude_position),
Domain = readcolstringrow(S, R, Dbtype, Domain_position),
Zipcode = readcolstringrow(S, R, Dbtype, Zipcode_position),
Timezone = readcolstringrow(S, R, Dbtype, Timezone_position),
Netspeed = readcolstringrow(S, R, Dbtype, Netspeed_position),
Iddcode = readcolstringrow(S, R, Dbtype, Iddcode_position),
Areacode = readcolstringrow(S, R, Dbtype, Areacode_position),
Weatherstationcode = readcolstringrow(S, R, Dbtype, Weatherstationcode_position),
Weatherstationname = readcolstringrow(S, R, Dbtype, Weatherstationname_position),
Mcc = readcolstringrow(S, R, Dbtype, Mcc_position),
Mnc = readcolstringrow(S, R, Dbtype, Mnc_position),
Mobilebrand = readcolstringrow(S, R, Dbtype, Mobilebrand_position),
Elevation = readcolfloatstringrow(S, R, Dbtype, Elevation_position),
Usagetype = readcolstringrow(S, R, Dbtype, Usagetype_position),
Addresstype = readcolstringrow(S, R, Dbtype, Addresstype_position),
Category = readcolstringrow(S, R, Dbtype, Category_position),

#ip2locationrecord{
country_short = Country_short,
country_long = Country_long,
region = Region,
city = City,
isp = Isp,
latitude = Latitude,
longitude = Longitude,
domain = Domain,
zipcode = Zipcode,
timezone = Timezone,
netspeed = Netspeed,
iddcode = Iddcode,
areacode = Areacode,
weatherstationcode = Weatherstationcode,
weatherstationname = Weatherstationname,
mcc = Mcc,
mnc = Mnc,
mobilebrand = Mobilebrand,
elevation = Elevation,
usagetype = Usagetype,
addresstype = Addresstype,
category = Category
}
end.
#ip2locationrecord{
country_short = Country_short,
country_long = Country_long,
region = Region,
city = City,
isp = Isp,
latitude = Latitude,
longitude = Longitude,
domain = Domain,
zipcode = Zipcode,
timezone = Timezone,
netspeed = Netspeed,
iddcode = Iddcode,
areacode = Areacode,
weatherstationcode = Weatherstationcode,
weatherstationname = Weatherstationname,
mcc = Mcc,
mnc = Mnc,
mobilebrand = Mobilebrand,
elevation = Elevation,
usagetype = Usagetype,
addresstype = Addresstype,
category = Category
}.

searchtree(S, Ipnum, Dbtype, Low, High, BaseAddr, Colsize, Iptype) ->
if
Low =< High ->
Mid = ((Low + High) bsr 1),
Rowoffset = BaseAddr + (Mid * Colsize),
Rowoffset2 = Rowoffset + Colsize,

if
Iptype == ipv4 ->
Ipfrom = readuint32(S, Rowoffset),
Ipto = readuint32(S, Rowoffset2);
true ->
Ipfrom = readuint128(S, Rowoffset),
Ipto = readuint128(S, Rowoffset2)
case Iptype of
ipv4 ->
Firstcol = 4; % 4 bytes
ipv6 ->
Firstcol = 16 % 16 bytes
end,

if
Ipnum >= Ipfrom andalso Ipnum < Ipto ->
if
Iptype == ipv4 ->
readrecord(S, Dbtype + 1, Rowoffset + 4);
true ->
readrecord(S, Dbtype + 1, Rowoffset + 16)
end;
true ->
if
Ipnum < Ipfrom ->
searchtree(S, Ipnum, Dbtype, Low, Mid - 1, BaseAddr, Colsize, Iptype);
true ->
searchtree(S, Ipnum, Dbtype, Mid + 1, High, BaseAddr, Colsize, Iptype)
end
Readlen = Colsize + Firstcol,
case file:pread(S, Rowoffset - 1, Readlen) of % reading IP From + whole row + next IP From
eof ->
io:format("Error: IP address not found.~n", []),
{}; % return empty
{ok, R} ->
if
Iptype == ipv4 ->
Ipfrom = readuint32row(R, 0),
Ipto = readuint32row(R, Colsize);
true ->
Ipfrom = readuint128row(R, 0),
Ipto = readuint128row(R, Colsize)
end,

if
Ipnum >= Ipfrom andalso Ipnum < Ipto ->
Rowlen = Colsize - Firstcol,
R2 = binary:part(R, Firstcol, Rowlen),

readrecord(S, R2, Dbtype + 1);
true ->
if
Ipnum < Ipfrom ->
searchtree(S, Ipnum, Dbtype, Low, Mid - 1, BaseAddr, Colsize, Iptype);
true ->
searchtree(S, Ipnum, Dbtype, Mid + 1, High, BaseAddr, Colsize, Iptype)
end
end
end;
true ->
io:format("Error: IP address not found.~n", []),
Expand All @@ -301,9 +293,15 @@ search4(S, Ipnum, Dbtype, Low, High, Baseaddr, Indexbaseaddr, Colsize) ->
if
Indexbaseaddr > 0 ->
Indexpos = ((Ipnum bsr 16) bsl 3) + Indexbaseaddr,
Low2 = readuint32(S, Indexpos),
High2 = readuint32(S, Indexpos + 4),
searchtree(S, Ipnum, Dbtype, Low2, High2, Baseaddr, Colsize, ipv4);
case file:pread(S, Indexpos - 1, 8) of % 4 bytes for each IP From & IP To
eof ->
io:format("Error: IP address not found.~n", []),
{}; % return empty
{ok, R} ->
Low2 = readuint32row(R, 0),
High2 = readuint32row(R, 4),
searchtree(S, Ipnum, Dbtype, Low2, High2, Baseaddr, Colsize, ipv4)
end;
true ->
searchtree(S, Ipnum, Dbtype, Low, High, Baseaddr, Colsize, ipv4)
end.
Expand All @@ -312,9 +310,15 @@ search6(S, Ipnum, Dbtype, Low, High, Baseaddr, Indexbaseaddr, Colsize) ->
if
Indexbaseaddr > 0 ->
Indexpos = ((Ipnum bsr 112) bsl 3) + Indexbaseaddr,
Low2 = readuint32(S, Indexpos),
High2 = readuint32(S, Indexpos + 4),
searchtree(S, Ipnum, Dbtype, Low2, High2, Baseaddr, Colsize, ipv6);
case file:pread(S, Indexpos - 1, 8) of % 4 bytes for each IP From & IP To
eof ->
io:format("Error: IP address not found.~n", []),
{}; % return empty
{ok, R} ->
Low2 = readuint32row(R, 0),
High2 = readuint32row(R, 4),
searchtree(S, Ipnum, Dbtype, Low2, High2, Baseaddr, Colsize, ipv6)
end;
true ->
searchtree(S, Ipnum, Dbtype, Low, High, Baseaddr, Colsize, ipv6)
end.
Expand Down

0 comments on commit f0e78c9

Please sign in to comment.