Merge pull request #6 from amanica/feature/support_https

Feature/support https
thank you @amanica

.github/workflows/ci.yml

@@ -31,7 +31,8 @@ jobs:
       # Label used to access the service container
       influxdb:
         # Docker Hub image
-        image: influxdb:2.4 #currently we are using 2.4 for testing purposes
+        image: influxdb:2.4 #currently we are using 2.4 for testing purposes, 
+        #WARN: github CI seems to fail on 2.6. Unclear if the API changed or maybe HTTPS is required for newer versions (or https is disabled?)
         # Provide the password 
         env:
           DOCKER_INFLUXDB_INIT_MODE: setup
@@ -67,5 +68,4 @@ jobs:
       - uses: julia-actions/julia-processcoverage@v1
       - uses: codecov/codecov-action@v3
         with:
-          files: lcov.info
-
+          files: lcov.info

Project.toml

@@ -1,7 +1,7 @@
 name = "InfluxDBClient"
 uuid = "7a087161-7d97-4627-bbdd-0c82161d9408"
-authors = ["bernhard.koenig"]
-version = "0.0.5"
+authors = ["bernhard.koenig", "marius.kruger"]
+version = "0.0.6"
 
 [deps]
 CSV = "336ed68f-0bac-5ca0-87d4-7b16caf5d00b"

README.md

@@ -19,7 +19,9 @@ There are three optoins to configure the database access (see function `get_sett
 1) environment variables
 * ENV["INFLUXDB_ORG"] the organization
 * ENV["INFLUXDB_TOKEN"] the token to access the InfluxDB
-* ENV["INFLUXDB_HOST"] should include the port, e.g. "10.14.15.10:8086"
+* ENV["INFLUXDB_URL"] should include protocol and the port, e.g. "http://10.14.15.10:8086"
+  or with https e.g. `ENV["INFLUXDB_URL"]="https://us-east-1-1.aws.cloud2.influxdata.com:443"`
+
 2) keyword argumetns to `get_settings`
 3) provide a space delimited file to `get_settings`
 
@@ -52,7 +54,7 @@ using DataFrames
 a_random_bucket_name = "test_InfluxDBClient.jl_asdfeafdfasefsIyxdFDYfadsfasdfa____l"
 
 #isettings should return a NamedTuple similar to 
-#(INFLUXDB_HOST = "10.14.15.10:8086", INFLUXDB_ORG = "bk", INFLUXDB_TOKEN = "5Ixxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx==")
+#(INFLUXDB_URL = "http://10.14.15.10:8086", INFLUXDB_ORG = "bk", INFLUXDB_TOKEN = "5Ixxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx==")
 isettings = get_settings()
 
 #check if the InfluxDB is reachable
@@ -109,7 +111,7 @@ delete_bucket(isettings,a_random_bucket_name)
 First make sure you have the following environment variables defined:
 
 ```Julia
-ENV["INFLUXDB_HOST"]="localhost:8086"
+ENV["INFLUXDB_URL"]="http://localhost:8086"
 ENV["INFLUXDB_ORG"]="<some org>"
 ENV["INFLUXDB_TOKEN"]="5Ixxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx=="
 ENV["INFLUXDB_USER"]="<admin user>"
@@ -135,4 +137,4 @@ For more information, see:
     <https://discourse.julialang.org/t/activating-test-dependencies/48121/10>
     <https://github.com/JuliaTesting/TestEnv.jl>
 
-also had to modify runtents.jl line 47 to not add "test/"
+also had to modify runtents.jl line 55 to not add "test/"

config_example.txt

@@ -1,4 +1,4 @@
-INFLUXDB_HOST 10.14.15.10:8086
+INFLUXDB_URL http://10.14.15.10:8086
 INFLUXDB_ORG bk
 INFLUXDB_TOKEN 5xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxoA==
 INFLUXDB_USER bernhard

config_host_example.txt

@@ -0,0 +1,5 @@
+INFLUXDB_HOST 10.14.15.10:8086
+INFLUXDB_ORG bk
+INFLUXDB_TOKEN 5xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxoA==
+INFLUXDB_USER bernhard
+INFLUXDB_PASSWORD aaaaaaaaaaaaaaaaaaaa9

dev/readme_code.jl

@@ -5,7 +5,7 @@ using DataFrames
 a_random_bucket_name = "test_InfluxDBClient.jl_asdfeafdfasefsIyxdFDYfadsfasdfa____l"
 
 #isettings should return a NamedTuple similar to 
-#(INFLUXDB_HOST = "10.14.15.10:8086", INFLUXDB_ORG = "bk", INFLUXDB_TOKEN = "5Ixxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx==")
+#(INFLUXDB_URL = "http://10.14.15.10:8086", INFLUXDB_ORG = "bk", INFLUXDB_TOKEN = "5Ixxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx==")
 isettings = get_settings()
 
 #check if the InfluxDB is reachable

src/buckets.jl

@@ -10,8 +10,8 @@ end
 export create_bucket
 function create_bucket(isettings,bucket,content::String="")
     #https://docs.influxdata.com/influxdb/cloud/api/#operation/PostBuckets
-    @unpack INFLUXDB_HOST,INFLUXDB_TOKEN,INFLUXDB_ORG = isettings
-    
+    @unpack INFLUXDB_URL,INFLUXDB_TOKEN,INFLUXDB_ORG = isettings
+
     buckets,_ = get_buckets(isettings;limit=100,offset=0)
     if in(bucket,buckets)
         throw(ArgumentError("Bucket $bucket already exists"))
@@ -24,7 +24,7 @@ function create_bucket(isettings,bucket,content::String="")
         content = """{"name": "$bucket", "orgID": "$ORG_ID"}"""
     end
 
-    r = HTTP.request("POST", """http://$(INFLUXDB_HOST)/api/v2/buckets""", ["Authorization" => "Token $(INFLUXDB_TOKEN)", "Accept"=>"application/json","Content-Type"=>"application/json"],body=content)
+    r = HTTP.request("POST", """$(INFLUXDB_URL)/api/v2/buckets""", ["Authorization" => "Token $(INFLUXDB_TOKEN)", "Accept"=>"application/json","Content-Type"=>"application/json"],body=content)
     if !in(r.status,[201])
         @warn "Unexpected status" r.status
     end
@@ -36,7 +36,7 @@ end
 
 export delete_bucket
 function delete_bucket(isettings,bucket)
-    @unpack INFLUXDB_HOST,INFLUXDB_TOKEN = isettings
+    @unpack INFLUXDB_URL,INFLUXDB_TOKEN = isettings
     buckets,json = get_buckets(isettings;limit=100,offset=0);
     if !in(bucket,buckets)
         msg = "Unable to delete bucket $bucket as it does not exist."
@@ -45,7 +45,7 @@ function delete_bucket(isettings,bucket)
 
     BUCKET_ID = get_bucketid(json,bucket)
 
-    r = HTTP.request("DELETE", """http://$(INFLUXDB_HOST)/api/v2/buckets/$BUCKET_ID""", ["Authorization" => "Token $(INFLUXDB_TOKEN)", "Accept"=>"application/json"])
+    r = HTTP.request("DELETE", """$(INFLUXDB_URL)/api/v2/buckets/$BUCKET_ID""", ["Authorization" => "Token $(INFLUXDB_TOKEN)", "Accept"=>"application/json"])
     #if !in(r.status,[200,204])
     if !in(r.status,[204]) #api shows that 204 should be returned
         @warn "Unexpected status" r.status
@@ -70,10 +70,10 @@ end
 
 export get_buckets
 function get_buckets(isettings;limit=100,offset=0)
-    @unpack INFLUXDB_HOST,INFLUXDB_TOKEN = isettings
+    @unpack INFLUXDB_URL,INFLUXDB_TOKEN = isettings
 
-    r = HTTP.request("GET", """http://$(INFLUXDB_HOST)/api/v2/buckets?limit=$limit&offset=$offset""", ["Authorization" => "Token $(INFLUXDB_TOKEN)", "Content-Type"=>"text/plain; charset=utf-8","Accept"=>"application/json"] )
-    if r.status != 200 
+    r = HTTP.request("GET", """$(INFLUXDB_URL)/api/v2/buckets?limit=$limit&offset=$offset""", ["Authorization" => "Token $(INFLUXDB_TOKEN)", "Content-Type"=>"text/plain; charset=utf-8","Accept"=>"application/json"] )
+    if r.status != 200
         @warn "Unexpected status" r.status
     end
 
@@ -85,9 +85,9 @@ end
 
 export get_organizations
 function get_organizations(isettings;limit=100,offset=0)
-    @unpack INFLUXDB_HOST,INFLUXDB_TOKEN = isettings
+    @unpack INFLUXDB_URL,INFLUXDB_TOKEN = isettings
 
-    r = HTTP.request("GET", """http://$(INFLUXDB_HOST)/api/v2/orgs?limit=$limit&offset=$offset""", ["Authorization" => "Token $(INFLUXDB_TOKEN)", "Content-Type"=>"text/plain; charset=utf-8","Accept"=>"application/json"] )
+    r = HTTP.request("GET", """$(INFLUXDB_URL)/api/v2/orgs?limit=$limit&offset=$offset""", ["Authorization" => "Token $(INFLUXDB_TOKEN)", "Content-Type"=>"text/plain; charset=utf-8","Accept"=>"application/json"] )
     if r.status != 200
         @warn "Unexpected status" r.status
     end
@@ -114,8 +114,8 @@ end
 #=
 export get_buckets_curl
 function get_buckets_curl(isettings;limit=100,offset=0)
-    @unpack INFLUXDB_HOST,INFLUXDB_TOKEN = isettings
-    cmd = `curl -s --request GET "http://$(INFLUXDB_HOST)/api/v2/buckets?limit=$limit&offset=$offset" --header "Authorization: Token $(INFLUXDB_TOKEN)" --header "Content-Type: text/plain; charset=utf-8" --header "Accept: application/json"`
+    @unpack INFLUXDB_URL,INFLUXDB_TOKEN = isettings
+    cmd = `curl -s --request GET "$(INFLUXDB_URL)/api/v2/buckets?limit=$limit&offset=$offset" --header "Authorization: Token $(INFLUXDB_TOKEN)" --header "Content-Type: text/plain; charset=utf-8" --header "Accept: application/json"`
     rs = read(cmd, String)
     json = JSON3.read(rs)
     bucket_names = map(x->x.name,json.buckets)

src/delete.jl

@@ -9,12 +9,12 @@ function delete(isettings,bucket::String;measurement::String="",start::Union{Dat
     #=
          bucket = a_random_bucket_name
     =#
-    @unpack INFLUXDB_HOST,INFLUXDB_TOKEN,INFLUXDB_ORG = isettings
+    @unpack INFLUXDB_URL,INFLUXDB_TOKEN,INFLUXDB_ORG = isettings
     organization_names,jsonORG = get_organizations(isettings)
     ORG_ID = get_orgid(jsonORG,INFLUXDB_ORG)
 
     hdrs = Dict("Authorization" => "Token $(INFLUXDB_TOKEN)", "Content-Type"=>"application/json")
-    url = """http://$(INFLUXDB_HOST)/api/v2/delete?org=$INFLUXDB_ORG&bucket=$bucket"""
+    url = """$(INFLUXDB_URL)/api/v2/delete?org=$INFLUXDB_ORG&bucket=$bucket"""
 
     if isa(start,DateTime) 
         start = string(start,"Z")

src/query.jl

@@ -3,8 +3,6 @@
 
 export query_flux
 
-
-
 """
     query_flux(isettings,bucket,measurement;parse_datetime=false,datetime_precision="ns",tzstr = "UTC",range=Dict{String,Any}(),fields::Vector{String}=String[],tags=Dict{String,Any}(),aggregate::String="")
     
@@ -52,6 +50,10 @@ function query_flux_postprocess_response(bdy,parse_datetime,datetime_precision,t
         return df 
     end
 
+    # influx sometimes returns extra header rows that ends up in our data :'(
+    # https://docs.influxdata.com/influxdb/v2.3/reference/syntax/annotated-csv/?t=Different+schema#csv-response-format
+    filter!(:_time => !=("_time"), df)
+
     DataFrames.select!(df,Not(:Column1)) #unclear what this could/would be (let us drop it for now)
 
     if parse_datetime
@@ -118,7 +120,7 @@ function query_flux_http_response(isettings,bucket,measurement;range=Dict{String
         measurement = "my_meas"
     =#
 
-    @unpack INFLUXDB_HOST,INFLUXDB_TOKEN,INFLUXDB_ORG = isettings
+    @unpack INFLUXDB_URL,INFLUXDB_TOKEN,INFLUXDB_ORG = isettings
 
     #@assert length(range) > 0 #I think this may be necessary for any query...
     rngstr = ""
@@ -197,10 +199,10 @@ function query_flux_http_response(isettings,bucket,measurement;range=Dict{String
 end 
 
 function query_flux(isettings,q::String)
-    @unpack INFLUXDB_HOST,INFLUXDB_TOKEN,INFLUXDB_ORG = isettings
+    @unpack INFLUXDB_URL,INFLUXDB_TOKEN,INFLUXDB_ORG = isettings
 
     hdrs = Dict("Authorization" => "Token $(INFLUXDB_TOKEN)", "Accept"=>"application/json","Content-Type"=>"application/vnd.flux; charset=utf-8","Content-Encoding" => "identity")    
-    url = """http://$(INFLUXDB_HOST)/api/v2/query?org=$INFLUXDB_ORG"""
+    url = """$(INFLUXDB_URL)/api/v2/query?org=$INFLUXDB_ORG"""
     bdy = q
     #@show q
 

src/settings.jl

@@ -1,24 +1,25 @@
 export get_settings
-function get_settings(;org::String="",token::String="",host::String="",user::String="",password::String="",file::String="") #,bucket::String="")
+function get_settings(;org::String="",token::String="",url::String="",host::String="",user::String="",password::String="",file::String="") #,bucket::String="")
 
     #maybe add an option to use something along ~/.influxdbconfig ? 
     if length(file)>0
         settings_from_file = get_settings_from_file(;file=file)
-        return settings_from_file
+        return _update_setting_keys(settings_from_file)
     else 
         settings_from_file = Dict{String,String}()
     end
 
     envsettings = Dict{String,String}()
-    for k in ["INFLUXDB_PASSWORD","INFLUXDB_USER","INFLUXDB_HOST","INFLUXDB_TOKEN","INFLUXDB_ORG"]
+    for k in ["INFLUXDB_PASSWORD","INFLUXDB_USER","INFLUXDB_URL","INFLUXDB_HOST","INFLUXDB_TOKEN","INFLUXDB_ORG"]
         if haskey(ENV,k)
-         envsettings[k] = ENV[k]
+            envsettings[k] = ENV[k]
         end
     end
 
     #kwargs should 'overwrite' environment variables
     kwsettings = envsettings 
     if length(org) > 0; kwsettings["INFLUXDB_ORG"] = org; end;
+    if length(url) > 0; kwsettings["INFLUXDB_URL"] = url; end;
     if length(host) > 0; kwsettings["INFLUXDB_HOST"] = host; end;
     if length(token) > 0 ; kwsettings["INFLUXDB_TOKEN"] = token; end;
     if length(user) > 0 ; kwsettings["INFLUXDB_USER"] = user; end;
@@ -32,16 +33,22 @@ function get_settings(;org::String="",token::String="",host::String="",user::Str
 
     #isettings=Dict{String,String}("INFLUXDB_HOST"=>host,"INFLUXDB_ORG"=>org,"INFLUXDB_TOKEN"=>token,"INFLUXDB_USER"=>user,"INFLUXDB_PASSWORD"=>password)
 
+    return _update_setting_keys(isettings)
+end
+
+function _update_setting_keys(isettings::Dict{String,String})
+    if !haskey(isettings, "INFLUXDB_URL")
+        # support INFLUXDB_HOST for backward compatibility
+        isettings["INFLUXDB_URL"] = "http://" * isettings["INFLUXDB_HOST"]
+    end
+    delete!(isettings, "INFLUXDB_HOST")
+
+    isettings["INFLUXDB_ORG"] = replace(isettings["INFLUXDB_ORG"], " " => "%20") 
     return isettings
 end
 
 export get_settings_from_file
 function get_settings_from_file(;file="")
-    #=
-    
-    fi = raw"
-    =#
-
     isettings = Dict{String,String}()
 
     @show file

src/write.jl

@@ -44,7 +44,7 @@ function write_data(isettings,bucket::String,payload::Union{String,Vector{UInt8}
                 airSensors,sensor_id=TLM0202 temperature=72.30007505999716,humidity=30.651929918691714,co=0.6141876544505826 1630424259000000000"""
     =#
 
-    @unpack INFLUXDB_HOST,INFLUXDB_TOKEN,INFLUXDB_ORG = isettings
+    @unpack INFLUXDB_URL,INFLUXDB_TOKEN,INFLUXDB_ORG = isettings
 
     if !bucket_exists(isettings,bucket)
         throw(ArgumentError("Bucket $bucket does  not exist"))
@@ -61,7 +61,7 @@ function write_data(isettings,bucket::String,payload::Union{String,Vector{UInt8}
 
     #To send a line protocol payload, pass Content-Type: text/plain; charset=utf-8.    
     hdrs = Dict("Authorization" => "Token $(INFLUXDB_TOKEN)", "Accept"=>"application/json","Content-Type"=>"application/json; charset=utf-8")
-    url = """http://$(INFLUXDB_HOST)/api/v2/write?org=$INFLUXDB_ORG&bucket=$bucket&precision=$influx_precision"""
+    url = """$(INFLUXDB_URL)/api/v2/write?org=$INFLUXDB_ORG&bucket=$bucket&precision=$influx_precision"""
     #@show typeof(payload)
     if compress
         #gzip

test/delete.jl

@@ -18,13 +18,13 @@ curl --request POST http://localhost:8086/api/v2/delete?org=example-org&bucket=e
 
     #buckets,_ = get_buckets(isettings;limit=100,offset=0)
     bucket = a_random_bucket_name
-    @unpack INFLUXDB_HOST,INFLUXDB_TOKEN,INFLUXDB_ORG = isettings
+    @unpack INFLUXDB_URL,INFLUXDB_TOKEN,INFLUXDB_ORG = isettings
     organization_names,jsonORG = get_organizations(isettings)
     ORG_ID = get_orgid(jsonORG,INFLUXDB_ORG)
     gzip_compression_is_enabled = false
 
     hdrs = Dict("Authorization" => "Token $(INFLUXDB_TOKEN)", "Content-Type"=>"application/json")
-    url = """http://$(INFLUXDB_HOST)/api/v2/delete?org=$INFLUXDB_ORG&bucket=$bucket"""
+    url = """$(INFLUXDB_URL)/api/v2/delete?org=$INFLUXDB_ORG&bucket=$bucket"""
     content = """{"name": "$bucket", "orgID": "$ORG_ID"}"""
 
     payload_to_write = """airSensors,sensor_id=TLM0201 temperature=73.97038159354763,humidity=35.23103248356096,co=0.48445310567793615 1630424257000000000

test/large_data.jl

@@ -20,7 +20,7 @@
 
     lp = lineprotocol("my_meas",df,["temperature","an_int_value","abool","humidity"],tags=["color","sensor_id"], :datetime);
     @time write_data(isettings,a_random_bucket_name,lp,"ns")
-    @test length(findall('\n',lp)) == nn - 1 #only works if data has no \n
+    @test length(findall("\n",lp)) == nn - 1 #only works if data has no \n
 
     ########################################################################
     #batch processing data

test/metadata.jl

@@ -33,7 +33,7 @@
     for measurement in measurements
         fields = query_measurement_field_keys(isettings, a_random_bucket_name, measurement)
         #on my local machine this vector is empty, not sure why
-        @warn("may need to improve this")
+        @warn("may need to improve this test")
         @show fields
         println("$measurement : $(join(fields, ", "))")
     end

test/runtests.jl

@@ -25,33 +25,37 @@ end
 @test length(a_random_bucket_name) > 0
 
 #smoketest 1 to see if DB is up
-#a get request to """http://$(INFLUXDB_HOST)/metrics""" is another possibility to check if the server is up
+#a get request to """$(INFLUXDB_URL)/metrics""" is another possibility to check if the server is up
 try 
-    metrics_url = """http://$(isettings["INFLUXDB_HOST"])/metrics"""
+    metrics_url = """$(isettings["INFLUXDB_URL"])/metrics"""
     @info("Trying to query $(metrics_url)...")
     r = HTTP.request("GET", metrics_url,status_exception = false)
     #maybe status is 200 when metrics are ENABLED and status is 403 when metrics are DISABLED
     @test in(r.status,[200,403])
     @info("Status is $(r.status)")
-    @info("Body is $(String(r.body))")
+    strbdy = String(r.body)
+    if length(strbdy) < 200
+        @info("Body is $(String(r.body))")
+    end
 catch er
-    @warn("failed to query: http://$(isettings["INFLUXDB_HOST"])/metrics")
+    @warn("failed to query: $(isettings["INFLUXDB_URL"])/metrics")
     @show er
 end
 
 #smoketest 2 to see if DB is up
 #https://docs.influxdata.com/influxdb/v2.4/write-data/developer-tools/api/
-bucket_names,json = try
-    try
-        get_buckets(isettings); #1.7 ms btime, (influxdb host is on a different machine)
-    catch er
-        @show er
-    end
-catch
-    "","";
-end;
-@test length(bucket_names) > 0
-@show bucket_names
+bucket_names = ""
+json = ""
+try
+    global bucket_names, json = get_buckets(isettings); #1.7 ms btime, (influxdb host is on a different machine)
+    @test length(bucket_names) > 0
+    @show bucket_names
+catch er
+    @warn("failed to get list of buckets. InfluxDB may not be reachable.")
+    @show er
+    global bucket_names = "" 
+    global json = ""
+end
 prefix = ifelse(isinteractive() , "test/", "")
 include(string(prefix,"functions.jl"))