Feature/roll ur own tls

README.md

@@ -10,7 +10,7 @@ Figure 1 - Accurate depiction of a man learning Rust ☠️
 
 ## Quick start docker ( Recommended)
 Alternatively we offer a Docker container, which can be found at https://www.github.com/looselab/icarust_docker. 
-This negates the need for any manual building, dependency management and is simple and easy to use.
+This negates the need for any manual building, dependency management and is simple(er) to use.
 #### Caveats
 MacOS runs docker volumes through virtualisation, rather than on the underlying OS. This results in very slow read/write for directories shared bewteen the host computer and the container. Whilst it is possible to run Icarust using docker on Mac, it may be better to run "natively", following the instructions below.
 
@@ -28,10 +28,23 @@ sudo apt install -y protobuf-compiler libprotobuf-dev
 
 #### TLS
 
-Minknow core 5.x requires a secure channel connection be made by the minknow API. IN order to do this it is neccessary to use the localhost certificates provided with an installed version of minknow. Therefore currently _MinKNOW MUST BE INSTALLED_. The certificates are read from:
+Minknow core 5.x requires a secure channel connection be made by the minknow API. IN order to do this, any programs connecting to Icarusts facsimile of the MinKNOW RPC will need to set the following environment variables:
+```python
+from minknow_api.manager import Manager
+import os         
+os.environ["MINKNOW_TRUSTED_CA"] = "/Path/to/Icarust/static/tls_certs/ca.crt"                  
+from minknow_api.manager import Manager                                                      
+m = Manager( port=9502)                                                                      
+pos = next(m.flow_cell_positions())                                                          
+con = pos.connect()                                                                          
+con.instance.get_version_info()    
+```
+
+Alternatively this can be exported on the command line.
 
-- **linux (ubuntu)**: `/opt/ont/minnow/conf/rpc-certs/`
-- **macOS**: `/Applications/MinKNOW.app/Contents/Resources/conf/rpc-certs`
+```bash
+export MINKNOW_TRUSTED_CA="/Path/to/icarust/static/tls_certs/ca.crt"
+```
 
 
 In order to run Icarust with and view the options - 

config.ini

@@ -1,6 +1,6 @@
 [TLS]
-cert-dir = /opt/ont/minknow/conf/rpc-certs/
-
+cert-dir = ./static/tls_certs
+; cert-dir = /opt/ont/minknow/conf/rpc-certs
 [PORTS]
 manager = 9502
 position = 10001

src/impl_services/data.rs

@@ -444,7 +444,7 @@ fn start_write_out_thread(
             let z = { *write_out_gracefully.lock().unwrap() };
 
             // this isn't perfect. if we are finsihing up a run and have more than 4000 reads waiting to be written out, we will lose the excess reads over 4000
-            if read_infos.len() >= 10 || z {
+            if read_infos.len() >= 4000 || z {
                 let extension = if x.pod5 { ".pod5" } else { ".fast5" };
                 let output_file_name = format!(
                     "{}/{}_pass_{}_{}{}",

src/main.rs

@@ -256,10 +256,10 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
             .get("TLS", "cert-dir")
             .expect("Tls cert dir not found in config.ini"),
     ); // Setup the TLS certifcates using the Minknow TLS certs
-    let cert = tokio::fs::read(format!("{}", tls_cert_path.join("localhost.crt").display()))
+    let cert = tokio::fs::read(format!("{}", tls_cert_path.join("server.crt").display()))
         .await
         .expect("No TLS certs found");
-    let key = tokio::fs::read(format!("{}", tls_cert_path.join("localhost.key").display())).await?;
+    let key = tokio::fs::read(format!("{}", tls_cert_path.join("server.key").display())).await?;
     let server_identity = Identity::from_pem(cert, key);
     let tls = ServerTlsConfig::new().identity(server_identity);
     let tls_position = tls.clone();

static/tls_certs/ca.crt

@@ -0,0 +1,19 @@
+-----BEGIN CERTIFICATE-----
+MIIDDTCCAfWgAwIBAgIUEKgtKA6S02Ozq5DAcn5Ok08JUtAwDQYJKoZIhvcNAQEL
+BQAwFjEUMBIGA1UEAwwLTG9jYWxob3N0Q0EwHhcNMjMxMjA4MTQwNTUwWhcNMzMx
+MjA1MTQwNTUwWjAWMRQwEgYDVQQDDAtMb2NhbGhvc3RDQTCCASIwDQYJKoZIhvcN
+AQEBBQADggEPADCCAQoCggEBAL08MBKQt4vyBhBjAXwu9REu3KB4AUBYvbFOONiC
+DeK8JzZPUSd8Lc1YIkm5+nkhJrY2GqTkdBuYBwUENvv/JLPDMR/k3Kllvr+dA5xB
+xhvUWmkXqf0oJy1yRoSMh4gwUdGdHwE1Osr87IMW7FVPuhtjW6wQXL7SdAiuEdXF
+n8OeZ4LAue6+Fu0tZykf12cwXUrRZtRqkCRrGlTmpb3bNJDGoqL0sqQQvFlJUNkm
+nuEWH8scIQUcfWPPzwqFv2DYXG6Z8AN7cErKL8GDDynyRElGUHCy53yqi6cSWYfR
+a3b22jpTnFRyWGBch4pQE9FqDnaEji9/Wa1VoOAlcAaNNpkCAwEAAaNTMFEwHQYD
+VR0OBBYEFJABIDvKQy5ttY4wWS43JVfFDtE3MB8GA1UdIwQYMBaAFJABIDvKQy5t
+tY4wWS43JVfFDtE3MA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZIhvcNAQELBQADggEB
+ACIGG3zxvvTGyQk8uG46EJ7DEy+1mwRLQneAN3VvYNYPMk5M8EruwicvEcyjL+9L
+1CB0Lvr1l04FnDOPJZYOuAm8rHSF0P3kH+Dn0ygS5l3mntH/c+GqGM5y0GDRdFFO
+vzI8WlJ3d62mDKqgqHXo3NAWpMOHlsBWAR76RuhQMHRhNtGiw5A7Nz6y2FKNe9Cb
+J9XKYgEwu41O3dKumLM4lvacw45SGiHJHTdOU8vJGPYDdX2SOSbIgfnZ+ZncXKXp
+jO3X2m7hYOH2uCCp3+lDTCwl38aViKpdAbXJ9e/Rn/XqNSjkf1qaf39AFxvIlc3j
+a4y3MmraUZebdbntpa0h1zQ=
+-----END CERTIFICATE-----

static/tls_certs/ca.key

@@ -0,0 +1,28 @@
+-----BEGIN PRIVATE KEY-----
+MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQC9PDASkLeL8gYQ
+YwF8LvURLtygeAFAWL2xTjjYgg3ivCc2T1EnfC3NWCJJufp5ISa2Nhqk5HQbmAcF
+BDb7/ySzwzEf5NypZb6/nQOcQcYb1FppF6n9KCctckaEjIeIMFHRnR8BNTrK/OyD
+FuxVT7obY1usEFy+0nQIrhHVxZ/DnmeCwLnuvhbtLWcpH9dnMF1K0WbUapAkaxpU
+5qW92zSQxqKi9LKkELxZSVDZJp7hFh/LHCEFHH1jz88Khb9g2FxumfADe3BKyi/B
+gw8p8kRJRlBwsud8qounElmH0Wt29to6U5xUclhgXIeKUBPRag52hI4vf1mtVaDg
+JXAGjTaZAgMBAAECggEAN7ZFFuopR5fJBkEHumZeuyDXTp0WDTozGsTDQqbDbMnl
+J32hkGTkXSzB53/MCKZkbceu/3XM4yTWiDkN8RHUXeItD81C09KDHomc+v94UlAq
+D/+qCcCtDBm1PifbbRKAvBwseSARP5rIDO4ORS2URqTxY85rZ7iw9a89y/iDSUVS
+a/NfooYUaQRCC1n2yo406gsH3qiTzezyPyUx0xz/fQncvPkf12JLqFDPfj0PA26O
+VgwBre1jwawtB4iveZkS/FGVjsVWd5uXVQZ74KH8k7B6D0ORcfAD/+PQNgRG0PTx
+VcbASq9gPI/FRGHaPRhLFDL8ksxqeRSDIflwmIxu8wKBgQD5S5gEX+y0l9LrBreR
+k3IKsQ83wB/2sCyR7Kcohe+gahTpVehn29W9z3+i1o3zxZUV+OKB5ZDuzClVQQF9
+3cTxsEmq46Vj3oDTK2bPIekAIiX0NUbhoOrIVhu9VemGzieVtPbjyZ0x8axjGpDL
+bggNjSlVfaxImqdhKzl42RC/lwKBgQDCUxPiR3WoDj3AjmHm95Lj4dnhuxPvhhRu
+XGFqd3Tf++09lCqeYJQrDQ40eyEMEE4PHSANE4X9FnQpInnVcK3XqzGCfaz3sUJ4
+P6iHFIQ+q7n0TpzLG4QH91+ZRYeHm40rSfQU5uP/gu2nunAfxNKj47W9LcLjrSx9
+yAiLP8GBTwKBgCFttZDAnCkLQskpOWxX7S7p1zSOmrqlnRRkO3X/n8ZO80bmsidb
+Hyo4rNDVHGvXzsoHebgZZHng/pnkAaMzHbma/gXgSMk/iN7ILx0PWwMdkSFSE5Ml
+1NXQVh/paUseUciNX53j5yRPV+JiKZBdzM6kYyqHk5aYxmENPsOpt8k5AoGAPDHZ
+7fUu2BEZReWVJCO6p8aVZJw7z62b9Je/QWihlqSGV9Xm4cqO+0r/BHjDwuXmGD5w
+aDyM+9oRv5BDZz+RSkZofsCJbOzG51Lo1Hchg/xUDP6xCSWL35/cSDFGi1Y/A5LT
+/gmLxOF81sEk7GJe0Y+/A0wCm95P8uKc6u2j9ncCgYEA2YtX2towh7d7WM6tU4wD
+JG7m/B8x5GVOmht7NgFaKfAF4QRLdbu39tbm5UG0nhIehX799CRUzDG0Usens7FL
+juk1wGMngVJ168NrOjt3gtixUfYvirGyO4+lG0Gyov6Nw1CnvLG23bm9hQeb8JG0
+0GATczzMSKYYmmJddjogEdI=
+-----END PRIVATE KEY-----

static/tls_certs/server.crt

@@ -0,0 +1,17 @@
+-----BEGIN CERTIFICATE-----
+MIICsTCCAZkCFBZ3N7bt9q+ys6LaRU6OcDzOwn1ZMA0GCSqGSIb3DQEBCwUAMBYx
+FDASBgNVBAMMC0xvY2FsaG9zdENBMB4XDTIzMTIwODE0MDYxMVoXDTI0MTIwNzE0
+MDYxMVowFDESMBAGA1UEAwwJbG9jYWxob3N0MIIBIjANBgkqhkiG9w0BAQEFAAOC
+AQ8AMIIBCgKCAQEAzhcleEe7IuE2MLel9V3jSe6TsxuCd1nAwGHWjiQOyz4MJKvK
+9z+dOA217JPYtybymCukgHL8wP2GAPvYf2R4g+dsJ2hJfp1su01Nu06FRIAbbYkm
+GTfJl77I3uIuOOhvUKwynxDsrFCsEU7Kk3OzpBlzMujwapdhiCNHikeDjfeU8iVp
+YZREGCpC1s/CWNG1e/PXZmK+jzVTimq1rocvZ1wRILVU3vSZPls5bl9LtR7n0tgR
+Lj5EaNCeobNWyBk4lJuiLjao+lH80iFQBkwDULCiWD2EdKIVxcKaf1JtXhU6fIjM
+v3N7ufvSHuqxQ0O5GxAuTuvwjEq1Q/mHhLObLQIDAQABMA0GCSqGSIb3DQEBCwUA
+A4IBAQAz8uCIrpF41Q/+Gt4b5N60XC0Zfp5an2GItX+gDVytaLwnfOdkAjfSMCHB
+DFKI4luGuuddj2dN6+LGzqCS9ftt80N7OwETpL2VgwhDvgB9JVZJu0xNoonb8k0v
+UgpH8IjqVUV1eOEhwHWCsftzu8mW3lvtcHGt6oYz7PGF72vf8+PXsXdM5eQmrO2S
+ws1Gs5k4QjDnsDtC5u43nD9V+3MgKgNXwsOM3lVGcWNrJEoIiW0BDyoL5k7XE7M7
+uZTsW9wygtyS+g68E0KntkoTyL2AJFkJKoqdVbPf/VuRXvqWtPr8VHcVblKhOJtE
+dGv79WdddlHk6xh2DxezVKmzrubr
+-----END CERTIFICATE-----

static/tls_certs/server.key

@@ -0,0 +1,28 @@
+-----BEGIN PRIVATE KEY-----
+MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQDOFyV4R7si4TYw
+t6X1XeNJ7pOzG4J3WcDAYdaOJA7LPgwkq8r3P504DbXsk9i3JvKYK6SAcvzA/YYA
++9h/ZHiD52wnaEl+nWy7TU27ToVEgBttiSYZN8mXvsje4i446G9QrDKfEOysUKwR
+TsqTc7OkGXMy6PBql2GII0eKR4ON95TyJWlhlEQYKkLWz8JY0bV789dmYr6PNVOK
+arWuhy9nXBEgtVTe9Jk+WzluX0u1HufS2BEuPkRo0J6hs1bIGTiUm6IuNqj6UfzS
+IVAGTANQsKJYPYR0ohXFwpp/Um1eFTp8iMy/c3u5+9Ie6rFDQ7kbEC5O6/CMSrVD
++YeEs5stAgMBAAECggEAEJ1eGHXT+OpmJ4ROvVrN3aJ9xtVDbeLO/tOEbB4tESaV
+7QjqiodW4fN9BK8grBdYPHtcPjzG8rodqQuktxR/AMdR+VVXg30mjG0dhKm+Zbjc
+PTbu2t6cAn9Pi2R5rH3Up5DSXe+L3RJ95MFQ5JVR2LtyhZqw8y70HnkVpejBCttT
+XC78LKeGDjXw0J3MtdXrqi+BHlw03i2qU8S1ycznZNSF6l/0en8zMUglPkPMP64u
++VnSlLtOYSn3YeXDpB5PUcvPuwUWOqE8Lz3eCmMVpgjxR/O29ygLVTIhZlM1Sl6Z
+MTaMb8XCzptlYm+V0HD01XVs5Pweny+i0cPw2QJj0wKBgQDqVOvkCA9J22XzaExK
+7RpSz/wG0zZfS2FdBGU6IGllBpJvE1hXwc4KTYNgVNk9osfdo7peMQD+BIiSdflB
+gUSKc6uxaBRXJbrP7ojJFn2ZKgsLKfToxxZlmIoDYLVUK8M62KwF1VZwTy6WyvnT
+YMsItAW7D+r6/92AMwHusq9qNwKBgQDhJbMF41VpNP2IHl3/J4+tq+McVllyQ5WI
+B8B1f3+hpGhNxgwOOFLukCnglwnYghl5HWqY0nYILgvLWP+1M1rr5pgFkX+/qYrB
+l3MoB51g9qAyba8pjcKcITsUvEYMKLUz22bcK3FAaBpL1Y2sAFRWEQZQiLlRD5mQ
+xJGfaZCjuwKBgQCQGjUN/97W/nLNor/3+6MKdhh4Yf63z3VKbF+LxU7K76WxZOYY
+4XfBYjV+Ue9eNfJO3FbJgb0oeMU7vAzSOZpawhM9f9GyXeNp5zh64LfQLsinzt9r
+HrMpOwr0VM5tupPOZlwzIiSL4EW4OY40bPuumHZFuN46CIrAQZ3Ymk3BNQKBgFsA
+OtjBZw9L/MGczI9J9TfUBsq4WBToHjNKNs8HBpZV9lahGtRQjayhFlwRkEgzZYpf
+imtUfaKWweuiAFDD/hblMELn6UOVimDvDvt6AHFlnaBkVu0rwCINCG08H+/gMEVS
+g8Ovx2mp6h8GpOmFhJD49p1sdcK+1GoAGX/eUjjrAoGAFKdZW1lLzIUfxIsJw6jH
+Gk1hDfifBQ8ZnxCCRk+7/S1PKYvG47VqcF1ClOewzReCj7gx95Q8FbW3Vxm9c/1i
+e8ouqlLaZtG6wzY+MlUHyys64saeabKfoA/8OtPl48CAt+bZO8xo9feibDjZyrzA
+pOUTGqXSKZ8G8cLizrGwYwE=
+-----END PRIVATE KEY-----