Initial alpha release

the-carlisle-group · Sep 10, 2019 · 13f3dca · 13f3dca
1 parent e68cdf0
commit 13f3dca
Show file tree

Hide file tree

Showing 22 changed files with 92 additions and 63 deletions.
diff --git a/APLSource/Client.aplc b/APLSource/Client.aplc
@@ -31,7 +31,7 @@
 
     :endproperty
 
-    :field private secretKey
+    :field private secretKey ←''
     :Property SecretKey
     :access public instance
 
@@ -56,7 +56,7 @@
         ∇
     :EndProperty
 
-    :field private accessKey
+    :field private accessKey ←''
     :Property AccessKey
     :access public instance
         ∇ r←get
@@ -219,7 +219,6 @@
 
     :endproperty
 
-
     ∇ Z←New(sn rg)
       :Access public shared
       ⍝ sn = Service Name
@@ -232,25 +231,13 @@
       :Access public
       :Implements constructor
       timeStamp←GetTimeStamp
-      SetDefaultKeys ##.##.AcreConfig.ProjectFolder
       rumba←#.CarlisleGroup.Rumba.Core
       uri←##.##.URI.New
       uri.HostService←sn
       uri.HostRegion←rg
       uri.HostPrefix←''
     ∇
 
-      SetDefaultKeys←{
-          p←⍵
-          0=⎕NEXISTS ⍵:_←0
-          fp←⍵,'defaultKeys.csv'
-          0=⎕NEXISTS fp:_←0
-          keys←0⊃⎕CSV fp'UTF-8'⍬ 1
-          0=≢keys:_←0
-          ⎕THIS.(accessKey secretKey)←keys[0;]
-      }
-
-
     ∇ Z←ConstructConicalRequest;cr;qp;q;ct;n;v
       :Access public instance
 
@@ -285,10 +272,8 @@
       ss,←Date,'/',URI.HostRegion,'/',URI.HostService,'/aws4_request',NewLine
       ss,←Hash_SHA256 ConstructConicalRequest
       Z←ss
-
     ∇
 
-
     ∇ Z←ConstructSignatureKey;dateKey;dateRegionKey;dateRegionServiceKey;signingKey
       :Access public instance
       dateKey←('AWS4',SecretKey)HMAC_SHA256 Date
@@ -355,6 +340,7 @@
 
       c←rumba.NewClient''
       c.ClientTimeout←60000
+      c.TLSFlag←0
       c.Host,←URI.Host
       c.Secure←URI.Scheme≡'https'
       c.Port←URI.Port

diff --git a/APLSource/Init.aplf b/APLSource/Init.aplf
@@ -1,4 +1,12 @@
- {Z}←Init;_
- #.DCL.Crypt.Init #.DCL.AcreConfig.ProjectFolder,'Assets/Runtime/'
- _←#.CarlisleGroup.Rumba.Core.InitConga''
- Z←0
+ Init←{
+     ⍝ Goal is remove DCL requirement either with Conga or standalone hashing algorithm
+     pf←{
+         'CLEAR WS'≡⍵:AcreConfig.ProjectFolder,'../'
+         ('\\'⎕R'/')0⊃⎕NPARTS ⍵
+     }⎕WSID
+     #.DCL.AcreConfig.ProjectFolder←pf,'DyalogDCL/'
+     ##.Rumba.AcreConfig.ProjectFolder←pf,'Rumba/'
+     _←#.DCL.Crypt.Init #.DCL.AcreConfig.ProjectFolder,'Assets/Runtime/'
+     _←#.CarlisleGroup.Rumba.Core.InitConga''
+     1:×_←0
+ }
diff --git a/APLSource/S3/Bucket.aplc b/APLSource/S3/Bucket.aplc
@@ -77,13 +77,16 @@
       name←n
       client←c.Clone
       client.URI.HostPrefix←n
-      client.URI.HostRegion←Region
+      ⍝⍝      client.URI.HostRegion←''⍝Region
+      ⍝⍝      Region reguired to be set before calling any requst
     ∇
 
 
     ∇ Z←GetObjects(startAfterId maxKeys prefix);res;xml;bc;b;tb;params;pi
       :Access public instance
-
+      ⍝ startAfterId    : Token to list maxKeys count from when iterating a large list.  AWS ContinuationToken value
+      ⍝ maxKeys         : Max object count.  1000 is max 
+      ⍝ prefix          : only object where the key start with this value.
       Z←⍬
 
       isTruncated←0
@@ -202,7 +205,7 @@
     :endproperty
 
     ∇ Z←getBucketRegion;res;xml;v;c2
-
+      ;msg
       c2←client.Clone
       c2.URI.QueryParams←,⊂'location' ''
       res←c2.SendRequest 0
@@ -222,7 +225,8 @@
               Z←client.URI.HostRegion
               :Return
           :ElseIf res.StatusCode=403
-              ⎕SIGNAL⊂('EN'res.StatusCode)('EM'(2⊃xml[1;]))('Message' 'Does the bucket exist or belong to someone else?')
+              msg←'Does bucket "',name,'" exist or belong to someone else?'
+              ⎕SIGNAL⊂('EN'res.StatusCode)('EM'(2⊃xml[1;]))('Message'msg)
           :Else
               SignalError res
           :EndIf

diff --git a/APLSource/S3/Service.aplc b/APLSource/S3/Service.aplc
@@ -20,9 +20,11 @@
       client←_client.Clone
     ∇
 
-    ∇ Z←GetBuckets;res;xml;bc;b;tb;bn;cd
-      :Access public instance
-      res←client.SendRequest 1
+    ∇ Z←GetBuckets;res;xml;bc;b;tb;bn;cd;c
+      :Access public instance    
+      c←client.Clone
+      res←c.SendRequest 1
+
       CheckContentTypeXML res
 
       xml←⎕XML res.Content

diff --git a/APLSource/Tests/DynamoDB/ListTables.aplf b/APLSource/Tests/DynamoDB/ListTables.aplf
@@ -1,4 +1,5 @@
  ListTables←{
+     1
      c←##.##.Client.New'dynamodb'##.defaultRegion
      db←##.##.DynamoDB.New c
      t←db.ListTables

diff --git a/APLSource/Tests/GetClient.aplf b/APLSource/Tests/GetClient.aplf
@@ -1 +1,7 @@
- GetClient
+ getClient←{
+     c←##.Client.New ⍵
+     c.AccessKey←2 ⎕NQ'.' 'GetEnvironment' 'AWS_AccessKey'
+     c.SecretKey←2 ⎕NQ'.' 'GetEnvironment' 'AWS_SecretKey'
+     c.Token←2 ⎕NQ'.' 'GetEnvironment' 'AWS_Token'
+     c
+ }
diff --git a/APLSource/Tests/RunTests.aplf b/APLSource/Tests/RunTests.aplf
@@ -1,12 +1,15 @@
  RunTests←{
      _←⎕SE.acre.OpenProject ##.AcreConfig.ProjectFolder,'../DyalogDCL'
-     _←##.Init
+     _←##.Init''
      r←1↓refs ⎕THIS
-     S3.defaultBucket←'awstoolsdefault'
-     defaultRegion∘←'us-east-1'
-     out←{
+     ⎕THIS._defaultBucket←'d19apps'
+     client←getClient's3' 'us-east-2'
+     out←client{
          list←⍵.⎕NL ¯3
-         err←⍵.⍎¨list,¨⊂' 0'
+         err←⍵{
+             exp←'⍺.',⍵,' client'
+             ⍎exp
+         }¨list
          ⎕←err/list
          ⎕←(⊂(⍕⍵),'.'),¨(err/list)
          err

diff --git a/APLSource/Tests/S3/DeleteAllObjects.aplf b/APLSource/Tests/S3/DeleteAllObjects.aplf
@@ -1,7 +1,8 @@
  DeleteAllObjects←{
-     c←##.##.Client.New's3'##.defaultRegion
+     c←⍵
      s←##.##.S3.Service.New c
-     b←0⊃s.GetBuckets
+     b←s.GetBuckets
+     b←0⊃b
 
      uriz←b{
          ob←##.##.S3.Object.New ⍺

diff --git a/APLSource/Tests/S3/DeleteBucketIncompleteMultiPart.aplf b/APLSource/Tests/S3/DeleteBucketIncompleteMultiPart.aplf
@@ -1,5 +1,5 @@
  DeleteBucketIncompleteMultiPart←{
-     c←##.##.Client.New's3'##.defaultRegion
+     c←⍵
      s←##.##.S3.Service.New c
      b←0⊃s.GetBuckets
      obs←b.GetIncompleteMultiPart

diff --git a/APLSource/Tests/S3/DeleteObject.aplf b/APLSource/Tests/S3/DeleteObject.aplf
@@ -1,9 +1,9 @@
  DeleteObject←{
-     c←##.##.Client.New's3'##.defaultRegion
+     c←⍵
      s←##.##.S3.Service.New c
      b←0⊃s.GetBuckets
-     _←PutObject''
-     ob←0⊃b.GetObjects'' 1
+     _←PutObject ⍵
+     ob←0⊃b.GetObjects'' 1 ''
      _←ob.Delete
      0
  }
diff --git a/APLSource/Tests/S3/GetBucketCollection.aplf b/APLSource/Tests/S3/GetBucketCollection.aplf
@@ -1,6 +1,6 @@
  GetBucketCollection←{
 
-     c←##.##.Client.New's3'##.defaultRegion
+     c←⍵
      s←##.##.S3.Service.New c
      b←s.GetBuckets
      0

diff --git a/APLSource/Tests/S3/GetBucketIncompleteMultiPart.aplf b/APLSource/Tests/S3/GetBucketIncompleteMultiPart.aplf
@@ -1,5 +1,5 @@
  GetBucketIncompleteMultiPart←{
-     c←##.##.Client.New's3'##.defaultRegion
+     c←⍵
      s←##.##.S3.Service.New c
      b←0⊃s.GetBuckets
      obs←b.GetIncompleteMultiPart

diff --git a/APLSource/Tests/S3/GetObject.aplf b/APLSource/Tests/S3/GetObject.aplf
@@ -1,10 +1,9 @@
  GetObject←{
-
-     c←##.##.Client.New's3'##.defaultRegion
+     c←⍵
      s←##.##.S3.Service.New c
      b←0⊃s.GetBuckets
-     _←PutObject''
-     ob←0⊃b.GetObjects'' 1
+     _←PutObject c
+     ob←0⊃b.GetObjects'' 1 ''
      _←ob.GetMetaData
      _←ob.GetValue
      0

diff --git a/APLSource/Tests/S3/GetObjectCollection.aplf b/APLSource/Tests/S3/GetObjectCollection.aplf
@@ -1,5 +1,5 @@
  GetObjectCollection←{
-     c←##.##.Client.New's3'##.defaultRegion
+     c←⍵
      s←##.##.S3.Service.New c
      b←0⊃s.GetBuckets
      ob←b.GetAllObjects

diff --git a/APLSource/Tests/S3/GetObjectPreSignedURI.aplf b/APLSource/Tests/S3/GetObjectPreSignedURI.aplf
@@ -1,7 +1,7 @@
  GetObjectPreSignedURI←{
-     c←##.##.Client.New's3'##.defaultRegion
+     c←⍵
      bn←##.##.Utils.GetTimeStamp
-     b←##.##.S3.Bucket.New c defaultBucket
+     b←##.##.S3.Bucket.New c ##._defaultBucket
      ob←##.##.S3.Object.New b
      ob.ContentType←'text/plain'
      ob.Value←'this is some text',⎕UCS 13 10

diff --git a/APLSource/Tests/S3/PutBucket.aplf b/APLSource/Tests/S3/PutBucket.aplf
@@ -1,9 +1,9 @@
  PutBucket←{
-     c←##.##.Client.New's3'##.defaultRegion
-     b←##.##.S3.Bucket.New c'badname'
+     c←⍵
+     b←##.##.S3.Bucket.New c('badname',⍕⌊10000000×?0)
      e←b.Exists
      e≠0:1
-     _←b.Create
+     _←b.Create  ⍝ must have access to create bucket in the account
      e←b.Exists
      e≠1:1
      _←b.Delete

diff --git a/APLSource/Tests/S3/PutObject.aplf b/APLSource/Tests/S3/PutObject.aplf
@@ -1,7 +1,6 @@
  PutObject←{
-     c←##.##.Client.New's3'##.defaultRegion
-     s←##.##.S3.Service.New c
-     b←##.##.S3.Bucket.New c defaultBucket
+     c←⍵
+     b←##.##.S3.Bucket.New c ##._defaultBucket
      ob←##.##.S3.Object.New b
 ⍝     ob.ACL←'public-read'
      ob.ContentType←'text/plain'
@@ -25,7 +24,7 @@
      ob.Value←'this is some text',⎕UCS 13 10
      ob.Value,←'Current Time: ',##.##.Utils.GetTimeStamp
      ob.Key←'test_',(' '~⍨∊⍕⎕TS),'.txt'
-     ob.ACL←'public-read'
+⍝     ob.ACL←'public-read'  ⍝ <<<< will not work without bucket having public content enabled!!!
      ok←ob.Put
      url←'https://',b.Name,'.s3.amazonaws.com/',ob.Key,'#somepw'
      ⍝ will be error becaues private!

diff --git a/APLSource/Tests/S3/PutObjectAutoBucket.aplf b/APLSource/Tests/S3/PutObjectAutoBucket.aplf
@@ -1,5 +1,5 @@
  PutObjectAutoBucket←{
-     c←##.##.Client.New's3'##.defaultRegion
+     c←⍵
      s←##.##.S3.Service.New c
      b←0⊃s.GetBuckets
      ob←##.##.S3.Object.New b

diff --git a/APLSource/Tests/S3/PutObjectInFolders.aplf b/APLSource/Tests/S3/PutObjectInFolders.aplf
@@ -1,6 +1,6 @@
  PutObjectInFolders←{
-     c←##.##.Client.New's3'##.defaultRegion
-     b←##.##.S3.Bucket.New c defaultBucket
+     c←⍵
+     b←##.##.S3.Bucket.New c ##._defaultBucket
 
      ob←##.##.S3.Object.New b
 ⍝     ob.ACL←'public-read'
@@ -13,7 +13,7 @@
      logLink←ob.URI
 
      ob←##.##.S3.Object.New b
-     ob.ACL←'public-read'
+⍝     ob.ACL←'public-read'
      ob.ContentType←'application/json'
      ob.Value←⎕JSON ⎕DMX
      ob.Folder←'errors'
@@ -31,7 +31,7 @@
      html,←'</body></html>'
 
      ob←##.##.S3.Object.New b
-     ob.ACL←'public-read'
+⍝     ob.ACL←'public-read'
      ob.ContentType←'text/html'
      ob.Value←html
      ob.Key←'page_',(' '~⍨∊⍕⎕TS),'.html'

diff --git a/APLSource/Tests/S3/PutObjectMultiPart.aplf b/APLSource/Tests/S3/PutObjectMultiPart.aplf
@@ -1,7 +1,7 @@
  PutObjectMultiPart←{
-     c←##.##.Client.New's3'##.defaultRegion
+     c←⍵
      s←##.##.S3.Service.New c
-     b←##.##.S3.Bucket.New c defaultBucket
+     b←##.##.S3.Bucket.New c ##._defaultBucket
      ob←##.##.S3.Object.New b
      ob.ContentType←'text/plain'
      ob.Value←13002300⍴'this is some text',⎕UCS 13 10

diff --git a/APLSource/Utils/GetAWSKeys.aplf b/APLSource/Utils/GetAWSKeys.aplf
@@ -0,0 +1,19 @@
+ Z←GetAWSKeys;ak;sk;mt;tk
+ Z←⎕NS''
+
+ Z.Token←''  ⍝ default to none
+ Z.AccessKey←''
+ Z.SecretKey←''
+
+ :Trap 0
+     mt←#.CarlisleGroup.AWSTools.EC2.MetaData.IAM
+     Z.(AccessKey SecretKey Token)←mt.securitycredentials.role.(AccessKeyId SecretAccessKey Token)
+     :Return
+ :EndTrap
+
+ Z.AccessKey←GetEnv'AWS_AccessKey'
+ Z.SecretKey←GetEnv'AWS_SecretKey'
+ Z.Token←GetEnv'AWS_Token'
+ →(∧/0≠≢¨ak sk)/0
+
+ 'Keys not available.'⎕SIGNAL 11
diff --git a/APLSource/Utils/GetEnv.aplf b/APLSource/Utils/GetEnv.aplf
@@ -0,0 +1 @@
+ GetEnv←{2 ⎕NQ'.' 'GetEnvironment'⍵}