1508 avro schemas

.circleci/config.yml

@@ -55,7 +55,6 @@ jobs:
 #      - slack/status:
 #          fail_only: false
 
-# cli build
   cli-build:
     executor: docker/docker
     steps:
@@ -75,30 +74,55 @@ jobs:
       - slack/status:
           fail_only: false
 
-# clients build
-  client-build:
+  onestop-s3-handler-build:
     executor: docker/docker
     steps:
       - setup_remote_docker
       - checkout
       - docker/check
       - docker/build:
-          path: onestop-python-client
+          path: scripts/sqs-to-registry
+          image: cedardevs/onestop-s3-handler
+          tag: latest
+      - docker/push:
+          image: cedardevs/onestop-s3-handler
+          tag: latest
+      - slack/status:
+          fail_only: false
+
+  onestop-sme-build:
+    executor: docker/docker
+    steps:
+      - setup_remote_docker
+      - checkout
+      - docker/check
+      - docker/build:
+          path: scripts/sme/
+          image: cedardevs/onestop-sme
+          tag: latest
+      - docker/push:
+          image: cedardevs/onestop-sme
+          tag: latest
+      - slack/status:
+          fail_only: false
+
+  onestop-python-client-build:
+    executor: docker/docker
+    steps:
+      - setup_remote_docker
+      - checkout
+      - docker/check
+      - docker/build:
+          path: ./
           image: cedardevs/onestop-python-client
-          tag: ${CIRCLE_BRANCH}-SNAPSHOT
-      - run:
-          name: "What branch am I on now?"
-          command: echo $CIRCLE_BRANCH
-#no need to push this image yet
+          tag: latest
       - docker/push:
           image: cedardevs/onestop-python-client
-          tag: ${CIRCLE_BRANCH}-SNAPSHOT
+          tag: latest
       - slack/status:
           fail_only: false
-  # Base test configuration for Go library tests Each distinct version should
-  # inherit this base, and override (at least) the container image used.
 
-  python-client-test: &python-client-test
+  onestop-python-client-test: &python-client-test
     executor: python/default
     steps: &steps
       - checkout
@@ -107,10 +131,16 @@ jobs:
           app-dir: ./onestop-python-client
           pkg-manager: pip
       - run:
-          name: "Run util tests"
+          name: "Run unit tests"
           command: >
             cd onestop-python-client/;
-            python -m unittest tests/util/*.py
+            python -m unittest discover -s test/unit
+#          This is commented out only because the OneStop we have running on cedardevs doesn't have its registry exposed. You can only reach it via sshing to another machine.
+#      - run:
+#          name: "Run integration tests"
+#          command: >
+#            cd onestop-python-client/;
+#            python -m unittest discover -s test/integration
 
 orbs:
   slack: circleci/slack@3.4.2
@@ -121,9 +151,15 @@ version: 2.1
 workflows:
   main:
     jobs:
-#      - "latest"
 #      - cli-test
 #      - cli-build
-#      - client-build
-       - python-client-test
-
+       - onestop-python-client-test
+       - onestop-python-client-build:
+           requires:
+             - onestop-python-client-test
+       - onestop-sme-build:
+           requires:
+             - onestop-python-client-build
+       - onestop-s3-handler-build:
+           requires:
+             - onestop-python-client-build

Dockerfile

@@ -1,10 +1,15 @@
 FROM python:3.8
+
 COPY ./onestop-python-client /onestop-python-client
 COPY ./scripts /scripts
+
 RUN apt-get update
 RUN pip install --upgrade pip
-RUN pip install ./onestop-python-client
 RUN pip install -r ./onestop-python-client/requirements.txt
 
+# Needed for scripts - do here since directory out of scope when in scripts/* dockerfiles.
+# Unsure if possible this isn't latest build, like doing pip install before this is built.
+RUN pip install ./onestop-python-client
+
 #Base image stays up for dev access
 CMD tail -f /dev/null

helm/onestop-sqs-consumer/values.yaml

@@ -7,7 +7,7 @@ replicaCount: 1
 image:
   repository: cedardevs/onestop-sme
   tag: latest
-  pullPolicy: IfNotPresent
+  pullPolicy: Always
 
 imagePullSecrets: []
 nameOverride: ""
@@ -57,35 +57,51 @@ config: |-
   log_level: INFO
 
   # AWS config values
-  sqs_url: https://sqs.us-east-2.amazonaws.com/798276211865/cloud-archive-client-sqs
-  sqs_max_polls: 100
+  sqs_name: cloud-archive-client-sqs
   s3_region: us-east-2
   s3_bucket: archive-testing-demo
+  sqs_max_polls: 100
 
   #AWS config values for 2nd vault in different region
   vault_name: archive-vault-new
   s3_region2: us-east-2
-  s3_bucket2: noaa-nccf-dev-archive
+  s3_bucket2: archive-testing-testing-test
 
   #CSB stream config
   format: csv
   headers: UNIQUE_ID,FILE_UUID,LON,LAT,DEPTH,TIME,PLATFORM_NAME,PROVIDER
-  type: COLLECTION
   collection_id: fdb56230-87f4-49f2-ab83-104cfd073177
   access_bucket: https://archive-testing-demo.s3-us-east-2.amazonaws.com
   #access_bucket: https://odp-noaa-nesdis-ncei-test.s3-us-west-2.amazonaws.com
-  file_identifier_prefix: "gov.noaa.ncei.csb:"
+  file_id_prefix: "gov.noaa.ncei.csb:"
 
   # COLLECTION or GRANULE
-  metadata_type: granule
-  registry_base_url: http://onestop-registry:80
-  onestop_base_url: http://onestop-search:8080
+  kafka_consumer_metadata_type: GRANULE
+  kafka_publisher_metadata_type: GRANULE
+  s3_message_adapter_metadata_type: COLLECTION
 
-  security:
-    enabled: True
+  registry_base_url: http://os-registry:80
+  onestop_base_url: http://os-search:8080
 
   prefixMap:
     NESDIS/CSB: 'fdb56230-87f4-49f2-ab83-104cfd073177'
     NESDIS/H8: '0fad03df-0805-434a-86a6-7dc42d68480e'
     NESDIS/GOES: '11111111-1111-1111-1111-111111111111'
-    NESDIS/SAB: '98e03b47-069a-4f2c-8071-649e8c4254d6'
+    NESDIS/SAB: '98e03b47-069a-4f2c-8071-649e8c4254d6'
+
+  # Kafka config values
+  brokers: cp-cp-kafka:9092
+  schema_registry: http://cp-cp-schema-registry:8081
+  collection_topic_publish: psi-granules-by-collection
+  granule_topic_publish: psi-granule-parsed
+  collection_topic_consume: psi-collection-input-unknown
+  granule_topic_consume: psi-granule-input-unknown
+  group_id: sme-test
+  auto_offset_reset: earliest
+  security:
+    # True/False
+    enabled: False
+    # If security is enabled then need these:
+    caLoc: /etc/pki/tls/cert.pem
+    keyLoc: /etc/pki/tls/private/kafka-user.key
+    certLoc: /etc/pki/tls/certs/kafka-user.crt

helm/sme-chart/values.yaml

@@ -1,7 +1,7 @@
 image:
   repository: cedardevs/onestop-e2e-demo
   tag: latest
-  pullPolicy: IfNotPresent
+  pullPolicy: Always
 
 secret:
   registry_username: <value>
@@ -14,29 +14,52 @@ config: |-
   log_level: INFO
 
   # AWS config values
-  sqs_url: https://sqs.us-east-2.amazonaws.com/798276211865/cloud-archive-client-sqs
-  sqs_max_polls: 100
+  sqs_name: cloud-archive-client-sqs
   s3_region: us-east-2
   s3_bucket: archive-testing-demo
+  sqs_max_polls: 100
 
   #AWS config values for 2nd vault in different region
   vault_name: archive-vault-new
   s3_region2: us-east-2
-  s3_bucket2: noaa-nccf-dev-archive
+  s3_bucket2: archive-testing-testing-test
 
   #CSB stream config
   format: csv
   headers: UNIQUE_ID,FILE_UUID,LON,LAT,DEPTH,TIME,PLATFORM_NAME,PROVIDER
-  type: COLLECTION
   collection_id: fdb56230-87f4-49f2-ab83-104cfd073177
   access_bucket: https://archive-testing-demo.s3-us-east-2.amazonaws.com
   #access_bucket: https://odp-noaa-nesdis-ncei-test.s3-us-west-2.amazonaws.com
-  file_identifier_prefix: "gov.noaa.ncei.csb:"
+  file_id_prefix: "gov.noaa.ncei.csb:"
 
   # COLLECTION or GRANULE
-  metadata_type: granule
-  registry_base_url: http://onestop-registry:80
-  onestop_base_url: http://onestop-search:8080
+  kafka_consumer_metadata_type: GRANULE
+  kafka_producer_metadata_type: GRANULE
+  web_publisher_metadata_type: GRANULE
+  s3_message_adapter_metadata_type: COLLECTION
+
+  registry_base_url: http://os-registry:80
+  onestop_base_url: http://os-search:8080
+
+  prefixMap:
+    NESDIS/CSB: 'fdb56230-87f4-49f2-ab83-104cfd073177'
+    NESDIS/H8: '0fad03df-0805-434a-86a6-7dc42d68480e'
+    NESDIS/GOES: '11111111-1111-1111-1111-111111111111'
+    NESDIS/SAB: '98e03b47-069a-4f2c-8071-649e8c4254d6'
 
+  # Kafka config values
+  brokers: cp-cp-kafka:9092
+  schema_registry: http://cp-cp-schema-registry:8081
+  collection_topic_publish: psi-granules-by-collection
+  granule_topic_publish: psi-granule-parsed
+  collection_topic_consume: psi-collection-input-unknown
+  granule_topic_consume: psi-granule-input-unknown
+  group_id: sme-test
+  auto_offset_reset: earliest
   security:
-    enabled: True
+    # True/False
+    enabled: False
+    # If security is enabled then need these:
+    caLoc: /etc/pki/tls/cert.pem
+    keyLoc: /etc/pki/tls/private/kafka-user.key
+    certLoc: /etc/pki/tls/certs/kafka-user.crt

kubernetes/pyconsumer-pod.yaml

@@ -70,19 +70,21 @@ data:
     csb:
       format: csv
       headers: UNIQUE_ID,FILE_UUID,LON,LAT,DEPTH,TIME,PLATFORM_NAME,PROVIDER
-      type: COLLECTION
       collection_id: fdb56230-87f4-49f2-ab83-104cfd073177
-      psi_registry_url: https://cedardevs.org/
+      registry_base_url: https://cedardevs.org/
       access_bucket: https://archive-testing-demo.s3-us-east-2.amazonaws.com
       #access_bucket: https://odp-noaa-nesdis-ncei-test.s3-us-west-2.amazonaws.com
       file_identifier_prefix: "gov.noaa.ncei.csb:"
 
     # Web Publisher
     web:
-      # COLLECTION or GRANULE
-      metadata_type: granule
       registry_base_url: https://cedardevs.org/onestop/registry-api
       onestop_base_url: https://cedardevs.org/onestop/search-api
 
       security:
-        enabled: True
+        enabled: True
+
+    # COLLECTION or GRANULE
+    kafka_consumer_metadata_type: GRANULE
+    kafka_publisher_metadata_type: GRANULE
+    s3_message_adapter_metadata_type: COLLECTION

onestop-python-client/config/aws-util-config-dev.yml

@@ -1,11 +1,12 @@
 # Example config values for osim client
-log_level: INFO
 
 # AWS config values
 sqs_url: https://sqs.us-east-2.amazonaws.com/798276211865/cloud-archive-client-sqs
+sqs_name: 'foobar'
 sqs_max_polls: 2
 s3_region: "us-east-2"
 s3_bucket: archive-testing-demo
+s3_key: 'ABI-L1b-RadF/2019/298/15/OR_ABI-L1b-RadF-M6C15_G16_s20192981500369_e20192981510082_c20192981510166.nc'
 
 #AWS config values for 2nd vault in different region
 vault_name: archive-vault-new

onestop-python-client/config/credentials-template.yml

@@ -9,5 +9,4 @@ registry:
   username: rw_user
   password: rw_user_pwd
 
-
-
+log_level: INFO

onestop-python-client/config/csb-data-stream-config-template.yml

@@ -1,15 +1,18 @@
-log_level: INFO
+# COLLECTION or GRANULE
+kafka_consumer_metadata_type: COLLECTION
+kafka_publisher_metadata_type: COLLECTION
+s3_message_adapter_metadata_type: COLLECTION
+
 format: csv
 headers: UNIQUE_ID,FILE_UUID,LON,LAT,DEPTH,TIME,PLATFORM_NAME,PROVIDER
-type: COLLECTION
 collection_id: fdb56230-87f4-49f2-ab83-104cfd073177
 #registry_base_url: https://internal-a683c98a66fb011eaa4230e0d5e5657f-369075387.us-east-1.elb.amazonaws.com
 registry_base_url: http://localhost/onestop/api/registry
 #onestop_base_url: https://cedardevs.org/onestop/search
 onestop_base_url: http://localhost/onestop/api/search/search
 access_bucket: https://archive-testing-demo.s3-us-east-2.amazonaws.com
 #access_bucket: https://odp-noaa-nesdis-ncei-test.s3-us-west-2.amazonaws.com
-file_identifier_prefix: "gov.noaa.ncei.csb:"
+file_id_prefix: "gov.noaa.ncei.csb:"
 
 prefixMap:
   NESDIS/CSB: 'fdb56230-87f4-49f2-ab83-104cfd073177'