[CE-494] Connection profile use public IP

Currently ansible agent setup components in
the same k8s cluster, using the componment
names to communicate is fine. However when
a fabric network uses multiple clusters, use
the component names in the connection profile
won't work. This patch fixed that problem.

Change-Id: I1a7220fea9389fbb0d398be46934c450db65e34d
Signed-off-by: Tong Li <litong01@us.ibm.com>

src/agent/ansible/roles/deploy_compose/certsetup/templates/configtx.j2

@@ -130,8 +130,8 @@ Application: &ApplicationDefaults
           Rule: "MAJORITY Admins"
 {% endif %}
 {% if project_version is version_compare('1.1.0','>=') or 'stable' in project_version or project_version == 'latest' %}
-    Capabilities:
-      <<: *ApplicationCapabilities
+  Capabilities:
+    <<: *ApplicationCapabilities
 {% endif %}
 
 {% if project_version is version_compare('1.2.0','>=') or 'stable' in project_version or project_version == 'latest' %}

src/agent/ansible/roles/deploy_k8s/plays.yml

@@ -33,6 +33,18 @@
     - include_tasks: "fabricsetup/tasks/{{ mode }}.yml"
   tags: "fabricsetup"
 
+- name: Generate connection profile using public IPs
+  hosts: cloud
+  connection: local
+  become: true
+  become_user: root
+  vars_files:
+    - "./../../vars/{{ env }}.yml"
+  tasks:
+    - include_tasks: "pubconn/tasks/{{ mode }}.yml"
+      when: fabric.k8s.exposeserviceport == True
+  tags: "conngen"
+
 - name: setup composer
   hosts: cloud
   connection: local

src/agent/ansible/roles/deploy_k8s/pubconn/tasks/apply.yml

@@ -0,0 +1,72 @@
+---
+- name: Reuse common code
+  include_tasks: "{{ playbook_dir }}/../common/config_apply.yml"
+
+- name: Ensure certificate directory exist
+  file:
+    path: "{{ fabricworkdir }}/keyfiles"
+    state: "{{ item }}"
+  with_items:
+    - "directory"
+
+- name: Query node public ip address
+  command: >-
+    ./kubectl --kubeconfig kubeconfig  get nodes -o
+    jsonpath={.items[*].status.addresses[?\(@.type==\'ExternalIP\'\)].address}
+  register: ips
+  args:
+    chdir: "{{ playbook_dir }}/../../vars/"
+
+- name: Query service ports
+  shell: >-
+    kubectl --kubeconfig kubeconfig get services
+    | grep NodePort | awk '{print $1, $5}'
+  register: ports
+  args:
+    chdir: "{{ playbook_dir }}/../../vars/"
+
+- debug:
+    var: ports.stdout_lines
+
+- set_fact:
+    allips: "{{ ips.stdout.split(' ') }}"
+    k8sports: []
+    sports: {}
+
+- set_fact:
+    k8sports: >-
+      {{ k8sports +
+      [{ 'name':item.split(' ')[0], 'value':(item.split(' ')[1]|replace('/TCP','')).split(',') }] }}
+  with_items: "{{ ports.stdout_lines }}"
+
+- set_fact:
+    sports: >-
+      {{ k8sports | subelements('value', skip_missing=True) }}
+
+- set_fact:
+    k8sports: {}
+
+- set_fact:
+    k8sports: >-
+      {{ k8sports|combine( {item[0].name+':'+item[1].split(':')[0]:item[1].split(':')[1]} ) }}
+  with_items: "{{ [sports] }}"
+
+- debug:
+    var: k8sports
+
+- name: Create connection profiles
+  template:
+    src: "{{ playbook_dir }}/../deploy_k8s/pubconn/templates/connection.j2"
+    dest: "{{ fabricworkdir }}/keyfiles/{{ item }}/connection.json"
+  with_items: "{{ allorgs }}"
+
+- name: Create yaml connection profiles
+  template:
+    src: "{{ playbook_dir }}/../deploy_k8s/pubconn/templates/connyaml.j2"
+    dest: "{{ fabricworkdir }}/keyfiles/{{ item }}/connection.yml"
+  with_items: "{{ allorgs }}"
+
+- name: Create a zip file of all certificates
+  archive:
+    path: "{{ fabricworkdir }}/keyfiles"
+    dest: "{{ fabricworkdir }}/certs.tgz"

src/agent/ansible/roles/deploy_k8s/pubconn/tasks/destroy.yml

@@ -0,0 +1 @@
+---

src/agent/ansible/roles/deploy_k8s/pubconn/templates/connection.j2

@@ -0,0 +1,83 @@
+{
+  "name": "{{ item }}-network",
+  "x-type": "hlfv1",
+  "version": "1.0.0",
+  "client": {
+    "organization": "{{ item }}",
+    "connection": {
+      "timeout": {
+        "peer": { "endorser": "300", "eventHub": "300", "eventReg": "300"},
+        "orderer": "300"
+      }
+    }
+  },
+  "channels": {
+    "firstchannel": {
+      "orderers": [
+{% for orderer in allorderers %}
+        "{{ orderer.name }}"{{ '' if loop.last else ',' }}
+{% endfor %}
+      ],
+      "peers": {
+{% for peer in allpeers %}
+        "{{ peer.name }}": {
+          "endorsingPeer": true, "chaincodeQuery": true, "eventSource": true
+        }{{ '' if loop.last else ',' }}
+{% endfor %}
+      }
+    }
+  },
+  "organizations": {
+{% for org in allorgs %}
+    "{{ org }}": {
+      "mspid": "{{ org }}",
+      "peers": [
+{% for peer in allpeers|selectattr('org', 'equalto', org)|list %}
+        "{{ peer.name }}"{{ '' if loop.last else ',' }}
+{% endfor %}
+      ],
+      "certificateAuthorities": [
+{% for ca in allcas|selectattr('org', 'equalto', org)|list %}
+        "{{ ca.name }}"{{ '' if loop.last else ',' }}
+{% endfor %}
+      ]
+    }{{ '' if loop.last else ',' }}
+{% endfor %}
+  },
+  "orderers": {
+{% for orderer in allorderers %}
+    "{{ orderer.name }}": {
+      "url": "{{ tls|ternary('grpcs','grpc') }}://{{ allips|random }}:{{ k8sports[orderer.name+':7050'] }}",
+      "grpcOptions": {
+        "ssl-target-name-override": "{{ orderer.name }}"
+      },
+      "tlsCACerts": {
+        "pem": "{{ fabricworkdir+'/run/keyfiles/'+orderer.org+'/orderers/'+orderer.name+'.'+orderer.org+'/tls/ca.crt' }}"
+      }
+    }{{ '' if loop.last else ',' }}
+{% endfor %}
+  },
+  "peers": {
+{% for peer in allpeers|selectattr('org', 'equalto', item)|list %}
+    "{{ peer.name }}": {
+      "url": "{{ tls|ternary('grpcs','grpc') }}://{{ allips|random }}:{{ k8sports[peer.name+':7051'] }}",
+      "eventUrl": "{{ tls|ternary('grpcs','grpc') }}://{{ allips|random }}:{{ k8sports[peer.name+':7053'] }}",
+      "grpcOptions": {
+        "ssl-target-name-override": "{{ peer.name }}"
+      },
+      "tlsCACerts": {
+        "pem": "{{ fabricworkdir+'/run/keyfiles/'+peer.org+'/tlsca/tlsca.'+peer.org+'-cert.pem' }}"
+      }
+    }{{ '' if loop.last else ',' }}
+{% endfor %}
+  },
+  "certificateAuthorities": {
+{% for ca in allcas|selectattr('org', 'equalto', item)|list %}
+    "{{ ca.name }}": {
+      "url": "{{ tls|ternary('https','http') }}://{{ allips|random }}:{{ k8sports[ca.name+':7054'] }}",
+      "caName": "{{ ca.name }}",
+      "httpOptions": {"verify": false}
+    }{{ '' if loop.last else ',' }}
+{% endfor %}
+  }
+}

src/agent/ansible/roles/deploy_k8s/pubconn/templates/connyaml.j2

@@ -0,0 +1,71 @@
+---
+name: "{{ item }}-network"
+x-type: "hlfv1"
+version: "1.0.0"
+client:
+  organization: "{{ item }}"
+  cryptoconfig:
+    path: "/fabric/keyfiles"
+  connection:
+    timeout:
+      peer:
+        endorser: 300
+        eventHub: 300
+        eventReg: 300
+      orderer: 300
+channels:
+  firstchannel:
+    orderers:
+{% for orderer in allorderers %}
+      - {{ orderer.name }}
+{% endfor %}
+    peers:
+{% for peer in allpeers %}
+      {{ peer.name }}:
+        endorsingPeer: true
+        chaincodeQuery: true
+        eventSource: true
+{% endfor %}
+organizations:
+{% for org in allorgs %}
+  {{ org }}:
+    mspid: "{{ org }}"
+    cryptoPath: "{{ org }}/users/{username}@{{ org}}/msp"
+    peers:
+{% for peer in allpeers|selectattr('org', 'equalto', org)|list %}
+      - {{ peer.name }}
+{% endfor %}
+    certificateAuthorities:
+{% for ca in allcas|selectattr('org', 'equalto', org)|list %}
+      - {{ ca.name }}
+{% endfor %}
+{% endfor %}
+orderers:
+{% for orderer in allorderers %}
+  {{ orderer.name }}:
+    url: "{{ tls|ternary('grpcs','grpc') }}://{{ allips|random }}:{{ k8sports[orderer.name+':7050'] }}"
+    grpcOptions:
+      ssl-target-name-override: "{{ orderer.name }}"
+      tlsCACerts:
+        path: "{{ '/fabric/keyfiles/'+orderer.org+'/orderers/'+orderer.name+'.'+orderer.org+'/tls/ca.crt' }}"
+{% endfor %}
+peers:
+{% for peer in allpeers|selectattr('org', 'equalto', item)|list %}
+  {{ peer.name }}:
+    url: "{{ tls|ternary('grpcs','grpc') }}://{{ allips|random }}:{{ k8sports[peer.name+':7051'] }}"
+    eventUrl: "{{ tls|ternary('grpcs','grpc') }}://{{ allips|random }}:{{ k8sports[peer.name+':7053'] }}"
+    grpcOptions:
+      ssl-target-name-override: "{{ peer.name }}"
+    tlsCACerts:
+      path: "{{ '/fabric/keyfiles/'+peer.org+'/tlsca/tlsca.'+peer.org+'-cert.pem' }}"
+{% endfor %}
+certificateAuthorities:
+{% for ca in allcas|selectattr('org', 'equalto', item)|list %}
+  {{ ca.name }}:
+    url: "{{ tls|ternary('https','http') }}://{{ allips|random }}:{{ k8sports[ca.name+':7054'] }}"
+    caName: "{{ ca.name }}"
+    httpOptions:
+      verify: false
+    tlsCACerts:
+      path: "{{ '/fabric/keyfiles/'+ca.org+'/tlsca/tlsca.'+ca.org+'-cert.pem' }}"
+{% endfor %}