gopa.yml

cluster:
  name: gopa
  minimum_nodes: 1
#  seeds:
#  - localhost:20000
#  - localhost:20001
#  - localhost:20002
#node.name: node1

path.data: data
path.logs: log
path.certs: test/cert

cookie_secret: YOUR-GOPA-SECRET

allow_multi_instance: true
max_num_of_instances: 30

tls_enabled: false

logging.level: info

node.master: true
node.data: true
node.proxy: true

node.tag: ["external","hot"]


network:
  http_bind: 0.0.0.0:9000
  api_bind: 0.0.0.0:8000
  rpc_bind: 0.0.0.0:20000
  broadcast_bind: 224.3.2.2:9876

elasticsearch:
- name: default
  enabled: true
  endpoint: http://localhost:9200
  index_prefix: gopa-
  basic_auth:
    username: elastic
    password: changeme

- name: monitoring
  enabled: true
  endpoint: http://localhost:9200
  index_prefix: gopa-
  basic_auth:
    username: elastic
    password: changeme


pipelines:
- name: url_filter_flow
  start:
    joint: init_task
    enabled: true
  process:
    - joint: url_normalization
      enabled: true
      parameters:
        follow_all_domain: false
        follow_sub_domain: true
        max_filename_length: 1000
    - joint: url_filter
      enabled: true
      parameters:
        url_match_rule:
          must_not:
            prefix:
            - "mailto:"
            - "data:image/"
            - "#"
            - "javascript:"
            contain:
            - "crond/run"
            - "login"
            - "logout"
            suffix: []
          must:
            prefix: []
            contain: []
            suffix: []
          should:
            prefix: []
            contain: []
            suffix: []
        host_match_rule:
          must:
            prefix: []
            contain: []
            suffix: []
          should:
            prefix: []
            contain: []
            suffix: []
          must_not:
            prefix: []
            contain: []
            suffix: []
        file_match_rule:
          must:
            prefix: []
            contain: []
            suffix: []
          should:
            prefix: []
            contain: []
            suffix: []
          must_not:
            prefix: []
            contain: []
            suffix: []
        file_ext_match_rule:
          should:
            prefix: []
            contain: []
            suffix: []
          must:
            prefix: []
            contain: []
            suffix: []
          must_not:
            contain: [zip, exe, jar, js, css, rar, gz, zip, bmp, jpeg, gif,svg, png, jpg, apk]
            prefix: []
            suffix: []
    - joint: filter_check
      enabled: true
      parameters:
        filter_key: check_filter
    - joint: task_deduplication
      enabled: true
  end:
    joint: save_task
    enabled: true
    parameters:
      is_create: true
  error:
    joint: save_task
    enabled: true
- name: url_fetch_flow
  start:
    joint: init_task
    enabled: true
  process:
    - joint: url_normalization
      enabled: true
      parameters:
        follow_all_domain: false
        follow_sub_domain: true
        max_filename_length: 1000
    - joint: ignore_timeout
      parameters:
        ignore_timeout_after_count: 500
      enabled: true
    - joint: chrome_fetch
      enabled: false
      parameters:
        chrome_host: http://127.0.0.1:9223
        save_screenshot: true
        compress_enabled: true
        timeout_in_seconds: 60
        screenshot_quality: 50
    - joint: fetch
      enabled: true
      parameters:
        timeout_in_seconds: 100
#          proxy: "socks5://127.0.0.1:1080"
#          cookie: ""
    - joint: parse_pdf
      enabled: true
      parameters:
        accepted_content_type:
        - application/octet-stream
        - application/pdf
    - joint: parse
      enabled: true
      parameters:
        dispatch_links: true
        save_images: true
        max_depth: 100
        max_breadth: 1
    - joint: extract
      enabled: false
      parameters:
        html_block:
          your_tag_name1: ".tag_class"
          your_tag_name2: "#tag_id"
    - joint: html2text
      parameters:
        remove_nonscript: true
        merge_whitespace: false
      enabled: true
    - joint: hash
      enabled: true
    - joint: content_deduplication
      enabled: true
    - joint: update_check_time
      enabled: true
      parameters:
        decelerate_steps: "24h,48h,72h,168h,360h,720h"
        accelerate_steps: "24h,12h,6h,3h,1h30m,45m,30m,20m,10m"
    - joint: lang_detect
      enabled: true
    - joint : save_snapshot_fs
      enabled: false
    - joint : save_snapshot_db
      enabled: true
      parameters:
        compress_enabled: true
        bucket: "Snapshot"
        max_revision: 5
    - joint: index
      enabled: true
  end:
    joint: save_task
    enabled: true
    keep_404: true
    keep_redirected: true
  error:
    joint: save_task
    enabled: true

modules:
- name: pipeline
  enabled: true
  runners:
    - name: checker
      enabled: true
      input_queue: check
      pipeline_id: url_filter_flow
      max_go_routine: 1
      threshold_in_ms: 0
      timeout_in_ms: 5000
    - name: fetch
      enabled: true
      input_queue: fetch
      pipeline_id: url_fetch_flow
      max_go_routine: 1
      threshold_in_ms: 0
      timeout_in_ms: 30000

- name: cluster
  enabled: true


- name: api
  enabled: true

- name: web
  enabled: true
  auth:
    enabled: false
    oauth_provider: github
    oauth_authorize_url: https://github.com/login/oauth/authorize
    oauth_token_url: https://github.com/login/oauth/access_token
    client_id: 850d747174ace88ce889
    client_secret: 3d437b64e06371d6f62769320438d3dfc95a8d8e
    authorized_admin:
      - medcl
      - your_github_login_name

- name: filter
  enabled: true

- name: queue
  enabled: true

- name: index
  enabled: true
  ui:
    enabled: true
#    site_name: Elasticsearch
#    logo: https://static-www.elastic.co/cn/assets/blt6050efb80ceabd47/elastic-logo (2).svg?q=294
#    favicon: https://www.elastic.co/favicon.ico
  elasticsearch: default

- name: elastic
  enabled: true
  store_enabled: true
  orm_enabled: true
  indexer_enabled: true
  elasticsearch: default

- name: stats
  enabled: true

- name: statsd
  enabled: false
  host: 127.0.0.1
  port: 8125
  namespace: gopa.
  interval_in_seconds: 1


plugins:
- name: chrome
  enabled: false
  command: "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome"
#  command: "/usr/bin/google-chrome"
  debug_port: 9223

- name: dispatch
  enabled: true
  failed_retry: true
  new_task: true
  update_task: true
  max_concurrent_fetch_tasks: 10
  fetch_size: 10
  get_host_config_by_url: false


rpc:
  enabled: true
  tls_enabled: true
  tls_cert_file: test/cert/server.crt
  tls_key_file: test/cert/server.key
#  tls_skip_insecure_verify: false