obmp-psql.yml

# Openbmp PostgreSQL consumer configuration

base:

  # Interval at which the customer statistics should be printed/logged
  stats_interval: 300


  # In minutes; The maximum age between received collector heartbeats.
  #     The collector sends heatbeat messages every interval.  This
  #     is how the collector is detected as being up or down.  This max
  #     age is the maximum allowed time before declaring the collector down.
  heartbeat_max_age: 6

  # Number of writer threads per processing type.
  #     The number of threads and psql connections are
  #     [types * writer_max_threads_per_type]. Each writer makes
  #     a connection to psql in order to execute SQL statements in parallel.
  #     The number of threads are auto-scaled up and down based on partition
  #     load.  If there is high load, additional threads will be added, up
  #     to the writer_max_threads_per_type.
  #
  #  Following types are implemented.
  #   - Default
  #   - base attributes
  writer_max_threads_per_type: 6

  # Number of consecutive times the writer queue (per type) can sustain over
  #    the high threshold mark.  If queue is above threshold for a consecutive
  #    writer_allowed_over_queue_times value, a new thread will be added for
  #    the writer type, providing it isn't already at max threads (per type).
  writer_allowed_over_queue_times: 2

  # Number of seconds the writer needs to sustain below the low queue threshold mark
  #     in order to trigger scaling back the number of threads in use.  Only one thread
  #     is scaled back a time.  It can take
  #     [writer_seconds_thread_scale_back * writer_max_threads_per_type - 1] time to scale
  #     back to one 1 (per type).
  writer_seconds_thread_scale_back: 4800

  # Number of seconds between rebalacing of writer threads
  #    Rebalance will drain writer queues at this interval if at least one writer is above threshold
  writer_rebalance_seconds: 600

  # Maximum input queue size
  #   Normally within the range of 10000 - 50000 is enough
  writer_queue_size: 30000

  # Maximum input queue size for the consumer
  #   A good starting size is 2 times the size of the writer queue size
  consumer_queue_size: 60000

  # By default AS Path indexing is enabled.  This can be very resource intensive to psql
  #    and disk.   You can disable updating the psql 'as_path_analysis' table if
  #    this analysis is not needed/used.  Disabling this will effect the gen_asn_stats table,
  #    which will also effect anything that uses that table.
  disable_as_path_indexing: false

postgres:
  host    : "localhost:5432"
  db_name : "openbmp"
  username: "openbmp"
  password: "openbmp"


  # The number of statements or records to batch in a single bulk update/insert/delete
  #   NOTE: It's more efficient to have more threads with a low batch size
  batch_records: 300

  # The time in milliseconds to wait for batching records in a bulk update/insert/delete
  batch_time_millis: 100

  # The number of times to retry a statement
  retries: 10

kafka:

  # Consumer configuration per https://kafka.apache.org/0102/documentation.html#consumerconfigs
  #   Any and all consumer configs are supported.  Simply add them as they are documented.
  #   Values should always be strings.    Integers will be converted to strings. If config
  #   uses boolean value, make sure to double quote "true" and "false" so that they are
  #   strings, not of type boolean.
  consumer_config:
    bootstrap.servers: "localhost:9092"

    group.id  : "obmp-psql-consumer"
    client.id : "obmp-psql-consumer"

    session.timeout.ms: 15000
    heartbeat.interval.ms: 5000
    max.poll.interval.ms: 300000
    auto.offset.reset: "earliest"
    max.partition.fetch.bytes: 2000000
    max.poll.records: 1000
    fetch.max.wait.ms: 50

    # For TLS/SSL config follow Kafka consumer configuration guide for setting up SSL/TLS.

  # Delay in milliseconds between subscribing to topic patterns
  #    Delay is introduced between each pattern when subscribing
  topic_subscribe_delay_millis: 15000

  # List of topic patterns to subscribe to.  Pattern is always a regex, but exact
  #     topic name can be specified.
  #
  #     It's import to subscribe to the inventory topics before the NLRI topics, for
  #     example router and peer, so that peer level state changes are handled first
  #     when starting the consumer.  This is why we introduce a delay between subscriptions.
  subscribe_topic_patterns:
    - "openbmp[.]parsed[.]collector"
    - "openbmp[.]parsed[.]router"
    - "openbmp[.]parsed[.]peer"
    #- "openbmp[.]parsed[.]ls.*"
    #- "openbmp[.]parsed[.]bmp_stat"
    - "openbmp[.]parsed[.]base_attribute"
    #- "openbmp[.]parsed[.]l3vpn"
    - "openbmp[.]parsed[.]unicast_prefix"