# Copyright 2018-present Open Networking Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# This file provides a template of all available configurations in Atomix.
# All options can be configured via *.conf, *.json, or *.properties files.
# Individual options can be overridden in system properties as well.
# The option values displayed here are the current defaults for each option.

# The cluster configuration specifies how the Atomix node locates and communicates with peers in the Atomix cluster.
cluster {
  # The unique cluster identifier. This must match on all nodes to successfully form a cluster.
  # This option does not typically need to be changed.
  clusterId: atomix

  # The node object defines the local node information. This is a required object that must be supplied
  # either via configuration files or via system properties or command line arguments.
  node {
    # The node identifier is an optional unique identifier that can be used to communicate with this node. The node
    # ID is also used in the Raft protocol to identify Raft partition group members. If no node ID is specified,
    # a universally unique identifier (UUID) will be generated. Note that for certain protocols (e.g. Raft) a fixed
    # node identifier is required.
    id: (uuid)

    # The host is the hostname for the local node to broadcast to peers. This should be the IP or hostname
    # through which peers can connect to this node.
    host: localhost

    # The port is the port number for the local node to broadcast to peers. This should be the port through
    # which peers can connect to this node. Defaults to 5679.
    port: 5679

    # The address is the address through which other nodes can reach this node for cluster membership and general
    # communication. The address is a host:port tuple and supports DNS lookups.
    address: "localhost:5679"

    # The 'zoneId' is metadata that may be used for zone-aware partitioning in certain protocols.
    zoneId: null

    # The 'rackId' is metadata that may be used for zone-aware partitioning in certain protocols.
    rackId: null

    # The 'hostId' is metadata that may be used for zone-aware partitioning in certain protocols.
    hostId: null

    # Properties is an arbitrary mutable mapping of node properties that will be replicated to all peers.
    properties {

    }
  }

  # Node discovery is an extensible mechanism by which the node joins the Atomix cluster.
  # This is a multicast based discovery configuration. When multicast discovery is used, the node will broadcast
  # its information via the provided multicast group to discover other nodes.
  discovery {
    # The 'multicast' type indicates the discovery provider to use for node discovery.
    type: multicast

    # The broadcast interval is the rate at which the node advertises its existence via multicast.
    # The format allows the interval to be specified in ms, s, m, h, d, etc.
    broadcastInterval: 1s
  }

  # Alternatively, node discovery can be done via the bootstrap discovery provider. The bootstrap provider works by
  # joining a pre-defined list of peers via TCP to discover the complete set of nodes in the cluster.
  discovery {

    # The 'bootstrap' type indicates the discovery provider to use for node discovery.
    type: bootstrap

    # The list of 'nodes' is a pre-defined set of peers to which to connect via the TCP-based MessagingService to request
    # information about the set of nodes in the cluster. At least one of the provided peers must be reachable in order
    # for the node to successfully join the cluster.
    nodes.1 {
      id: atomix-1
      host: 10.192.19.171
      port: 5679
    }
    nodes.2 {
      id: atomix-2
      host: 10.192.19.172
      port: 5679
    }
    nodes.3 {
      id: atomix-3
      host: 10.192.19.173
      port: 5679
    }
  }

  # The 'dns' discovery protocol uses DNS SRV records to locate peers.
  discovery {

    # The 'dns' type enables the DNS protocol.
    type: dns

    # The 'service' indicates the SRV record name to lookup. This is a required property.
    service: atomix

    # The resolution interval indicates the interval at which to attempt to resolve DNS SRV records.
    # The format allows the interval to be specified in ms, s, m, h, d, etc.
    resolutionInterval: 15s
  }

  # Cluster membership protocols dictate how membership information is replicated and failures are detected.
  # The protocol is defined by a 'type' name, and each protocol has specific configuration options (below).
  protocol {
    type: (type)
  }

  # The 'heartbeat' protocol uses simple periodic heartbeats to all peers and phi-accrual failure detectors to
  # detect and replicate changes to the cluster membership.
  protocol {
    # The 'heartbeat' type enables the heartbeat membership protocol.
    type: heartbeat

    # The 'heartbeatInterval' defines the interval at which to send heartbeats to all peers.
    # The format allows the interval to be specified in ms, s, m, h, d, etc.
    heartbeatInterval: 1s

    # The 'phiFailureThreshold' is the phi value threshold at which to consider a peer dead.
    # Typical values range from 8 to 12.
    phiFailureThreshold: 10

    # The 'failureTimeout' is the maximum amount of time that may pass without hearing from a peer before
    # marking it dead. This is used when not enough heartbeats have been recorded to reliably detect
    # failures using the phi accrual algorithm.
    failureTimeout: 10s
  }

  # The 'swim' protocol
  protocol {
    # The 'swim' type enables the SWIM membership protocol.
    type: swim

    # Indicates whether to immediately broadcast membership updates to all members. Enabling this option may
    # increase network traffic but reduce the time it takes to propagate membership changes.
    broadcastUpdates: false

    # Indicates whether to immediately broadcast disputes to all members. Enabling this option may increase
    # network traffic but reduce the time it takes to propagate membership changes.
    broadcastDisputes: true

    # Indicates whether to immediately notify suspected members on membership updates. Enabling this option
    # will increase network traffic but may avoid false positives in failure detectors.
    notifySuspect: false

    # Defines the interval at which to gossip pending updates with a random set of peers.
    # The format allows the interval to be specified in ms, s, m, h, d, etc.
    gossipInterval: 250ms

    # Defines the number of peers with which to gossip pending updates on each round.
    gossipFanout: 2

    # Defines the interval at which to attempt to probe a random peer for failure detection.
    # The format allows the interval to be specified in ms, s, m, h, d, etc.
    probeInterval: 1s

    # Defines the timeout for probe messages.
    probeTimeout: 2s

    # Indicates the number of probe attempt failures that must occur before marking a member suspect.
    suspectProbes: 3

    # The time to allow a suspect member to refute the suspicion before declaring it dead.
    # The format allows the interval to be specified in ms, s, m, h, d, etc.
    failureTimeout: 10s
  }

  # The messaging configuration specifies how the local node communicates with its peers.
  messaging {
    # The interfaces array lists the local interfaces to which to bind the Atomix messaging service. These interfaces
    # may differ from the 'address' broadcast to other nodes in the cluster e.g. in containerized environments where
    # the internal container IP may differ from the host IP.
    interfaces: ["0.0.0.0"]

    # The bind port indicates the port to which to bind the node.
    port: null

    # The number of connections per peer. Persistent connections will typically only be created for connections
    # between a client and server node primitive client and all nodes in the primitive's partition group). Nodes
    # that do not communicate with each other directly will not maintain persistent connections to one another.
    connectionPoolSize: 8

    # The timeout for TCP connection attempts to other nodes.
    # The format allows the interval to be specified in ms, s, m, h, d, etc.
    connectTimeout: 10s

    # The messaging TLS configuration.
    tls {
      # Whether to enable TLS for the messaging service.
      enabled: false

      # The key store path and password, optionally specified by system properties.
      keyStore: conf/atomix.jks
      keyStore: ${?javax.net.ssl.keyStore}
      keyStorePassword: changeit
      keyStorePassword: ${?javax.net.ssl.keyStorePassword}

      # The trust store path and password, optionally specified by system properties.
      trustStore: conf/atomix.jks
      trustStore: ${?javax.net.ssl.trustStore}
      trustStorePassword: changeit
      trustStorePassword: ${?javax.net.ssl.trustStorePassword}
    }
  }

  # The multicast object defines the configuration for the multicast-based BroadcastService.
  # Multicast must be explicitly enabled in order to use multicast node discovery.
  multicast {

    # To enable multicast, set this value to true.
    enabled: true

    # The multicast group to be used by the BroadcastService when multicast is enabled..
    group: 230.0.0.1

    # The multicast port to be used by the BroadcastService when multicast is enabled.
    port: 54321
  }
}

# The `managementGroup` is a group of partitions used for managing primitives, configuration, and transactions.
# It must be configured on at least one discoverable node in the Atomix cluster.

# A management group using the Raft consensus protocol.
managementGroup {
  # The 'raft' type indicates that the Raft protocol should be used for replication.
  type: raft

  # The number of partitions indicates the number of distinct Raft clusters to use in the group.
  partitions: 1

  # The partition size is the number of nodes in each Raft partition. This should be an odd
  # number, typically 3, 5, or 7. The partition size must be less than the number of
  # 'members' in the group. The default value 0 sizes the partitions equal to the number
  # of 'members' in the group.
  partitionSize: 0

  # The 'members' is a list of member IDs for all members participating in the group.
  # Fixed member identifiers are necessary to preserve the safety of the Raft consensus algorithm.
  members: [atomix-1]

  # The 'storage' object configures how Raft partitions store data.
  storage {
    # The directory in which to store Raft configuration, logs, and snapshots.
    # The default storage directory for all Raft partition groups is the atomix.data system property
    # plus the partition group name.
    directory: ${atomix.data}/system

    # The type of storage to use for Raft logs, either 'disk' or 'mapped' for memory mapped files.
    level: disk

    # The maximum size of each entry in the Raft log. Increasing this value will increase the amount
    # of memory used by each partition but will also allow larger primitive writes.
    # This value is specified in memory size format, e.g. 100KB, 10MB, etc.
    maxEntrySize: 1MB

    # The maximum size of each segment in the Raft log. Increasing the size may reduce the frequency
    # of service snapshots but reduces the ability of the Raft partitions to conserve disk space.
    # This value is specified in memory size format, e.g. 100KB, 10MB, etc.
    maxSegmentSize: 32MB

    # Whether to flush Raft logs to disk on every commit. Enabling this option ensures durability for
    # every write to Raft-based primitives but will increase the latency of Raft partitions. Disabling
    # this option risks losing some recent writes after a majority of a Raft partition is lost.
    flushOnCommit: false
  }

  # The 'compaction' configuration is used by Raft partitions to determine when to free disk space
  # consumed by Raft logs and snapshots.
  compaction {
    # Whether to enable dynamic compaction, allowing Raft partitions to optionally skip compaction
    # during periods of high load.
    dynamic: true

    # A percentage value of free disk space to require before Raft partitions will force compact logs.
    freeDiskBuffer: .2

    # A percentage value of free memory to require before Raft partitions will force compact logs.
    freeMemoryBuffer: .2
  }
}

# 'partitionGroups' is a mapping of named partition groups in which the node participates to
# replicate distributed primitives. Nodes may participate in any number of partition groups,
# and groups are identified by protocols when primitives are constructed.

# A primary-backup (data grid) partition group.
partitionGroups.data {
  # The 'primary-backup' type indicates the primary-backup protocol should be used for configuration
  # and replication.
  type: primary-backup

  # The number of instances of the primary-backup protocol to run in the partition group. Partitions
  # will be evenly distributed among all the nodes participating in the group. More partitions increases
  # concurrency and improves the distribution of state in the cluster but also comes with overhead.
  partitions: 71

  # Defines how partitions are distributed among the cluster members. Backups for a partition will be
  # distributed first among distinct groups. For example, to distribute backups among nodes on different
  # racks, configure the 'cluster.node.rackId' for each node and use the 'rack-aware' strategy.
  # The value may be one of 'node-aware', 'host-aware', 'rack-aware', or 'zone-aware'.
  memberGroupStrategy: node-aware
}

# A distribtued log partition group.
partitionGroups.log {
  # the 'log' type indicates the distributed log protocol should be used for replication.
  type: log

  # Defines how partitions are distributed among the cluster members. Replicas for a partition will be
  # distributed first among distinct groups. For example, to distribute replicas among nodes on different
  # racks, configure the 'cluster.node.rack' for each node and use the 'rack-aware' strategy.
  # The value may be one of 'node-aware', 'host-aware', 'rack-aware', or 'zone-aware'.
  memberGroupStrategy: node-aware

  # The storage configuration defines how and where the distributed log is stored on this node.
  storage {
    # The directory in which to store the distributed log on this node.
    # The default storage directory for all distributed log partition groups is the atomix.data
    # system property plus the partition group name.
    directory: ${atomix.data}/log

    # The type of storage to use for distributed logs, either 'disk' or 'mapped' for memory mapped files.
    level: mapped

    # The maximum size of each entry in the distributed log. Increasing this value will increase the amount
    # of memory used by each partition but will also allow larger primitive writes.
    # This value is specified in memory size format, e.g. 100KB, 10MB, etc.
    maxEntrySize: 1MB

    # The size of each segment in the distributed log. Increasing the size may reduce the frequency
    # of log compaction but reduces the ability of the log partitions to conserve disk space.
    # This value is specified in memory size format, e.g. 100KB, 10MB, etc.
    segmentSize: 32MB

    # Whether to flush logs to disk on every commit. Enabling this option ensures durability for
    # every write to the distributed log but will increase the latency of log partitions.
    flushOnCommit: false
  }

  # The compaction configuration defines how long this node should retain persisted logs.
  compaction {
    # When size-based compaction is non-null, logs will be retained on this node until the given
    # size is exceeded by all segments. Once exceeded, the oldest segments will be deleted from
    # disk until the given size is retained. The real size of logs on disk can be up to
    # compaction.size plus storage.segmentSize.
    size: 1GB

    # When age-based compaction is non-null, log segments that are older than the given age will
    # be deleted from disk. The age can be specified in human readable format with a time unit,
    # e.g. 1D or 12H.
    age: null
  }
}

# Primitive defaults allow configuration to override the programmatic default values for primitives by type.
# The 'primitiveDefaults' map is keyed by primitive type names; values are the default configuration options
# for the primitive type.
primitiveDefaults {
  // Example:
  lock {
    protocol {
      type: multi-raft
      readConsistency: linearizable
    }
  }
}

# Specific named primitives may be configured via the 'primitives' object. When a primitive
# is constructed programmatically, the default configuration will be read from the 'primitives' object.

# An example primitive configuration using a Raft partition group.
primitives.foo {
  # The primitive type
  type: map

  # The protocol to use to replicate the primitive
  protocol {
    # The 'multi-raft' protocol indicates that a Raft partition group must be used to replicate the primitive.
    type: multi-raft

    # The 'group' indicates the name of the partition group in which to replicate the primitive.
    # The configured partition group must support the protocol indicated in 'type' above.
    # If no partition group is configured, a single partition group of the given type must be present
    # If multiple groups of the multi-raft type are present an exception will be thrown on primitive creation.
    group: null

    # The approximate minimum Raft session timeout. Session timeouts are used to detect client failures,
    # e.g. to release a lock primitive or elect a new leader.
    minTimeout: 250ms

    # The approximate maximum Raft session timeout. Session timeouts are used to detect client failures,
    # e.g. to release a lock primitive or elect a new leader.
    maxTimeout: 30s

    # The read consistency indicates the consistency guarantee of reads on a Raft partition.
    # 'sequential' reads guarantee state will not go back in time but do not provide a real-time guarantee
    # 'linearizable-lease' reads guarantee linearizability assuming clock accuracy
    # 'linearizable' guarantees linearizable reads by verifying the Raft leader
    readConsistency: sequential

    # The communication strategy indicates the node(s) to which the primitive should communicate in each partition.
    # 'leader' indicates the primitive should communicate directly with the Raft leader
    # 'followers' indicates the primitive should favor Raft followers
    # 'any' indicates the primitive can communicate with any node in each partition
    communicationStrategy: leader

    # The strategy to use when a primitive session has been expired. The 'recover' strategy seamlessly opens
    # a new session. May be one of 'recover' or 'close'.
    recoveryStrategy: recover

    # The maximum number of retries to perform per operation. Note that retries break program order guarantees.
    maxRetries: 0

    # The delay between each operation retry. This may be specified in human readable format with a time unit,
    # e.g. 100ms or 5s.
    retryDelay: 100ms
  }
}

# An example primitive configuration using a primary-backup partition group.
primitives.bar {
  # The primitive type
  type: set

  # The protocol to use to replicate the primitive
  protocol {
    # The 'multi-primary' protocol indicates that a primary-backup partition group must be used to replicate the primitive.
    type: multi-primary

    # The 'group' indicates the name of the partition group in which to replicate the primitive.
    # The configured partition group must support the protocol indicated in 'type' above.
    # If no partition group is configured, a single partition group of the given type must be present
    # If multiple groups of the 'primary-backup' type are present an exception will be thrown on primitive creation.
    group: null

    # The 'consistency' is a primary-backup specific setting indicating whether the primitive should communicate
    # with the primary or backups on reads.
    consistency: sequential

    # The 'replication' is a primary-backup specific setting indicating whether replication should be
    # 'synchronous' or 'asynchronous'
    replication: asynchronous

    # The 'backups' indicates the number of copies to replicate in addition to the primary copy.
    # In other words, 2 backups means the primitive will be replicated on 3 nodes in each partition.
    backups: 1

    # 'maxRetries' is the maximum number of attempts to allow for any read or write.
    # Note that retries break program order guarantees.
    maxRetries: 0

    # The 'retryDelay' is the time to wait between retries.
    # This may be specified in human readable format with a time unit, e.g. 100ms or 5s.
    retryDelay: 100ms
  }
}

# An example primitive configuration using a distributed log partition group.
# When a distributed log partition group is used for a service-based primitive (other than
# the log primitive), the primitive service will be maintained by the client and consumers
# reading from the (potentially remote) distributed log will populate the local primitive state.
primitives.baz {
  # The primitive type.
  type: multimap

  # The protocol to use to replicate the primitive.
  protocol {
    # The 'multi-log' protocol indicates that a distributed log partition group must be used to replicate the primitive.
    type: multi-log

    # The 'group' indicates the name of the partition group in which to replicate the primitive.
    # The configured partition group must support the protocol indicated in 'type' above.
    # If no partition group is configured, a single partition group of the given type must be present
    # If multiple groups of the 'log' type are present an exception will be thrown on primitive creation.
    group: null

    # The 'consistency' option indicates whether the primitive should communicate with the leader
    # or followers of each partition when consuming logs.
    consistency: sequential

    # The 'replication' option indicates whether log replication for the primitive should be
    # 'synchronous' or 'asynchronous'
    replication: asynchronous

    # The strategy to use when a primitive session has been expired. The 'recover' strategy seamlessly opens
    # a new session. May be one of 'recover' or 'close'.
    recovery: recover

    # 'maxRetries' is the maximum number of attempts to allow for any read or write.
    # Note that retries break program order guarantees.
    maxRetries: 0

    # The 'retryDelay' is the time to wait between retries.
    # This may be specified in human readable format with a time unit, e.g. 100ms or 5s.
    retryDelay: 100ms
  }
}

# An example distributed log primitive.
primitives.log {
  # The primitive type.
  type: log

  # The protcol to use to replicate the primitive.
  protocol {
    # The 'multi-log' protocol indicates that a distributed log partition group must be used to replicate the primitive.
    type: multi-log

    # ... (see primitives.baz)
  }
}