docker compose on multiple environments

docker-compose is a pretty cool tool that allows you to bootstrap and run multiple docker containers with 1 configuration file

Below is an example docker-compose file for staring a 3 node confluent kafka cluster

---
version: '2'
services:
  zookeeper:
    image: confluentinc/cp-zookeeper:latest
    network_mode: host
    environment:
      ZOOKEEPER_CLIENT_PORT: 32181
      ZOOKEEPER_TICK_TIME: 2000
    extra_hosts:
      - "moby:127.0.0.1"
    restart: always

  kafka-1:
    image: confluentinc/cp-kafka:latest
    network_mode: host
    depends_on:
      - zookeeper
    environment:
      KAFKA_BROKER_ID: 1
      KAFKA_ZOOKEEPER_CONNECT: localhost:32181
      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://myserver.com:29092
    restart: always
    extra_hosts:
      - "moby:127.0.0.1"

  kafka-2:
      image: confluentinc/cp-kafka:latest
      network_mode: host
      depends_on:
        - zookeeper
      environment:
        KAFKA_BROKER_ID: 2
        KAFKA_ZOOKEEPER_CONNECT: localhost:32181
        KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://myserver.com:29093
      restart: always
      extra_hosts:
        - "moby:127.0.0.1"

  kafka-3:
        image: confluentinc/cp-kafka:latest
        network_mode: host
        depends_on:
          - zookeeper
        environment:
          KAFKA_BROKER_ID: 3
          KAFKA_ZOOKEEPER_CONNECT: localhost:32181
          KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://myserver.com:29094
        restart: always
        extra_hosts:
          - "moby:127.0.0.1"

  kafka-rest:
    image: confluentinc/cp-kafka-rest:latest
    network_mode: host
    depends_on:
      - kafka-1
      - kafka-2
      - kafka-3
    restart: always
    environment:
      KAFKA_REST_ZOOKEEPER_CONNECT: localhost:32181
      KAFKA_REST_LISTENERS: http://0.0.0.0:8082
      KAFKA_REST_HOST_NAME: kafka-rest
    extra_hosts:
      - "moby:127.0.0.1"

---
version: '2'
services:
  zookeeper:
    image: confluentinc/cp-zookeeper:latest
    network_mode: host
    environment:
      ZOOKEEPER_CLIENT_PORT: 32181
      ZOOKEEPER_TICK_TIME: 2000
    extra_hosts:
      - "moby:127.0.0.1"
    restart: always

  kafka-1:
    image: confluentinc/cp-kafka:latest
    network_mode: host
    depends_on:
      - zookeeper
    environment:
      KAFKA_BROKER_ID: 1
      KAFKA_ZOOKEEPER_CONNECT: localhost:32181
      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://test.myserver.com:29092
    restart: always
    extra_hosts:
      - "moby:127.0.0.1"

  kafka-2:
      image: confluentinc/cp-kafka:latest
      network_mode: host
      depends_on:
        - zookeeper
      environment:
        KAFKA_BROKER_ID: 2
        KAFKA_ZOOKEEPER_CONNECT: localhost:32181
        KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://test.myserver.com:29093
      restart: always
      extra_hosts:
        - "moby:127.0.0.1"

  kafka-3:
        image: confluentinc/cp-kafka:latest
        network_mode: host
        depends_on:
          - zookeeper
        environment:
          KAFKA_BROKER_ID: 3
          KAFKA_ZOOKEEPER_CONNECT: localhost:32181
          KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://test.myserver.com:29094
        restart: always
        extra_hosts:
          - "moby:127.0.0.1"

  kafka-rest:
    image: confluentinc/cp-kafka-rest:latest
    network_mode: host
    depends_on:
      - kafka-1
      - kafka-2
      - kafka-3
    restart: always
    environment:
      KAFKA_REST_ZOOKEEPER_CONNECT: localhost:32181
      KAFKA_REST_LISTENERS: http://0.0.0.0:8082
      KAFKA_REST_HOST_NAME: kafka-rest
    extra_hosts:
      - "moby:127.0.0.1"

You can now run docker-compose by with a second -f and the redefined properties in the test file will ovewrite those in the base file

docker-compose -f docker-compose.yml -f docker-compose.test.yml up -d

More information regarding docker compose advanced configuration can be found here