From docker-compose to AWS

Question

I have a docker-compose.yml:

version: '2'

services:
  scheduler:
    build:
      context: .
      dockerfile: Dockerfile
    hostname: dask-scheduler
    ports:
      - "8786:8786"
      - "8787:8787"
    command: dask-scheduler

  worker:
    build:
      context: .
      dockerfile: Dockerfile
    hostname: dask-worker
    ports:
      - "8789:8789"
    command: dask-worker scheduler:8786

and the Dockerfile:

FROM continuumio/miniconda3

RUN apt-get update && apt-get install -y build-essential freetds-dev

RUN mkdir project

COPY requirements.txt /project/requirements.txt
COPY src/ /project/src
COPY setup.py /project/setup.py
WORKDIR /project
RUN pip install -r requirements.txt
RUN python -m nltk.downloader punkt

I can use it locally by docker-compose up. Next, I want to have the same composition on multiple EC2 nodes. So, I tried to use ecs-cli but for no success. I followed the instructions but so far with no success. The error I get is:

WARN[0000] Skipping unsupported YAML option...           option name=networks
WARN[0000] Skipping unsupported YAML option for service...  option name=build service name=scheduler
WARN[0000] Skipping unsupported YAML option for service...  option name=networks service name=scheduler
WARN[0000] Skipping unsupported YAML option for service...  option name=build service name=worker
WARN[0000] Skipping unsupported YAML option for service...  option name=networks service name=worker
ERRO[0000] Error registering task definition             error="ClientException: Container.image should not be null or empty.\n\tstatus code: 400, request id: 1966ba7f-dbf6-11e7-a974-15857d851822" family="cs_emails"
ERRO[0000] Create task definition failed                 error="ClientException: Container.image should not be null or empty.\n\tstatus code: 400, request id: 1966ba7f-dbf6-11e7-a974-15857d851822"
FATA[0000] ClientException: Container.image should not be null or empty.
    status code: 400, request id: 1966ba7f-dbf6-11e7-a974-15857d851822

To be honest I don't know how to get around this. Any idea?

Edit: Trying to push the image to ECR

As suggested in this answer I built the image and pushed it to ECR. In addition I changed the yml:

version: '2'

services:
  scheduler:
    image: xxxx.dkr.ecr.eu-west-1.amazonaws.com/dask-poc
    hostname: dask-scheduler
    ports:
      - "8786:8786"
      - "8787:8787"
    command: dask-scheduler

  worker:
    # build:
    #   context: .
    image: xxxx.dkr.ecr.eu-west-1.amazonaws.com/dask-poc
    hostname: dask-worker
    ports:
      - "8789:8789"
    command: dask-worker scheduler:8786

Now, ecs-cli compose --file docker-compose.yml --cluster dask-poc-cluster --verbose up yields:

INFO[0000] Using ECS task definition                     TaskDefinition="dir_name_of_projec:1"
ERRO[0000] Error running tasks                           error="InvalidParameterException: No Container Instances were found in your cluster.\n\tstatus code: 400, request id: 809171a2-dc02-11e7-bb31-a98d8b09e45a" task definition="arn:aws:ecs:eu-west-1:345891532217:task-definition/dir_name_of_project:1"
FATA[0000] InvalidParameterException: No Container Instances were found in your cluster.
    status code: 400, request id: 809171a2-dc02-11e7-bb31-a98d8b09e45a

Answer 1

you must use docker compose v3 or stack file to use compose in nodes environment.