From 33760847bdb212f5e74e07a4cc55de6c7890fe1d Mon Sep 17 00:00:00 2001 From: Thilo Wischmeyer <thwischm@gmail.com> Date: Sat, 19 Jun 2021 16:43:56 +0200 Subject: [PATCH 1/7] Reduced the MTU on the network bridge used by docker-compose There must have been some change to the network that the CI is running on and it caused HTTPS connection to github to hang forever. Lowering the MTU on the docker network fixes it. I also added this to the /etc/docker/daemon.json on our CI server. --- docker-compose.yml | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 563b5f04..43fef06d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -6,7 +6,10 @@ services: image: postgres:13 restart: always networks: - - default + default: + driver: bridge + driver_opts: + com.docker.network.driver.mtu: 1300 ports: - 6543:5432 @@ -20,7 +23,10 @@ services: - postgres restart: always networks: - - default + default: + driver: bridge + driver_opts: + com.docker.network.driver.mtu: 1300 expose: - "8000" ports: -- GitLab From f6c96a419252babad1af08799bfc8aa8c6a4dda8 Mon Sep 17 00:00:00 2001 From: Thilo Wischmeyer <thwischm@gmail.com> Date: Sat, 19 Jun 2021 17:09:33 +0200 Subject: [PATCH 2/7] Revert "Reduced the MTU on the network bridge used by docker-compose" This reverts commit 33760847bdb212f5e74e07a4cc55de6c7890fe1d. --- docker-compose.yml | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 43fef06d..563b5f04 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -6,10 +6,7 @@ services: image: postgres:13 restart: always networks: - default: - driver: bridge - driver_opts: - com.docker.network.driver.mtu: 1300 + - default ports: - 6543:5432 @@ -23,10 +20,7 @@ services: - postgres restart: always networks: - default: - driver: bridge - driver_opts: - com.docker.network.driver.mtu: 1300 + - default expose: - "8000" ports: -- GitLab From 507ca1268e5dfd6d731953fc9a7442c389106463 Mon Sep 17 00:00:00 2001 From: Thilo Wischmeyer <thwischm@gmail.com> Date: Sat, 19 Jun 2021 17:13:42 +0200 Subject: [PATCH 3/7] Attempt to clear some cache used by docker build when building the test image. --- .gitlab-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b616628a..b48e25fa 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -81,9 +81,9 @@ build_test_image: script: - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY - docker pull $DEV_IMAGE_BASE || true - - docker build --cache-from $DEV_IMAGE_BASE -t $DEV_IMAGE_BASE --target node . + - docker build -t $DEV_IMAGE_BASE --target node . - docker pull $DEV_IMAGE || true - - docker build --cache-from $DEV_IMAGE --cache-from $DEV_IMAGE_BASE -t $DEV_IMAGE . + - docker build -t $DEV_IMAGE . - docker push $DEV_IMAGE_BASE - docker push $DEV_IMAGE tags: -- GitLab From ea6891af7e2e8a49ac72f88f9b46a420e3685f70 Mon Sep 17 00:00:00 2001 From: Thilo Wischmeyer <thwischm@gmail.com> Date: Sat, 19 Jun 2021 17:30:37 +0200 Subject: [PATCH 4/7] Revert "Attempt to clear some cache used by docker build when building the test image." This reverts commit 507ca1268e5dfd6d731953fc9a7442c389106463. --- .gitlab-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b48e25fa..b616628a 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -81,9 +81,9 @@ build_test_image: script: - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY - docker pull $DEV_IMAGE_BASE || true - - docker build -t $DEV_IMAGE_BASE --target node . + - docker build --cache-from $DEV_IMAGE_BASE -t $DEV_IMAGE_BASE --target node . - docker pull $DEV_IMAGE || true - - docker build -t $DEV_IMAGE . + - docker build --cache-from $DEV_IMAGE --cache-from $DEV_IMAGE_BASE -t $DEV_IMAGE . - docker push $DEV_IMAGE_BASE - docker push $DEV_IMAGE tags: -- GitLab From fffb30c0b5cfa5bef34cfc56a4fba55e8c77dea9 Mon Sep 17 00:00:00 2001 From: Thilo Wischmeyer <thwischm@gmail.com> Date: Sun, 20 Jun 2021 13:49:47 +0200 Subject: [PATCH 5/7] Set the mtu in daemon.json inside the script. --- .gitlab-ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b616628a..a9d7b379 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -79,6 +79,8 @@ build_test_image: DOCKER_HOST: tcp://docker:2375/ DOCKER_DRIVER: overlay2 script: + - "echo '{ \"mtu\": 1300 }' > /etc/docker/daemon.json" + - systemctl restart docker - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY - docker pull $DEV_IMAGE_BASE || true - docker build --cache-from $DEV_IMAGE_BASE -t $DEV_IMAGE_BASE --target node . -- GitLab From 1af82947860efb2e9b3df2a63bdfee2cb991ab71 Mon Sep 17 00:00:00 2001 From: Thilo Wischmeyer <thwischm@gmail.com> Date: Sun, 20 Jun 2021 14:03:53 +0200 Subject: [PATCH 6/7] Create the directory first --- .gitlab-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index a9d7b379..f8fea0bd 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -79,6 +79,7 @@ build_test_image: DOCKER_HOST: tcp://docker:2375/ DOCKER_DRIVER: overlay2 script: + - mkdir -p /etc/docker - "echo '{ \"mtu\": 1300 }' > /etc/docker/daemon.json" - systemctl restart docker - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY -- GitLab From 538f74f96d80cbe35e893ffa408d53e7acc64030 Mon Sep 17 00:00:00 2001 From: Thilo Wischmeyer <thwischm@gmail.com> Date: Sun, 20 Jun 2021 14:20:05 +0200 Subject: [PATCH 7/7] Lets try using --network=host --- .gitlab-ci.yml | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index f8fea0bd..a235b565 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -79,14 +79,11 @@ build_test_image: DOCKER_HOST: tcp://docker:2375/ DOCKER_DRIVER: overlay2 script: - - mkdir -p /etc/docker - - "echo '{ \"mtu\": 1300 }' > /etc/docker/daemon.json" - - systemctl restart docker - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY - docker pull $DEV_IMAGE_BASE || true - - docker build --cache-from $DEV_IMAGE_BASE -t $DEV_IMAGE_BASE --target node . + - docker build --network=host --cache-from $DEV_IMAGE_BASE -t $DEV_IMAGE_BASE --target node . - docker pull $DEV_IMAGE || true - - docker build --cache-from $DEV_IMAGE --cache-from $DEV_IMAGE_BASE -t $DEV_IMAGE . + - docker build --network=host --cache-from $DEV_IMAGE --cache-from $DEV_IMAGE_BASE -t $DEV_IMAGE . - docker push $DEV_IMAGE_BASE - docker push $DEV_IMAGE tags: -- GitLab