From 05468851f34e3813c0f75e49ec9799fce2462d63 Mon Sep 17 00:00:00 2001 From: Ubbo Veentjer <veentjer@sub.uni-goettingen.de> Date: Tue, 5 Oct 2021 21:49:11 +0200 Subject: [PATCH 1/9] enable container-, dependency- and secret scanning in ci --- .gitlab-ci.yml | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 0d4d497..c09d86a 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -3,6 +3,11 @@ image: docker:19.03.0 services: - docker:19.03.0-dind +include: + - template: Dependency-Scanning.gitlab-ci.yml + - template: Security/Container-Scanning.gitlab-ci.yml + - template: Security/Secret-Detection.gitlab-ci.yml + variables: CONTAINER_IMAGE: $CI_REGISTRY_IMAGE/$CI_COMMIT_REF_SLUG:$CI_COMMIT_SHA CONTAINER_IMAGE_LATEST: $CI_REGISTRY_IMAGE/$CI_COMMIT_REF_SLUG:latest @@ -16,6 +21,7 @@ before_script: stages: - build + - test - deploy package: @@ -23,6 +29,13 @@ package: script: - docker build --target production -t $CONTAINER_IMAGE -f compose/django/Dockerfile . - docker push $CONTAINER_IMAGE + except: + - postgres-image + +tag_dev_image: + stage: deploy + script: + - docker pull $CONTAINER_IMAGE - docker tag $CONTAINER_IMAGE $CONTAINER_IMAGE_LATEST - docker push $CONTAINER_IMAGE_LATEST except: @@ -30,10 +43,11 @@ package: - tags - postgres-image -package_release: - stage: build +tag_release_image: + stage: deploy script: - - docker build --target production -t $CONTAINER_RELEASE_IMAGE -f compose/django/Dockerfile . + - docker pull $CONTAINER_IMAGE + - docker tag $CONTAINER_IMAGE $CONTAINER_RELEASE_IMAGE_LATEST - docker push $CONTAINER_RELEASE_IMAGE - docker tag $CONTAINER_RELEASE_IMAGE $CONTAINER_RELEASE_IMAGE_LATEST - docker push $CONTAINER_RELEASE_IMAGE_LATEST -- GitLab From 2fe4abeb4f3714549803b51beffa3127ea4bb3fb Mon Sep 17 00:00:00 2001 From: Ubbo Veentjer <veentjer@sub.uni-goettingen.de> Date: Tue, 5 Oct 2021 21:57:01 +0200 Subject: [PATCH 2/9] no docker command in gitlab sec containers --- .gitlab-ci.yml | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index c09d86a..22cce48 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -16,7 +16,7 @@ variables: POSTGRES_IMAGE: $CI_REGISTRY_IMAGE/$CI_COMMIT_REF_SLUG:$CI_COMMIT_SHA POSTGRES_IMAGE_TAG: $CI_REGISTRY_IMAGE/$CI_COMMIT_REF_SLUG:latest -before_script: +.docker-setup_template: &docker-setup - docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN $CI_REGISTRY stages: @@ -26,6 +26,8 @@ stages: package: stage: build + before_script: + - *docker-setup script: - docker build --target production -t $CONTAINER_IMAGE -f compose/django/Dockerfile . - docker push $CONTAINER_IMAGE @@ -34,6 +36,8 @@ package: tag_dev_image: stage: deploy + before_script: + - *docker-setup script: - docker pull $CONTAINER_IMAGE - docker tag $CONTAINER_IMAGE $CONTAINER_IMAGE_LATEST @@ -45,6 +49,8 @@ tag_dev_image: tag_release_image: stage: deploy + before_script: + - *docker-setup script: - docker pull $CONTAINER_IMAGE - docker tag $CONTAINER_IMAGE $CONTAINER_RELEASE_IMAGE_LATEST @@ -56,6 +62,8 @@ tag_release_image: build_postgres-image: stage: build + before_script: + - *docker-setup script: - docker build -t $POSTGRES_IMAGE -f compose/postgres/Dockerfile . - docker push $POSTGRES_IMAGE @@ -64,6 +72,8 @@ build_postgres-image: deploy_postgres-image: stage: deploy + before_script: + - *docker-setup script: - docker pull $POSTGRES_IMAGE - docker tag $POSTGRES_IMAGE $POSTGRES_IMAGE_TAG -- GitLab From 241d319999470a420fac82bbaf7462306806740a Mon Sep 17 00:00:00 2001 From: Ubbo Veentjer <veentjer@sub.uni-goettingen.de> Date: Tue, 5 Oct 2021 22:16:04 +0200 Subject: [PATCH 3/9] bullseye! --- compose/django/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/compose/django/Dockerfile b/compose/django/Dockerfile index ce13725..4df65bf 100644 --- a/compose/django/Dockerfile +++ b/compose/django/Dockerfile @@ -1,7 +1,7 @@ ### # The builder ### -FROM python:3.7-slim-buster as builder +FROM python:3.9.7-slim-bullseye as builder RUN apt-get update -y && apt-get upgrade -y && apt-get install --no-install-recommends -y \ gettext \ @@ -28,7 +28,7 @@ RUN git clone --depth 1 https://github.com/rdmorganiser/rdmo-catalog.git /rdmo-c ### # The base image ### -FROM python:3.7-slim-buster as base +FROM python:3.9.7-slim-bullseye as base RUN apt-get update -y && apt-get upgrade -y && apt-get install --no-install-recommends -y \ libpq5 \ -- GitLab From 486dd52a016bdb39245002b9a8cf5046276c66ae Mon Sep 17 00:00:00 2001 From: Ubbo Veentjer <veentjer@sub.uni-goettingen.de> Date: Tue, 5 Oct 2021 22:59:14 +0200 Subject: [PATCH 4/9] get dep scanning up? --- .gitlab-ci.yml | 4 +++- requirements-for-scanner.txt | 4 ++++ 2 files changed, 7 insertions(+), 1 deletion(-) create mode 100644 requirements-for-scanner.txt diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 22cce48..5b32e93 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -16,6 +16,9 @@ variables: POSTGRES_IMAGE: $CI_REGISTRY_IMAGE/$CI_COMMIT_REF_SLUG:$CI_COMMIT_SHA POSTGRES_IMAGE_TAG: $CI_REGISTRY_IMAGE/$CI_COMMIT_REF_SLUG:latest + # test if we get it to work with concat file from https://github.com/rdmorganiser/rdmo-app/tree/master/requirements + PIP_REQUIREMENTS_FILE: "requirements-for-scanner.txt" + .docker-setup_template: &docker-setup - docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN $CI_REGISTRY @@ -80,4 +83,3 @@ deploy_postgres-image: - docker push $POSTGRES_IMAGE_TAG only: - postgres-image - diff --git a/requirements-for-scanner.txt b/requirements-for-scanner.txt new file mode 100644 index 0000000..3f6c6c4 --- /dev/null +++ b/requirements-for-scanner.txt @@ -0,0 +1,4 @@ +rdmo==1.6 +psycopg2==2.8.6 +gunicorn>=19.9 +git+https://github.com/Brown-University-Library/django-shibboleth-remoteuser.git -- GitLab From 5b9e796d4f122cbe3eae6c8053c72cc546757161 Mon Sep 17 00:00:00 2001 From: Ubbo Veentjer <veentjer@sub.uni-goettingen.de> Date: Wed, 10 Nov 2021 11:53:58 +0100 Subject: [PATCH 5/9] unrelated commit, should not change the docker image! --- docker-compose.prod.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docker-compose.prod.yaml b/docker-compose.prod.yaml index 968ac7d..46d1dda 100644 --- a/docker-compose.prod.yaml +++ b/docker-compose.prod.yaml @@ -9,7 +9,8 @@ services: django: build: context: . - dockerfile: ./compose/django/Dockerfile.production + dockerfile: ./compose/django/Dockerfile + target: production image: rdmo_local_django depends_on: - postgres -- GitLab From 347bd9d7f0b9bc00ca97871e1244bc928c062e62 Mon Sep 17 00:00:00 2001 From: Ubbo Veentjer <veentjer@sub.uni-goettingen.de> Date: Wed, 18 May 2022 20:07:07 +0200 Subject: [PATCH 6/9] kaniko and crane for postgres image --- .gitlab-ci.yml | 40 +++++++++++++++++----------------------- 1 file changed, 17 insertions(+), 23 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index d1ec501..c647c56 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -8,12 +8,8 @@ variables: CONTAINER_RELEASE_IMAGE: $CI_REGISTRY_IMAGE:$CI_COMMIT_TAG POSTGRES_BASE_IMAGE_VERSION: '11.13-bullseye' POSTGRES_IMAGE: $CI_REGISTRY_IMAGE/$CI_COMMIT_REF_SLUG:$CI_COMMIT_SHA - POSTGRES_IMAGE_VERSION_TAG: $CI_REGISTRY_IMAGE/$CI_COMMIT_REF_SLUG:$POSTGRES_BASE_IMAGE_VERSION PIP_REQUIREMENTS_FILE: "requirements/production.txt" -.docker-setup_template: &docker-setup - - docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN $CI_REGISTRY - .kaniko-setup_template: &kaniko-setup - mkdir -p /kaniko/.docker - echo "{\"auths\":{\"$CI_REGISTRY\":{\"auth\":\"$(echo -n ${CI_REGISTRY_USER}:${CI_REGISTRY_PASSWORD} | base64 | tr -d '\n')\"}}}" > /kaniko/.docker/config.json @@ -73,32 +69,30 @@ tag-release-image: build_postgres-image: - image: docker:19.03.0 - services: - - docker:19.03.0-dind - tags: - - docker + image: + name: gcr.io/kaniko-project/executor:debug + entrypoint: [""] stage: build before_script: - - *docker-setup + - *kaniko-setup script: - - docker build -t $POSTGRES_IMAGE --build-arg POSTGRES_IMAGE_TAG=$POSTGRES_BASE_IMAGE_VERSION -f compose/postgres/Dockerfile . - - docker push $POSTGRES_IMAGE + - /kaniko/executor + --context $CI_PROJECT_DIR + --dockerfile $CI_PROJECT_DIR/compose/postgres/Dockerfile + --build-arg POSTGRES_IMAGE_TAG=$POSTGRES_BASE_IMAGE_VERSION + --destination $POSTGRES_IMAGE only: - postgres-image -deploy_postgres-image: - image: docker:19.03.0 - services: - - docker:19.03.0-dind - tags: - - docker +tag_postgres-image: + image: + name: gcr.io/go-containerregistry/crane:debug + entrypoint: [""] stage: deploy before_script: - - *docker-setup + - *crane-setup script: - - docker pull $POSTGRES_IMAGE - - docker tag $POSTGRES_IMAGE $POSTGRES_IMAGE_VERSION_TAG - - docker push $POSTGRES_IMAGE_TAG - only: + - crane tag $POSTGRES_IMAGE $POSTGRES_BASE_IMAGE_VERSION + only: - postgres-image + -- GitLab From b35cffcb77b23097e98d3c83caab5c2b164bb9de Mon Sep 17 00:00:00 2001 From: Ubbo Veentjer <veentjer@sub.uni-goettingen.de> Date: Wed, 18 May 2022 20:24:14 +0200 Subject: [PATCH 7/9] path to requirements --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index c647c56..2e2cbd6 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -8,7 +8,7 @@ variables: CONTAINER_RELEASE_IMAGE: $CI_REGISTRY_IMAGE:$CI_COMMIT_TAG POSTGRES_BASE_IMAGE_VERSION: '11.13-bullseye' POSTGRES_IMAGE: $CI_REGISTRY_IMAGE/$CI_COMMIT_REF_SLUG:$CI_COMMIT_SHA - PIP_REQUIREMENTS_FILE: "requirements/production.txt" + PIP_REQUIREMENTS_FILE: /app/requirements/production.txt .kaniko-setup_template: &kaniko-setup - mkdir -p /kaniko/.docker -- GitLab From ec7ef774d8cd38126691a90e388d29080cbac787 Mon Sep 17 00:00:00 2001 From: Ubbo Veentjer <veentjer@sub.uni-goettingen.de> Date: Wed, 18 May 2022 20:30:21 +0200 Subject: [PATCH 8/9] fix image --- config/settings/docker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/settings/docker.py b/config/settings/docker.py index e982b3d..87c6239 100644 --- a/config/settings/docker.py +++ b/config/settings/docker.py @@ -1,5 +1,5 @@ import os -from . import BASE_DIR, INSTALLED_APPS, PROJECT_EXPORTS, PROJECT_IMPORTS, VENDOR +from . import BASE_DIR, INSTALLED_APPS, PROJECT_EXPORTS, PROJECT_IMPORTS, VENDOR, AUTHENTICATION_BACKENDS, MIDDLEWARE from django.utils.translation import ugettext_lazy as _ ''' -- GitLab From 03cd2e71300bf7eb82404c51892941b32b653199 Mon Sep 17 00:00:00 2001 From: Ubbo Veentjer <veentjer@sub.uni-goettingen.de> Date: Wed, 18 May 2022 18:54:25 +0200 Subject: [PATCH 9/9] activate autosave and sending issues again for testing. relates to #103, #113 and #128 --- config/settings/docker.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/config/settings/docker.py b/config/settings/docker.py index 87c6239..c46d8cc 100644 --- a/config/settings/docker.py +++ b/config/settings/docker.py @@ -293,13 +293,13 @@ if (str(os.getenv('USE_PROXY')).lower() == 'true'): PROJECT_SEND_ISSUE = False OVERLAYS = {} -#PROJECT_SEND_ISSUE = True -#EMAIL_RECIPIENTS_CHOICES = [ -# ('esteban.huanqui@gwdg.de', 'eRA Support (Esteban) <esteban.huanqui@gwdg.de>'), -# ('thenne@gwdg.de', 'eRA Support (Timo) <thenne@gwdg.de>'), -# ('uveentj@gwdg.de', 'eRA Support (Ubbo) <uveentj@gwdg.de>'), -#] - -#PROJECT_QUESTIONS_AUTOSAVE = True +PROJECT_SEND_ISSUE = True +EMAIL_RECIPIENTS_CHOICES = [ + ('esteban.huanqui@gwdg.de', 'eRA Support (Esteban) <esteban.huanqui@gwdg.de>'), + ('thenne@gwdg.de', 'eRA Support (Timo) <thenne@gwdg.de>'), + ('uveentj@gwdg.de', 'eRA Support (Ubbo) <uveentj@gwdg.de>'), +] + +PROJECT_QUESTIONS_AUTOSAVE = True -- GitLab