.gitlab-ci.yml from psono/psono-server

.gitlab-ci.yml
Summary

Maintainability

Test Coverage

Issues
variables:
  CONTAINER_TEST_IMAGE: $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_NAME
  CONTAINER_TEST_ARTIFACTORY_IMAGE: psono-docker.jfrog.io/psono/psono-server:$CI_COMMIT_REF_NAME
  CONTAINER_LATEST_IMAGE: psono-docker.jfrog.io/psono/psono-server:latest

stages:
  - translate
  - build
  - test
  - release
  - deploy

upload-translations:
  except:
    - schedules
  stage: translate
  image: psono-docker.jfrog.io/ubuntu:22.04
  services:
    - name: docker:25-dind
      alias: docker
      command: ["--tls=false"]
  variables:
    DOCKER_DRIVER: overlay2
    DOCKER_HOST: tcp://docker:2375/
    DOCKER_TLS_CERTDIR: ""
  script:
    - apt-get update && apt-get install -y curl iptables libdevmapper1.02.1 python3 python3-pip
    - curl -fSL "https://download.docker.com/linux/ubuntu/dists/jammy/pool/stable/amd64/containerd.io_1.6.28-1_amd64.deb" -o containerd.deb && echo "3fcc3e44e6d507c346ae05bbaef8f4bb04dfa8da9f04af658537f7e373c91340 containerd.deb" | sha256sum -c -
    - curl -fSL "https://download.docker.com/linux/ubuntu/dists/jammy/pool/stable/amd64/docker-buildx-plugin_0.12.1-1~ubuntu.22.04~jammy_amd64.deb" -o docker-buildx-plugin.deb && echo "3fb7cd11fd9bf7c31d81c1afa96c023519d9e6e17422192aa4d0aa2767ed0776 docker-buildx-plugin.deb" | sha256sum -c -
    - curl -fSL "https://download.docker.com/linux/ubuntu/dists/jammy/pool/stable/amd64/docker-ce-cli_25.0.3-1~ubuntu.22.04~jammy_amd64.deb" -o docker-ce-cli.deb && echo "624abe860a3d9dd190c278263b3ae115d54ce1135774a79f84026a6ec6ebafd1 docker-ce-cli.deb" | sha256sum -c -
    - curl -fSL "https://download.docker.com/linux/ubuntu/dists/jammy/pool/stable/amd64/docker-ce_25.0.3-1~ubuntu.22.04~jammy_amd64.deb" -o docker-ce.deb && echo "63c7703814558a829ab3cc37c8018c050c9dad56f99c261090500f6c87821fd1 docker-ce.deb" | sha256sum -c -
    - dpkg -i ./containerd.deb ./docker-ce.deb ./docker-ce-cli.deb ./docker-buildx-plugin.deb
    - docker info
    - docker build -f DockerfileUbuntu -t ubu-testimage --pull .
    - mkdir locale
    - docker run -v $(pwd)/locale:/root/psono/locale ubu-testimage bash -c "cd /root/psono && django-admin makemessages --locale=en -i 'restapi' -i 'fileserver' -i 'administration'"
    - pip3 install requests
    - python3 var/upload_translations.py
  only:
    - develop@psono/psono-server

create-translations:
  except:
    - schedules
  stage: translate
  image: psono-docker.jfrog.io/ubuntu:22.04
  script:
    - apt-get update && apt-get install -y python3 python3-pip
    - pip3 install requests openai==1.34.0
    - python3 var/translate.py
  only:
    - master@psono/psono-server

build-container-alpine:
  except:
    - schedules
  stage: build
  image: psono-docker.jfrog.io/ubuntu:22.04
  services:
    - name: docker:25-dind
      alias: docker
      command: ["--tls=false"]
  variables:
    DOCKER_DRIVER: overlay2
    DOCKER_HOST: tcp://docker:2375/
    DOCKER_TLS_CERTDIR: ""
  script:
    - sh ./var/update_version.sh
    - apt-get update || true
    - apt install -y curl ca-certificates iptables libdevmapper1.02.1 python3 python3-pip
    - pip3 install requests
    - python3 var/download_translations.py
    - curl -fSL "https://download.docker.com/linux/ubuntu/dists/jammy/pool/stable/amd64/containerd.io_1.6.28-1_amd64.deb" -o containerd.deb && echo "3fcc3e44e6d507c346ae05bbaef8f4bb04dfa8da9f04af658537f7e373c91340 containerd.deb" | sha256sum -c -
    - curl -fSL "https://download.docker.com/linux/ubuntu/dists/jammy/pool/stable/amd64/docker-buildx-plugin_0.12.1-1~ubuntu.22.04~jammy_amd64.deb" -o docker-buildx-plugin.deb && echo "3fb7cd11fd9bf7c31d81c1afa96c023519d9e6e17422192aa4d0aa2767ed0776 docker-buildx-plugin.deb" | sha256sum -c -
    - curl -fSL "https://download.docker.com/linux/ubuntu/dists/jammy/pool/stable/amd64/docker-ce-cli_25.0.3-1~ubuntu.22.04~jammy_amd64.deb" -o docker-ce-cli.deb && echo "624abe860a3d9dd190c278263b3ae115d54ce1135774a79f84026a6ec6ebafd1 docker-ce-cli.deb" | sha256sum -c -
    - curl -fSL "https://download.docker.com/linux/ubuntu/dists/jammy/pool/stable/amd64/docker-ce_25.0.3-1~ubuntu.22.04~jammy_amd64.deb" -o docker-ce.deb && echo "63c7703814558a829ab3cc37c8018c050c9dad56f99c261090500f6c87821fd1 docker-ce.deb" | sha256sum -c -
    - dpkg -i ./containerd.deb ./docker-ce.deb ./docker-ce-cli.deb ./docker-buildx-plugin.deb
    - docker info
    - echo $CI_JOB_TOKEN | docker login --username=gitlab-ci-token --password-stdin registry.gitlab.com
    - docker context create builder
    - docker buildx create builder --use
    - docker buildx build -f DockerfileAlpine --platform linux/amd64,linux/arm64 -t $CONTAINER_TEST_IMAGE --push --pull .
  only:
    - branches@psono/psono-server
    - tags@psono/psono-server


run-unittests-ubuntu:
  except:
    - schedules
  stage: test
  image: psono-docker.jfrog.io/docker:25.0.3-git
  services:
    - name: docker:25-dind
      alias: docker
      command: ["--tls=false"]
  variables:
    DOCKER_DRIVER: overlay2
    DOCKER_HOST: tcp://docker:2375/
    DOCKER_TLS_CERTDIR: ""
    POSTGRES_DB: postgres
    POSTGRES_USER: postgres
    POSTGRES_PASSWORD: ""
    PSONO_EMAIL_HOST: 172.17.0.1
    PSONO_EMAIL_FROM: test@example.com
    PSONO_ACTIVATION_LINK_SECRET: 9SruC2qPmKScVzGaF4378LW4rvNNkK2G3Gddqy9kPQqgkjeDQjs7jaLBCstgtJTt
    PSONO_SECRET_KEY: RQTKawYQv4w6KkuphcLzLu7r5ap7xE5DSDu5SkKXjMnWBQ93mcMKjdZfeZkY2Y7C
  script:
    - docker info
    - sh ./var/update_version.sh
    - docker build -f DockerfileUbuntu -t ubu-testimage --pull .
    - docker run -d -e POSTGRES_HOST_AUTH_METHOD=trust --name db postgres:15.5-alpine
    - sleep 20
    - docker run --link db:postgres -e "PSONO_THROTTLING=False" ubu-testimage bash -c "apt-get update && apt-get install -y python3-pip && pip3 install -r requirements-dev.txt && python3 ./psono/manage.py presetup && python3 ./psono/manage.py migrate && coverage3 run --source='.' ./psono/manage.py test restapi.tests administration.tests fileserver.tests && coverage3 report --omit=psono/restapi/migrations/*,psono/administration/tests*,psono/administration/migrations/*,psono/restapi/tests*,psono/fileserver/migrations/*,psono/fileserver/tests*"


run-unittests-alpine:
  except:
    - schedules
  stage: test
  image: psono-docker.jfrog.io/docker:25.0.3-git
  services:
    - name: docker:25-dind
      alias: docker
      command: ["--tls=false"]
  variables:
    DOCKER_DRIVER: overlay2
    DOCKER_HOST: tcp://docker:2375/
    DOCKER_TLS_CERTDIR: ""
    POSTGRES_DB: postgres
    POSTGRES_USER: postgres
    POSTGRES_PASSWORD: ""
    PSONO_EMAIL_HOST: 172.17.0.1
    PSONO_EMAIL_FROM: test@example.com
    PSONO_ACTIVATION_LINK_SECRET: 9SruC2qPmKScVzGaF4378LW4rvNNkK2G3Gddqy9kPQqgkjeDQjs7jaLBCstgtJTt
    PSONO_SECRET_KEY: RQTKawYQv4w6KkuphcLzLu7r5ap7xE5DSDu5SkKXjMnWBQ93mcMKjdZfeZkY2Y7C
  script:
    - docker info
    - echo $CI_JOB_TOKEN | docker login --username=gitlab-ci-token --password-stdin registry.gitlab.com
    - docker pull $CONTAINER_TEST_IMAGE
    - docker run -d -e POSTGRES_HOST_AUTH_METHOD=trust --name db postgres:15.5-alpine
    - sleep 20
    - docker run --link db:postgres -e "PSONO_THROTTLING=False" $CONTAINER_TEST_IMAGE /bin/sh -c "pip3 install -r requirements-dev.txt && python3 ./psono/manage.py presetup && python3 ./psono/manage.py migrate && python3 ./psono/manage.py test --parallel=8 restapi.tests administration.tests fileserver.tests"
  only:
    - branches@psono/psono-server
    - tags@psono/psono-server

run-unittests-alpine-arm64:
  except:
    - schedules
  stage: test
  image: psono-docker.jfrog.io/docker:25.0.3-git
  tags:
    - arm64
  services:
    - name: docker:25-dind
      alias: docker
      command: ["--tls=false"]
  variables:
    DOCKER_DRIVER: overlay2
    DOCKER_HOST: tcp://docker:2375/
    DOCKER_TLS_CERTDIR: ""
    POSTGRES_DB: postgres
    POSTGRES_USER: postgres
    POSTGRES_PASSWORD: ""
    PSONO_EMAIL_HOST: 172.17.0.1
    PSONO_EMAIL_FROM: test@example.com
    PSONO_ACTIVATION_LINK_SECRET: 9SruC2qPmKScVzGaF4378LW4rvNNkK2G3Gddqy9kPQqgkjeDQjs7jaLBCstgtJTt
    PSONO_SECRET_KEY: RQTKawYQv4w6KkuphcLzLu7r5ap7xE5DSDu5SkKXjMnWBQ93mcMKjdZfeZkY2Y7C
  script:
    - docker info
    - echo $CI_JOB_TOKEN | docker login --username=gitlab-ci-token --password-stdin registry.gitlab.com
    - docker pull $CONTAINER_TEST_IMAGE
    - docker run -d -e POSTGRES_HOST_AUTH_METHOD=trust --name db postgres:15.5-alpine
    - sleep 20
    - docker run --link db:postgres -e "PSONO_THROTTLING=False" $CONTAINER_TEST_IMAGE /bin/sh -c "pip3 install -r requirements-dev.txt && python3 ./psono/manage.py presetup && python3 ./psono/manage.py migrate && python3 ./psono/manage.py test --parallel=8 restapi.tests administration.tests fileserver.tests"
  only:
    - branches@psono/psono-server
    - tags@psono/psono-server

run-vulnerability-python-dependencies:
  except:
    - schedules
  stage: test
  image: python:3.11-buster
  script:
    - pip3 install pip-audit
    - pip-audit -r ./requirements.txt
  allow_failure: true

run-vulnerability-scan:
  except:
    - schedules
  stage: test
  image: psono-docker.jfrog.io/docker:25.0.3-git
  services:
    - name: docker:25-dind
      alias: docker
      command: ["--tls=false"]
  variables:
    DOCKER_DRIVER: overlay2
    DOCKER_HOST: tcp://docker:2375/
    DOCKER_TLS_CERTDIR: ""
  script:
    - docker info
    - echo $CI_JOB_TOKEN | docker login --username=gitlab-ci-token --password-stdin registry.gitlab.com
#    - echo $docker_hub_credentials | docker login --username=psonogitlab --password-stdin
    - docker pull $CONTAINER_TEST_IMAGE
    - docker run -e "LANG=C.UTF-8" $CONTAINER_TEST_IMAGE sh -c "pip3 install bandit && bandit -r /root -x /root/examples,/root/psono/restapi/tests,/root/psono/administration/tests,/root/psono/fileserver/tests"
  only:
    - branches@psono/psono-server
    - tags@psono/psono-server
  allow_failure: true



#deploy-security-scan-image:
#  except:
#    - schedules
#  stage: deploy
#  image: psono-docker.jfrog.io/docker:25.0.3-git
#  services:
#    - name: docker:25-dind
#      alias: docker
#      command: ["--tls=false"]
#  variables:
#    DOCKER_DRIVER: overlay2
#    DOCKER_HOST: tcp://docker:2375/
#    DOCKER_TLS_CERTDIR: ""
#  script:
#    - docker info
#    - echo $CI_JOB_TOKEN | docker login --username=gitlab-ci-token --password-stdin registry.gitlab.com
#    - echo $artifactory_credentials | docker login --username=gitlab --password-stdin psono-docker.jfrog.io
#    - docker pull $CONTAINER_TEST_IMAGE
#    - docker tag $CONTAINER_TEST_IMAGE psono/security-scans:psono-server-ce-$CI_COMMIT_REF_NAME
#    - docker push psono/security-scans:psono-server-ce-$CI_COMMIT_REF_NAME

release-container:
  except:
    - schedules
  stage: release
  image: psono-docker.jfrog.io/docker:25.0.3-git
  services:
    - name: docker:25-dind
      alias: docker
      command: ["--tls=false"]
  variables:
    DOCKER_DRIVER: overlay2
    DOCKER_HOST: tcp://docker:2375/
    DOCKER_TLS_CERTDIR: ""
  script:
    - apk add skopeo
    - docker info
    - echo $CI_JOB_TOKEN | docker login --username=gitlab-ci-token --password-stdin registry.gitlab.com
    - echo $artifactory_credentials | docker login --username=gitlab --password-stdin psono-docker.jfrog.io
    - echo $docker_hub_credentials | docker login --username=psonogitlab --password-stdin
    - skopeo copy --all docker://$CONTAINER_TEST_IMAGE docker://$CONTAINER_LATEST_IMAGE
    - skopeo copy --all docker://$CONTAINER_TEST_IMAGE docker://$CONTAINER_TEST_ARTIFACTORY_IMAGE
  only:
    - /^v[0-9]*\.[0-9]*\.[0-9]*$/

release-version-bump:
  except:
    - schedules
  stage: release
  image: psono-docker.jfrog.io/docker:25.0.3-git
  script:
    - mkdir -p /root/.ssh
    - echo "$github_deploy_key" > /root/.ssh/id_rsa
    - chmod 600 /root/.ssh/id_rsa
    - ssh-keyscan -t rsa gitlab.com >> ~/.ssh/known_hosts
    - git remote set-url --push origin "git@gitlab.com:${CI_PROJECT_PATH}.git"
    - git config --global user.email "buildserver@psono.com"
    - git config --global user.name "Build Server"
    - git checkout master
    - git pull
    - sh ./var/update_version.sh
    - git add psono/VERSION.txt
    - git add psono/SHA.txt
    - git commit -m "Bump version [skip ci]"
    - git push
  only:
    - /^v[0-9]*\.[0-9]*\.[0-9]*$/


deploy:
  except:
    - schedules
  stage: deploy
  image: psono-docker.jfrog.io/docker:25.0.3-git
  services:
    - name: docker:25-dind
      alias: docker
      command: ["--tls=false"]
  variables:
    DOCKER_DRIVER: overlay2
    DOCKER_HOST: tcp://docker:2375/
    DOCKER_TLS_CERTDIR: ""
  script:
    - docker info
    - echo $CI_JOB_TOKEN | docker login --username=gitlab-ci-token --password-stdin registry.gitlab.com
    - echo $artifactory_credentials | docker login --username=gitlab --password-stdin psono-docker.jfrog.io
    - echo $docker_hub_credentials | docker login --username=psonogitlab --password-stdin
    - sh ./var/deploy.sh
  environment:
    name: production
    url: https://psono.pw
  only:
    - /^v[0-9]*\.[0-9]*\.[0-9]*$/


deploy-changelog:
  except:
    - schedules
  stage: deploy
  image: psono-docker.jfrog.io/ubuntu:22.04
  script:
    - sh ./var/deploy_changelog.sh
  environment:
    name: static.psono.com
    url: https://static.psono.com/gitlab.com/psono/psono-server/changelog.json
  only:
    - /^v[0-9]*\.[0-9]*\.[0-9]*$/