From bef2d7b6b515bc90d8a58e3fa9776ab52fc48039 Mon Sep 17 00:00:00 2001 From: Fijxu Date: Thu, 15 May 2025 01:07:40 -0400 Subject: [PATCH 1/8] CI: Use public ARM64 Github actions runners for ARM64 builds. Currently, Invidious uses QEMU to build it's ARM64 Invidious image, which is slow (since we are basically using a virtual machine). This helps with the speed of building ARM64 binaries for Invidious on each release/commit. More information about the public ARM64 runners here: https://github.com/orgs/community/discussions/148648 CI: Use ARM64 compose file for build-docker-arm64 --- .github/workflows/build-nightly-container.yml | 55 +++++++------------ .github/workflows/build-stable-container.yml | 55 +++++++------------ .github/workflows/ci.yml | 20 ++----- docker-compose-arm64.yml | 55 +++++++++++++++++++ 4 files changed, 101 insertions(+), 84 deletions(-) create mode 100644 docker-compose-arm64.yml diff --git a/.github/workflows/build-nightly-container.yml b/.github/workflows/build-nightly-container.yml index 4149bd0b..3277c015 100644 --- a/.github/workflows/build-nightly-container.yml +++ b/.github/workflows/build-nightly-container.yml @@ -17,17 +17,27 @@ on: jobs: release: - runs-on: ubuntu-latest + strategy: + matrix: + include: + - os: ubuntu-latest + platforms: linux/amd64 + name: "AMD64" + dockerfile: "docker/Dockerfile" + tag_suffix: "" + # GitHub doesn't has a ubuntu-latest-arm runner + - os: ubuntu-24.04-arm + platforms: linux/arm64/v8 + name: "ARM64" + dockerfile: "docker/Dockerfile.arm64" + tag_suffix: "-arm64" + + runs-on: ${{ matrix.os }} steps: - name: Checkout uses: actions/checkout@v4 - - name: Set up QEMU - uses: docker/setup-qemu-action@v3 - with: - platforms: arm64 - - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 @@ -43,45 +53,22 @@ jobs: uses: docker/metadata-action@v5 with: images: quay.io/invidious/invidious + flavor: | + suffix=${{ matrix.tag_suffix }} tags: | type=sha,format=short,prefix={{date 'YYYY.MM.DD'}}-,enable=${{ github.ref == format('refs/heads/{0}', 'master') }} type=raw,value=master,enable=${{ github.ref == format('refs/heads/{0}', 'master') }} labels: | quay.expires-after=12w - - name: Build and push Docker AMD64 image for Push Event + - name: Build and push Docker ${{ matrix.name }} image for Push Event uses: docker/build-push-action@v6 with: context: . - file: docker/Dockerfile - platforms: linux/amd64 + file: ${{ matrix.dockerfile }} + platforms: ${{ matrix.platform }} labels: ${{ steps.meta.outputs.labels }} push: true tags: ${{ steps.meta.outputs.tags }} build-args: | "release=1" - - - name: Docker meta - id: meta-arm64 - uses: docker/metadata-action@v5 - with: - images: quay.io/invidious/invidious - flavor: | - suffix=-arm64 - tags: | - type=sha,format=short,prefix={{date 'YYYY.MM.DD'}}-,enable=${{ github.ref == format('refs/heads/{0}', 'master') }} - type=raw,value=master,enable=${{ github.ref == format('refs/heads/{0}', 'master') }} - labels: | - quay.expires-after=12w - - - name: Build and push Docker ARM64 image for Push Event - uses: docker/build-push-action@v6 - with: - context: . - file: docker/Dockerfile.arm64 - platforms: linux/arm64/v8 - labels: ${{ steps.meta-arm64.outputs.labels }} - push: true - tags: ${{ steps.meta-arm64.outputs.tags }} - build-args: | - "release=1" diff --git a/.github/workflows/build-stable-container.yml b/.github/workflows/build-stable-container.yml index 1a23e68c..1498dc2e 100644 --- a/.github/workflows/build-stable-container.yml +++ b/.github/workflows/build-stable-container.yml @@ -8,17 +8,27 @@ on: jobs: release: - runs-on: ubuntu-latest + strategy: + matrix: + include: + - os: ubuntu-latest + platforms: linux/amd64 + name: "AMD64" + dockerfile: "docker/Dockerfile" + tag_suffix: "" + # GitHub doesn't has a ubuntu-latest-arm runner + - os: ubuntu-24.04-arm + platforms: linux/arm64/v8 + name: "ARM64" + dockerfile: "docker/Dockerfile.arm64" + tag_suffix: "-arm64" + + runs-on: ${{ matrix.os }} steps: - name: Checkout uses: actions/checkout@v4 - - name: Set up QEMU - uses: docker/setup-qemu-action@v3 - with: - platforms: arm64 - - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 @@ -36,46 +46,21 @@ jobs: images: quay.io/invidious/invidious flavor: | latest=false + suffix=${{ matrix.tag_suffix }} tags: | type=semver,pattern={{version}} type=raw,value=latest labels: | quay.expires-after=12w - - name: Build and push Docker AMD64 image for Push Event + - name: Build and push Docker ${{ matrix.name }} image for Push Event uses: docker/build-push-action@v6 with: context: . - file: docker/Dockerfile - platforms: linux/amd64 + file: ${{ matrix.dockerfile }} + platforms: ${{ matrix.platform }} labels: ${{ steps.meta.outputs.labels }} push: true tags: ${{ steps.meta.outputs.tags }} build-args: | "release=1" - - - name: Docker meta - id: meta-arm64 - uses: docker/metadata-action@v5 - with: - images: quay.io/invidious/invidious - flavor: | - latest=false - suffix=-arm64 - tags: | - type=semver,pattern={{version}} - type=raw,value=latest - labels: | - quay.expires-after=12w - - - name: Build and push Docker ARM64 image for Push Event - uses: docker/build-push-action@v6 - with: - context: . - file: docker/Dockerfile.arm64 - platforms: linux/arm64/v8 - labels: ${{ steps.meta-arm64.outputs.labels }} - push: true - tags: ${{ steps.meta-arm64.outputs.tags }} - build-args: | - "release=1" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9d6a930a..c8805d10 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -100,26 +100,16 @@ jobs: build-docker-arm64: - runs-on: ubuntu-latest + runs-on: ubuntu-24.04-arm steps: - uses: actions/checkout@v4 - - name: Set up QEMU - uses: docker/setup-qemu-action@v3 - with: - platforms: arm64 + - name: Build Docker + run: docker compose -f docker-compose-arm64.yml build --build-arg release=0 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - - name: Build Docker ARM64 image - uses: docker/build-push-action@v6 - with: - context: . - file: docker/Dockerfile.arm64 - platforms: linux/arm64/v8 - build-args: release=0 + - name: Run Docker + run: docker compose -f docker-compose-arm64.yml up -d - name: Test Docker run: while curl -Isf http://localhost:3000; do sleep 1; done diff --git a/docker-compose-arm64.yml b/docker-compose-arm64.yml new file mode 100644 index 00000000..ba9e0a3f --- /dev/null +++ b/docker-compose-arm64.yml @@ -0,0 +1,55 @@ +# Warning: This docker-compose file is made for development purposes. +# Using it will build an image from the locally cloned repository. +# +# If you want to use Invidious in production, see the docker-compose.yml file provided +# in the installation documentation: https://docs.invidious.io/installation/ + +version: "3" +services: + + invidious: + build: + context: . + dockerfile: docker/Dockerfile.arm64 + restart: unless-stopped + ports: + - "127.0.0.1:3000:3000" + environment: + # Please read the following file for a comprehensive list of all available + # configuration options and their associated syntax: + # https://github.com/iv-org/invidious/blob/master/config/config.example.yml + INVIDIOUS_CONFIG: | + db: + dbname: invidious + user: kemal + password: kemal + host: invidious-db + port: 5432 + check_tables: true + # external_port: + # domain: + # https_only: false + # statistics_enabled: false + hmac_key: "CHANGE_ME!!" + healthcheck: + test: wget -nv --tries=1 --spider http://127.0.0.1:3000/api/v1/trending || exit 1 + interval: 30s + timeout: 5s + retries: 2 + + invidious-db: + image: docker.io/library/postgres:14 + restart: unless-stopped + volumes: + - postgresdata:/var/lib/postgresql/data + - ./config/sql:/config/sql + - ./docker/init-invidious-db.sh:/docker-entrypoint-initdb.d/init-invidious-db.sh + environment: + POSTGRES_DB: invidious + POSTGRES_USER: kemal + POSTGRES_PASSWORD: kemal + healthcheck: + test: ["CMD-SHELL", "pg_isready -U $$POSTGRES_USER -d $$POSTGRES_DB"] + +volumes: + postgresdata: From cef0097a309847f6075d7e9173c0362dcc83c757 Mon Sep 17 00:00:00 2001 From: Fijxu Date: Thu, 15 May 2025 15:28:14 -0400 Subject: [PATCH 2/8] CI: fix typo on matrix platforms --- .github/workflows/build-nightly-container.yml | 4 ++-- .github/workflows/build-stable-container.yml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build-nightly-container.yml b/.github/workflows/build-nightly-container.yml index 3277c015..6b9d4a87 100644 --- a/.github/workflows/build-nightly-container.yml +++ b/.github/workflows/build-nightly-container.yml @@ -21,13 +21,13 @@ jobs: matrix: include: - os: ubuntu-latest - platforms: linux/amd64 + platform: linux/amd64 name: "AMD64" dockerfile: "docker/Dockerfile" tag_suffix: "" # GitHub doesn't has a ubuntu-latest-arm runner - os: ubuntu-24.04-arm - platforms: linux/arm64/v8 + platform: linux/arm64/v8 name: "ARM64" dockerfile: "docker/Dockerfile.arm64" tag_suffix: "-arm64" diff --git a/.github/workflows/build-stable-container.yml b/.github/workflows/build-stable-container.yml index 1498dc2e..07a3520b 100644 --- a/.github/workflows/build-stable-container.yml +++ b/.github/workflows/build-stable-container.yml @@ -12,13 +12,13 @@ jobs: matrix: include: - os: ubuntu-latest - platforms: linux/amd64 + platform: linux/amd64 name: "AMD64" dockerfile: "docker/Dockerfile" tag_suffix: "" # GitHub doesn't has a ubuntu-latest-arm runner - os: ubuntu-24.04-arm - platforms: linux/arm64/v8 + platform: linux/arm64/v8 name: "ARM64" dockerfile: "docker/Dockerfile.arm64" tag_suffix: "-arm64" From 1d2f4b68133231c66e18d878706e8e263e47a66f Mon Sep 17 00:00:00 2001 From: Fijxu Date: Thu, 15 May 2025 15:29:24 -0400 Subject: [PATCH 3/8] CI: fix typo on comment about the os used on the ARM64 builder --- .github/workflows/build-nightly-container.yml | 2 +- .github/workflows/build-stable-container.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-nightly-container.yml b/.github/workflows/build-nightly-container.yml index 6b9d4a87..1a5abeea 100644 --- a/.github/workflows/build-nightly-container.yml +++ b/.github/workflows/build-nightly-container.yml @@ -25,7 +25,7 @@ jobs: name: "AMD64" dockerfile: "docker/Dockerfile" tag_suffix: "" - # GitHub doesn't has a ubuntu-latest-arm runner + # GitHub doesn't have a ubuntu-latest-arm runner - os: ubuntu-24.04-arm platform: linux/arm64/v8 name: "ARM64" diff --git a/.github/workflows/build-stable-container.yml b/.github/workflows/build-stable-container.yml index 07a3520b..7c2a276b 100644 --- a/.github/workflows/build-stable-container.yml +++ b/.github/workflows/build-stable-container.yml @@ -16,7 +16,7 @@ jobs: name: "AMD64" dockerfile: "docker/Dockerfile" tag_suffix: "" - # GitHub doesn't has a ubuntu-latest-arm runner + # GitHub doesn't have a ubuntu-latest-arm runner - os: ubuntu-24.04-arm platform: linux/arm64/v8 name: "ARM64" From 94f0a7a9d22e46e58447810fbb0da05508162fad Mon Sep 17 00:00:00 2001 From: Fijxu Date: Thu, 15 May 2025 15:31:17 -0400 Subject: [PATCH 4/8] CI: remove --build-arg Dockerfile and Dockerfile.arm64 already build Invidious without release mode if `release` argument is not present. --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c8805d10..1bb92101 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -90,7 +90,7 @@ jobs: - uses: actions/checkout@v4 - name: Build Docker - run: docker compose build --build-arg release=0 + run: docker compose build - name: Run Docker run: docker compose up -d @@ -106,7 +106,7 @@ jobs: - uses: actions/checkout@v4 - name: Build Docker - run: docker compose -f docker-compose-arm64.yml build --build-arg release=0 + run: docker compose -f docker-compose-arm64.yml build - name: Run Docker run: docker compose -f docker-compose-arm64.yml up -d From 1d664c759f17b5455d1ffbbe5e276a35dd4202e9 Mon Sep 17 00:00:00 2001 From: Fijxu Date: Thu, 15 May 2025 16:33:03 -0400 Subject: [PATCH 5/8] CI: Use matrix for `build-docker` on ci.yml --- .github/workflows/ci.yml | 28 ++++++++++------------------ 1 file changed, 10 insertions(+), 18 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1bb92101..51a5052d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -83,14 +83,22 @@ jobs: run: crystal build --warnings all --error-on-warnings --error-trace src/invidious.cr build-docker: + strategy: + matrix: + include: + - os: ubuntu-latest + docker_compose_file: "docker-compose.yml" + # GitHub doesn't have a ubuntu-latest-arm runner + - os: ubuntu-24.04-arm + docker_compose_file: "docker-compose-arm64.yml" - runs-on: ubuntu-latest + runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v4 - name: Build Docker - run: docker compose build + run: docker compose -f ${{ matrix.docker_compose_file }} build - name: Run Docker run: docker compose up -d @@ -98,22 +106,6 @@ jobs: - name: Test Docker run: while curl -Isf http://localhost:3000; do sleep 1; done - build-docker-arm64: - - runs-on: ubuntu-24.04-arm - - steps: - - uses: actions/checkout@v4 - - - name: Build Docker - run: docker compose -f docker-compose-arm64.yml build - - - name: Run Docker - run: docker compose -f docker-compose-arm64.yml up -d - - - name: Test Docker - run: while curl -Isf http://localhost:3000; do sleep 1; done - lint: runs-on: ubuntu-latest From a3375e512edf00c5c0c00089d370392c37bbe550 Mon Sep 17 00:00:00 2001 From: Fijxu Date: Thu, 15 May 2025 17:43:03 -0400 Subject: [PATCH 6/8] CI: Add name attribute to `build-docker` job --- .github/workflows/ci.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 51a5052d..80cb81a0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -88,10 +88,13 @@ jobs: include: - os: ubuntu-latest docker_compose_file: "docker-compose.yml" + name: "AMD64" # GitHub doesn't have a ubuntu-latest-arm runner - os: ubuntu-24.04-arm docker_compose_file: "docker-compose-arm64.yml" + name: "ARM64" + name: Test ${{ matrix.name }} Docker build runs-on: ${{ matrix.os }} steps: From 033a44fab574df56dd63a41d63d089b84cdb31f5 Mon Sep 17 00:00:00 2001 From: Fijxu Date: Thu, 15 May 2025 17:58:24 -0400 Subject: [PATCH 7/8] CI: Also use `matrix.docker_compose_file` for `Run Docker` step --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 80cb81a0..d3b6455a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -104,7 +104,7 @@ jobs: run: docker compose -f ${{ matrix.docker_compose_file }} build - name: Run Docker - run: docker compose up -d + run: docker compose -f ${{ matrix.docker_compose_file }} up -d - name: Test Docker run: while curl -Isf http://localhost:3000; do sleep 1; done From 381074fce1f3e405d8c527a672f524c4700aead5 Mon Sep 17 00:00:00 2001 From: Fijxu Date: Thu, 15 May 2025 19:38:21 -0400 Subject: [PATCH 8/8] CI: Replace Dockerfile path depending of the os used --- .github/workflows/ci.yml | 10 +++++--- docker-compose-arm64.yml | 55 ---------------------------------------- 2 files changed, 6 insertions(+), 59 deletions(-) delete mode 100644 docker-compose-arm64.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d3b6455a..7a5e8850 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -87,11 +87,9 @@ jobs: matrix: include: - os: ubuntu-latest - docker_compose_file: "docker-compose.yml" name: "AMD64" # GitHub doesn't have a ubuntu-latest-arm runner - os: ubuntu-24.04-arm - docker_compose_file: "docker-compose-arm64.yml" name: "ARM64" name: Test ${{ matrix.name }} Docker build @@ -100,11 +98,15 @@ jobs: steps: - uses: actions/checkout@v4 + - name: Use ARM64 Dockerfile if ARM64 + if: ${{ matrix.name }} == "ARM64" + run: sed -i 's/Dockerfile/Dockerfile.arm64/' docker-compose.yml + - name: Build Docker - run: docker compose -f ${{ matrix.docker_compose_file }} build + run: docker compose build - name: Run Docker - run: docker compose -f ${{ matrix.docker_compose_file }} up -d + run: docker compose up -d - name: Test Docker run: while curl -Isf http://localhost:3000; do sleep 1; done diff --git a/docker-compose-arm64.yml b/docker-compose-arm64.yml deleted file mode 100644 index ba9e0a3f..00000000 --- a/docker-compose-arm64.yml +++ /dev/null @@ -1,55 +0,0 @@ -# Warning: This docker-compose file is made for development purposes. -# Using it will build an image from the locally cloned repository. -# -# If you want to use Invidious in production, see the docker-compose.yml file provided -# in the installation documentation: https://docs.invidious.io/installation/ - -version: "3" -services: - - invidious: - build: - context: . - dockerfile: docker/Dockerfile.arm64 - restart: unless-stopped - ports: - - "127.0.0.1:3000:3000" - environment: - # Please read the following file for a comprehensive list of all available - # configuration options and their associated syntax: - # https://github.com/iv-org/invidious/blob/master/config/config.example.yml - INVIDIOUS_CONFIG: | - db: - dbname: invidious - user: kemal - password: kemal - host: invidious-db - port: 5432 - check_tables: true - # external_port: - # domain: - # https_only: false - # statistics_enabled: false - hmac_key: "CHANGE_ME!!" - healthcheck: - test: wget -nv --tries=1 --spider http://127.0.0.1:3000/api/v1/trending || exit 1 - interval: 30s - timeout: 5s - retries: 2 - - invidious-db: - image: docker.io/library/postgres:14 - restart: unless-stopped - volumes: - - postgresdata:/var/lib/postgresql/data - - ./config/sql:/config/sql - - ./docker/init-invidious-db.sh:/docker-entrypoint-initdb.d/init-invidious-db.sh - environment: - POSTGRES_DB: invidious - POSTGRES_USER: kemal - POSTGRES_PASSWORD: kemal - healthcheck: - test: ["CMD-SHELL", "pg_isready -U $$POSTGRES_USER -d $$POSTGRES_DB"] - -volumes: - postgresdata: