-
Notifications
You must be signed in to change notification settings - Fork 191
132 lines (126 loc) · 4.68 KB
/
docker-publish.yml
File metadata and controls
132 lines (126 loc) · 4.68 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
name: Build And Push Docker Image
on:
push:
branches:
- main
workflow_dispatch:
env:
DOCKER_REPOSITORY: quay.io/unstructured-io/unstructured-api
DOCKER_BUILD_REPOSITORY: quay.io/unstructured-io/build-unstructured-api
PACKAGE: "unstructured-api"
PIPELINE_FAMILY: "general"
jobs:
set-short-sha:
runs-on: opensource-linux-8core
outputs:
short_sha: ${{ steps.set_short_sha.outputs.short_sha }}
steps:
- name: Set Short SHA
id: set_short_sha
run: echo "short_sha=$(echo ${{ github.sha }} | cut -c1-7)" >> $GITHUB_OUTPUT
build-images:
strategy:
matrix:
arch: ["arm64", "amd64"]
runs-on: ${{ matrix.arch == 'arm64' && 'ubuntu-24.04-arm' || 'opensource-linux-8core' }}
needs: set-short-sha
env:
SHORT_SHA: ${{ needs.set-short-sha.outputs.short_sha }}
DOCKER_PLATFORM: linux/${{ matrix.arch }}
steps:
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v4
with:
driver: docker
- name: Checkout code
uses: actions/checkout@v5
- name: Read Python version from .python-version
run: echo "PYTHON_VERSION=$(cat .python-version)" >> $GITHUB_ENV
- name: Login to Quay.io
uses: docker/login-action@v4
with:
registry: quay.io
username: ${{ secrets.QUAY_IO_ROBOT_USERNAME }}
password: ${{ secrets.QUAY_IO_ROBOT_TOKEN }}
- name: Free up disk space
run: |
# Clear some space (https://github.com/actions/runner-images/issues/2840)
echo "Disk usage before cleanup:"
df -h
# Remove unnecessary pre-installed software
sudo rm -rf /usr/share/dotnet
sudo rm -rf /opt/ghc
sudo rm -rf /usr/local/share/boost
sudo rm -rf /usr/local/lib/android
sudo rm -rf /opt/hostedtoolcache/CodeQL
sudo rm -rf /usr/local/.ghcup
sudo rm -rf /usr/share/swift
# Clean up docker to ensure we start fresh
docker system prune -af --volumes
echo "Disk usage after cleanup:"
df -h
- name: Build image
run: |
DOCKER_BUILDKIT=1 docker buildx build --load -f Dockerfile \
--platform=$DOCKER_PLATFORM \
--build-arg BUILDKIT_INLINE_CACHE=1 \
--build-arg PIPELINE_PACKAGE=${{ env.PIPELINE_FAMILY }} \
--provenance=false \
--progress plain \
--cache-from $DOCKER_BUILD_REPOSITORY:${{ matrix.arch }} \
-t $DOCKER_BUILD_REPOSITORY:${{ matrix.arch }}-$SHORT_SHA .
- name: Install uv
uses: astral-sh/setup-uv@v7
with:
enable-cache: true
cache-dependency-glob: "uv.lock"
- name: Set up Python ${{ env.PYTHON_VERSION }}
run: uv python install ${{ env.PYTHON_VERSION }}
- name: Install test dependencies
run: uv sync --group test --locked
- name: Test image
run: |
export DOCKER_IMAGE="$DOCKER_BUILD_REPOSITORY:${{ matrix.arch }}-$SHORT_SHA"
SKIP_INFERENCE_TESTS=true make docker-test
- name: Push image
run: |
# write to the build repository to cache for the publish-images job
docker push $DOCKER_BUILD_REPOSITORY:${{ matrix.arch }}-$SHORT_SHA
publish-images:
runs-on: opensource-linux-8core
needs: [set-short-sha, build-images]
env:
SHORT_SHA: ${{ needs.set-short-sha.outputs.short_sha }}
steps:
- name: Checkout code
uses: actions/checkout@v5
- name: Set SHORT_SHA
run: echo "SHORT_SHA=$(git rev-parse --short HEAD)" >> $GITHUB_ENV
- name: Login to Quay.io
uses: docker/login-action@v4
with:
registry: quay.io
username: ${{ secrets.QUAY_IO_ROBOT_USERNAME }}
password: ${{ secrets.QUAY_IO_ROBOT_TOKEN }}
- name: Pull AMD image
run: |
docker pull $DOCKER_BUILD_REPOSITORY:amd64-$SHORT_SHA
- name: Pull ARM image
run: |
docker pull $DOCKER_BUILD_REPOSITORY:arm64-$SHORT_SHA
- name: Push AMD and ARM tags
run: |
# these are used to construct the final manifest but also cache-from in subsequent runs
docker tag $DOCKER_BUILD_REPOSITORY:amd64-$SHORT_SHA $DOCKER_BUILD_REPOSITORY:amd64
docker push $DOCKER_BUILD_REPOSITORY:amd64
docker tag $DOCKER_BUILD_REPOSITORY:arm64-$SHORT_SHA $DOCKER_BUILD_REPOSITORY:arm64
docker push $DOCKER_BUILD_REPOSITORY:arm64
- name: Push multiarch manifest
run: |
VERSION=$(grep -oP '(?<=__version__ = ")[^"]+' prepline_general/api/__version__.py)
docker buildx imagetools create \
-t ${DOCKER_REPOSITORY}:latest \
-t ${DOCKER_REPOSITORY}:$SHORT_SHA \
-t ${DOCKER_REPOSITORY}:$VERSION \
$DOCKER_BUILD_REPOSITORY:amd64 \
$DOCKER_BUILD_REPOSITORY:arm64