Repository navigation
Expand file tree
/
Copy pathDockerfile
More file actions
269 lines (226 loc) · 12.2 KB
/
Copy pathDockerfile
File metadata and controls
269 lines (226 loc) · 12.2 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
# base stage
FROM ubuntu:24.04 AS base
USER root
SHELL ["/bin/bash", "-c"]
ARG NEED_MIRROR=0
#Optional parameter
#If set NEED_MIRROR=1, and set GITEE_TOKEN="xxxxx" , donwload source from gitee.
#If don't set GITEE_TOKEN , download from github
ARG GITEE_TOKEN
WORKDIR /ragflow
# copy models downloaded via download_deps.py
# layout.laws/manual/paper.onnx are byte-identical to layout.onnx, so we
# exclude them from the tar extract and symlink them to layout.onnx instead,
# saving ~219MB in the image.
RUN mkdir -p /ragflow/internal/rag/res/deepdoc /root/.ragflow
RUN --mount=type=bind,from=infiniflow/ragflow_deps:latest,source=/huggingface.co,target=/huggingface.co \
tar --exclude='.*' \
--exclude='layout.laws.onnx' \
--exclude='layout.manual.onnx' \
--exclude='layout.paper.onnx' \
--exclude='layout.onnx' \
--exclude='det.onnx' \
--exclude='rec.onnx' \
--exclude='tsr.onnx' \
--exclude='layout.laws.ort' \
--exclude='layout.manual.ort' \
--exclude='layout.paper.ort' \
-cf - \
/huggingface.co/InfiniFlow/text_concat_xgb_v1.0 \
/huggingface.co/InfiniFlow/deepdoc \
| tar -xf - --strip-components=3 -C /ragflow/internal/rag/res/deepdoc && \
#ln -s layout.onnx /ragflow/internal/rag/res/deepdoc/layout.laws.onnx && \
#ln -s layout.onnx /ragflow/internal/rag/res/deepdoc/layout.manual.onnx && \
#ln -s layout.onnx /ragflow/internal/rag/res/deepdoc/layout.paper.onnx
ln -s layout.ort /ragflow/internal/rag/res/deepdoc/layout.laws.ort && \
ln -s layout.ort /ragflow/internal/rag/res/deepdoc/layout.manual.ort && \
ln -s layout.ort /ragflow/internal/rag/res/deepdoc/layout.paper.ort
# Copy the cl100k_base BPE table used by the Go tokenizer (tiktoken-go
# cl100k_base). The deps image ships it at its root; the Go image previously
# mounted only /huggingface.co and omitted this file, so NumTokensFromString
# silently returned 0. localBpeLoader resolves it from <workdir>/ragflow_deps/,
# so dropping it here makes the table load offline at startup (see
# InitCL100KEncoder fail-fast guard).
RUN --mount=type=bind,from=infiniflow/ragflow_deps:latest,source=/cl100k_base.tiktoken,target=/tmp/cl100k_base.tiktoken \
mkdir -p /ragflow/ragflow_deps && \
cp /tmp/cl100k_base.tiktoken /ragflow/ragflow_deps/cl100k_base.tiktoken
# Embedding tokenizer assets for the Go counters (internal/tokenizer). Same story as
# the cl100k table above: the deps image carries them and the Go image has to put them
# where the loaders search (<workdir>/ragflow_deps/huggingface.co/<repo>/<file>). A
# missing asset FAILS THE BUILD here rather than shipping an image where every tagged
# model silently counts with the calibrated cl100k estimate. The tokenizer.json files
# that exist only as cross-check oracles are deliberately not shipped.
RUN --mount=type=bind,from=infiniflow/ragflow_deps:latest,source=/huggingface.co,target=/huggingface.co \
for asset in \
BAAI/bge-m3/sentencepiece.bpe.model \
BAAI/bge-large-en-v1.5/vocab.txt \
Qwen/Qwen3-Embedding-0.6B/tokenizer.json \
intfloat/e5-mistral-7b-instruct/tokenizer.json ; do \
if [ -f "/huggingface.co/$asset" ]; then \
mkdir -p "/ragflow/ragflow_deps/huggingface.co/$(dirname "$asset")" && \
cp "/huggingface.co/$asset" "/ragflow/ragflow_deps/huggingface.co/$asset" ; \
else \
echo "ERROR: tokenizer asset $asset is missing from the infiniflow/ragflow_deps image; this image would count with the calibrated estimate instead of the model's own tokenizer" >&2 ; \
exit 1 ; \
fi ; \
done
ENV DEBIAN_FRONTEND=noninteractive
# Setup apt
RUN --mount=type=cache,id=ragflow_apt,target=/var/cache/apt,sharing=locked \
if [ "$NEED_MIRROR" == "1" ]; then \
# CI runners may inject a proxy whose TLS certificate is not trusted inside
# the fresh Ubuntu base image yet. Keep the Ubuntu mirror on HTTP here so
# the mirror switch remains usable before the full CA store is available.
sed -i 's|http://archive.ubuntu.com/ubuntu|http://mirrors.aliyun.com/ubuntu|g' /etc/apt/sources.list.d/ubuntu.sources; \
sed -i 's|http://security.ubuntu.com/ubuntu|http://mirrors.aliyun.com/ubuntu|g' /etc/apt/sources.list.d/ubuntu.sources; \
fi; \
rm -f /etc/apt/apt.conf.d/docker-clean && \
echo 'Binary::apt::APT::Keep-Downloaded-Packages "true";' > /etc/apt/apt.conf.d/keep-cache && \
chmod 1777 /tmp && \
apt update && \
apt --no-install-recommends install -y ca-certificates curl vim unzip iproute2 fonts-dejavu fontconfig && \
mkdir -p /usr/local/share/fonts/truetype/noto && \
curl --retry 5 --retry-delay 2 --retry-all-errors -fsSL \
-o /usr/local/share/fonts/truetype/noto/NotoSansCJKsc-VF.ttf \
https://github.com/notofonts/noto-cjk/raw/main/Sans/Variable/TTF/NotoSansCJKsc-VF.ttf && \
fc-cache -f && \
rm -rf /var/lib/apt/lists/*
# Download resource from GitHub to /usr/share/infinity
# Ship only the directories required by the runtime tokenizer:
# rag - base analyzer dictionaries (mandatory)
# opencc - Traditional/Simplified Chinese conversion
# wordnet - WordNet resources
RUN --mount=type=cache,id=ragflow_apt,target=/var/cache/apt,sharing=locked \
apt-get update && \
apt-get install -y --no-install-recommends git && \
mkdir -p /usr/share/infinity/resource && \
if [ "$NEED_MIRROR" == "1" ]; then \
if [ -n "$GITEE_TOKEN" ]; then \
git clone --depth 1 --single-branch "https://oauth2:${GITEE_TOKEN}@gitee.com/infiniflow/resource" /tmp/resource; \
else \
git clone --depth 1 --single-branch https://github.com/infiniflow/resource.git /tmp/resource; \
fi; \
else \
git clone --depth 1 --single-branch https://github.com/infiniflow/resource.git /tmp/resource; \
fi && \
for d in rag opencc wordnet; do \
cp -r "/tmp/resource/$d" /usr/share/infinity/resource/; \
done && \
rm -rf /tmp/resource && \
apt-get purge -y git && \
apt-get autoremove -y && \
rm -rf /var/lib/apt/lists/*
ARG NGINX_VERSION=1.31.3-1~noble
RUN --mount=type=cache,id=ragflow_apt,target=/var/cache/apt,sharing=locked \
apt -o Acquire::Retries=5 update && \
apt -o Acquire::Retries=5 install -y --no-install-recommends gnupg && \
mkdir -p /etc/apt/keyrings && \
curl --retry 5 --retry-delay 2 --retry-all-errors -fsSL https://nginx.org/keys/nginx_signing.key | gpg --dearmor -o /etc/apt/keyrings/nginx-archive-keyring.gpg && \
echo "deb [signed-by=/etc/apt/keyrings/nginx-archive-keyring.gpg] https://nginx.org/packages/mainline/ubuntu/ noble nginx" > /etc/apt/sources.list.d/nginx.list && \
apt -o Acquire::Retries=5 update && \
apt -o Acquire::Retries=5 install -y --no-install-recommends nginx=${NGINX_VERSION} && \
apt-mark hold nginx && \
apt-get purge -y gnupg && \
apt-get autoremove -y && \
rm -rf /var/lib/apt/lists/*
# ── web-builder stage ──
FROM infiniflow/github_action_runner:latest AS web-builder
USER root
WORKDIR /ragflow
# Install frontend dependencies — depends only on package manifests so
# web source / docs changes don't invalidate this layer.
COPY web/package.json web/package-lock.json web/.npmrc ./web/
RUN --mount=type=cache,id=ragflow_npm,target=/root/.npm,sharing=locked \
cd web && NODE_OPTIONS="--max-old-space-size=8192" npm install
# Copy full web source and docs for the frontend build.
COPY web web
COPY docs docs
RUN --mount=type=cache,id=ragflow_npm,target=/root/.npm,sharing=locked \
cd web && NODE_OPTIONS="--max-old-space-size=8192" VITE_BUILD_SOURCEMAP=false VITE_MINIFY=esbuild npm run build
# Stamp the build version into /ragflow/VERSION. Requires git, which must be
# preinstalled in the github_action_runner base image (the former apt-get install
# git step was removed), and the host .git tree bound in at build time.
RUN --mount=type=bind,source=.git,target=/ragflow/.git \
version_info=$(git describe --tags --match=v* --first-parent --always) && \
echo "$version_info" > /ragflow/VERSION
# ── go-builder stage ──
FROM infiniflow/github_action_runner:latest AS go-builder
USER root
SHELL ["/bin/bash", "-c"]
WORKDIR /ragflow
# Cache Go modules BEFORE copying source (mirrors the Dockerfile_ci fix):
# copy only the manifests, download the full module graph into a persistent
# BuildKit cache mount, then bring in source. GOMODCACHE/GOCACHE are pinned to the
# mounted paths so `go mod download` and `build.sh --go` share the same cache and
# dependencies are never re-fetched when only source changes.
COPY go.mod go.sum ./
RUN --mount=type=cache,id=ragflow_gomod,target=/root/.cache/gomod \
--mount=type=cache,id=ragflow_gobuild,target=/root/.cache/gobuild \
GOMODCACHE=/root/.cache/gomod GOCACHE=/root/.cache/gobuild \
GOPROXY=${GOPROXY:-https://goproxy.cn,https://proxy.golang.org,direct} \
go mod download
COPY internal internal
COPY cmd cmd
COPY build.sh ./
# build.sh's check_ort_version_consistency (run via `./build.sh --go`) greps the
# ORT version pins from these files; without them the --go build fails with
# "could not parse the ONNX Runtime version from one of the pinned locations".
COPY ragflow_deps/download_go_deps.py ragflow_deps/download_deps.py ./ragflow_deps/
COPY Dockerfile ./
# ONNX Runtime static archives: build.sh's _seed_from_system looks for the ORT
# static libs under ONNXRUNTIME_STATIC_PREFIX (default ~/ragflow-native-libs/onnxruntime).
# The github_action_runner base image pre-bakes them at /opt/ragflow-native-libs,
# so we copy them into the expected user-cache path before building. These .a files
# are consumed at link time only; they do NOT enter the final image (only the
# compiled /ragflow/bin is COPY --from=go-builder). Without this, ORT linking is
# silently skipped and the binary fails at startup with
# "no in-process DeepDoc backend serving" (dlopen(NULL)/dlsym can't find OrtGetApiBase).
# ${HOME} (not hard-coded /root) keeps the path consistent with build.sh regardless of HOME.
ARG ORT_VERSION=1.29.0
RUN set -eux; \
mkdir -p "${HOME}/ragflow-native-libs/onnxruntime/static_lib"; \
_src="/opt/ragflow-native-libs/onnxruntime/static_lib"; \
_v="$(ls -d "${_src}"/onnxruntime-linux-x64-static_lib-"${ORT_VERSION}"-* 2>/dev/null | head -1)"; \
if [ -z "${_v}" ]; then \
_v="$(ls -d "${_src}"/onnxruntime-linux-x64-static_lib-* 2>/dev/null | head -1)"; \
fi; \
if [ -n "${_v}" ]; then \
cp -r "${_v}" "${HOME}/ragflow-native-libs/onnxruntime/static_lib/"; \
else \
echo "Warning: no pre-baked onnxruntime static_lib in runner image; build.sh will attempt download" >&2; \
fi; \
find "${HOME}/ragflow-native-libs/onnxruntime/static_lib" -name '*.a' | head -5
RUN git config --global safe.directory "*" && \
cd /ragflow && ./build.sh --cpp
RUN --mount=type=cache,id=ragflow_gomod,target=/root/.cache/gomod \
--mount=type=cache,id=ragflow_gobuild,target=/root/.cache/gobuild \
GOMODCACHE=/root/.cache/gomod GOCACHE=/root/.cache/gobuild \
git config --global safe.directory "*" && \
./build.sh --go
##### production stage
FROM base AS production
USER root
WORKDIR /ragflow
# Copy the compiled Go backend binaries (set exec bits at copy time to avoid a redundant chmod layer)
COPY --from=go-builder --chmod=755 /ragflow/bin/ragflow_server /ragflow/bin/ragflow_server
ENV PYTHONPATH=/ragflow/
COPY docker/service_conf.yaml.template ./conf/service_conf.yaml.template
COPY --chmod=755 docker/entrypoint*.sh ./
# Copy nginx configuration for frontend serving
RUN mkdir -p /etc/nginx/conf.d /var/log/nginx
COPY docker/nginx/nginx.conf docker/nginx/proxy.conf /etc/nginx/
COPY docker/nginx/ragflow.conf /etc/nginx/conf.d/
RUN rm -f /etc/nginx/sites-enabled/default
COPY conf conf
COPY internal/agent/templates agent/templates
COPY rag/prompts rag/prompts
# Wiki page-structure presets read at runtime by the Go backend
# (CompilationTemplateService.LoadWikiPresets).
COPY internal/ingestion/knowledge_compile/templates ./internal/ingestion/knowledge_compile/templates
# Copy compiled web pages
COPY --from=web-builder /ragflow/web/dist /ragflow/web/dist
# Copy version info
COPY --from=web-builder /ragflow/VERSION /ragflow/VERSION
# Set environment variables
ENV HF_ENDPOINT=https://hf-mirror.com
ENTRYPOINT ["./entrypoint-go.sh"]