Skip to content

Commit 3768246

Browse files
vlaskyvlasky
committed
Add fuzz targets and nightly fuzz CI workflow
Ports upstream's fuzz testing infrastructure (asg017/sqlite-vec a61d451, 4418a34, 0dd0765 and follow-ups), which found most of the bugs recently fixed in this fork. Takes the 8 new libFuzzer targets that apply to this fork's feature set (shadow-corrupt, vec0-operations, scalar-functions, vec0-create-full, metadata-columns, vec-each, vec-mismatch, vec0-delete-completeness) alongside the 4 existing ones, and trims the Makefile's DiskANN/IVF/rescore targets, which cover features this fork does not have. The fuzz.yaml workflow runs all targets for 60s each under ASan+UBSan on push to main and nightly (Linux required; macOS and Windows best-effort), uploading crash artifacts on failure. One change from upstream: corpus directories are looked up by the hyphenated target name so the tracked seed corpus is actually used. All 12 targets build with Homebrew LLVM on macOS and ran locally for 15s each with no findings.
1 parent 3a64182 commit 3768246

12 files changed

Lines changed: 1027 additions & 40 deletions

‎.github/workflows/fuzz.yaml‎

Lines changed: 167 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,167 @@
1+
name: "Fuzz"
2+
on:
3+
push:
4+
branches: [main]
5+
schedule:
6+
# Nightly at 2am UTC for longer fuzzing sessions
7+
- cron: "0 2 * * *"
8+
workflow_dispatch:
9+
inputs:
10+
duration:
11+
description: "Fuzz duration per target (seconds)"
12+
default: "60"
13+
14+
permissions:
15+
contents: read
16+
17+
jobs:
18+
fuzz-linux:
19+
runs-on: ubuntu-22.04
20+
steps:
21+
- uses: actions/checkout@v4
22+
- name: Install LLVM 18
23+
run: |
24+
wget -qO- https://apt.llvm.org/llvm.sh | sudo bash -s -- 18
25+
echo "FUZZ_CC=clang-18" >> $GITHUB_ENV
26+
- run: ./scripts/vendor.sh
27+
- name: Generate sqlite-vec.h
28+
run: make sqlite-vec.h
29+
- name: Build fuzz targets
30+
run: make -C tests/fuzz all FUZZ_CC=$FUZZ_CC FUZZ_LDFLAGS=
31+
- name: Run fuzz targets
32+
run: |
33+
DURATION=${{ github.event.inputs.duration || '60' }}
34+
EXIT_CODE=0
35+
for target in tests/fuzz/targets/*; do
36+
[ -f "$target" ] && [ -x "$target" ] || continue
37+
name=$(basename "$target")
38+
echo "::group::Fuzzing $name ($DURATION seconds)"
39+
corpus="tests/fuzz/corpus/${name//_/-}"
40+
mkdir -p "$corpus"
41+
dict="tests/fuzz/${name//_/-}.dict"
42+
dict_flag=""
43+
[ -f "$dict" ] && dict_flag="-dict=$dict"
44+
if ! ASAN_OPTIONS=detect_leaks=1 "$target" $dict_flag \
45+
-max_total_time="$DURATION" "$corpus" 2>&1; then
46+
echo "::error::Fuzz target $name found a crash!"
47+
EXIT_CODE=1
48+
fi
49+
echo "::endgroup::"
50+
done
51+
exit $EXIT_CODE
52+
- name: Upload crash artifacts
53+
if: failure()
54+
uses: actions/upload-artifact@v4
55+
with:
56+
name: fuzz-crashes-linux
57+
path: |
58+
crash-*
59+
leak-*
60+
timeout-*
61+
62+
fuzz-macos:
63+
runs-on: macos-14
64+
# Best-effort: Homebrew LLVM 18 runtime dylibs pull in
65+
# __ZnwmSt19__type_descriptor_t (typed allocation ABI) which
66+
# macOS 14's system libc++ doesn't provide, causing dyld to abort.
67+
# Xcode clang doesn't ship libclang_rt.fuzzer_osx.a (no libFuzzer).
68+
# TODO: fix macOS fuzzing (pin older compiler-rt, or static runtime).
69+
continue-on-error: true
70+
steps:
71+
- uses: actions/checkout@v4
72+
- name: Install LLVM 18
73+
run: brew install llvm@18
74+
- run: ./scripts/vendor.sh
75+
- name: Generate sqlite-vec.h
76+
run: make sqlite-vec.h
77+
- name: Build fuzz targets
78+
run: |
79+
LLVM=/opt/homebrew/opt/llvm@18
80+
make -C tests/fuzz all \
81+
FUZZ_CC=$LLVM/bin/clang \
82+
FUZZ_LDFLAGS="-Wl,-ld_classic"
83+
- name: Run fuzz targets
84+
run: |
85+
DURATION=${{ github.event.inputs.duration || '60' }}
86+
EXIT_CODE=0
87+
for target in tests/fuzz/targets/*; do
88+
[ -f "$target" ] && [ -x "$target" ] || continue
89+
name=$(basename "$target")
90+
echo "::group::Fuzzing $name ($DURATION seconds)"
91+
corpus="tests/fuzz/corpus/${name//_/-}"
92+
mkdir -p "$corpus"
93+
dict="tests/fuzz/${name//_/-}.dict"
94+
dict_flag=""
95+
[ -f "$dict" ] && dict_flag="-dict=$dict"
96+
if ! "$target" $dict_flag \
97+
-max_total_time="$DURATION" "$corpus" 2>&1; then
98+
echo "::error::Fuzz target $name found a crash!"
99+
EXIT_CODE=1
100+
fi
101+
echo "::endgroup::"
102+
done
103+
exit $EXIT_CODE
104+
- name: Upload crash artifacts
105+
if: failure()
106+
uses: actions/upload-artifact@v4
107+
with:
108+
name: fuzz-crashes-macos
109+
path: |
110+
crash-*
111+
leak-*
112+
timeout-*
113+
114+
fuzz-windows:
115+
# Best-effort: libFuzzer works on Windows via LLVM but ASAN/UBSAN
116+
# support is less reliable. Leak detection is not available.
117+
runs-on: windows-2022
118+
continue-on-error: true
119+
steps:
120+
- uses: actions/checkout@v4
121+
- name: Install LLVM
122+
run: choco install llvm -y
123+
- run: bash ./scripts/vendor.sh
124+
shell: bash
125+
- name: Generate sqlite-vec.h
126+
shell: bash
127+
run: make sqlite-vec.h
128+
- name: Build fuzz targets
129+
shell: bash
130+
run: |
131+
export PATH="/c/Program Files/LLVM/bin:$PATH"
132+
cd tests/fuzz
133+
mkdir -p targets
134+
for src in *.c; do
135+
name="${src%.c}"
136+
target_name="${name//-/_}"
137+
echo "Building $target_name from $src"
138+
clang -fsanitize=address,fuzzer \
139+
-I ../../ -I ../../vendor -DSQLITE_CORE -g \
140+
../../vendor/sqlite3.c ../../sqlite-vec.c \
141+
"$src" -o "targets/${target_name}.exe" || {
142+
echo "Warning: failed to build $target_name (best-effort)"
143+
}
144+
done
145+
- name: Run fuzz targets
146+
shell: bash
147+
run: |
148+
export PATH="/c/Program Files/LLVM/bin:$PATH"
149+
DURATION=${{ github.event.inputs.duration || '60' }}
150+
for target in tests/fuzz/targets/*.exe; do
151+
[ -f "$target" ] || continue
152+
name=$(basename "$target" .exe)
153+
echo "=== Fuzzing $name ($DURATION seconds) ==="
154+
corpus="tests/fuzz/corpus/${name//_/-}"
155+
mkdir -p "$corpus"
156+
"$target" -max_total_time="$DURATION" "$corpus" 2>&1 || {
157+
echo "Warning: $name found an issue or failed"
158+
}
159+
done
160+
- name: Upload crash artifacts
161+
if: failure()
162+
uses: actions/upload-artifact@v4
163+
with:
164+
name: fuzz-crashes-windows
165+
path: |
166+
tests/fuzz/crash-*
167+
tests/fuzz/leak-*

‎tests/fuzz/.gitignore‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,7 @@
11
*.dSYM
22
targets/
3+
corpus/
4+
crash-*
5+
leak-*
6+
timeout-*
7+
*.log

‎tests/fuzz/Makefile‎

Lines changed: 75 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -1,48 +1,83 @@
1+
# Auto-detect clang with libFuzzer support.
2+
# Priority: Homebrew LLVM (macOS ARM) → Homebrew LLVM (macOS Intel) →
3+
# versioned clang (Linux) → system clang
4+
FUZZ_CC ?= $(shell \
5+
if [ -x /opt/homebrew/opt/llvm/bin/clang ]; then \
6+
echo "/opt/homebrew/opt/llvm/bin/clang"; \
7+
elif [ -x /usr/local/opt/llvm/bin/clang ]; then \
8+
echo "/usr/local/opt/llvm/bin/clang"; \
9+
elif command -v clang-18 >/dev/null 2>&1; then \
10+
echo "clang-18"; \
11+
elif command -v clang-17 >/dev/null 2>&1; then \
12+
echo "clang-17"; \
13+
elif command -v clang >/dev/null 2>&1; then \
14+
echo "clang"; \
15+
else \
16+
echo "FUZZ_CC_NOT_FOUND"; \
17+
fi)
118

2-
TARGET_DIR=./targets
19+
# AddressSanitizer + UndefinedBehaviorSanitizer + libFuzzer.
20+
# Override FUZZ_SANITIZERS to change (e.g., drop ubsan on Windows).
21+
FUZZ_SANITIZERS ?= -fsanitize=address,undefined,fuzzer
22+
23+
# On macOS, Homebrew LLVM may need -Wl,-ld_classic to work with the system linker.
24+
FUZZ_LDFLAGS ?= $(shell \
25+
if [ "$$(uname -s)" = "Darwin" ]; then \
26+
echo "-Wl,-ld_classic"; \
27+
fi)
28+
29+
FUZZ_CFLAGS = $(FUZZ_SANITIZERS) -I ../../ -I ../../vendor -DSQLITE_CORE -g $(FUZZ_LDFLAGS)
30+
FUZZ_SRCS = ../../vendor/sqlite3.c ../../sqlite-vec.c
31+
32+
TARGET_DIR = ./targets
333

434
$(TARGET_DIR):
535
mkdir -p $@
636

7-
# ASAN_OPTIONS=detect_leaks=1 ./fuzz_json -detect_leaks=1 '-trace_malloc=[12]' tmp
8-
$(TARGET_DIR)/json: json.c $(TARGET_DIR)
9-
/opt/homebrew/opt/llvm/bin/clang \
10-
-fsanitize=address,fuzzer \
11-
-I ../../ -I ../../vendor -DSQLITE_CORE -g \
12-
../../vendor/sqlite3.c \
13-
../../sqlite-vec.c \
14-
$< \
15-
-o $@
16-
17-
18-
$(TARGET_DIR)/vec0_create: vec0-create.c ../../sqlite-vec.c $(TARGET_DIR)
19-
/opt/homebrew/opt/llvm/bin/clang \
20-
-fsanitize=address,fuzzer \
21-
-I ../../ -I ../../vendor -DSQLITE_CORE -g \
22-
../../vendor/sqlite3.c \
23-
../../sqlite-vec.c \
24-
$< \
25-
-o $@
26-
27-
$(TARGET_DIR)/numpy: numpy.c ../../sqlite-vec.c $(TARGET_DIR)
28-
/opt/homebrew/opt/llvm/bin/clang \
29-
-fsanitize=address,fuzzer \
30-
-I ../../ -I ../../vendor -DSQLITE_CORE -g \
31-
../../vendor/sqlite3.c \
32-
../../sqlite-vec.c \
33-
$< \
34-
-o $@
35-
36-
$(TARGET_DIR)/exec: exec.c ../../sqlite-vec.c $(TARGET_DIR)
37-
/opt/homebrew/opt/llvm/bin/clang \
38-
-fsanitize=address,fuzzer \
39-
-I ../../ -I ../../vendor -DSQLITE_CORE -g \
40-
../../vendor/sqlite3.c \
41-
../../sqlite-vec.c \
42-
$< \
43-
-o $@
44-
45-
all: $(TARGET_DIR)/json $(TARGET_DIR)/numpy $(TARGET_DIR)/json $(TARGET_DIR)/exec
37+
$(TARGET_DIR)/vec0_create: vec0-create.c $(FUZZ_SRCS) | $(TARGET_DIR)
38+
$(FUZZ_CC) $(FUZZ_CFLAGS) $(FUZZ_SRCS) $< -o $@
39+
40+
$(TARGET_DIR)/exec: exec.c $(FUZZ_SRCS) | $(TARGET_DIR)
41+
$(FUZZ_CC) $(FUZZ_CFLAGS) $(FUZZ_SRCS) $< -o $@
42+
43+
$(TARGET_DIR)/json: json.c $(FUZZ_SRCS) | $(TARGET_DIR)
44+
$(FUZZ_CC) $(FUZZ_CFLAGS) $(FUZZ_SRCS) $< -o $@
45+
46+
$(TARGET_DIR)/numpy: numpy.c $(FUZZ_SRCS) | $(TARGET_DIR)
47+
$(FUZZ_CC) $(FUZZ_CFLAGS) $(FUZZ_SRCS) $< -o $@
48+
49+
$(TARGET_DIR)/shadow_corrupt: shadow-corrupt.c $(FUZZ_SRCS) | $(TARGET_DIR)
50+
$(FUZZ_CC) $(FUZZ_CFLAGS) $(FUZZ_SRCS) $< -o $@
51+
52+
$(TARGET_DIR)/vec0_operations: vec0-operations.c $(FUZZ_SRCS) | $(TARGET_DIR)
53+
$(FUZZ_CC) $(FUZZ_CFLAGS) $(FUZZ_SRCS) $< -o $@
54+
55+
$(TARGET_DIR)/scalar_functions: scalar-functions.c $(FUZZ_SRCS) | $(TARGET_DIR)
56+
$(FUZZ_CC) $(FUZZ_CFLAGS) $(FUZZ_SRCS) $< -o $@
57+
58+
$(TARGET_DIR)/vec0_create_full: vec0-create-full.c $(FUZZ_SRCS) | $(TARGET_DIR)
59+
$(FUZZ_CC) $(FUZZ_CFLAGS) $(FUZZ_SRCS) $< -o $@
60+
61+
$(TARGET_DIR)/metadata_columns: metadata-columns.c $(FUZZ_SRCS) | $(TARGET_DIR)
62+
$(FUZZ_CC) $(FUZZ_CFLAGS) $(FUZZ_SRCS) $< -o $@
63+
64+
$(TARGET_DIR)/vec_each: vec-each.c $(FUZZ_SRCS) | $(TARGET_DIR)
65+
$(FUZZ_CC) $(FUZZ_CFLAGS) $(FUZZ_SRCS) $< -o $@
66+
67+
$(TARGET_DIR)/vec_mismatch: vec-mismatch.c $(FUZZ_SRCS) | $(TARGET_DIR)
68+
$(FUZZ_CC) $(FUZZ_CFLAGS) $(FUZZ_SRCS) $< -o $@
69+
70+
$(TARGET_DIR)/vec0_delete_completeness: vec0-delete-completeness.c $(FUZZ_SRCS) | $(TARGET_DIR)
71+
$(FUZZ_CC) $(FUZZ_CFLAGS) $(FUZZ_SRCS) $< -o $@
72+
73+
FUZZ_TARGETS = vec0_create exec json numpy \
74+
shadow_corrupt vec0_operations scalar_functions \
75+
vec0_create_full metadata_columns vec_each vec_mismatch \
76+
vec0_delete_completeness
77+
78+
all: $(addprefix $(TARGET_DIR)/,$(FUZZ_TARGETS))
4679

4780
clean:
4881
rm -rf $(TARGET_DIR)/*
82+
83+
.PHONY: all clean

0 commit comments

Comments
 (0)