Compare commits

...

6 Commits

Author SHA1 Message Date
Evgeny Vereshchagin 135a1add7b turn off fuzzit part 2
it's just a follow-up to https://github.com/systemd/systemd/pull/16064
2020-06-11 18:59:44 +02:00
Evgeny Vereshchagin 60950f3c25
Merge pull request #16144 from mrc0mmand/gh-actions-followup
GH Actions followup
2020-06-11 18:04:13 +03:00
Frantisek Sumsal 86a23f38ab ci: tweak the dependency installation 2020-06-11 16:11:26 +02:00
Frantisek Sumsal 3d0a45d5da ci: tweak the compilation options
Build each build with tests (slow tests and fuzzer tests as well), and
combine the LTO build with -O3.
2020-06-11 16:11:19 +02:00
Frantisek Sumsal 4e1a13db89 ci: move the build check script to workflows directory 2020-06-11 14:17:40 +02:00
Frantisek Sumsal 0664249ffc ci: do the compiler-detection magic in the test script
so we have all PPA definitions at one place.
2020-06-11 14:17:37 +02:00
6 changed files with 122 additions and 192 deletions

View File

@ -10,7 +10,6 @@ on:
- 'meson_options.txt'
- 'src/**'
- 'test/fuzz/**'
- 'travis-ci/managers/ubuntu-build-check.sh'
jobs:
build:
@ -19,26 +18,11 @@ jobs:
fail-fast: false
matrix:
env:
# As we use postfixed clang/gcc binaries, we need to override $AR
# as well, otherwise meson falls back to ar from binutils which
# doesn't work with LTO
- { CC: "clang-10", CXX: "clang++-10", AR: "llvm-ar-10" }
- { CC: "gcc-10", CXX: "g++-10", AR: "gcc-ar-10" }
- { COMPILER: "gcc", COMPILER_VERSION: "10" }
- { COMPILER: "clang", COMPILER_VERSION: "10" }
env: ${{ matrix.env }}
steps:
- name: Repository checkout
uses: actions/checkout@v1
- name: Configure custom APT repositories for ${{ env.CC }}
run: |
if [[ "$CC" == clang-* ]]; then
# Latest LLVM stack deb packages provided by https://apt.llvm.org/
sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)"
sudo apt-get install clang-10 llvm-10
else
# Latest gcc stack deb packages provided by
# https://launchpad.net/~ubuntu-toolchain-r/+archive/ubuntu/test
sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test
sudo apt-get install gcc-10
fi
- name: Build check (${{ env.CC }})
run: sudo -E travis-ci/managers/ubuntu-build-check.sh
- name: Build check (${{ env.COMPILER }}-${{ env.COMPILER_VERSION }})
run: sudo -E .github/workflows/ubuntu-build-check.sh

117
.github/workflows/ubuntu-build-check.sh vendored Executable file
View File

@ -0,0 +1,117 @@
#!/bin/bash
set -ex
info() { echo -e "\033[33;1m$1\033[0m"; }
fatal() { echo >&2 -e "\033[31;1m$1\033[0m"; exit 1; }
success() { echo >&2 -e "\033[32;1m$1\033[0m"; }
ARGS=(
"--optimization=0"
"--optimization=2"
"--optimization=s"
"--optimization=3 -Db_lto=true"
"--optimization=3 -Db_lto=false"
"-Db_ndebug=true"
)
PACKAGES=(
cryptsetup-bin
expect
fdisk
gettext
iptables-dev
iputils-ping
isc-dhcp-client
itstool
kbd
libblkid-dev
libcap-dev
libcurl4-gnutls-dev
libfdisk-dev
libgpg-error-dev
liblz4-dev
liblzma-dev
libmicrohttpd-dev
libmount-dev
libp11-kit-dev
libpwquality-dev
libqrencode-dev
libssl-dev
libxkbcommon-dev
libzstd-dev
mount
net-tools
perl
python-lxml
python3-evdev
python3-lxml
python3-pip
python3-pyparsing
python3-setuptools
quota
strace
unifont
util-linux
zstd
)
COMPILER="${COMPILER:?}"
COMPILER_VERSION="${COMPILER_VERSION:?}"
RELEASE="$(lsb_release -cs)"
bash -c "echo 'deb-src http://archive.ubuntu.com/ubuntu/ $RELEASE main restricted universe multiverse' >>/etc/apt/sources.list"
# Note: As we use postfixed clang/gcc binaries, we need to override $AR
# as well, otherwise meson falls back to ar from binutils which
# doesn't work with LTO
if [[ "$COMPILER" == clang ]]; then
CC="clang-$COMPILER_VERSION"
CXX="clang++-$COMPILER_VERSION"
AR="llvm-ar-$COMPILER_VERSION"
# Latest LLVM stack deb packages provided by https://apt.llvm.org/
# Following snippet was borrowed from https://apt.llvm.org/llvm.sh
wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add -
add-apt-repository -y "deb http://apt.llvm.org/$RELEASE/ llvm-toolchain-$RELEASE-$COMPILER_VERSION main"
PACKAGES+=(clang-$COMPILER_VERSION lldb-$COMPILER_VERSION lld-$COMPILER_VERSION clangd-$COMPILER_VERSION)
elif [[ "$COMPILER" == gcc ]]; then
CC="gcc-$COMPILER_VERSION"
CXX="g++-$COMPILER_VERSION"
AR="gcc-ar-$COMPILER_VERSION"
# Latest gcc stack deb packages provided by
# https://launchpad.net/~ubuntu-toolchain-r/+archive/ubuntu/test
sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test
PACKAGES+=(gcc-$COMPILER_VERSION)
else
fatal "Unknown compiler: $COMPILER"
fi
# PPA with some newer build dependencies (like zstd)
add-apt-repository -y ppa:upstream-systemd-ci/systemd-ci
apt-get -y update
apt-get -y build-dep systemd
apt-get -y install "${PACKAGES[@]}"
# Install the latest meson and ninja form pip, since the distro versions don't
# support all the features we need (like --optimization=). Since the build-dep
# command above installs the distro versions, let's install the pip ones just
# locally and add the local bin directory to the $PATH.
pip3 install --user -U meson ninja
export PATH="$HOME/.local/bin:$PATH"
$CC --version
for args in "${ARGS[@]}"; do
SECONDS=0
info "Checking build with $args"
if ! AR="$AR" CC="$CC" CXX="$CXX" meson -Dtests=unsafe -Dslow-tests=true --werror $args build; then
fatal "meson failed with $args"
fi
ninja --version
if ! ninja -C build; then
fatal "ninja failed with $args"
fi
git clean -dxf
success "Build with $args passed in $SECONDS seconds"
done

View File

@ -6,7 +6,6 @@ System and Service Manager
<a href="https://in.waw.pl/systemd-github-state/systemd-systemd-pull-requests.svg"><img align="right" src="https://in.waw.pl/systemd-github-state/systemd-systemd-pull-requests-small.svg" alt="Count of open pull requests over time"></a>
[![Semaphore CI Build Status](https://semaphoreci.com/api/v1/projects/28a5a3ca-3c56-4078-8b5e-7ed6ef912e14/443470/shields_badge.svg)](https://semaphoreci.com/systemd/systemd)<br/>
[![Coverity Scan Status](https://scan.coverity.com/projects/350/badge.svg)](https://scan.coverity.com/projects/350)<br/>
[![Fuzzit Status](https://app.fuzzit.dev/badge?org_id=systemd&branch=master)](https://app.fuzzit.dev/orgs/systemd/dashboard)<br/>
[![OSS-Fuzz Status](https://oss-fuzz-build-logs.storage.googleapis.com/badges/systemd.svg)](https://oss-fuzz-build-logs.storage.googleapis.com/index.html#systemd)<br/>
[![CIFuzz](https://github.com/systemd/systemd/workflows/CIFuzz/badge.svg)](https://github.com/systemd/systemd/actions)<br/>
[![CII Best Practices](https://bestpractices.coreinfrastructure.org/projects/1369/badge)](https://bestpractices.coreinfrastructure.org/projects/1369)<br/>

View File

@ -98,7 +98,7 @@ Happy hacking!
## Fuzzers
systemd includes fuzzers in `src/fuzz/` that use libFuzzer and are automatically
run by [OSS-Fuzz](https://github.com/google/oss-fuzz) and [Fuzzit](https://fuzzit.dev) with sanitizers.
run by [OSS-Fuzz](https://github.com/google/oss-fuzz) with sanitizers.
To add a fuzz target, create a new `src/fuzz/fuzz-foo.c` file with a `LLVMFuzzerTestOneInput`
function and add it to the list in `src/fuzz/meson.build`.
@ -118,10 +118,6 @@ python infra/helper.py build_fuzzers --sanitizer memory systemd ../systemd
python infra/helper.py run_fuzzer systemd fuzz-foo
```
When you add a new target you should also add the target on [Fuzzit](https://app.fuzzit.dev/admin/RxqRpGNXquIvqrmp4iJS/dashboard)
(Please ask someone with permissions). One the target is configured on Fuzzit you need to add it to
`travis-ci/managers/fuzzit.sh` so the new target will run sanity tests on every pull-request and periodic fuzzing jobs.
If you find a bug that impacts the security of systemd, please follow the
guidance in [CONTRIBUTING.md](CONTRIBUTING.md) on how to report a security vulnerability.

View File

@ -1,73 +0,0 @@
#!/bin/bash
set -e
set -x
set -u
# This should help to protect the systemd organization on Fuzzit from forks
# that are activated on Travis CI.
[[ "$TRAVIS_REPO_SLUG" = "systemd/systemd" ]] || exit 0
REPO_ROOT=${REPO_ROOT:-$(pwd)}
sudo bash -c "echo 'deb-src http://archive.ubuntu.com/ubuntu/ xenial main restricted universe multiverse' >>/etc/apt/sources.list"
sudo apt-get update -y
sudo apt-get build-dep systemd -y
sudo apt-get install -y python3-pip python3-setuptools
# The following should be dropped when debian packaging has been updated to include them
sudo apt-get install -y libfdisk-dev libp11-kit-dev libssl-dev libpwquality-dev
pip3 install ninja meson
cd $REPO_ROOT
export PATH="$HOME/.local/bin/:$PATH"
# TODO: figure out what to do about unsigned-integer-overflow: https://github.com/google/oss-fuzz/issues/910
export SANITIZER="address -fsanitize=undefined,unsigned-integer-overflow"
tools/oss-fuzz.sh
FUZZING_TYPE=${1:-regression}
if [ "$TRAVIS_PULL_REQUEST" = "false" ]; then
FUZZIT_BRANCH="${TRAVIS_BRANCH}"
else
FUZZIT_BRANCH="PR-${TRAVIS_PULL_REQUEST}"
fi
# Because we want Fuzzit to run on every pull-request and Travis/Azure doesn't support encrypted keys
# on pull-request we use a write-only key which is ok for now. maybe there will be a better solution in the future
export FUZZIT_API_KEY=af6992074353998676713818cc6435ef4a750439932dab58b51e9354d6742c54d740a3cd9fc1fc001db82f51734a24bc
FUZZIT_ADDITIONAL_FILES="./out/src/shared/libsystemd-shared-*.so"
# ASan options are borrowed almost verbatim from OSS-Fuzz
ASAN_OPTIONS=redzone=32:print_summary=1:handle_sigill=1:allocator_release_to_os_interval_ms=500:print_suppressions=0:strict_memcmp=1:allow_user_segv_handler=0:allocator_may_return_null=1:use_sigaltstack=1:handle_sigfpe=1:handle_sigbus=1:detect_stack_use_after_return=1:alloc_dealloc_mismatch=0:detect_leaks=1:print_scariness=1:max_uar_stack_size_log=16:handle_abort=1:check_malloc_usable_size=0:quarantine_size_mb=64:detect_odr_violation=0:handle_segv=1:fast_unwind_on_fatal=0
UBSAN_OPTIONS=print_stacktrace=1:print_summary=1:halt_on_error=1:silence_unsigned_overflow=1
FUZZIT_ARGS="--type ${FUZZING_TYPE} --branch ${FUZZIT_BRANCH} --revision ${TRAVIS_COMMIT} -e ASAN_OPTIONS=${ASAN_OPTIONS} -e UBSAN_OPTIONS=${UBSAN_OPTIONS}"
wget -O fuzzit https://github.com/fuzzitdev/fuzzit/releases/latest/download/fuzzit_Linux_x86_64
chmod +x fuzzit
# Simple wrapper which retries given command up to three times if it fails
_retry() {
local EC=1
for _ in {0..2}; do
if "$@"; then
EC=0
break
fi
sleep 1
done
return $EC
}
find out/ -maxdepth 1 -name 'fuzz-*' -executable -type f -exec basename '{}' \; | while read -r fuzzer; do
_retry ./fuzzit create job ${FUZZIT_ARGS} ${fuzzer}-asan-ubsan out/${fuzzer} ${FUZZIT_ADDITIONAL_FILES}
done
export SANITIZER="memory -fsanitize-memory-track-origins"
FUZZIT_ARGS="--type ${FUZZING_TYPE} --branch ${FUZZIT_BRANCH} --revision ${TRAVIS_COMMIT}"
tools/oss-fuzz.sh
find out/ -maxdepth 1 -name 'fuzz-*' -executable -type f -exec basename '{}' \; | while read -r fuzzer; do
_retry ./fuzzit create job ${FUZZIT_ARGS} ${fuzzer}-msan out/${fuzzer} ${FUZZIT_ADDITIONAL_FILES}
done

View File

@ -1,93 +0,0 @@
#!/bin/bash
set -e
info() { echo -e "\033[33;1m$1\033[0m"; }
error() { echo >&2 -e "\033[31;1m$1\033[0m"; }
success() { echo >&2 -e "\033[32;1m$1\033[0m"; }
ARGS=(
"--optimization=0"
"--optimization=2"
"--optimization=3"
"--optimization=s"
"-Db_lto=true"
"-Db_ndebug=true"
)
PACKAGES=(
cryptsetup-bin
expect
fdisk
gettext
iptables-dev
iputils-ping
isc-dhcp-client
itstool
kbd
libblkid-dev
libcap-dev
libcurl4-gnutls-dev
libfdisk-dev
libgpg-error-dev
liblz4-dev
liblzma-dev
libmicrohttpd-dev
libmount-dev
libp11-kit-dev
libpwquality-dev
libqrencode-dev
libssl-dev
libxkbcommon-dev
libzstd-dev
mount
net-tools
ninja-build
perl
python-lxml
python3-evdev
python3-lxml
python3-pip
python3-pyparsing
python3-setuptools
quota
strace
unifont
util-linux
zstd
)
CC="${CC:?}"
CXX="${CXX:?}"
AR="${AR:-""}"
RELEASE="$(lsb_release -cs)"
bash -c "echo 'deb-src http://archive.ubuntu.com/ubuntu/ $RELEASE main restricted universe multiverse' >>/etc/apt/sources.list"
# PPA with some newer build dependencies (like zstd)
add-apt-repository -y ppa:upstream-systemd-ci/systemd-ci
apt-get update
apt-get build-dep systemd -y
apt-get install -y "${PACKAGES[@]}"
# Install latest meson from pip, as the distro-one doesn't support
# --optimization=
pip3 install meson
$CC --version
for args in "${ARGS[@]}"; do
SECONDS=0
info "Checking build with $args"
if ! AR="$AR" CC="$CC" CXX="$CXX" meson --werror $args build; then
error "meson failed with $args"
exit 1
fi
if ! ninja -C build; then
error "ninja failed with $args"
exit 1
fi
git clean -dxf
success "Build with $args passed in $SECONDS seconds"
done