[U-Boot-Custodians] Switch to GitLab-CI after GitLab?
Tom Rini
trini at konsulko.com
Mon Apr 22 13:25:21 UTC 2019
Hey all,
One of the things that moving to GitLab will bring us is easier access
to GitLab CI. I've got a proof-of-concept rewrite of our current
.travis.yml as a .gitlab-ci.yml file here:
# SPDX-License-Identifier: GPL-2.0+
# Grab our configured image
image: trini/u-boot-gitlab-ci-runner:xenial-20190222-10April2019
# We run some tests in different order, to catch some failures quicker.
stages:
- test.py
- testsuites
- world build
.buildman_and_testpy_template: &buildman_and_testpy_dfn
stage: test.py
before_script:
# Clone uboot-test-hooks
- git clone --depth=1 git://github.com/swarren/uboot-test-hooks.git /tmp/uboot-test-hooks
- ln -s travis-ci /tmp/uboot-test-hooks/bin/`hostname`
- ln -s travis-ci /tmp/uboot-test-hooks/py/`hostname`
- virtualenv /tmp/venv
- . /tmp/venv/bin/activate
- pip install pytest==2.8.7
- pip install python-subunit
- grub-mkimage -o ~/grub_x86.efi -O i386-efi normal echo lsefimmap lsefi lsefisystab efinet tftp minicmd
- grub-mkimage -o ~/grub_x64.efi -O x86_64-efi normal echo lsefimmap lsefi lsefisystab efinet tftp minicmd
- mkdir ~/grub2-arm
- ( cd ~/grub2-arm; wget -O - http://download.opensuse.org/ports/armv7hl/distribution/leap/42.2/repo/oss/suse/armv7hl/grub2-arm-efi-2.02~beta2-87.1.armv7hl.rpm | rpm2cpio | cpio -di )
- mkdir ~/grub2-arm64
- ( cd ~/grub2-arm64; wget -O - http://download.opensuse.org/ports/aarch64/distribution/leap/42.2/repo/oss/suse/aarch64/grub2-arm64-efi-2.02~beta2-87.1.aarch64.rpm | rpm2cpio | cpio -di )
- if [[ "${QEMU_TARGET}" != "" ]]; then
git clone git://git.qemu.org/qemu.git /tmp/qemu;
pushd /tmp/qemu;
git submodule update --init dtc &&
git checkout ${QEMU_VERSION} &&
./configure --prefix=/tmp/qemu-install --target-list=${QEMU_TARGET} &&
make -j$(nproc) all install;
popd;
fi
after_script:
- rm -rf ~/grub2* /tmp/uboot-test-hooks /tmp/qemu /tmp/venv
script:
# From buildman, exit code 129 means warnings only. If we've been asked to
# use clang only do one configuration.
- if [[ "${BUILDMAN}" != "" ]]; then
ret=0;
tools/buildman/buildman -P -E ${BUILDMAN} ${OVERRIDE}|| ret=$?;
if [[ $ret -ne 0 && $ret -ne 129 ]]; then
tools/buildman/buildman -sdeP ${BUILDMAN};
exit $ret;
fi;
fi
# "not a_test_which_does_not_exist" is a dummy -k parameter which will
# never prevent any test from running. That way, we can always pass
# "-k something" even when $TEST_PY_TEST_SPEC doesnt need a custom
# value.
- export UBOOT_TRAVIS_BUILD_DIR=`cd .. && pwd`/.bm-work/${TEST_PY_BD};
export PATH=/tmp/qemu-install/bin:/tmp/uboot-test-hooks/bin:/usr/bin:/bin;
export PYTHONPATH=/tmp/uboot-test-hooks/py/travis-ci;
if [[ "${TEST_PY_BD}" != "" ]]; then
./test/py/test.py --bd ${TEST_PY_BD} ${TEST_PY_ID}
-k "${TEST_PY_TEST_SPEC:-not a_test_which_does_not_exist}"
--build-dir "$UBOOT_TRAVIS_BUILD_DIR";
ret=$?;
if [[ $ret -ne 0 ]]; then
exit $ret;
fi;
fi;
build all plaforms:
stage: world build
script:
- ret=0;
./tools/buildman/buildman -P -E -x openrd || ret=$?;
if [[ $ret -ne 0 && $ret -ne 129 ]]; then
./tools/buildman/buildman -sdeP -x openrd;
exit $ret;
fi;
# QA jobs for code analytics
# static code analysis with cppcheck (we can add --enable=all later)
cppcheck:
stage: testsuites
script:
- cppcheck --force --quiet --inline-suppr .
# search for TODO within source tree
grep TODO/FIXME/HACK:
stage: testsuites
script:
- grep -r TODO .
- grep -r FIXME .
# search for HACK within source tree and ignore HACKKIT board
- grep -r HACK . | grep -v HACKKIT
# some statistics about the code base
sloccount:
stage: testsuites
script:
- sloccount .
# ensure all configs have MAINTAINERS entries
Check for configs without MAINTAINERS entry:
stage: testsuites
script:
- if [ `./tools/genboardscfg.py -f 2>&1 | wc -l` -ne 0 ]; then exit 1; fi
# Ensure host tools build
Build tools-only:
stage: testsuites
script:
- make tools-only_config tools-only -j$(nproc)
# Run various tool tests
Run patman testsuite:
stage: testsuites
script:
- git config --global user.name "GitLab CI Runner"
- git config --global user.email trini at konsulko.com
- ./tools/patman/patman --test
Run buildman testsuite:
stage: testsuites
script:
- ./tools/buildman/buildman -t
Run binman and dtoc testsuite:
stage: testsuites
script:
- export UBOOT_TRAVIS_BUILD_DIR=`cd .. && pwd`/.bm-work/sandbox_spl;
./tools/buildman/buildman -P sandbox_spl &&
export PYTHONPATH="${UBOOT_TRAVIS_BUILD_DIR}/scripts/dtc/pylibfdt";
export PATH="${UBOOT_TRAVIS_BUILD_DIR}/scripts/dtc:${PATH}";
./tools/binman/binman -t &&
./tools/dtoc/dtoc -t
# Test sandbox with test.py
sandbox test.py:
variables:
TEST_PY_BD: "sandbox"
BUILDMAN: "^sandbox$"
<<: *buildman_and_testpy_dfn
sandbox_spl test.py:
variables:
TEST_PY_BD: "sandbox_spl"
BUILDMAN: "^sandbox_spl$"
TEST_PY_TEST_SPEC: "test_ofplatdata"
<<: *buildman_and_testpy_dfn
sandbox_flattree test.py:
variables:
TEST_PY_BD: "sandbox_flattree"
BUILDMAN: "^sandbox_flattree$"
<<: *buildman_and_testpy_dfn
vexpress_ca15_tc2 test.py:
variables:
TEST_PY_BD: "vexpress_ca15_tc2"
TEST_PY_ID: "--id qemu"
QEMU_TARGET: "arm-softmmu"
QEMU_VERSION: "v3.0.0"
BUILDMAN: "^vexpress_ca15_tc2$"
<<: *buildman_and_testpy_dfn
vexpress_ca9x4 test.py:
variables:
TEST_PY_BD: "vexpress_ca9x4"
TEST_PY_ID: "--id qemu"
QEMU_TARGET: "arm-softmmu"
BUILDMAN: "^vexpress_ca9x4$"
<<: *buildman_and_testpy_dfn
integratorcp_cm926ejs test.py:
variables:
TEST_PY_BD: "integratorcp_cm926ejs"
TEST_PY_TEST_SPEC: "not sleep"
TEST_PY_ID: "--id qemu"
QEMU_TARGET: "arm-softmmu"
BUILDMAN: "^integratorcp_cm926ejs$"
<<: *buildman_and_testpy_dfn
qemu_arm test.py:
variables:
TEST_PY_BD: "qemu_arm"
TEST_PY_TEST_SPEC: "not sleep"
QEMU_TARGET: "arm-softmmu"
BUILDMAN: "^qemu_arm$"
<<: *buildman_and_testpy_dfn
qemu_arm64 test.py:
variables:
TEST_PY_BD: "qemu_arm64"
TEST_PY_TEST_SPEC: "not sleep"
QEMU_TARGET: "aarch64-softmmu"
BUILDMAN: "^qemu_arm64$"
<<: *buildman_and_testpy_dfn
qemu_mips test.py:
variables:
TEST_PY_BD: "qemu_mips"
TEST_PY_TEST_SPEC: "not sleep"
QEMU_TARGET: "mips-softmmu"
BUILDMAN: "^qemu_mips$"
TOOLCHAIN: "mips"
<<: *buildman_and_testpy_dfn
qemu_mipsel test.py:
variables:
TEST_PY_BD: "qemu_mipsel"
TEST_PY_TEST_SPEC: "not sleep"
QEMU_TARGET: "mipsel-softmmu"
BUILDMAN: "^qemu_mipsel$"
TOOLCHAIN: "mips"
<<: *buildman_and_testpy_dfn
qemu_mips64 test.py:
variables:
TEST_PY_BD: "qemu_mips64"
TEST_PY_TEST_SPEC: "not sleep"
QEMU_TARGET: "mips64-softmmu"
BUILDMAN: "^qemu_mips64$"
TOOLCHAIN: "mips"
<<: *buildman_and_testpy_dfn
qemu_mips64el test.py:
variables:
TEST_PY_BD: "qemu_mips64el"
TEST_PY_TEST_SPEC: "not sleep"
QEMU_TARGET: "mips64el-softmmu"
BUILDMAN: "^qemu_mips64el$"
TOOLCHAIN: "mips"
<<: *buildman_and_testpy_dfn
qemu-ppce500 test.py:
variables:
TEST_PY_BD: "qemu-ppce500"
TEST_PY_TEST_SPEC: "not sleep"
QEMU_TARGET: "ppc-softmmu"
BUILDMAN: "^qemu-ppce500$"
TOOLCHAIN: "powerpc"
<<: *buildman_and_testpy_dfn
qemu-x86 test.py:
variables:
TEST_PY_BD: "qemu-x86"
TEST_PY_TEST_SPEC: "not sleep"
QEMU_TARGET: "i386-softmmu"
BUILDMAN: "^qemu-x86$"
TOOLCHAIN: "i386"
<<: *buildman_and_testpy_dfn
qemu-x86_64 test.py:
variables:
TEST_PY_BD: "qemu-x86_64"
TEST_PY_TEST_SPEC: "not sleep"
QEMU_TARGET: "x86_64-softmmu"
BUILDMAN: "^qemu-x86_64$"
TOOLCHAIN: "i386"
<<: *buildman_and_testpy_dfn
zynq_zc702 test.py:
variables:
TEST_PY_BD: "zynq_zc702"
TEST_PY_TEST_SPEC: "not sleep"
QEMU_TARGET: "arm-softmmu"
TEST_PY_ID: "--id qemu"
BUILDMAN: "^zynq_zc702$"
<<: *buildman_and_testpy_dfn
xilinx_versal_virt test.py:
variables:
TEST_PY_BD: "xilinx_versal_virt"
TEST_PY_TEST_SPEC: "not sleep"
QEMU_TARGET: "aarch64-softmmu"
TEST_PY_ID: "--id qemu"
BUILDMAN: "^xilinx_versal_virt$"
<<: *buildman_and_testpy_dfn
xtfpga test.py:
variables:
TEST_PY_BD: "xtfpga"
TEST_PY_TEST_SPEC: "not sleep"
QEMU_TARGET: "xtensa-softmmu"
TEST_PY_ID: "--id qemu"
BUILDMAN: "^xtfpga$"
TOOLCHAIN: "xtensa-dc233c-elf"
<<: *buildman_and_testpy_dfn
It's using a Docker container with a Dockerfile that looks like:
# SPDX-License-Identifier: GPL-2.0+
# This Dockerfile is used to build an image containing basic stuff to be used
# to build U-Boot and run our test suites.
FROM ubuntu:xenial-20190222
MAINTAINER Tom Rini <trini at konsulko.com>
LABEL Description=" This image is for building U-Boot inside a container"
# Make sure apt is happy
ENV DEBIAN_FRONTEND=noninteractive
# Add LLVM repository
RUN apt-get update && apt-get install -y wget xz-utils && rm -rf /var/lib/apt/lists/*
RUN wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add -
RUN echo deb http://apt.llvm.org/xenial/ llvm-toolchain-xenial-7 main | tee /etc/apt/sources.list.d/llvm.list
# Manually install the kernel.org "Crosstool" based toolchains for gcc-7.3
RUN wget -O - https://mirrors.edge.kernel.org/pub/tools/crosstool/files/bin/x86_64/7.3.0/x86_64-gcc-7.3.0-nolibc_aarch64-linux.tar.xz | tar -C /opt -xJ
RUN wget -O - https://mirrors.edge.kernel.org/pub/tools/crosstool/files/bin/x86_64/7.3.0/x86_64-gcc-7.3.0-nolibc_arm-linux-gnueabi.tar.xz | tar -C /opt -xJ
RUN wget -O - https://mirrors.edge.kernel.org/pub/tools/crosstool/files/bin/x86_64/7.3.0/x86_64-gcc-7.3.0-nolibc_i386-linux.tar.xz | tar -C /opt -xJ
RUN wget -O - https://mirrors.edge.kernel.org/pub/tools/crosstool/files/bin/x86_64/7.3.0/x86_64-gcc-7.3.0-nolibc_m68k-linux.tar.xz | tar -C /opt -xJ
RUN wget -O - https://mirrors.edge.kernel.org/pub/tools/crosstool/files/bin/x86_64/7.3.0/x86_64-gcc-7.3.0-nolibc_mips-linux.tar.xz | tar -C /opt -xJ
RUN wget -O - https://mirrors.edge.kernel.org/pub/tools/crosstool/files/bin/x86_64/7.3.0/x86_64-gcc-7.3.0-nolibc_microblaze-linux.tar.xz | tar -C /opt -xJ
RUN wget -O - https://mirrors.edge.kernel.org/pub/tools/crosstool/files/bin/x86_64/7.3.0/x86_64-gcc-7.3.0-nolibc_nios2-linux.tar.xz | tar -C /opt -xJ
RUN wget -O - https://mirrors.edge.kernel.org/pub/tools/crosstool/files/bin/x86_64/7.3.0/x86_64-gcc-7.3.0-nolibc_powerpc-linux.tar.xz | tar -C /opt -xJ
RUN wget -O - https://mirrors.edge.kernel.org/pub/tools/crosstool/files/bin/x86_64/7.3.0/x86_64-gcc-7.3.0-nolibc_riscv64-linux.tar.xz | tar -C /opt -xJ
RUN wget -O - https://mirrors.edge.kernel.org/pub/tools/crosstool/files/bin/x86_64/7.3.0/x86_64-gcc-7.3.0-nolibc_sh2-linux.tar.xz | tar -C /opt -xJ
# Manually install other toolchains
RUN wget -O - https://github.com/foss-xtensa/toolchain/releases/download/2018.02/x86_64-2018.02-xtensa-dc233c-elf.tar.gz | tar -C /opt -xz
RUN wget -O - https://github.com/foss-for-synopsys-dwc-arc-processors/toolchain/releases/download/arc-2018.09-release/arc_gnu_2018.09_prebuilt_uclibc_le_archs_linux_install.tar.gz | tar -C /opt -xz
RUN wget -O - https://github.com/vincentzwc/prebuilt-nds32-toolchain/releases/download/20180521/nds32le-linux-glibc-v3-upstream.tar.gz | tar -C /opt -xz
# Updat and install things from apt now
RUN apt-get update && apt-get install -y \
bc \
bison \
build-essential \
clang-7 \
cpio \
cppcheck \
curl \
device-tree-compiler \
dosfstools \
e2fsprogs \
flex \
gdisk \
git \
grub-efi-amd64-bin \
grub-efi-ia32-bin \
iasl \
iputils-ping \
liblz4-tool \
libpixman-1-dev \
libpython-dev \
libsdl1.2-dev \
libssl-dev \
libudev-dev \
libusb-1.0-0-dev \
lzop \
picocom \
python \
python-coverage \
python-dev \
python-pip \
python-pytest \
python-virtualenv \
rpm2cpio \
sloccount \
sparse \
sudo \
swig \
virtualenv \
zip \
&& rm -rf /var/lib/apt/lists/*
# Create the buildman config file
RUN /bin/echo -e "[toolchain]\nroot = /usr" > ~/.buildman
RUN /bin/echo -e "kernelorg = /opt/gcc-7.3.0-nolibc/*" >> ~/.buildman
RUN /bin/echo -e "arc = /opt/arc_gnu_2018.09_prebuilt_uclibc_le_archs_linux_install" >> ~/.buildman
RUN /bin/echo -e "\n[toolchain-prefix]\nxtensa = /opt/2018.02/xtensa-dc233c-elf/bin/xtensa-dc233c-elf-" >> ~/.buildman;
RUN /bin/echo -e "\nnds32 = /opt/nds32le-linux-glibc-v3-upstream/bin/nds32le-linux-" >> ~/.buildman;
RUN /bin/echo -e "\n[toolchain-alias]\nsh = sh2" >> ~/.buildman
RUN /bin/echo -e "\nriscv = riscv64" >> ~/.buildman
RUN /bin/echo -e "\nsandbox = x86_64" >> ~/.buildman
RUN /bin/echo -e "\nx86 = i386" >> ~/.buildman;
This shows a few things that are a positive change over Travis. First,
we can define pipelines in the job which are done in order. In this
version, all of the test.py jobs go first, as that gives us some good
build coverage and also functional testing. After that all of the other
testsuites run. The next thing this brings us is the final job in the
pipeline, one world build of everything.
With GitLab we'll control the runners and as such, won't hit the "free
compute, time is up" limits. Where will we build however? Good
question! Konsulko is going to provide a build box that should get the
whole suite run in 2 hours or so. I don't have exact times since
we're putting that box together still, but based on other build machines
we have, that's a reasonable time. It may be quicker. It's also
entirely possible that other folks could contribute a "runner" in GitLab
CI terms, if they want. It's also very possible to configure a runner
on your own machine, for your own testing.
--
Tom
-------------- next part --------------
A non-text attachment was scrubbed...
Name: signature.asc
Type: application/pgp-signature
Size: 819 bytes
Desc: not available
URL: <http://lists.denx.de/pipermail/u-boot-custodians/attachments/20190422/4283f931/attachment.sig>
More information about the U-Boot-Custodians
mailing list