Reimplement new pipeline and allow limiting number of cores used fo build

Co-authored-by: belliash <belliash@codingworkshop.eu.org>
Reviewed-on: xt-sys/xtchain#3
Reviewed-by: Piotr Likoski <likoski@noreply.codingworkshop.git>

.build.yml

@@ -1,8 +1,14 @@
-kind: pipeline
-type: exec
-name: XT Toolchain Pipeline
-
-steps:
-- name: Compile XT Toolchain
-  commands:
-    - sh build-linux.sh
+pipeline:
+  build:
+    image: codingworkshop/oscw-runner:latest
+    commands:
+      - CORES=10 ./build-linux.sh
+  publish:
+    image: codingworkshop/oscw-runner:latest
+    secrets:
+      - OSCW_GITHUB_USERNAME
+      - OSCW_GITHUB_PASSWORD
+    when:
+      event: tag
+    commands:
+      - github_publish $OSCW_GITHUB_USERNAME $OSCW_GITHUB_PASSWORD $(ls xtchain-*-linux.tar.zst)

README.md

@@ -1,7 +1,7 @@
 ## XT Toolchain
 This is a LLVM/Clang/LLD based mingw-w64 toolchain. It currently supports C and C++, and provides
 a variety of tools including IDL, message and resource compilers. The XT Toolchain is also the
-official build environment for compiling XT software, including the FerretOS. Currently, it is
+official build environment for compiling XT software, including the XT OS. Currently, it is
 targeted at Linux host only, however it should be possible to build it in MSYS2 as well.
 
 Benefits of a LLVM based MinGW toolchain are:

build-linux.sh

@@ -18,12 +18,12 @@ GENERIC="generic-w64-mingw32"
 
 # CMake Settings
 CMAKEDIR="${SRCDIR}/cmake"
-CMAKETAG="v3.19.2"
+CMAKETAG="v3.23.1"
 CMAKEVCS="https://gitlab.kitware.com/cmake/cmake.git"
 
 # LLVM Settings
 LLVMDIR="${SRCDIR}/llvm"
-LLVMTAG="llvmorg-11.0.0"
+LLVMTAG="llvmorg-14.0.6"
 LLVMVCS="https://github.com/llvm/llvm-project.git"
 
 # Make Settings
@@ -34,7 +34,7 @@ MAKEVCS="git://git.savannah.gnu.org/make"
 # Mingw-w64 Settings
 MINGWDIR="${SRCDIR}/mingw-w64"
 MINGWLIB="ucrt"
-MINGWTAG="v8.0.0"
+MINGWTAG="v10.0.0"
 MINGWNTV="0x601"
 MINGWVCS="https://github.com/mirror/mingw-w64.git"
 
@@ -45,12 +45,12 @@ NASMVCS="https://github.com/netwide-assembler/nasm.git"
 
 # Ninja Settings
 NINJADIR="${SRCDIR}/ninja"
-NINJATAG="v1.10.0"
+NINJATAG="v1.11.0"
 NINJAVCS="https://github.com/ninja-build/ninja.git"
 
 # Wine Settings
 WINEDIR="${SRCDIR}/wine"
-WINETAG="wine-5.15"
+WINETAG="wine-7.9"
 WINEVCS="git://source.winehq.org/git/wine.git"
 
 
@@ -129,7 +129,7 @@ llvm_build()
     done
     LLVM_ARCHS=( $(for ARCH in ${LLVM_ARCHS[@]}; do echo ${ARCH}; done | sort -u) )
     cd ${LLVMDIR}/llvm/tools
-    for UTIL in clang lld lldb; do
+    for UTIL in clang lld; do
         if [ ! -e ${UTIL} ]; then
             ln -sf ../../${UTIL} .
         fi
@@ -139,28 +139,26 @@ llvm_build()
     cmake \
         -DCMAKE_BUILD_TYPE="Release" \
         -DCMAKE_INSTALL_PREFIX=${BINDIR} \
-        -DLLDB_ENABLE_CURSES=FALSE \
-        -DLLDB_ENABLE_LIBEDIT=FALSE \
-        -DLLDB_ENABLE_LUA=FALSE \
-        -DLLDB_ENABLE_PYTHON=FALSE \
         -DLLDB_INCLUDE_TESTS=FALSE \
         -DLLVM_ENABLE_ASSERTIONS=FALSE \
-        -DLLVM_INSTALL_TOOLCHAIN_ONLY=TRUE \
+        -DLLVM_ENABLE_PROJECTS="clang;clang-tools-extra;lld" \
+        -DLLVM_INSTALL_TOOLCHAIN_ONLY=ON \
+        -DLLVM_LINK_LLVM_DYLIB=ON \
         -DLLVM_TARGETS_TO_BUILD="$(echo ${LLVM_ARCHS[@]} | tr ' ' ';')" \
-        -DLLVM_TOOLCHAIN_TOOLS="clang;llvm-addr2line;llvm-ar;llvm-as;llvm-cov;llvm-cvtres;llvm-dlltool;llvm-nm;llvm-objdump;llvm-objcopy;llvm-pdbutil;llvm-profdata;llvm-ranlib;llvm-rc;llvm-readobj;llvm-strings;llvm-strip;llvm-symbolizer" \
+        -DLLVM_TOOLCHAIN_TOOLS="llvm-addr2line;llvm-ar;llvm-as;llvm-cov;llvm-cvtres;llvm-dlltool;llvm-ml;llvm-nm;llvm-objdump;llvm-objcopy;llvm-pdbutil;llvm-profdata;llvm-ranlib;llvm-rc;llvm-readelf;llvm-readobj;llvm-strings;llvm-strip;llvm-symbolizer;llvm-windres" \
         ..
         make -j${CORES} install/strip
         cd ${WRKDIR}
 }
 
-# This function compiles and install LIBCXX
-llvm_build_libcxx()
+# This function compiles and install LIBCXX & LIBUNWIND
+llvm_build_libs()
 {
     echo ">>> Building LLVM libraries (libcxx) ..."
     for ARCH in ${ARCHS}; do
-        [ -z ${CLEAN} ] || rm -rf ${LLVMDIR}/libcxx/build-${ARCH}
-        mkdir -p ${LLVMDIR}/libcxx/build-${ARCH}
-        cd ${LLVMDIR}/libcxx/build-${ARCH}
+        [ -z ${CLEAN} ] || rm -rf ${LLVMDIR}/runtimes/build-${ARCH}
+        mkdir -p ${LLVMDIR}/runtimes/build-${ARCH}
+        cd ${LLVMDIR}/runtimes/build-${ARCH}
         cmake \
             -DCMAKE_BUILD_TYPE="Release" \
             -DCMAKE_INSTALL_PREFIX=${BINDIR}/${ARCH}-w64-mingw32 \
@@ -168,117 +166,32 @@ llvm_build_libcxx()
             -DCMAKE_C_COMPILER="${BINDIR}/bin/${ARCH}-w64-mingw32-clang" \
             -DCMAKE_C_COMPILER_WORKS=1 \
             -DCMAKE_CXX_COMPILER="${BINDIR}/bin/${ARCH}-w64-mingw32-clang++" \
+            -DCMAKE_CXX_COMPILER_TARGET=${ARCH}-w64-windows-gnu \
             -DCMAKE_CXX_COMPILER_WORKS=1 \
             -DCMAKE_CROSSCOMPILING=TRUE \
             -DCMAKE_RANLIB="${BINDIR}/bin/llvm-ranlib" \
-            -DCMAKE_SHARED_LINKER_FLAGS="-lunwind" \
             -DCMAKE_SYSTEM_NAME="Windows" \
             -DLLVM_PATH="${LLVMDIR}/llvm" \
-            -DLIBCXX_CXX_ABI="libcxxabi" \
-            -DLIBCXX_CXX_ABI_INCLUDE_PATHS="../../libcxxabi/include" \
-            -DLIBCXX_CXX_ABI_LIBRARY_PATH="../../libcxxabi/build-${ARCH}/lib" \
-            -DLIBCXX_ENABLE_ABI_LINKER_SCRIPT=FALSE \
-            -DLIBCXX_ENABLE_EXCEPTIONS=TRUE \
-            -DLIBCXX_ENABLE_EXPERIMENTAL_LIBRARY=FALSE \
-            -DLIBCXX_ENABLE_FILESYSTEM=FALSE \
-            -DLIBCXX_ENABLE_MONOTONIC_CLOCK=TRUE \
-            -DLIBCXX_ENABLE_SHARED=TRUE \
-            -DLIBCXX_ENABLE_STATIC=TRUE \
-            -DLIBCXX_ENABLE_STATIC_ABI_LIBRARY=TRUE \
-            -DLIBCXX_ENABLE_THREADS=TRUE \
-            -DLIBCXX_HAS_WIN32_THREAD_API=TRUE \
-            -DLIBCXX_HAVE_CXX_ATOMICS_WITHOUT_LIB=TRUE \
-            -DLIBCXX_INCLUDE_TESTS=FALSE \
-            -DLIBCXX_INSTALL_HEADERS=TRUE \
-            -DLIBCXX_LIBDIR_SUFFIX="" \
-            -DLIBCXX_USE_COMPILER_RT=TRUE \
-            ..
-        make -j${CORES}
-        make install
-        ${BINDIR}/bin/llvm-ar qcsL \
-            ${BINDIR}/${ARCH}-w64-mingw32/lib/libc++.dll.a \
-            ${BINDIR}/${ARCH}-w64-mingw32/lib/libunwind.dll.a
-        ${BINDIR}/bin/llvm-ar qcsL \
-            ${BINDIR}/${ARCH}-w64-mingw32/lib/libc++.a \
-            ${BINDIR}/${ARCH}-w64-mingw32/lib/libunwind.a
-        if [ ! -e ${BINDIR}/${ARCH}-w64-mingw32/bin ]; then
-            mkdir -p ${BINDIR}/${ARCH}-w64-mingw32/bin
-        fi
-        cp lib/libc++.dll ${BINDIR}/${ARCH}-w64-mingw32/bin/
-    done
-    cd ${WRKDIR}
-}
-
-# This function compiles LIBCXXABI
-llvm_build_libcxxabi()
-{
-    echo ">>> Building LLVM libraries (libcxxabi) ..."
-    for ARCH in ${ARCHS}; do
-        [ -z ${CLEAN} ] || rm -rf ${LLVMDIR}/libcxxabi/build-${ARCH}
-        mkdir -p ${LLVMDIR}/libcxxabi/build-${ARCH}
-        cd ${LLVMDIR}/libcxxabi/build-${ARCH}
-        cmake \
-            -DCMAKE_BUILD_TYPE="Release" \
-            -DCMAKE_INSTALL_PREFIX=${BINDIR}/${ARCH}-w64-mingw32 \
-            -DCMAKE_AR="${BINDIR}/bin/llvm-ar" \
-            -DCMAKE_C_COMPILER="${BINDIR}/bin/${ARCH}-w64-mingw32-clang" \
-            -DCMAKE_C_COMPILER_WORKS=1 \
-            -DCMAKE_CROSSCOMPILING=TRUE \
-            -DCMAKE_CXX_COMPILER="${BINDIR}/bin/${ARCH}-w64-mingw32-clang++" \
-            -DCMAKE_CXX_COMPILER_WORKS=1 \
-            -DCMAKE_CXX_FLAGS="-D_LIBCPP_HAS_THREAD_API_WIN32 -D_LIBCPP_DISABLE_VISIBILITY_ANNOTATIONS" \
-            -DCMAKE_RANLIB="${BINDIR}/bin/llvm-ranlib" \
-            -DCMAKE_SYSTEM_NAME="Windows" \
-            -DLLVM_PATH="${LLVMDIR}/llvm" \
-            -DLIBCXXABI_ENABLE_EXCEPTIONS=TRUE \
-            -DLIBCXXABI_ENABLE_NEW_DELETE_DEFINITIONS=FALSE \
-            -DLIBCXXABI_ENABLE_SHARED=FALSE \
-            -DLIBCXXABI_ENABLE_THREADS=TRUE \
-            -DLIBCXXABI_LIBCXX_INCLUDES="../../libcxx/include" \
-            -DLIBCXXABI_LIBDIR_SUFFIX="" \
-            -DLIBCXXABI_TARGET_TRIPLE="${ARCH}-w64-mingw32" \
-            -DLIBCXXABI_USE_COMPILER_RT=TRUE \
-            ..
-        make -j${CORES}
-    done
-    cd ${WRKDIR}
-}
-
-# This function compiles and installs LIBUNWIND
-llvm_build_libunwind()
-{
-    echo ">>> Building LLVM libraries (libunwind) ..."
-    for ARCH in ${ARCHS}; do
-        [ -z ${CLEAN} ] || rm -rf ${LLVMDIR}/libunwind/build-${ARCH}
-        mkdir -p ${LLVMDIR}/libunwind/build-${ARCH}
-        cd ${LLVMDIR}/libunwind/build-${ARCH}
-        cmake \
-            -DCMAKE_BUILD_TYPE="Release" \
-            -DCMAKE_INSTALL_PREFIX=${BINDIR}/${ARCH}-w64-mingw32 \
-            -DCMAKE_AR="${BINDIR}/bin/llvm-ar" \
-            -DCMAKE_C_COMPILER="${BINDIR}/bin/${ARCH}-w64-mingw32-clang" \
-            -DCMAKE_C_COMPILER_WORKS=1 \
-            -DCMAKE_C_FLAGS="-Wno-dll-attribute-on-redeclaration" \
-            -DCMAKE_CROSSCOMPILING=TRUE \
-            -DCMAKE_CXX_COMPILER="${BINDIR}/bin/${ARCH}-w64-mingw32-clang++" \
-            -DCMAKE_CXX_COMPILER_WORKS=1 \
-            -DCMAKE_CXX_FLAGS="-Wno-dll-attribute-on-redeclaration" \
-            -DCMAKE_SYSTEM_NAME="Windows" \
-            -DCMAKE_RANLIB="${BINDIR}/bin/llvm-ranlib" \
-            -DLLVM_COMPILER_CHECKED=TRUE \
-            -DLLVM_PATH="${LLVMDIR}/llvm" \
-            -DLIBUNWIND_ENABLE_CROSS_UNWINDING=FALSE \
-            -DLIBUNWIND_ENABLE_SHARED=TRUE \
-            -DLIBUNWIND_ENABLE_STATIC=TRUE \
-            -DLIBUNWIND_ENABLE_THREADS=TRUE \
+            -DLLVM_ENABLE_RUNTIMES="libunwind;libcxxabi;libcxx" \
             -DLIBUNWIND_USE_COMPILER_RT=TRUE \
+            -DLIBUNWIND_ENABLE_SHARED=TRUE \
+            -DLIBUNWIND_ENABLE_STATIC=FALSE \
+            -DLIBCXX_USE_COMPILER_RT=ON \
+            -DLIBCXX_ENABLE_SHARED=TRUE \
+            -DLIBCXX_ENABLE_STATIC=FALSE \
+            -DLIBCXX_ENABLE_EXPERIMENTAL_LIBRARY=OFF \
+            -DLIBCXX_ENABLE_STATIC_ABI_LIBRARY=TRUE \
+            -DLIBCXX_CXX_ABI=libcxxabi \
+            -DLIBCXX_LIBDIR_SUFFIX="" \
+            -DLIBCXX_INCLUDE_TESTS=FALSE \
+            -DLIBCXX_ENABLE_ABI_LINKER_SCRIPT=FALSE \
+            -DLIBCXXABI_USE_COMPILER_RT=ON \
+            -DLIBCXXABI_ENABLE_SHARED=OFF \
+            -DLIBCXXABI_LIBDIR_SUFFIX="" \
+            -DLIBCXXABI_USE_LLVM_UNWINDER=ON \
             ..
         make -j${CORES}
         make install
-        if [ ! -e ${BINDIR}/${ARCH}-w64-mingw32/bin ]; then
-            mkdir -p ${BINDIR}/${ARCH}-w64-mingw32/bin
-        fi
-        cp lib/libunwind.dll ${BINDIR}/${ARCH}-w64-mingw32/bin/
     done
     cd ${WRKDIR}
 }
@@ -291,32 +204,13 @@ llvm_build_runtime()
         [ -z ${CLEAN} ] || rm -rf ${LLVMDIR}/compiler-rt/build-${ARCH}
         mkdir -p ${LLVMDIR}/compiler-rt/build-${ARCH}
         cd ${LLVMDIR}/compiler-rt/build-${ARCH}
-        case ${ARCH} in
-            "armv7")
-                BARCH="armv7"
-                LARCH="arm"
-                ;;
-            "i686")
-                if [ ! -e ${BINDIR}/i386-w64-mingw32 ]; then
-                    ln -sf i686-w64-mingw32 ${BINDIR}/i386-w64-mingw32
-                fi
-                BARCH="i386"
-                LARCH="i386"
-                ;;
-            *)
-                BARCH="${ARCH}"
-                LARCH="${ARCH}"
-                ;;
-        esac
         cmake \
             -DCMAKE_BUILD_TYPE="Release" \
-            -DCMAKE_INSTALL_PREFIX=${BINDIR}/${ARCH}-w64-mingw32 \
             -DCMAKE_AR="${BINDIR}/bin/llvm-ar" \
             -DCMAKE_C_COMPILER="${BINDIR}/bin/${ARCH}-w64-mingw32-clang" \
-            -DCMAKE_C_COMPILER_TARGET="${BARCH}-windows-gnu" \
-            -DCMAKE_C_COMPILER_WORKS=1 \
+            -DCMAKE_C_COMPILER_TARGET="${ARCH}-windows-gnu" \
             -DCMAKE_CXX_COMPILER="${BINDIR}/bin/${ARCH}-w64-mingw32-clang++" \
-            -DCMAKE_CXX_COMPILER_WORKS=1 \
+            -DCMAKE_INSTALL_PREFIX=${BINDIR}/lib/clang/${LLVMTAG##*-} \
             -DCMAKE_RANLIB="${BINDIR}/bin/llvm-ranlib" \
             -DCMAKE_SYSTEM_NAME="Windows" \
             -DCOMPILER_RT_DEFAULT_TARGET_ONLY=TRUE \
@@ -324,10 +218,7 @@ llvm_build_runtime()
             -DSANITIZER_CXX_ABI=libc++ \
             ../lib/builtins
             make -j${CORES}
-            mkdir -p ${BINDIR}/lib/clang/${LLVMTAG#*-}/lib/windows
-            for LIB in lib/windows/libclang_rt.*.a; do
-                cp ${LIB} ${BINDIR}/lib/clang/${LLVMTAG#*-}/lib/windows/$(basename ${LIB} | sed s/${BARCH}/${LARCH}/)
-            done
+            make install
     done
     cd ${WRKDIR}
 }
@@ -557,9 +448,10 @@ wine_build()
     cd ${WINEDIR}/build
     ../configure \
         -enable-win64 \
+        --without-freetype \
         --without-x
     for TOOL in winedump wmc wrc; do
-        make -j${CORES} tools/${TOOL}
+        make -j${CORES} tools/${TOOL}/all
         cp tools/${TOOL}/${TOOL} ${BINDIR}/bin/
         for ARCH in ${ARCHS}; do
             if [ ! -e ${BINDIR}/bin/${ARCH}-w64-mingw32-${TOOL} ]; then
@@ -616,16 +508,12 @@ xtchain_build()
 # Exit immediately on any failure
 set -e
 
-# Check if script launched as root
-if [ "$(whoami)" = "root" ]; then
-    echo "This script cannot be run as root!"
-    exit 1
-fi
-
 # Check number of CPU cores available
-: ${CORES:=$(sysctl -n hw.ncpu 2>/dev/null)}
-: ${CORES:=$(nproc 2>/dev/null)}
-: ${CORES:=1}
+if [[ ! -n ${CORES} ]]; then
+	: ${CORES:=$(sysctl -n hw.ncpu 2>/dev/null)}
+	: ${CORES:=$(nproc 2>/dev/null)}
+	: ${CORES:=1}
+fi
 
 # Create working directories
 mkdir -p ${BINDIR}
@@ -658,17 +546,15 @@ mingw_build_crt
 # Build and install LLVM compiler runtime
 llvm_build_runtime
 
+# Build and install LLVM compiler libraries
+llvm_build_libs
+
 # Build and install Mingw-W64 libraries
 mingw_build_libs
 
 # Build and install Mingw-W64 tools
 mingw_build_tools
 
-# Build LLVM libraries
-llvm_build_libunwind
-llvm_build_libcxxabi
-llvm_build_libcxx
-
 # Download Wine
 wine_fetch
 

scripts/xtchain

@@ -6,12 +6,6 @@
 # DEVELOPERS:  Rafal Kupiec <belliash@codingworkshop.eu.org>
 
 
-# Check if script launched as root
-if [ "$(whoami)" = "root" ]; then
-    echo "This script cannot be run as root!"
-    exit 1
-fi
-
 # Get the absolute path to the XTchain
 export XTCDIR="$(realpath $(dirname ${0}))"
 

tools/windres.c

@@ -87,7 +87,7 @@ static
 void print_version(void)
 {
     printf("XTchain windres (GNU windres compatible) %s\n", WINDRES_VERSION);
-    exit(-1);
+    exit(0);
 }
 
 static
@@ -126,7 +126,7 @@ void print_help(void)
     "  pei-x86-64\n"
     "  pe-i386\n"
     "  pei-i386\n");
-    exit(1);
+    exit(0);
 }
 
 static