From 22a28564cf13d9e9bf0e648784ab8b9a12b65bda Mon Sep 17 00:00:00 2001 From: Konrad Kleine Date: Apr 04 2023 08:05:21 +0000 Subject: PGO instrumentation Add two PGO build conditions * pgo_instrumented_build * pgo_optimized_build Add `pgo-instrumentation-macros` subpackage. There were a couple off error that I ran into: `Error: LLVM Profile Warning: Unable to track new values: Running out of static counters. Consider using option -mllvm -vp-counters-per-site= to allocate more value profile counters at compile time.` *Solution:* Add `--vp-counters-per-site` option Then a follow-up error occurred: `Error: clang (LLVM option parsing): for the --vp-counters-per-site option: may only occur zero or one times!` *Solution:* Modify `vp-counters-per-site` option through `LLVM_VP_COUNTERS_PER_SITE` instead of adding it. --- diff --git a/llvm.spec b/llvm.spec index 4520c5c..36f5a3a 100644 --- a/llvm.spec +++ b/llvm.spec @@ -16,6 +16,18 @@ %bcond_with compat_build %bcond_without check +# https://llvm.org/docs/HowToBuildWithPGO.html#building-clang-with-pgo +# tag::pgo_bcond_instrumented[] +%bcond_with pgo_instrumented_build +# end::pgo_bcond_instrumented[] +# tag::pgo_bcond_optimized[] +%bcond_with pgo_optimized_build +# end::pgo_bcond_optimized[] + +%if %{with pgo_instrumented_build} || %{with pgo_optimized_build} +# Disable PGO data generation +%global _toolchain_profile_subpackages %{nil} +%endif #global rc_ver 4 %global maj_ver 16 @@ -75,7 +87,7 @@ Name: %{pkg_name} Version: %{maj_ver}.%{min_ver}.%{patch_ver}%{?rc_ver:~rc%{rc_ver}} -Release: 1%{?dist} +Release: 2%{?dist} Summary: The Low Level Virtual Machine License: Apache-2.0 WITH LLVM-exception OR NCSA @@ -129,6 +141,17 @@ BuildRequires: python3-setuptools # For origin certification BuildRequires: gnupg2 +# tag::pgo_build_requires_instrumented[] +%if %{with pgo_instrumented_build} +BuildRequires: compiler-rt +%endif +# end::pgo_build_requires_instrumented[] +# tag::pgo_build_requires_optimized[] +%if %{with pgo_optimized_build} +BuildRequires: llvm-pgo-profdata +%endif +# end::pgo_build_requires_optimized[] + Requires: %{name}-libs%{?_isa} = %{version}-%{release} @@ -209,6 +232,22 @@ Summary: LLVM's modified googletest sources %description googletest LLVM's modified googletest sources. +%package pgo-instrumentation-macros +Summary: Provides llvm_pgo_instrumented_XXX macros + +%description pgo-instrumentation-macros +Provides llvm_pgo_instrumented_XXX macros to be used in the "redhat-rpm-config" +package to tap into the build system and automatically generate a +"--pgo-profdata" subpackage. This subpackage will contain +indexed PGO profiling data. Make sure that you have a PGO instrumented +LLVM toolchain before installing this package. + +BuildArch: noarch + +Requires: inotify-tools + +%global rrcdir /usr/lib/rpm/redhat + %endif %prep @@ -302,6 +341,14 @@ export ASMFLAGS=$CFLAGS %if %{without compat_build} -DLLVM_VERSION_SUFFIX='' \ %endif +%if %{with pgo_instrumented_build} + -DLLVM_BUILD_INSTRUMENTED=IR \ + -DLLVM_BUILD_RUNTIME=No \ + -DLLVM_VP_COUNTERS_PER_SITE=8 \ +%endif +%if %{with pgo_optimized_build} + -DLLVM_PROFDATA_FILE=%{_libdir}/%{toolchain}-pgo-profdata/llvm-pgo.profdata \ +%endif -DLLVM_BUILD_LLVM_DYLIB:BOOL=ON \ -DLLVM_LINK_LLVM_DYLIB:BOOL=ON \ -DLLVM_BUILD_EXTERNAL_COMPILER_RT:BOOL=ON \ @@ -431,6 +478,13 @@ touch %{buildroot}%{_bindir}/llvm-config%{exec_suffix} cp -Rv ../cmake/Modules/* %{buildroot}%{pkg_libdir}/cmake/llvm +# PGO instrumentation macros +%if %{without compat_build} +mkdir -p %{buildroot}%{rrcdir} +install -p -m0644 -D macros.llvm-pgo-instrumentation %{buildroot}%{_rpmmacrodir}/macros.llvm-pgo-instrumentation +install -p -m 755 -t %{buildroot}%{rrcdir} pgo-background-merge.sh +%endif + %check # Disable check section on arm due to some kind of memory related failure. # Possibly related to https://bugzilla.redhat.com/show_bug.cgi?id=1920183 @@ -568,9 +622,21 @@ fi %{_includedir}/llvm-gtest %{_includedir}/llvm-gmock +%files pgo-instrumentation-macros +%license LICENSE +%doc llvm-pgo-instrumentation-macros.md +%dir %{rrcdir} +%{_rpmmacrodir}/macros.llvm-pgo-instrumentation +%{rrcdir}/pgo-background-merge.sh +%attr(0755,-,-) %{rrcdir}/pgo-background-merge.sh + %endif %changelog +* Tue Apr 04 2023 Konrad Kleine - 16.0.0-2 +- Add pgo_instrumented and pgo_optimized_build build conditions (both off by default) +- Add pgo-instrumentation-macros subpackage + * Mon Mar 20 2023 Tulio Magno Quites Machado Filho - 16.0.0-1 - Update to LLVM 16.0.0 diff --git a/macros.llvm-pgo-instrumentation b/macros.llvm-pgo-instrumentation new file mode 100644 index 0000000..453a81d --- /dev/null +++ b/macros.llvm-pgo-instrumentation @@ -0,0 +1,99 @@ +# llvm-pgo-instrumentation-macros +# +# Author: Konrad Kleine + +# To disable generation of profile subpackage, specify this in specfile: +# %global _toolchain_profile_subpackages %{nil} +# TODO(kwk): Make sure to only enable it when building with clang. +# tag::_toolchain_profile_subpackages[] +%_toolchain_profile_subpackages 1 +# end::_toolchain_profile_subpackages[] + +# tag::subpackage_template[] +# Generate profiledata packages for the compiler +%_toolchain_profile_subpackage_template \ +%package -n %{name}-%{toolchain}-pgo-profdata \ +Summary: Indexed PGO profile data from %{name} package \ +%description -n %{name}-%{toolchain}-pgo-profdata \ +This package contains profiledata for %{toolchain} that was generated while \ +compiling %{name}. This can be used for doing Profile Guided Optimizations \ +(PGO) builds of %{toolchain} \ +%files -n %{name}-%{toolchain}-pgo-profdata \ +%{_libdir}/%{toolchain}-pgo-profdata/%{name}/%{name}.%{toolchain}.profdata \ +%{nil} +# end::subpackage_template[] + +# tag::_pgo_macros_misc[] +# Auxilliary PGO profile to which the background job merges continously +%_pgo_background_merge_target %{_builddir}/%{name}.%{toolchain}.background.merge + +# Place where the background job stores its PID file +# %%_pgo_pid_file %{_builddir}/background-merge.pid +%_pgo_pid_file /tmp/background-merge.pid + +# How to specify the LLVM_PROFILE_FILE +%_pgo_llvm_profile_file %t/%{name}.%{toolchain}.%m.%p.profraw + +# Where to store all raw PGO profiles +%_pgo_tmpdir %{_builddir}/raw-pgo-profdata + +%_pgo_shutdown_file %{_pgo_tmpdir}/background-merge.shutdown + +# Use this before calling an instrumented LLVM binary. +%_pgo_env \\\ + TMPDIR='%{_pgo_tmpdir}' \\\ + && export TMPDIR \\\ + && mkdir -pv $TMPDIR \\\ + && LLVM_PROFILE_FILE='%{_pgo_llvm_profile_file}' \\\ + && export LLVM_PROFILE_FILE +# end::_pgo_macros_misc[] + +# tag::find_and_merge_profiles[] +%_pgo_merge_profdata %[ 0%{_toolchain_profile_subpackages} > 0 ? "\\\ + mkdir -pv %{buildroot}%{_libdir}/%{toolchain}-pgo-profdata/%{name} \\\ + && %{_pgo_env} \\\ + && llvm-profdata merge \\\ + --compress-all-sections \\\ + --sparse \\\ + %{_pgo_background_merge_target} \\\ + $(find %{_builddir}/raw-pgo-profdata -type f -name '*.profraw') \\\ + -o %{buildroot}%{_libdir}/%{toolchain}-pgo-profdata/%{name}/%{name}.%{toolchain}.profdata \\\ + " : "%{nil}" ] +# end::find_and_merge_profiles[] + +# The following %%llvm_pgo_instrumented_XXX macros +# will be used in redhat-rpm-config. +# ------------------------------------------------ + +# tag::llvm_pgo_instrumented_build_flags[] +%llvm_pgo_instrumented_build_flags [ 0%{_toolchain_profile_subpackages} > 0 ] && %{_pgo_env} +# end::llvm_pgo_instrumented_build_flags[] + +# tag::llvm_pgo_instrumented_spec_build_pre[] +%llvm_pgo_instrumented_spec_build_pre \ + [ 0%{_toolchain_profile_subpackages} > 0 ] \\\ + && %{_pgo_env} \\\ + && /usr/lib/rpm/redhat/pgo-background-merge.sh \\\ + -d %{_pgo_tmpdir} \\\ + -f %{_pgo_background_merge_target} \\\ + -p %{_pgo_pid_file} & \ +# end::llvm_pgo_instrumented_spec_build_pre[] + +# tag::llvm_pgo_instrumented_spec_build_post[] +# Overriding __spec_build_post macro from /usr/lib/rpm/macros +%llvm_pgo_instrumented_spec_build_post \ + if [ 0%{_toolchain_profile_subpackages} > 0 ]\ + then\ + echo 'please exit' > %{_pgo_shutdown_file};\ + [ -e %{_pgo_pid_file} ] && inotifywait -e delete_self %{_pgo_pid_file} || true;\ + fi\ +# end::llvm_pgo_instrumented_spec_build_post[] + +# tag::llvm_pgo_instrumented_os_install_post[] +%llvm_pgo_instrumented_os_install_post %{?_toolchain_profile_subpackages:%{?_pgo_merge_profdata}} +# end::llvm_pgo_instrumented_os_install_post[] + +# tag::llvm_pgo_instrumented_install[] +%llvm_pgo_instrumented_install\ + %[ 0%{_toolchain_profile_subpackages} > 0 ? "%{_toolchain_profile_subpackage_template}" : "%{nil}" ] +# end::llvm_pgo_instrumented_install[] \ No newline at end of file diff --git a/pgo-background-merge.sh b/pgo-background-merge.sh new file mode 100755 index 0000000..5937bf1 --- /dev/null +++ b/pgo-background-merge.sh @@ -0,0 +1,189 @@ +#!/bin/bash +# This is a script to run in the background and continously merge +# PGO profile data. + +# Author: Konrad Kleine +# Copyright (c) 2023 Red Hat. +# +# Run this script in the background. +# +# To gracefully shutdown the background job, write anything to +# and wait until is deleted: +# +# echo "foobar" > $shutdown_file +# [ -e $pid_file ] && inotifywait -e delete_self $pid_file || true + +function show_usage() +{ +cat < + -r + -s + -b + -u + -p + -f + -l + -x + -h;; +EOF +exit 0 +} + +while getopts "d:r:s:b:u:p:f:l:x:h" flag; do + case "${flag}" + in + d) observe_dir=${OPTARG};; + r) files_regex=${OPTARG};; + s) min_batch_size=${OPTARG};; + b) batch_file=${OPTARG};; + u) batch_file_in_process=${OPTARG};; + p) pid_file=${OPTARG};; + f) target_merge_file=${OPTARG};; + l) log_file=${OPTARG};; + x) shutdown_file=${OPTARG};; + h) show_usage;; + esac +done + +# Handle defaults +# Directory in which raw PGO profiles are stored +# NOTE: Normally PGO raw profiles are stored in the location where the +# instrumented binary is invoked. We make the assumption that all profiles are +# stored in the same directory. +# See %t here: +# https://clang.llvm.org/docs/SourceBasedCodeCoverage.html#running-the-instrumented-program +observe_dir=${observe_dir:-$PWD} +# Regex for the files to look out for +files_regex=${file_regex:-'.*\.profraw$'} +# Number of files that have to exist before we're processing them. +min_batch_size=${min_batch_size:-10} +# This file acts as to queue up file paths that we want to work on. +batch_file=${batch_file:-$observe_dir/background-merge.batch.txt} +# Once the number of lines in the batch_file reach the min_batch_size we move +# the content of batch_file over to this file and then processing can happen +# while batch_file can collect more files. +batch_file_in_process=${batch_file_in_process:-/tmp/background-merge.batch_in_process.txt} +# File to store the PID of this process. Once this file is deleted, the outer script will +pid_file=${pid_file:-/tmp/background-merge.pid} +target_merge_file=${target_merge_file:--/tmp/background-merge.target} +log_file=${log_file:-/tmp/background-merge.log} +# Once there's a write event to this file, the program exits gracefully. +shutdown_file=${shutdown_file:-$observe_dir/background-merge.shutdown} + +function show_config() +{ +cat < /dev/null 2>&1 & + # end::wait_for_profraw[] + + # Observe if a new profile was added to the list of the current batch. + # If the shutdown file was modified, gracefully shutdown. + # NOTE: batch file and shutdown file need to be in the same directory! + inotifywait -q -m -e modify \ + --include "($(basename $batch_file)|$(basename $shutdown_file))" \ + $observe_dir \ + | while read -r directory event filename + do + if [ "$filename" = "$(basename $shutdown_file)" ]; then + echo "Exiting gracefully..." + rm -f $pid_file + exit 0 + fi + batch_size=$(wc -l < $batch_file) + if [ $batch_size -le 0 ]; then + # This event happens when we empty the batch file + continue + fi + if [ $batch_size -lt $min_batch_size ]; then + echo "Batch is still too small: $batch_size must be at least $min_batch_size" + continue + fi + cat $batch_file > $batch_file_in_process + empty_batch_file + echo "Processing batch (size: $batch_size) in 5 seconds: " + cat $batch_file_in_process + process_batch + done +} + +# tag::setup[] +function setup() { + # Handle if PID file exists and whether process is still running or not. + if [ -e $pid_file ]; then + echo "ERROR: PID file already in use: $pid_file" + exit 0 + fi + + # Save this PID to a file + echo $$ > $pid_file + + # Create log backup + if [ -e $log_file ]; then + echo "Backing up existing log file: $log_file" + cp -bv $log_file $log_file.bak + fi +} +# end::setup[] + +show_config +setup + +main >> $log_file 2>&1