e1a8a9c
#!/usr/bin/python3
5a44c73
"""
5a44c73
build helper script for edk2, see
5a44c73
https://gitlab.com/kraxel/edk2-build-config
5a44c73
5a44c73
"""
e1a8a9c
import os
e1a8a9c
import sys
d88cf98
import time
e1a8a9c
import shutil
5a44c73
import argparse
e1a8a9c
import subprocess
e1a8a9c
import configparser
e1a8a9c
4405543
rebase_prefix    = ""
4405543
version_override = None
e70e465
release_date     = None
4405543
5a44c73
# pylint: disable=unused-variable
e1a8a9c
def check_rebase():
4405543
    """ detect 'git rebase -x edk2-build.py master' testbuilds """
4405543
    global rebase_prefix
4405543
    global version_override
676bf77
    gitdir = '.git'
4405543
676bf77
    if os.path.isfile(gitdir):
5a44c73
        with open(gitdir, 'r', encoding = 'utf-8') as f:
676bf77
            (unused, gitdir) = f.read().split()
676bf77
676bf77
    if not os.path.exists(f'{gitdir}/rebase-merge/msgnum'):
5a44c73
        return
5a44c73
    with open(f'{gitdir}/rebase-merge/msgnum', 'r', encoding = 'utf-8') as f:
e1a8a9c
        msgnum = int(f.read())
5a44c73
    with open(f'{gitdir}/rebase-merge/end', 'r', encoding = 'utf-8') as f:
e1a8a9c
        end = int(f.read())
5a44c73
    with open(f'{gitdir}/rebase-merge/head-name', 'r', encoding = 'utf-8') as f:
e1a8a9c
        head = f.read().strip().split('/')
4405543
4405543
    rebase_prefix = f'[ {int(msgnum/2)} / {int(end/2)} - {head[-1]} ] '
e70e465
    if msgnum != end and not version_override:
4405543
        # fixed version speeds up builds
4405543
        version_override = "test-build-patch-series"
e1a8a9c
e1a8a9c
def get_coredir(cfg):
e1a8a9c
    if cfg.has_option('global', 'core'):
e1a8a9c
        return os.path.abspath(cfg['global']['core'])
5a44c73
    return os.getcwd()
5a44c73
5a44c73
def get_toolchain(cfg, build):
5a44c73
    if cfg.has_option(build, 'tool'):
5a44c73
        return cfg[build]['tool']
5a44c73
    if cfg.has_option('global', 'tool'):
5a44c73
        return cfg['global']['tool']
5a44c73
    return 'GCC5'
e1a8a9c
d88cf98
def get_version(cfg, silent = False):
e1a8a9c
    coredir = get_coredir(cfg)
4405543
    if version_override:
4405543
        version = version_override
d88cf98
        if not silent:
d88cf98
            print('')
d88cf98
            print(f'### version [override]: {version}')
4405543
        return version
e1a8a9c
    if os.environ.get('RPM_PACKAGE_NAME'):
5a44c73
        version = os.environ.get('RPM_PACKAGE_NAME')
5a44c73
        version += '-' + os.environ.get('RPM_PACKAGE_VERSION')
5a44c73
        version += '-' + os.environ.get('RPM_PACKAGE_RELEASE')
d88cf98
        if not silent:
d88cf98
            print('')
d88cf98
            print(f'### version [rpmbuild]: {version}')
e1a8a9c
        return version
e1a8a9c
    if os.path.exists(coredir + '/.git'):
5a44c73
        cmdline = [ 'git', 'describe', '--tags', '--abbrev=8',
5a44c73
                    '--match=edk2-stable*' ]
5a44c73
        result = subprocess.run(cmdline, cwd = coredir,
5a44c73
                                stdout = subprocess.PIPE,
5a44c73
                                check = True)
e1a8a9c
        version = result.stdout.decode().strip()
d88cf98
        if not silent:
d88cf98
            print('')
d88cf98
            print(f'### version [git]: {version}')
e1a8a9c
        return version
e1a8a9c
    return None
e1a8a9c
e1a8a9c
def pcd_string(name, value):
e9d9632
    return f'{name}=L{value}\\0'
e1a8a9c
d88cf98
def pcd_version(cfg, silent = False):
d88cf98
    version = get_version(cfg, silent)
e1a8a9c
    if version is None:
e1a8a9c
        return []
e1a8a9c
    return [ '--pcd', pcd_string('PcdFirmwareVersionString', version) ]
e1a8a9c
5a44c73
def pcd_release_date():
e70e465
    if release_date is None:
e70e465
        return []
e70e465
    return [ '--pcd', pcd_string('PcdFirmwareReleaseDateString', release_date) ]
e70e465
d88cf98
def build_message(line, line2 = None, silent = False):
e1a8a9c
    if os.environ.get('TERM') in [ 'xterm', 'xterm-256color' ]:
e1a8a9c
        # setxterm  title
e1a8a9c
        start  = '\x1b]2;'
e1a8a9c
        end    = '\x07'
4405543
        print(f'{start}{rebase_prefix}{line}{end}', end = '')
e1a8a9c
d88cf98
    if silent:
d88cf98
        print(f'### {rebase_prefix}{line}', flush = True)
d88cf98
    else:
d88cf98
        print('')
d88cf98
        print('###')
d88cf98
        print(f'### {rebase_prefix}{line}')
d88cf98
        if line2:
d88cf98
            print(f'### {line2}')
d88cf98
        print('###', flush = True)
e1a8a9c
d88cf98
def build_run(cmdline, name, section, silent = False, nologs = False):
e70e465
    if silent:
d88cf98
        logfile = f'{section}.log'
d88cf98
        if nologs:
d88cf98
            print(f'### building in silent mode [no log] ...', flush = True)
d88cf98
        else:
d88cf98
            print(f'### building in silent mode [{logfile}] ...', flush = True)
d88cf98
        start = time.time()
5a44c73
        result = subprocess.run(cmdline, check = False,
e70e465
                                stdout = subprocess.PIPE,
e70e465
                                stderr = subprocess.STDOUT)
d88cf98
        if not nologs:
d88cf98
            with open(logfile, 'wb') as f:
d88cf98
                f.write(result.stdout)
e70e465
e70e465
        if result.returncode:
e70e465
            print('### BUILD FAILURE')
d88cf98
            print('### cmdline')
d88cf98
            print(cmdline)
e70e465
            print('### output')
e70e465
            print(result.stdout.decode())
e70e465
            print(f'### exit code: {result.returncode}')
e70e465
        else:
d88cf98
            secs = int(time.time() - start)
d88cf98
            print(f'### OK ({int(secs/60)}:{secs%60:02d})')
e70e465
    else:
d88cf98
        print(cmdline, flush = True)
5a44c73
        result = subprocess.run(cmdline, check = False)
e1a8a9c
    if result.returncode:
5a44c73
        print(f'ERROR: {cmdline[0]} exited with {result.returncode}'
5a44c73
              f' while building {name}')
e1a8a9c
        sys.exit(result.returncode)
e1a8a9c
5a44c73
def build_copy(plat, tgt, toolchain, dstdir, copy):
5a44c73
    srcdir = f'Build/{plat}/{tgt}_{toolchain}'
e1a8a9c
    names = copy.split()
e1a8a9c
    srcfile = names[0]
e1a8a9c
    if len(names) > 1:
e1a8a9c
        dstfile = names[1]
e1a8a9c
    else:
e1a8a9c
        dstfile = os.path.basename(srcfile)
e1a8a9c
    print(f'# copy: {srcdir} / {srcfile}  =>  {dstdir} / {dstfile}')
e1a8a9c
e70e465
    src = srcdir + '/' + srcfile
e70e465
    dst = dstdir + '/' + dstfile
e70e465
    os.makedirs(os.path.dirname(dst), exist_ok = True)
e70e465
    shutil.copy(src, dst)
e1a8a9c
e1a8a9c
def pad_file(dstdir, pad):
e1a8a9c
    args = pad.split()
e1a8a9c
    if len(args) < 2:
e1a8a9c
        raise RuntimeError(f'missing arg for pad ({args})')
e1a8a9c
    name = args[0]
e1a8a9c
    size = args[1]
e1a8a9c
    cmdline = [
e1a8a9c
        'truncate',
e1a8a9c
        '--size', size,
e1a8a9c
        dstdir + '/' + name,
e1a8a9c
    ]
e1a8a9c
    print(f'# padding: {dstdir} / {name}  =>  {size}')
5a44c73
    subprocess.run(cmdline, check = True)
e1a8a9c
5a44c73
# pylint: disable=too-many-branches
d88cf98
def build_one(cfg, build, jobs = None, silent = False, nologs = False):
5a44c73
    b = cfg[build]
5a44c73
e1a8a9c
    cmdline  = [ 'build' ]
5a44c73
    cmdline += [ '-t', get_toolchain(cfg, build) ]
5a44c73
    cmdline += [ '-p', b['conf'] ]
e1a8a9c
5a44c73
    if (b['conf'].startswith('OvmfPkg/') or
5a44c73
        b['conf'].startswith('ArmVirtPkg/')):
d88cf98
        cmdline += pcd_version(cfg, silent)
5a44c73
        cmdline += pcd_release_date()
e1a8a9c
e1a8a9c
    if jobs:
e1a8a9c
        cmdline += [ '-n', jobs ]
5a44c73
    for arch in b['arch'].split():
e1a8a9c
        cmdline += [ '-a', arch ]
5a44c73
    if 'opts' in b:
5a44c73
        for name in b['opts'].split():
e1a8a9c
            section = 'opts.' + name
e1a8a9c
            for opt in cfg[section]:
139b9bb
                cmdline += [ '-D', opt + '=' + cfg[section][opt] ]
5a44c73
    if 'pcds' in b:
5a44c73
        for name in b['pcds'].split():
139b9bb
            section = 'pcds.' + name
139b9bb
            for pcd in cfg[section]:
139b9bb
                cmdline += [ '--pcd', pcd + '=' + cfg[section][pcd] ]
5a44c73
    if 'tgts' in b:
5a44c73
        tgts = b['tgts'].split()
e1a8a9c
    else:
e1a8a9c
        tgts = [ 'DEBUG' ]
e1a8a9c
    for tgt in tgts:
e70e465
        desc = None
5a44c73
        if 'desc' in b:
5a44c73
            desc = b['desc']
5a44c73
        build_message(f'building: {b["conf"]} ({b["arch"]}, {tgt})',
d88cf98
                      f'description: {desc}',
d88cf98
                      silent = silent)
e1a8a9c
        build_run(cmdline + [ '-b', tgt ],
5a44c73
                  b['conf'],
e70e465
                  build + '.' + tgt,
d88cf98
                  silent,
d88cf98
                  nologs)
e1a8a9c
5a44c73
        if 'plat' in b:
e1a8a9c
            # copy files
5a44c73
            for cpy in b:
e1a8a9c
                if not cpy.startswith('cpy'):
e1a8a9c
                    continue
5a44c73
                build_copy(b['plat'], tgt,
5a44c73
                           get_toolchain(cfg, build),
5a44c73
                           b['dest'], b[cpy])
e1a8a9c
            # pad builds
5a44c73
            for pad in b:
e1a8a9c
                if not pad.startswith('pad'):
e1a8a9c
                    continue
5a44c73
                pad_file(b['dest'], b[pad])
e1a8a9c
d88cf98
def build_basetools(silent = False, nologs = False):
d88cf98
    build_message('building: BaseTools', silent = silent)
e1a8a9c
    basedir = os.environ['EDK_TOOLS_PATH']
e1a8a9c
    cmdline = [ 'make', '-C', basedir ]
d88cf98
    build_run(cmdline, 'BaseTools', 'build.basetools', silent, nologs)
e1a8a9c
e1a8a9c
def binary_exists(name):
5a44c73
    for pdir in os.environ['PATH'].split(':'):
5a44c73
        if os.path.exists(pdir + '/' + name):
e1a8a9c
            return True
e1a8a9c
    return False
e1a8a9c
d88cf98
def prepare_env(cfg, silent = False):
e1a8a9c
    """ mimic Conf/BuildEnv.sh """
e1a8a9c
    workspace = os.getcwd()
e1a8a9c
    packages = [ workspace, ]
e1a8a9c
    path = os.environ['PATH'].split(':')
e1a8a9c
    dirs = [
e1a8a9c
        'BaseTools/Bin/Linux-x86_64',
e1a8a9c
        'BaseTools/BinWrappers/PosixLike'
e1a8a9c
    ]
e1a8a9c
e1a8a9c
    if cfg.has_option('global', 'pkgs'):
e1a8a9c
        for pkgdir in cfg['global']['pkgs'].split():
e1a8a9c
            packages.append(os.path.abspath(pkgdir))
e70e465
    coredir = get_coredir(cfg)
e70e465
    if coredir != workspace:
e70e465
        packages.append(coredir)
e1a8a9c
e1a8a9c
    # add basetools to path
5a44c73
    for pdir in dirs:
5a44c73
        p = coredir + '/' + pdir
e1a8a9c
        if not os.path.exists(p):
e1a8a9c
            continue
e1a8a9c
        if p in path:
e1a8a9c
            continue
e1a8a9c
        path.insert(0, p)
e1a8a9c
e1a8a9c
    # run edksetup if needed
5a44c73
    toolsdef = coredir + '/Conf/tools_def.txt'
e1a8a9c
    if not os.path.exists(toolsdef):
e70e465
        os.makedirs(os.path.dirname(toolsdef), exist_ok = True)
d88cf98
        build_message('running BaseTools/BuildEnv', silent = silent)
5a44c73
        cmdline = [ 'bash', 'BaseTools/BuildEnv' ]
5a44c73
        subprocess.run(cmdline, cwd = coredir, check = True)
e1a8a9c
e1a8a9c
    # set variables
e1a8a9c
    os.environ['PATH'] = ':'.join(path)
e1a8a9c
    os.environ['PACKAGES_PATH'] = ':'.join(packages)
e1a8a9c
    os.environ['WORKSPACE'] = workspace
e1a8a9c
    os.environ['EDK_TOOLS_PATH'] = coredir + '/BaseTools'
e1a8a9c
    os.environ['CONF_PATH'] = coredir + '/Conf'
e1a8a9c
    os.environ['PYTHON_COMMAND'] = '/usr/bin/python3'
6d59876
    os.environ['PYTHONHASHSEED'] = '1'
e1a8a9c
e1a8a9c
    # for cross builds
d88cf98
    if binary_exists('arm-linux-gnueabi-gcc'):
d88cf98
        # ubuntu
d88cf98
        os.environ['GCC5_ARM_PREFIX'] = 'arm-linux-gnueabi-'
d88cf98
        os.environ['GCC_ARM_PREFIX'] = 'arm-linux-gnueabi-'
d88cf98
    elif binary_exists('arm-linux-gnu-gcc'):
d88cf98
        # fedora
e1a8a9c
        os.environ['GCC5_ARM_PREFIX'] = 'arm-linux-gnu-'
d88cf98
        os.environ['GCC_ARM_PREFIX'] = 'arm-linux-gnu-'
6d59876
    if binary_exists('loongarch64-linux-gnu-gcc'):
6d59876
        os.environ['GCC5_LOONGARCH64_PREFIX'] = 'loongarch64-linux-gnu-'
d88cf98
        os.environ['GCC_LOONGARCH64_PREFIX'] = 'loongarch64-linux-gnu-'
676bf77
676bf77
    hostarch = os.uname().machine
676bf77
    if binary_exists('aarch64-linux-gnu-gcc') and hostarch != 'aarch64':
676bf77
        os.environ['GCC5_AARCH64_PREFIX'] = 'aarch64-linux-gnu-'
d88cf98
        os.environ['GCC_AARCH64_PREFIX'] = 'aarch64-linux-gnu-'
676bf77
    if binary_exists('riscv64-linux-gnu-gcc') and hostarch != 'riscv64':
676bf77
        os.environ['GCC5_RISCV64_PREFIX'] = 'riscv64-linux-gnu-'
d88cf98
        os.environ['GCC_RISCV64_PREFIX'] = 'riscv64-linux-gnu-'
676bf77
    if binary_exists('x86_64-linux-gnu-gcc') and hostarch != 'x86_64':
e1a8a9c
        os.environ['GCC5_IA32_PREFIX'] = 'x86_64-linux-gnu-'
e1a8a9c
        os.environ['GCC5_X64_PREFIX'] = 'x86_64-linux-gnu-'
6d59876
        os.environ['GCC5_BIN'] = 'x86_64-linux-gnu-'
d88cf98
        os.environ['GCC_IA32_PREFIX'] = 'x86_64-linux-gnu-'
d88cf98
        os.environ['GCC_X64_PREFIX'] = 'x86_64-linux-gnu-'
d88cf98
        os.environ['GCC_BIN'] = 'x86_64-linux-gnu-'
e1a8a9c
e1a8a9c
def build_list(cfg):
e1a8a9c
    for build in cfg.sections():
e1a8a9c
        if not build.startswith('build.'):
e1a8a9c
            continue
e1a8a9c
        name = build.lstrip('build.')
e1a8a9c
        desc = 'no description'
e1a8a9c
        if 'desc' in cfg[build]:
e1a8a9c
            desc = cfg[build]['desc']
4405543
        print(f'# {name:20s} - {desc}')
5a44c73
e1a8a9c
def main():
5a44c73
    parser = argparse.ArgumentParser(prog = 'edk2-build',
5a44c73
                                     description = 'edk2 build helper script')
5a44c73
    parser.add_argument('-c', '--config', dest = 'configfile',
5a44c73
                        type = str, default = '.edk2.builds', metavar = 'FILE',
5a44c73
                        help = 'read configuration from FILE (default: .edk2.builds)')
5a44c73
    parser.add_argument('-C', '--directory', dest = 'directory', type = str,
5a44c73
                        help = 'change to DIR before building', metavar = 'DIR')
5a44c73
    parser.add_argument('-j', '--jobs', dest = 'jobs', type = str,
5a44c73
                        help = 'allow up to JOBS parallel build jobs',
5a44c73
                        metavar = 'JOBS')
58f180d
    parser.add_argument('-m', '--match', dest = 'match',
58f180d
                        type = str, action = 'append',
5a44c73
                        help = 'only run builds matching INCLUDE (substring)',
5a44c73
                        metavar = 'INCLUDE')
d88cf98
    parser.add_argument('-x', '--exclude', dest = 'exclude',
d88cf98
                        type = str, action = 'append',
5a44c73
                        help = 'skip builds matching EXCLUDE (substring)',
5a44c73
                        metavar = 'EXCLUDE')
5a44c73
    parser.add_argument('-l', '--list', dest = 'list',
5a44c73
                        action = 'store_true', default = False,
5a44c73
                        help = 'list build configs available')
5a44c73
    parser.add_argument('--silent', dest = 'silent',
5a44c73
                        action = 'store_true', default = False,
5a44c73
                        help = 'write build output to logfiles, '
5a44c73
                        'write to console only on errors')
d88cf98
    parser.add_argument('--no-logs', dest = 'nologs',
d88cf98
                        action = 'store_true', default = False,
d88cf98
                        help = 'do not write build log files (with --silent)')
5a44c73
    parser.add_argument('--core', dest = 'core', type = str, metavar = 'DIR',
5a44c73
                        help = 'location of the core edk2 repository '
5a44c73
                        '(i.e. where BuildTools are located)')
5a44c73
    parser.add_argument('--pkg', '--package', dest = 'pkgs',
5a44c73
                        type = str, action = 'append', metavar = 'DIR',
5a44c73
                        help = 'location(s) of additional packages '
5a44c73
                        '(can be specified multiple times)')
5a44c73
    parser.add_argument('-t', '--toolchain', dest = 'toolchain', type = str, metavar = 'NAME',
5a44c73
                        help = 'tool chain to be used to build edk2')
5a44c73
    parser.add_argument('--version-override', dest = 'version_override',
5a44c73
                        type = str, metavar = 'VERSION',
5a44c73
                        help = 'set firmware build version')
5a44c73
    parser.add_argument('--release-date', dest = 'release_date',
5a44c73
                        type = str, metavar = 'DATE',
5a44c73
                        help = 'set firmware build release date (in MM/DD/YYYY format)')
5a44c73
    options = parser.parse_args()
e1a8a9c
6d59876
    if options.directory:
6d59876
        os.chdir(options.directory)
6d59876
5a44c73
    if not os.path.exists(options.configfile):
d88cf98
        print(f'config file "{options.configfile}" not found')
5a44c73
        return 1
5a44c73
e1a8a9c
    cfg = configparser.ConfigParser()
139b9bb
    cfg.optionxform = str
e1a8a9c
    cfg.read(options.configfile)
e1a8a9c
e1a8a9c
    if options.list:
e1a8a9c
        build_list(cfg)
5a44c73
        return 0
e1a8a9c
e1a8a9c
    if not cfg.has_section('global'):
e1a8a9c
        cfg.add_section('global')
e1a8a9c
    if options.core:
e1a8a9c
        cfg.set('global', 'core', options.core)
e70e465
    if options.pkgs:
e70e465
        cfg.set('global', 'pkgs', ' '.join(options.pkgs))
5a44c73
    if options.toolchain:
5a44c73
        cfg.set('global', 'tool', options.toolchain)
e1a8a9c
139b9bb
    global version_override
e70e465
    global release_date
4405543
    check_rebase()
4405543
    if options.version_override:
4405543
        version_override = options.version_override
e70e465
    if options.release_date:
e70e465
        release_date = options.release_date
4405543
d88cf98
    prepare_env(cfg, options.silent)
d88cf98
    build_basetools(options.silent, options.nologs)
e1a8a9c
    for build in cfg.sections():
e1a8a9c
        if not build.startswith('build.'):
e1a8a9c
            continue
58f180d
        if options.match:
58f180d
            matching = False
58f180d
            for item in options.match:
58f180d
                if item in build:
58f180d
                    matching = True
58f180d
            if not matching:
58f180d
                print(f'# skipping "{build}" (not matching "{"|".join(options.match)}")')
58f180d
                continue
d88cf98
        if options.exclude:
d88cf98
            exclude = False
d88cf98
            for item in options.exclude:
d88cf98
                if item in build:
d88cf98
                    print(f'# skipping "{build}" (matching "{item}")')
d88cf98
                    exclude = True
d88cf98
            if exclude:
d88cf98
                continue
d88cf98
        build_one(cfg, build, options.jobs, options.silent, options.nologs)
e1a8a9c
5a44c73
    return 0
5a44c73
e1a8a9c
if __name__ == '__main__':
e1a8a9c
    sys.exit(main())