rtems/rtems-source-builder
1
0
Fork 0
mirror of https://git.rtems.org/rtems-source-builder synced 2024-10-09 07:15:10 +08:00
Code Issues Packages Projects Releases Wiki Activity

sb: Monitor the build disk usage. Report the usage, total and various sizes

Browse Source 
- Track the size of a build of a package in a build set to determine the
  maximum amout of disk space used. This can be used as a guide to
  documenting how much space a user needs to set aside to build a specific
  set of tools.

- The `%clean` stage of a build is now split into a separate script.
  I do not think this is an issue because I could not find any `%clean`
  sections in any build configs we have. In time support for the
  `%clean` section will be removed, the package builder cleans up.

Closes #3516
This commit is contained in:
Chris Johns 2018-09-28 07:27:57 +10:00
parent a16bfe19ef
commit 38fd56c8a8
4 changed files with 199 additions and 49 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

125
source-builder/sb/build.py
View File

@ -1,6 +1,6 @@
#
# RTEMS Tools Project (http://www.rtems.org/)
# Copyright 2010-2013 Chris Johns (chrisj@rtems.org)
# Copyright 2010-2018 Chris Johns (chrisj@rtems.org)
# All rights reserved.
#
# This file is part of the RTEMS Tools package in 'rtems-tools'.
@ -51,6 +51,13 @@ except:
print('error: unknown application load error')
sys.exit(1)
def humanize_number(num, suffix):
for unit in ['','K','M','G','T','P','E','Z']:
if abs(num) < 1024.0:
return "%5.3f%s%s" % (num, unit, suffix)
num /= 1024.0
return "%.3f%s%s" % (size, 'Y', suffix)
class script:
"""Create and manage a shell script."""
@ -124,7 +131,8 @@ class build:
log.notice('config: ' + name)
self.set_macros(macros)
self.config = config.file(name, opts, self.macros)
self.script = script()
self.script_build = script()
self.script_clean = script()
self.macros['buildname'] = self._name_(self.macros['name'])
except error.general as gerr:
log.notice(str(gerr))
@ -288,21 +296,21 @@ class build:
raise error.general('setup source tag not found: %d' % (source_tag))
else:
name = opt_name
self.script.append(self.config.expand('cd %{_builddir}'))
self.script_build.append(self.config.expand('cd %{_builddir}'))
if not deleted_dir and delete_before_unpack:
self.script.append(self.config.expand('%{__rm} -rf ' + name))
self.script_build.append(self.config.expand('%{__rm} -rf ' + name))
deleted_dir = True
if not created_dir and create_dir:
self.script.append(self.config.expand('%{__mkdir_p} ' + name))
self.script_build.append(self.config.expand('%{__mkdir_p} ' + name))
created_dir = True
if not changed_dir and (not unpack_before_chdir or create_dir):
self.script.append(self.config.expand('cd ' + name))
self.script_build.append(self.config.expand('cd ' + name))
changed_dir = True
self.script.append(self.config.expand(source['script']))
self.script_build.append(self.config.expand(source['script']))
if not changed_dir and (unpack_before_chdir and not create_dir):
self.script.append(self.config.expand('cd ' + name))
self.script_build.append(self.config.expand('cd ' + name))
changed_dir = True
self.script.append(self.config.expand('%{__setup_post}'))
self.script_build.append(self.config.expand('%{__setup_post}'))
def patch_setup(self, package, args):
name = args[1]
@ -360,7 +368,7 @@ class build:
else:
patch['script'] = '%{__cat} ' + patch['local']
patch['script'] += ' | %%{__patch} %s' % (opts)
self.script.append(self.config.expand(patch['script']))
self.script_build.append(self.config.expand(patch['script']))
def run(self, command, shell_opts = '', cwd = None):
e = execute.capture_execution(log = log.default, dump = self.opts.quiet())
@ -378,7 +386,7 @@ class build:
self.mkdir(builddir)
def prep(self, package):
self.script.append('echo "==> %prep:"')
self.script_build.append('echo "==> %prep:"')
_prep = package.prep()
if _prep:
for l in _prep:
@ -400,59 +408,78 @@ class build:
sources.hash(args[1:], self.macros, err)
self.hash(package, args)
else:
self.script.append(' '.join(args))
self.script_build.append(' '.join(args))
def build(self, package):
self.script.append('echo "==> clean %{buildroot}: ${SB_BUILD_ROOT}"')
self.script.append('%s ${SB_BUILD_ROOT}' %
(self.config.expand('%{__rmdir}')))
self.script.append('%s ${SB_BUILD_ROOT}' %
(self.config.expand('%{__mkdir_p}')))
self.script.append('echo "==> %build:"')
self.script_build.append('echo "==> clean %{buildroot}: ${SB_BUILD_ROOT}"')
self.script_build.append('%s ${SB_BUILD_ROOT}' %
(self.config.expand('%{__rmdir}')))
self.script_build.append('%s ${SB_BUILD_ROOT}' %
(self.config.expand('%{__mkdir_p}')))
self.script_build.append('echo "==> %build:"')
_build = package.build()
if _build:
for l in _build:
self.script.append(l)
self.script_build.append(l)
def install(self, package):
self.script.append('echo "==> %install:"')
self.script_build.append('echo "==> %install:"')
_install = package.install()
if _install:
for l in _install:
args = l.split()
self.script.append(' '.join(args))
self.script_build.append(' '.join(args))
def files(self, package):
if self.create_tar_files \
and not self.macros.get('%{_disable_packaging'):
self.script.append('echo "==> %files:"')
self.script_build.append('echo "==> %files:"')
inpath = path.abspath(self.config.expand('%{buildroot}'))
tardir = path.abspath(self.config.expand('%{_tardir}'))
self.script.append(self.config.expand('if test -d %s; then' % (inpath)))
self.script.append(self.config.expand(' %%{__mkdir_p} %s' % tardir))
self.script.append(self.config.expand(' cd ' + inpath))
self.script_build.append(self.config.expand('if test -d %s; then' % (inpath)))
self.script_build.append(self.config.expand(' %%{__mkdir_p} %s' % tardir))
self.script_build.append(self.config.expand(' cd ' + inpath))
tar = path.join(tardir, package.long_name() + '.tar.bz2')
cmd = self.config.expand(' %{__tar} -cf - . ' + '| %{__bzip2} > ' + tar)
self.script.append(cmd)
self.script.append(self.config.expand(' cd %{_builddir}'))
self.script.append('fi')
self.script_build.append(cmd)
self.script_build.append(self.config.expand(' cd %{_builddir}'))
self.script_build.append('fi')
def clean(self, package):
self.script.append('echo "==> %clean:"')
self.script_clean.reset()
self.script_clean.append(self.config.expand('%{___build_template}'))
self.script_clean.append('echo "=> ' + package.name() + ': CLEAN"')
self.script_clean.append('echo "==> %clean:"')
_clean = package.clean()
if _clean is not None:
for l in _clean:
args = l.split()
self.script.append(' '.join(args))
self.script_clean.append(' '.join(args))
def sizes(self, package):
def _sizes(package, what, path):
package.set_size(what, path)
s = humanize_number(package.get_size(what), 'B')
log.trace('size: %s (%s): %s (%d)' % (what, path, s, package.get_size(what)))
return s
s = {}
for p in [('build', '%{_builddir}'),
('build', '%{buildroot}'),
('installed', '%{buildroot}')]:
hs = _sizes(package, p[0], self.config.expand(p[1]))
s[p[0]] = hs
log.notice('sizes: %s: %s (installed: %s)' % (package.name(),
s['build'],
s['installed']))
def build_package(self, package):
if self.canadian_cross():
if not self.config.defined('%{allow_cxc}'):
raise error.general('Canadian Cross is not allowed')
self.script.append('echo "==> Candian-cross build/target:"')
self.script.append('SB_CXC="yes"')
self.script_build.append('echo "==> Candian-cross build/target:"')
self.script_build.append('SB_CXC="yes"')
else:
self.script.append('SB_CXC="no"')
self.script_build.append('SB_CXC="no"')
self.build(package)
self.install(package)
self.files(package)
@ -498,18 +525,24 @@ class build:
log.trace('---- macro maps %s' % ('-' * 55))
log.trace('%s' % (str(self.config.macros)))
log.trace('-' * 70)
self.script.reset()
self.script.append(self.config.expand('%{___build_template}'))
self.script.append('echo "=> ' + name + ':"')
self.script_build.reset()
self.script_build.append(self.config.expand('%{___build_template}'))
self.script_build.append('echo "=> ' + name + ': BUILD"')
self.prep(package)
self.build_package(package)
if not self.opts.dry_run():
self.builddir()
sn = path.join(self.config.expand('%{_builddir}'), 'doit')
log.output('write script: ' + sn)
self.script.write(sn)
build_sn = path.join(self.config.expand('%{_builddir}'), 'do-build')
log.output('write script: ' + build_sn)
self.script_build.write(build_sn)
clean_sn = path.join(self.config.expand('%{_builddir}'), 'do-clean')
log.output('write script: ' + clean_sn)
self.script_clean.write(clean_sn)
log.notice('building: %s%s' % (cxc_label, name))
self.run(sn)
self.run(build_sn)
self.sizes(package)
log.notice('cleaning: %s%s' % (cxc_label, name))
self.run(clean_sn)
except error.general as gerr:
log.notice(str(gerr))
log.stderr('Build FAILED')
@ -536,6 +569,18 @@ class build:
package = packages['main']
return package.disabled()
def get_build_size(self):
package = self.main_package()
if package.disabled():
return 0
return package.get_size('build')
def get_installed_size(self):
package = self.main_package()
if package.disabled():
return 0
return package.get_size('installed')
def get_configs(opts):
def _scan(_path, ext):

13
source-builder/sb/config.py
View File

@ -1,6 +1,6 @@
#
# RTEMS Tools Project (http://www.rtems.org/)
# Copyright 2010-2016 Chris Johns (chrisj@rtems.org)
# Copyright 2010-2018 Chris Johns (chrisj@rtems.org)
# All rights reserved.
#
# This file is part of the RTEMS Tools package in 'rtems-tools'.
@ -73,6 +73,7 @@ class package:
self.config = config
self.directives = {}
self.infos = {}
self.sizes = {}
def __str__(self):
@ -218,6 +219,16 @@ class package:
def disabled(self):
return len(self.name()) == 0
def set_size(self, what, path_):
if what not in self.sizes:
self.sizes[what] = 0
self.sizes[what] += path.get_size(path_)
def get_size(self, what):
if what in self.sizes:
return self.sizes[what]
return 0
class file:
"""Parse a config file."""

68
source-builder/sb/path.py
View File

@ -1,6 +1,6 @@
#
# RTEMS Tools Project (http://www.rtems.org/)
# Copyright 2010-2016 Chris Johns (chrisj@rtems.org)
# Copyright 2010-2018 Chris Johns (chrisj@rtems.org)
# All rights reserved.
#
# This file is part of the RTEMS Tools package in 'rtems-tools'.
@ -53,11 +53,6 @@ def host(path):
path = u'\\'.join([u'\\\\?', path])
return path
def is_abspath(path):
if path is not None and len(path) > 0:
return '/' == path[0]
return False
def shell(path):
if path is not None:
if windows:
@ -79,6 +74,11 @@ def dirname(path):
path = shell(path)
return shell(os.path.dirname(path))
def is_abspath(path):
if path is not None and len(path) > 0:
return '/' == path[0]
return False
def join(path, *args):
path = shell(path)
for arg in args:
@ -304,6 +304,58 @@ def copy_tree(src, dst):
else:
raise error.general('copying tree (4): %s -> %s: %s' % (hsrc, hdst, str(why)))
def get_size(path, depth = -1):
#
# Get the size the directory tree manually to the required depth.
# This makes sure on Windows the files are correctly encoded to avoid
# the file name size limit. On Windows the os.walk fails once we
# get to the max path length on Windows.
#
def _isdir(path):
hpath = host(path)
return os.path.isdir(hpath) and not os.path.islink(hpath)
def _node_size(path):
hpath = host(path)
size = 0
if not os.path.islink(hpath):
size = os.path.getsize(hpath)
return size
def _get_size(path, depth, level = 0):
level += 1
dirs = []
size = 0
for name in listdir(path):
path_ = join(path, shell(name))
hname = host(path_)
if _isdir(path_):
dirs += [shell(name)]
else:
size += _node_size(path_)
if depth < 0 or level < depth:
for name in dirs:
dir = join(path, name)
size += _get_size(dir, depth, level)
return size
path = shell(path)
hpath = host(path)
size = 0
if os.path.exists(hpath):
size = _get_size(path, depth)
return size
def get_humanize_size(path, depth = -1):
size = get_size(path, depth)
for unit in ['','K','M','G','T','P','E','Z']:
if abs(size) < 1024.0:
return "%5.3f%sB" % (size, unit)
size /= 1024.0
return "%.3f%sB" % (size, 'Y')
if __name__ == '__main__':
print(host('/a/b/c/d-e-f'))
print(host('//a/b//c/d-e-f'))
@ -311,6 +363,10 @@ if __name__ == '__main__':
print(basename('/as/sd/df/fg/me.txt'))
print(dirname('/as/sd/df/fg/me.txt'))
print(join('/d', 'g', '/tyty/fgfg'))
print('size of . depth all: ', get_size('.'))
print('size of . depth 1: ', get_size('.', 1))
print('size of . depth 2: ', get_size('.', 2))
print('size of . as human : ', get_humanize_size('.'))
windows = True
print(host('/a/b/c/d-e-f'))
print(host('//a/b//c/d-e-f'))

42
source-builder/sb/setbuilder.py
View File

@ -1,6 +1,6 @@
#
# RTEMS Tools Project (http://www.rtems.org/)
# Copyright 2010-2016 Chris Johns (chrisj@rtems.org)
# Copyright 2010-2018 Chris Johns (chrisj@rtems.org)
# All rights reserved.
#
# This file is part of the RTEMS Tools package in 'rtems-tools'.
@ -447,13 +447,48 @@ class buildset:
self.install(b.name(),
b.config.expand('%{buildroot}'),
b.config.expand('%{_prefix}'))
#
# Sizes ...
#
if len(builds) > 1:
size_build = 0
size_installed = 0
size_build_max = 0
for b in builds:
s = b.get_build_size()
size_build += s
if s > size_build_max:
size_build_max = s
size_installed += b.get_installed_size()
size_sources = 0
for p in builds[0].config.expand('%{_sourcedir}').split(':'):
size_sources += path.get_size(p)
size_patches = 0
for p in builds[0].config.expand('%{_patchdir}').split(':'):
size_patches += path.get_size(p)
size_total = size_sources + size_patches + size_installed
build_size = 'usage: %s' % (build.humanize_number(size_build_max + size_installed, 'B'))
build_size += ' total: %s' % (build.humanize_number(size_total, 'B'))
build_size += ' (sources: %s' % (build.humanize_number(size_sources, 'B'))
build_size += ', patches: %s' % (build.humanize_number(size_patches, 'B'))
build_size += ', installed %s)' % (build.humanize_number(size_installed, 'B'))
#
# Cleaning ...
#
if deps is None and \
(not self.opts.no_clean() or self.opts.always_clean()):
for b in builds:
if not b.disabled():
log.notice('cleaning: %s' % (b.name()))
b.cleanup()
#
# Log the build size message
#
if len(builds) > 1:
log.notice('Build Sizes: %s' % (build_size))
#
# Clear out the builds ...
#
for b in builds:
del b
except error.general as gerr:
@ -484,6 +519,9 @@ class buildset:
self.write_mail_header('')
log.notice('Mailing report: %s' % (mail['to']))
body = self.get_mail_header()
body += 'Sizes' + os.linesep
body += '=====' + os.linesep + os.linesep
body += 'Output' + os.linesep
body += '======' + os.linesep + os.linesep
body += os.linesep.join(mail['output'].get())