sb: Monitor the build disk usage. Report the usage, total and various sizes

- Track the size of a build of a package in a build set to determine the
  maximum amout of disk space used. This can be used as a guide to
  documenting how much space a user needs to set aside to build a specific
  set of tools.

- The `%clean` stage of a build is now split into a separate script.
  I do not think this is an issue because I could not find any `%clean`
  sections in any build configs we have. In time support for the
  `%clean` section will be removed, the package builder cleans up.

Closes #3516

source-builder/sb/path.py

@@ -1,6 +1,6 @@
 #
 # RTEMS Tools Project (http://www.rtems.org/)
-# Copyright 2010-2016 Chris Johns (chrisj@rtems.org)
+# Copyright 2010-2018 Chris Johns (chrisj@rtems.org)
 # All rights reserved.
 #
 # This file is part of the RTEMS Tools package in 'rtems-tools'.
@@ -53,11 +53,6 @@ def host(path):
                 path = u'\\'.join([u'\\\\?', path])
     return path
 
-def is_abspath(path):
-    if path is not None and len(path) > 0:
-        return '/' == path[0]
-    return False
-
 def shell(path):
     if path is not None:
         if windows:
@@ -79,6 +74,11 @@ def dirname(path):
     path = shell(path)
     return shell(os.path.dirname(path))
 
+def is_abspath(path):
+    if path is not None and len(path) > 0:
+        return '/' == path[0]
+    return False
+
 def join(path, *args):
     path = shell(path)
     for arg in args:
@@ -304,6 +304,58 @@ def copy_tree(src, dst):
         else:
             raise error.general('copying tree (4): %s -> %s: %s' % (hsrc, hdst, str(why)))
 
+def get_size(path, depth = -1):
+    #
+    # Get the size the directory tree manually to the required depth.
+    # This makes sure on Windows the files are correctly encoded to avoid
+    # the file name size limit. On Windows the os.walk fails once we
+    # get to the max path length on Windows.
+    #
+    def _isdir(path):
+        hpath = host(path)
+        return os.path.isdir(hpath) and not os.path.islink(hpath)
+
+    def _node_size(path):
+        hpath = host(path)
+        size = 0
+        if not os.path.islink(hpath):
+            size = os.path.getsize(hpath)
+        return size
+
+    def _get_size(path, depth, level = 0):
+        level += 1
+        dirs = []
+        size = 0
+        for name in listdir(path):
+            path_ = join(path, shell(name))
+            hname = host(path_)
+            if _isdir(path_):
+                dirs += [shell(name)]
+            else:
+                size += _node_size(path_)
+        if depth < 0 or level < depth:
+            for name in dirs:
+                dir = join(path, name)
+                size += _get_size(dir, depth, level)
+        return size
+
+    path = shell(path)
+    hpath = host(path)
+    size = 0
+
+    if os.path.exists(hpath):
+        size = _get_size(path, depth)
+
+    return size
+
+def get_humanize_size(path, depth = -1):
+    size = get_size(path, depth)
+    for unit in ['','K','M','G','T','P','E','Z']:
+        if abs(size) < 1024.0:
+            return "%5.3f%sB" % (size, unit)
+        size /= 1024.0
+    return "%.3f%sB" % (size, 'Y')
+
 if __name__ == '__main__':
     print(host('/a/b/c/d-e-f'))
     print(host('//a/b//c/d-e-f'))
@@ -311,6 +363,10 @@ if __name__ == '__main__':
     print(basename('/as/sd/df/fg/me.txt'))
     print(dirname('/as/sd/df/fg/me.txt'))
     print(join('/d', 'g', '/tyty/fgfg'))
+    print('size of . depth all: ', get_size('.'))
+    print('size of . depth   1: ', get_size('.', 1))
+    print('size of . depth   2: ', get_size('.', 2))
+    print('size of . as human : ', get_humanize_size('.'))
     windows = True
     print(host('/a/b/c/d-e-f'))
     print(host('//a/b//c/d-e-f'))