waf: Refector the builder to work with Python3 and UTF-8 source files.

Python 3 requires better UTF-8 handling of files and FreeBSD has UTF-8
characters in some files.

Refactor builder.py to clean up the code and remove the need to have
a temporary file. Update other scripts to use the new code.

builder.py

@@ -43,33 +43,71 @@ import sys
 import getopt
 import filecmp
 import difflib
+import codecs
 
 #
 # Global controls.
 #
 RTEMS_DIR = "."
 FreeBSD_DIR = "freebsd-org"
-isVerbose = False
+verboseLevel = 0
 isDryRun = False
 isDiffMode = False
 filesProcessedCount = 0
 filesProcessed = []
 
+verboseInfo = 1
+verboseDetail = 2
+verboseMoreDetail = 3
+verboseDebug = 4
+
+def verbose(level = verboseInfo):
+    return verboseLevel >= level
+
 def changedFileSummary():
     if isDiffMode == False:
         print('%d file(s) were changed:' % (filesProcessedCount))
-        for f in sorted(filesProcessed):
+        if verbose():
-            print(' %s' % (f))
+            for f in sorted(filesProcessed):
+                print(' %s' % (f))
 
+def read_file(name):
+    try:
+        contents = codecs.open(name, mode = 'r', encoding = 'utf-8', errors = 'ignore').read()
+    except UnicodeDecodeError as ude:
+        print('error: reading: %s: %s' % (name, ude))
+        sys.exit(1)
+    return contents
+
+def write_file(name, contents):
+    path = os.path.dirname(name)
+    if not os.path.exists(path):
+        try:
+            os.makedirs(path)
+        except OSError as oe:
+            print('error: cannot create directory: %s: %s' % (dst_path, oe))
+            sys.exit(1)
+    try:
+        codecs.open(name, mode = 'w',  encoding = 'utf-8', errors = 'ignore').write(contents)
+    except UnicodeDecodeError as ude:
+        print('error: write: %s: %s' % (name, ude))
+        sys.exit(1)
+
+#
+# A builder error.
+#
 class error(Exception):
     """Base class for exceptions."""
-    def __init(self, msg):
+    def __init__(self, msg):
         self.msg = 'error: %s' % (msg)
     def set_output(self, msg):
         self.msg = msg
     def __str__(self):
         return self.msg
 
+#
+# This stuff needs to move to libbsd.py.
+#
 def common_flags():
     return ['-O2',
             '-g',
@@ -136,61 +174,34 @@ def header_paths():
             ('mDNSResponder/mDNSShared',      'dns_sd.h',          ''),
             ('mDNSResponder/mDNSPosix',       'mDNSPosix.h',       '')]
 
-# compare and process file only if different
-#  + copy or diff depending on execution mode
-def processIfDifferent(new, old, src):
-
-    global filesProcessedCount
-    global filesProcessed
-    global isVerbose, isDryRun, isEarlyExit
-
-    if not os.path.exists(old) or \
-       filecmp.cmp(new, old, shallow = False) == False:
-        filesProcessed += [old]
-        filesProcessedCount += 1
-        if isDiffMode == False:
-            if isVerbose == True:
-                print("Move " + src + " to " + old)
-            if isDryRun == False:
-                shutil.move(new, old)
-        else:
-            if isVerbose == True:
-                print("Diff %s => %s" % (src, new))
-            old_contents = open(old).readlines()
-            new_contents = open(new).readlines()
-            for line in \
-                difflib.unified_diff(old_contents, new_contents,
-                                     fromfile = src, tofile = new, n = 5):
-                sys.stdout.write(line)
-
 # Move target dependent files under a machine directory
 def mapCPUDependentPath(path):
   return path.replace("include/", "include/machine/")
 
 def fixIncludes(data):
-    data = re.sub('#include <sys/lock.h>', '#include <rtems/bsd/sys/lock.h>', data)
+    data = re.sub('#include <sys/lock.h>',     '#include <rtems/bsd/sys/lock.h>', data)
-    data = re.sub('#include <sys/time.h>', '#include <rtems/bsd/sys/time.h>', data)
+    data = re.sub('#include <sys/time.h>',     '#include <rtems/bsd/sys/time.h>', data)
-    data = re.sub('#include <sys/cpuset.h>', '#include <rtems/bsd/sys/cpuset.h>', data)
+    data = re.sub('#include <sys/cpuset.h>',   '#include <rtems/bsd/sys/cpuset.h>', data)
-    data = re.sub('#include <sys/errno.h>', '#include <rtems/bsd/sys/errno.h>', data)
+    data = re.sub('#include <sys/errno.h>',    '#include <rtems/bsd/sys/errno.h>', data)
-    data = re.sub('#include <sys/param.h>', '#include <rtems/bsd/sys/param.h>', data)
+    data = re.sub('#include <sys/param.h>',    '#include <rtems/bsd/sys/param.h>', data)
-    data = re.sub('#include <sys/types.h>', '#include <rtems/bsd/sys/types.h>', data)
+    data = re.sub('#include <sys/types.h>',    '#include <rtems/bsd/sys/types.h>', data)
     data = re.sub('#include <sys/resource.h>', '#include <rtems/bsd/sys/resource.h>', data)
-    data = re.sub('#include <sys/unistd.h>', '#include <rtems/bsd/sys/unistd.h>', data)
+    data = re.sub('#include <sys/unistd.h>',   '#include <rtems/bsd/sys/unistd.h>', data)
-    data = re.sub('#include <sys/_types.h>', '#include <rtems/bsd/sys/_types.h>', data)
+    data = re.sub('#include <sys/_types.h>',   '#include <rtems/bsd/sys/_types.h>', data)
     return data
 
 # revert fixing the include paths inside a C or .h file
 def revertFixIncludes(data):
-    data = re.sub('#include <rtems/bsd/', '#include <', data)
+    data = re.sub('#include <rtems/bsd/',  '#include <', data)
-    data = re.sub('#include <util.h>', '#include <rtems/bsd/util.h>', data)
+    data = re.sub('#include <util.h>',     '#include <rtems/bsd/util.h>', data)
-    data = re.sub('#include <bsd.h>', '#include <rtems/bsd/bsd.h>', data)
+    data = re.sub('#include <bsd.h>',      '#include <rtems/bsd/bsd.h>', data)
     data = re.sub('#include <zerocopy.h>', '#include <rtems/bsd/zerocopy.h>', data)
     return data
 
 # fix include paths inside a C or .h file
 def fixLocalIncludes(data):
-    data = re.sub('#include "opt_([^"]*)"', '#include <rtems/bsd/local/opt_\\1>', data)
+    data = re.sub('#include "opt_([^"]*)"',    '#include <rtems/bsd/local/opt_\\1>', data)
-    data = re.sub('#include "([^"]*)_if.h"', '#include <rtems/bsd/local/\\1_if.h>', data)
+    data = re.sub('#include "([^"]*)_if.h"',   '#include <rtems/bsd/local/\\1_if.h>', data)
     data = re.sub('#include "miidevs([^"]*)"', '#include <rtems/bsd/local/miidevs\\1>', data)
     data = re.sub('#include "usbdevs([^"]*)"', '#include <rtems/bsd/local/usbdevs\\1>', data)
     return data
@@ -212,90 +223,178 @@ def assertSourceFile(path):
         print("*** Move it to a header file list")
         sys.exit(2)
 
+#
+# Converters provide a way to alter the various types of code. The conversion
+# process filters a file as it is copies from the source path to the
+# destination path. Specialised versions are provided for different types of
+# source.
+#
 class Converter(object):
-    def convert(self, src):
-        return open(src).read()
 
-    def isConvertible(self):
+    def convert(self, src, dst, has_source = True, source_filter = None, src_contents = None):
-        return True
+
+        global filesProcessed, filesProcessedCount
+
+        if verbose(verboseDebug):
+            print("convert: filter:%s: %s -> %s" % \
+                  (['yes', 'no'][source_filter is None], src, dst))
+
+        #
+        # If there is no source raise an error if we expect source else print a
+        # warning and do not try and convert.
+        #
+        if src_contents is None:
+            if not os.path.exists(src):
+                if has_source:
+                    raise error('source not found: %s' % (src))
+                else:
+                    print('warning: no source: %s' % (src))
+                    return
+
+            #
+            # Files read as a single string if not passed in.
+            #
+            src_contents = read_file(src)
+
+        if os.path.exists(dst):
+            dst_contents = read_file(dst)
+        else:
+            print('warning: no destination: %s' % (dst))
+            dst_contents = ''
+
+        #
+        # Filter the source.
+        #
+        if source_filter is not None:
+            src_contents = source_filter(src_contents)
+
+        #
+        # Split into a list of lines.
+        #
+        src_lines = src_contents.split(os.linesep)
+        dst_lines = dst_contents.split(os.linesep)
+
+        if verbose(verboseDebug):
+            print('Unified diff: %s (lines:%d)' % (src, len(src_lines)))
+
+        #
+        # Diff, note there is no line termination on each string.  Expand the
+        # generator to list because the generator is not reusable.
+        #
+        diff = list(difflib.unified_diff(dst_lines,
+                                         src_lines,
+                                         fromfile = src,
+                                         tofile = dst,
+                                         n = 5,
+                                         lineterm = ''))
+
+        #
+        # The diff list is empty if the files are the same.
+        #
+        if len(diff) > 0:
+
+            if verbose(verboseDebug):
+                print('Unified diff length: %d' % len(diff))
+
+            filesProcessed += [dst]
+            filesProcessedCount += 1
+            if isDiffMode == False:
+                if verbose(verboseDetail):
+                    print("UPDATE: %s -> %s" % (src, dst))
+                if isDryRun == False:
+                    write_file(dst, src_contents)
+            else:
+                print("diff -u %s %s" % (src, dst))
+                for l in diff:
+                    print(l)
 
 class NoConverter(Converter):
-    def convert(self, src):
+    def convert(self, src, dst, has_source = True, source_filter = None):
-        raise
-
-    def isConvertible(self):
-        return False
-
-class EmptyConverter(Converter):
-    def convert(self, src):
         return '/* EMPTY */\n'
 
 class FromFreeBSDToRTEMSHeaderConverter(Converter):
-    def convert(self, src):
+    def source_filter(self, data):
-        data = super(FromFreeBSDToRTEMSHeaderConverter, self).convert(src)
         data = fixLocalIncludes(data)
         data = fixIncludes(data)
         return data
 
+    def convert(self, src, dst):
+        sconverter = super(FromFreeBSDToRTEMSHeaderConverter, self)
+        sconverter.convert(src, dst, source_filter = self.source_filter)
+
 class FromFreeBSDToRTEMSUserSpaceHeaderConverter(Converter):
-    def convert(self, src):
+    def source_filter(self, data):
-        data = super(FromFreeBSDToRTEMSUserSpaceHeaderConverter, self).convert(src)
         data = fixIncludes(data)
         return data
 
+    def convert(self, src, dst):
+        sconverter = super(FromFreeBSDToRTEMSUserSpaceHeaderConverter, self)
+        sconverter.convert(src, dst, source_filter = self.source_filter)
+
 class FromFreeBSDToRTEMSSourceConverter(Converter):
-    def convert(self, src):
+    def source_filter(self, data):
-        data = super(FromFreeBSDToRTEMSSourceConverter, self).convert(src)
         data = fixLocalIncludes(data)
         data = fixIncludes(data)
         data = '#include <machine/rtems-bsd-kernel-space.h>\n\n' + data
         return data
 
+    def convert(self, src, dst):
+        sconverter = super(FromFreeBSDToRTEMSSourceConverter, self)
+        sconverter.convert(src, dst, source_filter = self.source_filter)
+
 class FromFreeBSDToRTEMSUserSpaceSourceConverter(Converter):
-    def convert(self, src):
+    def source_filter(self, data):
-        data = super(FromFreeBSDToRTEMSUserSpaceSourceConverter, self).convert(src)
         data = fixIncludes(data)
         data = '#include <machine/rtems-bsd-user-space.h>\n\n' + data
         return data
 
+    def convert(self, src, dst):
+        sconverter = super(FromFreeBSDToRTEMSUserSpaceSourceConverter, self)
+        sconverter.convert(src, dst, source_filter = self.source_filter)
+
 class FromRTEMSToFreeBSDHeaderConverter(Converter):
-    def convert(self, src):
+    def source_filter(self, data):
-        data = super(FromRTEMSToFreeBSDHeaderConverter, self).convert(src)
         data = revertFixLocalIncludes(data)
         data = revertFixIncludes(data)
         return data
 
+    def convert(self, src, dst):
+        sconverter = super(FromRTEMSToFreeBSDHeaderConverter, self)
+        sconverter.convert(src, dst, has_source = False,  source_filter = self.source_filter)
+
 class FromRTEMSToFreeBSDSourceConverter(Converter):
-    def convert(self, src):
+    def source_filter(self, data):
-        data = super(FromRTEMSToFreeBSDSourceConverter, self).convert(src)
         data = re.sub('#include <machine/rtems-bsd-kernel-space.h>\n\n', '', data)
         data = re.sub('#include <machine/rtems-bsd-user-space.h>\n\n', '', data)
-        data = revertFixLocalIncludes(data)
-        data = revertFixIncludes(data)
         return data
 
+    def convert(self, src, dst):
+        sconverter = super(FromRTEMSToFreeBSDSourceConverter, self)
+        sconverter.convert(src, dst, has_source = False, source_filter = self.source_filter)
+
+#
+# Compose a path based for the various parts of the source tree.
+#
 class PathComposer(object):
     def composeFreeBSDPath(self, path):
         return path
 
     def composeRTEMSPath(self, path, prefix):
-        path = prefix + path
+        return os.path.join(prefix, path)
-        return path
 
 class FreeBSDPathComposer(PathComposer):
     def composeFreeBSDPath(self, path):
-        return FreeBSD_DIR + '/' + path
+        return os.path.join(FreeBSD_DIR, path)
 
     def composeRTEMSPath(self, path, prefix):
-        return prefix + 'freebsd/' + path
+        return os.path.join(prefix, 'freebsd', path)
 
 class RTEMSPathComposer(PathComposer):
     def composeFreeBSDPath(self, path):
         return path
 
     def composeRTEMSPath(self, path, prefix):
-        path = prefix + 'rtemsbsd/' + path
+        return os.path.join(prefix, 'rtemsbsd', path)
-        return path
 
 class CPUDependentPathComposer(FreeBSDPathComposer):
     def composeRTEMSPath(self, path, prefix):
@@ -323,49 +422,42 @@ class BuildSystemFragmentComposer(object):
     def compose(self, path):
         return ''
 
+#
+# File - a file in the source we move backwards and forwards.
+#
 class File(object):
     def __init__(self, path, pathComposer,
-                 fromFreeBSDToRTEMSConverter, fromRTEMSToFreeBSDConverter, buildSystemComposer):
+                 forwardConverter, reverseConverter, buildSystemComposer):
+        if verbose(verboseMoreDetail):
+            print("FILE: %-50s F:%-45s R:%-45s" % \
+                  (path,
+                   forwardConverter.__class__.__name__,
+                   reverseConverter.__class__.__name__))
         self.path = path
         self.pathComposer = pathComposer
-        self.fromFreeBSDToRTEMSConverter = fromFreeBSDToRTEMSConverter
+        self.freebsdPath = self.pathComposer.composeFreeBSDPath(self.path)
-        self.fromRTEMSToFreeBSDConverter = fromRTEMSToFreeBSDConverter
+        self.rtemsPath = self.pathComposer.composeRTEMSPath(self.path, RTEMS_DIR)
+        self.forwardConverter = forwardConverter
+        self.reverseConverter = reverseConverter
         self.buildSystemComposer = buildSystemComposer
 
-    def copy(self, dst, src, converter = None):
+    def processSource(self, forward):
-        import tempfile
+        if forward:
-        if converter is not None and converter.isConvertible():
+            if verbose(verboseDetail):
-            try:
+                print("process source: %s => %s" % (self.freebsdPath, self.rtemsPath))
-                if isDryRun == False:
+            self.forwardConverter.convert(self.freebsdPath, self.rtemsPath)
-                    os.makedirs(os.path.dirname(dst))
+        else:
-            except OSError:
+            if verbose(verboseDetail):
-                pass
+                print("process source: %s => %s converter:%s" % \
-            data = converter.convert(src)
+                      (self.rtemsPath, self.freebsdPath, self.reverseConverter.__class__.__name__))
-            try:
+            self.reverseConverter.convert(self.rtemsPath, self.freebsdPath)
-                out = tempfile.NamedTemporaryFile(delete = False)
-                out.write(data)
-                out.close()
-                processIfDifferent(out.name, dst, src)
-            finally:
-                try:
-                    os.remove(out.name)
-                except:
-                    pass
-
-    def copyFromFreeBSDToRTEMS(self):
-        src = self.pathComposer.composeFreeBSDPath(self.path)
-        dst = self.pathComposer.composeRTEMSPath(self.path, RTEMS_DIR + '/')
-        self.copy(dst, src, self.fromFreeBSDToRTEMSConverter)
-
-    def copyFromRTEMSToFreeBSD(self):
-        src = self.pathComposer.composeRTEMSPath(self.path, RTEMS_DIR + '/')
-        dst = self.pathComposer.composeFreeBSDPath(self.path)
-        self.copy(dst, src, self.fromRTEMSToFreeBSDConverter)
 
     def getFragment(self):
         return self.buildSystemComposer.compose(self.pathComposer.composeRTEMSPath(self.path, ''))
 
+#
 # Module - logical group of related files we can perform actions on
+#
 class Module:
     def __init__(self, name):
         self.name = name
@@ -378,19 +470,14 @@ class Module:
         if cpu not in self.cpuDependentSourceFiles:
             self.cpuDependentSourceFiles[cpu] = []
 
-    def copyFromFreeBSDToRTEMS(self):
+    def processSource(self, direction):
+        if verbose(verboseDetail):
+            print("process module: %s" % (self.name))
         for f in self.files:
-            f.copyFromFreeBSDToRTEMS()
+            f.processSource(direction)
         for cpu, files in self.cpuDependentSourceFiles.items():
             for f in files:
-                f.copyFromFreeBSDToRTEMS()
+                f.processSource(direction)
-
-    def copyFromRTEMSToFreeBSD(self):
-        for f in self.files:
-            f.copyFromRTEMSToFreeBSD()
-        for cpu, files in self.cpuDependentSourceFiles.items():
-            for f in files:
-                f.copyFromRTEMSToFreeBSD()
 
     def addFiles(self, newFiles, buildSystemComposer = BuildSystemFragmentComposer()):
         files = []
@@ -478,6 +565,9 @@ class Module:
     def addDependency(self, dep):
         self.dependencies += [dep]
 
+#
+# Manager - a collection of modules.
+#
 class ModuleManager:
     def __init__(self):
         self.modules = {}
@@ -495,10 +585,8 @@ class ModuleManager:
     def addModule(self, module):
         self.modules[module.name] = module
 
-    def copyFromFreeBSDToRTEMS(self):
+    def processSource(self, direction):
+        if verbose(verboseDetail):
+            print("process modules:")
         for m in sorted(self.modules):
-            self.modules[m].copyFromFreeBSDToRTEMS()
+            self.modules[m].processSource(direction)
-
-    def copyFromRTEMSToFreeBSD(self):
-        for m in sorted(self.modules):
-            self.modules[m].copyFromRTEMSToFreeBSD()

freebsd-to-rtems.py

@@ -87,7 +87,7 @@ def parseArguments():
         sys.exit(2)
     for o, a in opts:
         if o in ("-v", "--verbose"):
-            builder.isVerbose = True
+            builder.verboseLevel += 1
         elif o in ("-h", "--help", "-?"):
             usage()
             sys.exit()
@@ -110,13 +110,14 @@ def parseArguments():
 
 parseArguments()
 
-print("Verbose:                     " + ("no", "yes")[builder.isVerbose])
+print("Verbose:                     %s (%d)" % (("no", "yes")[builder.verbose()],
-print("Dry Run:                     " + ("no", "yes")[builder.isDryRun])
+                                                builder.verboseLevel))
-print("Diff Mode Enabled:           " + ("no", "yes")[builder.isDiffMode])
+print("Dry Run:                     %s" % (("no", "yes")[builder.isDryRun]))
-print("Only Generate Build Scripts: " + ("no", "yes")[isOnlyBuildScripts])
+print("Diff Mode Enabled:           %s" % (("no", "yes")[builder.isDiffMode]))
-print("RTEMS Libbsd Directory:      " + builder.RTEMS_DIR)
+print("Only Generate Build Scripts: %s" % (("no", "yes")[isOnlyBuildScripts]))
-print("FreeBSD SVN Directory:       " + builder.FreeBSD_DIR)
+print("RTEMS Libbsd Directory:      %s" % (builder.RTEMS_DIR))
-print("Direction:                   " + ("reverse", "forward")[isForward])
+print("FreeBSD SVN Directory:       %s" % (builder.FreeBSD_DIR))
+print("Direction:                   %s" % (("reverse", "forward")[isForward]))
 
 # Check directory argument was set and exist
 def wasDirectorySet(desc, path):
@@ -147,16 +148,14 @@ if isEarlyExit == True:
 
 try:
     waf_gen = waf_generator.ModuleManager()
-
     libbsd.sources(waf_gen)
-
+    if not isOnlyBuildScripts:
-    # Perform the actual file manipulation
+        waf_gen.processSource(isForward)
-    if isForward:
+    waf_gen.generate(libbsd.rtems_version())
-        if not isOnlyBuildScripts:
-            waf_gen.copyFromFreeBSDToRTEMS()
-        waf_gen.generate(libbsd.rtems_version())
-    else:
-        waf_gen.copyFromRTEMSToFreeBSD()
     builder.changedFileSummary()
 except IOError as ioe:
-    print('error: %s' % (ioe))
+    print('error: %s' % (str(ioe)))
+except builder.error as be:
+    print('error: %s' % (be))
+except KeyboardInterrupt:
+    print('user abort')

waf_generator.py

@@ -147,20 +147,9 @@ class ModuleManager(builder.ModuleManager):
         self.script += line + os.linesep
 
     def write(self):
-        try:
+        name = os.path.join(builder.RTEMS_DIR, 'libbsd_waf.py')
-            out = tempfile.NamedTemporaryFile(delete = False)
+        converter = builder.Converter()
-            try:
+        converter.convert(name, name, src_contents = self.script)
-                out.write(bytes(self.script, sys.stdin.encoding))
-            except:
-                out.write(self.script)
-            out.close()
-            wscript = builder.RTEMS_DIR + '/libbsd_waf.py'
-            builder.processIfDifferent(out.name, wscript, "libbsd_waf.py")
-        finally:
-            try:
-                os.remove(out.name)
-            except:
-                pass
 
     def setGenerators(self):
         self.generator['convert'] = builder.Converter