Add a stats report command.

The report shows the level of changes we have made to the FreeBSD code.

builder.py

@@ -55,6 +55,11 @@ isDryRun = False
 isDiffMode = False
 filesProcessedCount = 0
 filesProcessed = []
+filesTotal = 0
+filesTotalLines = 0
+filesTotalInserts = 0
+filesTotalDeletes = 0
+diffDetails = { }
 
 verboseInfo = 1
 verboseDetail = 2
@@ -64,12 +69,32 @@ verboseDebug = 4
 def verbose(level = verboseInfo):
     return verboseLevel >= level
 
-def changedFileSummary():
+def changedFileSummary(statsReport = False):
+
+    global filesTotal, filesTotalLines, filesTotalInserts, filesTotalDeletes
+
     if isDiffMode == False:
-        print('%d file(s) were changed:' % (filesProcessedCount))
         if verbose():
+            print('%d file(s) were changed:' % (filesProcessedCount))
             for f in sorted(filesProcessed):
                 print(' %s' % (f))
+        else:
+            print('%d file(s) were changed.' % (filesProcessedCount))
+    if statsReport:
+        print('Stats Report:')
+        transparent = filesTotal - len(diffDetails)
+        changes = filesTotalInserts + filesTotalDeletes
+        opacity = (float(changes) / (filesTotalLines + changes)) * 100.0
+        print(' Total File(s):%d  Unchanged:%d (%.1f%%)  Changed:%d' \
+              '   Opacity:%5.1f%% Lines:%d Edits:%d (+):%d (-):%d'  % \
+              (filesTotal, transparent, (float(transparent) / filesTotal) * 100.0, len(diffDetails), \
+               opacity, filesTotalLines, changes, filesTotalInserts, filesTotalDeletes))
+        #
+        # Sort by opacity.
+        #
+        ordered_diffs = sorted(diffDetails.items(), key = lambda diff: diff[1].opacity, reverse = True)
+        for f in ordered_diffs:
+            print('  %s' % (diffDetails[f[0]].status()))
 
 def readFile(name):
     try:
@@ -105,6 +130,28 @@ class error(Exception):
     def __str__(self):
         return self.msg
 
+#
+# Diff Record
+#
+class diffRecord:
+    def __init__(self, src, dst, orig, diff, inserts, deletes):
+        self.src = src
+        self.dst = dst
+        self.orig = orig
+        self.diff = diff
+        self.lines = len(orig)
+        self.inserts = inserts
+        self.deletes = deletes
+        self.changes = inserts + deletes
+        self.opacity = (float(self.changes) / (self.lines + self.changes)) * 100.0
+
+    def __repr__(self):
+        return self.src
+
+    def status(self):
+        return 'opacity:%5.1f%% edits:%4d (+):%-4d (-):%-4d %s' % \
+            (self.opacity, self.changes, self.inserts, self.deletes, self.src)
+
 #
 # This stuff needs to move to libbsd.py.
 #
@@ -221,6 +268,41 @@ def assertSourceFile(path):
         print("*** Move it to a header file list")
         sys.exit(2)
 
+def diffSource(dstLines, srcLines, src, dst):
+    global filesTotal, filesTotalLines, filesTotalInserts, filesTotalDeletes
+    #
+    # Diff, note there is no line termination on each string.  Expand the
+    # generator to list because the generator is not reusable.
+    #
+    diff = list(difflib.unified_diff(dstLines,
+                                     srcLines,
+                                     fromfile = src,
+                                     tofile = dst,
+                                     n = 5,
+                                     lineterm = ''))
+    inserts = 0
+    deletes = 0
+    if len(diff) > 0:
+        if src in diffDetails and \
+           diffDetails[src].dst != dst and diffDetails[src].diff != diff:
+            raise error('repeated diff of file different: src:%s dst:%s' % (src, dst))
+        for l in diff:
+            if l[0] == '-':
+                deletes += 1
+            elif l[0] == '+':
+                inserts += 1
+        diffDetails[src] = diffRecord(src, dst, srcLines, diff, inserts, deletes)
+
+    #
+    # Count the total files, lines and the level of changes.
+    #
+    filesTotal += 1
+    filesTotalLines += len(srcLines)
+    filesTotalInserts += inserts
+    filesTotalDeletes += deletes
+
+    return diff
+
 #
 # Converters provide a way to alter the various types of code. The conversion
 # process filters a file as it is copies from the source path to the
@@ -276,15 +358,9 @@ class Converter(object):
             print('Unified diff: %s (lines:%d)' % (src, len(srcLines)))
 
         #
-        # Diff, note there is no line termination on each string.  Expand the
+        # Diff, note there is no line termination on each string.
-        # generator to list because the generator is not reusable.
         #
-        diff = list(difflib.unified_diff(dstLines,
+        diff = diffSource(dstLines, srcLines, src, dst)
-                                         srcLines,
-                                         fromfile = src,
-                                         tofile = dst,
-                                         n = 5,
-                                         lineterm = ''))
 
         #
         # The diff list is empty if the files are the same.

freebsd-to-rtems.py

@@ -48,6 +48,7 @@ import libbsd
 isForward = True
 isEarlyExit = False
 isOnlyBuildScripts = False
+statsReport = False
 
 def usage():
     print("freebsd-to-rtems.py [args]")
@@ -57,6 +58,7 @@ def usage():
     print("  -e|--early-exit   evaluate arguments, print results, and exit")
     print("  -m|--makefile     Warning: depreciated and will be removed ")
     print("  -b|--buildscripts just generate the build scripts")
+    print("  -S|--stats        Print a statistics report")
     print("  -R|--reverse      default FreeBSD -> RTEMS, reverse that")
     print("  -r|--rtems        RTEMS Libbsd directory (default: '.')")
     print("  -f|--freebsd      FreeBSD SVN directory (default: 'freebsd-org')")
@@ -64,11 +66,11 @@ def usage():
 
 # Parse the arguments
 def parseArguments():
-    global isForward, isEarlyExit
+    global isForward, isEarlyExit, statsReport
     global isOnlyBuildScripts
     try:
         opts, args = getopt.getopt(sys.argv[1:],
-                                   "?hdDembRr:f:v",
+                                   "?hdDembSRr:f:v",
                                    [ "help",
                                      "help",
                                      "dry-run"
@@ -77,6 +79,7 @@ def parseArguments():
                                      "makefile"
                                      "buildscripts"
                                      "reverse"
+                                     "stats"
                                      "rtems="
                                      "freebsd="
                                      "verbose" ])
@@ -99,6 +102,8 @@ def parseArguments():
             isEarlyExit = True
         elif o in ("-b", "--buildscripts") or o in ("-m", "--makefile"):
             isOnlyBuildScripts = True
+        elif o in ("-S", "--stats"):
+            statsReport = True
         elif o in ("-R", "--reverse"):
             isForward = False
         elif o in ("-r", "--rtems"):
@@ -152,7 +157,7 @@ try:
     if not isOnlyBuildScripts:
         wafGen.processSource(isForward)
     wafGen.generate(libbsd.rtems_version())
-    builder.changedFileSummary()
+    builder.changedFileSummary(statsReport)
 except IOError as ioe:
     print('error: %s' % (str(ioe)))
 except builder.error as be:

libbsd.txt

@@ -614,9 +614,10 @@ freebsd-to-rtems.py [args]
   -e|--early-exit   evaluate arguments, print results, and exit
   -m|--makefile     Warning: depreciated and will be removed
   -b|--buildscripts just generate the build scripts
+  -S|--stats        Print a statistics report
   -R|--reverse      default FreeBSD -> RTEMS, reverse that
-  -r|--rtems        RTEMS directory
+  -r|--rtems        RTEMS Libbsd directory (default: '.')
-  -f|--freebsd      FreeBSD directory
+  -f|--freebsd      FreeBSD SVN directory (default: 'freebsd-org')
   -v|--verbose      enable verbose output mode
 ----
 
@@ -682,6 +683,19 @@ detail and debug level information from the command.
 . Run `./create-kernel-namespace.sh` if you imported kernel space headers.  Add only your new defines via `git add -p rtemsbsd/include/machine/rtems-bsd-kernel-namespace.h`.
 . Create one commit from this.
 
+The -S or --stats option generates reports the changes we have made to
+FreeBSD. If the code has been reserved into the original FreeBSD tree it will
+show nothing has changed. To see what we have change:
+
+ $ cd freebsd-org
+ $ git checkout -- .
+ $ cd ..
+ $ ./freebsd-to-rtems.py -R -S -d
+
+The report lists the files change based on the opacity level. The opacity is a
+measure on how much of a file differs from the original FreeBSD source. The
+lower the value the more transparent the source file it.
+
 == Initialization of the BSD Library
 
 The initialization of the BSD library is based on the FreeBSD SYSINIT(9)