Clean up some of our python scripts.
Change-Id: Ifa75345db43434298cfb6113fbe2f7a33b88c79d
diff --git a/libc/kernel/tools/clean_header.py b/libc/kernel/tools/clean_header.py
index 6601817..ebebe80 100755
--- a/libc/kernel/tools/clean_header.py
+++ b/libc/kernel/tools/clean_header.py
@@ -199,8 +199,7 @@
if opt == '-u':
noUpdate = 0
elif opt == '-v':
- verbose = 1
- D_setlevel(1)
+ logging.basicConfig(level=logging.DEBUG)
elif opt == '-k':
kernel_original_path = arg
elif opt == '-d':
diff --git a/libc/kernel/tools/cpp.py b/libc/kernel/tools/cpp.py
index 2be9532..0c098de 100644
--- a/libc/kernel/tools/cpp.py
+++ b/libc/kernel/tools/cpp.py
@@ -1711,7 +1711,7 @@
while j < n and not blocks[j].isIf():
j += 1
if j > i:
- D2("appending lines %d to %d" % (blocks[i].lineno, blocks[j-1].lineno))
+ logging.debug("appending lines %d to %d" % (blocks[i].lineno, blocks[j-1].lineno))
result += blocks[i:j]
if j >= n:
break
@@ -1730,17 +1730,17 @@
break
dir = blocks[j].directive
if dir == "endif":
- D2("remove 'if 0' .. 'endif' (lines %d to %d)" % (blocks[i].lineno, blocks[j].lineno))
+ logging.debug("remove 'if 0' .. 'endif' (lines %d to %d)" % (blocks[i].lineno, blocks[j].lineno))
i = j + 1
elif dir == "else":
# convert 'else' into 'if 1'
- D2("convert 'if 0' .. 'else' into 'if 1' (lines %d to %d)" % (blocks[i].lineno, blocks[j-1].lineno))
+ logging.debug("convert 'if 0' .. 'else' into 'if 1' (lines %d to %d)" % (blocks[i].lineno, blocks[j-1].lineno))
blocks[j].directive = "if"
blocks[j].expr = CppExpr( CppLineTokenizer("1").toTokenList() )
i = j
elif dir == "elif":
# convert 'elif' into 'if'
- D2("convert 'if 0' .. 'elif' into 'if'")
+ logging.debug("convert 'if 0' .. 'elif' into 'if'")
blocks[j].directive = "if"
i = j
continue
@@ -1749,25 +1749,25 @@
k = find_matching_endif( blocks, j+1 )
if k >= n:
# unterminated #if 1, finish here
- D2("unterminated 'if 1'")
+ logging.debug("unterminated 'if 1'")
result += blocks[j+1:k]
break
dir = blocks[k].directive
if dir == "endif":
- D2("convert 'if 1' .. 'endif' (lines %d to %d)" % (blocks[j].lineno, blocks[k].lineno))
+ logging.debug("convert 'if 1' .. 'endif' (lines %d to %d)" % (blocks[j].lineno, blocks[k].lineno))
result += optimize_if01(blocks[j+1:k])
i = k+1
elif dir == "else":
# convert 'else' into 'if 0'
- D2("convert 'if 1' .. 'else' (lines %d to %d)" % (blocks[j].lineno, blocks[k].lineno))
+ logging.debug("convert 'if 1' .. 'else' (lines %d to %d)" % (blocks[j].lineno, blocks[k].lineno))
result += optimize_if01(blocks[j+1:k])
blocks[k].directive = "if"
blocks[k].expr = CppExpr( CppLineTokenizer("0").toTokenList() )
i = k
elif dir == "elif":
# convert 'elif' into 'if 0'
- D2("convert 'if 1' .. 'elif' (lines %d to %d)" % (blocks[j].lineno, blocks[k].lineno))
+ logging.debug("convert 'if 1' .. 'elif' (lines %d to %d)" % (blocks[j].lineno, blocks[k].lineno))
result += optimize_if01(blocks[j+1:k])
blocks[k].expr = CppExpr( CppLineTokenizer("0").toTokenList() )
i = k
@@ -1835,7 +1835,6 @@
out = StringOutput()
lines = string.split(text, '\n')
list = BlockParser().parse( CppLinesTokenizer(lines) )
- #D_setlevel(2)
list.replaceTokens( kernel_token_replacements )
list.optimizeAll( {"__KERNEL__":kCppUndefinedMacro} )
list.write(out)
diff --git a/libc/kernel/tools/utils.py b/libc/kernel/tools/utils.py
index 0478e93..e5a310e 100644
--- a/libc/kernel/tools/utils.py
+++ b/libc/kernel/tools/utils.py
@@ -1,59 +1,29 @@
# common python utility routines for the Bionic tool scripts
-import sys, os, commands, string, commands
+import commands
+import logging
+import os
+import string
+import sys
-# basic debugging trace support
-# call D_setlevel to set the verbosity level
-# and D(), D2(), D3(), D4() to add traces
-#
-verbose = 0
def panic(msg):
- sys.stderr.write( find_program_name() + ": error: " )
- sys.stderr.write( msg )
+ sys.stderr.write(os.path.basename(sys.argv[0]) + ": error: ")
+ sys.stderr.write(msg)
sys.exit(1)
-def D(msg):
- global verbose
- if verbose > 0:
- print msg
-
-def D2(msg):
- global verbose
- if verbose >= 2:
- print msg
-
-def D3(msg):
- global verbose
- if verbose >= 3:
- print msg
-
-def D4(msg):
- global verbose
- if verbose >= 4:
- print msg
-
-def D_setlevel(level):
- global verbose
- verbose = level
-
-
-# other stuff
-#
-#
-def find_program_name():
- return os.path.basename(sys.argv[0])
def find_program_dir():
return os.path.dirname(sys.argv[0])
+
class StringOutput:
def __init__(self):
self.line = ""
def write(self,msg):
self.line += msg
- D2("write '%s'" % msg)
+ logging.debug("write '%s'" % msg)
def get(self):
return self.line
@@ -76,47 +46,6 @@
continue
os.mkdir(dir)
-def walk_source_files(paths,callback,args,excludes=[]):
- """recursively walk a list of paths and files, only keeping the source files in directories"""
- for path in paths:
- if len(path) > 0 and path[0] == '@':
- # this is the name of another file, include it and parse it
- path = path[1:]
- if os.path.exists(path):
- for line in open(path):
- if len(line) > 0 and line[-1] == '\n':
- line = line[:-1]
- walk_source_files([line],callback,args,excludes)
- continue
- if not os.path.isdir(path):
- callback(path,args)
- else:
- for root, dirs, files in os.walk(path):
- #print "w-- %s (ex: %s)" % (repr((root,dirs)), repr(excludes))
- if len(excludes):
- for d in dirs[:]:
- if os.path.join(root,d) in excludes:
- dirs.remove(d)
- for f in files:
- r, ext = os.path.splitext(f)
- if ext in [ ".h", ".c", ".cpp", ".S" ]:
- callback( "%s/%s" % (root,f), args )
-
-def cleanup_dir(path):
- """create a directory if needed, and ensure that it is totally empty
- by removing any existing content in it"""
- if not os.path.exists(path):
- os.mkdir(path)
- else:
- for root, dirs, files in os.walk(path, topdown=False):
- if root.endswith("kernel_headers/"):
- # skip 'kernel_headers'
- continue
- for name in files:
- os.remove(os.path.join(root, name))
- for name in dirs:
- os.rmdir(os.path.join(root, name))
-
class BatchFileUpdater:
"""a class used to edit several files at once"""
diff --git a/libc/tools/bionic_utils.py b/libc/tools/bionic_utils.py
deleted file mode 100644
index c38efb5..0000000
--- a/libc/tools/bionic_utils.py
+++ /dev/null
@@ -1,165 +0,0 @@
-# common python utility routines for the Bionic tool scripts
-
-import sys, os, commands, string
-
-all_arches = [ "arm", "arm64", "mips", "mips64", "x86", "x86_64" ]
-
-# basic debugging trace support
-# call D_setlevel to set the verbosity level
-# and D(), D2(), D3(), D4() to add traces
-#
-verbose = 0
-
-def D(msg):
- global verbose
- if verbose > 0:
- print msg
-
-def D2(msg):
- global verbose
- if verbose >= 2:
- print msg
-
-def D3(msg):
- global verbose
- if verbose >= 3:
- print msg
-
-def D4(msg):
- global verbose
- if verbose >= 4:
- print msg
-
-def D_setlevel(level):
- global verbose
- verbose = level
-
-
-# parser for the SYSCALLS.TXT file
-#
-class SysCallsTxtParser:
- def __init__(self):
- self.syscalls = []
- self.lineno = 0
-
- def E(self, msg):
- print "%d: %s" % (self.lineno, msg)
-
- def parse_line(self, line):
- """ parse a syscall spec line.
-
- line processing, format is
- return type func_name[|alias_list][:syscall_name[:socketcall_id]] ( [paramlist] ) architecture_list
- """
- pos_lparen = line.find('(')
- E = self.E
- if pos_lparen < 0:
- E("missing left parenthesis in '%s'" % line)
- return
-
- pos_rparen = line.rfind(')')
- if pos_rparen < 0 or pos_rparen <= pos_lparen:
- E("missing or misplaced right parenthesis in '%s'" % line)
- return
-
- return_type = line[:pos_lparen].strip().split()
- if len(return_type) < 2:
- E("missing return type in '%s'" % line)
- return
-
- syscall_func = return_type[-1]
- return_type = string.join(return_type[:-1],' ')
- socketcall_id = -1
-
- pos_colon = syscall_func.find(':')
- if pos_colon < 0:
- syscall_name = syscall_func
- else:
- if pos_colon == 0 or pos_colon+1 >= len(syscall_func):
- E("misplaced colon in '%s'" % line)
- return
-
- # now find if there is a socketcall_id for a dispatch-type syscall
- # after the optional 2nd colon
- pos_colon2 = syscall_func.find(':', pos_colon + 1)
- if pos_colon2 < 0:
- syscall_name = syscall_func[pos_colon+1:]
- syscall_func = syscall_func[:pos_colon]
- else:
- if pos_colon2+1 >= len(syscall_func):
- E("misplaced colon2 in '%s'" % line)
- return
- syscall_name = syscall_func[(pos_colon+1):pos_colon2]
- socketcall_id = int(syscall_func[pos_colon2+1:])
- syscall_func = syscall_func[:pos_colon]
-
- alias_delim = syscall_func.find('|')
- if alias_delim > 0:
- alias_list = syscall_func[alias_delim+1:].strip()
- syscall_func = syscall_func[:alias_delim]
- alias_delim = syscall_name.find('|')
- if alias_delim > 0:
- syscall_name = syscall_name[:alias_delim]
- syscall_aliases = string.split(alias_list, ',')
- else:
- syscall_aliases = []
-
- if pos_rparen > pos_lparen+1:
- syscall_params = line[pos_lparen+1:pos_rparen].split(',')
- params = string.join(syscall_params,',')
- else:
- syscall_params = []
- params = "void"
-
- t = {
- "name" : syscall_name,
- "func" : syscall_func,
- "aliases" : syscall_aliases,
- "params" : syscall_params,
- "decl" : "%-15s %s (%s);" % (return_type, syscall_func, params),
- "socketcall_id" : socketcall_id
- }
-
- # Parse the architecture list.
- arch_list = line[pos_rparen+1:].strip()
- if arch_list == "all":
- for arch in all_arches:
- t[arch] = True
- else:
- for arch in string.split(arch_list, ','):
- if arch in all_arches:
- t[arch] = True
- else:
- E("invalid syscall architecture '%s' in '%s'" % (arch, line))
- return
-
- self.syscalls.append(t)
-
- global verbose
- if verbose >= 2:
- print t
-
-
- def parse_file(self, file_path):
- D2("parse_file: %s" % file_path)
- fp = open(file_path)
- for line in fp.xreadlines():
- self.lineno += 1
- line = line.strip()
- if not line: continue
- if line[0] == '#': continue
- self.parse_line(line)
-
- fp.close()
-
-
-class StringOutput:
- def __init__(self):
- self.line = ""
-
- def write(self,msg):
- self.line += msg
- D2("write '%s'" % msg)
-
- def get(self):
- return self.line
diff --git a/libc/tools/gensyscalls.py b/libc/tools/gensyscalls.py
index 4619ec6..929bec4 100755
--- a/libc/tools/gensyscalls.py
+++ b/libc/tools/gensyscalls.py
@@ -8,14 +8,18 @@
import commands
import filecmp
import glob
+import logging
import os.path
import re
import shutil
import stat
+import string
import sys
import tempfile
-from bionic_utils import *
+
+all_arches = [ "arm", "arm64", "mips", "mips64", "x86", "x86_64" ]
+
# temp directory where we store all intermediate files
bionic_temp = tempfile.mkdtemp(prefix="bionic_gensyscalls");
@@ -387,6 +391,120 @@
return result
+class SysCallsTxtParser:
+ def __init__(self):
+ self.syscalls = []
+ self.lineno = 0
+
+ def E(self, msg):
+ print "%d: %s" % (self.lineno, msg)
+
+ def parse_line(self, line):
+ """ parse a syscall spec line.
+
+ line processing, format is
+ return type func_name[|alias_list][:syscall_name[:socketcall_id]] ( [paramlist] ) architecture_list
+ """
+ pos_lparen = line.find('(')
+ E = self.E
+ if pos_lparen < 0:
+ E("missing left parenthesis in '%s'" % line)
+ return
+
+ pos_rparen = line.rfind(')')
+ if pos_rparen < 0 or pos_rparen <= pos_lparen:
+ E("missing or misplaced right parenthesis in '%s'" % line)
+ return
+
+ return_type = line[:pos_lparen].strip().split()
+ if len(return_type) < 2:
+ E("missing return type in '%s'" % line)
+ return
+
+ syscall_func = return_type[-1]
+ return_type = string.join(return_type[:-1],' ')
+ socketcall_id = -1
+
+ pos_colon = syscall_func.find(':')
+ if pos_colon < 0:
+ syscall_name = syscall_func
+ else:
+ if pos_colon == 0 or pos_colon+1 >= len(syscall_func):
+ E("misplaced colon in '%s'" % line)
+ return
+
+ # now find if there is a socketcall_id for a dispatch-type syscall
+ # after the optional 2nd colon
+ pos_colon2 = syscall_func.find(':', pos_colon + 1)
+ if pos_colon2 < 0:
+ syscall_name = syscall_func[pos_colon+1:]
+ syscall_func = syscall_func[:pos_colon]
+ else:
+ if pos_colon2+1 >= len(syscall_func):
+ E("misplaced colon2 in '%s'" % line)
+ return
+ syscall_name = syscall_func[(pos_colon+1):pos_colon2]
+ socketcall_id = int(syscall_func[pos_colon2+1:])
+ syscall_func = syscall_func[:pos_colon]
+
+ alias_delim = syscall_func.find('|')
+ if alias_delim > 0:
+ alias_list = syscall_func[alias_delim+1:].strip()
+ syscall_func = syscall_func[:alias_delim]
+ alias_delim = syscall_name.find('|')
+ if alias_delim > 0:
+ syscall_name = syscall_name[:alias_delim]
+ syscall_aliases = string.split(alias_list, ',')
+ else:
+ syscall_aliases = []
+
+ if pos_rparen > pos_lparen+1:
+ syscall_params = line[pos_lparen+1:pos_rparen].split(',')
+ params = string.join(syscall_params,',')
+ else:
+ syscall_params = []
+ params = "void"
+
+ t = {
+ "name" : syscall_name,
+ "func" : syscall_func,
+ "aliases" : syscall_aliases,
+ "params" : syscall_params,
+ "decl" : "%-15s %s (%s);" % (return_type, syscall_func, params),
+ "socketcall_id" : socketcall_id
+ }
+
+ # Parse the architecture list.
+ arch_list = line[pos_rparen+1:].strip()
+ if arch_list == "all":
+ for arch in all_arches:
+ t[arch] = True
+ else:
+ for arch in string.split(arch_list, ','):
+ if arch in all_arches:
+ t[arch] = True
+ else:
+ E("invalid syscall architecture '%s' in '%s'" % (arch, line))
+ return
+
+ self.syscalls.append(t)
+
+ logging.debug(t)
+
+
+ def parse_file(self, file_path):
+ logging.debug("parse_file: %s" % file_path)
+ fp = open(file_path)
+ for line in fp.xreadlines():
+ self.lineno += 1
+ line = line.strip()
+ if not line: continue
+ if line[0] == '#': continue
+ self.parse_line(line)
+
+ fp.close()
+
+
class State:
def __init__(self):
self.old_stubs = []
@@ -444,7 +562,7 @@
def gen_glibc_syscalls_h(self):
# TODO: generate a separate file for each architecture, like glibc's bits/syscall.h.
glibc_syscalls_h_path = "include/sys/glibc-syscalls.h"
- D("generating " + glibc_syscalls_h_path)
+ logging.info("generating " + glibc_syscalls_h_path)
glibc_fp = create_file(glibc_syscalls_h_path)
glibc_fp.write("/* %s */\n" % warning)
glibc_fp.write("#ifndef _BIONIC_GLIBC_SYSCALLS_H_\n")
@@ -473,7 +591,7 @@
for arch in all_arches:
if syscall.has_key("asm-%s" % arch):
filename = "arch-%s/syscalls/%s.S" % (arch, syscall["func"])
- D2(">>> generating " + filename)
+ logging.info(">>> generating " + filename)
fp = create_file(filename)
fp.write(syscall["asm-%s" % arch])
fp.close()
@@ -481,28 +599,28 @@
def regenerate(self):
- D("scanning for existing architecture-specific stub files...")
+ logging.info("scanning for existing architecture-specific stub files...")
for arch in all_arches:
arch_dir = "arch-" + arch
- D("scanning " + os.path.join(bionic_libc_root, arch_dir))
+ logging.info("scanning " + os.path.join(bionic_libc_root, arch_dir))
rel_path = os.path.join(arch_dir, "syscalls")
for file in os.listdir(os.path.join(bionic_libc_root, rel_path)):
if file.endswith(".S"):
self.old_stubs.append(os.path.join(rel_path, file))
- D("found %d stub files" % len(self.old_stubs))
+ logging.info("found %d stub files" % len(self.old_stubs))
if not os.path.exists(bionic_temp):
- D("creating %s..." % bionic_temp)
+ logging.info("creating %s..." % bionic_temp)
make_dir(bionic_temp)
- D("re-generating stubs and support files...")
+ logging.info("re-generating stubs and support files...")
self.gen_glibc_syscalls_h()
self.gen_syscall_stubs()
- D("comparing files...")
+ logging.info("comparing files...")
adds = []
edits = []
@@ -511,18 +629,18 @@
libc_file = os.path.join(bionic_libc_root, stub)
if not os.path.exists(libc_file):
# new file, git add it
- D("new file: " + stub)
+ logging.info("new file: " + stub)
adds.append(libc_file)
shutil.copyfile(tmp_file, libc_file)
elif not filecmp.cmp(tmp_file, libc_file):
- D("changed file: " + stub)
+ logging.info("changed file: " + stub)
edits.append(stub)
deletes = []
for stub in self.old_stubs:
if not stub in self.new_stubs:
- D("deleted file: " + stub)
+ logging.info("deleted file: " + stub)
deletes.append(os.path.join(bionic_libc_root, stub))
if not DRY_RUN:
@@ -539,11 +657,11 @@
commands.getoutput("git add %s" % (os.path.join(bionic_libc_root, "SYSCALLS.TXT")))
if (not adds) and (not deletes) and (not edits):
- D("no changes detected!")
+ logging.info("no changes detected!")
else:
- D("ready to go!!")
+ logging.info("ready to go!!")
-D_setlevel(1)
+logging.basicConfig(level=logging.INFO)
state = State()
state.process_file(os.path.join(bionic_libc_root, "SYSCALLS.TXT"))