mirror of
https://git.rtems.org/rtems-source-builder
synced 2024-10-09 07:15:10 +08:00
Add options support for --with/--without processing. Add --with-download support.
You can now test downloading by using --dry-run and --with-download. Print the redirect if one is taken.
This commit is contained in:
@@ -36,6 +36,16 @@ import log
|
|||||||
import path
|
import path
|
||||||
import sources
|
import sources
|
||||||
|
|
||||||
|
def _do_download(opts):
|
||||||
|
download = True
|
||||||
|
if opts.dry_run():
|
||||||
|
download = False
|
||||||
|
wa = opts.with_arg('download')
|
||||||
|
if wa is not None:
|
||||||
|
if wa[0] == 'with_download' and wa[1] == 'yes':
|
||||||
|
download = True
|
||||||
|
return download
|
||||||
|
|
||||||
def _humanize_bytes(bytes, precision = 1):
|
def _humanize_bytes(bytes, precision = 1):
|
||||||
abbrevs = (
|
abbrevs = (
|
||||||
(1 << 50L, 'PB'),
|
(1 << 50L, 'PB'),
|
||||||
@@ -102,7 +112,56 @@ def _hash_check(file_, absfile, macros, remove = True):
|
|||||||
log.warning('%s: no hash found' % (file_))
|
log.warning('%s: no hash found' % (file_))
|
||||||
return not failed
|
return not failed
|
||||||
|
|
||||||
def _http_parser(source, config, opts):
|
def _local_path(source, pathkey, config):
|
||||||
|
for p in config.define(pathkey).split(':'):
|
||||||
|
local = path.join(path.abspath(p), source['file'])
|
||||||
|
if source['local'] is None:
|
||||||
|
source['local_prefix'] = path.abspath(p)
|
||||||
|
source['local'] = local
|
||||||
|
if path.exists(local):
|
||||||
|
source['local_prefix'] = path.abspath(p)
|
||||||
|
source['local'] = local
|
||||||
|
_hash_check(source['file'], local, config.macros)
|
||||||
|
break
|
||||||
|
|
||||||
|
def _http_parser(source, pathkey, config, opts):
|
||||||
|
#
|
||||||
|
# Hack for gitweb.cgi patch downloads. We rewrite the various fields.
|
||||||
|
#
|
||||||
|
if 'gitweb.cgi' in source['url']:
|
||||||
|
url = source['url']
|
||||||
|
if '?' not in url:
|
||||||
|
raise error.general('invalid gitweb.cgi request: %s' % (url))
|
||||||
|
req = url.split('?')[1]
|
||||||
|
if len(req) == 0:
|
||||||
|
raise error.general('invalid gitweb.cgi request: %s' % (url))
|
||||||
|
#
|
||||||
|
# The gitweb.cgi request should have:
|
||||||
|
# p=<what>
|
||||||
|
# a=patch
|
||||||
|
# h=<hash>
|
||||||
|
# so extract the p and h parts to make the local name.
|
||||||
|
#
|
||||||
|
p = None
|
||||||
|
a = None
|
||||||
|
h = None
|
||||||
|
for r in req.split(';'):
|
||||||
|
if '=' not in r:
|
||||||
|
raise error.general('invalid gitweb.cgi path: %s' % (url))
|
||||||
|
rs = r.split('=')
|
||||||
|
if rs[0] == 'p':
|
||||||
|
p = rs[1].replace('.', '-')
|
||||||
|
elif rs[0] == 'a':
|
||||||
|
a = rs[1]
|
||||||
|
elif rs[0] == 'h':
|
||||||
|
h = rs[1]
|
||||||
|
if p is None or h is None:
|
||||||
|
raise error.general('gitweb.cgi path missing p or h: %s' % (url))
|
||||||
|
source['file'] = '%s-%s.patch' % (p, h)
|
||||||
|
#
|
||||||
|
# Check local path
|
||||||
|
#
|
||||||
|
_local_path(source, pathkey, config)
|
||||||
#
|
#
|
||||||
# Is the file compressed ?
|
# Is the file compressed ?
|
||||||
#
|
#
|
||||||
@@ -120,10 +179,18 @@ def _http_parser(source, config, opts):
|
|||||||
source['compressed-type'] = 'xz'
|
source['compressed-type'] = 'xz'
|
||||||
source['compressed'] = '%{__xz} -dc'
|
source['compressed'] = '%{__xz} -dc'
|
||||||
|
|
||||||
def _patchworks_parser(source, config, opts):
|
def _patchworks_parser(source, pathkey, config, opts):
|
||||||
|
#
|
||||||
|
# Check local path
|
||||||
|
#
|
||||||
|
_local_path(source, pathkey, config)
|
||||||
source['url'] = 'http%s' % (source['path'][2:])
|
source['url'] = 'http%s' % (source['path'][2:])
|
||||||
|
|
||||||
def _git_parser(source, config, opts):
|
def _git_parser(source, pathkey, config, opts):
|
||||||
|
#
|
||||||
|
# Check local path
|
||||||
|
#
|
||||||
|
_local_path(source, pathkey, config)
|
||||||
#
|
#
|
||||||
# Symlink.
|
# Symlink.
|
||||||
#
|
#
|
||||||
@@ -137,7 +204,11 @@ def _git_parser(source, config, opts):
|
|||||||
path.join(source['local_prefix'], 'git', source['file'])
|
path.join(source['local_prefix'], 'git', source['file'])
|
||||||
source['symlink'] = source['local']
|
source['symlink'] = source['local']
|
||||||
|
|
||||||
def _cvs_parser(source, config, opts):
|
def _cvs_parser(source, pathkey, config, opts):
|
||||||
|
#
|
||||||
|
# Check local path
|
||||||
|
#
|
||||||
|
_local_path(source, pathkey, config)
|
||||||
#
|
#
|
||||||
# Symlink.
|
# Symlink.
|
||||||
#
|
#
|
||||||
@@ -185,7 +256,11 @@ def _cvs_parser(source, config, opts):
|
|||||||
else:
|
else:
|
||||||
source['symlink'] = source['local']
|
source['symlink'] = source['local']
|
||||||
|
|
||||||
def _file_parser(source, config, opts):
|
def _file_parser(source, pathkey, config, opts):
|
||||||
|
#
|
||||||
|
# Check local path
|
||||||
|
#
|
||||||
|
_local_path(source, pathkey, config)
|
||||||
#
|
#
|
||||||
# Symlink.
|
# Symlink.
|
||||||
#
|
#
|
||||||
@@ -217,22 +292,12 @@ def parse_url(url, pathkey, config, opts):
|
|||||||
# Get the file. Checks the local source directory first.
|
# Get the file. Checks the local source directory first.
|
||||||
#
|
#
|
||||||
source['local'] = None
|
source['local'] = None
|
||||||
for p in config.define(pathkey).split(':'):
|
|
||||||
local = path.join(path.abspath(p), source['file'])
|
|
||||||
if source['local'] is None:
|
|
||||||
source['local_prefix'] = path.abspath(p)
|
|
||||||
source['local'] = local
|
|
||||||
if path.exists(local):
|
|
||||||
source['local_prefix'] = path.abspath(p)
|
|
||||||
source['local'] = local
|
|
||||||
_hash_check(source['file'], local, config.macros)
|
|
||||||
break
|
|
||||||
source['script'] = ''
|
|
||||||
for p in parsers:
|
for p in parsers:
|
||||||
if url.startswith(p):
|
if url.startswith(p):
|
||||||
source['type'] = p
|
source['type'] = p
|
||||||
if parsers[p](source, config, opts):
|
if parsers[p](source, pathkey, config, opts):
|
||||||
break
|
break
|
||||||
|
source['script'] = ''
|
||||||
return source
|
return source
|
||||||
|
|
||||||
def _http_downloader(url, local, config, opts):
|
def _http_downloader(url, local, config, opts):
|
||||||
@@ -246,7 +311,7 @@ def _http_downloader(url, local, config, opts):
|
|||||||
dst = os.path.relpath(path.host(local))
|
dst = os.path.relpath(path.host(local))
|
||||||
log.notice('download: %s -> %s' % (url, dst))
|
log.notice('download: %s -> %s' % (url, dst))
|
||||||
failed = False
|
failed = False
|
||||||
if not opts.dry_run():
|
if _do_download(opts):
|
||||||
_in = None
|
_in = None
|
||||||
_out = None
|
_out = None
|
||||||
_length = None
|
_length = None
|
||||||
@@ -258,7 +323,10 @@ def _http_downloader(url, local, config, opts):
|
|||||||
_wipe_output = False
|
_wipe_output = False
|
||||||
try:
|
try:
|
||||||
try:
|
try:
|
||||||
|
_in = None
|
||||||
_in = urllib2.urlopen(url)
|
_in = urllib2.urlopen(url)
|
||||||
|
if url != _in.geturl():
|
||||||
|
log.notice(' redirect: %s' % (_in.geturl()))
|
||||||
_out = open(path.host(local), 'wb')
|
_out = open(path.host(local), 'wb')
|
||||||
try:
|
try:
|
||||||
_length = int(_in.info().getheader('Content-Length').strip())
|
_length = int(_in.info().getheader('Content-Length').strip())
|
||||||
@@ -300,12 +368,15 @@ def _http_downloader(url, local, config, opts):
|
|||||||
msg = 'download: %s: error' % (url)
|
msg = 'download: %s: error' % (url)
|
||||||
log.stderr(msg)
|
log.stderr(msg)
|
||||||
log.notice(msg)
|
log.notice(msg)
|
||||||
|
if _in is not None:
|
||||||
|
_in.close()
|
||||||
if _out is not None:
|
if _out is not None:
|
||||||
_out.close()
|
_out.close()
|
||||||
raise
|
raise
|
||||||
if _out is not None:
|
if _out is not None:
|
||||||
_out.close()
|
_out.close()
|
||||||
if _in is not None:
|
if _in is not None:
|
||||||
|
_in.close()
|
||||||
del _in
|
del _in
|
||||||
if not failed:
|
if not failed:
|
||||||
if not path.isfile(local):
|
if not path.isfile(local):
|
||||||
@@ -338,7 +409,7 @@ def _git_downloader(url, local, config, opts):
|
|||||||
us[0] = _as[1] + url_base
|
us[0] = _as[1] + url_base
|
||||||
if not repo.valid():
|
if not repo.valid():
|
||||||
log.notice('git: clone: %s -> %s' % (us[0], rlp))
|
log.notice('git: clone: %s -> %s' % (us[0], rlp))
|
||||||
if not opts.dry_run():
|
if _do_download(opts):
|
||||||
repo.clone(us[0], local)
|
repo.clone(us[0], local)
|
||||||
else:
|
else:
|
||||||
repo.clean(['-f', '-d'])
|
repo.clean(['-f', '-d'])
|
||||||
@@ -350,28 +421,28 @@ def _git_downloader(url, local, config, opts):
|
|||||||
if len(_as) != 2:
|
if len(_as) != 2:
|
||||||
raise error.general('invalid git branch/checkout: %s' % (_as))
|
raise error.general('invalid git branch/checkout: %s' % (_as))
|
||||||
log.notice('git: checkout: %s => %s' % (us[0], _as[1]))
|
log.notice('git: checkout: %s => %s' % (us[0], _as[1]))
|
||||||
if not opts.dry_run():
|
if _do_download(opts):
|
||||||
repo.checkout(_as[1])
|
repo.checkout(_as[1])
|
||||||
elif _as[0] == 'pull':
|
elif _as[0] == 'pull':
|
||||||
log.notice('git: pull: %s' % (us[0]))
|
log.notice('git: pull: %s' % (us[0]))
|
||||||
if not opts.dry_run():
|
if _do_download(opts):
|
||||||
repo.pull()
|
repo.pull()
|
||||||
elif _as[0] == 'submodule':
|
elif _as[0] == 'submodule':
|
||||||
if len(_as) != 2:
|
if len(_as) != 2:
|
||||||
raise error.general('invalid git submodule: %s' % (_as))
|
raise error.general('invalid git submodule: %s' % (_as))
|
||||||
log.notice('git: submodule: %s <= %s' % (us[0], _as[1]))
|
log.notice('git: submodule: %s <= %s' % (us[0], _as[1]))
|
||||||
if not opts.dry_run():
|
if _do_download(opts):
|
||||||
repo.submodule(_as[1])
|
repo.submodule(_as[1])
|
||||||
elif _as[0] == 'fetch':
|
elif _as[0] == 'fetch':
|
||||||
log.notice('git: fetch: %s -> %s' % (us[0], rlp))
|
log.notice('git: fetch: %s -> %s' % (us[0], rlp))
|
||||||
if not opts.dry_run():
|
if _do_download(opts):
|
||||||
repo.fetch()
|
repo.fetch()
|
||||||
elif _as[0] == 'reset':
|
elif _as[0] == 'reset':
|
||||||
arg = []
|
arg = []
|
||||||
if len(_as) > 1:
|
if len(_as) > 1:
|
||||||
arg = ['--%s' % (_as[1])]
|
arg = ['--%s' % (_as[1])]
|
||||||
log.notice('git: reset: %s' % (us[0]))
|
log.notice('git: reset: %s' % (us[0]))
|
||||||
if not opts.dry_run():
|
if _do_download(opts):
|
||||||
repo.reset(arg)
|
repo.reset(arg)
|
||||||
elif _as[0] == 'protocol':
|
elif _as[0] == 'protocol':
|
||||||
pass
|
pass
|
||||||
@@ -409,20 +480,20 @@ def _cvs_downloader(url, local, config, opts):
|
|||||||
if not path.isdir(local):
|
if not path.isdir(local):
|
||||||
log.notice('Creating source directory: %s' % \
|
log.notice('Creating source directory: %s' % \
|
||||||
(os.path.relpath(path.host(local))))
|
(os.path.relpath(path.host(local))))
|
||||||
if not opts.dry_run():
|
if _do_download(opts):
|
||||||
path.mkdir(local)
|
path.mkdir(local)
|
||||||
log.notice('cvs: checkout: %s -> %s' % (us[0], rlp))
|
log.notice('cvs: checkout: %s -> %s' % (us[0], rlp))
|
||||||
if not opts.dry_run():
|
if _do_download(opts):
|
||||||
repo.checkout(':%s' % (us[0][6:]), module, tag, date)
|
repo.checkout(':%s' % (us[0][6:]), module, tag, date)
|
||||||
for a in us[1:]:
|
for a in us[1:]:
|
||||||
_as = a.split('=')
|
_as = a.split('=')
|
||||||
if _as[0] == 'update':
|
if _as[0] == 'update':
|
||||||
log.notice('cvs: update: %s' % (us[0]))
|
log.notice('cvs: update: %s' % (us[0]))
|
||||||
if not opts.dry_run():
|
if _do_download(opts):
|
||||||
repo.update()
|
repo.update()
|
||||||
elif _as[0] == 'reset':
|
elif _as[0] == 'reset':
|
||||||
log.notice('cvs: reset: %s' % (us[0]))
|
log.notice('cvs: reset: %s' % (us[0]))
|
||||||
if not opts.dry_run():
|
if _do_download(opts):
|
||||||
repo.reset()
|
repo.reset()
|
||||||
return True
|
return True
|
||||||
|
|
||||||
@@ -445,7 +516,7 @@ def get_file(url, local, opts, config):
|
|||||||
log.notice('Creating source directory: %s' % \
|
log.notice('Creating source directory: %s' % \
|
||||||
(os.path.relpath(path.host(path.dirname(local)))))
|
(os.path.relpath(path.host(path.dirname(local)))))
|
||||||
log.output('making dir: %s' % (path.host(path.dirname(local))))
|
log.output('making dir: %s' % (path.host(path.dirname(local))))
|
||||||
if not opts.dry_run():
|
if _do_download(opts):
|
||||||
path.mkdir(path.dirname(local))
|
path.mkdir(path.dirname(local))
|
||||||
if not path.exists(local) and opts.download_disabled():
|
if not path.exists(local) and opts.download_disabled():
|
||||||
raise error.general('source not found: %s' % (path.host(local)))
|
raise error.general('source not found: %s' % (path.host(local)))
|
||||||
@@ -472,5 +543,5 @@ def get_file(url, local, opts, config):
|
|||||||
if url.startswith(dl):
|
if url.startswith(dl):
|
||||||
if downloaders[dl](url, local, config, opts):
|
if downloaders[dl](url, local, config, opts):
|
||||||
return
|
return
|
||||||
if not opts.dry_run():
|
if _do_download(opts):
|
||||||
raise error.general('downloading %s: all paths have failed, giving up' % (url))
|
raise error.general('downloading %s: all paths have failed, giving up' % (url))
|
||||||
|
@@ -190,7 +190,7 @@ class command_line:
|
|||||||
|
|
||||||
def help(self):
|
def help(self):
|
||||||
print '%s: [options] [args]' % (self.command_name)
|
print '%s: [options] [args]' % (self.command_name)
|
||||||
print 'RTEMS Source Builder, an RTEMS Tools Project (c) 2012-2013 Chris Johns'
|
print 'RTEMS Source Builder, an RTEMS Tools Project (c) 2012-2015 Chris Johns'
|
||||||
print 'Options and arguments:'
|
print 'Options and arguments:'
|
||||||
print '--force : Force the build to proceed'
|
print '--force : Force the build to proceed'
|
||||||
print '--quiet : Quiet output (not used)'
|
print '--quiet : Quiet output (not used)'
|
||||||
@@ -427,14 +427,49 @@ class command_line:
|
|||||||
def params(self):
|
def params(self):
|
||||||
return self.opts['params']
|
return self.opts['params']
|
||||||
|
|
||||||
|
def parse_args(self, arg, error = True):
|
||||||
|
for a in range(0, len(self.args)):
|
||||||
|
if self.args[a].startswith(arg):
|
||||||
|
lhs = None
|
||||||
|
rhs = None
|
||||||
|
if '=' in self.args[a]:
|
||||||
|
eqs = self.args[a].split('=')
|
||||||
|
lhs = eqs[0]
|
||||||
|
if len(eqs) > 2:
|
||||||
|
rhs = '='.join(eqs[1:])
|
||||||
|
else:
|
||||||
|
rhs = eqs[1]
|
||||||
|
else:
|
||||||
|
lhs = self.args[a]
|
||||||
|
a += 1
|
||||||
|
if a >= len(self.args):
|
||||||
|
return [arg, None]
|
||||||
|
rhs = self.args[a]
|
||||||
|
return [lhs, rhs]
|
||||||
|
a += 1
|
||||||
|
return None
|
||||||
|
|
||||||
def get_arg(self, arg):
|
def get_arg(self, arg):
|
||||||
if self.optargs is None or arg not in self.optargs:
|
if self.optargs is None or arg not in self.optargs:
|
||||||
raise error.internal('bad arg: %s' % (arg))
|
raise error.internal('bad arg: %s' % (arg))
|
||||||
for a in self.args:
|
return self.parse_args(arg)
|
||||||
sa = a.split('=')
|
|
||||||
if sa[0].startswith(arg):
|
def with_arg(self, label):
|
||||||
return sa
|
for pre in ['with', 'without']:
|
||||||
return None
|
arg_str = '--%s-%s' % (pre, label)
|
||||||
|
arg_label = '%s_%s' % (pre, label)
|
||||||
|
arg = self.parse_args(arg_str, error = False)
|
||||||
|
if arg is not None:
|
||||||
|
if arg[1] is None:
|
||||||
|
result = 'yes'
|
||||||
|
else:
|
||||||
|
result = arg[1]
|
||||||
|
break
|
||||||
|
if pre == 'with':
|
||||||
|
result = 'yes'
|
||||||
|
else:
|
||||||
|
result = 'no'
|
||||||
|
return [arg_label, result]
|
||||||
|
|
||||||
def get_config_files(self, config):
|
def get_config_files(self, config):
|
||||||
#
|
#
|
||||||
@@ -570,6 +605,8 @@ def run(args):
|
|||||||
log.notice(str(_opts))
|
log.notice(str(_opts))
|
||||||
log.notice('Defaults:')
|
log.notice('Defaults:')
|
||||||
log.notice(str(_opts.defaults))
|
log.notice(str(_opts.defaults))
|
||||||
|
log.notice('with-opt1: %r' % (_opts.with_arg('opt1')))
|
||||||
|
log.notice('without-opt2: %r' % (_opts.with_arg('opt2')))
|
||||||
except error.general, gerr:
|
except error.general, gerr:
|
||||||
print gerr
|
print gerr
|
||||||
sys.exit(1)
|
sys.exit(1)
|
||||||
|
Reference in New Issue
Block a user