@@ -25,8 +25,7 @@ except RuntimeError as exc:
from bb import cookerdata
from bb.main import bitbake_main, BitBakeConfigParameters, BBMainException
-if sys.getfilesystemencoding() != "utf-8":
- sys.exit("Please use a locale setting which supports UTF-8 (such as LANG=en_US.UTF-8).\nPython can't change the filesystem locale after loading so we need a UTF-8 when Python starts or things won't work.")
+bb.utils.check_system_locale()
__version__ = "2.0.0"
@@ -11,6 +11,7 @@
import os
import sys
import warnings
+
warnings.simplefilter("default")
import argparse
import logging
@@ -27,6 +28,7 @@ logger = bb.msg.logger_create(myname)
is_dump = myname == 'bitbake-dumpsig'
+
def find_siginfo(tinfoil, pn, taskname, sigs=None):
result = None
tinfoil.set_event_mask(['bb.event.FindSigInfoResult',
@@ -52,6 +54,7 @@ def find_siginfo(tinfoil, pn, taskname, sigs=None):
sys.exit(2)
return result
+
def find_siginfo_task(bbhandler, pn, taskname, sig1=None, sig2=None):
""" Find the most recent signature files for the specified PN/task """
@@ -63,10 +66,10 @@ def find_siginfo_task(bbhandler, pn, taskname, sig1=None, sig2=None):
if not sigfiles:
logger.error('No sigdata files found matching %s %s matching either %s or %s' % (pn, taskname, sig1, sig2))
sys.exit(1)
- elif not sig1 in sigfiles:
+ elif sig1 not in sigfiles:
logger.error('No sigdata files found matching %s %s with signature %s' % (pn, taskname, sig1))
sys.exit(1)
- elif not sig2 in sigfiles:
+ elif sig2 not in sigfiles:
logger.error('No sigdata files found matching %s %s with signature %s' % (pn, taskname, sig2))
sys.exit(1)
latestfiles = [sigfiles[sig1], sigfiles[sig2]]
@@ -88,9 +91,9 @@ def recursecb(key, hash1, hash2):
recout = []
if not hashfiles:
recout.append("Unable to find matching sigdata for %s with hashes %s or %s" % (key, hash1, hash2))
- elif not hash1 in hashfiles:
+ elif hash1 not in hashfiles:
recout.append("Unable to find matching sigdata for %s with hash %s" % (key, hash1))
- elif not hash2 in hashfiles:
+ elif hash2 not in hashfiles:
recout.append("Unable to find matching sigdata for %s with hash %s" % (key, hash2))
else:
out2 = bb.siggen.compare_sigfiles(hashfiles[hash1], hashfiles[hash2], recursecb, color=color)
@@ -110,36 +113,36 @@ parser.add_argument('-D', '--debug',
if is_dump:
parser.add_argument("-t", "--task",
- help="find the signature data file for the last run of the specified task",
- action="store", dest="taskargs", nargs=2, metavar=('recipename', 'taskname'))
+ help="find the signature data file for the last run of the specified task",
+ action="store", dest="taskargs", nargs=2, metavar=('recipename', 'taskname'))
parser.add_argument("sigdatafile1",
- help="Signature file to dump. Not used when using -t/--task.",
- action="store", nargs='?', metavar="sigdatafile")
+ help="Signature file to dump. Not used when using -t/--task.",
+ action="store", nargs='?', metavar="sigdatafile")
else:
parser.add_argument('-c', '--color',
- help='Colorize the output (where %(metavar)s is %(choices)s)',
- choices=['auto', 'always', 'never'], default='auto', metavar='color')
+ help='Colorize the output (where %(metavar)s is %(choices)s)',
+ choices=['auto', 'always', 'never'], default='auto', metavar='color')
parser.add_argument('-d', '--dump',
- help='Dump the last signature data instead of comparing (equivalent to using bitbake-dumpsig)',
- action='store_true')
+ help='Dump the last signature data instead of comparing (equivalent to using bitbake-dumpsig)',
+ action='store_true')
parser.add_argument("-t", "--task",
- help="find the signature data files for the last two runs of the specified task and compare them",
- action="store", dest="taskargs", nargs=2, metavar=('recipename', 'taskname'))
+ help="find the signature data files for the last two runs of the specified task and compare them",
+ action="store", dest="taskargs", nargs=2, metavar=('recipename', 'taskname'))
parser.add_argument("-s", "--signature",
- help="With -t/--task, specify the signatures to look for instead of taking the last two",
- action="store", dest="sigargs", nargs=2, metavar=('fromsig', 'tosig'))
+ help="With -t/--task, specify the signatures to look for instead of taking the last two",
+ action="store", dest="sigargs", nargs=2, metavar=('fromsig', 'tosig'))
parser.add_argument("sigdatafile1",
- help="First signature file to compare (or signature file to dump, if second not specified). Not used when using -t/--task.",
- action="store", nargs='?')
+ help="First signature file to compare (or signature file to dump, if second not specified). Not used when using -t/--task.",
+ action="store", nargs='?')
parser.add_argument("sigdatafile2",
- help="Second signature file to compare",
- action="store", nargs='?')
+ help="Second signature file to compare",
+ action="store", nargs='?')
options = parser.parse_args()
if is_dump:
@@ -157,7 +160,8 @@ if options.taskargs:
with bb.tinfoil.Tinfoil() as tinfoil:
tinfoil.prepare(config_only=True)
if not options.dump and options.sigargs:
- files = find_siginfo_task(tinfoil, options.taskargs[0], options.taskargs[1], options.sigargs[0], options.sigargs[1])
+ files = find_siginfo_task(tinfoil, options.taskargs[0], options.taskargs[1], options.sigargs[0],
+ options.sigargs[1])
else:
files = find_siginfo_task(tinfoil, options.taskargs[0], options.taskargs[1])
@@ -166,7 +170,8 @@ if options.taskargs:
output = bb.siggen.dump_sigfile(files[-1])
else:
if len(files) < 2:
- logger.error('Only one matching sigdata file found for the specified task (%s %s)' % (options.taskargs[0], options.taskargs[1]))
+ logger.error('Only one matching sigdata file found for the specified task (%s %s)' % (
+ options.taskargs[0], options.taskargs[1]))
sys.exit(1)
# Recurse into signature comparison
@@ -68,11 +68,11 @@ def main():
registered = False
for plugin in plugins:
+ if hasattr(plugin, 'tinfoil_init'):
+ plugin.tinfoil_init(tinfoil)
if hasattr(plugin, 'register_commands'):
registered = True
plugin.register_commands(subparsers)
- if hasattr(plugin, 'tinfoil_init'):
- plugin.tinfoil_init(tinfoil)
if not registered:
logger.error("No commands registered - missing plugins?")
@@ -12,8 +12,9 @@ warnings.simplefilter("default")
import logging
sys.path.insert(0, os.path.join(os.path.dirname(os.path.dirname(sys.argv[0])), 'lib'))
-if sys.getfilesystemencoding() != "utf-8":
- sys.exit("Please use a locale setting which supports UTF-8 (such as LANG=en_US.UTF-8).\nPython can't change the filesystem locale after loading so we need a UTF-8 when Python starts or things won't work.")
+import bb
+
+bb.utils.check_system_locale()
# Users shouldn't be running this code directly
if len(sys.argv) != 10 or not sys.argv[1].startswith("decafbad"):
@@ -24,8 +24,7 @@ import subprocess
from multiprocessing import Lock
from threading import Thread
-if sys.getfilesystemencoding() != "utf-8":
- sys.exit("Please use a locale setting which supports UTF-8 (such as LANG=en_US.UTF-8).\nPython can't change the filesystem locale after loading so we need a UTF-8 when Python starts or things won't work.")
+bb.utils.check_system_locale()
# Users shouldn't be running this code directly
if len(sys.argv) != 2 or not sys.argv[1].startswith("decafbad"):
@@ -424,8 +424,8 @@ This fetcher supports the following parameters:
- *"nobranch":* Tells the fetcher to not check the SHA validation for
the branch when set to "1". The default is "0". Set this option for
- the recipe that refers to the commit that is valid for a tag instead
- of the branch.
+ the recipe that refers to the commit that is valid for any namespace
+ (branch, tag, ...) instead of the branch.
- *"bareclone":* Tells the fetcher to clone a bare clone into the
destination directory without checking out a working tree. Only the
@@ -160,12 +160,7 @@ def catch_parse_error(func):
def wrapped(fn, *args):
try:
return func(fn, *args)
- except IOError as exc:
- import traceback
- parselog.critical(traceback.format_exc())
- parselog.critical("Unable to parse %s: %s" % (fn, exc))
- raise bb.BBHandledException()
- except bb.data_smart.ExpansionError as exc:
+ except Exception as exc:
import traceback
bbdir = os.path.dirname(__file__) + os.sep
@@ -177,9 +172,6 @@ def catch_parse_error(func):
break
parselog.critical("Unable to parse %s" % fn, exc_info=(exc_class, exc, tb))
raise bb.BBHandledException()
- except bb.parse.ParseError as exc:
- parselog.critical(str(exc))
- raise bb.BBHandledException()
return wrapped
@catch_parse_error
@@ -301,14 +293,9 @@ class CookerDataBuilder(object):
bb.event.fire(bb.event.MultiConfigParsed(self.mcdata), self.data)
self.data_hash = data_hash.hexdigest()
- except (SyntaxError, bb.BBHandledException):
- raise bb.BBHandledException()
except bb.data_smart.ExpansionError as e:
logger.error(str(e))
raise bb.BBHandledException()
- except Exception:
- logger.exception("Error parsing configuration files")
- raise bb.BBHandledException()
# Handle obsolete variable names
@@ -435,7 +422,7 @@ class CookerDataBuilder(object):
msg += (" and bitbake did not find a conf/bblayers.conf file in"
" the expected location.\nMaybe you accidentally"
" invoked bitbake from the wrong directory?")
- raise SystemExit(msg)
+ bb.fatal(msg)
if not data.getVar("TOPDIR"):
data.setVar("TOPDIR", os.path.abspath(os.getcwd()))
@@ -44,7 +44,8 @@ Supported SRC_URI options are:
- nobranch
Don't check the SHA validation for branch. set this option for the recipe
- referring to commit which is valid in tag instead of branch.
+ referring to commit which is valid in any namespace (branch, tag, ...)
+ instead of branch.
The default is "0", set nobranch=1 if needed.
- usehead
@@ -358,9 +359,13 @@ class Git(FetchMethod):
# If the repo still doesn't exist, fallback to cloning it
if not os.path.exists(ud.clonedir):
- # We do this since git will use a "-l" option automatically for local urls where possible
+ # We do this since git will use a "-l" option automatically for local urls where possible,
+ # but it doesn't work when git/objects is a symlink, only works when it is a directory.
if repourl.startswith("file://"):
- repourl = repourl[7:]
+ repourl_path = repourl[7:]
+ objects = os.path.join(repourl_path, 'objects')
+ if os.path.isdir(objects) and not os.path.islink(objects):
+ repourl = repourl_path
clone_cmd = "LANG=C %s clone --bare --mirror %s %s --progress" % (ud.basecmd, shlex.quote(repourl), ud.clonedir)
if ud.proto.lower() != 'file':
bb.fetch2.check_network_access(d, clone_cmd, ud.url)
@@ -374,7 +379,11 @@ class Git(FetchMethod):
runfetchcmd("%s remote rm origin" % ud.basecmd, d, workdir=ud.clonedir)
runfetchcmd("%s remote add --mirror=fetch origin %s" % (ud.basecmd, shlex.quote(repourl)), d, workdir=ud.clonedir)
- fetch_cmd = "LANG=C %s fetch -f --progress %s refs/*:refs/*" % (ud.basecmd, shlex.quote(repourl))
+
+ if ud.nobranch:
+ fetch_cmd = "LANG=C %s fetch -f --progress %s refs/*:refs/*" % (ud.basecmd, shlex.quote(repourl))
+ else:
+ fetch_cmd = "LANG=C %s fetch -f --progress %s refs/heads/*:refs/heads/* refs/tags/*:refs/tags/*" % (ud.basecmd, shlex.quote(repourl))
if ud.proto.lower() != 'file':
bb.fetch2.check_network_access(d, fetch_cmd, ud.url)
progresshandler = GitProgressHandler(d)
@@ -731,7 +740,7 @@ class Git(FetchMethod):
Compute the HEAD revision for the url
"""
if not d.getVar("__BBSEENSRCREV"):
- raise bb.fetch2.FetchError("Recipe uses a floating tag/branch without a fixed SRCREV yet doesn't call bb.fetch2.get_srcrev() (use SRCPV in PV for OE).")
+ raise bb.fetch2.FetchError("Recipe uses a floating tag/branch '%s' for repo '%s' without a fixed SRCREV yet doesn't call bb.fetch2.get_srcrev() (use SRCPV in PV for OE)." % (ud.unresolvedrev[name], ud.host+ud.path))
# Ensure we mark as not cached
bb.fetch2.get_autorev(d)
@@ -115,7 +115,7 @@ class GitSM(Git):
# This has to be a file reference
proto = "file"
url = "gitsm://" + uris[module]
- if "{}{}".format(ud.host, ud.path) in url:
+ if url.endswith("{}{}".format(ud.host, ud.path)):
raise bb.fetch2.FetchError("Submodule refers to the parent repository. This will cause deadlock situation in current version of Bitbake." \
"Consider using git fetcher instead.")
@@ -1974,11 +1974,19 @@ class RunQueueExecute:
self.setbuildable(revdep)
logger.debug("Marking task %s as buildable", revdep)
- for t in self.sq_deferred.copy():
+ found = None
+ for t in sorted(self.sq_deferred.copy()):
if self.sq_deferred[t] == task:
- logger.debug2("Deferred task %s now buildable" % t)
- del self.sq_deferred[t]
- update_scenequeue_data([t], self.sqdata, self.rqdata, self.rq, self.cooker, self.stampcache, self, summary=False)
+ # Allow the next deferred task to run. Any other deferred tasks should be deferred after that task.
+ # We shouldn't allow all to run at once as it is prone to races.
+ if not found:
+ bb.note("Deferred task %s now buildable" % t)
+ del self.sq_deferred[t]
+ update_scenequeue_data([t], self.sqdata, self.rqdata, self.rq, self.cooker, self.stampcache, self, summary=False)
+ found = t
+ else:
+ bb.note("Deferring %s after %s" % (t, found))
+ self.sq_deferred[t] = found
def task_complete(self, task):
self.stats.taskCompleted()
@@ -2489,17 +2497,6 @@ class RunQueueExecute:
self.sq_buildable.remove(tid)
if tid in self.sq_running:
self.sq_running.remove(tid)
- harddepfail = False
- for t in self.sqdata.sq_harddeps:
- if tid in self.sqdata.sq_harddeps[t] and t in self.scenequeue_notcovered:
- harddepfail = True
- break
- if not harddepfail and self.sqdata.sq_revdeps[tid].issubset(self.scenequeue_covered | self.scenequeue_notcovered):
- if tid not in self.sq_buildable:
- self.sq_buildable.add(tid)
- if not self.sqdata.sq_revdeps[tid]:
- self.sq_buildable.add(tid)
-
if tid in self.sqdata.outrightfail:
self.sqdata.outrightfail.remove(tid)
if tid in self.scenequeue_notcovered:
@@ -2518,21 +2515,36 @@ class RunQueueExecute:
if tid in self.build_stamps:
del self.build_stamps[tid]
- update_tasks.append((tid, harddepfail, tid in self.sqdata.valid))
+ update_tasks.append(tid)
+
+ update_tasks2 = []
+ for tid in update_tasks:
+ harddepfail = False
+ for t in self.sqdata.sq_harddeps:
+ if tid in self.sqdata.sq_harddeps[t] and t in self.scenequeue_notcovered:
+ harddepfail = True
+ break
+ if not harddepfail and self.sqdata.sq_revdeps[tid].issubset(self.scenequeue_covered | self.scenequeue_notcovered):
+ if tid not in self.sq_buildable:
+ self.sq_buildable.add(tid)
+ if not self.sqdata.sq_revdeps[tid]:
+ self.sq_buildable.add(tid)
+
+ update_tasks2.append((tid, harddepfail, tid in self.sqdata.valid))
- if update_tasks:
+ if update_tasks2:
self.sqdone = False
for mc in sorted(self.sqdata.multiconfigs):
- for tid in sorted([t[0] for t in update_tasks]):
+ for tid in sorted([t[0] for t in update_tasks2]):
if mc_from_tid(tid) != mc:
continue
h = pending_hash_index(tid, self.rqdata)
if h in self.sqdata.hashes and tid != self.sqdata.hashes[h]:
self.sq_deferred[tid] = self.sqdata.hashes[h]
bb.note("Deferring %s after %s" % (tid, self.sqdata.hashes[h]))
- update_scenequeue_data([t[0] for t in update_tasks], self.sqdata, self.rqdata, self.rq, self.cooker, self.stampcache, self, summary=False)
+ update_scenequeue_data([t[0] for t in update_tasks2], self.sqdata, self.rqdata, self.rq, self.cooker, self.stampcache, self, summary=False)
- for (tid, harddepfail, origvalid) in update_tasks:
+ for (tid, harddepfail, origvalid) in update_tasks2:
if tid in self.sqdata.valid and not origvalid:
hashequiv_logger.verbose("Setscene task %s became valid" % tid)
if harddepfail:
@@ -329,19 +329,19 @@ class SignatureGeneratorBasic(SignatureGenerator):
data = self.basehash[tid]
for dep in self.runtaskdeps[tid]:
- data = data + self.get_unihash(dep)
+ data += self.get_unihash(dep)
for (f, cs) in self.file_checksum_values[tid]:
if cs:
if "/./" in f:
- data = data + "./" + f.split("/./")[1]
- data = data + cs
+ data += "./" + f.split("/./")[1]
+ data += cs
if tid in self.taints:
if self.taints[tid].startswith("nostamp:"):
- data = data + self.taints[tid][8:]
+ data += self.taints[tid][8:]
else:
- data = data + self.taints[tid]
+ data += self.taints[tid]
h = hashlib.sha256(data.encode("utf-8")).hexdigest()
self.taskhash[tid] = h
@@ -1028,6 +1028,7 @@ def compare_sigfiles(a, b, recursecb=None, color=False, collapsed=False):
# If a dependent hash changed, might as well print the line above and then defer to the changes in
# that hash since in all likelyhood, they're the same changes this task also saw.
output = [output[-1]] + recout
+ break
a_taint = a_data.get('taint', None)
b_taint = b_data.get('taint', None)
@@ -13,6 +13,7 @@ import errno
import logging
import bb
import bb.msg
+import locale
import multiprocessing
import fcntl
import importlib
@@ -545,7 +546,12 @@ def md5_file(filename):
Return the hex string representation of the MD5 checksum of filename.
"""
import hashlib
- return _hasher(hashlib.new('MD5', usedforsecurity=False), filename)
+ try:
+ sig = hashlib.new('MD5', usedforsecurity=False)
+ except TypeError:
+ # Some configurations don't appear to support two arguments
+ sig = hashlib.new('MD5')
+ return _hasher(sig, filename)
def sha256_file(filename):
"""
@@ -601,6 +607,21 @@ def preserved_envvars():
]
return v + preserved_envvars_exported()
+def check_system_locale():
+ """Make sure the required system locale are available and configured"""
+ default_locale = locale.getlocale(locale.LC_CTYPE)
+
+ try:
+ locale.setlocale(locale.LC_CTYPE, ("en_US", "UTF-8"))
+ except:
+ sys.exit("Please make sure locale 'en_US.UTF-8' is available on your system")
+ else:
+ locale.setlocale(locale.LC_CTYPE, default_locale)
+
+ if sys.getfilesystemencoding() != "utf-8":
+ sys.exit("Please use a locale setting which supports UTF-8 (such as LANG=en_US.UTF-8).\n"
+ "Python can't change the filesystem locale after loading so we need a UTF-8 when Python starts or things won't work.")
+
def filter_environment(good_vars):
"""
Create a pristine environment for bitbake. This will remove variables that
@@ -985,6 +1006,9 @@ def to_boolean(string, default=None):
if not string:
return default
+ if isinstance(string, int):
+ return string != 0
+
normalized = string.lower()
if normalized in ("y", "yes", "1", "true"):
return True
@@ -1635,23 +1659,20 @@ def disable_network(uid=None, gid=None):
def export_proxies(d):
""" export common proxies variables from datastore to environment """
- import os
variables = ['http_proxy', 'HTTP_PROXY', 'https_proxy', 'HTTPS_PROXY',
'ftp_proxy', 'FTP_PROXY', 'no_proxy', 'NO_PROXY',
- 'GIT_PROXY_COMMAND']
- exported = False
+ 'GIT_PROXY_COMMAND', 'SSL_CERT_FILE', 'SSL_CERT_DIR']
- for v in variables:
- if v in os.environ.keys():
- exported = True
- else:
- v_proxy = d.getVar(v)
- if v_proxy is not None:
- os.environ[v] = v_proxy
- exported = True
+ origenv = d.getVar("BB_ORIGENV")
+
+ for name in variables:
+ value = d.getVar(name)
+ if not value and origenv:
+ value = origenv.getVar(name)
+ if value:
+ os.environ[name] = value
- return exported
def load_plugins(logger, plugins, pluginpath):
@@ -2798,7 +2798,14 @@ class ParserReflect(object):
def signature(self):
try:
import hashlib
+ except ImportError:
+ raise RuntimeError("Unable to import hashlib")
+ try:
sig = hashlib.new('MD5', usedforsecurity=False)
+ except TypeError:
+ # Some configurations don't appear to support two arguments
+ sig = hashlib.new('MD5')
+ try:
if self.start:
sig.update(self.start.encode('latin-1'))
if self.prec:
Upstream commit 41b6684489d0261753344956042be2cc4adb0159. Signed-off-by: Anton Mikanovich <amikan@ilbers.de> --- bitbake/bin/bitbake | 3 +- bitbake/bin/bitbake-diffsigs | 49 +++++++++-------- bitbake/bin/bitbake-layers | 4 +- bitbake/bin/bitbake-server | 5 +- bitbake/bin/bitbake-worker | 3 +- .../bitbake-user-manual-fetching.rst | 4 +- bitbake/lib/bb/cookerdata.py | 17 +----- bitbake/lib/bb/fetch2/git.py | 19 +++++-- bitbake/lib/bb/fetch2/gitsm.py | 2 +- bitbake/lib/bb/runqueue.py | 52 ++++++++++++------- bitbake/lib/bb/siggen.py | 11 ++-- bitbake/lib/bb/utils.py | 47 ++++++++++++----- bitbake/lib/ply/yacc.py | 7 +++ 13 files changed, 132 insertions(+), 91 deletions(-)