From ba574b2358c0bef6b6fb5d7c9c6622f8dfb6256e Mon Sep 17 00:00:00 2001 From: Fabio Berton Date: Tue, 11 Oct 2016 18:15:10 -0300 Subject: [PATCH 01/13] Migrate code to Python 3 Adapt code to run with Python3 and Bitbake version 1.32. Change-Id: I5ce6f81117107dd7d4f0a4e6905db13bd4df48ed Signed-off-by: Fabio Berton --- bb-expand-vars.py | 75 ++++++++++++++++++++++------------------------- ye | 24 +++++++-------- 2 files changed, 46 insertions(+), 53 deletions(-) diff --git a/bb-expand-vars.py b/bb-expand-vars.py index bf51c76..50e35e8 100644 --- a/bb-expand-vars.py +++ b/bb-expand-vars.py @@ -1,4 +1,4 @@ -#! /usr/bin/env python +#! /usr/bin/env python3 # -*- encoding: utf-8 -*- """ From https://github.com/kergoth/bb/blob/master/libexec/bbcmd.py """ @@ -49,7 +49,7 @@ def __init__(self, output=sys.stdout): bb.providers.logger.setLevel(logging.ERROR) bb.taskdata.logger.setLevel(logging.CRITICAL) self.cooker_data = None - self.taskdata = None + self.taskdata = {} self.localdata = bb.data.createCopy(self.config_data) self.localdata.finalize() @@ -58,9 +58,8 @@ def __init__(self, output=sys.stdout): def prepare_taskdata(self, provided=None, rprovided=None): - self.cache_data = self.cooker.recipecache - if self.taskdata is None: - self.taskdata = bb.taskdata.TaskData(abort=False) + self.cache_data = self.cooker.recipecaches[''] + self.taskdata[''] = self.taskdata.get('', bb.taskdata.TaskData(abort=False)) if provided: self.add_provided(provided) @@ -70,9 +69,9 @@ def prepare_taskdata(self, provided=None, rprovided=None): def add_rprovided(self, rprovided): for item in rprovided: - self.taskdata.add_rprovider(self.localdata, self.cache_data, item) + self.taskdata[''].add_rprovider(self.localdata, self.cache_data, item) - self.taskdata.add_unresolved(self.localdata, self.cache_data) + self.taskdata[''].add_unresolved(self.localdata, self.cache_data) def add_provided(self, provided): if 'world' in provided: @@ -86,42 +85,42 @@ def add_provided(self, provided): provided.extend(self.cache_data.universe_target) for item in provided: - self.taskdata.add_provider(self.localdata, self.cache_data, item) + self.taskdata[''].add_provider(self.localdata, self.cache_data, item) - self.taskdata.add_unresolved(self.localdata, self.cache_data) + self.taskdata[''].add_unresolved(self.localdata, self.cache_data) def get_buildid(self, target): - if not self.taskdata.have_build_target(target): - if target in self.cooker.recipecache.ignored_dependencies: + if not self.taskdata[''].have_build_target(target): + if target in self.cooker.recipecaches[''].ignored_dependencies: return - reasons = self.taskdata.get_reasons(target) + reasons = self.taskdata[''].get_reasons(target) if reasons: self.logger.error("No buildable '%s' recipe found:\n%s", target, "\n".join(reasons)) else: self.logger.error("No '%s' recipe found", target) return else: - return self.taskdata.getbuild_id(target) + return self.taskdata[''].getbuild_id(target) def target_filenames(self): """Return the filenames of all of taskdata's targets""" filenames = set() - for targetid in self.taskdata.build_targets: - fnid = self.taskdata.build_targets[targetid][0] - fn = self.taskdata.fn_index[fnid] + for targetid in self.taskdata[''].build_targets: + fnid = self.taskdata[''].build_targets[targetid][0] + fn = self.taskdata[''].fn_index[fnid] filenames.add(fn) - for targetid in self.taskdata.run_targets: - fnid = self.taskdata.run_targets[targetid][0] - fn = self.taskdata.fn_index[fnid] + for targetid in self.taskdata[''].run_targets: + fnid = self.taskdata[''].run_targets[targetid][0] + fn = self.taskdata[''].fn_index[fnid] filenames.add(fn) return filenames def all_filenames(self): - return self.cooker.recipecache.file_checksums.keys() + return self.cooker.recipecaches[''].file_checksums.keys() def all_preferred_filenames(self): """Return all the recipes we have cached, filtered by providers. @@ -130,10 +129,10 @@ def all_preferred_filenames(self): """ filenames = set() excluded = set() - for provide, fns in self.cooker.recipecache.providers.iteritems(): + for provide, fns in self.cooker.recipecaches[''].providers.items(): eligible, foundUnique = bb.providers.filterProviders(fns, provide, self.localdata, - self.cooker.recipecache) + self.cooker.recipecaches['']) preferred = eligible[0] if len(fns) > 1: # Excluding non-preferred providers in multiple-provider @@ -148,27 +147,23 @@ def all_preferred_filenames(self): def provide_to_fn(self, provide): """Return the preferred recipe for the specified provide""" - filenames = self.cooker.recipecache.providers[provide] + filenames = self.cooker.recipecaches.providers[provide] eligible, foundUnique = bb.providers.filterProviders(filenames, provide, self.localdata) return eligible[0] def build_target_to_fn(self, target): """Given a target, prepare taskdata and return a filename""" self.prepare_taskdata([target]) - targetid = self.get_buildid(target) - if targetid is None: - return - fnid = self.taskdata.build_targets[targetid][0] - fn = self.taskdata.fn_index[fnid] + if target in self.taskdata[''].build_targets and self.taskdata[''].build_targets[target]: + fn = self.taskdata[''].build_targets[target][0] return fn def parse_recipe_file(self, recipe_filename): """Given a recipe filename, do a full parse of it""" + bb_cache = bb.cache.NoCache(self.cooker.databuilder) appends = self.cooker.collection.get_file_appends(recipe_filename) try: - recipe_data = bb.cache.Cache.loadDataFull(recipe_filename, - appends, - self.config_data) + recipe_data = bb_cache.loadDataFull(recipe_filename, appends) except Exception: raise return recipe_data @@ -242,19 +237,19 @@ def next(pos, expr): def parse_expr(expr): if expr[0] == '$': - return BBVar(map(parse_expr, tokenize_expr(expr[2:-1]))) + return BBVar([parse_expr(token) for token in tokenize_expr(expr[2:-1])]) else: return expr def parse_exprs(exprs): - return map(parse_expr, exprs) + return [parse_expr(expr) for expr in exprs] def unparse_expr(expr): if isinstance(expr, list): - return ''.join(map(unparse_expr, expr)) + return ''.join(unparse_expr(exp) for exp in expr) elif isinstance(expr, BBVar): if isinstance(expr.name, list): - return ''.join(map(unparse_expr, expr.name)) + return ''.join(unparse_expr(exp) for exp in expr.name) else: return expr.name else: @@ -266,7 +261,7 @@ def show_expansion(var, val, indent): margin = '' else: margin = ' ' * margin_spacing - print('%s%s ==> %s' % (margin, unparse_expr(var), val)) + print(('%s%s ==> %s' % (margin, unparse_expr(var), val))) def get_var_val(var, metadata): val = None @@ -305,7 +300,7 @@ def expand_var(var, metadata, indent): return var def expand_vars(vars, metadata, indent=0): - return ''.join(map(lambda v: expand_var(v, metadata, indent), vars)) + return ''.join([expand_var(v, metadata, indent) for v in vars]) def expand_expr(expr, metadata): return expand_vars(parse_exprs(tokenize_expr(expr)), metadata) @@ -336,10 +331,10 @@ def show_var_expansions(recipe, var, plumbing_mode=False): val = metadata.getVar(var, True) if val is not None: - print '=== Final value' - print '%s = %s' % (var, val) + print('=== Final value') + print( '%s = %s' % (var, val)) - print '\n=== Expansion' + print('\n=== Expansion') expand_expr('${' + var + '}', metadata) else: sys.stderr.write('%s: no such variable.\n' % var) diff --git a/ye b/ye index f9b2674..2d832ba 100755 --- a/ye +++ b/ye @@ -1,4 +1,4 @@ -#! /usr/bin/env python +#! /usr/bin/env python3 ## ye: Yocto Explorer ## @@ -20,14 +20,12 @@ ### TODO # Implement exclusion pattern for grep (option should not clash with git grep's) -from __future__ import print_function - import os import re import sys import time import glob -import urllib +import urllib.request, urllib.parse, urllib.error import locale import subprocess @@ -141,10 +139,10 @@ def download_file(uri, output_file): if dir and not os.path.exists(dir): os.makedirs(dir) try: - uri_fd = urllib.urlopen(uri) + uri_fd = urllib.request.urlopen(uri) except: die('Could not download %s. Aborting' % uri) - with open(output_file, 'w') as output_fd: + with open(output_file, 'wb') as output_fd: output_fd.write(uri_fd.read()) def find_yocto_root(): @@ -214,7 +212,7 @@ def prompt(options, option_formatter=lambda x: x, valid_shortcuts=[]): for i, text in enumerate(options): text_options += '[%d] %s [%d]\n' % (i, option_formatter(text), i) paginate(text_options) - print('Option (ENTER to cancel): ', end='') + print('Option (ENTER to cancel): ') return input() while True: @@ -229,7 +227,7 @@ def prompt_action(options): def inner_prompt(): for short, text in options: print('[%s] %s' % (short, text)) - print('Option (ENTER to cancel): ', end='') + print('Option (ENTER to cancel): ') return input() short_options = [ o[0] for o in options ] @@ -340,10 +338,10 @@ def print_matches(matches, plumbing_cmd=None): dest = sys.stdout if matches.__class__.__name__ == 'list': for match in matches: - dest.write(match.encode(encoding) + '\n') + dest.write(match.encode(encoding).decode() + '\n') else: # a string - dest.write(matches.encode(encoding) + '\n') + dest.write(matches.encode(encoding).decode() + '\n') if plumbing_cmd: dest.close() @@ -560,7 +558,7 @@ def git_log_grep(pattern, max_commits, case_sensitive): gitlog.wait() out = grep.communicate()[0] if out: - results[repo_path] = out.strip().split('\n') + results[repo_path] = out.strip().split(b'\n') sources_dir = os.path.realpath(os.path.join(root_dir, 'sources')) options = [] @@ -707,7 +705,7 @@ def doc(pattern, exact): text.append(parse_paragraph(p)) docs[glossterm] = '\n\n'.join(text) if docs: - options = docs.keys() + options = list(docs.keys()) if len(options) == 1: show_doc(options[0], docs[options[0]]) elif sys.stdout.isatty(): @@ -730,7 +728,7 @@ def write_plumbing_selection(cmd, val): os.makedirs(plumbing_selection_dir) fd = open(cmd_file, 'w') if isinstance(val, dict): - for var, value in val.items(): + for var, value in list(val.items()): fd.write('%s=%s\n' % (var, value)) else: print(val) From 4e644de0692c58a71d310e01e8ed87e0db2ca034 Mon Sep 17 00:00:00 2001 From: Fabio Berton Date: Fri, 24 Feb 2017 17:10:20 -0300 Subject: [PATCH 02/13] bb-expand-vars: Migrate to use Tinfoil2 API bb-expand-vars uses internal tinfoil api and this api was changed in bitbake commit 3bbf8d611c859f74d563778115677a04f5c4ab43. Using tinfoil2 api, like other tools e.g. devtool and recipetool, we can simplify code and be compatible with recent bitbake version. Change-Id: I2ae67dc01b62dd3e0e2535404ccbc3b15b484905 Signed-off-by: Fabio Berton --- bb-expand-vars.py | 258 +++++++++------------------------------------- 1 file changed, 51 insertions(+), 207 deletions(-) diff --git a/bb-expand-vars.py b/bb-expand-vars.py index 50e35e8..4ff453d 100644 --- a/bb-expand-vars.py +++ b/bb-expand-vars.py @@ -1,15 +1,7 @@ #! /usr/bin/env python3 # -*- encoding: utf-8 -*- - -""" From https://github.com/kergoth/bb/blob/master/libexec/bbcmd.py """ - -import re -import argparse -import contextlib -import logging import os import sys -import warnings PATH = os.getenv('PATH').split(':') bitbake_paths = [os.path.join(path, '..', 'lib') @@ -19,181 +11,36 @@ sys.path[0:0] = bitbake_paths -import bb.msg -import bb.utils -import bb.providers -import bb.tinfoil -from bb.cookerdata import CookerConfiguration, ConfigParameters - - -class Terminate(BaseException): - pass - - -class Tinfoil(bb.tinfoil.Tinfoil): - def __init__(self, output=sys.stdout): - # Needed to avoid deprecation warnings with python 2.6 - warnings.filterwarnings("ignore", category=DeprecationWarning) - - # Set up logging - self.logger = logging.getLogger('BitBake') - if output is not None: - setup_log_handler(self.logger, output) - - self.config = self.config = CookerConfiguration() - configparams = bb.tinfoil.TinfoilConfigParameters(parse_only=True) - self.config.setConfigParameters(configparams) - self.config.setServerRegIdleCallback(self.register_idle_function) - self.cooker = bb.cooker.BBCooker(self.config) - self.config_data = self.cooker.data - bb.providers.logger.setLevel(logging.ERROR) - bb.taskdata.logger.setLevel(logging.CRITICAL) - self.cooker_data = None - self.taskdata = {} - - self.localdata = bb.data.createCopy(self.config_data) - self.localdata.finalize() - # TODO: why isn't expandKeys a method of DataSmart? - bb.data.expandKeys(self.localdata) - - - def prepare_taskdata(self, provided=None, rprovided=None): - self.cache_data = self.cooker.recipecaches[''] - self.taskdata[''] = self.taskdata.get('', bb.taskdata.TaskData(abort=False)) - - if provided: - self.add_provided(provided) - - if rprovided: - self.add_rprovided(rprovided) - - def add_rprovided(self, rprovided): - for item in rprovided: - self.taskdata[''].add_rprovider(self.localdata, self.cache_data, item) - - self.taskdata[''].add_unresolved(self.localdata, self.cache_data) - - def add_provided(self, provided): - if 'world' in provided: - if not self.cache_data.world_target: - self.cooker.buildWorldTargetList() - provided.remove('world') - provided.extend(self.cache_data.world_target) - - if 'universe' in provided: - provided.remove('universe') - provided.extend(self.cache_data.universe_target) - - for item in provided: - self.taskdata[''].add_provider(self.localdata, self.cache_data, item) - - self.taskdata[''].add_unresolved(self.localdata, self.cache_data) - - def get_buildid(self, target): - if not self.taskdata[''].have_build_target(target): - if target in self.cooker.recipecaches[''].ignored_dependencies: - return - - reasons = self.taskdata[''].get_reasons(target) - if reasons: - self.logger.error("No buildable '%s' recipe found:\n%s", target, "\n".join(reasons)) - else: - self.logger.error("No '%s' recipe found", target) - return - else: - return self.taskdata[''].getbuild_id(target) - - def target_filenames(self): - """Return the filenames of all of taskdata's targets""" - filenames = set() - - for targetid in self.taskdata[''].build_targets: - fnid = self.taskdata[''].build_targets[targetid][0] - fn = self.taskdata[''].fn_index[fnid] - filenames.add(fn) - - for targetid in self.taskdata[''].run_targets: - fnid = self.taskdata[''].run_targets[targetid][0] - fn = self.taskdata[''].fn_index[fnid] - filenames.add(fn) - - return filenames - - def all_filenames(self): - return self.cooker.recipecaches[''].file_checksums.keys() - - def all_preferred_filenames(self): - """Return all the recipes we have cached, filtered by providers. - - Unlike target_filenames, this doesn't operate against taskdata. - """ - filenames = set() - excluded = set() - for provide, fns in self.cooker.recipecaches[''].providers.items(): - eligible, foundUnique = bb.providers.filterProviders(fns, provide, - self.localdata, - self.cooker.recipecaches['']) - preferred = eligible[0] - if len(fns) > 1: - # Excluding non-preferred providers in multiple-provider - # situations. - for fn in fns: - if fn != preferred: - excluded.add(fn) - filenames.add(preferred) - - filenames -= excluded - return filenames - - def provide_to_fn(self, provide): - """Return the preferred recipe for the specified provide""" - filenames = self.cooker.recipecaches.providers[provide] - eligible, foundUnique = bb.providers.filterProviders(filenames, provide, self.localdata) - return eligible[0] - - def build_target_to_fn(self, target): - """Given a target, prepare taskdata and return a filename""" - self.prepare_taskdata([target]) - if target in self.taskdata[''].build_targets and self.taskdata[''].build_targets[target]: - fn = self.taskdata[''].build_targets[target][0] - return fn - - def parse_recipe_file(self, recipe_filename): - """Given a recipe filename, do a full parse of it""" - bb_cache = bb.cache.NoCache(self.cooker.databuilder) - appends = self.cooker.collection.get_file_appends(recipe_filename) - try: - recipe_data = bb_cache.loadDataFull(recipe_filename, appends) - except Exception: - raise - return recipe_data - - def parse_metadata(self, recipe=None): - """Return metadata, either global or for a particular recipe""" - if recipe: - self.prepare_taskdata([recipe]) - filename = self.build_target_to_fn(recipe) - return self.parse_recipe_file(filename) +def find_yocto_root(): + def inner_find(dir): + repo_dir = os.path.join(dir, '.repo') + if os.path.exists(repo_dir) and os.path.isdir(repo_dir): + return dir + elif dir == '/': + return False else: - return self.localdata - - -def setup_log_handler(logger, output=sys.stderr): - log_format = bb.msg.BBLogFormatter("%(levelname)s: %(message)s") - if output.isatty() and hasattr(log_format, 'enable_color'): - log_format.enable_color() - handler = logging.StreamHandler(output) - handler.setFormatter(log_format) - - bb.msg.addDefaultlogFilter(handler) - logger.addHandler(handler) - logger.setLevel(logging.INFO) - + return inner_find(os.path.dirname(dir)) + BUILDDIR = os.environ.get('BUILDDIR') + if BUILDDIR: + yocto_root = inner_find(BUILDDIR) + else: + yocto_root = inner_find(os.getcwd()) + return yocto_root or die("ERROR: won't search from /.") + +def get_yocto_path(): + types = ['poky', 'openembedded-core', 'oe'] + base = find_yocto_root() + paths = [os.path.join(base, 'sources', t, 'scripts/lib') for t in types] + path = list(filter(os.path.exists, paths)) + if len(path) != 1: + print("ERROR: Can't find scripts path") + sys.exit(1) + sys.path.append(path[0]) -def sigterm_exception(signum, stackframe): - raise Terminate() +basepath = '' -###### end of bbcmd +get_yocto_path() +from devtool import setup_tinfoil # For printing indented expression expansions indent_step = 4 @@ -307,38 +154,35 @@ def expand_expr(expr, metadata): def show_var_expansions(recipe, var, plumbing_mode=False): # When plumbing_mode is truthy, var is a list of variables + try: + tinfoil = setup_tinfoil(config_only=True, basepath=basepath) + tinfoil.parseRecipes() - ## tinfoil sets up log output for the bitbake loggers, but bb uses - ## a separate namespace at this time - setup_log_handler(logging.getLogger('bb')) - - tinfoil = Tinfoil(output=sys.stderr) - tinfoil.prepare(config_only=True) - - tinfoil.parseRecipes() + try: + metadata = tinfoil.parse_recipe(recipe) + except: + sys.exit(1) - try: - metadata = tinfoil.parse_metadata(recipe) - except: - sys.exit(1) + if plumbing_mode: + vars_vals = {} + for v in var: + vars_vals[v] = metadata.getVar(v, True) + return vars_vals + else: + val = metadata.getVar(var, True) - if plumbing_mode: - vars_vals = {} - for v in var: - vars_vals[v] = metadata.getVar(v, True) - return vars_vals - else: - val = metadata.getVar(var, True) + if val is not None: + print('=== Final value') + print( '%s = %s' % (var, val)) - if val is not None: - print('=== Final value') - print( '%s = %s' % (var, val)) + print('\n=== Expansion') + expand_expr('${' + var + '}', metadata) + else: + sys.stderr.write('%s: no such variable.\n' % var) + sys.exit(1) - print('\n=== Expansion') - expand_expr('${' + var + '}', metadata) - else: - sys.stderr.write('%s: no such variable.\n' % var) - sys.exit(1) + finally: + tinfoil.shutdown() if __name__ == '__main__': From b5a9244adcdfcfa276b669010fb481e6f57b3dd5 Mon Sep 17 00:00:00 2001 From: Fabio Berton Date: Thu, 25 May 2017 10:33:07 -0300 Subject: [PATCH 03/13] Find sysroot files inside sysroots-components directory Change-Id: I72e9a7797bf3a6db3ca6bb99c83bb72398ba9a93 Signed-off-by: Fabio Berton --- ye | 6 +++--- ye-cd | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/ye b/ye index 2d832ba..b7a14f8 100755 --- a/ye +++ b/ye @@ -899,7 +899,7 @@ grep sg sysroot-grep - Run 'grep -r $BUILDDIR/tmp/sysroots/$MACHINE'. + Run 'grep -r $BUILDDIR/tmp/sysroots-components/$MACHINE'. glg [-n ] [-i] git-log-grep [-n ] [-i] @@ -1078,7 +1078,7 @@ def main(args): machine = val['MACHINE'] else: die('Could not determine MACHINE.') - sysroot_dir = os.path.join(BUILDDIR, 'tmp', 'sysroots', machine) + sysroot_dir = os.path.join(BUILDDIR, 'tmp', 'sysroots-components', machine) print_matches(grep(cmd_args, [view_action, edit_action], case_sensitive, @@ -1098,7 +1098,7 @@ def main(args): elif cmd in ['sf', 'sysroot-find', 'wsf', 'work-shared-find']: check_builddir() if cmd in ['sf', 'sysroot-find']: - search_dir = os.path.join(BUILDDIR, 'tmp', 'sysroots') + search_dir = os.path.join(BUILDDIR, 'tmp', 'sysroots-components') else: search_dir = os.path.join(BUILDDIR, 'tmp', 'work-shared') if os.path.isdir(search_dir): diff --git a/ye-cd b/ye-cd index cd942e6..a05a347 100644 --- a/ye-cd +++ b/ye-cd @@ -75,7 +75,7 @@ ye() { "sd"|"sysroot"|"sysrootdir") $YE_SCRIPT plumbing x MACHINE local machine=`yecd_get_bb_var MACHINE` - [ -n "$machine" ] && cd $BUILDDIR/tmp/sysroots/$machine + [ -n "$machine" ] && cd $BUILDDIR/tmp/sysroots-components/$machine ;; "src"|"sources") local recipe=$2 From f4f6decd0c724db132a8b1820799f43413e2e0ac Mon Sep 17 00:00:00 2001 From: Otavio Salvador Date: Wed, 1 Nov 2017 16:18:37 -0200 Subject: [PATCH 04/13] Fix .deb support as it uses xz packages Change-Id: I4aa8bb7f9641b3f392f1e2750e9ccecf05a01aa6 Signed-off-by: Otavio Salvador --- ye | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/ye b/ye index b7a14f8..9b048bd 100755 --- a/ye +++ b/ye @@ -2,7 +2,7 @@ ## ye: Yocto Explorer ## -## Copyright (C) 2014, 2015 O.S. Systems Software LTDA +## Copyright (C) 2014, 2015, 2017 O.S. Systems Software LTDA ## ## This program is free software: you can redistribute it and/or modify ## it under the terms of the GNU Affero General Public License as @@ -348,8 +348,10 @@ def print_matches(matches, plumbing_cmd=None): def pkg_view(file): ext = os.path.splitext(file)[1] - if ext in ['.ipk', '.deb']: + if ext in '.ipk': os.system("ar p %s data.tar.gz | tar tvzf -" % file) + if ext in '.deb': + os.system("ar p %s data.tar.xz | tar tvJf -" % file) elif ext == '.rpm': os.system("rpm -qlvp %s" % file) else: @@ -380,13 +382,20 @@ def pkg_extract(pkg_file, file, from_control): if os.path.exists(out_dir): die("%s already exists. Won't clobber. Aborting." % out_dir) os.makedirs(out_dir) - if ext in ['.ipk', '.deb']: + if ext in '.ipk': tarball = 'data.tar.gz' if from_control: tarball = 'control.tar.gz' status = os.system("ar p %s %s | tar xzf - %s -C %s" % (pkg_file, tarball, file, out_dir)) if not status: print('Extracted to %s' % out_dir) + if ext in '.deb': + tarball = 'data.tar.xz' + if from_control: + tarball = 'control.tar.gz' + status = os.system("ar p %s %s | tar xJf - %s -C %s" % (pkg_file, tarball, file, out_dir)) + if not status: + print('Extracted to %s' % out_dir) elif ext == '.rpm': if is_program_available('rpm2cpio'): os.chdir(out_dir) From e469cac4b385d331bd488da230b3c0465413381b Mon Sep 17 00:00:00 2001 From: Fabio Berton Date: Fri, 23 Feb 2018 16:01:58 -0300 Subject: [PATCH 05/13] ye: Add support .ipk and .deb with xz and gzip compression types Change-Id: Idbf868a706f61b12aaf3423d1877fe776c4e5fcf Signed-off-by: Fabio Berton --- ye | 42 ++++++++++++++++++++++++------------------ 1 file changed, 24 insertions(+), 18 deletions(-) diff --git a/ye b/ye index 9b048bd..31b8487 100755 --- a/ye +++ b/ye @@ -348,10 +348,14 @@ def print_matches(matches, plumbing_cmd=None): def pkg_view(file): ext = os.path.splitext(file)[1] - if ext in '.ipk': - os.system("ar p %s data.tar.gz | tar tvzf -" % file) - if ext in '.deb': - os.system("ar p %s data.tar.xz | tar tvJf -" % file) + if ext in [ '.deb', '.ipk' ]: + comp_type = os.popen("ar t %s" % file).read() + if 'data.tar.gz' in comp_type: + os.system("ar p %s data.tar.gz | tar tvzf -" % file) + elif 'data.tar.xz' in comp_type: + os.system("ar p %s data.tar.xz | tar tvJf -" % file) + else: + die("ERROR: Compression type not supported for %s" % ext) elif ext == '.rpm': os.system("rpm -qlvp %s" % file) else: @@ -382,20 +386,22 @@ def pkg_extract(pkg_file, file, from_control): if os.path.exists(out_dir): die("%s already exists. Won't clobber. Aborting." % out_dir) os.makedirs(out_dir) - if ext in '.ipk': - tarball = 'data.tar.gz' - if from_control: - tarball = 'control.tar.gz' - status = os.system("ar p %s %s | tar xzf - %s -C %s" % (pkg_file, tarball, file, out_dir)) - if not status: - print('Extracted to %s' % out_dir) - if ext in '.deb': - tarball = 'data.tar.xz' - if from_control: - tarball = 'control.tar.gz' - status = os.system("ar p %s %s | tar xJf - %s -C %s" % (pkg_file, tarball, file, out_dir)) - if not status: - print('Extracted to %s' % out_dir) + if ext in [ '.deb', '.ipk' ]: + comp_type = os.popen("ar t %s" % pkg_file).read() + if 'data.tar.gz' in comp_type: + tarball = 'data.tar.gz' + if from_control: + tarball = 'control.tar.gz' + status = os.system("ar p %s %s | tar xzf - %s -C %s" % (pkg_file, tarball, file, out_dir)) + if not status: + print('Extracted to %s' % out_dir) + if 'data.tar.xz' in comp_type: + tarball = 'data.tar.xz' + if from_control: + tarball = 'control.tar.gz' + status = os.system("ar p %s %s | tar xJf - %s -C %s" % (pkg_file, tarball, file, out_dir)) + if not status: + print('Extracted to %s' % out_dir) elif ext == '.rpm': if is_program_available('rpm2cpio'): os.chdir(out_dir) From dc9fc8a0bebc4214f458c5b43a50e71ad5d0a1c7 Mon Sep 17 00:00:00 2001 From: Fabio Berton Date: Sun, 10 May 2020 17:44:28 -0300 Subject: [PATCH 06/13] ye: Add PV-PR to located workdir output The WORKDIR directory is defined as follows: ${TMPDIR}/work/${MULTIMACH_TARGET_SYS}/${PN}/${EXTENDPE}${PV}-${PR} Append ${PV}-${PR} when print the workdir output from command ye wd. Signed-off-by: Fabio Berton Change-Id: I9127702de0955a4a5473496b2927439337a9f409 --- ye | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ye b/ye index 31b8487..0466558 100755 --- a/ye +++ b/ye @@ -303,10 +303,10 @@ def find_workdir(pattern, exact): if not exact: pattern = '.*%s.*' % pattern re_pattern = re.compile(pattern) - paths = glob.glob(os.path.join(BUILDDIR, 'tmp', 'work', '*', '*')) + paths = glob.glob(os.path.join(BUILDDIR, 'tmp', 'work', '*', '*', '*')) for path in paths: if os.path.isdir(path): - dir = os.path.basename(path) + dir = os.path.abspath(os.path.join(path, os.pardir)) if re_pattern.match(dir): results.append(path) return results From a87b3a2740fe7550e57010f7f97cb9942dcf0041 Mon Sep 17 00:00:00 2001 From: Otavio Salvador Date: Wed, 23 Feb 2022 09:04:59 -0300 Subject: [PATCH 07/13] Revert "ye: Add PV-PR to located workdir output" This reverts commit dc9fc8a0bebc4214f458c5b43a50e71ad5d0a1c7. --- ye | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ye b/ye index 0466558..31b8487 100755 --- a/ye +++ b/ye @@ -303,10 +303,10 @@ def find_workdir(pattern, exact): if not exact: pattern = '.*%s.*' % pattern re_pattern = re.compile(pattern) - paths = glob.glob(os.path.join(BUILDDIR, 'tmp', 'work', '*', '*', '*')) + paths = glob.glob(os.path.join(BUILDDIR, 'tmp', 'work', '*', '*')) for path in paths: if os.path.isdir(path): - dir = os.path.abspath(os.path.join(path, os.pardir)) + dir = os.path.basename(path) if re_pattern.match(dir): results.append(path) return results From 0921b2ee825016597a14c371cc670d1b4e3ce93b Mon Sep 17 00:00:00 2001 From: Otavio Salvador Date: Wed, 23 Feb 2022 09:07:05 -0300 Subject: [PATCH 08/13] ye-cd: Fix machine name lookup as we use _ instead of - Signed-off-by: Otavio Salvador --- ye-cd | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ye-cd b/ye-cd index a05a347..7708a42 100644 --- a/ye-cd +++ b/ye-cd @@ -74,7 +74,7 @@ ye() { ;; "sd"|"sysroot"|"sysrootdir") $YE_SCRIPT plumbing x MACHINE - local machine=`yecd_get_bb_var MACHINE` + local machine=`yecd_get_bb_var MACHINE | tr - _` [ -n "$machine" ] && cd $BUILDDIR/tmp/sysroots-components/$machine ;; "src"|"sources") @@ -96,7 +96,7 @@ ye() { "pkg"|"packages") $YE_SCRIPT plumbing x IMAGE_PKGTYPE MACHINE local pkg_type=`yecd_get_bb_var IMAGE_PKGTYPE` - local machine=`yecd_get_bb_var MACHINE` + local machine=`yecd_get_bb_var MACHINE | tr - _` [ -n "$pkg_type" ] && [ -n "$machine" ] && \ cd $BUILDDIR/tmp/deploy/$pkg_type/$machine ;; From 4b06f78c2746f66e2377761d4f6d430aadf909e1 Mon Sep 17 00:00:00 2001 From: Luciano Dittgen Date: Fri, 29 May 2026 00:51:43 -0300 Subject: [PATCH 09/13] ye: support modern Yocto layer layouts Add compatibility helpers for current Yocto projects that do not use the legacy .repo/sources layout. Detect BUILDDIR from conf/bblayers.conf, parse BBLAYERS, honor OEROOT/COREBASE and YE_SOURCE_DIRS, and keep fallbacks for legacy sources/ and layers/ trees. Update find/view/edit, grep, git-log-grep, sysroot search, and cd plumbing to use detected layer, source, sysroot, and package deploy directories. Fall back from repo grep to git grep or recursive grep when repo is not available. Update BitBake variable expansion to find scripts/lib in modern trees and read global config metadata without requiring core-image-minimal. Refresh the doc command to parse current docs.yoctoproject.org variable pages. Update setup-environment integration and README documentation for both legacy O.S. Systems layouts and newer Yocto layouts such as Scarthgap and Wrynose. --- README.adoc | 100 ++++++------ bb-expand-vars.py | 71 +++++--- setup-environment.d/ye.py | 14 +- ye | 290 +++++++++++++++++++++++---------- ye-cd | 11 +- ye_compat.py | 331 ++++++++++++++++++++++++++++++++++++++ 6 files changed, 652 insertions(+), 165 deletions(-) create mode 100644 ye_compat.py diff --git a/README.adoc b/README.adoc index b9fccc4..c017036 100644 --- a/README.adoc +++ b/README.adoc @@ -9,10 +9,9 @@ the directory structure adopted by O.S. Systems to organize projects that use the Yocto Project. `ye` provides means to quickly locate files and directories into projects' directory tree, and act on them. -=== The directory layout assumed by ye +=== The directory layouts understood by ye -The infrastructure adopted by O.S. Systems assumes the following -directory layout: +`ye` supports the legacy O.S. Systems layout: .... @@ -31,6 +30,12 @@ of all projects (git repositories) will be stored. The directory layout in the `` directory follows the standard layout as determined by the Yocto Project. +For current Yocto Project releases, `ye` also understands projects +set up with `oe-init-build-env`, `OEROOT`, `/conf/bblayers.conf`, +and layer checkouts under either `sources/` or `layers/`. If automatic +detection is not enough, `YE_SOURCE_DIRS` can be set to a colon-separated +list of layer or source directories. + === Notes on arguments, environment and interactive behavior Most `ye` commands accept a regular expression as argument. Thus, it's @@ -62,12 +67,13 @@ Usage: ye [ ] f [-e] find [-e] - Locate paths (in the "sources" directory) that match the given - expression . is case insensitive and implicitly - surrounded by '.*'. -e disables the implicit use of '.*' around - the given . Note that, unless contains /, matching - is attempted on filenames only (not on dirnames). If - contains /, matching is attempted on the full path. + Locate paths in configured Yocto layer/source directories that + match the given expression . is case insensitive + and implicitly surrounded by '.*'. -e disables the implicit use + of '.*' around the given . Note that, unless + contains /, matching is attempted on filenames only (not on + dirnames). If contains /, matching is attempted on the + full path. v [-e] view [-e] @@ -157,11 +163,12 @@ run [-e] [] g grep - Run 'repo grep '. + Run 'repo grep ' in repo-based workspaces, otherwise run + 'git grep ' across configured layers. sg sysroot-grep - Run 'grep -r $BUILDDIR/tmp/sysroots/$MACHINE'. + Run 'grep -r ' across available sysroot directories. glg [-n ] [-i] git-log-grep [-n ] [-i] @@ -214,14 +221,14 @@ cd [] Change to the sysroot directory for MACHINE src [] - Change to 's source dir or to TOPDIR/sources - if is not provided + Change to 's source dir or to the detected source/layers + directory if is not provided img Change to BUILDDIR/tmp/deploy/MACHINE/image/ pkg - Change to BUILDDIR/tmp/deploy/PKG_TYPE/image/ + Change to BUILDDIR/tmp/deploy/PKG_TYPE or its MACHINE subdir manifest Change to TOPDIR/.repo/manifests @@ -240,13 +247,16 @@ some file types (e.g., packages). The following sections provide a more in-depth explanation about them. -==== Finding and operating on files in the `sources` directory +==== Finding and operating on files in layer/source directories The `find` command (short: `f`) can be used to locate files under the -`sources` directory. It's argument is a regular expression that will -be matched against pathnames. If the given regex contains `/`, -matching is attempted on filenames only (not on dirnames). If the -given regex contains `/`, matching is attempted on the full path. +configured Yocto layers. `ye` gets those layers from `bblayers.conf` +when possible, otherwise it falls back to legacy `sources/`, modern +`layers/`, `OEROOT` or `YE_SOURCE_DIRS`. Its argument is a regular +expression that will be matched against pathnames. If the given regex +does not contain `/`, matching is attempted on filenames only (not on +dirnames). If the given regex contains `/`, matching is attempted on +the full path. After locating files that match the given pattern, `ye` will prompt you to select one of the matches and, next, what to do with it. In @@ -341,7 +351,10 @@ for the one you really want to see. The `sysroot-find` (short: `sf`) command is pretty much equivalent to the `find` command, except it locates files under the sysroots -directory (`/tmp/sysroots`). +directories. Current Yocto releases primarily use +`/tmp/sysroots-components` and per-recipe +`recipe-sysroot` directories; older `/tmp/sysroots` trees +are still used as a fallback when present. ==== Finding and displaying files in the `work-shared` directory @@ -648,19 +661,15 @@ detail. ==== Finding text in source files -The `grep` command is a thin wrapper around `repo grep` (`repo` is the -tool used by O.S. Systems to manage multiple git repositories -- see -the http://doc.ossystems.com.br/managing-platforms.html[Managing -platforms based on the Yocto Project] document for more information). -Basically, `repo grep ` will run `git grep ` on -each repository (in the `sources` directory) which is part of the -project. +The `grep` command searches configured Yocto layers. In legacy +repo-based workspaces it uses `repo grep`; otherwise it runs `git grep` +across detected layer repositories. If the layers are not git +repositories, it falls back to recursive `grep`. -The `grep` command will run `repo grep` plus the arguments provided on -the command line (any valid argument for `git grep`) and will prompt -you to select one of the matches, then the action to apply on the -selected file. In case of a single match, you'll be only prompted for -the action. +The `grep` command will pass the arguments provided on the command +line to the underlying grep command and will prompt you to select one +of the matches, then the action to apply on the selected file. In +case of a single match, you'll be only prompted for the action. Example: @@ -683,9 +692,8 @@ Option (ENTER to cancel): 2v ==== Finding text in files in the sysroot directory The `sysroot-grep` (short: `sg`) command is similar to the `grep` -command, but instead of searching for matches in the `sources` -directory, it recursively searches for matches in the `sysroot` -directory (`/tmp/sysroots/`, specifically). +command, but instead of searching for matches in source directories, +it recursively searches for matches in available sysroot directories. Example: @@ -852,10 +860,11 @@ STAGING_DIR_TARGET ==> ${STAGING_DIR}/${MACHINE} .... -Except for the `find` and `grep` commands, all commands expect the -`BUILDDIR` environment variable to be set in the environment. This -variable is automatically set by the `setup-environment` script -provided by O.S. Systems for the Yocto Project-based projects. +Most build-output commands expect the `BUILDDIR` environment variable +to be set in the environment. This variable is set by the usual Yocto +environment setup scripts. If it is not set, `ye` will also try to +detect a build directory by walking up from the current directory until +it finds `conf/bblayers.conf`. === Moving around: changing directories @@ -909,7 +918,9 @@ will do that automatically if you have `ye` in your source tree. displaying and editing files, respectively. The configuration is via environment variables. `ye` uses `YE_PAGER` -and `YE_EDITOR` for pager and editor, respectively. +and `YE_EDITOR` for pager and editor, respectively. `YE_SOURCE_DIRS` +can be set to a colon-separated list of source or layer directories +when the automatic layer detection is not appropriate. For the editor, `ye` first checks if `YE_EDITOR` is set in the environment. If it is not set, it checks the `EDITOR` environment @@ -926,12 +937,9 @@ cannot be found, you'll get an error. == Requirements -A Python installation and the directory structure in the layout -created by O.S. System's Yocto Project-based platforms. - -`ye` has been more extensively tested with Python version 2.7.3, but -it should work with other recent Python 2.x versions and with Python -3.x. +A Python 3 installation and a Yocto Project build tree. `ye` supports +both the legacy O.S. Systems `.repo`/`sources` layout and current +Yocto layouts that expose layers through `conf/bblayers.conf`. For the `doc` command, the http://lxml.de/[lxml] module for Python is required. diff --git a/bb-expand-vars.py b/bb-expand-vars.py index 4ff453d..7cd22bf 100644 --- a/bb-expand-vars.py +++ b/bb-expand-vars.py @@ -2,6 +2,7 @@ # -*- encoding: utf-8 -*- import os import sys +import ye_compat PATH = os.getenv('PATH').split(':') bitbake_paths = [os.path.join(path, '..', 'lib') @@ -12,30 +13,37 @@ sys.path[0:0] = bitbake_paths def find_yocto_root(): - def inner_find(dir): - repo_dir = os.path.join(dir, '.repo') - if os.path.exists(repo_dir) and os.path.isdir(repo_dir): - return dir - elif dir == '/': - return False - else: - return inner_find(os.path.dirname(dir)) - BUILDDIR = os.environ.get('BUILDDIR') - if BUILDDIR: - yocto_root = inner_find(BUILDDIR) - else: - yocto_root = inner_find(os.getcwd()) - return yocto_root or die("ERROR: won't search from /.") + return ye_compat.workspace_root() def get_yocto_path(): - types = ['poky', 'openembedded-core', 'oe'] - base = find_yocto_root() - paths = [os.path.join(base, 'sources', t, 'scripts/lib') for t in types] - path = list(filter(os.path.exists, paths)) - if len(path) != 1: + candidates = [] + for env_name in ['OEROOT', 'COREBASE']: + if os.environ.get(env_name): + candidates.append(os.environ[env_name]) + + root = find_yocto_root() + if root: + candidates.extend([ + os.path.join(root, 'poky'), + os.path.join(root, 'openembedded-core'), + os.path.join(root, 'sources', 'poky'), + os.path.join(root, 'sources', 'openembedded-core'), + os.path.join(root, 'sources', 'oe'), + ]) + + for layer in ye_compat.source_dirs(): + candidates.extend([ + layer, + os.path.dirname(layer), + os.path.dirname(os.path.dirname(layer)), + ]) + + paths = ye_compat.existing_dirs( + [os.path.join(path, 'scripts', 'lib') for path in candidates]) + if len(paths) < 1: print("ERROR: Can't find scripts path") sys.exit(1) - sys.path.append(path[0]) + sys.path.append(paths[0]) basepath = '' @@ -154,13 +162,23 @@ def expand_expr(expr, metadata): def show_var_expansions(recipe, var, plumbing_mode=False): # When plumbing_mode is truthy, var is a list of variables + tinfoil = None try: tinfoil = setup_tinfoil(config_only=True, basepath=basepath) - tinfoil.parseRecipes() - - try: - metadata = tinfoil.parse_recipe(recipe) - except: + metadata = getattr(tinfoil, 'config_data', None) + if recipe: + if hasattr(tinfoil, 'parseRecipes'): + tinfoil.parseRecipes() + elif hasattr(tinfoil, 'parse_recipes'): + tinfoil.parse_recipes() + + try: + metadata = tinfoil.parse_recipe(recipe) + except: + sys.exit(1) + + if metadata is None: + sys.stderr.write('Could not load BitBake metadata.\n') sys.exit(1) if plumbing_mode: @@ -182,7 +200,8 @@ def show_var_expansions(recipe, var, plumbing_mode=False): sys.exit(1) finally: - tinfoil.shutdown() + if tinfoil: + tinfoil.shutdown() if __name__ == '__main__': diff --git a/setup-environment.d/ye.py b/setup-environment.d/ye.py index a60f481..7e10d07 100644 --- a/setup-environment.d/ye.py +++ b/setup-environment.d/ye.py @@ -1,6 +1,16 @@ +import os + + def __after_init_ye_yocto(): PLATFORM_ROOT_DIR = os.environ['PLATFORM_ROOT_DIR'] - yedir = os.path.join(PLATFORM_ROOT_DIR, 'sources', 'ye') - os.environ['PATH'] = os.environ['PATH'] + ':' + yedir + candidates = [ + os.path.join(PLATFORM_ROOT_DIR, 'sources', 'ye'), + os.path.join(PLATFORM_ROOT_DIR, 'layers', 'ye'), + os.path.join(PLATFORM_ROOT_DIR, 'ye'), + ] + for yedir in candidates: + if os.path.exists(os.path.join(yedir, 'ye')): + os.environ['PATH'] = os.environ['PATH'] + ':' + yedir + break run_after_init(__after_init_ye_yocto) diff --git a/ye b/ye index 31b8487..42baa25 100755 --- a/ye +++ b/ye @@ -28,6 +28,7 @@ import glob import urllib.request, urllib.parse, urllib.error import locale import subprocess +import ye_compat # lxml is required for parsing the documentation XML files is_lxml_available = True @@ -36,7 +37,10 @@ try: except: is_lxml_available = False -encoding = locale.getdefaultlocale()[1] +try: + encoding = locale.getencoding() +except AttributeError: + encoding = locale.getdefaultlocale()[1] if not encoding: sys.stderr.write('WARNING: could not determine locale. Forcing UTF-8.\n') encoding = 'UTF-8' @@ -89,7 +93,7 @@ YE_EDITOR += ' "%s"' BUILDDIR = None try: - BUILDDIR = os.environ['BUILDDIR'] + BUILDDIR = ye_compat.get_builddir() except: pass @@ -107,13 +111,16 @@ bold = '\033[1m{}\033[0m' def colorize_layer(path): path_parts = path.split('/') - sources_idx = None + source_idx = None try: - sources_idx = path_parts.index('sources') + source_idx = path_parts.index('sources') except ValueError: - return path - if sources_idx < (len(path_parts) - 2): - layer_idx = sources_idx + 1 + try: + source_idx = path_parts.index('layers') + except ValueError: + source_idx = None + if source_idx is not None and source_idx < (len(path_parts) - 2): + layer_idx = source_idx + 1 path_parts[layer_idx] = bold.format(path_parts[layer_idx]) return '/'.join(path_parts) @@ -146,25 +153,24 @@ def download_file(uri, output_file): output_fd.write(uri_fd.read()) def find_yocto_root(): - def inner_find(dir): - repo_dir = os.path.join(dir, '.repo') - if os.path.exists(repo_dir) and os.path.isdir(repo_dir): - return dir - elif dir == '/': - return False - else: - return inner_find(os.path.dirname(dir)) - if BUILDDIR: - yocto_root = inner_find(BUILDDIR) - else: - yocto_root = inner_find(os.getcwd()) + yocto_root = ye_compat.workspace_root() return yocto_root or die("ERROR: won't search from /.") +def find_source_dirs(): + dirs = ye_compat.source_dirs() + return dirs or die("ERROR: could not determine Yocto layer/source directories.") + + def find_files(basedir, maxdepth=None, pattern='.*'): + if isinstance(basedir, list): + results = [] + for dir in basedir: + results.extend(find_files(dir, maxdepth, pattern)) + return results depth = 0 re_pattern = re.compile(pattern) - re_skip_pattern = re.compile('.*~$|^\.#|^#.*#$|.*\.pyc$') + re_skip_pattern = re.compile(r'.*~$|^\.#|^#.*#$|.*\.pyc$') results = [] match_whole_path = '/' in pattern for root, dirs, files in os.walk(basedir): @@ -482,7 +488,7 @@ def list_temp_files(file_type, recipe_pattern, log_pattern, exact, raw, apply_re version_dir = version_dirs[0] log_dir = os.path.join(version_dir, 'temp') all_files = find_files(log_dir, - pattern='%s\..*%s.*' % (file_type, log_pattern)) + pattern=r'%s\..*%s.*' % (file_type, log_pattern)) logs = [ f for f in all_files if os.path.islink(f) ] log = None if logs: @@ -503,23 +509,87 @@ def list_logs(recipe_pattern, log_pattern, exact, raw, apply_replacements): def list_run_scripts(recipe_pattern, log_pattern, exact): list_temp_files('run', recipe_pattern, log_pattern, exact, True, False) + +def ensure_list(value): + if value is None: + return [] + if isinstance(value, list): + return value + return [value] + + +def prefix_grep_paths(output, root): + lines = output.decode(encoding, errors='replace').split('\n')[:-1] + prefixed = [] + for line in lines: + file, sep, rest = line.partition(':') + if sep and not os.path.isabs(file): + line = os.path.join(root, file) + sep + rest + prefixed.append(line) + return prefixed + + +def run_git_grep(args, dirs): + results = [] + for repo in ye_compat.git_repos(dirs): + p = subprocess.Popen(['git', '-C', repo, 'grep'] + args, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + out, _ = p.communicate() + results.extend(prefix_grep_paths(out, repo)) + return results + + +def run_recursive_grep(args, dirs): + results = [] + for dir in dirs: + p = subprocess.Popen(['grep', '-r'] + args + [dir], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + out, _ = p.communicate() + results.extend(out.decode(encoding, errors='replace').split('\n')[:-1]) + return results + + +def apply_action(file, actions, shortcut=None): + if len(actions) == 1: + actions[0][2](file) + return file + if shortcut: + for action in actions: + if action[1] == shortcut: + action[2](file) + return file + action = prompt_action([ (a[1], a[0]) for a in actions ]) + actions[action][2](file) + return file + + def grep(args, actions, case_sensitive, dir=None, git=False): - if git and dir: - os.chdir(dir) - p = subprocess.Popen(['git', 'grep'] + args, stdout=subprocess.PIPE) - elif dir: - os.chdir(dir) - p = subprocess.Popen(['grep', '-r'] + args, stdout=subprocess.PIPE) - else: - p = subprocess.Popen(['repo', 'grep'] + args, stdout=subprocess.PIPE) regex_pattern = [ arg for arg in args if not arg.startswith('-') ] if not regex_pattern: die('Could not determine the pattern for grep.') if len(regex_pattern) > 1: die('Multiple patterns for grep is not supported.') regex_pattern = regex_pattern[0] - output = p.communicate()[0] - results = output.decode(encoding).split('\n')[:-1] # the last line is always empty + + if git and dir: + results = run_git_grep(args, ensure_list(dir)) + elif dir: + results = run_recursive_grep(args, ensure_list(dir)) + else: + repo_root = ye_compat.get_repo_root() + if repo_root and ye_compat.is_program_available('repo'): + p = subprocess.Popen(['repo', 'grep'] + args, + cwd=repo_root, + stdout=subprocess.PIPE) + results = p.communicate()[0].decode(encoding, errors='replace').split('\n')[:-1] + else: + source_dirs = find_source_dirs() + results = run_git_grep(args, source_dirs) + if not results: + results = run_recursive_grep(args, source_dirs) + if sys.stdout.isatty(): shortcut = None if len(results) == 0: @@ -541,27 +611,22 @@ def grep(args, actions, case_sensitive, dir=None, git=False): else: actions = [view_action, edit_action] file = match.split(':')[0] # let's hope filenames don't contain ':' + if os.path.isabs(file) and os.path.exists(file): + return apply_action(file, actions, shortcut) return find(file, - dir or os.path.join(find_yocto_root(), 'sources'), + dir or find_source_dirs(), False, actions) else: return results def git_log_grep(pattern, max_commits, case_sensitive): - root_dir = find_yocto_root() - os.chdir(root_dir) - repos = [] - repop = subprocess.Popen(['repo', 'list'], stdout=subprocess.PIPE) - repos = repop.communicate()[0].strip() - repos = [ line.split(':')[0].strip() for line in repos.decode(encoding).split('\n') ] - repo_paths = [ os.path.realpath(os.path.join(root_dir, repo)) for repo in repos ] - + repo_paths = ye_compat.git_repos(find_source_dirs()) results = {} for repo_path in repo_paths: results[repo_path] = None - os.chdir(repo_path) - gitlog = subprocess.Popen(('git', 'log', '-n', str(max_commits), '--oneline'), + gitlog = subprocess.Popen(('git', '-C', repo_path, 'log', '-n', + str(max_commits), '--oneline'), stdout=subprocess.PIPE) if case_sensitive: grep_cmd = ('grep', pattern) @@ -575,13 +640,13 @@ def git_log_grep(pattern, max_commits, case_sensitive): if out: results[repo_path] = out.strip().split(b'\n') - sources_dir = os.path.realpath(os.path.join(root_dir, 'sources')) + root_dir = find_yocto_root() options = [] for repo_path, commits in results.items(): - repo_dir = repo_path[len(sources_dir) + 1:] # drop sources_dir + repo_dir = os.path.relpath(repo_path, root_dir) if commits: for commit in commits: - options.append('%s %s' % (repo_dir, commit)) + options.append('%s %s' % (repo_dir, commit.decode(encoding))) if sys.stdout.isatty(): shortcut = None @@ -600,9 +665,8 @@ def git_log_grep(pattern, max_commits, case_sensitive): match = options[0] repo_dir, commit_hash = match.split()[0:2] - repo_path = os.path.join(sources_dir, repo_dir) - os.chdir(repo_path) - os.system('git show %s' % commit_hash) + repo_path = os.path.join(root_dir, repo_dir) + os.system('git -C "%s" show %s' % (repo_path, commit_hash)) else: for option in options: print(option) @@ -687,38 +751,68 @@ def doc(pattern, exact): print('\n=== ' + var) print(doc) + def element_text(elt): + return remove_duplicate_spaces(' '.join(elt.itertext())).strip() + + def parse_current_html(path): + parser = etree.HTMLParser() + tree = etree.parse(path, parser) + result = {} + for dt in tree.xpath('//dt[starts-with(@id, "term-")]'): + var = element_text(dt).replace('ΒΆ', '').strip() + if not var: + continue + var = var.split()[0] + var = re.sub(r'[^A-Za-z0-9_:+.-].*$', '', var) + dd = dt.getnext() + while dd is not None and dd.tag != 'dd': + dd = dd.getnext() + if dd is not None: + result[var] = element_text(dd) + return result + + def parse_legacy_xml(path): + tree = etree.parse(path) + result = {} + for e in tree.findall('//glossentry'): + glossterm = e.find('glossterm').text + glossdef = e.find('glossdef') + paragraphs = glossdef.findall('para') + text = [] + for p in paragraphs: + text.append(parse_paragraph(p)) + result[glossterm] = '\n\n'.join(text) + return result + doc_data_dir = os.path.join(YE_DIR, 'doc-data') + variables_html = os.path.join(doc_data_dir, 'ref-manual', 'variables.html') ref_variables_xml = os.path.join(doc_data_dir, 'ref-manual', 'ref-variables.xml') # Check if we need to update documentation data update_window = 7 * 24 * 3600 # 7 days - # We use documentation from master - doc_base_uri = 'https://git.yoctoproject.org/cgit/cgit.cgi/yocto-docs/plain/documentation' - if (not os.path.exists(ref_variables_xml) or - os.stat(ref_variables_xml).st_mtime < time.time() - update_window): + doc_base_uri = 'https://docs.yoctoproject.org' + if (not os.path.exists(variables_html) or + os.stat(variables_html).st_mtime < time.time() - update_window): print('Updating documentation data...') - download_file('%s/poky.ent' % doc_base_uri, - os.path.join(doc_data_dir, 'poky.ent')) - download_file('%s/ref-manual/ref-variables.xml' % doc_base_uri, - ref_variables_xml) + download_file('%s/ref-manual/variables.html' % doc_base_uri, + variables_html) - tree = etree.parse(ref_variables_xml) - glossentries = tree.findall('//glossentry') - docs = {} + try: + all_docs = parse_current_html(variables_html) + except: + all_docs = {} + if os.path.exists(ref_variables_xml): + all_docs = parse_legacy_xml(ref_variables_xml) orig_pattern = pattern if not exact: pattern = '.*%s.*' % pattern + re_pattern = re.compile(pattern, re.IGNORECASE) - for e in glossentries: - glossterm = e.find('glossterm').text - if re.compile(pattern).match(glossterm): - glossdef = e.find('glossdef') - paragraphs = glossdef.findall('para') - text = [] - for p in paragraphs: - text.append(parse_paragraph(p)) - docs[glossterm] = '\n\n'.join(text) + docs = {} + for var, text in all_docs.items(): + if re_pattern.match(var): + docs[var] = text if docs: options = list(docs.keys()) if len(options) == 1: @@ -782,11 +876,31 @@ def handle_plumbing(args): die('plumbing wd: missing argument.') elif cmd == 'topdir': print(find_yocto_root()) + elif cmd == 'sourcedir': + source_dir = ye_compat.source_root_dir() + if source_dir: + print(source_dir) + else: + die('plumbing sourcedir: could not determine source directory.') + elif cmd == 'sysrootdir': + dirs = ye_compat.sysroot_dirs(cmd_args[0] if cmd_args else None) + if dirs: + print(dirs[0]) + else: + die('plumbing sysrootdir: could not determine sysroot directory.') + elif cmd == 'pkgdir': + pkg_type = cmd_args[0] if len(cmd_args) > 0 else None + machine = cmd_args[1] if len(cmd_args) > 1 else None + pkg_dir = ye_compat.deploy_pkg_dir(pkg_type, machine) + if pkg_dir: + print(pkg_dir) + else: + die('plumbing pkgdir: could not determine package deploy directory.') elif cmd == 'x': if cmd_args: expand_mod = __import__('bb-expand-vars') os.chdir(BUILDDIR) - val = expand_mod.show_var_expansions('core-image-minimal', + val = expand_mod.show_var_expansions(None, cmd_args, True) if val: @@ -795,7 +909,7 @@ def handle_plumbing(args): die('plumbing x: missing argument.') elif cmd == 'find': print_matches(find(cmd_args[0], - os.path.join(find_yocto_root(), 'sources'), + find_source_dirs(), exact, [('Print', 'p', lambda file: print(file))]), plumbing_cmd=cmd) @@ -815,12 +929,13 @@ def usage(exit_code=None, cmd=None): f [-e] find [-e] - Locate paths (in the "sources" directory) that match the given - expression . is case insensitive and implicitly - surrounded by '.*'. -e disables the implicit use of '.*' around - the given . Note that, unless contains /, matching - is attempted on filenames only (not on dirnames). If - contains /, matching is attempted on the full path. + Locate paths in configured Yocto layer/source directories that + match the given expression . is case insensitive + and implicitly surrounded by '.*'. -e disables the implicit use + of '.*' around the given . Note that, unless + contains /, matching is attempted on filenames only (not on + dirnames). If contains /, matching is attempted on the + full path. v [-e] view [-e] @@ -910,11 +1025,12 @@ run [-e] [] g grep - Run 'repo grep '. + Run 'repo grep ' in repo-based workspaces, otherwise run + 'git grep ' across configured layers. sg sysroot-grep - Run 'grep -r $BUILDDIR/tmp/sysroots-components/$MACHINE'. + Run 'grep -r ' across available sysroot directories. glg [-n ] [-i] git-log-grep [-n ] [-i] @@ -967,14 +1083,14 @@ cd [] Change to the sysroot directory for MACHINE src [] - Change to 's source dir or to TOPDIR/sources - if is not provided + Change to 's source dir or to the detected source/layers + directory if is not provided img Change to BUILDDIR/tmp/deploy/MACHINE/image/ pkg - Change to BUILDDIR/tmp/deploy/PKG_TYPE/image/ + Change to BUILDDIR/tmp/deploy/PKG_TYPE or its MACHINE subdir manifest Change to TOPDIR/.repo/manifests @@ -1086,14 +1202,16 @@ def main(args): check_builddir() expand_mod = __import__('bb-expand-vars') os.chdir(BUILDDIR) - val = expand_mod.show_var_expansions('core-image-minimal', + val = expand_mod.show_var_expansions(None, ['MACHINE'], True) if val: machine = val['MACHINE'] else: die('Could not determine MACHINE.') - sysroot_dir = os.path.join(BUILDDIR, 'tmp', 'sysroots-components', machine) + sysroot_dir = ye_compat.sysroot_dirs(machine) + if not sysroot_dir: + die('Could not determine a sysroot directory.') print_matches(grep(cmd_args, [view_action, edit_action], case_sensitive, @@ -1107,16 +1225,16 @@ def main(args): else: actions = [view_action, edit_action] print_matches(find(pattern, - os.path.join(find_yocto_root(), 'sources'), + find_source_dirs(), exact, actions)) elif cmd in ['sf', 'sysroot-find', 'wsf', 'work-shared-find']: check_builddir() if cmd in ['sf', 'sysroot-find']: - search_dir = os.path.join(BUILDDIR, 'tmp', 'sysroots-components') + search_dir = ye_compat.sysroot_dirs() else: search_dir = os.path.join(BUILDDIR, 'tmp', 'work-shared') - if os.path.isdir(search_dir): + if isinstance(search_dir, list) or os.path.isdir(search_dir): print_matches(find(pattern, search_dir, exact, diff --git a/ye-cd b/ye-cd index 7708a42..69c4e53 100644 --- a/ye-cd +++ b/ye-cd @@ -75,13 +75,14 @@ ye() { "sd"|"sysroot"|"sysrootdir") $YE_SCRIPT plumbing x MACHINE local machine=`yecd_get_bb_var MACHINE | tr - _` - [ -n "$machine" ] && cd $BUILDDIR/tmp/sysroots-components/$machine + local sysrootdir=`$YE_SCRIPT plumbing sysrootdir "$machine"` + [ -n "$sysrootdir" ] && cd "$sysrootdir" ;; "src"|"sources") local recipe=$2 if [ -z "$recipe" ]; then - topdir=`$YE_SCRIPT plumbing topdir` - [ $? -eq 0 ] && cd $topdir/sources + local sourcedir=`$YE_SCRIPT plumbing sourcedir` + [ -n "$sourcedir" ] && cd "$sourcedir" else cd $BUILDDIR $YE_SCRIPT plumbing find $recipe @@ -97,8 +98,8 @@ ye() { $YE_SCRIPT plumbing x IMAGE_PKGTYPE MACHINE local pkg_type=`yecd_get_bb_var IMAGE_PKGTYPE` local machine=`yecd_get_bb_var MACHINE | tr - _` - [ -n "$pkg_type" ] && [ -n "$machine" ] && \ - cd $BUILDDIR/tmp/deploy/$pkg_type/$machine + local pkgdir=`$YE_SCRIPT plumbing pkgdir "$pkg_type" "$machine"` + [ -n "$pkgdir" ] && cd "$pkgdir" ;; "manifest"|"manifests") topdir=`$YE_SCRIPT plumbing topdir` diff --git a/ye_compat.py b/ye_compat.py new file mode 100644 index 0000000..1aebcc3 --- /dev/null +++ b/ye_compat.py @@ -0,0 +1,331 @@ +import glob +import os +import re +import subprocess + + +def uniq(paths): + result = [] + seen = set() + for path in paths: + if not path: + continue + real = os.path.realpath(os.path.abspath(os.path.expanduser(path))) + if real not in seen: + seen.add(real) + result.append(real) + return result + + +def existing_dirs(paths): + return [path for path in uniq(paths) if os.path.isdir(path)] + + +def find_upwards(start, names): + start = os.path.realpath(os.path.abspath(start or os.getcwd())) + if os.path.isfile(start): + start = os.path.dirname(start) + while True: + for name in names: + path = os.path.join(start, name) + if os.path.exists(path): + return start + parent = os.path.dirname(start) + if parent == start: + return None + start = parent + + +def is_program_available(program): + for directory in os.environ.get('PATH', '').split(os.pathsep): + path = os.path.join(directory, program) + if os.path.isfile(path) and os.access(path, os.X_OK): + return True + return False + + +def get_builddir(): + builddir = os.environ.get('BUILDDIR') + if builddir: + return os.path.realpath(os.path.abspath(builddir)) + + found = find_upwards(os.getcwd(), + [os.path.join('conf', 'bblayers.conf')]) + if found: + return found + return None + + +def get_repo_root(): + builddir = get_builddir() + starts = [os.getcwd()] + if builddir: + starts.insert(0, builddir) + for start in starts: + root = find_upwards(start, ['.repo']) + if root: + return root + return None + + +def join_bitbake_lines(text): + logical_lines = [] + current = '' + for line in text.splitlines(): + stripped = line.rstrip() + if stripped.endswith('\\'): + current += stripped[:-1] + ' ' + continue + logical_lines.append(current + stripped) + current = '' + if current: + logical_lines.append(current) + return logical_lines + + +def strip_comment(line): + result = [] + quote = None + escaped = False + for char in line: + if escaped: + result.append(char) + escaped = False + continue + if char == '\\': + result.append(char) + escaped = True + continue + if char in ['"', "'"]: + if quote == char: + quote = None + elif quote is None: + quote = char + if char == '#' and quote is None: + break + result.append(char) + return ''.join(result) + + +def expand_bitbake_path(path, variables): + value = os.path.expanduser(path) + for _ in range(8): + expanded = re.sub(r'\${([^}]+)}', + lambda m: variables.get(m.group(1), + os.environ.get(m.group(1), m.group(0))), + value) + if expanded == value: + break + value = expanded + return os.path.expandvars(value) + + +def parse_bblayers_conf(builddir=None): + builddir = builddir or get_builddir() + if not builddir: + return [] + + conf = os.path.join(builddir, 'conf', 'bblayers.conf') + if not os.path.exists(conf): + return [] + + try: + data = open(conf, 'r').read() + except OSError: + return [] + + variables = { + 'TOPDIR': builddir, + 'BUILDDIR': builddir, + 'HOME': os.environ.get('HOME', ''), + } + for var in ['OEROOT', 'COREBASE', 'POKYBASE']: + if os.environ.get(var): + variables[var] = os.environ[var] + + layers = [] + for line in join_bitbake_lines(data): + line = strip_comment(line).strip() + if not re.match(r'^BBLAYERS(?:\s|[:?+.=:]|$)', line): + continue + for match in re.finditer(r'"([^"]*)"', line): + for token in match.group(1).split(): + expanded = expand_bitbake_path(token, variables) + if not os.path.isabs(expanded): + expanded = os.path.join(builddir, expanded) + matches = glob.glob(expanded) + layers.extend(matches or [expanded]) + return existing_dirs(layers) + + +def fallback_source_dirs(builddir=None): + candidates = [] + repo_root = get_repo_root() + if repo_root: + candidates.extend([ + os.path.join(repo_root, 'sources'), + os.path.join(repo_root, 'layers'), + ]) + + for env_name in ['PLATFORM_ROOT_DIR', 'YE_TOPDIR']: + if os.environ.get(env_name): + root = os.environ[env_name] + candidates.extend([ + os.path.join(root, 'sources'), + os.path.join(root, 'layers'), + root, + ]) + + for env_name in ['OEROOT', 'COREBASE']: + if os.environ.get(env_name): + candidates.append(os.environ[env_name]) + + builddir = builddir or get_builddir() + if builddir: + parent = os.path.dirname(builddir) + candidates.extend([ + os.path.join(parent, 'sources'), + os.path.join(parent, 'layers'), + os.path.join(parent, 'poky'), + os.path.join(parent, 'openembedded-core'), + ]) + + root = find_upwards(os.getcwd(), ['sources', 'layers', 'poky', + 'openembedded-core']) + if root: + candidates.extend([ + os.path.join(root, 'sources'), + os.path.join(root, 'layers'), + os.path.join(root, 'poky'), + os.path.join(root, 'openembedded-core'), + ]) + + return existing_dirs(candidates) + + +def source_dirs(): + env_dirs = os.environ.get('YE_SOURCE_DIRS') + if env_dirs: + return existing_dirs(env_dirs.split(os.pathsep)) + + builddir = get_builddir() + layers = parse_bblayers_conf(builddir) + if layers: + return layers + return fallback_source_dirs(builddir) + + +def source_root_dir(): + root = workspace_root() + if root: + for name in ['sources', 'layers']: + path = os.path.join(root, name) + if os.path.isdir(path): + return os.path.realpath(path) + + dirs = source_dirs() + if not dirs: + return None + try: + common = os.path.commonpath(dirs) + if common and common != os.path.sep and os.path.isdir(common): + return os.path.realpath(common) + except ValueError: + pass + return dirs[0] + + +def workspace_root(): + repo_root = get_repo_root() + if repo_root: + return repo_root + + roots = [] + builddir = get_builddir() + if builddir: + roots.append(builddir) + roots.extend(source_dirs()) + if roots: + try: + common = os.path.commonpath(roots) + if common and common != os.path.sep: + return os.path.realpath(common) + except ValueError: + pass + + for env_name in ['PLATFORM_ROOT_DIR', 'YE_TOPDIR', 'OEROOT', 'COREBASE']: + if os.environ.get(env_name): + return os.path.realpath(os.path.abspath(os.environ[env_name])) + return find_upwards(os.getcwd(), ['sources', 'layers', 'poky', + 'openembedded-core']) + + +def find_git_root(path): + if not path or not is_program_available('git'): + return None + try: + out = subprocess.check_output( + ['git', '-C', path, 'rev-parse', '--show-toplevel'], + stderr=subprocess.DEVNULL) + return os.path.realpath(out.decode().strip()) + except (subprocess.CalledProcessError, OSError): + return None + + +def git_repos(dirs=None): + dirs = dirs or source_dirs() + repos = [] + for directory in dirs: + root = find_git_root(directory) + if root: + repos.append(root) + continue + for current, subdirs, _ in os.walk(directory): + subdirs[:] = [d for d in subdirs + if d not in ['.git', 'tmp', 'downloads', + 'sstate-cache', 'cache']] + if '.git' in subdirs: + repos.append(current) + subdirs[:] = [] + return existing_dirs(repos) + + +def sysroot_dirs(machine=None): + builddir = get_builddir() + if not builddir: + return [] + + candidates = [] + components = os.path.join(builddir, 'tmp', 'sysroots-components') + if machine: + candidates.append(os.path.join(components, machine)) + candidates.append(os.path.join(components, machine.replace('-', '_'))) + candidates.append(components) + + old_sysroots = os.path.join(builddir, 'tmp', 'sysroots') + if machine: + candidates.append(os.path.join(old_sysroots, machine)) + candidates.append(old_sysroots) + + candidates.extend(glob.glob(os.path.join(builddir, 'tmp', 'work', '*', + '*', '*', 'recipe-sysroot'))) + candidates.extend(glob.glob(os.path.join(builddir, 'tmp', 'work', '*', + '*', '*', 'recipe-sysroot-native'))) + return existing_dirs(candidates) + + +def deploy_pkg_dir(pkg_type=None, machine=None): + builddir = get_builddir() + if not builddir: + return None + deploy = os.path.join(builddir, 'tmp', 'deploy') + candidates = [] + if pkg_type: + pkg_base = os.path.join(deploy, pkg_type) + if machine: + candidates.append(os.path.join(pkg_base, machine.replace('-', '_'))) + candidates.append(os.path.join(pkg_base, machine)) + candidates.append(pkg_base) + candidates.append(deploy) + dirs = existing_dirs(candidates) + return dirs[0] if dirs else None From 7aa71b442f04d34aab71afad821b221a6dbd81c0 Mon Sep 17 00:00:00 2001 From: Luciano Dittgen Date: Sat, 30 May 2026 19:18:07 -0300 Subject: [PATCH 10/13] Fix depth-limited file search and BBLAYERS parsing Update ye file discovery to calculate depth from each path relative to the search root and prune child traversal when maxdepth is reached. The old counter advanced per os.walk iteration instead of per directory level, so searches could stop early and skip sibling directories. Keep the current directory's visible child names available before pruning so directories at the max depth can still be matched while deeper traversal is blocked. Add quoted value parsing for bblayers.conf in ye_compat.py, including single quotes, double quotes, and escaped characters. This handles valid BitBake layer definitions that the previous double-quote-only regex could miss or parse incorrectly. Read bblayers.conf with a context manager so the file handle is closed reliably after parsing. --- ye | 13 +++++++------ ye_compat.py | 38 +++++++++++++++++++++++++++++++++++--- 2 files changed, 42 insertions(+), 9 deletions(-) diff --git a/ye b/ye index 42baa25..6d7139e 100755 --- a/ye +++ b/ye @@ -168,21 +168,22 @@ def find_files(basedir, maxdepth=None, pattern='.*'): for dir in basedir: results.extend(find_files(dir, maxdepth, pattern)) return results - depth = 0 re_pattern = re.compile(pattern) re_skip_pattern = re.compile(r'.*~$|^\.#|^#.*#$|.*\.pyc$') results = [] match_whole_path = '/' in pattern for root, dirs, files in os.walk(basedir): dirs[:] = [d for d in dirs if not d[0] == '.'] - depth += 1 - if maxdepth and depth > maxdepth: - return results + visible_dirs = list(dirs) + relroot = os.path.relpath(root, basedir) + depth = 1 if relroot == '.' else relroot.count(os.sep) + 2 + if maxdepth and depth >= maxdepth: + dirs[:] = [] all_files = None if match_whole_path: - all_files = [ os.path.join(root, f) for f in dirs + files ] + all_files = [ os.path.join(root, f) for f in visible_dirs + files ] else: - all_files = dirs + files + all_files = visible_dirs + files for file in all_files: if not re_skip_pattern.match(file) and re_pattern.match(file): results.append(os.path.join(root, file)) diff --git a/ye_compat.py b/ye_compat.py index 1aebcc3..a683d78 100644 --- a/ye_compat.py +++ b/ye_compat.py @@ -120,6 +120,37 @@ def expand_bitbake_path(path, variables): return os.path.expandvars(value) +def quoted_values(line): + values = [] + quote = None + value = [] + escaped = False + for char in line: + if escaped: + if quote: + value.append(char) + escaped = False + continue + if char == '\\': + escaped = True + if quote: + value.append(char) + continue + if char in ['"', "'"]: + if quote == char: + values.append(''.join(value)) + value = [] + quote = None + elif quote is None: + quote = char + elif quote: + value.append(char) + continue + if quote: + value.append(char) + return values + + def parse_bblayers_conf(builddir=None): builddir = builddir or get_builddir() if not builddir: @@ -130,7 +161,8 @@ def parse_bblayers_conf(builddir=None): return [] try: - data = open(conf, 'r').read() + with open(conf, 'r') as fd: + data = fd.read() except OSError: return [] @@ -148,8 +180,8 @@ def parse_bblayers_conf(builddir=None): line = strip_comment(line).strip() if not re.match(r'^BBLAYERS(?:\s|[:?+.=:]|$)', line): continue - for match in re.finditer(r'"([^"]*)"', line): - for token in match.group(1).split(): + for value in quoted_values(line): + for token in value.split(): expanded = expand_bitbake_path(token, variables) if not os.path.isabs(expanded): expanded = os.path.join(builddir, expanded) From 6e3b5a8279640c19c45164d28c3cdf5810a6b03c Mon Sep 17 00:00:00 2001 From: Luciano Dittgen Date: Sat, 30 May 2026 19:29:53 -0300 Subject: [PATCH 11/13] Support modern Yocto layer and IPK package layouts Parse single-quoted BBLAYERS entries so source discovery stays scoped to configured layers in generated bblayers.conf files. Detect data.tar.* and control.tar.* members in deb/ipk archives instead of assuming fixed gzip/xz names. Add zstd support for Scarthgap/ Wrynose IPK payloads and share archive handling across package view, info, scripts, and extract commands. Add focused tests for gzip, xz, and zstd package archive selection. --- README.adoc | 2 +- test_ye_packages.py | 50 ++++++++++++++++++++++ ye | 101 ++++++++++++++++++++++++++++++++------------ 3 files changed, 124 insertions(+), 29 deletions(-) create mode 100644 test_ye_packages.py diff --git a/README.adoc b/README.adoc index c017036..52b1ee2 100644 --- a/README.adoc +++ b/README.adoc @@ -131,7 +131,7 @@ pkg-extract [-e] [-c] [] the package filename (without extension). must match the exact file name in the package (usually starts with ./). -c is specific to .deb and .ipk packages -- ye will extract files - from the control.tar.gz tarball in packages. + from the control archive in packages. wd [-e] workdir [-e] diff --git a/test_ye_packages.py b/test_ye_packages.py new file mode 100644 index 0000000..72e8afb --- /dev/null +++ b/test_ye_packages.py @@ -0,0 +1,50 @@ +import importlib.machinery +import importlib.util +import os +import pathlib +import unittest +from unittest import mock + + +def load_ye(): + os.environ.setdefault('YE_EDITOR', 'true') + path = pathlib.Path(__file__).with_name('ye') + loader = importlib.machinery.SourceFileLoader('ye_module', str(path)) + spec = importlib.util.spec_from_loader(loader.name, loader) + module = importlib.util.module_from_spec(spec) + loader.exec_module(module) + return module + + +class PackageArchiveTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.ye = load_ye() + + def test_pkg_tarball_selects_gzip_data_archive(self): + members = ['debian-binary', 'control.tar.gz', 'data.tar.gz'] + with mock.patch.object(self.ye, 'ar_members', return_value=members): + self.assertEqual(self.ye.pkg_tarball('package.ipk', 'data'), + 'data.tar.gz') + + def test_pkg_tarball_selects_xz_data_archive(self): + members = ['debian-binary', 'control.tar.gz', 'data.tar.xz'] + with mock.patch.object(self.ye, 'ar_members', return_value=members): + self.assertEqual(self.ye.pkg_tarball('package.ipk', 'data'), + 'data.tar.xz') + + def test_pkg_tarball_selects_zstd_data_archive(self): + members = ['debian-binary', 'control.tar.gz', 'data.tar.zst'] + with mock.patch.object(self.ye, 'ar_members', return_value=members): + self.assertEqual(self.ye.pkg_tarball('package.ipk', 'data'), + 'data.tar.zst') + + def test_pkg_tarball_selects_compressed_control_archive(self): + members = ['debian-binary', 'control.tar.zst', 'data.tar.zst'] + with mock.patch.object(self.ye, 'ar_members', return_value=members): + self.assertEqual(self.ye.pkg_tarball('package.ipk', 'control'), + 'control.tar.zst') + + +if __name__ == '__main__': + unittest.main() diff --git a/ye b/ye index 6d7139e..6460b64 100755 --- a/ye +++ b/ye @@ -353,16 +353,65 @@ def print_matches(matches, plumbing_cmd=None): dest.close() +TAR_COMPRESSION_ARGS = [ + ('.tar.gz', ['--gzip']), + ('.tar.xz', ['--xz']), + ('.tar.zst', ['--zstd']), + ('.tar', []), +] + + +def tar_compression_args(tarball): + for suffix, args in TAR_COMPRESSION_ARGS: + if tarball.endswith(suffix): + return args + return None + + +def ar_members(pkg_file): + try: + output = subprocess.check_output(['ar', 't', pkg_file], + universal_newlines=True) + except subprocess.CalledProcessError: + die("ERROR: could not read package file: %s" % pkg_file) + return [line.strip() for line in output.splitlines() if line.strip()] + + +def pkg_tarball(pkg_file, archive_name): + ext = os.path.splitext(pkg_file)[1] + prefix = archive_name + '.tar' + unsupported = [] + for member in ar_members(pkg_file): + if member == prefix or member.startswith(prefix + '.'): + if tar_compression_args(member) is not None: + return member + unsupported.append(member) + if unsupported: + die("ERROR: Compression type not supported for %s" % ext) + die("ERROR: %s archive not found in %s" % (archive_name, ext)) + + +def run_ar_tar(pkg_file, tarball, tar_args): + compression_args = tar_compression_args(tarball) + if compression_args is None: + die("ERROR: Compression type not supported for %s" % + os.path.splitext(pkg_file)[1]) + + ar = subprocess.Popen(['ar', 'p', pkg_file, tarball], + stdout=subprocess.PIPE) + tar = subprocess.Popen(['tar'] + compression_args + tar_args, + stdin=ar.stdout) + ar.stdout.close() + tar_status = tar.wait() + ar_status = ar.wait() + return ar_status or tar_status + + def pkg_view(file): ext = os.path.splitext(file)[1] if ext in [ '.deb', '.ipk' ]: - comp_type = os.popen("ar t %s" % file).read() - if 'data.tar.gz' in comp_type: - os.system("ar p %s data.tar.gz | tar tvzf -" % file) - elif 'data.tar.xz' in comp_type: - os.system("ar p %s data.tar.xz | tar tvJf -" % file) - else: - die("ERROR: Compression type not supported for %s" % ext) + tarball = pkg_tarball(file, 'data') + run_ar_tar(file, tarball, ['-tvf', '-']) elif ext == '.rpm': os.system("rpm -qlvp %s" % file) else: @@ -372,7 +421,8 @@ def pkg_view(file): def pkg_info(file): ext = os.path.splitext(file)[1] if ext in ['.ipk', '.deb']: - os.system("ar p %s control.tar.gz | tar xzf - ./control -O" % file) + tarball = pkg_tarball(file, 'control') + run_ar_tar(file, tarball, ['-xOf', '-', './control']) elif ext == '.rpm': os.system("rpm -qip %s" % file) else: @@ -382,7 +432,8 @@ def pkg_info(file): def pkg_scripts(file): ext = os.path.splitext(file)[1] if ext in ['.ipk', '.deb']: - os.system("ar p %s control.tar.gz | tar tzf -" % file) + tarball = pkg_tarball(file, 'control') + run_ar_tar(file, tarball, ['-tf', '-']) else: die("ERROR: unsupported package file extension: %s" % ext) @@ -392,24 +443,18 @@ def pkg_extract(pkg_file, file, from_control): out_dir = os.path.basename(out_dir) if os.path.exists(out_dir): die("%s already exists. Won't clobber. Aborting." % out_dir) - os.makedirs(out_dir) if ext in [ '.deb', '.ipk' ]: - comp_type = os.popen("ar t %s" % pkg_file).read() - if 'data.tar.gz' in comp_type: - tarball = 'data.tar.gz' - if from_control: - tarball = 'control.tar.gz' - status = os.system("ar p %s %s | tar xzf - %s -C %s" % (pkg_file, tarball, file, out_dir)) - if not status: - print('Extracted to %s' % out_dir) - if 'data.tar.xz' in comp_type: - tarball = 'data.tar.xz' - if from_control: - tarball = 'control.tar.gz' - status = os.system("ar p %s %s | tar xJf - %s -C %s" % (pkg_file, tarball, file, out_dir)) - if not status: - print('Extracted to %s' % out_dir) + archive_name = 'control' if from_control else 'data' + tarball = pkg_tarball(pkg_file, archive_name) + os.makedirs(out_dir) + tar_args = ['-xf', '-', '-C', out_dir] + if file: + tar_args.append(file) + status = run_ar_tar(pkg_file, tarball, tar_args) + if not status: + print('Extracted to %s' % out_dir) elif ext == '.rpm': + os.makedirs(out_dir) if is_program_available('rpm2cpio'): os.chdir(out_dir) status = os.system("rpm2cpio %s | cpio -id %s" % (pkg_file, file)) @@ -994,7 +1039,7 @@ pkg-extract [-e] [-c] [] the package filename (without extension). must match the exact file name in the package (usually starts with ./). -c is specific to .deb and .ipk packages -- ye will extract files - from the control.tar.gz tarball in packages. + from the control archive in packages. wd [-e] workdir [-e] @@ -1291,5 +1336,5 @@ def main(args): else: usage(1) - -main(sys.argv[1:]) +if __name__ == '__main__': + main(sys.argv[1:]) From 1dedd425d53f5c1c4ced1f78fe59c63b279d57fb Mon Sep 17 00:00:00 2001 From: Luciano Dittgen Date: Sat, 30 May 2026 19:35:21 -0300 Subject: [PATCH 12/13] docs: expand Yocto Explorer usage and compatibility guide Document installation, shell setup, quick-start flow, command summary, modern layer discovery, package archive support, release compatibility, requirements, testing, and troubleshooting. Also update package documentation for control archive handling and zstd-compressed IPK payloads used by current Yocto releases. --- README.adoc | 415 +++++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 398 insertions(+), 17 deletions(-) diff --git a/README.adoc b/README.adoc index 52b1ee2..4f6fab6 100644 --- a/README.adoc +++ b/README.adoc @@ -36,6 +36,24 @@ and layer checkouts under either `sources/` or `layers/`. If automatic detection is not enough, `YE_SOURCE_DIRS` can be set to a colon-separated list of layer or source directories. +In practice, `ye` determines the workspace in this order: + +. If `BUILDDIR` is set, use it as the build directory. +. Otherwise, walk upward from the current directory until + `conf/bblayers.conf` is found. +. If a repo workspace exists, use the directory containing `.repo` as + the project root. +. Prefer layer directories declared in `conf/bblayers.conf`. +. If layer detection fails, fall back to `YE_SOURCE_DIRS`, + `PLATFORM_ROOT_DIR`, `YE_TOPDIR`, `OEROOT`, `COREBASE`, and common + sibling directories such as `sources/`, `layers/`, `poky/`, and + `openembedded-core/`. + +Both single-quoted and double-quoted `BBLAYERS` values are supported. +This matters for current Yocto Project setups, whose generated +configuration files commonly use shell-like multi-line single-quoted +assignments. + === Notes on arguments, environment and interactive behavior Most `ye` commands accept a regular expression as argument. Thus, it's @@ -53,6 +71,99 @@ to find out things). This variable is automatically set by the `setup-environment` script provided by O.S. Systems for Yocto Project-based projects. +Commands that print matches are useful in scripts too. When stdout is +not a terminal, `ye` disables the interactive prompts and prints all +matches to stdout, except for shortcut commands such as `view`, +`pkg-info`, or `pkg-extract`, which perform their requested action +when a single unambiguous match exists. + + +== Installation and shell setup + +`ye` is shipped as an executable Python script. It does not require a +Python package installation step; put the checkout directory in +`PATH`, or create a symlink to the `ye` executable from a directory +already in `PATH`. + +Example: + +.... +$ export PATH=/path/to/ye:$PATH +$ ye --help +.... + +For shell completion, source `ye-completion.sh` from your shell +startup file: + +.... +$ . /path/to/ye-completion.sh +.... + +The `ye cd` command needs to affect the current shell process, so it +is implemented by the `ye-cd` shell wrapper. Source it after `ye` is +available in `PATH`: + +.... +$ . /path/to/ye-cd +.... + +O.S. Systems' `setup-environment` script can do this automatically +when `ye` is present in the source tree. In other Yocto setups, add +the `PATH`, completion, and `ye-cd` lines to your preferred shell +startup file or project environment setup script. + + +== Quick start + +. Enter or source a Yocto build environment: ++ +.... +$ source oe-init-build-env +.... ++ +or, in O.S. Systems workspaces: ++ +.... +$ source setup-environment +.... + +. Confirm `ye` can see the build and configured layers: ++ +.... +$ ye plumbing topdir +$ ye f -e layer.conf +.... + +. Find and open a recipe or configuration file: ++ +.... +$ ye f busybox +$ ye v conf/machine/qemuarm.conf +.... + +. Locate build output: ++ +.... +$ ye wd linux-yocto +$ ye pf 'kernel_.*\.ipk' +$ ye sf stdio.h +.... + +. Inspect a package: ++ +.... +$ ye pi 'kernel_.*\.ipk' +$ ye pv 'kernel-image.*\.ipk' +.... + +. Use directory shortcuts by sourcing `ye-cd`: ++ +.... +$ . /path/to/ye-cd +$ ye cd src +$ ye cd wd linux-yocto +.... + == Using ye @@ -143,8 +254,8 @@ log [-e] [-H] [] Show the log files for . -e is only applied to . is always implicitly surrounded by '.*', if provided. If -H ("human readable") is given on the - command line, ye will try to make the lines that contain calls - to gcc/g++ look more readable. If -R is provided, ye will apply + command line, ye will try to make the lines that contain calls to + clang/clang++/gcc/g++ look more readable. If -R is provided, ye will apply some text replacements to make the output more readable. Currently, ye reverse expands some common variables whose expansion pollutes log files with long paths. The following variables are reverse @@ -239,6 +350,98 @@ cd [] .... +=== Command summary + +[cols="1,2,4",options="header"] +|=== +|Short |Long |Purpose + +|`f` +|`find` +|Find files in configured Yocto layer/source directories. + +|`v` +|`view` +|Find one file and open it with the configured pager. + +|`e` +|`edit` +|Find one file and open it with the configured editor. + +|`sf` +|`sysroot-find` +|Find files in available sysroot directories. + +|`wsf` +|`work-shared-find` +|Find files in `tmp/work-shared`. + +|`pf` +|`pkg-find` +|Find packages under `tmp/deploy/{deb,ipk,rpm}`. + +|`pv` +|`pkg-view` +|List files contained in a package. + +|`pi` +|`pkg-info` +|Show package metadata. + +|`ps` +|`pkg-scripts` +|List package control scripts or control members. + +|`px` +|`pkg-extract` +|Extract package data files, or control files with `-c`. + +|`wd` +|`workdir` +|Print recipe work directories under `tmp/work`. + +|`l` +|`log` +|Open BitBake task logs. + +|`r` +|`run` +|Open BitBake task run scripts. + +|`g` +|`grep` +|Search configured layers with `repo grep`, `git grep`, or recursive `grep`. + +|`sg` +|`sysroot-grep` +|Search text under available sysroot directories. + +|`glg` +|`git-log-grep` +|Search git commit summaries across layer repositories. + +|`gbh` +|`grep-buildhistory` +|Run `git grep` in the buildhistory repository. + +|`bh` +|`buildhistory` +|Show recent buildhistory changes. + +|`d` +|`doc` +|Search Yocto Project variable documentation. + +|`x` +|`expand` +|Expand BitBake variables and show recursive expansion. + +|`cd` +|`cd` +|Change to common project/build directories through the `ye-cd` shell wrapper. +|=== + + === Finding and operating on files `ye` provides commands to locate files and operate on them. Some @@ -372,6 +575,25 @@ packages (`/tmp/deploy/`). `ye` supports the most common package formats generated by Yocto Project: `.ipk`, `.deb` and `.rpm`. +For `.ipk` and `.deb` packages, `ye` reads the package as an `ar` +archive and then processes the embedded `data.tar.*` or +`control.tar.*` member. The following tar payload formats are +supported: + +* `data.tar` +* `data.tar.gz` +* `data.tar.xz` +* `data.tar.zst` +* `control.tar` +* `control.tar.gz` +* `control.tar.xz` +* `control.tar.zst` + +The zstd variants are important for Scarthgap, Wrynose, and newer +Yocto Project releases where IPK package payloads can use +`data.tar.zst`. RPM packages are handled through `rpm` and +`rpm2cpio`. + The actions for packages are different from the `find` command. `ye` supports the following actions on packages: @@ -384,6 +606,10 @@ supports the following actions on packages: `extract`:: extract package contents to a directory named after the package filename +The `pkg-extract` command refuses to overwrite an existing extraction +directory. Run it from a scratch directory if you want to inspect a +package without modifying the build tree. + Just like the `view` and `edit` counterparts to the `find` command, `ye` provides `pkg-view` (short: `pv`), `pkg-info` (short: `pi`), `pkg-scripts` (short: `ps`) and `pkg-extract` (short: `px`) command @@ -543,8 +769,9 @@ Option (ENTER to cancel): 4 .... The `log` command also handles the `-H` option, which tries to make -compiler command lines more readable (and numbers them). See some -examples below: +compiler command lines more readable (and numbers them). It +recognizes common C/C++ compiler invocations such as `gcc`, `g++`, +`clang`, and `clang++`. See some examples below: Without `-H`: @@ -717,7 +944,7 @@ Option (ENTER to cancel): 1v The `git-log-grep` command (short: `glg`) basically runs .... -git log --oneline | grep " +git log --oneline | grep .... for the given regular expressions on the summary lines of all git @@ -767,7 +994,7 @@ index f4ea800..0c92ff9 100644 ==== Finding text in the buildhistory repository -The `grep-buildhistory` commmand (short: `gbh`) is a wrapper around +The `grep-buildhistory` command (short: `gbh`) is a wrapper around `git grep` in the buildhistory directory. Example: @@ -832,7 +1059,7 @@ Option (ENTER to cancel): 3 .... `ye` maintains a cache of the Yocto project Reference Manual for seven -days (under `$YE_DIR/doc-data`). If the cache is older than seven +days (under `$HOME/.ye/doc-data`). If the cache is older than seven days, it will fetch the reference manual data and update the cache. @@ -866,6 +1093,11 @@ environment setup scripts. If it is not set, `ye` will also try to detect a build directory by walking up from the current directory until it finds `conf/bblayers.conf`. +The `expand` command starts BitBake through Tinfoil. It therefore has +the same constraints as other BitBake clients: it needs a valid build +environment and may fail if another process is holding the same +BitBake server in an incompatible state. + === Moving around: changing directories @@ -914,18 +1146,48 @@ will do that automatically if you have `ye` in your source tree. == Configuration -`ye` allows you to customize the pager and the editor it uses for -displaying and editing files, respectively. +`ye` is configured through environment variables. + +[cols="1,4",options="header"] +|=== +|Variable |Meaning + +|`BUILDDIR` +|Yocto build directory. If unset, `ye` tries to find a parent directory containing `conf/bblayers.conf`. + +|`YE_SOURCE_DIRS` +|Colon-separated list of layer/source directories. This overrides automatic source discovery. + +|`PLATFORM_ROOT_DIR` +|Project root hint used by O.S. Systems environments. -The configuration is via environment variables. `ye` uses `YE_PAGER` -and `YE_EDITOR` for pager and editor, respectively. `YE_SOURCE_DIRS` -can be set to a colon-separated list of source or layer directories -when the automatic layer detection is not appropriate. +|`YE_TOPDIR` +|Project root hint used when automatic discovery is not enough. + +|`OEROOT` +|OpenEmbedded-Core or Poky root hint, usually set by `oe-init-build-env`. + +|`COREBASE` +|OpenEmbedded-Core base directory hint. + +|`YE_PAGER` +|Command template used to view files. Use `%s` where the filename should be inserted. + +|`PAGER` +|Fallback pager when `YE_PAGER` is not set. + +|`YE_EDITOR` +|Command used to edit files. + +|`EDITOR` +|Fallback editor when `YE_EDITOR` is not set. +|=== For the editor, `ye` first checks if `YE_EDITOR` is set in the environment. If it is not set, it checks the `EDITOR` environment -variable. If it is not set, it resorts to `emacs`. If `emacs` cannot -be found, you'll get an error. +variable. If neither is set, it searches for a usable terminal editor +in this order: `emacs -nw`, `mg`, `zile`, `qemacs`, `nano`, `vim`, +and `vi`. If no editor is found, `ye` prints an error. For the pager, `ye` first checks if `YE_PAGER` is set in the environment. If it is not set, it checks the `PAGER` environment @@ -934,26 +1196,145 @@ cannot be found, you'll get an error. `%s` can be used as a placeholder for the file to act upon. +`ye` stores cached documentation and shell plumbing files below +`$HOME/.ye`. + + +== Compatibility with current Yocto releases + +`ye` is intended to work with both older O.S. Systems Yocto layouts +and current Yocto Project layouts. The compatibility-sensitive areas +are source/layer discovery, sysroot discovery, and package archive +format support. + +Kirkstone 4.0 LTS:: + Compatible for the tested surfaces. The older `tmp/sysroots` + layout remains supported as a fallback. + +Scarthgap 5.0 LTS:: + Compatible for source discovery, workdir lookup, sysroot lookup, and + package discovery. IPK zstd payloads are supported through + `data.tar.zst`. + +Wrynose 6.0 LTS:: + Compatible for the tested surfaces in a Wrynose build tree using + `PACKAGE_CLASSES = "package_ipk"`. `ye` correctly handles + configured layers from single-quoted `BBLAYERS`, per-recipe + sysroots, `tmp/sysroots-components`, and zstd-compressed IPK + payloads. + +When migrating between Yocto releases, pay special attention to: + +* package payload compression (`data.tar.zst` for modern IPK output); +* source layout changes around `WORKDIR` and `UNPACKDIR`; +* whether the build setup still creates `conf/bblayers.conf`; +* whether package, image, sysroot, and buildhistory directories exist + in the expected build directory. + == Requirements -A Python 3 installation and a Yocto Project build tree. `ye` supports +A Python 3 installation and a Yocto Project build tree are required. +Use the Python version supported by your Yocto release. `ye` supports both the legacy O.S. Systems `.repo`/`sources` layout and current Yocto layouts that expose layers through `conf/bblayers.conf`. +Core host tools: + +* `git` +* `grep` +* `less` or another pager +* a terminal editor for `ye edit` +* `ar` and GNU `tar` for `.deb` and `.ipk` inspection +* `zstd` or GNU `tar` with zstd support for modern `.ipk` payloads +* `rpm` and `rpm2cpio` for RPM package inspection/extraction + +Optional host tools: + +* `repo`, for legacy repo-based workspace grep support +* `buildhistory-diff`, for formatted buildhistory output + For the `doc` command, the http://lxml.de/[lxml] module for Python is required. For the `cd` command, a Bourne-compatible shell is required. +== Testing and validation + +The lightweight checks below are useful before changing `ye`: + +.... +$ python3 -m py_compile ye ye_compat.py bb-expand-vars.py format-command-lines.py setup-environment.d/ye.py +$ bash -n ye-cd +$ bash -n ye-completion.sh +.... + +Package archive selection is covered by `test_ye_packages.py`: + +.... +$ python3 -B test_ye_packages.py +.... + +Useful smoke tests in a real build tree: + +.... +$ ye --help +$ ye plumbing topdir +$ ye f -e layer.conf +$ ye wd -e linux-yocto +$ ye pf -e 'kernel_.*\.ipk' +$ ye pi -e 'kernel_.*\.ipk' +$ ye pv -e 'kernel-image.*\.ipk' +$ ye sf stdio.h +.... + +Commands that call BitBake, such as `ye expand` and `ye plumbing x`, +should be treated as integration tests. They need a working BitBake +server and a valid build environment. + + +== Troubleshooting + +`ye` searches too many files or files from unconfigured layers:: + Check `conf/bblayers.conf` and `YE_SOURCE_DIRS`. `ye` prefers + configured `BBLAYERS`; if parsing fails or no layer exists, it falls + back to broader source directories. + +`ERROR: Could not determine the Yocto Project's build directory.`:: + Run from inside the build directory, set `BUILDDIR`, or source the + Yocto setup script again. + +`pkg-view` reports an unsupported compression type:: + Confirm the package members with `ar t `. Supported + `.deb` and `.ipk` data/control archives are uncompressed tar, gzip, + xz, and zstd tar files. + +`pkg-extract` refuses to run because the output directory exists:: + `ye` never clobbers extraction directories. Remove or rename the + existing directory, or run the command from a scratch directory. + +`doc` cannot update documentation data:: + The `doc` command needs Python `lxml` and network access when its + cache under `$HOME/.ye/doc-data` is missing or older than seven + days. + +`expand` or `plumbing x` cannot start BitBake:: + Ensure the Yocto environment is sourced and that no incompatible + BitBake server is active for the same build directory. + + == Limitations Some `ye` commands use `bitbake` behind the scenes, and since -`bitbake` doesn't support running multipl instances in parallel under +`bitbake` doesn't support running multiple instances in parallel under the same build directory, some `ye` features may not work while you are using `bitbake`. +`ye` is a navigation and inspection helper. It does not replace +BitBake, `devtool`, package managers, or release migration tools; it +uses the build tree that those tools create. + == License From e8813bc82698335c4025657625cd20b0dacc11db Mon Sep 17 00:00:00 2001 From: Otavio Salvador Date: Tue, 2 Jun 2026 10:34:48 -0300 Subject: [PATCH 13/13] refactor: simplify and de-duplicate Yocto layout discovery Quality cleanup over the modern-layout support, with no behavior change: - Make find() reuse the extracted apply_action() helper instead of carrying an identical copy of the action-dispatch block. - Drop ye's local is_program_available() in favor of the one in ye_compat, so the helper exists in a single place. - Memoize get_builddir() and source_dirs() so a single command no longer re-walks the filesystem and re-parses bblayers.conf several times. - Extract a common_dir() helper shared by workspace_root() and source_root_dir(), replacing two copies of the commonpath dance. - Collapse the repeated well-known layer directory lists into a single LAYER_DIR_NAMES constant plus a layer_subdirs() helper. - Name the bblayers.conf variable-expansion pass bound (MAX_VAR_EXPANSION_PASSES). --- ye | 24 +++---------------- ye_compat.py | 66 ++++++++++++++++++++++++++++------------------------ 2 files changed, 38 insertions(+), 52 deletions(-) diff --git a/ye b/ye index 6460b64..e46863e 100755 --- a/ye +++ b/ye @@ -49,14 +49,6 @@ def die(msg): sys.stderr.write(msg + '\n') sys.exit(1) -def is_program_available(program): - progpaths = os.environ['PATH'].split(':') - for dir in progpaths: - progpath = os.path.join(dir, program) - if os.path.isfile(progpath) and os.access(progpath, os.X_OK): - return True - return False - YE_DIR = os.path.join(os.environ['HOME'], '.ye') EDITOR_ALTERNATIVES = [('emacs', '-nw'), 'mg', 'zile', 'qemacs', 'nano', 'vim', 'vi'] @@ -70,7 +62,7 @@ def find_editor(): args = option[1] else: editor = option - if is_program_available(editor): + if ye_compat.is_program_available(editor): return '%s %s' % (editor, args) die('Could not find an editor. Please, set the YE_EDITOR environment variable.') @@ -281,17 +273,7 @@ def find(pattern, basedir, exact, actions): else: file = results[0] print(colorize(shorten_path(file), orig_pattern)) - if len(actions) == 1: - actions[0][2](file) - return file - if shortcut: - for action in actions: - if action[1] == shortcut: - action[2](file) - else: - action = prompt_action([ (a[1], a[0]) for a in actions ]) - actions[action][2](file) - return file + return apply_action(file, actions, shortcut) else: if len(results) == 1 and len(actions) == 1: actions[0][2](results[0]) @@ -455,7 +437,7 @@ def pkg_extract(pkg_file, file, from_control): print('Extracted to %s' % out_dir) elif ext == '.rpm': os.makedirs(out_dir) - if is_program_available('rpm2cpio'): + if ye_compat.is_program_available('rpm2cpio'): os.chdir(out_dir) status = os.system("rpm2cpio %s | cpio -id %s" % (pkg_file, file)) if not status: diff --git a/ye_compat.py b/ye_compat.py index a683d78..0a32d8b 100644 --- a/ye_compat.py +++ b/ye_compat.py @@ -1,8 +1,32 @@ +import functools import glob import os import re import subprocess +# Well-known sibling directory names that may hold Yocto layers/sources. +LAYER_DIR_NAMES = ['sources', 'layers', 'poky', 'openembedded-core'] + +# Upper bound on the number of passes used to resolve nested ${VAR} +# references when expanding a bblayers.conf path. +MAX_VAR_EXPANSION_PASSES = 8 + + +def layer_subdirs(root): + return [os.path.join(root, name) for name in LAYER_DIR_NAMES] + + +def common_dir(dirs, require_dir=False): + try: + common = os.path.commonpath(dirs) + except ValueError: + return None + if not common or common == os.path.sep: + return None + if require_dir and not os.path.isdir(common): + return None + return os.path.realpath(common) + def uniq(paths): result = [] @@ -44,6 +68,7 @@ def is_program_available(program): return False +@functools.lru_cache(maxsize=None) def get_builddir(): builddir = os.environ.get('BUILDDIR') if builddir: @@ -109,7 +134,7 @@ def strip_comment(line): def expand_bitbake_path(path, variables): value = os.path.expanduser(path) - for _ in range(8): + for _ in range(MAX_VAR_EXPANSION_PASSES): expanded = re.sub(r'\${([^}]+)}', lambda m: variables.get(m.group(1), os.environ.get(m.group(1), m.group(0))), @@ -214,27 +239,16 @@ def fallback_source_dirs(builddir=None): builddir = builddir or get_builddir() if builddir: - parent = os.path.dirname(builddir) - candidates.extend([ - os.path.join(parent, 'sources'), - os.path.join(parent, 'layers'), - os.path.join(parent, 'poky'), - os.path.join(parent, 'openembedded-core'), - ]) + candidates.extend(layer_subdirs(os.path.dirname(builddir))) - root = find_upwards(os.getcwd(), ['sources', 'layers', 'poky', - 'openembedded-core']) + root = find_upwards(os.getcwd(), LAYER_DIR_NAMES) if root: - candidates.extend([ - os.path.join(root, 'sources'), - os.path.join(root, 'layers'), - os.path.join(root, 'poky'), - os.path.join(root, 'openembedded-core'), - ]) + candidates.extend(layer_subdirs(root)) return existing_dirs(candidates) +@functools.lru_cache(maxsize=None) def source_dirs(): env_dirs = os.environ.get('YE_SOURCE_DIRS') if env_dirs: @@ -258,13 +272,7 @@ def source_root_dir(): dirs = source_dirs() if not dirs: return None - try: - common = os.path.commonpath(dirs) - if common and common != os.path.sep and os.path.isdir(common): - return os.path.realpath(common) - except ValueError: - pass - return dirs[0] + return common_dir(dirs, require_dir=True) or dirs[0] def workspace_root(): @@ -278,18 +286,14 @@ def workspace_root(): roots.append(builddir) roots.extend(source_dirs()) if roots: - try: - common = os.path.commonpath(roots) - if common and common != os.path.sep: - return os.path.realpath(common) - except ValueError: - pass + common = common_dir(roots) + if common: + return common for env_name in ['PLATFORM_ROOT_DIR', 'YE_TOPDIR', 'OEROOT', 'COREBASE']: if os.environ.get(env_name): return os.path.realpath(os.path.abspath(os.environ[env_name])) - return find_upwards(os.getcwd(), ['sources', 'layers', 'poky', - 'openembedded-core']) + return find_upwards(os.getcwd(), LAYER_DIR_NAMES) def find_git_root(path):