diff --git a/README.adoc b/README.adoc index b9fccc4..4f6fab6 100644 --- a/README.adoc +++ b/README.adoc @@ -9,10 +9,9 @@ the directory structure adopted by O.S. Systems to organize projects that use the Yocto Project. `ye` provides means to quickly locate files and directories into projects' directory tree, and act on them. -=== The directory layout assumed by ye +=== The directory layouts understood by ye -The infrastructure adopted by O.S. Systems assumes the following -directory layout: +`ye` supports the legacy O.S. Systems layout: .... @@ -31,6 +30,30 @@ of all projects (git repositories) will be stored. The directory layout in the `` directory follows the standard layout as determined by the Yocto Project. +For current Yocto Project releases, `ye` also understands projects +set up with `oe-init-build-env`, `OEROOT`, `/conf/bblayers.conf`, +and layer checkouts under either `sources/` or `layers/`. If automatic +detection is not enough, `YE_SOURCE_DIRS` can be set to a colon-separated +list of layer or source directories. + +In practice, `ye` determines the workspace in this order: + +. If `BUILDDIR` is set, use it as the build directory. +. Otherwise, walk upward from the current directory until + `conf/bblayers.conf` is found. +. If a repo workspace exists, use the directory containing `.repo` as + the project root. +. Prefer layer directories declared in `conf/bblayers.conf`. +. If layer detection fails, fall back to `YE_SOURCE_DIRS`, + `PLATFORM_ROOT_DIR`, `YE_TOPDIR`, `OEROOT`, `COREBASE`, and common + sibling directories such as `sources/`, `layers/`, `poky/`, and + `openembedded-core/`. + +Both single-quoted and double-quoted `BBLAYERS` values are supported. +This matters for current Yocto Project setups, whose generated +configuration files commonly use shell-like multi-line single-quoted +assignments. + === Notes on arguments, environment and interactive behavior Most `ye` commands accept a regular expression as argument. Thus, it's @@ -48,6 +71,99 @@ to find out things). This variable is automatically set by the `setup-environment` script provided by O.S. Systems for Yocto Project-based projects. +Commands that print matches are useful in scripts too. When stdout is +not a terminal, `ye` disables the interactive prompts and prints all +matches to stdout, except for shortcut commands such as `view`, +`pkg-info`, or `pkg-extract`, which perform their requested action +when a single unambiguous match exists. + + +== Installation and shell setup + +`ye` is shipped as an executable Python script. It does not require a +Python package installation step; put the checkout directory in +`PATH`, or create a symlink to the `ye` executable from a directory +already in `PATH`. + +Example: + +.... +$ export PATH=/path/to/ye:$PATH +$ ye --help +.... + +For shell completion, source `ye-completion.sh` from your shell +startup file: + +.... +$ . /path/to/ye-completion.sh +.... + +The `ye cd` command needs to affect the current shell process, so it +is implemented by the `ye-cd` shell wrapper. Source it after `ye` is +available in `PATH`: + +.... +$ . /path/to/ye-cd +.... + +O.S. Systems' `setup-environment` script can do this automatically +when `ye` is present in the source tree. In other Yocto setups, add +the `PATH`, completion, and `ye-cd` lines to your preferred shell +startup file or project environment setup script. + + +== Quick start + +. Enter or source a Yocto build environment: ++ +.... +$ source oe-init-build-env +.... ++ +or, in O.S. Systems workspaces: ++ +.... +$ source setup-environment +.... + +. Confirm `ye` can see the build and configured layers: ++ +.... +$ ye plumbing topdir +$ ye f -e layer.conf +.... + +. Find and open a recipe or configuration file: ++ +.... +$ ye f busybox +$ ye v conf/machine/qemuarm.conf +.... + +. Locate build output: ++ +.... +$ ye wd linux-yocto +$ ye pf 'kernel_.*\.ipk' +$ ye sf stdio.h +.... + +. Inspect a package: ++ +.... +$ ye pi 'kernel_.*\.ipk' +$ ye pv 'kernel-image.*\.ipk' +.... + +. Use directory shortcuts by sourcing `ye-cd`: ++ +.... +$ . /path/to/ye-cd +$ ye cd src +$ ye cd wd linux-yocto +.... + == Using ye @@ -62,12 +178,13 @@ Usage: ye [ ] f [-e] find [-e] - Locate paths (in the "sources" directory) that match the given - expression . is case insensitive and implicitly - surrounded by '.*'. -e disables the implicit use of '.*' around - the given . Note that, unless contains /, matching - is attempted on filenames only (not on dirnames). If - contains /, matching is attempted on the full path. + Locate paths in configured Yocto layer/source directories that + match the given expression . is case insensitive + and implicitly surrounded by '.*'. -e disables the implicit use + of '.*' around the given . Note that, unless + contains /, matching is attempted on filenames only (not on + dirnames). If contains /, matching is attempted on the + full path. v [-e] view [-e] @@ -125,7 +242,7 @@ pkg-extract [-e] [-c] [] the package filename (without extension). must match the exact file name in the package (usually starts with ./). -c is specific to .deb and .ipk packages -- ye will extract files - from the control.tar.gz tarball in packages. + from the control archive in packages. wd [-e] workdir [-e] @@ -137,8 +254,8 @@ log [-e] [-H] [] Show the log files for . -e is only applied to . is always implicitly surrounded by '.*', if provided. If -H ("human readable") is given on the - command line, ye will try to make the lines that contain calls - to gcc/g++ look more readable. If -R is provided, ye will apply + command line, ye will try to make the lines that contain calls to + clang/clang++/gcc/g++ look more readable. If -R is provided, ye will apply some text replacements to make the output more readable. Currently, ye reverse expands some common variables whose expansion pollutes log files with long paths. The following variables are reverse @@ -157,11 +274,12 @@ run [-e] [] g grep - Run 'repo grep '. + Run 'repo grep ' in repo-based workspaces, otherwise run + 'git grep ' across configured layers. sg sysroot-grep - Run 'grep -r $BUILDDIR/tmp/sysroots/$MACHINE'. + Run 'grep -r ' across available sysroot directories. glg [-n ] [-i] git-log-grep [-n ] [-i] @@ -214,14 +332,14 @@ cd [] Change to the sysroot directory for MACHINE src [] - Change to 's source dir or to TOPDIR/sources - if is not provided + Change to 's source dir or to the detected source/layers + directory if is not provided img Change to BUILDDIR/tmp/deploy/MACHINE/image/ pkg - Change to BUILDDIR/tmp/deploy/PKG_TYPE/image/ + Change to BUILDDIR/tmp/deploy/PKG_TYPE or its MACHINE subdir manifest Change to TOPDIR/.repo/manifests @@ -232,6 +350,98 @@ cd [] .... +=== Command summary + +[cols="1,2,4",options="header"] +|=== +|Short |Long |Purpose + +|`f` +|`find` +|Find files in configured Yocto layer/source directories. + +|`v` +|`view` +|Find one file and open it with the configured pager. + +|`e` +|`edit` +|Find one file and open it with the configured editor. + +|`sf` +|`sysroot-find` +|Find files in available sysroot directories. + +|`wsf` +|`work-shared-find` +|Find files in `tmp/work-shared`. + +|`pf` +|`pkg-find` +|Find packages under `tmp/deploy/{deb,ipk,rpm}`. + +|`pv` +|`pkg-view` +|List files contained in a package. + +|`pi` +|`pkg-info` +|Show package metadata. + +|`ps` +|`pkg-scripts` +|List package control scripts or control members. + +|`px` +|`pkg-extract` +|Extract package data files, or control files with `-c`. + +|`wd` +|`workdir` +|Print recipe work directories under `tmp/work`. + +|`l` +|`log` +|Open BitBake task logs. + +|`r` +|`run` +|Open BitBake task run scripts. + +|`g` +|`grep` +|Search configured layers with `repo grep`, `git grep`, or recursive `grep`. + +|`sg` +|`sysroot-grep` +|Search text under available sysroot directories. + +|`glg` +|`git-log-grep` +|Search git commit summaries across layer repositories. + +|`gbh` +|`grep-buildhistory` +|Run `git grep` in the buildhistory repository. + +|`bh` +|`buildhistory` +|Show recent buildhistory changes. + +|`d` +|`doc` +|Search Yocto Project variable documentation. + +|`x` +|`expand` +|Expand BitBake variables and show recursive expansion. + +|`cd` +|`cd` +|Change to common project/build directories through the `ye-cd` shell wrapper. +|=== + + === Finding and operating on files `ye` provides commands to locate files and operate on them. Some @@ -240,13 +450,16 @@ some file types (e.g., packages). The following sections provide a more in-depth explanation about them. -==== Finding and operating on files in the `sources` directory +==== Finding and operating on files in layer/source directories The `find` command (short: `f`) can be used to locate files under the -`sources` directory. It's argument is a regular expression that will -be matched against pathnames. If the given regex contains `/`, -matching is attempted on filenames only (not on dirnames). If the -given regex contains `/`, matching is attempted on the full path. +configured Yocto layers. `ye` gets those layers from `bblayers.conf` +when possible, otherwise it falls back to legacy `sources/`, modern +`layers/`, `OEROOT` or `YE_SOURCE_DIRS`. Its argument is a regular +expression that will be matched against pathnames. If the given regex +does not contain `/`, matching is attempted on filenames only (not on +dirnames). If the given regex contains `/`, matching is attempted on +the full path. After locating files that match the given pattern, `ye` will prompt you to select one of the matches and, next, what to do with it. In @@ -341,7 +554,10 @@ for the one you really want to see. The `sysroot-find` (short: `sf`) command is pretty much equivalent to the `find` command, except it locates files under the sysroots -directory (`/tmp/sysroots`). +directories. Current Yocto releases primarily use +`/tmp/sysroots-components` and per-recipe +`recipe-sysroot` directories; older `/tmp/sysroots` trees +are still used as a fallback when present. ==== Finding and displaying files in the `work-shared` directory @@ -359,6 +575,25 @@ packages (`/tmp/deploy/`). `ye` supports the most common package formats generated by Yocto Project: `.ipk`, `.deb` and `.rpm`. +For `.ipk` and `.deb` packages, `ye` reads the package as an `ar` +archive and then processes the embedded `data.tar.*` or +`control.tar.*` member. The following tar payload formats are +supported: + +* `data.tar` +* `data.tar.gz` +* `data.tar.xz` +* `data.tar.zst` +* `control.tar` +* `control.tar.gz` +* `control.tar.xz` +* `control.tar.zst` + +The zstd variants are important for Scarthgap, Wrynose, and newer +Yocto Project releases where IPK package payloads can use +`data.tar.zst`. RPM packages are handled through `rpm` and +`rpm2cpio`. + The actions for packages are different from the `find` command. `ye` supports the following actions on packages: @@ -371,6 +606,10 @@ supports the following actions on packages: `extract`:: extract package contents to a directory named after the package filename +The `pkg-extract` command refuses to overwrite an existing extraction +directory. Run it from a scratch directory if you want to inspect a +package without modifying the build tree. + Just like the `view` and `edit` counterparts to the `find` command, `ye` provides `pkg-view` (short: `pv`), `pkg-info` (short: `pi`), `pkg-scripts` (short: `ps`) and `pkg-extract` (short: `px`) command @@ -530,8 +769,9 @@ Option (ENTER to cancel): 4 .... The `log` command also handles the `-H` option, which tries to make -compiler command lines more readable (and numbers them). See some -examples below: +compiler command lines more readable (and numbers them). It +recognizes common C/C++ compiler invocations such as `gcc`, `g++`, +`clang`, and `clang++`. See some examples below: Without `-H`: @@ -648,19 +888,15 @@ detail. ==== Finding text in source files -The `grep` command is a thin wrapper around `repo grep` (`repo` is the -tool used by O.S. Systems to manage multiple git repositories -- see -the http://doc.ossystems.com.br/managing-platforms.html[Managing -platforms based on the Yocto Project] document for more information). -Basically, `repo grep ` will run `git grep ` on -each repository (in the `sources` directory) which is part of the -project. +The `grep` command searches configured Yocto layers. In legacy +repo-based workspaces it uses `repo grep`; otherwise it runs `git grep` +across detected layer repositories. If the layers are not git +repositories, it falls back to recursive `grep`. -The `grep` command will run `repo grep` plus the arguments provided on -the command line (any valid argument for `git grep`) and will prompt -you to select one of the matches, then the action to apply on the -selected file. In case of a single match, you'll be only prompted for -the action. +The `grep` command will pass the arguments provided on the command +line to the underlying grep command and will prompt you to select one +of the matches, then the action to apply on the selected file. In +case of a single match, you'll be only prompted for the action. Example: @@ -683,9 +919,8 @@ Option (ENTER to cancel): 2v ==== Finding text in files in the sysroot directory The `sysroot-grep` (short: `sg`) command is similar to the `grep` -command, but instead of searching for matches in the `sources` -directory, it recursively searches for matches in the `sysroot` -directory (`/tmp/sysroots/`, specifically). +command, but instead of searching for matches in source directories, +it recursively searches for matches in available sysroot directories. Example: @@ -709,7 +944,7 @@ Option (ENTER to cancel): 1v The `git-log-grep` command (short: `glg`) basically runs .... -git log --oneline | grep " +git log --oneline | grep .... for the given regular expressions on the summary lines of all git @@ -759,7 +994,7 @@ index f4ea800..0c92ff9 100644 ==== Finding text in the buildhistory repository -The `grep-buildhistory` commmand (short: `gbh`) is a wrapper around +The `grep-buildhistory` command (short: `gbh`) is a wrapper around `git grep` in the buildhistory directory. Example: @@ -824,7 +1059,7 @@ Option (ENTER to cancel): 3 .... `ye` maintains a cache of the Yocto project Reference Manual for seven -days (under `$YE_DIR/doc-data`). If the cache is older than seven +days (under `$HOME/.ye/doc-data`). If the cache is older than seven days, it will fetch the reference manual data and update the cache. @@ -852,10 +1087,16 @@ STAGING_DIR_TARGET ==> ${STAGING_DIR}/${MACHINE} .... -Except for the `find` and `grep` commands, all commands expect the -`BUILDDIR` environment variable to be set in the environment. This -variable is automatically set by the `setup-environment` script -provided by O.S. Systems for the Yocto Project-based projects. +Most build-output commands expect the `BUILDDIR` environment variable +to be set in the environment. This variable is set by the usual Yocto +environment setup scripts. If it is not set, `ye` will also try to +detect a build directory by walking up from the current directory until +it finds `conf/bblayers.conf`. + +The `expand` command starts BitBake through Tinfoil. It therefore has +the same constraints as other BitBake clients: it needs a valid build +environment and may fail if another process is holding the same +BitBake server in an incompatible state. === Moving around: changing directories @@ -905,16 +1146,48 @@ will do that automatically if you have `ye` in your source tree. == Configuration -`ye` allows you to customize the pager and the editor it uses for -displaying and editing files, respectively. +`ye` is configured through environment variables. + +[cols="1,4",options="header"] +|=== +|Variable |Meaning -The configuration is via environment variables. `ye` uses `YE_PAGER` -and `YE_EDITOR` for pager and editor, respectively. +|`BUILDDIR` +|Yocto build directory. If unset, `ye` tries to find a parent directory containing `conf/bblayers.conf`. + +|`YE_SOURCE_DIRS` +|Colon-separated list of layer/source directories. This overrides automatic source discovery. + +|`PLATFORM_ROOT_DIR` +|Project root hint used by O.S. Systems environments. + +|`YE_TOPDIR` +|Project root hint used when automatic discovery is not enough. + +|`OEROOT` +|OpenEmbedded-Core or Poky root hint, usually set by `oe-init-build-env`. + +|`COREBASE` +|OpenEmbedded-Core base directory hint. + +|`YE_PAGER` +|Command template used to view files. Use `%s` where the filename should be inserted. + +|`PAGER` +|Fallback pager when `YE_PAGER` is not set. + +|`YE_EDITOR` +|Command used to edit files. + +|`EDITOR` +|Fallback editor when `YE_EDITOR` is not set. +|=== For the editor, `ye` first checks if `YE_EDITOR` is set in the environment. If it is not set, it checks the `EDITOR` environment -variable. If it is not set, it resorts to `emacs`. If `emacs` cannot -be found, you'll get an error. +variable. If neither is set, it searches for a usable terminal editor +in this order: `emacs -nw`, `mg`, `zile`, `qemacs`, `nano`, `vim`, +and `vi`. If no editor is found, `ye` prints an error. For the pager, `ye` first checks if `YE_PAGER` is set in the environment. If it is not set, it checks the `PAGER` environment @@ -923,15 +1196,63 @@ cannot be found, you'll get an error. `%s` can be used as a placeholder for the file to act upon. +`ye` stores cached documentation and shell plumbing files below +`$HOME/.ye`. + + +== Compatibility with current Yocto releases + +`ye` is intended to work with both older O.S. Systems Yocto layouts +and current Yocto Project layouts. The compatibility-sensitive areas +are source/layer discovery, sysroot discovery, and package archive +format support. + +Kirkstone 4.0 LTS:: + Compatible for the tested surfaces. The older `tmp/sysroots` + layout remains supported as a fallback. + +Scarthgap 5.0 LTS:: + Compatible for source discovery, workdir lookup, sysroot lookup, and + package discovery. IPK zstd payloads are supported through + `data.tar.zst`. + +Wrynose 6.0 LTS:: + Compatible for the tested surfaces in a Wrynose build tree using + `PACKAGE_CLASSES = "package_ipk"`. `ye` correctly handles + configured layers from single-quoted `BBLAYERS`, per-recipe + sysroots, `tmp/sysroots-components`, and zstd-compressed IPK + payloads. + +When migrating between Yocto releases, pay special attention to: + +* package payload compression (`data.tar.zst` for modern IPK output); +* source layout changes around `WORKDIR` and `UNPACKDIR`; +* whether the build setup still creates `conf/bblayers.conf`; +* whether package, image, sysroot, and buildhistory directories exist + in the expected build directory. + == Requirements -A Python installation and the directory structure in the layout -created by O.S. System's Yocto Project-based platforms. +A Python 3 installation and a Yocto Project build tree are required. +Use the Python version supported by your Yocto release. `ye` supports +both the legacy O.S. Systems `.repo`/`sources` layout and current +Yocto layouts that expose layers through `conf/bblayers.conf`. + +Core host tools: -`ye` has been more extensively tested with Python version 2.7.3, but -it should work with other recent Python 2.x versions and with Python -3.x. +* `git` +* `grep` +* `less` or another pager +* a terminal editor for `ye edit` +* `ar` and GNU `tar` for `.deb` and `.ipk` inspection +* `zstd` or GNU `tar` with zstd support for modern `.ipk` payloads +* `rpm` and `rpm2cpio` for RPM package inspection/extraction + +Optional host tools: + +* `repo`, for legacy repo-based workspace grep support +* `buildhistory-diff`, for formatted buildhistory output For the `doc` command, the http://lxml.de/[lxml] module for Python is required. @@ -939,13 +1260,81 @@ required. For the `cd` command, a Bourne-compatible shell is required. +== Testing and validation + +The lightweight checks below are useful before changing `ye`: + +.... +$ python3 -m py_compile ye ye_compat.py bb-expand-vars.py format-command-lines.py setup-environment.d/ye.py +$ bash -n ye-cd +$ bash -n ye-completion.sh +.... + +Package archive selection is covered by `test_ye_packages.py`: + +.... +$ python3 -B test_ye_packages.py +.... + +Useful smoke tests in a real build tree: + +.... +$ ye --help +$ ye plumbing topdir +$ ye f -e layer.conf +$ ye wd -e linux-yocto +$ ye pf -e 'kernel_.*\.ipk' +$ ye pi -e 'kernel_.*\.ipk' +$ ye pv -e 'kernel-image.*\.ipk' +$ ye sf stdio.h +.... + +Commands that call BitBake, such as `ye expand` and `ye plumbing x`, +should be treated as integration tests. They need a working BitBake +server and a valid build environment. + + +== Troubleshooting + +`ye` searches too many files or files from unconfigured layers:: + Check `conf/bblayers.conf` and `YE_SOURCE_DIRS`. `ye` prefers + configured `BBLAYERS`; if parsing fails or no layer exists, it falls + back to broader source directories. + +`ERROR: Could not determine the Yocto Project's build directory.`:: + Run from inside the build directory, set `BUILDDIR`, or source the + Yocto setup script again. + +`pkg-view` reports an unsupported compression type:: + Confirm the package members with `ar t `. Supported + `.deb` and `.ipk` data/control archives are uncompressed tar, gzip, + xz, and zstd tar files. + +`pkg-extract` refuses to run because the output directory exists:: + `ye` never clobbers extraction directories. Remove or rename the + existing directory, or run the command from a scratch directory. + +`doc` cannot update documentation data:: + The `doc` command needs Python `lxml` and network access when its + cache under `$HOME/.ye/doc-data` is missing or older than seven + days. + +`expand` or `plumbing x` cannot start BitBake:: + Ensure the Yocto environment is sourced and that no incompatible + BitBake server is active for the same build directory. + + == Limitations Some `ye` commands use `bitbake` behind the scenes, and since -`bitbake` doesn't support running multipl instances in parallel under +`bitbake` doesn't support running multiple instances in parallel under the same build directory, some `ye` features may not work while you are using `bitbake`. +`ye` is a navigation and inspection helper. It does not replace +BitBake, `devtool`, package managers, or release migration tools; it +uses the build tree that those tools create. + == License diff --git a/bb-expand-vars.py b/bb-expand-vars.py index bf51c76..7cd22bf 100644 --- a/bb-expand-vars.py +++ b/bb-expand-vars.py @@ -1,15 +1,8 @@ -#! /usr/bin/env python +#! /usr/bin/env python3 # -*- encoding: utf-8 -*- - -""" From https://github.com/kergoth/bb/blob/master/libexec/bbcmd.py """ - -import re -import argparse -import contextlib -import logging import os import sys -import warnings +import ye_compat PATH = os.getenv('PATH').split(':') bitbake_paths = [os.path.join(path, '..', 'lib') @@ -19,186 +12,43 @@ sys.path[0:0] = bitbake_paths -import bb.msg -import bb.utils -import bb.providers -import bb.tinfoil -from bb.cookerdata import CookerConfiguration, ConfigParameters - - -class Terminate(BaseException): - pass - - -class Tinfoil(bb.tinfoil.Tinfoil): - def __init__(self, output=sys.stdout): - # Needed to avoid deprecation warnings with python 2.6 - warnings.filterwarnings("ignore", category=DeprecationWarning) - - # Set up logging - self.logger = logging.getLogger('BitBake') - if output is not None: - setup_log_handler(self.logger, output) - - self.config = self.config = CookerConfiguration() - configparams = bb.tinfoil.TinfoilConfigParameters(parse_only=True) - self.config.setConfigParameters(configparams) - self.config.setServerRegIdleCallback(self.register_idle_function) - self.cooker = bb.cooker.BBCooker(self.config) - self.config_data = self.cooker.data - bb.providers.logger.setLevel(logging.ERROR) - bb.taskdata.logger.setLevel(logging.CRITICAL) - self.cooker_data = None - self.taskdata = None - - self.localdata = bb.data.createCopy(self.config_data) - self.localdata.finalize() - # TODO: why isn't expandKeys a method of DataSmart? - bb.data.expandKeys(self.localdata) - - - def prepare_taskdata(self, provided=None, rprovided=None): - self.cache_data = self.cooker.recipecache - if self.taskdata is None: - self.taskdata = bb.taskdata.TaskData(abort=False) - - if provided: - self.add_provided(provided) - - if rprovided: - self.add_rprovided(rprovided) - - def add_rprovided(self, rprovided): - for item in rprovided: - self.taskdata.add_rprovider(self.localdata, self.cache_data, item) - - self.taskdata.add_unresolved(self.localdata, self.cache_data) - - def add_provided(self, provided): - if 'world' in provided: - if not self.cache_data.world_target: - self.cooker.buildWorldTargetList() - provided.remove('world') - provided.extend(self.cache_data.world_target) - - if 'universe' in provided: - provided.remove('universe') - provided.extend(self.cache_data.universe_target) - - for item in provided: - self.taskdata.add_provider(self.localdata, self.cache_data, item) - - self.taskdata.add_unresolved(self.localdata, self.cache_data) - - def get_buildid(self, target): - if not self.taskdata.have_build_target(target): - if target in self.cooker.recipecache.ignored_dependencies: - return - - reasons = self.taskdata.get_reasons(target) - if reasons: - self.logger.error("No buildable '%s' recipe found:\n%s", target, "\n".join(reasons)) - else: - self.logger.error("No '%s' recipe found", target) - return - else: - return self.taskdata.getbuild_id(target) - - def target_filenames(self): - """Return the filenames of all of taskdata's targets""" - filenames = set() - - for targetid in self.taskdata.build_targets: - fnid = self.taskdata.build_targets[targetid][0] - fn = self.taskdata.fn_index[fnid] - filenames.add(fn) - - for targetid in self.taskdata.run_targets: - fnid = self.taskdata.run_targets[targetid][0] - fn = self.taskdata.fn_index[fnid] - filenames.add(fn) - - return filenames - - def all_filenames(self): - return self.cooker.recipecache.file_checksums.keys() - - def all_preferred_filenames(self): - """Return all the recipes we have cached, filtered by providers. - - Unlike target_filenames, this doesn't operate against taskdata. - """ - filenames = set() - excluded = set() - for provide, fns in self.cooker.recipecache.providers.iteritems(): - eligible, foundUnique = bb.providers.filterProviders(fns, provide, - self.localdata, - self.cooker.recipecache) - preferred = eligible[0] - if len(fns) > 1: - # Excluding non-preferred providers in multiple-provider - # situations. - for fn in fns: - if fn != preferred: - excluded.add(fn) - filenames.add(preferred) - - filenames -= excluded - return filenames - - def provide_to_fn(self, provide): - """Return the preferred recipe for the specified provide""" - filenames = self.cooker.recipecache.providers[provide] - eligible, foundUnique = bb.providers.filterProviders(filenames, provide, self.localdata) - return eligible[0] - - def build_target_to_fn(self, target): - """Given a target, prepare taskdata and return a filename""" - self.prepare_taskdata([target]) - targetid = self.get_buildid(target) - if targetid is None: - return - fnid = self.taskdata.build_targets[targetid][0] - fn = self.taskdata.fn_index[fnid] - return fn - - def parse_recipe_file(self, recipe_filename): - """Given a recipe filename, do a full parse of it""" - appends = self.cooker.collection.get_file_appends(recipe_filename) - try: - recipe_data = bb.cache.Cache.loadDataFull(recipe_filename, - appends, - self.config_data) - except Exception: - raise - return recipe_data - - def parse_metadata(self, recipe=None): - """Return metadata, either global or for a particular recipe""" - if recipe: - self.prepare_taskdata([recipe]) - filename = self.build_target_to_fn(recipe) - return self.parse_recipe_file(filename) - else: - return self.localdata - - -def setup_log_handler(logger, output=sys.stderr): - log_format = bb.msg.BBLogFormatter("%(levelname)s: %(message)s") - if output.isatty() and hasattr(log_format, 'enable_color'): - log_format.enable_color() - handler = logging.StreamHandler(output) - handler.setFormatter(log_format) - - bb.msg.addDefaultlogFilter(handler) - logger.addHandler(handler) - logger.setLevel(logging.INFO) - +def find_yocto_root(): + return ye_compat.workspace_root() + +def get_yocto_path(): + candidates = [] + for env_name in ['OEROOT', 'COREBASE']: + if os.environ.get(env_name): + candidates.append(os.environ[env_name]) + + root = find_yocto_root() + if root: + candidates.extend([ + os.path.join(root, 'poky'), + os.path.join(root, 'openembedded-core'), + os.path.join(root, 'sources', 'poky'), + os.path.join(root, 'sources', 'openembedded-core'), + os.path.join(root, 'sources', 'oe'), + ]) + + for layer in ye_compat.source_dirs(): + candidates.extend([ + layer, + os.path.dirname(layer), + os.path.dirname(os.path.dirname(layer)), + ]) + + paths = ye_compat.existing_dirs( + [os.path.join(path, 'scripts', 'lib') for path in candidates]) + if len(paths) < 1: + print("ERROR: Can't find scripts path") + sys.exit(1) + sys.path.append(paths[0]) -def sigterm_exception(signum, stackframe): - raise Terminate() +basepath = '' -###### end of bbcmd +get_yocto_path() +from devtool import setup_tinfoil # For printing indented expression expansions indent_step = 4 @@ -242,19 +92,19 @@ def next(pos, expr): def parse_expr(expr): if expr[0] == '$': - return BBVar(map(parse_expr, tokenize_expr(expr[2:-1]))) + return BBVar([parse_expr(token) for token in tokenize_expr(expr[2:-1])]) else: return expr def parse_exprs(exprs): - return map(parse_expr, exprs) + return [parse_expr(expr) for expr in exprs] def unparse_expr(expr): if isinstance(expr, list): - return ''.join(map(unparse_expr, expr)) + return ''.join(unparse_expr(exp) for exp in expr) elif isinstance(expr, BBVar): if isinstance(expr.name, list): - return ''.join(map(unparse_expr, expr.name)) + return ''.join(unparse_expr(exp) for exp in expr.name) else: return expr.name else: @@ -266,7 +116,7 @@ def show_expansion(var, val, indent): margin = '' else: margin = ' ' * margin_spacing - print('%s%s ==> %s' % (margin, unparse_expr(var), val)) + print(('%s%s ==> %s' % (margin, unparse_expr(var), val))) def get_var_val(var, metadata): val = None @@ -305,45 +155,53 @@ def expand_var(var, metadata, indent): return var def expand_vars(vars, metadata, indent=0): - return ''.join(map(lambda v: expand_var(v, metadata, indent), vars)) + return ''.join([expand_var(v, metadata, indent) for v in vars]) def expand_expr(expr, metadata): return expand_vars(parse_exprs(tokenize_expr(expr)), metadata) def show_var_expansions(recipe, var, plumbing_mode=False): # When plumbing_mode is truthy, var is a list of variables - - ## tinfoil sets up log output for the bitbake loggers, but bb uses - ## a separate namespace at this time - setup_log_handler(logging.getLogger('bb')) - - tinfoil = Tinfoil(output=sys.stderr) - tinfoil.prepare(config_only=True) - - tinfoil.parseRecipes() - + tinfoil = None try: - metadata = tinfoil.parse_metadata(recipe) - except: - sys.exit(1) + tinfoil = setup_tinfoil(config_only=True, basepath=basepath) + metadata = getattr(tinfoil, 'config_data', None) + if recipe: + if hasattr(tinfoil, 'parseRecipes'): + tinfoil.parseRecipes() + elif hasattr(tinfoil, 'parse_recipes'): + tinfoil.parse_recipes() + + try: + metadata = tinfoil.parse_recipe(recipe) + except: + sys.exit(1) + + if metadata is None: + sys.stderr.write('Could not load BitBake metadata.\n') + sys.exit(1) + + if plumbing_mode: + vars_vals = {} + for v in var: + vars_vals[v] = metadata.getVar(v, True) + return vars_vals + else: + val = metadata.getVar(var, True) - if plumbing_mode: - vars_vals = {} - for v in var: - vars_vals[v] = metadata.getVar(v, True) - return vars_vals - else: - val = metadata.getVar(var, True) + if val is not None: + print('=== Final value') + print( '%s = %s' % (var, val)) - if val is not None: - print '=== Final value' - print '%s = %s' % (var, val) + print('\n=== Expansion') + expand_expr('${' + var + '}', metadata) + else: + sys.stderr.write('%s: no such variable.\n' % var) + sys.exit(1) - print '\n=== Expansion' - expand_expr('${' + var + '}', metadata) - else: - sys.stderr.write('%s: no such variable.\n' % var) - sys.exit(1) + finally: + if tinfoil: + tinfoil.shutdown() if __name__ == '__main__': diff --git a/setup-environment.d/ye.py b/setup-environment.d/ye.py index a60f481..7e10d07 100644 --- a/setup-environment.d/ye.py +++ b/setup-environment.d/ye.py @@ -1,6 +1,16 @@ +import os + + def __after_init_ye_yocto(): PLATFORM_ROOT_DIR = os.environ['PLATFORM_ROOT_DIR'] - yedir = os.path.join(PLATFORM_ROOT_DIR, 'sources', 'ye') - os.environ['PATH'] = os.environ['PATH'] + ':' + yedir + candidates = [ + os.path.join(PLATFORM_ROOT_DIR, 'sources', 'ye'), + os.path.join(PLATFORM_ROOT_DIR, 'layers', 'ye'), + os.path.join(PLATFORM_ROOT_DIR, 'ye'), + ] + for yedir in candidates: + if os.path.exists(os.path.join(yedir, 'ye')): + os.environ['PATH'] = os.environ['PATH'] + ':' + yedir + break run_after_init(__after_init_ye_yocto) diff --git a/test_ye_packages.py b/test_ye_packages.py new file mode 100644 index 0000000..72e8afb --- /dev/null +++ b/test_ye_packages.py @@ -0,0 +1,50 @@ +import importlib.machinery +import importlib.util +import os +import pathlib +import unittest +from unittest import mock + + +def load_ye(): + os.environ.setdefault('YE_EDITOR', 'true') + path = pathlib.Path(__file__).with_name('ye') + loader = importlib.machinery.SourceFileLoader('ye_module', str(path)) + spec = importlib.util.spec_from_loader(loader.name, loader) + module = importlib.util.module_from_spec(spec) + loader.exec_module(module) + return module + + +class PackageArchiveTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.ye = load_ye() + + def test_pkg_tarball_selects_gzip_data_archive(self): + members = ['debian-binary', 'control.tar.gz', 'data.tar.gz'] + with mock.patch.object(self.ye, 'ar_members', return_value=members): + self.assertEqual(self.ye.pkg_tarball('package.ipk', 'data'), + 'data.tar.gz') + + def test_pkg_tarball_selects_xz_data_archive(self): + members = ['debian-binary', 'control.tar.gz', 'data.tar.xz'] + with mock.patch.object(self.ye, 'ar_members', return_value=members): + self.assertEqual(self.ye.pkg_tarball('package.ipk', 'data'), + 'data.tar.xz') + + def test_pkg_tarball_selects_zstd_data_archive(self): + members = ['debian-binary', 'control.tar.gz', 'data.tar.zst'] + with mock.patch.object(self.ye, 'ar_members', return_value=members): + self.assertEqual(self.ye.pkg_tarball('package.ipk', 'data'), + 'data.tar.zst') + + def test_pkg_tarball_selects_compressed_control_archive(self): + members = ['debian-binary', 'control.tar.zst', 'data.tar.zst'] + with mock.patch.object(self.ye, 'ar_members', return_value=members): + self.assertEqual(self.ye.pkg_tarball('package.ipk', 'control'), + 'control.tar.zst') + + +if __name__ == '__main__': + unittest.main() diff --git a/ye b/ye index f9b2674..e46863e 100755 --- a/ye +++ b/ye @@ -1,8 +1,8 @@ -#! /usr/bin/env python +#! /usr/bin/env python3 ## ye: Yocto Explorer ## -## Copyright (C) 2014, 2015 O.S. Systems Software LTDA +## Copyright (C) 2014, 2015, 2017 O.S. Systems Software LTDA ## ## This program is free software: you can redistribute it and/or modify ## it under the terms of the GNU Affero General Public License as @@ -20,16 +20,15 @@ ### TODO # Implement exclusion pattern for grep (option should not clash with git grep's) -from __future__ import print_function - import os import re import sys import time import glob -import urllib +import urllib.request, urllib.parse, urllib.error import locale import subprocess +import ye_compat # lxml is required for parsing the documentation XML files is_lxml_available = True @@ -38,7 +37,10 @@ try: except: is_lxml_available = False -encoding = locale.getdefaultlocale()[1] +try: + encoding = locale.getencoding() +except AttributeError: + encoding = locale.getdefaultlocale()[1] if not encoding: sys.stderr.write('WARNING: could not determine locale. Forcing UTF-8.\n') encoding = 'UTF-8' @@ -47,14 +49,6 @@ def die(msg): sys.stderr.write(msg + '\n') sys.exit(1) -def is_program_available(program): - progpaths = os.environ['PATH'].split(':') - for dir in progpaths: - progpath = os.path.join(dir, program) - if os.path.isfile(progpath) and os.access(progpath, os.X_OK): - return True - return False - YE_DIR = os.path.join(os.environ['HOME'], '.ye') EDITOR_ALTERNATIVES = [('emacs', '-nw'), 'mg', 'zile', 'qemacs', 'nano', 'vim', 'vi'] @@ -68,7 +62,7 @@ def find_editor(): args = option[1] else: editor = option - if is_program_available(editor): + if ye_compat.is_program_available(editor): return '%s %s' % (editor, args) die('Could not find an editor. Please, set the YE_EDITOR environment variable.') @@ -91,7 +85,7 @@ YE_EDITOR += ' "%s"' BUILDDIR = None try: - BUILDDIR = os.environ['BUILDDIR'] + BUILDDIR = ye_compat.get_builddir() except: pass @@ -109,13 +103,16 @@ bold = '\033[1m{}\033[0m' def colorize_layer(path): path_parts = path.split('/') - sources_idx = None + source_idx = None try: - sources_idx = path_parts.index('sources') + source_idx = path_parts.index('sources') except ValueError: - return path - if sources_idx < (len(path_parts) - 2): - layer_idx = sources_idx + 1 + try: + source_idx = path_parts.index('layers') + except ValueError: + source_idx = None + if source_idx is not None and source_idx < (len(path_parts) - 2): + layer_idx = source_idx + 1 path_parts[layer_idx] = bold.format(path_parts[layer_idx]) return '/'.join(path_parts) @@ -141,44 +138,44 @@ def download_file(uri, output_file): if dir and not os.path.exists(dir): os.makedirs(dir) try: - uri_fd = urllib.urlopen(uri) + uri_fd = urllib.request.urlopen(uri) except: die('Could not download %s. Aborting' % uri) - with open(output_file, 'w') as output_fd: + with open(output_file, 'wb') as output_fd: output_fd.write(uri_fd.read()) def find_yocto_root(): - def inner_find(dir): - repo_dir = os.path.join(dir, '.repo') - if os.path.exists(repo_dir) and os.path.isdir(repo_dir): - return dir - elif dir == '/': - return False - else: - return inner_find(os.path.dirname(dir)) - if BUILDDIR: - yocto_root = inner_find(BUILDDIR) - else: - yocto_root = inner_find(os.getcwd()) + yocto_root = ye_compat.workspace_root() return yocto_root or die("ERROR: won't search from /.") +def find_source_dirs(): + dirs = ye_compat.source_dirs() + return dirs or die("ERROR: could not determine Yocto layer/source directories.") + + def find_files(basedir, maxdepth=None, pattern='.*'): - depth = 0 + if isinstance(basedir, list): + results = [] + for dir in basedir: + results.extend(find_files(dir, maxdepth, pattern)) + return results re_pattern = re.compile(pattern) - re_skip_pattern = re.compile('.*~$|^\.#|^#.*#$|.*\.pyc$') + re_skip_pattern = re.compile(r'.*~$|^\.#|^#.*#$|.*\.pyc$') results = [] match_whole_path = '/' in pattern for root, dirs, files in os.walk(basedir): dirs[:] = [d for d in dirs if not d[0] == '.'] - depth += 1 - if maxdepth and depth > maxdepth: - return results + visible_dirs = list(dirs) + relroot = os.path.relpath(root, basedir) + depth = 1 if relroot == '.' else relroot.count(os.sep) + 2 + if maxdepth and depth >= maxdepth: + dirs[:] = [] all_files = None if match_whole_path: - all_files = [ os.path.join(root, f) for f in dirs + files ] + all_files = [ os.path.join(root, f) for f in visible_dirs + files ] else: - all_files = dirs + files + all_files = visible_dirs + files for file in all_files: if not re_skip_pattern.match(file) and re_pattern.match(file): results.append(os.path.join(root, file)) @@ -214,7 +211,7 @@ def prompt(options, option_formatter=lambda x: x, valid_shortcuts=[]): for i, text in enumerate(options): text_options += '[%d] %s [%d]\n' % (i, option_formatter(text), i) paginate(text_options) - print('Option (ENTER to cancel): ', end='') + print('Option (ENTER to cancel): ') return input() while True: @@ -229,7 +226,7 @@ def prompt_action(options): def inner_prompt(): for short, text in options: print('[%s] %s' % (short, text)) - print('Option (ENTER to cancel): ', end='') + print('Option (ENTER to cancel): ') return input() short_options = [ o[0] for o in options ] @@ -276,17 +273,7 @@ def find(pattern, basedir, exact, actions): else: file = results[0] print(colorize(shorten_path(file), orig_pattern)) - if len(actions) == 1: - actions[0][2](file) - return file - if shortcut: - for action in actions: - if action[1] == shortcut: - action[2](file) - else: - action = prompt_action([ (a[1], a[0]) for a in actions ]) - actions[action][2](file) - return file + return apply_action(file, actions, shortcut) else: if len(results) == 1 and len(actions) == 1: actions[0][2](results[0]) @@ -340,18 +327,73 @@ def print_matches(matches, plumbing_cmd=None): dest = sys.stdout if matches.__class__.__name__ == 'list': for match in matches: - dest.write(match.encode(encoding) + '\n') + dest.write(match.encode(encoding).decode() + '\n') else: # a string - dest.write(matches.encode(encoding) + '\n') + dest.write(matches.encode(encoding).decode() + '\n') if plumbing_cmd: dest.close() +TAR_COMPRESSION_ARGS = [ + ('.tar.gz', ['--gzip']), + ('.tar.xz', ['--xz']), + ('.tar.zst', ['--zstd']), + ('.tar', []), +] + + +def tar_compression_args(tarball): + for suffix, args in TAR_COMPRESSION_ARGS: + if tarball.endswith(suffix): + return args + return None + + +def ar_members(pkg_file): + try: + output = subprocess.check_output(['ar', 't', pkg_file], + universal_newlines=True) + except subprocess.CalledProcessError: + die("ERROR: could not read package file: %s" % pkg_file) + return [line.strip() for line in output.splitlines() if line.strip()] + + +def pkg_tarball(pkg_file, archive_name): + ext = os.path.splitext(pkg_file)[1] + prefix = archive_name + '.tar' + unsupported = [] + for member in ar_members(pkg_file): + if member == prefix or member.startswith(prefix + '.'): + if tar_compression_args(member) is not None: + return member + unsupported.append(member) + if unsupported: + die("ERROR: Compression type not supported for %s" % ext) + die("ERROR: %s archive not found in %s" % (archive_name, ext)) + + +def run_ar_tar(pkg_file, tarball, tar_args): + compression_args = tar_compression_args(tarball) + if compression_args is None: + die("ERROR: Compression type not supported for %s" % + os.path.splitext(pkg_file)[1]) + + ar = subprocess.Popen(['ar', 'p', pkg_file, tarball], + stdout=subprocess.PIPE) + tar = subprocess.Popen(['tar'] + compression_args + tar_args, + stdin=ar.stdout) + ar.stdout.close() + tar_status = tar.wait() + ar_status = ar.wait() + return ar_status or tar_status + + def pkg_view(file): ext = os.path.splitext(file)[1] - if ext in ['.ipk', '.deb']: - os.system("ar p %s data.tar.gz | tar tvzf -" % file) + if ext in [ '.deb', '.ipk' ]: + tarball = pkg_tarball(file, 'data') + run_ar_tar(file, tarball, ['-tvf', '-']) elif ext == '.rpm': os.system("rpm -qlvp %s" % file) else: @@ -361,7 +403,8 @@ def pkg_view(file): def pkg_info(file): ext = os.path.splitext(file)[1] if ext in ['.ipk', '.deb']: - os.system("ar p %s control.tar.gz | tar xzf - ./control -O" % file) + tarball = pkg_tarball(file, 'control') + run_ar_tar(file, tarball, ['-xOf', '-', './control']) elif ext == '.rpm': os.system("rpm -qip %s" % file) else: @@ -371,7 +414,8 @@ def pkg_info(file): def pkg_scripts(file): ext = os.path.splitext(file)[1] if ext in ['.ipk', '.deb']: - os.system("ar p %s control.tar.gz | tar tzf -" % file) + tarball = pkg_tarball(file, 'control') + run_ar_tar(file, tarball, ['-tf', '-']) else: die("ERROR: unsupported package file extension: %s" % ext) @@ -381,16 +425,19 @@ def pkg_extract(pkg_file, file, from_control): out_dir = os.path.basename(out_dir) if os.path.exists(out_dir): die("%s already exists. Won't clobber. Aborting." % out_dir) - os.makedirs(out_dir) - if ext in ['.ipk', '.deb']: - tarball = 'data.tar.gz' - if from_control: - tarball = 'control.tar.gz' - status = os.system("ar p %s %s | tar xzf - %s -C %s" % (pkg_file, tarball, file, out_dir)) + if ext in [ '.deb', '.ipk' ]: + archive_name = 'control' if from_control else 'data' + tarball = pkg_tarball(pkg_file, archive_name) + os.makedirs(out_dir) + tar_args = ['-xf', '-', '-C', out_dir] + if file: + tar_args.append(file) + status = run_ar_tar(pkg_file, tarball, tar_args) if not status: print('Extracted to %s' % out_dir) elif ext == '.rpm': - if is_program_available('rpm2cpio'): + os.makedirs(out_dir) + if ye_compat.is_program_available('rpm2cpio'): os.chdir(out_dir) status = os.system("rpm2cpio %s | cpio -id %s" % (pkg_file, file)) if not status: @@ -469,7 +516,7 @@ def list_temp_files(file_type, recipe_pattern, log_pattern, exact, raw, apply_re version_dir = version_dirs[0] log_dir = os.path.join(version_dir, 'temp') all_files = find_files(log_dir, - pattern='%s\..*%s.*' % (file_type, log_pattern)) + pattern=r'%s\..*%s.*' % (file_type, log_pattern)) logs = [ f for f in all_files if os.path.islink(f) ] log = None if logs: @@ -490,23 +537,87 @@ def list_logs(recipe_pattern, log_pattern, exact, raw, apply_replacements): def list_run_scripts(recipe_pattern, log_pattern, exact): list_temp_files('run', recipe_pattern, log_pattern, exact, True, False) + +def ensure_list(value): + if value is None: + return [] + if isinstance(value, list): + return value + return [value] + + +def prefix_grep_paths(output, root): + lines = output.decode(encoding, errors='replace').split('\n')[:-1] + prefixed = [] + for line in lines: + file, sep, rest = line.partition(':') + if sep and not os.path.isabs(file): + line = os.path.join(root, file) + sep + rest + prefixed.append(line) + return prefixed + + +def run_git_grep(args, dirs): + results = [] + for repo in ye_compat.git_repos(dirs): + p = subprocess.Popen(['git', '-C', repo, 'grep'] + args, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + out, _ = p.communicate() + results.extend(prefix_grep_paths(out, repo)) + return results + + +def run_recursive_grep(args, dirs): + results = [] + for dir in dirs: + p = subprocess.Popen(['grep', '-r'] + args + [dir], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + out, _ = p.communicate() + results.extend(out.decode(encoding, errors='replace').split('\n')[:-1]) + return results + + +def apply_action(file, actions, shortcut=None): + if len(actions) == 1: + actions[0][2](file) + return file + if shortcut: + for action in actions: + if action[1] == shortcut: + action[2](file) + return file + action = prompt_action([ (a[1], a[0]) for a in actions ]) + actions[action][2](file) + return file + + def grep(args, actions, case_sensitive, dir=None, git=False): - if git and dir: - os.chdir(dir) - p = subprocess.Popen(['git', 'grep'] + args, stdout=subprocess.PIPE) - elif dir: - os.chdir(dir) - p = subprocess.Popen(['grep', '-r'] + args, stdout=subprocess.PIPE) - else: - p = subprocess.Popen(['repo', 'grep'] + args, stdout=subprocess.PIPE) regex_pattern = [ arg for arg in args if not arg.startswith('-') ] if not regex_pattern: die('Could not determine the pattern for grep.') if len(regex_pattern) > 1: die('Multiple patterns for grep is not supported.') regex_pattern = regex_pattern[0] - output = p.communicate()[0] - results = output.decode(encoding).split('\n')[:-1] # the last line is always empty + + if git and dir: + results = run_git_grep(args, ensure_list(dir)) + elif dir: + results = run_recursive_grep(args, ensure_list(dir)) + else: + repo_root = ye_compat.get_repo_root() + if repo_root and ye_compat.is_program_available('repo'): + p = subprocess.Popen(['repo', 'grep'] + args, + cwd=repo_root, + stdout=subprocess.PIPE) + results = p.communicate()[0].decode(encoding, errors='replace').split('\n')[:-1] + else: + source_dirs = find_source_dirs() + results = run_git_grep(args, source_dirs) + if not results: + results = run_recursive_grep(args, source_dirs) + if sys.stdout.isatty(): shortcut = None if len(results) == 0: @@ -528,27 +639,22 @@ def grep(args, actions, case_sensitive, dir=None, git=False): else: actions = [view_action, edit_action] file = match.split(':')[0] # let's hope filenames don't contain ':' + if os.path.isabs(file) and os.path.exists(file): + return apply_action(file, actions, shortcut) return find(file, - dir or os.path.join(find_yocto_root(), 'sources'), + dir or find_source_dirs(), False, actions) else: return results def git_log_grep(pattern, max_commits, case_sensitive): - root_dir = find_yocto_root() - os.chdir(root_dir) - repos = [] - repop = subprocess.Popen(['repo', 'list'], stdout=subprocess.PIPE) - repos = repop.communicate()[0].strip() - repos = [ line.split(':')[0].strip() for line in repos.decode(encoding).split('\n') ] - repo_paths = [ os.path.realpath(os.path.join(root_dir, repo)) for repo in repos ] - + repo_paths = ye_compat.git_repos(find_source_dirs()) results = {} for repo_path in repo_paths: results[repo_path] = None - os.chdir(repo_path) - gitlog = subprocess.Popen(('git', 'log', '-n', str(max_commits), '--oneline'), + gitlog = subprocess.Popen(('git', '-C', repo_path, 'log', '-n', + str(max_commits), '--oneline'), stdout=subprocess.PIPE) if case_sensitive: grep_cmd = ('grep', pattern) @@ -560,15 +666,15 @@ def git_log_grep(pattern, max_commits, case_sensitive): gitlog.wait() out = grep.communicate()[0] if out: - results[repo_path] = out.strip().split('\n') + results[repo_path] = out.strip().split(b'\n') - sources_dir = os.path.realpath(os.path.join(root_dir, 'sources')) + root_dir = find_yocto_root() options = [] for repo_path, commits in results.items(): - repo_dir = repo_path[len(sources_dir) + 1:] # drop sources_dir + repo_dir = os.path.relpath(repo_path, root_dir) if commits: for commit in commits: - options.append('%s %s' % (repo_dir, commit)) + options.append('%s %s' % (repo_dir, commit.decode(encoding))) if sys.stdout.isatty(): shortcut = None @@ -587,9 +693,8 @@ def git_log_grep(pattern, max_commits, case_sensitive): match = options[0] repo_dir, commit_hash = match.split()[0:2] - repo_path = os.path.join(sources_dir, repo_dir) - os.chdir(repo_path) - os.system('git show %s' % commit_hash) + repo_path = os.path.join(root_dir, repo_dir) + os.system('git -C "%s" show %s' % (repo_path, commit_hash)) else: for option in options: print(option) @@ -674,40 +779,70 @@ def doc(pattern, exact): print('\n=== ' + var) print(doc) + def element_text(elt): + return remove_duplicate_spaces(' '.join(elt.itertext())).strip() + + def parse_current_html(path): + parser = etree.HTMLParser() + tree = etree.parse(path, parser) + result = {} + for dt in tree.xpath('//dt[starts-with(@id, "term-")]'): + var = element_text(dt).replace('ΒΆ', '').strip() + if not var: + continue + var = var.split()[0] + var = re.sub(r'[^A-Za-z0-9_:+.-].*$', '', var) + dd = dt.getnext() + while dd is not None and dd.tag != 'dd': + dd = dd.getnext() + if dd is not None: + result[var] = element_text(dd) + return result + + def parse_legacy_xml(path): + tree = etree.parse(path) + result = {} + for e in tree.findall('//glossentry'): + glossterm = e.find('glossterm').text + glossdef = e.find('glossdef') + paragraphs = glossdef.findall('para') + text = [] + for p in paragraphs: + text.append(parse_paragraph(p)) + result[glossterm] = '\n\n'.join(text) + return result + doc_data_dir = os.path.join(YE_DIR, 'doc-data') + variables_html = os.path.join(doc_data_dir, 'ref-manual', 'variables.html') ref_variables_xml = os.path.join(doc_data_dir, 'ref-manual', 'ref-variables.xml') # Check if we need to update documentation data update_window = 7 * 24 * 3600 # 7 days - # We use documentation from master - doc_base_uri = 'https://git.yoctoproject.org/cgit/cgit.cgi/yocto-docs/plain/documentation' - if (not os.path.exists(ref_variables_xml) or - os.stat(ref_variables_xml).st_mtime < time.time() - update_window): + doc_base_uri = 'https://docs.yoctoproject.org' + if (not os.path.exists(variables_html) or + os.stat(variables_html).st_mtime < time.time() - update_window): print('Updating documentation data...') - download_file('%s/poky.ent' % doc_base_uri, - os.path.join(doc_data_dir, 'poky.ent')) - download_file('%s/ref-manual/ref-variables.xml' % doc_base_uri, - ref_variables_xml) + download_file('%s/ref-manual/variables.html' % doc_base_uri, + variables_html) - tree = etree.parse(ref_variables_xml) - glossentries = tree.findall('//glossentry') - docs = {} + try: + all_docs = parse_current_html(variables_html) + except: + all_docs = {} + if os.path.exists(ref_variables_xml): + all_docs = parse_legacy_xml(ref_variables_xml) orig_pattern = pattern if not exact: pattern = '.*%s.*' % pattern + re_pattern = re.compile(pattern, re.IGNORECASE) - for e in glossentries: - glossterm = e.find('glossterm').text - if re.compile(pattern).match(glossterm): - glossdef = e.find('glossdef') - paragraphs = glossdef.findall('para') - text = [] - for p in paragraphs: - text.append(parse_paragraph(p)) - docs[glossterm] = '\n\n'.join(text) + docs = {} + for var, text in all_docs.items(): + if re_pattern.match(var): + docs[var] = text if docs: - options = docs.keys() + options = list(docs.keys()) if len(options) == 1: show_doc(options[0], docs[options[0]]) elif sys.stdout.isatty(): @@ -730,7 +865,7 @@ def write_plumbing_selection(cmd, val): os.makedirs(plumbing_selection_dir) fd = open(cmd_file, 'w') if isinstance(val, dict): - for var, value in val.items(): + for var, value in list(val.items()): fd.write('%s=%s\n' % (var, value)) else: print(val) @@ -769,11 +904,31 @@ def handle_plumbing(args): die('plumbing wd: missing argument.') elif cmd == 'topdir': print(find_yocto_root()) + elif cmd == 'sourcedir': + source_dir = ye_compat.source_root_dir() + if source_dir: + print(source_dir) + else: + die('plumbing sourcedir: could not determine source directory.') + elif cmd == 'sysrootdir': + dirs = ye_compat.sysroot_dirs(cmd_args[0] if cmd_args else None) + if dirs: + print(dirs[0]) + else: + die('plumbing sysrootdir: could not determine sysroot directory.') + elif cmd == 'pkgdir': + pkg_type = cmd_args[0] if len(cmd_args) > 0 else None + machine = cmd_args[1] if len(cmd_args) > 1 else None + pkg_dir = ye_compat.deploy_pkg_dir(pkg_type, machine) + if pkg_dir: + print(pkg_dir) + else: + die('plumbing pkgdir: could not determine package deploy directory.') elif cmd == 'x': if cmd_args: expand_mod = __import__('bb-expand-vars') os.chdir(BUILDDIR) - val = expand_mod.show_var_expansions('core-image-minimal', + val = expand_mod.show_var_expansions(None, cmd_args, True) if val: @@ -782,7 +937,7 @@ def handle_plumbing(args): die('plumbing x: missing argument.') elif cmd == 'find': print_matches(find(cmd_args[0], - os.path.join(find_yocto_root(), 'sources'), + find_source_dirs(), exact, [('Print', 'p', lambda file: print(file))]), plumbing_cmd=cmd) @@ -802,12 +957,13 @@ def usage(exit_code=None, cmd=None): f [-e] find [-e] - Locate paths (in the "sources" directory) that match the given - expression . is case insensitive and implicitly - surrounded by '.*'. -e disables the implicit use of '.*' around - the given . Note that, unless contains /, matching - is attempted on filenames only (not on dirnames). If - contains /, matching is attempted on the full path. + Locate paths in configured Yocto layer/source directories that + match the given expression . is case insensitive + and implicitly surrounded by '.*'. -e disables the implicit use + of '.*' around the given . Note that, unless + contains /, matching is attempted on filenames only (not on + dirnames). If contains /, matching is attempted on the + full path. v [-e] view [-e] @@ -865,7 +1021,7 @@ pkg-extract [-e] [-c] [] the package filename (without extension). must match the exact file name in the package (usually starts with ./). -c is specific to .deb and .ipk packages -- ye will extract files - from the control.tar.gz tarball in packages. + from the control archive in packages. wd [-e] workdir [-e] @@ -897,11 +1053,12 @@ run [-e] [] g grep - Run 'repo grep '. + Run 'repo grep ' in repo-based workspaces, otherwise run + 'git grep ' across configured layers. sg sysroot-grep - Run 'grep -r $BUILDDIR/tmp/sysroots/$MACHINE'. + Run 'grep -r ' across available sysroot directories. glg [-n ] [-i] git-log-grep [-n ] [-i] @@ -954,14 +1111,14 @@ cd [] Change to the sysroot directory for MACHINE src [] - Change to 's source dir or to TOPDIR/sources - if is not provided + Change to 's source dir or to the detected source/layers + directory if is not provided img Change to BUILDDIR/tmp/deploy/MACHINE/image/ pkg - Change to BUILDDIR/tmp/deploy/PKG_TYPE/image/ + Change to BUILDDIR/tmp/deploy/PKG_TYPE or its MACHINE subdir manifest Change to TOPDIR/.repo/manifests @@ -1073,14 +1230,16 @@ def main(args): check_builddir() expand_mod = __import__('bb-expand-vars') os.chdir(BUILDDIR) - val = expand_mod.show_var_expansions('core-image-minimal', + val = expand_mod.show_var_expansions(None, ['MACHINE'], True) if val: machine = val['MACHINE'] else: die('Could not determine MACHINE.') - sysroot_dir = os.path.join(BUILDDIR, 'tmp', 'sysroots', machine) + sysroot_dir = ye_compat.sysroot_dirs(machine) + if not sysroot_dir: + die('Could not determine a sysroot directory.') print_matches(grep(cmd_args, [view_action, edit_action], case_sensitive, @@ -1094,16 +1253,16 @@ def main(args): else: actions = [view_action, edit_action] print_matches(find(pattern, - os.path.join(find_yocto_root(), 'sources'), + find_source_dirs(), exact, actions)) elif cmd in ['sf', 'sysroot-find', 'wsf', 'work-shared-find']: check_builddir() if cmd in ['sf', 'sysroot-find']: - search_dir = os.path.join(BUILDDIR, 'tmp', 'sysroots') + search_dir = ye_compat.sysroot_dirs() else: search_dir = os.path.join(BUILDDIR, 'tmp', 'work-shared') - if os.path.isdir(search_dir): + if isinstance(search_dir, list) or os.path.isdir(search_dir): print_matches(find(pattern, search_dir, exact, @@ -1159,5 +1318,5 @@ def main(args): else: usage(1) - -main(sys.argv[1:]) +if __name__ == '__main__': + main(sys.argv[1:]) diff --git a/ye-cd b/ye-cd index cd942e6..69c4e53 100644 --- a/ye-cd +++ b/ye-cd @@ -74,14 +74,15 @@ ye() { ;; "sd"|"sysroot"|"sysrootdir") $YE_SCRIPT plumbing x MACHINE - local machine=`yecd_get_bb_var MACHINE` - [ -n "$machine" ] && cd $BUILDDIR/tmp/sysroots/$machine + local machine=`yecd_get_bb_var MACHINE | tr - _` + local sysrootdir=`$YE_SCRIPT plumbing sysrootdir "$machine"` + [ -n "$sysrootdir" ] && cd "$sysrootdir" ;; "src"|"sources") local recipe=$2 if [ -z "$recipe" ]; then - topdir=`$YE_SCRIPT plumbing topdir` - [ $? -eq 0 ] && cd $topdir/sources + local sourcedir=`$YE_SCRIPT plumbing sourcedir` + [ -n "$sourcedir" ] && cd "$sourcedir" else cd $BUILDDIR $YE_SCRIPT plumbing find $recipe @@ -96,9 +97,9 @@ ye() { "pkg"|"packages") $YE_SCRIPT plumbing x IMAGE_PKGTYPE MACHINE local pkg_type=`yecd_get_bb_var IMAGE_PKGTYPE` - local machine=`yecd_get_bb_var MACHINE` - [ -n "$pkg_type" ] && [ -n "$machine" ] && \ - cd $BUILDDIR/tmp/deploy/$pkg_type/$machine + local machine=`yecd_get_bb_var MACHINE | tr - _` + local pkgdir=`$YE_SCRIPT plumbing pkgdir "$pkg_type" "$machine"` + [ -n "$pkgdir" ] && cd "$pkgdir" ;; "manifest"|"manifests") topdir=`$YE_SCRIPT plumbing topdir` diff --git a/ye_compat.py b/ye_compat.py new file mode 100644 index 0000000..0a32d8b --- /dev/null +++ b/ye_compat.py @@ -0,0 +1,367 @@ +import functools +import glob +import os +import re +import subprocess + +# Well-known sibling directory names that may hold Yocto layers/sources. +LAYER_DIR_NAMES = ['sources', 'layers', 'poky', 'openembedded-core'] + +# Upper bound on the number of passes used to resolve nested ${VAR} +# references when expanding a bblayers.conf path. +MAX_VAR_EXPANSION_PASSES = 8 + + +def layer_subdirs(root): + return [os.path.join(root, name) for name in LAYER_DIR_NAMES] + + +def common_dir(dirs, require_dir=False): + try: + common = os.path.commonpath(dirs) + except ValueError: + return None + if not common or common == os.path.sep: + return None + if require_dir and not os.path.isdir(common): + return None + return os.path.realpath(common) + + +def uniq(paths): + result = [] + seen = set() + for path in paths: + if not path: + continue + real = os.path.realpath(os.path.abspath(os.path.expanduser(path))) + if real not in seen: + seen.add(real) + result.append(real) + return result + + +def existing_dirs(paths): + return [path for path in uniq(paths) if os.path.isdir(path)] + + +def find_upwards(start, names): + start = os.path.realpath(os.path.abspath(start or os.getcwd())) + if os.path.isfile(start): + start = os.path.dirname(start) + while True: + for name in names: + path = os.path.join(start, name) + if os.path.exists(path): + return start + parent = os.path.dirname(start) + if parent == start: + return None + start = parent + + +def is_program_available(program): + for directory in os.environ.get('PATH', '').split(os.pathsep): + path = os.path.join(directory, program) + if os.path.isfile(path) and os.access(path, os.X_OK): + return True + return False + + +@functools.lru_cache(maxsize=None) +def get_builddir(): + builddir = os.environ.get('BUILDDIR') + if builddir: + return os.path.realpath(os.path.abspath(builddir)) + + found = find_upwards(os.getcwd(), + [os.path.join('conf', 'bblayers.conf')]) + if found: + return found + return None + + +def get_repo_root(): + builddir = get_builddir() + starts = [os.getcwd()] + if builddir: + starts.insert(0, builddir) + for start in starts: + root = find_upwards(start, ['.repo']) + if root: + return root + return None + + +def join_bitbake_lines(text): + logical_lines = [] + current = '' + for line in text.splitlines(): + stripped = line.rstrip() + if stripped.endswith('\\'): + current += stripped[:-1] + ' ' + continue + logical_lines.append(current + stripped) + current = '' + if current: + logical_lines.append(current) + return logical_lines + + +def strip_comment(line): + result = [] + quote = None + escaped = False + for char in line: + if escaped: + result.append(char) + escaped = False + continue + if char == '\\': + result.append(char) + escaped = True + continue + if char in ['"', "'"]: + if quote == char: + quote = None + elif quote is None: + quote = char + if char == '#' and quote is None: + break + result.append(char) + return ''.join(result) + + +def expand_bitbake_path(path, variables): + value = os.path.expanduser(path) + for _ in range(MAX_VAR_EXPANSION_PASSES): + expanded = re.sub(r'\${([^}]+)}', + lambda m: variables.get(m.group(1), + os.environ.get(m.group(1), m.group(0))), + value) + if expanded == value: + break + value = expanded + return os.path.expandvars(value) + + +def quoted_values(line): + values = [] + quote = None + value = [] + escaped = False + for char in line: + if escaped: + if quote: + value.append(char) + escaped = False + continue + if char == '\\': + escaped = True + if quote: + value.append(char) + continue + if char in ['"', "'"]: + if quote == char: + values.append(''.join(value)) + value = [] + quote = None + elif quote is None: + quote = char + elif quote: + value.append(char) + continue + if quote: + value.append(char) + return values + + +def parse_bblayers_conf(builddir=None): + builddir = builddir or get_builddir() + if not builddir: + return [] + + conf = os.path.join(builddir, 'conf', 'bblayers.conf') + if not os.path.exists(conf): + return [] + + try: + with open(conf, 'r') as fd: + data = fd.read() + except OSError: + return [] + + variables = { + 'TOPDIR': builddir, + 'BUILDDIR': builddir, + 'HOME': os.environ.get('HOME', ''), + } + for var in ['OEROOT', 'COREBASE', 'POKYBASE']: + if os.environ.get(var): + variables[var] = os.environ[var] + + layers = [] + for line in join_bitbake_lines(data): + line = strip_comment(line).strip() + if not re.match(r'^BBLAYERS(?:\s|[:?+.=:]|$)', line): + continue + for value in quoted_values(line): + for token in value.split(): + expanded = expand_bitbake_path(token, variables) + if not os.path.isabs(expanded): + expanded = os.path.join(builddir, expanded) + matches = glob.glob(expanded) + layers.extend(matches or [expanded]) + return existing_dirs(layers) + + +def fallback_source_dirs(builddir=None): + candidates = [] + repo_root = get_repo_root() + if repo_root: + candidates.extend([ + os.path.join(repo_root, 'sources'), + os.path.join(repo_root, 'layers'), + ]) + + for env_name in ['PLATFORM_ROOT_DIR', 'YE_TOPDIR']: + if os.environ.get(env_name): + root = os.environ[env_name] + candidates.extend([ + os.path.join(root, 'sources'), + os.path.join(root, 'layers'), + root, + ]) + + for env_name in ['OEROOT', 'COREBASE']: + if os.environ.get(env_name): + candidates.append(os.environ[env_name]) + + builddir = builddir or get_builddir() + if builddir: + candidates.extend(layer_subdirs(os.path.dirname(builddir))) + + root = find_upwards(os.getcwd(), LAYER_DIR_NAMES) + if root: + candidates.extend(layer_subdirs(root)) + + return existing_dirs(candidates) + + +@functools.lru_cache(maxsize=None) +def source_dirs(): + env_dirs = os.environ.get('YE_SOURCE_DIRS') + if env_dirs: + return existing_dirs(env_dirs.split(os.pathsep)) + + builddir = get_builddir() + layers = parse_bblayers_conf(builddir) + if layers: + return layers + return fallback_source_dirs(builddir) + + +def source_root_dir(): + root = workspace_root() + if root: + for name in ['sources', 'layers']: + path = os.path.join(root, name) + if os.path.isdir(path): + return os.path.realpath(path) + + dirs = source_dirs() + if not dirs: + return None + return common_dir(dirs, require_dir=True) or dirs[0] + + +def workspace_root(): + repo_root = get_repo_root() + if repo_root: + return repo_root + + roots = [] + builddir = get_builddir() + if builddir: + roots.append(builddir) + roots.extend(source_dirs()) + if roots: + common = common_dir(roots) + if common: + return common + + for env_name in ['PLATFORM_ROOT_DIR', 'YE_TOPDIR', 'OEROOT', 'COREBASE']: + if os.environ.get(env_name): + return os.path.realpath(os.path.abspath(os.environ[env_name])) + return find_upwards(os.getcwd(), LAYER_DIR_NAMES) + + +def find_git_root(path): + if not path or not is_program_available('git'): + return None + try: + out = subprocess.check_output( + ['git', '-C', path, 'rev-parse', '--show-toplevel'], + stderr=subprocess.DEVNULL) + return os.path.realpath(out.decode().strip()) + except (subprocess.CalledProcessError, OSError): + return None + + +def git_repos(dirs=None): + dirs = dirs or source_dirs() + repos = [] + for directory in dirs: + root = find_git_root(directory) + if root: + repos.append(root) + continue + for current, subdirs, _ in os.walk(directory): + subdirs[:] = [d for d in subdirs + if d not in ['.git', 'tmp', 'downloads', + 'sstate-cache', 'cache']] + if '.git' in subdirs: + repos.append(current) + subdirs[:] = [] + return existing_dirs(repos) + + +def sysroot_dirs(machine=None): + builddir = get_builddir() + if not builddir: + return [] + + candidates = [] + components = os.path.join(builddir, 'tmp', 'sysroots-components') + if machine: + candidates.append(os.path.join(components, machine)) + candidates.append(os.path.join(components, machine.replace('-', '_'))) + candidates.append(components) + + old_sysroots = os.path.join(builddir, 'tmp', 'sysroots') + if machine: + candidates.append(os.path.join(old_sysroots, machine)) + candidates.append(old_sysroots) + + candidates.extend(glob.glob(os.path.join(builddir, 'tmp', 'work', '*', + '*', '*', 'recipe-sysroot'))) + candidates.extend(glob.glob(os.path.join(builddir, 'tmp', 'work', '*', + '*', '*', 'recipe-sysroot-native'))) + return existing_dirs(candidates) + + +def deploy_pkg_dir(pkg_type=None, machine=None): + builddir = get_builddir() + if not builddir: + return None + deploy = os.path.join(builddir, 'tmp', 'deploy') + candidates = [] + if pkg_type: + pkg_base = os.path.join(deploy, pkg_type) + if machine: + candidates.append(os.path.join(pkg_base, machine.replace('-', '_'))) + candidates.append(os.path.join(pkg_base, machine)) + candidates.append(pkg_base) + candidates.append(deploy) + dirs = existing_dirs(candidates) + return dirs[0] if dirs else None