diff --git a/sphinx_presentation/.gitignore b/sphinx_presentation/.gitignore new file mode 100644 index 0000000..1cbc778 --- /dev/null +++ b/sphinx_presentation/.gitignore @@ -0,0 +1,4 @@ +.DS_Store + + + diff --git a/sphinx_presentation/2018-AnacondaCON-PackageBuilding.pptx b/sphinx_presentation/2018-AnacondaCON-PackageBuilding.pptx new file mode 100644 index 0000000..aa0e68a Binary files /dev/null and b/sphinx_presentation/2018-AnacondaCON-PackageBuilding.pptx differ diff --git a/sphinx_presentation/Makefile b/sphinx_presentation/Makefile new file mode 100644 index 0000000..f122f0d --- /dev/null +++ b/sphinx_presentation/Makefile @@ -0,0 +1,20 @@ +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line. +SPHINXOPTS = +SPHINXBUILD = sphinx-build +SPHINXPROJ = TheJoyofPackaging +SOURCEDIR = source +BUILDDIR = build + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) \ No newline at end of file diff --git a/sphinx_presentation/make.bat b/sphinx_presentation/make.bat new file mode 100644 index 0000000..6258635 --- /dev/null +++ b/sphinx_presentation/make.bat @@ -0,0 +1,36 @@ +@ECHO OFF + +pushd %~dp0 + +REM Command file for Sphinx documentation + +if "%SPHINXBUILD%" == "" ( + set SPHINXBUILD=sphinx-build +) +set SOURCEDIR=source +set BUILDDIR=build +set SPHINXPROJ=TheJoyofPackaging + +if "%1" == "" goto help + +%SPHINXBUILD% >NUL 2>NUL +if errorlevel 9009 ( + echo. + echo.The 'sphinx-build' command was not found. Make sure you have Sphinx + echo.installed, then set the SPHINXBUILD environment variable to point + echo.to the full path of the 'sphinx-build' executable. Alternatively you + echo.may add the Sphinx directory to PATH. + echo. + echo.If you don't have Sphinx installed, grab it from + echo.http://sphinx-doc.org/ + exit /b 1 +) + +%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% +goto end + +:help +%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% + +:end +popd diff --git a/sphinx_presentation/slides2rst.py b/sphinx_presentation/slides2rst.py new file mode 100755 index 0000000..c0373f8 --- /dev/null +++ b/sphinx_presentation/slides2rst.py @@ -0,0 +1,65 @@ +#!/usr/bin/env python + +""" +hacky script to convert the txt exported from google slides to rst for hieroglyph +""" + +inlines = open("2018-AnacondaCON-PackageBuilding.pptx.txt").readlines() +outfilename = "PackageBuilding.rst" + +rst = [] + +# first line is the title: +title = inlines.pop(0).strip() +rst.append("#" * len(title)) +rst.append(title) +rst.append("#" * len(title)) +rst.append("") +# find end of title page: +line = inlines.pop(0) +while line.strip() != '1': + rst.append(line) + line = inlines.pop(0) + +while inlines: + # look for slide break: + slide_num = 0 + for i, line in enumerate(inlines): + # numbers by themselves on a line is a new slide + try: + num = int(line) + except ValueError: + continue + if num != slide_num: + slide_num = num + start_ind = i + else: + # we've hit the second numbers + end_ind = i + break + # write content + # slide header is right before slide number + header = inlines[start_ind - 1].strip() + rst.append(header) + rst.append("-" * len(header)) + + # content is the stuff above it + for line in inlines[:start_ind - 1]: + rst.append(line) + # footer is the stuff in between + rst.append("") + for line in inlines[start_ind + 1:end_ind]: + rst.append(line) + # clear it all out: + del inlines[:end_ind + 1] + # and clear out any empty lines + while inlines and (not inlines[0].strip()): + del inlines[0] + + + + + + +open(outfilename, 'w').write("\n".join(rst)) + diff --git a/sphinx_presentation/source/conf.py b/sphinx_presentation/source/conf.py new file mode 100644 index 0000000..4a1e83f --- /dev/null +++ b/sphinx_presentation/source/conf.py @@ -0,0 +1,160 @@ +# -*- coding: utf-8 -*- +# +# Configuration file for the Sphinx documentation builder. +# +# This file does only contain a selection of the most common options. For a +# full list see the documentation: +# http://www.sphinx-doc.org/en/master/config + +# -- Path setup -------------------------------------------------------------- + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +# +# import os +# import sys +# sys.path.insert(0, os.path.abspath('.')) + + +# -- Project information ----------------------------------------------------- + +project = 'The Joy of Packaging' +copyright = '2018, Assorted' +author = 'Assorted' + +# The short X.Y version +version = '' +# The full version, including alpha/beta/rc tags +release = '0.1' + + +# -- General configuration --------------------------------------------------- + +# If your documentation needs a minimal Sphinx version, state it here. +# +# needs_sphinx = '1.0' + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = [ + 'sphinx.ext.githubpages', + 'hieroglyph', +] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['_templates'] + +# The suffix(es) of source filenames. +# You can specify multiple suffix as a list of string: +# +# source_suffix = ['.rst', '.md'] +source_suffix = '.rst' + +# The master toctree document. +master_doc = 'index' + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +# +# This is also used if you do content translation via gettext catalogs. +# Usually you set "language" from the command line for these cases. +language = None + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +# This pattern also affects html_static_path and html_extra_path . +exclude_patterns = [] + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = 'sphinx' + + +# -- Options for HTML output ------------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +# +html_theme = 'alabaster' + +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. +# +# html_theme_options = {} + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ['_static'] + +# Custom sidebar templates, must be a dictionary that maps document names +# to template names. +# +# The default sidebars (for documents that don't match any pattern) are +# defined by theme itself. Builtin themes are using these templates by +# default: ``['localtoc.html', 'relations.html', 'sourcelink.html', +# 'searchbox.html']``. +# +# html_sidebars = {} + + +# -- Options for HTMLHelp output --------------------------------------------- + +# Output file base name for HTML help builder. +htmlhelp_basename = 'TheJoyofPackagingdoc' + + +# -- Options for LaTeX output ------------------------------------------------ + +latex_elements = { + # The paper size ('letterpaper' or 'a4paper'). + # + # 'papersize': 'letterpaper', + + # The font size ('10pt', '11pt' or '12pt'). + # + # 'pointsize': '10pt', + + # Additional stuff for the LaTeX preamble. + # + # 'preamble': '', + + # Latex figure (float) alignment + # + # 'figure_align': 'htbp', +} + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, +# author, documentclass [howto, manual, or own class]). +latex_documents = [ + (master_doc, 'TheJoyofPackaging.tex', 'The Joy of Packaging Documentation', + 'Assorted', 'manual'), +] + + +# -- Options for manual page output ------------------------------------------ + +# One entry per manual page. List of tuples +# (source start file, name, description, authors, manual section). +man_pages = [ + (master_doc, 'thejoyofpackaging', 'The Joy of Packaging Documentation', + [author], 1) +] + + +# -- Options for Texinfo output ---------------------------------------------- + +# Grouping the document tree into Texinfo files. List of tuples +# (source start file, target name, title, author, +# dir menu entry, description, category) +texinfo_documents = [ + (master_doc, 'TheJoyofPackaging', 'The Joy of Packaging Documentation', + author, 'TheJoyofPackaging', 'One line description of project.', + 'Miscellaneous'), +] + + +# -- Extension configuration ------------------------------------------------- \ No newline at end of file diff --git a/sphinx_presentation/source/images/about_section.png b/sphinx_presentation/source/images/about_section.png new file mode 100644 index 0000000..d599bf8 Binary files /dev/null and b/sphinx_presentation/source/images/about_section.png differ diff --git a/sphinx_presentation/source/images/req_run_exports.png b/sphinx_presentation/source/images/req_run_exports.png new file mode 100644 index 0000000..5d6973f Binary files /dev/null and b/sphinx_presentation/source/images/req_run_exports.png differ diff --git a/sphinx_presentation/source/images/run_exports.png b/sphinx_presentation/source/images/run_exports.png new file mode 100644 index 0000000..be1e2d3 Binary files /dev/null and b/sphinx_presentation/source/images/run_exports.png differ diff --git a/sphinx_presentation/source/index.rst b/sphinx_presentation/source/index.rst new file mode 100644 index 0000000..405cd83 --- /dev/null +++ b/sphinx_presentation/source/index.rst @@ -0,0 +1,22 @@ +.. The Joy of Packaging documentation master file, created by + sphinx-quickstart on Fri Jul 6 14:55:10 2018. + You can adapt this file completely to your liking, but it should at least + contain the root `toctree` directive. + +Welcome to The Joy of Packaging's documentation! +================================================ + +.. toctree:: + :maxdepth: 1 + :caption: Contents: + + overview + package_building + + +Indices and tables +================== + +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search` diff --git a/sphinx_presentation/source/overview.rst b/sphinx_presentation/source/overview.rst new file mode 100644 index 0000000..353d12d --- /dev/null +++ b/sphinx_presentation/source/overview.rst @@ -0,0 +1,204 @@ +*************************** +Packaging Tutorial Overview +*************************** + +Outline: +======== + +0:00-00:20 Overview of packaging +-------------------------------- + + * Source/binary + * Wheel vs conda packages + * PyPI/anaconda.org + +0:20-0:45 setup.py +------------------ + +* Essential specifications +* Optional specifications +* Specifying requirements +* In setup.py vs requirements file +* When and how to "pin" requirements + +Exercise: +......... + +Fill in the missing pieces in a setup.py for a sample package +Build a source distribution for the package + +0:45-1:00 Building and uploading to PyPI: +----------------------------------------- + +Tools and package types + + +**flit:** great for simple packages + +**twine:** the secure way to upload to PyPI + +* Building a source distribution + +* Building a wheel + +* Multibuild - https://github.com/matthew-brett/multibuild + +* Manylinux docker image - https://github.com/pypa/manylinux + +* Delocate - https://github.com/matthew-brett/delocate + +* Auditwheel - https://github.com/pypa/auditwheel + +1:00-1:10 Break +--------------- + +1:10-1:30 Worked example/exercise: +---------------------------------- + +Building a package and uploading to pypi + +Continuing from the the previous exercise, build a wheel for the package + +Register the package on the pypi testing server + +Upload the built distributions using twine + +Delete one of the uploaded files on pypi and try re-uploading (will fail) + +Introduce the idea of .post releases (it will happen to everyone who uploads) + +1:30-1:45 Binaries and dependencies: +------------------------------------ + +how scikit-build and conda-build can make life easier + +1:45-2:00 Scikit-build overview: + +Why + Motivations + +From [distutils.core.Extension] to [scikit-build + CMake] in few lines +Support for developer mode (bonus) + +2:00-2:45 Exercise: +------------------- + +Add CMake project that generates python extension. Tie +it into previous python project. + +Cookie cutter template integrating conda, pypi, etc. will be provided. + +2:45-3:00 Break +--------------- + +3:00-3:15 Conda-build overview +------------------------------ + +3:15-3:30 Exercise: +------------------- + +Write a conda recipe for the sample package from previous exercises (pure python) + +noarch packages + +Upload to anaconda cloud + +3:15-3:45 Exercise: +------------------- + +Recipe for package with compiled extensions + +Add compiled extension (source will be provided to students) to sample package + +Modify recipe, if needed + +Rebuild the package + +Version pinning (python, numpy) + +Split packages - multi-ecosystem ones + +Compiler packages + pin_downstream + +Interoperation with scikit-build + +3:45-4:00 Automated building with cloud-based CI services: +---------------------------------------------------------- + +conda-forge (optional; as time allows) + +http://scikit-ci.readthedocs.io + +http://scikit-ci-addons.readthedocs.io + +CI service overview & Conda-forge -- what are the pieces and how do they fit together? + +Recipe format + +staged-recipes + +feedstocks + +Re-rendering and conda-smithy + +Updating package when new version released + +Future direction/community needs + +Invitation to sprints + +Contributing to Conda-forge + +Intro to conda-forge: staged-recipes, maintainer role, contributing to an existing package + +conda-smithy lint/rerender + +Example to go from the conda-skeleton to a PR on staged-recipes + +Comment on some special cases: cython extensions, non-python pkgs, the use of the CIs, etc. + +Exercise: put a package on staged-recipes + + +Tutorial code base layout: +-------------------------- + +Name of the organization: python-packaging-tutorial + +All projects should be associated with a cookiecutter template + +One organization with multiple repos (or multiple branches ?) + +0_readme +1_helloworld_pure + +Install python + +Work with virtual env + +Include pytest, documentation building, … + +2_helloworld_c + +Show how C extensions are included in setup.py, and how they are made available to python + +3_helloworld_with_ci + +Introduce Appveyor, CircleCi, Travis + +Difference between CI for testing and CI for creating packages (CD) + +4_helloworld_skbuild + +Introduce C extensions with cmake + +Show how scikitbuild can help tie python and cmake together nicely + +5_helloworld_skbuild_ci + +Show how scikitbuild-ci can be used to simplify and unify CI scripts + +6_helloworld_skbuild_conda + +Show how conda-build can be used to produce conda packages and wheels, using the build files we’ve already used from previous exercises. + +7_Uploading_to_PyPI_&_anaconda.org diff --git a/sphinx_presentation/source/package_building.rst b/sphinx_presentation/source/package_building.rst new file mode 100644 index 0000000..bf44c25 --- /dev/null +++ b/sphinx_presentation/source/package_building.rst @@ -0,0 +1,2417 @@ +#################### +The Joy of Packaging +#################### + + +Scipy 2018 Tutorial +=================== + +Instructors +----------- + +Michael Sarahan, PhD: Conda-build tech lead, Anaconda, Inc. + +Matt McCormick (thewtex): Maintainer of dockcross, of Python packages for the Insight Toolkit (ITK) + +Jean-Christophe Fillion-Robin (jcfr): Maintainer of scikit-build, scikit-ci, scikit-ci-addons and python-cmake-buildsystem + +Filipe Fernandes (ocefpaf): Conda-forge core team, Maintainer of folium and a variety of libraries for ocean sciences. + +Matt Craig (mwcraig): Maintainer of ccdproc, reducer, astropy, lead on conda packaging for astropy, Conda-forge core team. + +Chris Barker (PythonCHB): Python instructor for the Univ. Washington Continuing Education Program, Contributor to conda-forge project. + +Ray Donnelly (mingwandroid): Anaconda employee, working on Anaconda Distribution. MSYS2 co-founder, Likes build systems too much. + +Jonathan Helmus (jjhelmus): Anaconda employee, working on Anaconda Distribution Builds tensorflow for fun, conda-forge core team member +Contributor to various open source packages in the scientific Python ecosystem. + + +Outline +------- + +Packaging fundamentals + +Conda packaging + +Compatibility and automation + + +What is a “package”? +-------------------- + +In a broad sense, anything you install using your package manager + +some kinds of packages have implied behavior and requirements + +Unfortunate overloading: python “package”: a folder that python imports + + +Package Managers and Repos +-------------------------- + +NPM, apt, yum, dnf, chocolatey, pip, conda, homebrew, etc. + +PyPI, anaconda.org, CRAN, CPAN + +Some form of dependency management + +Artifact and/or source repository + + +Implicit behavior & Requirements +-------------------------------- + +Folder structure + +Directly usable, or must be unpacked/installed? + +Python packages + +:: + + sound/ + __init__.py + formats/ + __init__.py + wavwrite.py + effects/ + __init__.py + echo.py + + +``https://docs.python.org/3/tutorial/modules.html#packages`` + +Folders must have ``__init__.py`` file to make Python able to import them + +``__init__.py`` can be empty (and is, most of the time) + + +``from sound.effects.echo import somefunc`` + +Python packages - why? +---------------------- + +.. code-block:: python + + # import nested module + + import sound.effects.echo + + from sound.effects import echo + + # import function or variable from nested module + + +https://docs.python.org/3/tutorial/modules.html#packages + +:: + + mypkg/ + __init__.py + subpkg/ + __init__.py + a.py + + + +Let’s make a package +-------------------- + +Windows +....... + + +.. code-block:: bash + + mkdir mypkg/subpkg + + echo. > mypkg/__init__.py + + echo . > mypkg/subpkg/__init__.py + + echo . > mypkg/subpkg/a.py + + +Mac/Linux +......... + +.. code-block:: bash + + mkdir -p mypkg/subpkg + + touch mypkg/__init__.py + + touch mypkg/subpkg/__init__.py + + touch mypkg/subpkg/a.py + + +How Python finds packages +------------------------- + +In python interpreter: + +.. code-block: python + + import sys + from pprint import pprint + pprint(sys.path) + + + +``sys.path`` explanation: https://stackoverflow.com/a/38403654/1170370 + + + +How to get things on sys.path +----------------------------- + +.pth files in sys.path locations + +``PYTHONPATH`` environment variable (fragile) + +Installing packages (destination: site-packages folder) + + +Find your site-packages folder +------------------------------ + +Mac/Linux: ``(install root)/lib/pythonX.Y/site-packages`` + + +Windows: ``(install root)\Lib\site-packages`` + + +Installing packages +------------------- + +pip install -e . + + +Installing: + +.. code-block:: python + + python setup.py install + + pip install . + + Development installs: + + python setup.py develop + + + +Installing +========== + + +Development install +------------------- + +Copies package into site-packages + +Adds a ``.pth`` file to site-packages, pointed at package source root + +Used when creating conda packages + +Used when developing software locally + +Normal priority in sys.path + +End of ``sys.path`` (only found if nothing else comes first) + +https://grahamwideman.wikispaces.com/Python-+site-package+dirs+and+.pth+files + + +What about setup.py? +-------------------- + +.. code-block:: python + + #!/usr/bin/env + + pythonfrom setuptools import setups + + setup(name='Distutils', + version='1.0', + description='Python Distribution Utilities', + author='Greg Ward', + author_email='gward@python.net', + url='https://www.python.org/sigs/distutils-sig/', packages=['distutils', 'distutils.command'], + ) + +``https://docs.python.org/2/distutils/setupscript.html`` + +What does setup.py do? +---------------------- + +Version & package metadata + +List of packages to include + +List of other files to include + +Lists of dependencies + +Lists of extensions to be compiled + + +Let’s write setup.py +-------------------- + +.. code-block:: python + + #!/usr/bin/env python + + from setuptools import setup + + setup(name='mypkg', + version='1.0', # list folders, not files + packages=['mypkg', 'mypkg.subpkg'], + ) + + +Setuptools +---------- + +Separate library (ships with Python by default, though) + +Adds entry point capability + +provides find_packages function (use with caution) + +creates eggs by default (people spend time fighting this later in the process) + + + +Where does setup.py go? +----------------------- + +:: + + mypkg-src + setup.py + mypkg/ + __init__.py + subpkg/ + __init__.py + a.py + +New outer folder + +``setup.py`` alongside package to be installed + +``mypkg`` is what will get installed + +``mypkg-src`` is what gets linked to by develop + + +Try installing your package +--------------------------- + +.. code-block:: bash + + cd mypkg-src + + python setup.py install + + python -c “import mypkg.subpkg.a” + + + +Go look in your ``site-packages`` folder + + +Making packages the easy way +---------------------------- + +``https://github.com/audreyr/cookiecutter`` + +.. code-block:: bash + + conda install -c conda-forge cookiecutter + + +Let’s make a project +-------------------- + +cookiecutter: ``https://goo.gl/Jge1g8`` + + +That’s a shortened link to: + +``https://github.com/conda/cookiecutter-conda-python`` + + +What did we get? +---------------- + +Adding requirements in ``setup.py`` + +.. code-block:: python + + #!/usr/bin/env python + from distutils.core import setup + + setup(name='mypkg', + version='1.0', + # list folders, not files + packages=['mypkg', 'mypkg.subpkg'], + install_requires=['click'], + ) + + +Requirements in ``requirements.txt`` + +**common mistake:** + +``requirements.txt`` often from pip freeze + +Pinned way too tightly. OK for env creation, bad for packaging. + +Donald Stufft (PyPA): Abstract vs. Concrete dependencies + + +``https://caremad.io/posts/2013/07/setup-vs-requirement/`` + + + +Requirements in ``setup.cfg`` (ideal) + +[metadata]name = my_package +version = attr: +src.VERSION[options] +packages = find: +install_requires = click + + +``http://setuptools.readthedocs.io/en/latest/setuptools.html#configuring-setup-using-setup-cfg-files`` + +Parseable without execution, unlike setup.py + + + + +Break time! +----------- + +Up next: producing redistributable artifacts + + +Redistributable artifacts +------------------------- + + +sdists + +wheels + +conda packages + +eggs (deprecated) + + +When/how to use an sdist +------------------------ + +Pure python (no build requirements) + +.. code-block:: bash + + python setup.py sdist + + +Wheels vs. Conda packages +------------------------- + ++-------------------------------------+-------------------------------------+ +| Wheels | Conda packages | ++=====================================+=====================================+ +| Employed by pip, blessed by PyPA | Foundation of Anaconda ecosystem | ++-------------------------------------+-------------------------------------+ +| Used by any python installation | Used by conda python installations | ++-------------------------------------+-------------------------------------+ +| Mostly specific to Python ecosystem | General purpose (any ecosystem) | ++-------------------------------------+-------------------------------------+ +| Good mechanism for specifying range | Primitive support for multiple | +| of python compatibility | python versions (noarch) | ++-------------------------------------+-------------------------------------+ +| Depends on static linking or other | Can bundle core system-level shared | +| system package managers to provide | libraries as packages, and resolve | +| core libraries | dependencies | ++-------------------------------------+-------------------------------------+ + + + +Introducing conda-build +----------------------- + +Orchestrates environment creation, activation, and build/test processes + +Can build conda packages and/or wheels + +Separate project from conda, but very tightly integrated + +Open-source, actively developedhttps://github.com/conda/conda-build + + +Getting conda-build to work for you +----------------------------------- + +Input: meta.yaml files + +:: + + package: + + name: mypkg + + version: 1.0 + + +Let’s use conda-build +--------------------- + +.. code-block:: bash + + conda install conda-build + + conda build mypkg-src + + +What happened? +-------------- + +templates filled in, recipe interpreted + +build environment created (isolated) + +build script run + +new files in build environment bundled into package + +test environment created (isolated) + +tests run on new package + +cleanup + + +Obtaining recipes +------------------ + +Existing recipes (best) + +https://github.com/AnacondaRecipes + +https://github.com/conda-forge + +Skeletons from other repositories (PyPI, CRAN, CPAN, RPM) + +DIY + + +Anaconda Recipes +---------------- + +Official recipes that Anaconda uses for building packages + +Since Anaconda 5.0, forked from conda-forge recipes. + +Intended to be compatible with conda-forge long-term + +Presently, ahead of conda-forge on use of conda-build 3 features + + +Conda-forge +----------- + +Numfocus-affiliated community organization made up of volunteers + +One github repository per recipe + +Fine granularity over permissions + +Heavy use of automation for building, deploying, and updating recipes + +Free builds on public CI services (TravisCI, CircleCI, Appveyor) + + +Skeletons +--------- + +Read metadata from upstream repository + +Translate that into a recipe + +Will save you some boilerplate work + +Might work out of the box (should not assume automatic, though) + +.. nextslide:: + +.. code-block:: bash + + conda skeleton pypi + + conda skeleton pypi + + conda skeleton pypi click + + conda skeleton pypi --recursive pyinstrument + + + conda skeleton cran + + conda skeleton cran + + conda skeleton cran acs + + conda skeleton cran --recursive biwt + + + +When all else fails, write a recipe +----------------------------------- + +Only required section: + + +.. code-block:: yaml + + package: + + name: abc + + version: 1.2.3 + + +Source types +------------ + +url + +git + +hg + +svn + +local path + +https://conda.io/docs/user-guide/tasks/build-packages/define-metadata.html#source-section + + +Source patches +-------------- + +patch files live alongside meta.yaml + +create patches with diff, git diff, or git format-patch + + +https://conda.io/docs/user-guide/tasks/build-packages/define-metadata.html#source-section + + +.. code-block:: yaml + + package: + + name: test-patch + + version: 1.2.3 + + source: + + url: https://zlib.net/zlib-1.2.11.tar.gz + + build: + + script: exit 1 + +Exercise: let’s make a patch +---------------------------- + +https://conda.io/docs/user-guide/tasks/build-packages/define-metadata.html#source-section + +Builds that fail leave their build folders in place + +look in output for source tree in: ``*/conda-bld/test-patch_/work`` + +cd there + + +.. nextslide:: + +https://conda.io/docs/user-guide/tasks/build-packages/define-metadata.html#source-section + +.. code-block:: bash + + git init + + git add * + + git commit -am “init” + + edit file of choice + + git commit -m “changing file because …” + + git format-patch HEAD~1 + +.. nextslide:: + + +copy that patch back alongside meta.yaml + +modify meta.yaml to include the patch + + + +Multiple sources +---------------- + +.. code-block:: yaml + + source: + - url: https://package1.com/a.tar.bz2 + folder: stuff + - url: https://package1.com/b.tar.bz2 + folder: stuff + patches: + - something.patch + - git_url: https://github.com/conda/conda-build + folder: conda-build + + +https://conda.io/docs/user-guide/tasks/build-packages/define-metadata.html#source-section + +Build options +------------- + +number: version reference of recipe (as opposed to version of source code) + +script: quick build steps, avoid separate build.sh/bld.bat files + +skip: skip building recipe on some platforms + +entry_points: python code locations to create executables for + +run_exports: add dependencies to downstream consumers to ensure compatibility + +https://conda.io/docs/user-guide/tasks/build-packages/define-metadata.html#build-section + + + +Requirements +------------ + +build + +host + +run + + +Requirements: build vs. host +---------------------------- + +Historically, only build + +Still fine to use only build + +host introduced for cross compiling + +host also useful for separating build tools from packaging environment + + +.. nextslide:: + +If in doubt, put everything in host + +build is treated same as host for old-style recipes +(only build, no {{ compiler() }}) + +packages are bundled from host env, not build env + + +Post-build tests +---------------- + +Help ensure that you didn’t make a packaging mistake + +Ideally checks that necessary shared libraries are included as dependencies + + + +Post-build tests: dependencies +------------------------------ + +Describe dependencies that are required for the tests +(but not for normal package usage) + +..code-block:: yaml + + test: + requires: + - pytest + + + +Post-build tests: test files +---------------------------- + + +run_test.pl, run_test.py, run_test.r, run_test.lua + +Windows + +Linux/Mac + +run_test.bat + +run_test.sh + + + +Post-build tests +---------------- + +May have specific requirements + +May specify files that must be bundled for tests (source_files) + +imports: language specific imports to try, to verify correct installation + +commands: sequential shell-based commands to run (not OS-specific) + + +https://conda.io/docs/user-guide/tasks/build-packages/define-metadata.html#test-section + + +Import tests +------------ + +.. code-block:: yaml + + test: + + imports: + + - dateutil + + - dateutil.rrule + + - dateutil.parser + + - dateutil.tz + + +Test commands +------------- + +.. code-block:: yaml + +test: + commands: + - curl --version + - curl-config --features # [not win] + - curl-config --protocols # [not win] + - curl https://some.website.com + + +Outputs - more than one pkg per recipe +-------------------------------------- + +.. code-block:: yaml + + package: + name: some-split + version: 1.0 + + outputs: + - name: subpkg + + - name: subpkg2 + + +.. nextslide:: + +* Useful for consolidating related recipes that share (large) source + +* Reduce update burden + +* Reduce build time by keeping some parts of the build, while looping over other parts + +* Also output different types of packages from one recipe (wheels) + + +https://conda.io/docs/user-guide/tasks/build-packages/define-metadata.html#outputs-section + + +Outputs rules +------------- + +* list of dicts + +* each list must have name or type key + +* May use all entries from build, requirements, test, about sections + +* May specify files to bundle either using globs or by running a script + + +Outputs examples +---------------- + +https://github.com/AnacondaRecipes/curl-feedstock/blob/master/recipe/meta.yaml + + +https://github.com/AnacondaRecipes/aggregate/blob/master/ctng-compilers-activation-feedstock/recipe/meta.yaml + + +Exercise: split a package +------------------------- + +Curl is a library and an executable. Splitting them lets us clarify where Curl is only a build time dependency, and where it also needs to be a runtime dependency. + +Starting point: + +https://github.com/conda-forge/curl-feedstock/tree/master/recipe + + +.. nextslide:: + +Solution: + +https://github.com/AnacondaRecipes/curl-feedstock/tree/master/recipe + + +About section +------------- + +.. image:: images/about_section.png + + + +Extra section: free-for-all +--------------------------- + +* Used for external tools or state management + +* No schema + +* Conda-forge’s maintainer list + +* Conda-build’s notion of whether a recipe is “final” + + +https://conda.io/docs/user-guide/tasks/build-packages/define-metadata.html#extra-section + + +Break time! +----------- + +Advanced recipe tricks coming next + + + +Conditional lines (selectors) +----------------------------- + +:: + + some_content # [some expression] + + +content inside ``[...]`` is eval’ed + +namespace includes OS info, python info, and a few others + + +https://conda.io/docs/user-guide/tasks/build-packages/define-metadata.html#preprocessing-selectors + + +Exercise: limit a recipe to only Linux +-------------------------------------- + +.. code-block:: yaml + + package: + name: example_skip_recipe + version: 1.0 + + build: + skip: True + +.. nextslide:: + +.. code-block:: yaml + + package: + name: example_skip_recipe + version: 1.0 + + build: + skip: True# [not linux] + + +Intro to templating with Jinja2 +-------------------------------- + +* Fill in information dynamically + + - git tag info + + - setup.py recipe data + + - centralized version numbering + + - string manipulation + +How does templating save you time? +---------------------------------- + +.. code-block:: yaml + + {% set version = "3.0.2" %} + package: + name: example + version: {{ version }} + source: + url: https://site/{{version}}.tgz + + +Jinja2 templating in meta.yaml +------------------------------ + +Set variables:: + + {% set somevar=”someval” %} + +Use variables:: + + {{ somevar }} + +Expressions in ``{{ }}`` are roughly python + + +Jinja2 conditionals +------------------- + +Selectors are one line only. When you want to toggle a block, +use jinja2:: + + {%- if foo -%} + + toggled content + + on many lines + + {% endif %} + + +Exercise: use Jinja2 to reduce edits +------------------------------------ + +.. code-block:: yaml + + package: + + name: abc + + version: 1.2.3 + + source: + + url: http://my.web/abc-1.2.3.tgz + + +Variants: Jinja2 on steroids +---------------------------- + +Matrix specification in yaml files + +.. code-block:: yaml + + somevar: + + - 1.0 + + - 2.0 + + anothervar: + + - 1.0 + + +All variant variables exposed in jinja2 +--------------------------------------- + +In meta.yaml, + +``{{ somevar }}`` + +And this loops over values + + +Exercise: try looping +--------------------- + +meta.yaml: + +.. code-block:: yaml + + package: + + name: abc + + version: 1.2.3 + + build: + + skip: True # [skipvar] + + conda_build_config.yaml: + + skipvar: + + - True + + - False + +.. nextslide:: + +meta.yaml: + +.. code-block:: yaml + + package: + + name: abc + + version: 1.2.3 + + requirements: + + build: + + - python {{ python }} + + run: + + - python {{ python }} + +conda_build_config.yaml: + +.. code-block:: yaml + + python: + + - 2.7 + + - 3.6 + +.. nextslide:: + +meta.yaml: + +.. code-block:: yaml + + package: + + name: abc + + version: 1.2.3 + + requirements: + + build: + + - python + + run: + + - python + +.. nextslide:: + +conda_build_config.yaml: + +.. code-block:: yaml + + python: + + - 2.7 + + - 3.6 + + +Jinja2 functions +---------------- + +loading source data: + + ``load_setup_py_data`` + + ``load_file_regex`` + +Dynamic Pinning: + + ``pin_compatible`` + + ``pin_subpackage`` + +Compatibility Control: + + ``compiler`` + + ``cdt`` + + +Loading setup.py data +--------------------- + +.. code-block:: yaml + + {% set setup_data = load_setup_py_data() %} + + package: + name: abc + version: {{ setup_data[‘version’] }} + + +* Primarily a development recipe tool - release recipes specify version instead, and template source download link + +* Centralizing version info is very nice - see also versioneer, setuptools_scm, autover, and many other auto-version tools + +Loading arbitrary data +---------------------- + +.. code-block:: yaml + + {% set data = load_file_regex(load_file='meta.yaml', + regex_pattern='git_tag: ([\\d.]+)') %} + + package: + name: conda-build-test-get-regex-data + version: {{ data.group(1) }} + +* Useful when software provides version in some arbitrary file + +* Primarily a development recipe tool - release recipes specify version instead, and template source download link + + +Dynamic pinning +--------------- + +Use in meta.yaml, generally in requirements section: + +.. code-block:: yaml + + requirements: + + host: + + - numpy + + run: + + - {{ pin_compatible(‘numpy’) }} + +.. nextslide:: + + +Use in meta.yaml, generally in requirements section: + +.. code-block:: yaml + + requirements: + host: + - numpy + run: + - {{ pin_compatible(‘numpy’) }} + + +* Pin run req based on what is present at build time + + +Dynamic pinning in practice +--------------------------- + +Used a lot with numpy: + +https://github.com/AnacondaRecipes/scikit-image-feedstock/blob/master/recipe/meta.yaml + +Dynamic pinning within recipes +------------------------------ + +Refer to other outputs within the same recipe + + - When intradependencies exist + + - When shared libraries are consumed by other libraries + +https://github.com/AnacondaRecipes/aggregate/blob/master/clang/meta.yaml + + +Compilers +--------- + +Use in meta.yaml in requirements section: + +.. code-block:: yaml + + + requirements: + build: + - {{ compiler(‘c’) }} + +* explicitly declare language needs + +* compiler packages can be actual compilers, or just activation scripts + +* Compiler packages utilize run_exports to add necessary runtime dependencies automatically + + +``run_exports`` +--------------- + +“if you build and link against library abc, you need a runtime dependency on library abc” + +This is annoying to keep track of in recipes. + + +Upstream package “abc” (already built) + +.. code-block:: yaml + + + package: + name: abc + version: 1.0 + + build: + run_exports: + - abc 1.0.* + + +Downstream recipe + +.. code-block:: yaml + + requirements: + host: + - abc + + +Downstream package + +.. code-block:: yaml + + requirements: + host: + - abc 1.0 0 + run: + - abc 1.0.* + +.. nextslide:: + +.. image:: images/run_exports.png + + +.. nextslide:: + +* Add host or run dependencies for downstream packages that depend on upstream that specifies run_exports + +* Expresses idea that “if you build and link against library abc, you need a runtime dependency on library abc” + +* Simplifies version tracking + + +Requirements: run_exports +------------------------- + +.. image:: images/req_run_exports.png + + + + +Uploading packages: anaconda.org +-------------------------------- + +* Sign-up: + + - ``https://anaconda.org/`` + +* Requirement: + + - ``conda install anaconda-client`` + +* CLI: anaconda upload path-to-package + +* conda-build auto-upload: + + - ``conda config --set anaconda_upload True`` + + + +Uploading packages: PyPI +------------------------ + + +* Sign-up: ``https://pypi.org/account/register/`` + +* Twine: ``pip install twine`` + +* Upload with twine to Test PyPI: + + - ``twine upload --repository-url https://test.pypi.org/legacy/ dist/* + +* Upload to PyPI: ``twine upload dist/*`` + + +Anaconda Survey +--------------- + +https://www.surveymonkey.com/r/conda + + + +Install + +Development install + +Copies package into site-packages + +Adds a .pth file to site-packages, pointed at package source root + +Used when creating conda packages + +Used when developing software locally + +Normal priority in sys.path + +End of sys.path (only found if nothing else comes first) + +https://grahamwideman.wikispaces.com/Python-+site-package+dirs+and+.pth+files + + + +What about setup.py? + +#!/usr/bin/env pythonfrom setuptools import setupsetup(name='Distutils', version='1.0', description='Python Distribution Utilities', author='Greg Ward', author_email='gward@python.net', url='https://www.python.org/sigs/distutils-sig/', packages=['distutils', 'distutils.command'], ) + + +https://docs.python.org/2/distutils/setupscript.html + +Lists of extensions to be compiled +---------------------------------- +What does setup.py do? + +Version & package metadata + +List of packages to include + +List of other files to include + +Lists of dependencies + + +#!/usr/bin/env pythonfrom setuptools import setupsetup(name='mypkg', version='1.0', # list folders, not files packages=['mypkg', 'mypkg.subpkg'], ) +------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ +Let’s write setup.py + + +creates eggs by default (people spend time fighting this later in the process) +------------------------------------------------------------------------------ +Setuptools + +Separate library (ships with Python by default, though) + +Adds entry point capability + +provides find_packages function (use with caution) + + +mypkg/ __init__.py subpkg/ __init__.py a.py +------------------------------------------------------------------- +Where does setup.py go? + +mypkg-src + +setup.py + + +New outer folder + +setup.py alongside package to be installed + +mypkg is what will get installed + +mypkg-src is what gets linked to by develop + +Go look in your site-packages folder +------------------------------------ +Try installing your package + +cd mypkg-src + +python setup.py install + +python -c “import mypkg.subpkg.a” + + + + + + +Making packages the easy way + +https://github.com/audreyr/cookiecutter + + + + + +conda install -c conda-forge cookiecutter + + +https://github.com/conda/cookiecutter-conda-python +-------------------------------------------------- +Let’s make a project + +cookiecutter https://goo.gl/Jge1g8 + + + +That’s a shortened link to: + + +What did we get? +---------------- + +install_requires=['click'], ) +---------------------------------- +Adding requirements in setup.py + +#!/usr/bin/env pythonfrom distutils.core import setupsetup(name='mypkg', version='1.0', # list folders, not files packages=['mypkg', 'mypkg.subpkg'], + + +Donald Stufft (PyPA): Abstract vs. Concrete dependencies +-------------------------------------------------------- +Requirements in requirements.txt + +common mistake + +requirements.txt often from pip freeze + +Pinned way too tightly. OK for env creation, bad for packaging. + + +https://caremad.io/posts/2013/07/setup-vs-requirement/ + + + +Requirements in setup.cfg (ideal) + +[metadata]name = my_packageversion = attr: src.VERSION[options]packages = find:install_requires = click + + +http://setuptools.readthedocs.io/en/latest/setuptools.html#configuring-setup-using-setup-cfg-files + +Parseable without execution, unlike setup.py + +Up next: producing redistributable artifacts +-------------------------------------------- +Break time! + + +eggs (deprecated) +----------------- +Redistributable artifacts + +sdists + +wheels + +conda packages + + +python setup.py sdist +--------------------- +When/how to use an sdist + +Pure python (no build requirements) + + +Wheels vs. Conda packages +------------------------- + +Wheels + +Conda packages + +Employed by pip, blessed by PyPA + +Foundation of Anaconda ecosystem + +Used by any python installation + +Used by conda python installations + +Mostly specific to Python ecosystem + + General purpose (any ecosystem) + +Good mechanism for specifying range of python compatibility + + Primitive support for multiple python + + versions (noarch) + +Depends on static linking or other system package managers to provide core libraries + +Can bundle core system-level shared libraries as packages, and resolve dependencies + +Open-source, actively developedhttps://github.com/conda/conda-build +-------------------------------------------------------------------- +Introducing conda-build + +Orchestrates environment creation, activation, and build/test processes + +Can build conda packages and/or wheels + +Separate project from conda, but very tightly integrated + + +version: 1.0 +------------ +Getting conda-build to work for you + +Input: meta.yaml files + +package: + + name: mypkg + + +conda build mypkg-src +--------------------- +Let’s use conda-build + +conda install conda-build + + +cleanup +------- +What happened? + +templates filled in, recipe interpreted + +build environment created (isolated) + +build script run + +new files in build environment bundled into package + +test environment created (isolated) + +tests run on new package + + +DIY +--- +Obtaining recipes + +Existing recipes (best) + +https://github.com/AnacondaRecipes + +https://github.com/conda-forge + +Skeletons from other repositories (PyPI, CRAN, CPAN, RPM) + + + + +Presently, ahead of conda-forge on use of conda-build 3 features +---------------------------------------------------------------- +AnacondaRecipes + +Official recipes that Anaconda uses for building packages + +Since Anaconda 5.0, forked from conda-forge recipes. + +Intended to be compatible with conda-forge long-term + + +Free builds on public CI services (TravisCI, CircleCI, Appveyor) +---------------------------------------------------------------- +Conda-forge + +Numfocus-affiliated community organization made up of volunteers + +One github repository per recipe + +Fine granularity over permissions + +Heavy use of automation for building, deploying, and updating recipes + + +Might work out of the box (should not assume automatic, though) +--------------------------------------------------------------- +Skeletons + +Read metadata from upstream repository + +Translate that into a recipe + + + +Will save you some boilerplate work + + +conda skeleton pypi --recursive pyinstrument +-------------------------------------------- +conda skeleton pypi + +conda skeleton pypi + +conda skeleton pypi click + + + + +conda skeleton cran --recursive biwt +------------------------------------ +conda skeleton cran + +conda skeleton cran + +conda skeleton cran acs + + + + +version: 1.2.3 +-------------- +When all else fails, write a recipe + +Only required section: + +package: + + name: abc + + +local path +---------- +Source types + +url + +git + +hg + +svn + + +https://conda.io/docs/user-guide/tasks/build-packages/define-metadata.html#source-section + +create patches with diff, git diff, or git format-patch +------------------------------------------------------- +Source patches + +patch files live alongside meta.yaml + + +https://conda.io/docs/user-guide/tasks/build-packages/define-metadata.html#source-section + +Exercise: let’s make a patch +---------------------------- +package: + + name: test-patch + + version: 1.2.3 + +source: + + url: https://zlib.net/zlib-1.2.11.tar.gz + +build: + + script: exit 1 + + +https://conda.io/docs/user-guide/tasks/build-packages/define-metadata.html#source-section + +Exercise: let’s make a patch +---------------------------- + +Builds that fail leave their build folders in place + +look in output for source tree in: ``*/conda-bld/test-patch_/work`` + +``cd there`` + + + +Exercise: let’s make a patch +---------------------------- + +.. code-block:: bash + + git init + + git add * + + git commit -am “init” + + edit file of choice + + git commit -m “changing file because …” + + git format-patch HEAD~1 + + +.. nextslide:: + +copy that patch back alongside ``meta.yaml`` + +modify meta.yaml to include the patch + + +Multiple sources +---------------- + +.. .code-block:: yaml + +source: + - url: https://package1.com/a.tar.bz2 + folder: stuff + - url: https://package1.com/b.tar.bz2 + folder: stuff + patches: + - something.patch + - git_url: https://github.com/conda/conda-build + folder: conda-build + + +https://conda.io/docs/user-guide/tasks/build-packages/define-metadata.html#source-section + + +Build options +------------- + +number: + version reference of recipe (as opposed to version of source code) + +script: + quick build steps, avoid separate build.sh/bld.bat files + +skip: + skip building recipe on some platforms + +entry_points: + python code locations to create executables for + +run_exports: + add dependencies to downstream consumers to ensure compatibility + +https://conda.io/docs/user-guide/tasks/build-packages/define-metadata.html#build-section + +Requirements +------------ + +build + +host + +run + +host also useful for separating build tools from packaging environment +---------------------------------------------------------------------- + +Requirements: build vs. host + +Historically, only build + +Still fine to use only build + +host introduced for cross compiling + + +packages are bundled from host env, not build env +------------------------------------------------- + +Requirements: build vs. host + +If in doubt, put everything in host + +build is treated same as host for old-style recipes +(only build, no ``{{ compiler() }}``) + + +Post-build tests +---------------- + +Help ensure that you didn’t make a packaging mistake + +Ideally checks that necessary shared libraries are included as dependencies + + +-------- +Post-build tests: dependencies +------------------------------ + +Describe dependencies that are required for the tests (but not for normal package usage) + +.. code-block:: yaml + + test: + + requires: + + - pytest + + +Post-build tests: test files +---------------------------- + +:: + + run_test.pl, run_test.py, run_test.r, run_test.lua + +Windows:: + + run_test.bat + + +Linux/Mac:: + + run_test.sh + + +Post-build tests +---------------- + +May have specific requirements + +May specify files that must be bundled for tests (source_files) + +imports: + language specific imports to try, to verify correct installation + +commands: + sequential shell-based commands to run (not OS-specific) + +https://conda.io/docs/user-guide/tasks/build-packages/define-metadata.html#test-section + +Import tests +------------ + +..code-block:: yaml + + test: + imports: + - dateutil + - dateutil.rrule + - dateutil.parser + - dateutil.tz + +Test commands +------------- + +.. code-block:: yaml + + test: + commands: + - curl --version + - curl-config --features # [not win] + - curl-config --protocols # [not win] + - curl https://some.website.com + + +Outputs - more than one pkg per recipe +-------------------------------------- + +package: + + name: some-split version: 1.0 + +outputs: + + - name: subpkg + - name: subpkg2 + +subpkg + +subpkg2 + + +.. nextslide:: + +Useful for consolidating related recipes that share (large) source + +Reduce update burden + +Reduce build time by keeping some parts of the build, while looping over other parts + +Also output different types of packages from one recipe (wheels) + + +https://conda.io/docs/user-guide/tasks/build-packages/define-metadata.html#outputs-section + + +Outputs rules +------------- + +list of dicts + +each list must have name or type key + +May use all entries from build, requirements, test, about sections + +May specify files to bundle either using globs or by running a script + + +https://conda.io/docs/user-guide/tasks/build-packages/define-metadata.html#outputs-section + +https://github.com/AnacondaRecipes/aggregate/blob/master/ctng-compilers-activation-feedstock/recipe/meta.yaml + +Outputs examples +---------------- + +https://github.com/AnacondaRecipes/curl-feedstock/blob/master/recipe/meta.yaml + +https://github.com/conda-forge/curl-feedstock/tree/master/recipe + +Exercise: split a package +------------------------- + +Curl is a library and an executable. Splitting them lets us clarify where Curl is only a build time dependency, and where it also needs to be a runtime dependency. + +Starting point: + + +https://conda.io/docs/user-guide/tasks/build-packages/define-metadata.html#outputs-section + +https://github.com/AnacondaRecipes/curl-feedstock/tree/master/recipe +-------------------------------------------------------------------- +Exercise: split a package + +Solution: + + +https://conda.io/docs/user-guide/tasks/build-packages/define-metadata.html#outputs-section + +About section +------------- +https://conda.io/docs/user-guide/tasks/build-packages/define-metadata.html#about-section + + +Provide this stuff + +Conda-build’s notion of whether a recipe is “final” +--------------------------------------------------- +Extra section: free-for-all + +Used for external tools or state management + +No schema + +Conda-forge’s maintainer list + + +https://conda.io/docs/user-guide/tasks/build-packages/define-metadata.html#extra-section + +Advanced recipe tricks coming next +---------------------------------- +Break time! + + +namespace includes OS info, python info, and a few others +--------------------------------------------------------- +Conditional lines (selectors) + +some_content # [some expression] + + + +content inside [] is eval’ed + + +https://conda.io/docs/user-guide/tasks/build-packages/define-metadata.html#preprocessing-selectors + +Exercise: limit a recipe to only Linux +-------------------------------------- + +https://conda.io/docs/user-guide/tasks/build-packages/define-metadata.html#preprocessing-selectors + + + +Intro to templating with Jinja2 + +Fill in information dynamically + +git tag info + +setup.py recipe data + +centralized version numbering + +string manipulation + + +Expressions in {{ }} are roughly python +--------------------------------------- +Jinja2 templating in meta.yaml + +Set variables: + +{% set somevar=”someval” %} + +Use variables: + +{{ somevar }} + + + + +Jinja2 conditionals + +Selectors are one line only. When you want to toggle a block, use jinja2: + +{%- if foo -%} + +toggled content + +on many lines + +{% endif %} + + +url: http://my.web/abc-1.2.3.tgz +-------------------------------- +Exercise: use Jinja2 to reduce edits + +package: + + name: abc + + version: 1.2.3 + +source: + + +- 1.0 +----- +Variants: Jinja2 on steroids + +Matrix specification in yaml files + +somevar: + + - 1.0 + + - 2.0 + +anothervar: + + +And this loops over values +-------------------------- +All variant variables exposed in jinja2 + +In meta.yaml, + + + +{{ somevar }} + + + + +skip: True # [skipvar] +---------------------- +Exercise: try looping + +meta.yaml: + +package: + + name: abc + + version: 1.2.3 + +build: + + +conda_build_config.yaml: + +skipvar: + + - True + +- False + +- python {{ python }} +--------------------- +Exercise: try looping + +meta.yaml: + +package: + + name: abc + + version: 1.2.3 + +requirements: + + build: + + - python {{ python }} + + run: + + +conda_build_config.yaml: + +python: + + - 2.7 + +- 3.6 + +- python +-------- +Exercise: try looping + +meta.yaml: + +package: + + name: abc + + version: 1.2.3 + +requirements: + + build: + + - python + + run: + + +conda_build_config.yaml: + +python: + + - 2.7 + +- 3.6 + +cdt +--- +Jinja2 functions + +load_setup_py_data + +load_file_regex + +pin_compatible + +pin_subpackage + +compiler + + +Dynamic pinning + +Loading source data + +Compatibility control + +Loading setup.py data +--------------------- + +{% set setup_data = load_setup_py_data() %} + + + +package: + + name: abc + + version: {{ setup_data[‘version’] }} + + + + + +Primarily a development recipe tool - release recipes specify version instead, and template source download link + +Centralizing version info is very nice - see also versioneer, setuptools_scm, autover, and many other auto-version tools + +Primarily a development recipe tool - release recipes specify version instead, and template source download link +---------------------------------------------------------------------------------------------------------------- +Loading arbitrary data + +{% set data = load_file_regex(load_file='meta.yaml', regex_pattern='git_tag: ([\\d.]+)') %} + +package: + + name: conda-build-test-get-regex-data + + version: {{ data.group(1) }} + +Useful when software provides version in some arbitrary file + + +- {{ pin_compatible(‘numpy’) }} +------------------------------- +Dynamic pinning + +Use in meta.yaml, generally in requirements section: + +requirements: + + host: + + - numpy + + run: + + +- {{ pin_compatible(‘numpy’) }} +------------------------------- +Dynamic pinning + +Use in meta.yaml, generally in requirements section: + +requirements: + + host: + + - numpy + + run: + + +Pin run req based on what is present at build time + +https://github.com/AnacondaRecipes/scikit-image-feedstock/blob/master/recipe/meta.yaml +-------------------------------------------------------------------------------------- +Dynamic pinning in practice + +Used a lot with numpy: + + + + +Dynamic pinning within recipes + +Refer to other outputs within the same recipe + +when intradependencies exist + +when shared libraries are consumed by other libraries + +https://github.com/AnacondaRecipes/aggregate/blob/master/clang/meta.yaml + + +Compiler packages utilize run_exports to add necessary runtime dependencies automatically +----------------------------------------------------------------------------------------- +Compilers + +Use in meta.yaml in requirements section: + +requirements: build: - {{ compiler(‘c’) }} + +explicitly declare language needs + +compiler packages can be actual compilers, or just activation scripts + + +run_exports +----------- + +package: + + name: abc + + version: 1.0 + + + +build: + + run_exports: + + - abc 1.0.* + +Upstream package “abc” (already built) + +Downstream recipe + +requirements: + + host: + + - abc + + + +requirements: + + host: + + - abc 1.0 0 + + run: + + - abc 1.0.* + + + +Downstream package + +Simplifies version tracking +--------------------------- +run_exports + +Add host or run dependencies for downstream packages that depend on upstream that specifies run_exports + +expresses idea that “if you build and link against library abc, you need a runtime dependency on library abc” + + +Requirements: run_exports +------------------------- + +build + +host + +run + +“Strong” run_exports + +“Weak” run_exports + +Uploading packages: anaconda.org +-------------------------------- + +Uploading packages: PyPI +------------------------ + +https://www.surveymonkey.com/r/conda +------------------------------------ +Anaconda Survey +