diff --git a/.github/workflows/docs_publish.yaml b/.github/workflows/docs_publish.yaml new file mode 100644 index 00000000..a1ba5dbc --- /dev/null +++ b/.github/workflows/docs_publish.yaml @@ -0,0 +1,57 @@ +name: Publish Documentation + +on: + push: + branches: [ 'main' ] + pull_request: + branches: [ main ] +jobs: + build: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: [3.9] + + steps: + - uses: actions/checkout@v3 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + cache: 'pip' # caching pip dependencies + - name: Install Dependencies + run: | + pip install --upgrade pip + pip install sphinx sphinx-rtd-theme myst-parser + #---------------------------------------------- + # Build documentation + #---------------------------------------------- + - name: Build documentation + run: sphinx-build -b html docs/source docs/build + + #---------------------------------------------- + # Clone documentation + #---------------------------------------------- + - uses: actions/checkout@v3 + with: + ref: gh-pages + path: pages + #---------------------------------------------- + # Move documentation + #---------------------------------------------- + - name: Move documentation + run: | + rm -r pages/docs/* + mv -f docs/build/* pages/docs/ + #---------------------------------------------- + # Commit & Push changes + #---------------------------------------------- + - name: Commit and Push + continue-on-error: true + run: | + cd pages + git config user.name github-actions + git config user.email github-actions@github.com + git add -f docs/ + git commit -m "Updated documentation" + git push diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 858c91a7..65f74981 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -2,7 +2,7 @@ name: Tests on: push: - branches: [ '*' ] + branches-ignore: [ gh-pages ] pull_request: branches: [ main ] jobs: @@ -13,47 +13,44 @@ jobs: python-version: [3.9] steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} - #---------------------------------------------- - # ----- install & configure poetry ----- - #---------------------------------------------- - - name: Install Poetry - uses: snok/install-poetry@v1 - with: - virtualenvs-create: true - virtualenvs-in-project: true - installer-parallel: true - #---------------------------------------------- - # load cached venv if cache exists - #---------------------------------------------- - - name: Load cached venv - id: cached-poetry-dependencies - uses: actions/cache@v2 - with: - path: .venv - key: venv-${{ runner.os }}-${{ hashFiles('**/poetry.lock') }} - #---------------------------------------------- - # install dependencies if cache does not exist - #---------------------------------------------- - - name: Install dependencies - if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true' - continue-on-error: true - run: poetry install --no-interaction --no-root - #---------------------------------------------- - # install your root project, if required - #---------------------------------------------- - - name: Install library - run: poetry install --no-interaction - #---------------------------------------------- - # format, type check, and test your code - #---------------------------------------------- - - name: Black code formatting - run: | - poetry run black --check --diff . - - name: MyPy - run: | - poetry run mypy synth + # #---------------------------------------------- + # # ----- install & configure poetry ----- + # #---------------------------------------------- + # - name: Install Poetry + # uses: snok/install-poetry@v1 + # with: + # virtualenvs-create: true + # virtualenvs-in-project: true + # installer-parallel: true + # #---------------------------------------------- + # # load cached venv if cache exists + # #---------------------------------------------- + # - name: Load cached venv + # id: cached-poetry-dependencies + # uses: actions/cache@v3 + # with: + # path: .venv + # key: venv-${{ runner.os }}-${{ hashFiles('**/poetry.lock') }} + # #---------------------------------------------- + # # install dependencies if cache does not exist + # #---------------------------------------------- + # - name: Install dependencies + # if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true' + # continue-on-error: true + # run: poetry install --no-interaction --no-root + # #---------------------------------------------- + # # install your root project, if required + # #---------------------------------------------- + # - name: Install library + # run: poetry install --no-interaction + # #---------------------------------------------- + # # type check, and test your code + # #---------------------------------------------- + # - name: MyPy + # run: | + # poetry run mypy synth diff --git a/.gitignore b/.gitignore index ce1d1f92..d38d8b2a 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,6 @@ +# Mac files +.DS_store +_site # VS code .vscode @@ -21,4 +24,13 @@ docs/build *.pickle # csv files -*.csv \ No newline at end of file +*.csv + +# model files +*.pt + +# VS Code Counter +.VSCodeCounter + +# Parsing Sygus files +examples/sygus/parsing/**/*.py \ No newline at end of file diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 00000000..6d6a3cf0 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,9 @@ +- repo: https://github.com/astral-sh/ruff-pre-commit + # Ruff version. + rev: v0.4.3 + hooks: + # Run the linter. + - id: ruff + args: [ --fix ] + # Run the formatter. + - id: ruff-format \ No newline at end of file diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 65cdbd93..c5e9536d 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,4 +1,4 @@ -# Contributing to ProgSynth +# Contributing to DeepSynth2 Feel free to open an issue or pull request if you have any questions or suggestions. If you plan to work on an issue, let us know in the issue thread so we can avoid duplicate work. diff --git a/LICENSE.md b/LICENSE.md index 64697fd8..8a9047b4 100644 --- a/LICENSE.md +++ b/LICENSE.md @@ -1,6 +1,6 @@ MIT License -Copyright (c) 2022 Nathanaël FIJALKOW & Théo MATRICON +Copyright (c) 2025 Nathanaël FIJALKOW & Théo MATRICON Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/README.md b/README.md index 16675cfe..2c5cf56b 100644 --- a/README.md +++ b/README.md @@ -1,19 +1,26 @@ -![ProgSynth Logo](./images/logo.png) - -------------------------------------------------------------------------------- -[![Tests](https://github.com/nathanael-fijalkow/AutoSynth/actions/workflows/tests.yaml/badge.svg)](https://github.com/nathanael-fijalkow/AutoSynth/actions/workflows/tests.yaml) +[![Tests](https://github.com/SynthesisLab/DeepSynth/actions/workflows/tests.yaml/badge.svg)](https://github.com/SynthesisLab/DeepSynth/actions/workflows/tests.yaml) + +DeepSynth is a high-level framework that enables to leverage program synthesis for other domains such as reinforcement learning or system design. -ProgSynth is a high-level framework that enables to leverage program synthesis for other domains such as reinforcement learning or system design. +Key publications: +* [AAAI 2025 (Oral): EcoSearch: A Constant-Delay Best-First Search Algorithm for Program Synthesis](https://arxiv.org/abs/2412.17330) +* [SPIN 2023: WikiCoder: Learning to Write Knowledge-Powered Code](https://arxiv.org/abs/2303.08574) +* [AAAI 2022 (Oral): Scaling Neural Program Synthesis with Distribution-based Search](https://arxiv.org/abs/2110.12485) -- [More About ProgSynth](#more-about-progsynth) +-------------------------------------------------------------------------------- +Table of contents +- [More About DeepSynth](#more-about-DeepSynth) - [Combining Deep Learning with Theoretical Guarantees](#combining-deep-learning-with-theoretical-guarantees) - [A Scalable Framework](#a-scalable-framework) - [Installation](#installation) - [From Source](#from-source) - - [Install ProgSynth](#install-progsynth) + - [Install DeepSynth](#install-DeepSynth) - [Documentation](#documentation) + - [Online](https://theomat.github.io/DeepSynth/) + - [Local](#documentation) - [Troubleshooting](#troubleshooting) - [Examples](./examples) - [The Team](#the-team) @@ -21,17 +28,17 @@ ProgSynth is a high-level framework that enables to leverage program synthesis f -## More About ProgSynth +## More About DeepSynth -At a granular level, ProgSynth is a library that consists of the following components: +At a granular level, DeepSynth is a library that consists of the following components: | Component | Description | | ---- | --- | | [**synth**](./synth) | A high level synthesis libary | -| [**synth.generation**](./synth/generation) | A compilation of tools to generate objetcs needed for the synthesis, it is mainly used with deep learning | +| [**synth.generation**](./synth/generation) | A compilation of tools to generate objects needed for the synthesis, it is mainly used with deep learning | | [**synth.nn**](./synth/nn) | A library to build neural network with for synthesis | | [**synth.pbe**](./synth/pbe) | A library to work in the Programming By Example (PBE) framework | -| [**synth.pruning**](./synth/pruning) | A library with pruning strategies | +| [**synth.filter**](./synth/filter) | A library with filtering strategies | | [**synth.semantic**](./synth/semantic) | A library of program evaluators | | [**synth.syntax**](./synth/syntax) | A library to manipulate dsl, grammars, probabilistic grammars | | [**synth.utils**](./synth/utils) | Utility objects and functions that do not fit elsewhere | @@ -42,7 +49,7 @@ Elaborating Further: The advantage of "classic" algorithms are their theoretical guarantees. But many new deep learning based methods have emerged, they provide a tremendous efficiency but lose almost all theoretical guarantees. -ProgSynth provides already implemented algorithms that combine both approaches to get the best of both worlds: speed and guarantees! +DeepSynth provides already implemented algorithms that combine both approaches to get the best of both worlds: speed and guarantees! ### A Scalable Framework @@ -56,18 +63,29 @@ For example, you can split probabilistic grammars into disjoint sub grammars to ### From Source -If you are installing from source, you will need Python 3.7.1 or later. +If you are installing from source, you will need Python 3.8 or later. -#### Install ProgSynth +#### Install DeepSynth -ProgSynth can be installed from source with `pip`, `conda` or `poetry`. +DeepSynth can be installed from source with `pip`, `conda` or `poetry`. ```bash pip install . ``` +When using `poetry` in an CUDA environment, then you need to follow every `poetry install` or `poetry update` with: + +```bash +pip install torch +``` + +See this [open issue of poetry](https://github.com/python-poetry/poetry/issues/6409) for more information. + ## Documentation +[Online Documentation](https://synthesislab.github.io/DeepSynth/) + + You might want to generate html pages of the documentation locally, where usage, contribution guidelines and more can be found. In which case, you will need to use [Sphinx](https://www.sphinx-doc.org/en/master/). @@ -87,16 +105,21 @@ There are some known issues: - **seed = 0** is the **same as no seeding**. - if you get an error after installation try to update/upgrade ``numpy``, it is often due to a discrepancy between the version with which ``vose`` is compiled and the version the environment is running. -- some dependencies may be missing depending on the DSL you want to use, running any example script with -h will ist you the list of available DSL with your current installation. +- **if you have issues with ``vose``**, you can just uninstall ``vose``, generation speed will be slower but everything will work. +- some dependencies may be missing depending on the DSL you want to use, running any example script with -h will list you the list of available DSL with your current installation. ## The Team -ProgSynth is a project initiated by [Nathanaël Fijalkow](https://nathanael-fijalkow.github.io/) and joined by [Théo Matricon](https://theomat.github.io/). +DeepSynth is a project initiated by [Nathanaël Fijalkow](https://nathanael-fijalkow.github.io/) and by [Théo Matricon](https://theomat.github.io/). +It is based on the [DeepSynth](https://github.com/nathanael-fijalkow/DeepSynth) project of [Nathanaël Fijalkow](https://nathanael-fijalkow.github.io/), [Guillaume Lagarde](https://guillaume-lagarde.github.io/), [Théo Matricon](https://theomat.github.io/), [Kevin Ellis](https://www.cs.cornell.edu/~ellisk/), [Pierre Ohlmann](https://www.irif.fr/~ohlmann/), Akarsh Potta Former: -- [Gaëtan Margueritte](https://github.com/gaetanmargueritte) did a four-month internship. He created the regexp and transduction DSLs, the first tutorial and first drafts of code related to the use of user defined constants. +- (2023) [Félix Yvonnet](https://github.com/Felix-Yvonnet) did a 2 months internship to work on restarts, a future feature of DeepSynth. +- (2023) [Priscilla Tissot](https://fr.linkedin.com/in/priscilla-tissot-9493851b8) did a 7 weeks long internship working on the Carel neural network and trying to improve the performance of our prediction models. +- (2022) [Gaëtan Margueritte](https://github.com/gaetanmargueritte) did a four-month internship. He created the regexp and transduction DSLs, the first tutorial and first drafts of code related to the use of user defined constants. +- (2022) Utkarsh Rajan did a two-month internship. He contributed to the implementation of bucket search and worked on the tower DSL. ## License -ProgSynth has a MIT license, as found in the [LICENSE](LICENSE) file. +DeepSynth has a MIT license, as found in the [LICENSE](LICENSE.md) file. diff --git a/docs/source/about.md b/docs/source/about.md new file mode 100644 index 00000000..ad72011d --- /dev/null +++ b/docs/source/about.md @@ -0,0 +1,5 @@ +About +=== + +```{include} ../../README.md +--- diff --git a/docs/source/conf.py b/docs/source/conf.py index 1ea0e145..400c948f 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -3,7 +3,7 @@ # -- Project information project = "ProgSynth" -copyright = "2022, Nathanaël Fijalkow & Théo Matricon" +copyright = "2023, Nathanaël Fijalkow & Théo Matricon" author = "Nathanaël Fijalkow & Théo Matricon" release = "0.1" diff --git a/docs/source/grammars.md b/docs/source/grammars.md new file mode 100644 index 00000000..5ed3fae4 --- /dev/null +++ b/docs/source/grammars.md @@ -0,0 +1,80 @@ +# Grammars + +Grammars are programs generator, all our grammars are finite. +Typically when you instantiate a grammar you always specify a maximum depth. + +The main object of interest are probabilistic grammars on which most methods to enumerate and sample programs are provided. + + +Table of contents: + +- [Grammar Models](#grammar-models) + - [det-CFG](#det-cfg) + - [U-CFG](#u-cfg) +- [Probabilistic Grammars](#probabilistic-grammars) + + + +## Grammar Models + +Currently the only grammar model supported are [Context-free grammars](https://en.wikipedia.org/wiki/Context-free_grammar) (CFG). +All our rules have the following form: + +``` +S -> f S1 S2 ... Sk +S -> g +``` + +where ``S``, ``S1``, ..., ``Sk`` are non terminal and ``f`` is a primitive of arity ``k`` and ``g`` is a primitive of arity 0, in other words a constant. + +We have two different models: deterministic CFG and unambiguous CFG; while the latter is more expressive it is around 20% slower but used correctly the gains are huge. + +The ways to generate a grammar are mainly through static methods such as ``MyGrammarModel.depth_constraint(dsl, type_request)``. +Grammars albeit already complex objects are not the final object of interests in ProgSynth. +The most relevant methods are: + +- ``program in grammar`` which returns whether program belongs to the grammar or not; +- ``grammar.programs()`` which yields the number of programs contained in the grammar, do not convert it to float as this easily yield values over MAX_DOUBLE, hence we return an int to take advantage of the lack of limit for int in python; +- ``grammar.derive(...)`` which allows you to derive your program step by step; +- ``grammar.derive_all(...)`` which derives the whole given subtree for you and hands you the result; +- ``grammar.reduce_derivatons(...)`` which is like a fold over the derivation steps of the given program. + +### det-CFG + +A CFG which has the following property: +> For a given non-terminal ``S``, for any primitive ``f``, there is at most one derivation from ``S`` using primitive ``f`` + +In other words, it is deterministic to derive ``f`` from non-terminal ``S``. + +In ProgSynth this is the default model, that is ``CFG``. +If you do not use [sharpening](sharpening.md) for example, then ProgSynth uses this model when producing a grammar. + +### U-CFG + +A CFG which has the following property: +> For a tree/program ``t``, there exists at most one derivation for tree/program ``t`` in the grammar + +In other words, there is no ambiguity to derive a program from the grammar, but locally it may be ambiguous, that is you have to try all derivation rules for the primitive to find out later which is the one that allows deriving the program. + +``UCFG`` in ProgSynth can express all regular tree languages and is generated when you use [sharpening](sharpening.md). + +## Probabilistic Grammars + +We offer tagged grammars, those are grammars where derivations are tagged with a generic type, replacing 'probabilistic' with 'tagged' in what is following will work as well. +The most relevant one is when derivations are tagged with float giving you probabilistic grammars. +> For a given non-terminal ``S``, the set of all derivations from ``S`` make up a probability distribution, *i.e.* sum up to 1. + +There are two models: ``ProbGrammar`` and ``ProbUGrammar`` respectively working for ``CFG`` and ``UCFG``. +Basically adding a U for class and a u_ for methods to the classic method will yield the equivalent methods for the unambiguous model. + +Probabilistic grammars offer a wide range of interesting methods to generate programs: + +- ``pgrammar.sample()`` sample a random program from the grammar, you will need to first call ``pgrammar.init_sampling(seed)`` for sampling to work, sampling is optimised compared to naive sampling; +- ``enumerate_prob_(u_)_grammar(pgrammar)`` which gives you an enumerator that will enumerate programs in the grammar by decreasing order of probability; +- ``split(pgrammar, n)`` which gives you ``n`` disjoint probabilistic unambiguous grammars that make up a partition of the original given ``pgrammar``, the main intereset is to easily parallelise the enumeration. + +Of course, since probabilistic grammars are grammars they also offer the same methods as classic grammars. + +**But I want to enumerate programs by size?** + +Well, you can just use ``Prob(U)Grammar.uniform(grammar)`` and enumerate that probabilistic grammar will give you an enumeration by program size. diff --git a/docs/source/images/pipeline.png b/docs/source/images/pipeline.png new file mode 100644 index 00000000..65472db2 Binary files /dev/null and b/docs/source/images/pipeline.png differ diff --git a/docs/source/index.rst b/docs/source/index.rst index bd28456a..3a33815b 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -7,11 +7,15 @@ ProgSynth is a high-level framework that enables to leverage program synthesis f Contents --------- .. toctree:: + about introduction usage tutorial - examples + type_system + grammars + Specifications PBE - pruning + prediction + sharpening contributing license diff --git a/docs/source/introduction.md b/docs/source/introduction.md index a7cfe72d..7801464a 100644 --- a/docs/source/introduction.md +++ b/docs/source/introduction.md @@ -1,5 +1,78 @@ -Introduction -=== +# Introduction -```{include} ../../README.md ---- +This page seeks to answer the following question: +> How does ProgSynth works at a high level? + + +Table of contents: + +- [What do you give in?](#what-do-you-give-in) +- [What do you get out?](#what-do-you-get-out) +- [How does that work?](#how-does-that-work) + - [Sharpening (Optional)](#sharpening-optional) + - [Compilation](#compilation) + - [Prediction (Optional in the future)](#prediction-optional-in-the-future) + - [Splitting (Optional)](#splitting-optional) + - [Enumeration](#enumeration) + + + +## What do you give in? + +ProgSynth only needs two things: a language to work on and a check function. + +The provided language is called a domain specific language (DSL). +The different kind of functions that can manipulate your data are provided and the framework will automatically generate the associated grammar. +Specifyign a language with its types is quite fast, it can be done in matters of minutes with the help of ProgSynth which has helper functions to help you focus on what really matters here: the synthesis. + +The check function is a python function, it can contain whatever code you might need to check that a given program satisfy your constraints. An example of such a function is to check whether your program matches the given examples of input-output pairs. + +## What do you get out? + +At the end you get a program tthat satisfies your condition, or if all programs have been enumerated, you get that there is no program in the given grammar that matches your specification. In practice the negative answer is never given since the number of programs grow exponentially, it would be infeasible to enumerate them all, so we recommend to use a timeout after which the search is stopped. + +## How does that work? + +This section attemps to give you a high-level overview of how it works. +Here is a figure that gives you a rough overview of how it works: +![pipeline](./images/pipeline.png) + +### Sharpening (Optional) + +Sharepning enables you to add syntactic constraints on the search space of programs, this enables to have large speed-ups in the enumeration step. +See [the page on sharpening](sharpening.md) for more details. +If you want to add syntactic constraints to your grammar then you need to write them and give them to ProgSynth in the compilation step. + +### Compilation + +The language that you give is typed and you also give at least a depth constraint, to guarantee that programs have a maximum depth. +Of course, you also specify the type of the program that you want to generate. +This language is built into a context-free grammar (CFG). + +If you have specified constraints through [sharpening](sharpening.md), then the constraints and the grammar are compiled into deterministic bottom-up tree automata. +The intersection is then computed and transformed back into a CFG. + +### Prediction (Optional in the future) + +If you have trained a model to produce program distributions over grammars, then you can use this step otherwise fear not because this will not be mandatory in the future. + +A model (often a neural network) takes as an input your specification and other information that you trained it on and then produce a vector which we translate into probabilities for our CFG. +That means that we have a probabilistic CFG (PCFG). + +### Splitting (Optional) + +You have multiple CPUs and you want to parallelize the search? +Well, we have a ``split`` function that takes a PCFG and split it into as many fragments as you want. +This means that each fragment of the original PCFG is independent that is no other fragment contains its programs. +In other words, with splitting you can have linear speed-up in your enumeration with the number of CPUs. + +Do note that while splitting is provided in ProgSynth we do not provide the parallelization framework, you will have to do it yourself. + +### Enumeration + +This PCFG actually gives us an order over programs. Programs can be ordered by their probability. +Therefore ProgSynth will enumerate programs in order of decreasing probability. That means that the most likely program will be enumerated first. +All of the time for synthesis is spent here and in most cases the cost to call your check function is what bounds the runtime. + +When a program is enumerated, we call your check function with the program as argument. +If your function returns ``True`` then we just stop here and returns the program, otherwise we continue. diff --git a/docs/source/prediction.md b/docs/source/prediction.md new file mode 100644 index 00000000..a9e90439 --- /dev/null +++ b/docs/source/prediction.md @@ -0,0 +1,144 @@ +# Prediction + +In order to produce a P(U)CFG from a (U)CFG, we need what we call *a prediction*. +That is a function which tags derivation rules with probabilities. + + +Table of contents: + +- [Neural Network](#neural-network) + - [Prediction Layers](#prediction-layers) + - [Instanciation](#instanciation) + - [Learning](#learning) + - [Tensor to Grammar](#tensor-to-grammar) + - [Task2Tensor](#task-to-tensor) + - [Example Model for PBE](#example-model-for-pbe) + + + +## Neural Network + +ProgSynth offers tools to easily use Neural networks to predict probabilities. + +### Prediction Layers + +The main tools are: ``DetGrammarPredictorLayer`` and ``UGrammarPredictorLayer`` which are respectively the same object for CFGs and UCFGs. +There are layers which maps tensors from ``(B, I)`` to ``(B, N)`` where ``B`` is the batch size, ``I`` is a parameter of the layer and ``N`` depends on the grammar. +A ``(N,)`` tensor can then be transformed into a tensor log-probability grammars thanks to ``tensor2log_prob_grammar``. +In case one wants to enumerate, a tensor log-probability grammar can always be transformed into a P(U)CFG. + +These layers however give a constant probability to all ``Variable`` and ``Constant`` object of the grammar since they can hardly be predicted. +The given probability can be changed anytime through ``layer.variable_probability``. + +#### Instanciation + +First, a layer is instanciated for an iterable of grammars. +That is they can support a finite set of type requests, so that one can train one model for multiple use. +To make use of this feature, it is better when grammars have common derivations, obviously they should be derived from the same DSL. + +Second, to create such a layer, one needs an abstraction. +An abstraction is a function which maps non-temrinals to elements of type ``A`` (hashable). +The idea is that if two non-terminals are mapped onto the same abstraction then they will use the same part of the output of the NN. + +Using ```from synth.nn import abstractions``` can provide you with a few defaults abstractions which are the most frequently used. + +#### Learning + +Both layers provide already implemented loss computations: +``loss_mse`` and ``loss_negative_log_prob``. +Their aguments indicate if one needs to convert the tensors into tensor grammars or not. + +Here is an example learning step: + +```python +optim.zero_grad() +loss = model.prediction_layer.loss_mse( + batch_programs, batch_type_requests, batch_output_tensors +) +loss.backward() +optim.step() +``` + +#### Tensor to Grammar + +Here is the following code to go from a tensor ``(N,)`` to a P(U)CFG: + +```python +tensor_grammar = model.prediction_layer.tensor2log_prob_grammar(tensor, task.type_request) +out_p_grammar = tensor_grammar.to_prob_u_grammar() if unambiguous else tensor_grammar.to_prob_det_grammar() +``` + +### Task to Tensor + +This is a ``torch.nn.Module`` which is a pipeline to make it easy to map a task to a tensor. +It takes a ``SpecificationEncoder[T, Tensor]`` which encodes a task into a tensor. +An embedder which will consume the output of the encoder to produce a new tensor. +And then this tensor is now packed into a ``PackedSequence`` and padded with ``encoder.pad_symbol`` to reach size ``embed_size`` in last dimension. + +This model is espacially helpful when working with variable length specification. +What occurs for example in PBE is that each of the example is one hot encoded into tensors and these tensors are stacked then fed to a regular ``Embedding``, finally they are packed into ``PackedSequence`` which can be easily fed to transformers, RNN, LSTM... + +### Example Model for PBE + +Here we give an example model which works for PBE: + +```python +from typing import List, Union + +from torch import Tensor +import torch.nn as nn +from torch.nn.utils.rnn import PackedSequence + +from synth import PBE, Task +from synth.nn import ( + DetGrammarPredictorLayer, + UGrammarPredictorLayer, + abstractions, + Task2Tensor, +) +from synth.pbe import IOEncoder +from synth.syntax import UCFG, TTCFG + + +class MyPredictor(nn.Module): + def __init__( + self, + size: int, + unambiguous: bool, + cfgs: Union[List[TTCFG], List[UCFG]], + variable_probability: float, + encoding_dimension: int, + device: str, + lexicon, + ) -> None: + super().__init__() + layer = UGrammarPredictorLayer if unambiguous else DetGrammarPredictorLayer + abstraction = ( + abstractions.ucfg_bigram + if unambiguous + else abstractions.cfg_bigram_without_depth + ) + self.prediction_layer = layer( + size, + cfgs, + abstraction, + variable_probability, + ) + encoder = IOEncoder(encoding_dimension, lexicon) + self.packer = Task2Tensor( + encoder, nn.Embedding(len(encoder.lexicon), size), size, device=device + ) + self.rnn = nn.LSTM(size, size, 1) + self.end = nn.Sequential( + nn.Linear(size, size), + nn.ReLU(), + nn.Linear(size, size), + nn.ReLU(), + ) + + def forward(self, x: List[Task[PBE]]) -> Tensor: + seq: PackedSequence = self.packer(x) + _, (y, _) = self.rnn(seq) + y: Tensor = y.squeeze(0) + return self.prediction_layer(self.end(y)) +``` diff --git a/docs/source/pruning.md b/docs/source/pruning.md deleted file mode 100644 index c960ebbc..00000000 --- a/docs/source/pruning.md +++ /dev/null @@ -1,146 +0,0 @@ -# Pruning - -This submodule enables the pruning of the grammars generated by ProgSynth reducing drastically their size. -Since ProgSynth tries to enumerate all programs from a grammar reducing its size, pruning removes non relevant programs increasing the chance of finding quickly a solution to your task. -There are currently three ways to do pruning and a script for the PBE specification which tries to find automatically empiricallly such constaints and encode them. - - - -- [Forbidden Patterns](#forbidden-patterns) -- [Local Type Constraints](#local-type-constraints) - - [Pattern](#pattern) - - [Syntax](#syntax) -- [Sketch](#sketch) -- [Automatic Discovery](#automatic-discovery) -- [Converting programs from DSL to pruned DSL and vice-versa](#converting-programs-from-dsl-to-pruned-dsl-and-vice-versa) - - - -## Forbidden Patterns - -The first way to add constraints is when a DSL is instantiated. The second argument given is ``forbidden_patterns: Dict[Tuple[str, int], Str[str]]``. -A key is a tuple ``(name_of_parent_primitive, arg_no)`` and gives access to the set of all primitives that cannot be directly derived for this specific argument of the ``parent_primitive``. -For example: - -```python -forbidden = { - ("+", 1) : {"+"}, - ("-", 0): {"-", "+"} -} -``` - -We forbid the second argument of ``+`` from being ``+`` and the first argument of ``-`` from being ``-`` or ``+``. - -This mechanism is quite powerful but does not enable to encode all constraints however it has the advantage of having no drawbacks. - -## Local Type Constraints - -Since we are using type constraints to produce the grammar, constraints and information can be encoded into types. -However doing so can be tedious, long and prone to errors when done manually. -This is why we made the tools presented below to help you leverage type constaints. -This is a very powerful tool to add constraints, however it comes at a small cost if you are using neural networks, it is possible that it may increase the size of the grammar prediction layers depending on the abstractions that you use. -One of the advantages of type constraints is that it is a generic way to express constraitns for any typed grammar and ProgSynth only works with typed grammars. -Here is a small example: - -```python -from synth.pruning import produce_new_syntax_for_constraints - -old_dsl = DSL(syntax, forbidden_patterns) - -# my_constraints is a list of string -# type_request is mandatory only if you have variable constraints otheriwse it can be anythign and will be returned as is -# forbidden_patterns enables to avoid redundancies -# progress displays a tqdm progress bar - -new_syntax, new_type_request = produce_new_syntax_for_constraints(syntax, my_constraints, type_request, forbidden_patterns, progress=False) - -pruned_dsl = DSL(new_syntax, forbidden_patterns) -``` - -Note that in the pruned DSL they may be duplicated primitives and types you can notice these with the ``@`` in their name. For example you may have ``+`` and ``+@0: int@0 -> int -> int``, they have the same semantic however the express different constraints. -You may use ``from synth.semantic.evaluator import auto_complete_semantics`` to automatically complete your semantis dictionnary for the evaluator. - -### Pattern - -You will give a set of constraints on patterns, they specify particular structure of your program solutions but not the general structure. -For example you can use this to tell that the left child of a ``+`` must not be a ``+`` but you cannot tell it that the root primitive should be either a ``+`` or ``-``, this can be done with a [sketch](#sketch). -That means that constraints defined here apply everywhere and to all depth levels. - -## Syntax - -We will explain the syntax through an example, let us take the calculator DSL (basic arithmetic and constants 0 up to 5). -Here are some constraints: - -- ``+ ^+ _``: - The first ``+`` tell us that we start from a ``+``, then the first argument starts with ``^`` which stands for "anything but the following" followed by a ``+`` so anything but ``+``. - Then the ``_`` means anything goes so it imposes no constraint. - This is a *pattern* *constraint* because we would like this constraint to be enforced everywhere. -- ``x 2,3,4,5 _``: - Here we are working on ``x`` (multiplication), the first argument is ``2,3,4,5`` which is simply a list of primitives which means the first argument of ``x`` can be anything in the given list that is ``2,3,4,5``. The second argument is the same as previously and again this is a *pattern* *constraint*. -- ``+ $(var0) (- _ $(var1))``: - This one is a bit more complicated. - First we are working on ``+``, then the first argument is ``$(var0)`` which means it must be a program that can only depend on variable ``var0`` and not on any other variables. - It is not necessary that ``var0`` be of the matching type (here INT). - Then the second argument is another expression so we must process it first before coming back to ``+``. - In this second expression we work on ``-`` whose first argument can be anything and its second argument can only depend on ``var1``. - Now, that means that the second argument of ``+`` here must be a ``-`` with the desired structure. - Here is seems quite clear that the resulting constraints depend on the type request and this looks like much more of a *program* *structure* than a *pattern* since it seems relevant enough to do additions with ``var1`` for example. - Thus the constraint will only model the structure of solutions, that is a valid program in this grammar could be: - ``(+ (+ var0 1) (- 1 3))``. First, note that the first argument contains a ``+`` which does not have the constraints of the root ``+``. Second, note that the second argument does not depend on ``var1``, the most we can do is forbid a dependence on variables no enforce a dependence on one. - -Sometimes it is not possible to express all constraints for a primitive in a single constraint, in that case you can actually use two constraints. -The result will be semantically equivalent. - -## Sketch - -Sketches are a very powerful way to guide the generation towards what you need since you provide a sketch of the solution. -Giving a sketch of the solution is probably one if not the method that purnes most programs. -In our framework, the sketch is encoded into type constraints. -Here is a small example: - -```python -from synth.pruning import produce_new_syntax_for_sketch - -old_dsl = DSL(syntax, forbidden_patterns) - -# my_sketch is a string -# type_request is mandatory here -# forbidden_patterns enables to avoid redundancies - -new_syntax, new_type_request = produce_new_syntax_for_sketch(syntax, my_sketch, type_request, forbidden_patterns) - -pruned_dsl = DSL(new_syntax, forbidden_patterns) -``` - -The syntax is the same as for local type constraints, see [syntax](#syntax), the difference is that this is interpreted as a sketch and not a type constraint. -For example: - -- ``+ ^+ _``: - implies that the solutions generated by ProgSynth will all have this pattern, that is they will start with a ``+`` then the first argument is anything but a ``+`` and the second argument can be anything. - Contrary to type constraints it does not imply that every ``+`` must not have a ``+`` as a first argument, only the one from the root. -- ``* $(var0) (+ 2 _)``: - implies that the solution will start with a ``*``, then its first argument can depend at most only on ``var0`` and no other variables, its second argument is ``(+ 2 _)`` where ``_`` can be anything. - -## Automatic Discovery - -The process of finding and enumerating all possible constraints for a DSL is tedious and prone to errors. -ProgSynth provides for some specifications such as PBE a script ``dsl_analyser.py`` which tries to empirically find semantically equivalent programs then produces as many constraints as possible to remove as many as possible semantically equivalent programs. - -The script works by either reproducing the distriution from a given dataset or just taking inputs from a given dataset, this enables it to produce inputs to test programs. -It then evaluates programs up to depth 2 of the grammar and builds sets of semantically equivalent programs with respect to this set of inputs. -Some of these programs can then be forbidden using different types of constraints either local type constraints or forbidden patterns. -The script produces two files: - -- a python file containing the local type constraints and the forbidden patterns it produced; -- a JSON file containing all semantically equivalent classes of programs et depth 2. - -## Converting programs from DSL to pruned DSL and vice-versa - -Type -When using ``dsl.parse("(+ (- 1 var0) (+ 1 var1))")`` you will get a ``Program`` expressed in the current DSL but with the original primitives. -That is since there may be duplicated primitives and types with the ``@`` in their name, they will not be parsed as such since we didn't use ``+@0``. -Writing the programs with the ``@`` would depend on the constraints and be a rather long and tedious process. -Instead after a ``DetGrammar`` has been produced you can use ``grammar.embed(program)`` which will return the embedded program. -This enables the conversion from DSL to pruned DSL and vice-versa. -However, the ``program`` must be in the given ``grammar``, it implies that it is always the case that you can embed a program from the pruned DSL to the DSL but not the other way around. -If the ``program`` is in the original DSL but in the pruned DSL then ``grammar.embed(program)`` returns ``None``. diff --git a/docs/source/sharpening.md b/docs/source/sharpening.md new file mode 100644 index 00000000..b62878dd --- /dev/null +++ b/docs/source/sharpening.md @@ -0,0 +1,148 @@ +# Sharpening + +This submodule enables the sharpening of the grammars generated by ProgSynth reducing drastically their size. +Since ProgSynth tries to enumerate all programs from a grammar reducing its size is a relevant way to speed up the search, sharpening removes non relevant programs increasing the chance of finding quickly a solution to your task. +There are currently two ways to do sharpening and a script for the PBE specification which tries to find automatically empiricallly such constaints and encode them. + + +Table of contents: + +- [Forbidden Patterns](#forbidden-patterns) +- [Simplifying Rules](#simplifying-rules) + - [Syntax](#syntax) + - [An Example](#an-example) + - [Automatically Generated Equations](#automatically-generated-equations) + - [Limits](#limits) + + + +## Forbidden Patterns + +The first way to add constraints is when a DSL is instantiated. The second argument given is ``forbidden_patterns: Dict[Tuple[str, int], Str[str]]``. +A key is a tuple ``(name_of_parent_primitive, arg_no)`` and gives access to the set of all primitives that cannot be directly derived for this specific argument of the ``parent_primitive``. +For example: + +```python +forbidden = { + ("+", 1) : {"+"}, + ("-", 0): {"-", "+"} +} +``` + +We forbid the second argument of ``+`` from being ``+`` and the first argument of ``-`` from being ``-`` or ``+``. + +This mechanism is quite powerful but does not enable to encode all constraints however it has the advantage of having no drawbacks, it can also be done within the other framework. + +## Simplifying Rules + +More generally, we would like to remove regular tree languages from the grammar which also describes a regular tree language. +In order to do that, we define a syntax that describes a subset of regular tree language but covers the most relevant options for program synthesis. +This is quite straightforward with the following code: + +```python +from synth.pruning.constraints import add_dfta_constraints +from synth.syntax.grammars import UCFG + + +cfg = ... # your regular CFG +my_constraints = [...] # a list of string that express the constraints +sketch = ... # your sketch or None if there isn't one +ucfg = UCFG.from_DFTA_with_ngrams( + add_dfta_constraints(cfg, my_constraints, global_constraint, progress=False), 2 + ) + +# You can continue as usual though since it is an UCFG instead of a CFG +# Det objects need to be replaced by U objects +# e.g.: ProbDetGrammar -> ProbUGrammar, HeapSearch -> UHeapSearch, ... +``` + +### Syntax + +Let us write ``P`` for the set of primitives names and variables. Rules are manipulating a set of names: + +``` +NSet := f1,...,fk | ^f1,...,fk | _ +``` + +where ``f1,...,fk`` are from ``P``, ``^`` is the complement operator and ``_`` represents any symbol. +Rules have the following form: + +``` +Rules = (NSet Rules1 . . . Rulesk ) | #[NSet]<=N | #[NSet]>=N +``` + +where ``k`` and ``N`` are constant integers. The rule ``(f g,h _)`` specifies that for each occurrence of ``f``, its first argument must be either ``g`` or ``h``, and the second argument can be anything. +The rule ``#[Var0]>=1`` specifies that ``Var0`` appears at least once. +Remark that ``#[_]<=10`` says that the whole program has size at most 10. +Rules can be nested, for instance ``f #[g]<=1 #[h]>=1`` : for each occurrence of ``f``, the first argument contains at most one ``g``, and the second argument at least one ``h``. +Rules specify locally which primitives or variables can be combined together. + +A sketch has the same syntax as a constraint. +It specifies how the solution program should be, that is the derivations from the root, whereas constraints specify derivation anywhere in the programs. +For instance, the skecth ``(f _ g)`` specifies that the program starts with ``f`` and that the second argument of that particular ``f`` is ``g``. + +### An Example + +Let us consider the grammar of Boolean formulas over the binary operators ``And``, ``Or`` and unary ``Not``, with Boolean variables ``Var0``, ``Var1``. + +``` +bool -> And(bool, bool) | Or(bool, bool) | Not(bool) | Var0 | Var1 +``` + +Clearly, this grammar generates a lot of redundant programs. Let us specify some rules in order to enforce that all programs are in conjunctive normal form: + +``` +Or ¬And ¬And ; +Not ¬{And, Or} +``` + +The first rule specifies that ``And`` cannot be an argument of ``Or`` and the second one that ``And`` and ``Or`` cannot be arguments of ``Not``. The output of a compilation algorithm using these two rules could be the following grammar: + +``` +bool1 -> And(bool1, bool1 ) | Or(bool2 , bool2 ) | Not(bool3 ) | Var0 | Var1 +bool2 -> Or(bool2 , bool2) | Not(bool3) | Var0 | Var1 +bool3 -> Not(bool3) | Var0 | Var1 +``` + +There are still a lot of equivalent programs generated by this grammar. One could consider the following rules further reducing symmetries: + +``` +And ¬And _ ; +Or ¬{Or, And} ¬And ; +Not ¬{And, Or} +``` + +It ensures conjunctive normal form but also that both ``And`` and ``Or`` are associated to the right: in particular, the formula ``And(And(φ1 , φ2), φ3)`` is replaced +by ``And(φ1 , And(φ2 , φ3))``. + +### Automatically Generated Equations + +Since writing rules can be tedious, even more so for large grammars, we propose +an automated process for generating valid equations: + +1. We enumerate all programs up to some fixed depth (in practice, 3 or 4); +2. We check for program equivalence amongst all generated programs; +3. For each equivalence class of programs, we choose as representative the small- +est program of that class: the goal is to find rules rejecting all non represen- +tative programs; +4. We enumerate rules and for each check whether they are useful, meaning +reject only (new) non representative programs. +Note that program equivalence may be hard to solve; in practice we evaluate the +programs on a set of inputs that is either sampled or scrapped from a dataset +and declare two programs equivalent if their outputs coincide on them. + +ProgSynth provides for some type of specifications such as PBE a script ``dsl_analyser.py`` such a tool. +The script works by either reproducing the distriution from a given dataset or just taking inputs from a given dataset, this enables it to produce inputs to test programs. +It then evaluates programs up to depth 2 of the grammar and builds sets of semantically equivalent programs with respect to this set of inputs. +Some of these programs can then be removed from the grammar. +The script produces two files: + +- a python file containing all constraints that were automatically produced; +- a JSON file containing all semantically equivalent classes of programs et depth 2. + +### Limits + +The design of our syntax was guided by simplicity; although expressive enough for most use cases, it could be extended. +Indeed, some natural rules cannot be expressed, for instance forbidding the pattern ``(f a b)``, when there is an occurrence of ``f`` where the first argument is ``a`` and the second argument is ``b``. +To put this remark in a wider perspective: we note that all simplifyig rules defined in our syntax induce regular tree languages, but conversely that some regular tree languages, such as ‘trees without the pattern ``(f a b)``’, cannot be defined in our syntax. +However, with some small tweaking one can directly give an deterministic bottom-up tree automaton and use it as if it were a rule. \ No newline at end of file diff --git a/docs/source/tutorial.md b/docs/source/tutorial.md new file mode 100644 index 00000000..dc848cf9 --- /dev/null +++ b/docs/source/tutorial.md @@ -0,0 +1,267 @@ +# Tutorial + +This tutorial will show you how to: + +- [create a new DSL from scratch](#create-a-dsl-from-scratch); +- [add a semantic to the DSL](#add-a-semantic-to-the-dsl); + +afterward everything is PBE specific: + +- [add a DSL to the already existing PBE pipeline](#making-your-dsl-usable-by-scripts); +- [create your first dataset for this DSL](#creating-a-dataset); +- [explore a dataset](#explore-a-dataset); +- [create a task generator for that pipeline](#creating-a-task-generator); +- [generate synthetics datasets](#generating-a-synthetic-dataset); +- [train a model](#train-a-model); +- [evaluate a model](#evaluate-a-model); +- [synthesize a program](#simple-synthesis). + +This example is the *calculator* DSL, whose source code can be found in the folder ``./examples/pbe/calculator``. + +## Create a DSL from scratch + +A DSL is a syntactic object thus it only defines the syntax of our primitives. +The relevant file is ``calculator/calculator.py``. + +A primitive is a function or a constant that you might need to use in the solution program, it is typed and usually has a semantic, but the semantic is not defined in the DSL. + +The syntax is a mapping from primitives to their type. + +For detailed information about types [see the page on the type system](type_system.md). + +The syntax object is a dictionnary where keys are unique strings identifying your primitives and values are ProgSynth types. It might be a bit long to explain all the different type features supported by ProgSynth, however ProgSynth provides the ``auto_type`` function which dramatically speed up the syntax writing process. +Here is an example: + +```python +from synth.syntax import auto_type, DSL + +syntax = auto_type({ + "1": "int", + "2": "int", + "3": "int", + "3.0": "float" + "int2float": "int -> float", + "+": "'a [int | float] -> 'a [int | float] -> 'a [int | float]", + "-": "'a [int | float] -> 'a [int | float] -> 'a [int | float]", +}) + +dsl = DSL(syntax) +``` + +The notation might seem complex to you but we will briefly explain what happens. +When you put a string such as ``"int"`` or ``"float`` then this is transformed into a ground type, the ``"->"`` translates into a function. +The ``"'"`` prefix tells us ``"'a"`` is a polymorphic type; however the ``[int | float]`` right after that indicates this polymorphic type can only take the following values: ``int`` or ``float``. +So that means we will have a ``+`` only for ``int`` and a ``+`` only for ``float``, but both will share the same semantic, since they both are named ``"+"``. +For detailed information about types and on how this works [see the page on the type system](type_system.md). + +You can now use your DSL to generate [grammars](grammars.md)! + +You might want to add *syntactic constraints* on the generated grammars, this is covered in [sharpening](sharpening.md). + +## Add a semantic to the DSL + +The relevant file is ``calculator/calculator.py``. +It's great that we can produce grammars and everything with our DSL but we cannot execute our program! It is time to gave them a semantic! +The semantic object is a dictionnary where keys are unique strings identifying your primitives and values are unary functions or constants. + +Here is the semantic for the primitives we defined earlier: + +```python +from synth.semantic import DSLEvaluator + +semantic = { + "+": lambda a: lambda b: round(a + b, 1), + "-": lambda a: lambda b: round(a - b, 1), + "int2float": lambda a: float(a), + "1": 1, + "2": 2, + "3": 3, + "3.0": 3.0, +} + +evaluator = DSLEvaluator(semantic) +``` + +First for constants, they are just associated to their value. +Then for functions, notice that while ``+`` is a binary function, here we have a unary function that returns another unary function. +ProgSynth needs functions in unary form in order to be able to do partial applications. +Python's system to automatically transform a n-ary function to a unary function as of now induces a relatively high execution cost, which makes it prohibitive for ProgSynth. + +You can now use your evaluator to eval your program, the syntax is ``evaluator.eval(program, inputs_as_a_list)``. + +As a side note, it might happen that in your evaluation, exceptions occur and you do not want to interrupt the python process, in that case you can use ``evaluator.skip_exceptions.add(My_Exception)``. When such an exception occurs, it is caught and instead a ``None`` is returned. + +The evaluator cached the evaluation of programs, so the value is computed only once on the same input. However, in some cases, you might need to clear the cache since it can take a lot of space which can be done using: ``evaluator.clear_cache()``. + +--- +**Everything after is PBE specific.** + +--- + +## Making your DSL usable by scripts + +Most if not all scripts in the ``pbe`` folder should work with little to no changes for most DSLs. +These scripts use the ``dsl_loader.py`` file that manages DSLs and provides a streamline approach for all scripts to load and use them. +You should add your DSL to that script to be able to use all these scripts for free. + +But since this is PBE specific we need to define a lexicon in ``calculator/calculator.py``. + +### Lexicon + +In the PBE specification, a lexicon is needed in order to: + +- create synthetic tasks and thus synthetic datasets; +- use neural networks for prediction. + +A lexicon is a list of all base values that can be encountered in the DSL. + Here, we limit our DSL to float numbers rounded to one decimal, in the range [-256.0, 257[. +For example, if our DSL were to manipulate lists of int or float, we would not have to add anything to the lexicon since lists are not a base type (`PrimitiveType`). + +### Finally adding your DSL + +Your only point of interest in this file is the ``__dsl_funcs`` dictionnary that should be surrounded by comments. +Here is the line that we added to the dictionnary for our calculator DSL: + +```python +"calculator": __base_loader( + "calculator.calculator", + [ + "dsl", + "evaluator", + "lexicon", + ("reproduce_calculator_dataset", "reproduce_dataset"), + ], + ), +``` + +It tells the loader that the DSL is defined in the file ``calculator/calculator.py``, then when it loads this file, it loads the following variables ``dsl, evaluator, lexicon, reproduce_calculator_dataset``. +These variables will be made available under the following fields respectively ``dsl, evaluator, lexicon, reproduce_dataset``. +Notice that the tuple notation allows renaming. +The first three are necessary while the last one is optional, in the sense that you might not need to redefine a ``reproduce_dataset`` function. + +## Creating a dataset + +The relevant file is ``calculator/convert_calculator.py``. + +To generate a synthethic dataset we need to create a dataset. +For this example, we created a short JSON file named `dataset/calculator_dataset.json` that is built with the following fields: + +- *program*: that contains the representation of the program, the parsing is done automatically by the DSL object (`dsl.parse`) so you don't need to parse it yourself. Here is a representation of a program that computes `f(x, y)= x + y * x` in our DSL: `(+ var0 (* var1 var0))`; +- *examples*: displaying what are the expected inputs and outputs of the program. + +Once the dataset is done, we need to create a file converting it to the ProgSynth format, done here in `convert_calculator.py`. +An important point to note is that we need to develop the `PolymorphicType`, since our ``+`` and ``-`` depend on it so before parsing we need to call `dsl.instantiate_polymorphic_types()`. + +If you want to adapt the code of `calculator/convert_calculator` for your own custom DSL, it should work almost out of the box with ProgSynth, note that ProgSynth needs to guess your type request and it does so from your examples. If you are manipulating types that are not guessed by ProgSynth, it wil fill them with ``UnknownType`` silently, in that case you may need to add your own function to guess type request or modify the one from ProgSynth which is `synth/syntax/type_helper.py@guess_type`. + +We can simply use this file by command line, from the folder `./examples/pbe/calculator`. + +```bash +python convert_calculator.py dataset/calculator_dataset.json -o calculator.pickle +``` + +## Explore a dataset + +You might want to check that you correctly translated your task to the ProgSynth format. +This can be done easily by visualizing the tasks of a dataset with the dataset explorer. +A dataset can be explored using `dataset_explorer.py`. + +```bash +python examples/pbe/dataset_explorer.py --dsl calculator --dataset calculator.pickle +``` + +## Creating a Task Generator + +Most often you don't need to use a custom TaskGenerator and the default one will work, however if you have more than one ground type you will need to do so, this is the case with the calculator DSL. +The code is at the end of ``calculator/calculator.py``. + +**TODO: explain in more details** + +## Generating a synthetic dataset + +Now we can create synthetic datasets. +There is already existing script that does all of the job for us. + +The dataset generator works out of the box for our DSL but that may not always be the case, you can check out other DSLs files and look at the `task_generator_*.py` files. + +You can generate datasets using: + +```bash +python examples/pbe/dataset_generator_unique.py --dsl calculator --dataset calculator/calculator.pickle -o dataset.pickle --inputs 1 --programs 1000 +``` + +## Train a model + +For more information about model creation see [this page](prediction.md). + +You can easily train a model using: + +```bash +python examples/pbe/model_trainer.py --dsl calculator --dataset my_train_dataset.pickle --seed 42 --b 32 -o my_model.pt -e 2 +``` + +There are various options to configure your model and everything which we do not dwelve into. + +## Infer with a model + +A model can be used to produce PCFGs, this will produce a `pickle` file in the same folder as your model, you will need to pass this file to the solver. + +```bash +python examples/pbe/model_prediction.py --dsl calculator --dataset my_test_dataset.pickle --model my_model.pt --b 32 -support my_train_dataset.pickle +``` + +The ``--support my_train_dataset.pickle`` is only used to filter the test set on type requests that were also present in the train set. + +## Evaluate a model + +You might want to evaluate a model to see if it learned anything relevant, this can be easily done but is time consuming. +To evaluate a model, we actually try to solve program synthesis tasks for a DSL so this is not simply an inference task. +If you are directly interested in synthesizing your first program then jump over to [the next section](#simple-synthesis) which tells you exactly how to do that. +You can easily evaluate a model using: + +```bash +python examples/pbe/solve.py --dsl calculator --dataset my_test_dataset.pickle --pcfg pcfgs_my_test_dataset_my_model.pt -o . -t 60 --support my_train_dataset.pickle --solver cutoff +``` + +The most important parameter is perhaps ``-t 60`` which gives a timeout of 60 seconds per task. +You can also play with different solver, by default ``cutoff`` works pretty well on almost anything. + +This will produce a CSV file in the output folder (``.`` above). +This result file can then be plotted using: + +```bash +python examples/plot_solve_results.py --dataset my_test_dataset.pickle --folder . --support my_train_dataset.pickle +``` + +Again there's a plethora of options available, so feel free to play with them. + +## Simple synthesis + +Here is a simple function that takes your task, the PCFG and the evaluator and generates a synthetised program. +For more information about predictions and how to produce a P(U)CFG from a model, see [this page](prediction.md). +If you are perhaps more interested in solving then you should probably look at the files in ``synth.pbe.solvers`` which offer different ways of solving our synthesis problem. + +```python +from synth import Task, PBE +from synth.semantic import DSLEvaluator +from synth.syntax import bps_enumerate_prob_grammar, ProbDetGrammar +from synth.pbe import CutoffPBESolver + +def synthesis( + evaluator: DSLEvaluator, + task: Task[PBE], + pcfg: ProbDetGrammar, + task_timeout: float = 60 +): + solver = CutoffPBESolver(evaluator) + solution_generator = solver.solve(task, bps_enumerate_prob_grammar(pcfg), task_timeout) + try: + solution = next(solution_generator) + print("Solution:", solution) + except StopIteration: + # Failed generating a solution + print("No solution found under timeout") + for stats in solver.available_stats(): + print(f"\t{stats}: {solver.get_stats(stats)}") + +``` diff --git a/docs/source/tutorial.rst b/docs/source/tutorial.rst deleted file mode 100644 index 98dc8088..00000000 --- a/docs/source/tutorial.rst +++ /dev/null @@ -1,117 +0,0 @@ -Tutorial -======== - -In this section, an overview of how to create a DSL is shown, based on the example of the *calculator* DSL, whose source code can be found in the folder :code:`./examples/pbe/calculator`. - -Structure of a DSL ------------------- - -Several files have been implemented in order to create a DSL allowing int-to-int or float-to-float additions and substractions. - -* :code:`calculator/calculator.py`, containing the primitives of the DSL and the default evaluator. -* :code:`calculator/convert_calculator` is a runnable python script which enables you to convert the original calculator dataset file to the ProgSynth format. -* :code:`calculator/calculator_task_generator`, an adapted version of the file :code:`task_generator` that allows us to create a synthetic dataset, based from the one created with :code:`convert_calculator`. - -Some changes have to be made to other files present in :code:`./examples/pbe`, and this tutorial will display how to change them for a new DSL. - -DSL (:code:`calculator/calculator.py`) --------------------------------------- -The DSL is mainly represented by two dictionaries, representing the semantics of the DSL and the types of its primitives. - -Semantics of the DSL -~~~~~~~~~~~~~~~~~~~~ -Simply put, the semantics are short snippets of code that will make up your programs. This enables ProgSynth to combine these **primitives** in order to synthetise the program that we want to obtain. - -For instance, as we want to solve tasks made of additions and substractions between two numbers of the same type, we only need to define only 3 primitives: +, - and int2float. - -Here, we consider that an integer is a sub-type of float. Thus, we only need in this case to convert integers to float numbers using int2float. - -.. _Types of the DSL: - -Types of the DSL -~~~~~~~~~~~~~~~~ -A DSL has to be strongly typed in order to properly work. The addition and substraction primtives should accept either 2 ints or 2 floats as parameters, we wish to define a custom type that can represent both. - -A method is logically represented by arrows, indicating the parameters of the primitive and the output. - -**Example**:: - :code:`int2float` takes an int as input and will return a float. Thus, its type is :code:`int -> float` - -ProgSynth uses custom-defined types: - -* :code:`PrimitiveType`, where the type is solely defined by its name. - - - In our case, INT was already defined by ProgSynth. We simply need to define the :code:`PrimitiveType` FLOAT for our DSL. -* :code:`PolymorphicType`, where the DSL will replace later-on this type with every :code:`PrimitiveType` that can be encountered. - - - As we have defined 2 :code:`PrimitiveType`, the methods :code:`+` and :code:`-` will each be developped in two different versions: one allowing operations between integers and another one allowing operations between floats. - - As the DSL needs to know which :code:`PrimitiveType`s can be encountered in order to replace any :code:`PolymorphicType`, we define some constants in our DSL with the correct type (at least one per type). - -Lexicon -~~~~~~~ -A DSL is defined inside a specific lexicon. To avoid overflow or underflow, we limit our DSL to float numbers rounded to one decimal, in the range [-256.0, 257[ - -Forbidden patterns (Optional) -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -In some cases, we wish to stop the DSL to derive a specific primitive from another one: -For instance, let us say that we want to extend the `calculator` DSL with a primitive to `add1` to an integer and another one to `sub1` to an integer. -Because doing `(add1 (sub1 x))` or `(sub1 (add1 x))` is the same as doing nothing, we can forbid the second pattern from being derived from the first one, in that case the patterns would be :code:`{ "add1": {"sub1}, "sub1": {"add1"}`. -For more information see `pruning `_. - - -Creating a dataset (:code:`calculator/convert_calculator.py`) --------------------------------------------------------------- -To use PBE, we need to create a dataset. For this example, we created a short JSON file named :code:`dataset/calculator_dataset.json` that is built with the following fields - -* *program*, that contains therepresentation of the program, the parsing is done automatically from the DSL object so you don't need to parse it yourself. Here ia representation of a program that computes :code:`f(x, y)= x + y * x` in our DSL: :code:`(+ var0 (* var0 var1))`. - -* *examples*, displaying what are the expected inputs and outputs of the program. - -Once the dataset is done, we need to create a file converting it to the ProgSynth format, done here in :code:`convert_calculator.py`. -An important point to note is that we need to develop the :code:`PolymorphicType`, as described in the previous sub-section. - -It is done automatically by calling the method :code:`dsl.instantiate_polymorphic_types(upper_bound)`. -As we only want to develop :code:`+` and :code:`-` as methods with a size of 5 (INT -> INT -> INT or FLOAT -> FLOAT -> FLOAT, as we consider each arrow in the size), we define its upper bound type size to 5. - - -If you want to adapt the code of :code:`calculator/convert_calculator` for your own custom DSL, it should work almost out of the box with ProgSynth, some point that may hinder you is that ProgSynth needs to guess your type request, it does so from your examples. If you are manipulating types that are not guess by ProgSynth, it wil fill them with UnknownType silently, in that case you may need to add your own function to guess type request or modify the one from ProgSynth. - - -Usage -~~~~~ -We can simply use this file by command line, from the folder :code:`./examples/pbe/calculator`. - -.. code:: bash - - python convert_calculator.py dataset/calculator_dataset.json -o calculator.pickle - - -Generating a synthetic dataset (:code:`dataset_generator.py`) -------------------------------------------------------------- -Once the DSL and a short dataset are created, we wish to generate automatically a dataset reproducing the task distribution. - -The *deepcoder* and *dreamcoder* datasets did not require to use float numbers. Thus, the previous implementation of the :code:`task_generator.py` needs to be adapted to float numbers. -Hence, we need to create a function to enable reproduing our dataset. We recommend checking the documentation of ``reproduce_dataset`` which we will use. - -* the function :code:`analyser` needs to analyse the range of both int and float inputs, it is basically called on base types. -* the function :code:`get_element_sampler` produces the sampler for our base types, here uniform on our int and float ranges. -* the function :code:`get_validator` produces a function that takes an ouput produced from a program and should tell whether this output is allowed or not. -* the function :code:`get_lexicon` produces the lexicon of the DSL, it will be used for deep learning. Here, as the int lexicon is included in the float lexicon, we return the latter one. - -Usage -~~~~~ -Once the DSL has been added to the :code:`dsl_loader.py` then you can generate datasets using: -.. code:: bash - - python dataset_generator.py --dsl calculator --dataset calculator/calculator.pickle -o dataset.pickle - -The dataset generated can be explored using :code:`dataset_explorer.py`. - -.. code:: bash - - python dataset_explorer.py --dsl calculator --dataset dataset.pickle - - -Conclusion ----------- -Once the dataset and the DSL are done, we simple need to add our DSL to the :code:`dsl_loader.py` script, in-depth instructions are provided in the file. Then, the usage is the same as describe in the section :doc:`usage`. diff --git a/docs/source/type_system.md b/docs/source/type_system.md new file mode 100644 index 00000000..2a3b476e --- /dev/null +++ b/docs/source/type_system.md @@ -0,0 +1,110 @@ +# The Type System + +The type system in ProgSynth has the vocation of adding constraints for compilation from a DSL into a grammar. + +ProgSynth does not check at any time that the data you manipulate has the correct type, types are only used at compilation time. + + +Table of contents: + +- [Basic Type](#basic-type) +- [Advanced Types](#advanced-types) + - [Sum Type](#sum-type) + - [Arrow](#arrow) + - [Generic](#generic) +- [Polymorphic Types](#polymorphic-types) +- [Methods of interest](#methods-of-interest) + + + +## Basic Type + +Ground types are ``PrimitiveType``. +An ``int`` is represented as ``PrimitiveType("int")``. +Notice that it is uniquely identified by its name therefore two instances with the same name represent the same type. + +ProgSynth already defines the following types: ``INT``, ``BOOL``, ``STRING``, ``UNIT``. + +## Advanced Types + +Advanced types are built from other types. +There are three: + +- Sum Type; +- Arrow; +- Generic. + +### Sum Type + +Sum types are union in python. +They can be built two ways, first with ``Sum(t1, t2)`` or ``t1 | t2``. + +### Arrow + +Arrow represent functions. +They can be built with ``Arrow(t1, t2)``. +A function ``int->int->int`` would have type ``Arrow(int, Arrow(int, int))``. +An easier way to construct these arrows is to use ``FunctionType(int, int, int)`` in this case. While ``Arrow`` is a binary constructor, ``FunctionType`` allows any number of arguments and ensure that the ``Arrow``are built correctly especially if you are using higher order functions. + +### Generic + +Generic are parametric types. For example, the previous type ``Arrow``is *almost* a generic, the ``List``type is one. You can make a list type out of any type using ``List(t)``. +To instanciate a Generic builder you can use the ``GenericFunctor``: + +```python +List = GenericFunctor("list", min_args=1, max_args=1) +# Arrow behaves almost like a Generic defined the following way: +Arrow = GenericFunctor( + "->", + min_args=2, + max_args=2, + infix=True, +) + +``` + +## Polymorphic Types + +One can instantiate polymorphic types with ``PolymorphicType("a")`` as with ``PrimitiveType`` they are uniquely identified by their name. +At compilation, a polymorphic type will take as possible values any ground type that is present in the DSL and advanced types built on top of them recursively up to some type size. + +You can limit the set of types a polymorphic type can take with ``FixedPolymorphicType("f", t1, t2, t3)`` which will only take types that are ``t1``, ``t2`` or ``t3``. + +You can check if a polymorphic type ``poly_t`` can be assigned some type ``t`` with ``poly_t.can_be(t)``. + +## Methods of interest + +Creating a type can be done by instanciating the objects individually or you can use the ``auto_type`` method which takes either your string type or your syntax dictionnary and transform it into a real type. Here are a few examples: + +```python +from synth.syntax import auto_type + +t = auto_type("int") +# PrimitiveType("int") +t = auto_type("int | float -> float") +# Arrow(int | float, float) +t = auto_type("'a list ('a -> 'b ) -> 'b list") +# let +# a = PolymorphicType("a") +# b = PolymorphicType("b") +# in +# FunctionType(List(a), Arrow(a, b), List(b)) + +t = auto_type("'a[int | float] -> 'a[int | float]") +# let +# a = FixedPolymorphicType("a", PrimitiveType("int"), PrimitiveType("float)) +# or equivalently +# a = FixedPolymorphicType("a", PrimitiveType("int") | PrimitiveType("float)) +# in +# Arrow(a, a) + +t = auto_type("int optional") +# Generic("optional", PrimitiveType("int")) +``` + +- ``t1.is_instance(t2)`` computes wether type ``t1`` is an instance of ``t2``, notice that ``t2`` can be a python type, some type in our system or a ``TypeFunctor`` such as a ``GenericFunctor`` like ``List`` or ``Arrow``. +Note that there is notion of covariant or contravariant types in our type system; +- ``t.arguments()`` returns the list of arguments consumed by this type if this is an arrow, if this not an arrow then an empty list is returned; +- ``t.returns()`` returns the type returned by this arrow, if this is not an arrow return the type ``t`` itself. +- ``t.all_versions()`` returns the list of all types that this type can take, this is only relevant for sum types, basically each sum type will take all its possible values; +- ``t.unify({'a': INT})`` will return ``t`` where all polymorphic types named ``'a'`` take value ``INT``. diff --git a/docs/source/usage.rst b/docs/source/usage.rst index da1eb38e..d7b9f246 100644 --- a/docs/source/usage.rst +++ b/docs/source/usage.rst @@ -3,14 +3,11 @@ Usage of ProgSynth The ProgSynth framework currently focuses on the Programming By Exemples (PBE) specification of tasks. Other cases are planned to be supported, despite this most elements of ProgSynth can be used independently of the specification. -The :code::`synth` folder is a standalone library and does not provide ready to use code such as :code:`model.fit()`, instead we provide in :code:`./examples` different elements: +The :code:`synth` folder is a standalone library and does not provide ready to use code in the same manner a :code:`model.fit()` does, however we provide in :code:`./examples` scripts and DSLs that are ready to use. -- scripts that can directly be used; -- implementations of some DSLs. +These scripts enable you to reproduce the results from papers or can be modified to test your ideas. The scripts are pretty generic and in general can be used for your own custom DSL with little to no modification. -These scripts enable you to reproduce the results from papers or can be modified to test yoru ideas. The script are pretty generic and in general can be used for your own custom DSL witt=h little to no modification. - -For further information, in each specifcation folder inside :code:`./examples` there is a :code:`README` explaining the use of scripts, what DSLs are implemented from which paper and where to download datasets. +For further information, in each specification folder inside :code:`./examples` there is a :code:`README` explaining the use of scripts, what DSLs are implemented from which paper and where to download the datasets. The tutorial on section :doc:`tutorial` uses the example of tasks based on additions and substractions between integers or between floating point numbers, explaining step-by-step how to create a new DSL that can be used by the framework. diff --git a/examples/README.md b/examples/README.md index 472ec7d5..50caaf95 100644 --- a/examples/README.md +++ b/examples/README.md @@ -7,6 +7,7 @@ This folder contains ready to use scripts and files that you can leverage to rep - [Generics](#generics) - [Programming By Example](#programming-by-example) - [Programming from Natural Language](#programming-from-natural-language) +- [SyGuS](#sygus) ## Generics @@ -27,3 +28,7 @@ The available domains are: ## Programming from Natural Language This is the NLP folder. The specification of the task is given as a natural language string that explains the task. + +## SyGuS + +This is the SyGuS folder. It provides scripts to directly work with the SyGus format. The goal is to mainly edit the specification thanks to the tools offered by ProgSynth such as sharpening. diff --git a/examples/compare_enumeration.py b/examples/compare_enumeration.py new file mode 100644 index 00000000..e7094851 --- /dev/null +++ b/examples/compare_enumeration.py @@ -0,0 +1,371 @@ +from typing import Callable, Iterable, List, Optional, Tuple, Union +import csv +import time +import argparse + +from synth.syntax import ( + ProbDetGrammar, + ProbUGrammar, + CFG, + DSL, + hs_enumerate_prob_grammar, + bs_enumerate_prob_grammar, + bps_enumerate_prob_grammar, + as_enumerate_prob_grammar, + ProgramEnumerator, + auto_type, +) +from synth.syntax.grammars.enumeration.constant_delay import ( + enumerate_prob_grammar as cd, +) +import tqdm +import timeout_decorator + +SEARCH_ALGOS = { + "a_star": as_enumerate_prob_grammar, + "bee_search": bs_enumerate_prob_grammar, + "beap_search": bps_enumerate_prob_grammar, + "heap_search": hs_enumerate_prob_grammar, + "cd4": lambda x: cd(x, k=4), + "cd16": lambda x: cd(x, k=16), + "cd64": lambda x: cd(x, k=64), +} + +parser = argparse.ArgumentParser( + description="Compare search algorithms", fromfile_prefix_chars="@" +) +parser.add_argument( + "-o", + "--output", + type=str, + default="./enumeration.csv", + help="output file (default: './enumeration.csv')", +) +parser.add_argument( + dest="n", metavar="programs", type=int, help="number of programs to be enumerated" +) +# parser.add_argument( +# dest="max_rules", type=int, help="maximum number of derivation rules" +# ) +parser.add_argument( + dest="max_non_terminals", type=int, help="maximum number of non terminals" +) +parser.add_argument( + dest="scaling", + choices=["distance", "nonterminals", "derivations"], + help="maximum number of non terminals", +) +parser.add_argument( + "-t", "--timeout", type=int, default=300, help="timeout in seconds (default: 300)" +) +parser.add_argument( + "-s", "--seed", type=int, default=-1, help="seed (default: -1, uniform)" +) + + +parameters = parser.parse_args() +output_file: str = parameters.output +programs: int = parameters.n +timeout: int = parameters.timeout +seed: int = parameters.seed +scaling: str = parameters.scaling +max_non_terminals: int = parameters.max_non_terminals +file_name = output_file[: -len(".csv")] if output_file.endswith(".csv") else output_file + + +# ================================ +# Load constants specific to dataset +# ================================ + + +def save(trace: Iterable, name: str) -> None: + with open(name, "w") as fd: + writer = csv.writer(fd) + writer.writerows(trace) + + +# Enumeration methods ===================================================== +def summary_enumerative_search( + pcfg: ProbDetGrammar, + name: str, + custom_enumerate: Callable[ + [Union[ProbDetGrammar, ProbUGrammar]], ProgramEnumerator + ], + programs: int, + timeout: int = 300, + title: Optional[str] = None, + seed: int = -1, +) -> Tuple[str, int, int, float, int, int, int, int]: + n = 0 + non_terminals = len(pcfg.rules) + derivation_rules = sum(len(pcfg.rules[S]) for S in pcfg.rules) + used_time = 0 + + pbar = tqdm.tqdm(total=programs, desc=title or name) + enumerator = custom_enumerate(pcfg) + gen = enumerator.generator() + program = 1 + datum_each = 100000 + start = 0 + try: + + def fun(): + return next(gen) + + get_next = timeout_decorator.timeout(timeout, timeout_exception=StopIteration)( + fun + ) + start = time.perf_counter_ns() + while program is not None: + program = get_next() + n += 1 + if n % datum_each == 0 or n >= programs: + used_time = time.perf_counter_ns() - start + bef = time.perf_counter_ns() + if used_time >= timeout * 1e9: + break + pbar.update(datum_each) + if n >= programs: + break + rem_time = timeout - used_time / 1e9 + get_next = timeout_decorator.timeout( + rem_time, timeout_exception=StopIteration + )(fun) + start -= time.perf_counter_ns() - bef + except (StopIteration, RuntimeError): + used_time = time.perf_counter_ns() - start + pbar.close() + datum_each = 1000000 + factor = datum_each / n + return ( + name, + non_terminals, + derivation_rules, + used_time * factor / 1e9, + datum_each, + int(enumerator.programs_in_queues() * factor), + int(enumerator.programs_in_banks() * factor), + seed, + ) + + +def enumerative_search( + pcfg: ProbDetGrammar, + name: str, + custom_enumerate: Callable[ + [Union[ProbDetGrammar, ProbUGrammar]], ProgramEnumerator + ], + programs: int, + timeout: int = 300, + title: Optional[str] = None, + seed: int = -1, +) -> Tuple[ + Tuple[str, int, int, float, int, int, int, int], + List[Tuple[str, int, int, float, int, int, int, int]], +]: + n = 0 + non_terminals = len(pcfg.rules) + derivation_rules = sum(len(pcfg.rules[S]) for S in pcfg.rules) + used_time = 0 + + pbar = tqdm.tqdm(total=programs, desc=title or name, smoothing=0) + enumerator = custom_enumerate(pcfg) + gen = enumerator.generator() + program = 1 + datum_each = 100000 + target_generation_speed = 1000000 + start = 0 + detailed = [] + try: + + def fun(): + return next(gen) + + get_next = timeout_decorator.timeout(timeout, timeout_exception=StopIteration)( + fun + ) + start = time.perf_counter_ns() + while program is not None: + program = get_next() + n += 1 + if n % datum_each == 0 or n >= programs: + used_time = time.perf_counter_ns() - start + bef = time.perf_counter_ns() + if used_time >= timeout * 1e9: + break + pbar.update(datum_each) + if n >= programs: + break + detailed.append( + ( + name, + non_terminals, + derivation_rules, + used_time / 1e9, + n, + enumerator.programs_in_queues(), + enumerator.programs_in_banks(), + seed, + ) + ) + rem_time = timeout - used_time / 1e9 + get_next = timeout_decorator.timeout( + rem_time, timeout_exception=StopIteration + )(fun) + start -= time.perf_counter_ns() - bef + except (StopIteration, RuntimeError): + used_time = time.perf_counter_ns() - start + pbar.close() + factor = target_generation_speed / n + return ( + name, + non_terminals, + derivation_rules, + used_time * factor / 1e9, + target_generation_speed, + int(enumerator.programs_in_queues() * factor), + int(enumerator.programs_in_banks() * factor), + seed, + ), detailed + + +# Main ==================================================================== + + +def gen_distance(non_terminals: int) -> CFG: + syntax = { + "1": "s1", + } + for i in range(2, non_terminals + 1): + syntax[f"cast{i}"] = f"s{i} -> s1" + syntax[f"s{i}"] = f"s{i}" + syntax[f"+{i}"] = f"s1 -> s{i} -> s{i}" + syntax[f"*{i}"] = f"s{i-1} -> s{i} -> s{i+1} -> s{i}" + return CFG.infinite(DSL(auto_type(syntax)), auto_type("s1->s1"), n_gram=1) + + +def gen_nonterminals(non_terminals: int) -> CFG: + syntax = { + "1": "s1", + } + for i in range(2, non_terminals + 1): + syntax[f"cast{i}"] = f"s{i} -> s1" + syntax[f"s{i}"] = f"s{i}" + syntax[f"+{i}"] = f"s1 -> s{i}" + return CFG.infinite(DSL(auto_type(syntax)), auto_type("s1->s1"), n_gram=1) + + +def gen_derivations(non_terminals: int) -> CFG: + syntax = { + "1": "s1", + } + for i in range(2, non_terminals + 1): + syntax[f"m{i}"] = f"s1 -> s1 -> s1" + syntax[f"f{i}"] = f"s1 -> s1" + syntax[f"s{i}"] = f"s1" + return CFG.infinite(DSL(auto_type(syntax)), auto_type("s1->s1"), n_gram=1) + + +if __name__ == "__main__": + # trace_rules = [ + # ( + # "search", + # "non_terminals", + # "derivation_rules", + # "time", + # "programs", + # "queue", + # "bank", + # ) + # ] + # print("Working on derivation rules scaling") + # for derivation_rules in range(2, max_rules + 1, 10): + # syntax = { + # "+": "s -> s", + # "1": "s", + # } + # for i in range(2, derivation_rules): + # syntax[f"{i}"] = "s" + # cfg = CFG.infinite(DSL(auto_type(syntax)), auto_type("s->s"), n_gram=1) + # pcfg = ProbDetGrammar.uniform(cfg) + # for name, enum in SEARCH_ALGOS.items(): + # trace_rules.append( + # enumerative_search(pcfg, name, enum, programs, timeout=timeout, title=f"{name}-{derivation_rules}") # type: ignore + # ) + # save(trace_rules, file_name + "_rules.csv") + # print("csv file was saved as:", file_name + "_rules.csv") + gene = { + "distance": gen_distance, + "nonterminals": gen_nonterminals, + "derivations": gen_derivations, + } + summary_trace = [ + ( + "search", + "non_terminals", + "derivation_rules", + "time", + "programs", + "queue", + "bank", + "seed", + ) + ] + detailed_trace = [ + ( + "search", + "non_terminals", + "derivation_rules", + "time", + "programs", + "queue", + "bank", + "seed", + ) + ] + print("Working on non terminals scaling") + non_terminals_values = [4] + while non_terminals_values[-1] < max_non_terminals: + last = non_terminals_values[-1] + if last <= 10: + last += 2 + else: + last += 10 + non_terminals_values.append(last) + first = True + for non_terminals in non_terminals_values: + cfg = gene[scaling](non_terminals) + if seed < 0: + pcfg = ProbDetGrammar.uniform(cfg) + else: + pcfg = ProbDetGrammar.random(cfg, seed=seed) + for name, enum in SEARCH_ALGOS.items(): + if first: + summary, detailed = enumerative_search( + pcfg, + name, + enum, + programs, + timeout=timeout, + title=f"{name}-{non_terminals}", + seed=seed, + ) # type: ignore + summary_trace.append(summary) + detailed_trace += detailed + else: + summary_trace.append( + summary_enumerative_search( + pcfg, + name, + enum, + programs, + timeout=timeout, + title=f"{name}-{non_terminals}", + seed=seed, + ) # type: ignore + ) + save(summary_trace, file_name + "_growth.csv") + save(detailed_trace, file_name + "_detailed.csv") + first = False + print("growth csv file was saved as:", file_name + "_growth.csv") + print("detailed csv file was saved as:", file_name + "_detailed.csv") diff --git a/examples/nlp/README.md b/examples/nlp/README.md deleted file mode 100644 index 945f756d..00000000 --- a/examples/nlp/README.md +++ /dev/null @@ -1,34 +0,0 @@ -# Examples - -This folder contains ready to use scripts and files that you can leverage to reproduce results from papers for example or to test your new ideas. - - - -- [Programming from Natural Language](#programming-by-example) - - [CoNaLa](#conala) - - [Downloading CoNaLa](#downloading-conala) - - - -## Programming from Natural Language - -TODO - -### CoNaLa - -This is a dataset of: -> *Yin, Pengcheng and Deng, Bowen and Chen, Edgar and Vasilescu, Bogdan and Neubig, Graham.* Learning to Mine Aligned Code and Natural Language Pairs from Stack Overflow. In International Conference on Mining Software Repositories, MSR, 2018. URL . - -This folder contains two files. -The `conala.py` file contains an impelementation of the DSL along with a default evaluator. -The `convert_conala.py` is a runnable python script which enables you to convert the original CoNaLa dataset files to the ProgSynth format. - -#### Downloading CoNaLa - -You can download the archive from thei website: . Then you simply need to: - -```bash -unzip conala-corpus-v1.1.zip -``` - -You should see a few JSON files in a ``conala-corpus`` folder, these JSON files are now convertible with the `convert_conala.py` script. \ No newline at end of file diff --git a/examples/nlp/conala/conala.py b/examples/nlp/conala/conala.py deleted file mode 100644 index a4d2c9d8..00000000 --- a/examples/nlp/conala/conala.py +++ /dev/null @@ -1,200 +0,0 @@ -from synth.semantic import DSLEvaluator -from synth.syntax import DSL, INT, Arrow, PolymorphicType, List - -t0 = PolymorphicType("t0") -t1 = PolymorphicType("t1") - - -def __access__(i, l): - if i is None: - return None - elif (i >= 0 and len(l) > i) or (i < 0 and len(l) >= -i): - return l[i] - else: - return None - - -def __scanl__(op): - def aux(l): - if len(l) == 0: - return [] - else: - y = [l[0]] - for x in l[1:]: - last = y[-1] - y.append(op(last, x)) - return y - - return aux - - -__semantics = { - "HEAD": lambda l: l[0] if len(l) > 0 else None, - "TAIL": lambda l: l[-1] if len(l) > 0 else None, - "ACCESS": lambda i: lambda l: __access__(i, l), - "MINIMUM": lambda l: min(l) if len(l) > 0 else None, - "MAXIMUM": lambda l: max(l) if len(l) > 0 else None, - "LENGTH": lambda l: len(l), - "COUNT[<0]": lambda l: len([x for x in l if x < 0]), - "COUNT[>0]": lambda l: len([x for x in l if x > 0]), - "COUNT[EVEN]": lambda l: len([x for x in l if x % 2 == 0]), - "COUNT[ODD]": lambda l: len([x for x in l if x % 2 == 1]), - "SUM": lambda l: sum(l), - "TAKE": lambda i: lambda l: l[:i], - "DROP": lambda i: lambda l: l[i:], - "SORT": lambda l: sorted(l), - "REVERSE": lambda l: l[::-1], - "FILTER[<0]": lambda l: [x for x in l if x < 0], - "FILTER[>0]": lambda l: [x for x in l if x > 0], - "FILTER[EVEN]": lambda l: [x for x in l if x % 2 == 0], - "FILTER[ODD]": lambda l: [x for x in l if x % 2 == 1], - "MAP[+1]": lambda l: [x + 1 for x in l], - "MAP[-1]": lambda l: [x - 1 for x in l], - "MAP[*2]": lambda l: [x * 2 for x in l], - "MAP[/2]": lambda l: [int(x / 2) for x in l], - "MAP[*3]": lambda l: [x * 3 for x in l], - "MAP[/3]": lambda l: [int(x / 3) for x in l], - "MAP[*4]": lambda l: [x * 4 for x in l], - "MAP[/4]": lambda l: [int(x / 4) for x in l], - "MAP[**2]": lambda l: [x**2 for x in l], - "MAP[*-1]": lambda l: [-x for x in l], - "ZIPWITH[+]": lambda l1: lambda l2: [x + y for (x, y) in zip(l1, l2)], - "ZIPWITH[-]": lambda l1: lambda l2: [x - y for (x, y) in zip(l1, l2)], - "ZIPWITH[*]": lambda l1: lambda l2: [x * y for (x, y) in zip(l1, l2)], - "ZIPWITH[max]": lambda l1: lambda l2: [ - (x if x > y else y) for (x, y) in zip(l1, l2) - ], - "ZIPWITH[min]": lambda l1: lambda l2: [ - (y if x > y else x) for (x, y) in zip(l1, l2) - ], - "SCAN1L[+]": __scanl__(lambda x, y: x + y), - "SCAN1L[-]": __scanl__(lambda x, y: x - y), - "SCAN1L[*]": __scanl__(lambda x, y: x * y), - "SCAN1L[min]": __scanl__(lambda x, y: min(x, y)), - "SCAN1L[max]": __scanl__(lambda x, y: max(x, y)), - # 'MAP': lambda f: lambda l: list(map(f, l)), -} - -__primitive_types = { - "HEAD": Arrow(List(INT), INT), - "TAIL": Arrow(List(INT), INT), - "ACCESS": Arrow(INT, Arrow(List(INT), INT)), - "MINIMUM": Arrow(List(INT), INT), - "MAXIMUM": Arrow(List(INT), INT), - "LENGTH": Arrow(List(INT), INT), - "COUNT[<0]": Arrow(List(INT), INT), - "COUNT[>0]": Arrow(List(INT), INT), - "COUNT[EVEN]": Arrow(List(INT), INT), - "COUNT[ODD]": Arrow(List(INT), INT), - "SUM": Arrow(List(INT), INT), - "TAKE": Arrow(INT, Arrow(List(INT), List(INT))), - "DROP": Arrow(INT, Arrow(List(INT), List(INT))), - "SORT": Arrow(List(INT), List(INT)), - "REVERSE": Arrow(List(INT), List(INT)), - "FILTER[<0]": Arrow(List(INT), List(INT)), - "FILTER[>0]": Arrow(List(INT), List(INT)), - "FILTER[EVEN]": Arrow(List(INT), List(INT)), - "FILTER[ODD]": Arrow(List(INT), List(INT)), - "MAP[+1]": Arrow(List(INT), List(INT)), - "MAP[-1]": Arrow(List(INT), List(INT)), - "MAP[*2]": Arrow(List(INT), List(INT)), - "MAP[/2]": Arrow(List(INT), List(INT)), - "MAP[*-1]": Arrow(List(INT), List(INT)), - "MAP[**2]": Arrow(List(INT), List(INT)), - "MAP[*3]": Arrow(List(INT), List(INT)), - "MAP[/3]": Arrow(List(INT), List(INT)), - "MAP[*4]": Arrow(List(INT), List(INT)), - "MAP[/4]": Arrow(List(INT), List(INT)), - "ZIPWITH[+]": Arrow(List(INT), Arrow(List(INT), List(INT))), - "ZIPWITH[-]": Arrow(List(INT), Arrow(List(INT), List(INT))), - "ZIPWITH[*]": Arrow(List(INT), Arrow(List(INT), List(INT))), - "ZIPWITH[min]": Arrow(List(INT), Arrow(List(INT), List(INT))), - "ZIPWITH[max]": Arrow(List(INT), Arrow(List(INT), List(INT))), - "SCAN1L[+]": Arrow(List(INT), List(INT)), - "SCAN1L[-]": Arrow(List(INT), List(INT)), - "SCAN1L[*]": Arrow(List(INT), List(INT)), - "SCAN1L[min]": Arrow(List(INT), List(INT)), - "SCAN1L[max]": Arrow(List(INT), List(INT)), - # 'MAP': Arrow(Arrow(t0,t1),Arrow(List(t0),List(t1))), -} - - -__forbidden_patterns = [ - ["HEAD", "SCAN1L[-]"], - ["HEAD", "SCAN1L[min]"], - ["HEAD", "SCAN1L[*]"], - ["HEAD", "SCAN1L[max]"], - ["HEAD", "SCAN1L[+]"], - ["MINIMUM", "SORT"], - ["MINIMUM", "REVERSE"], - ["MINIMUM", "SCAN1L[min]"], - ["MAXIMUM", "REVERSE"], - ["MAXIMUM", "SORT"], - ["MAXIMUM", "SCAN1L[max]"], - ["LENGTH", "MAP[/4]"], - ["LENGTH", "MAP[**2]"], - ["LENGTH", "SCAN1L[max]"], - ["LENGTH", "MAP[+1]"], - ["LENGTH", "SORT"], - ["LENGTH", "MAP[/3]"], - ["LENGTH", "SCAN1L[*]"], - ["LENGTH", "MAP[*2]"], - ["LENGTH", "MAP[*3]"], - ["LENGTH", "SCAN1L[+]"], - ["LENGTH", "MAP[*-1]"], - ["LENGTH", "MAP[/2]"], - ["LENGTH", "SCAN1L[-]"], - ["LENGTH", "SCAN1L[min]"], - ["LENGTH", "REVERSE"], - ["LENGTH", "MAP[-1]"], - ["LENGTH", "MAP[*4]"], - ["COUNT[<0]", "MAP[*2]"], - ["COUNT[<0]", "SORT"], - ["COUNT[<0]", "MAP[*3]"], - ["COUNT[<0]", "MAP[*4]"], - ["COUNT[<0]", "REVERSE"], - ["COUNT[<0]", "FILTER[<0]"], - ["COUNT[>0]", "SORT"], - ["COUNT[>0]", "MAP[*3]"], - ["COUNT[>0]", "MAP[-1]"], - ["COUNT[>0]", "FILTER[>0]"], - ["COUNT[>0]", "MAP[/2]"], - ["COUNT[>0]", "MAP[*4]"], - ["COUNT[>0]", "REVERSE"], - ["COUNT[>0]", "MAP[*2]"], - ["COUNT[EVEN]", "REVERSE"], - ["COUNT[EVEN]", "MAP[**2]"], - ["COUNT[EVEN]", "FILTER[EVEN]"], - ["COUNT[EVEN]", "MAP[*3]"], - ["COUNT[EVEN]", "SORT"], - ["COUNT[EVEN]", "MAP[*-1]"], - ["COUNT[ODD]", "REVERSE"], - ["COUNT[ODD]", "FILTER[ODD]"], - ["COUNT[ODD]", "MAP[**2]"], - ["COUNT[ODD]", "MAP[*3]"], - ["COUNT[ODD]", "SORT"], - ["COUNT[ODD]", "MAP[*-1]"], - ["SUM", "SORT"], - ["SUM", "REVERSE"], - ["SORT", "REVERSE"], - ["SORT", "SORT"], - ["REVERSE", "REVERSE"], - ["FILTER[<0]", "FILTER[<0]"], - ["FILTER[>0]", "FILTER[>0]"], - ["FILTER[EVEN]", "FILTER[EVEN]"], - ["FILTER[ODD]", "FILTER[ODD]"], - ["MAP[+1]", "MAP[-1]"], - ["MAP[-1]", "MAP[+1]"], - ["MAP[/2]", "MAP[*2]"], - ["MAP[*-1]", "MAP[*-1]"], - ["MAP[**2]", "MAP[*-1]"], - ["MAP[/3]", "MAP[*3]"], - ["MAP[/4]", "MAP[*4]"], - ["SCAN1L[min]", "SCAN1L[min]"], - ["SCAN1L[max]", "SCAN1L[max]"], -] - -dsl = DSL(__primitive_types, __forbidden_patterns) -evaluator = DSLEvaluator(__semantics) -evaluator.skip_exceptions.add(OverflowError) -lexicon = list(range(-256, 256 + 1)) diff --git a/examples/nlp/conala/convert_conala.py b/examples/nlp/conala/convert_conala.py deleted file mode 100644 index 75f749fa..00000000 --- a/examples/nlp/conala/convert_conala.py +++ /dev/null @@ -1,258 +0,0 @@ -import ast -import json -import re -import importlib -from dataclasses import dataclass, field -from typing import Dict, Optional, Set, Tuple, List as TList -from pathlib import Path - -import tqdm - -from synth import Task, Dataset -from synth.specification import NLP -from synth.syntax import ( - INT, - FunctionType, - List, - Function, - Primitive, - Program, - Variable, - Arrow, - Type, - guess_type, -) - -from conala import dsl, evaluator -from synth.syntax.type_system import STRING, PolymorphicType, UnknownType - -VAR = re.compile("'([^' ]*)'") - -KNOWN_TYPES = { - "string": STRING, - "list": List(PolymorphicType("any")), - "integer": INT, -} - - -@dataclass -class ParseContext: - varno: int = field(default=0) - variables: Dict[str, Tuple[int, Type]] = field(default_factory=lambda: {}) - modules: Set[str] = field(default_factory=set) - rest: Set[str] = field(default_factory=set) - - def variable_by_no(self, no: int) -> Optional[Tuple[str, Type]]: - for name, (varno, t) in self.variables.items(): - if varno == no: - return name, t - - -def try_parse_variable(value: str, ctx: ParseContext) -> Optional[Variable]: - if value in ctx.variables: - (varno, var_type) = ctx.variables[value] - return Variable(varno, var_type) - return None - - -def build_program(node: ast.AST, ctx: ParseContext) -> Program: - if isinstance(node, ast.Call): - - func = build_program(node.func, ctx) - if len(node.args) == 0: - return func - args = [build_program(arg, ctx) for arg in node.args] - correct_type = FunctionType(*[arg.type for arg in args], INT) - if isinstance(func, Function): - func.function.type = FunctionType( - *[arg.type for arg in func.arguments], correct_type - ) - func = Function(func.function, func.arguments) - elif isinstance(func, Primitive): - func.type = correct_type - # print("call:", func, "\n\t", args, "\n\ttype:", correct_type, [arg.type for arg in args]) - assert isinstance( - func.type, Arrow - ), f"func={func}, type={func.type} type's type={type(func.type)}" - if len(node.keywords) > 0: - raise ValueError - return Function(func, args) - elif isinstance(node, ast.Name): - # This is basically module names and variables - assert isinstance(node.ctx, ast.Load) - value = str(node.id) - var_prog = try_parse_variable(value, ctx) - return var_prog if var_prog else Primitive(node.id, INT) - elif isinstance(node, ast.Attribute): - assert isinstance(node.ctx, ast.Load) - object = build_program(node.value, ctx) - # print("attribute:", node.attr, "of", object) - if isinstance(object, Primitive): - spec = importlib.util.find_spec(object.primitive) - if spec is not None: - ctx.modules.add(object.primitive) - return Primitive(object.primitive + "." + node.attr, INT) - else: - ctx.rest.add(object.primitive) - object.type = FunctionType(INT, INT) - return Function(object, [Primitive(node.attr, INT)]) - elif isinstance(object, Variable): - res = ctx.variable_by_no(object.variable) - assert res is not None - _, vartype = res - return Function( - Primitive(f"{vartype}.{node.attr}", Arrow(object.type, INT)), [object] - ) - return Function(Primitive(node.attr, Arrow(object.type, INT)), [object]) - elif isinstance(node, ast.Expr): - return build_program(node.value, ctx) - elif isinstance(node, ast.Constant): - value = str(node.value) - var_prog = try_parse_variable(value, ctx) - if var_prog is None: - # out = Variable(ctx.varno, guess_type(node.value)) - # ctx.variables[node.value] = ctx.varno - # ctx.varno += 1 - out = Primitive(node.value, guess_type(node.value)) - assert False - else: - return var_prog - elif isinstance(node, ast.List) or isinstance(node, ast.Tuple): - elements = [build_program(el, ctx) for el in node.elts] - if not all(isinstance(el, Primitive) for el in elements): - raise ValueError - return Function( - Primitive( - "list", FunctionType(*[el.type for el in elements], INT), elements - ) - ) - elif isinstance(node, ast.Subscript): - assert isinstance(node.ctx, ast.Load) - object = build_program(node.value, ctx) - # print("SLICE:", node.slice) - slice = build_program(node.slice, ctx) - # raise ValueError - elements = [slice, object] - return Function( - Primitive( - "slice", FunctionType(*[el.type for el in elements], INT), elements - ) - ) - else: - # print("CANNOT BUILD:", node) - raise ValueError - - -def try_parse_snippet( - snippet: str, variables: Dict[str, int] -) -> Optional[Tuple[Program, ParseContext]]: - tree = ast.parse(snippet) - lines = tree.body - if len(lines) > 1: - return None - content = lines[0] - ctx = ParseContext(len(variables), variables) - try: - return build_program(content, ctx), ctx - except: - return None - - -def extract_variables(intent: str) -> Tuple[str, Dict[str, Tuple[int, Type]]]: - variables = {} - for var in re.finditer(VAR, intent): - content = var.group(1).encode().decode("unicode_escape") - - start_pos = var.start() - 1 - word_before = intent[intent.rfind(" ", 0, start_pos) + 1 : start_pos] - var_type = KNOWN_TYPES.get(word_before, UnknownType()) - varno = len(variables) - variables[content] = (varno, var_type) - intent = intent.replace(var.group(0), f"var{varno}") - return intent, variables - - -def try_convert_task(task: Dict[str, str], tasks: TList[Task[NLP]]) -> None: - intent = task["intent"] - if task["rewritten_intent"] is None: - return - intent = task["rewritten_intent"].replace("`", "'").replace('"', "'") - metadata = {"question_id": int(task["question_id"])} - snippet: str = task["snippet"] - try: - intent, variables = extract_variables(intent) - except UnicodeDecodeError: - return - if len(tasks) > 0 and tasks[-1].specification.intent == intent: - return - out = try_parse_snippet(snippet, variables) - if out and len(out[1].rest) == 0: - solution, ctx = out - vars = list(variables.values()) - vars.sort(key=lambda x: x[0]) - if any(isinstance(var[1], UnknownType) for var in vars): - return - type_request = FunctionType(*[var[1] for var in vars], INT) - print("OG Description=", task["rewritten_intent"]) - print("Description=", intent) - print("TR=", type_request) - print("Variables=", ctx.variables) - print("Type=", solution.type) - # print("Modules=", ctx.modules) - # print("Rest=", ctx.rest) - print("Solution=", solution) - print("Original=", task["snippet"]) - print() - tasks.append(Task(solution.type, NLP(intent), solution, metadata)) - # else: - # print("FAILED:", intent) - - -def convert_conala(file: str) -> None: - filename: str = Path(file).stem - output_file = f"./{filename}.pickle" - - # 1st task processing - # Crude parsing of programs - tasks: List[Task[NLP]] = [] - with open(file) as fd: - data = json.load(fd) - for task in tqdm.tqdm(data): - try_convert_task(task, tasks) - - # 2nd task processing - # Remove constants out - primitives = set() - for task in tasks: - sol = task.solution - assert sol is not None - for P in sol.depth_first_iter(): - if isinstance(P, Primitive): - primitives.add(P.primitive) - print("Primitives:", primitives) - dataset = Dataset(tasks) - dataset.save(output_file) - print( - "Successfully saved converted dataset file as", - output_file, - "with", - len(tasks), - "tasks", - ) - - -if __name__ == "__main__": - import argparse - - argument_parser: argparse.ArgumentParser = argparse.ArgumentParser( - description="Convert and filter CoNaLa original dataset to ProgSynth format." - ) - - argument_parser.add_argument( - type=str, - dest="file", - action="store", - help="Source JSON CoNaLa file to be converted", - ) - parsed_parameters = argument_parser.parse_args() - convert_conala(parsed_parameters.file) diff --git a/examples/nlp/dataset_explorer.py b/examples/nlp/dataset_explorer.py deleted file mode 100644 index c48a1658..00000000 --- a/examples/nlp/dataset_explorer.py +++ /dev/null @@ -1,197 +0,0 @@ -import sys -from typing import Optional - -from colorama import Fore as F - -from synth import Dataset, NLP -from synth.syntax import CFG -from synth.task import Task -from synth.utils import chrono - -CONALA = "conala" - - -import argparse - -parser = argparse.ArgumentParser( - description="Generate a dataset copying the original distribution of another dataset" -) -parser.add_argument( - "--dsl", - type=str, - default=CONALA, - help=f"dsl (default: {CONALA})", - choices=[CONALA], -) -parser.add_argument( - "--dataset", - type=str, - default="{dsl_name}.pickle", - help="dataset file (default: {dsl_name}.pickle)", -) - -parameters = parser.parse_args() -dsl_name: str = parameters.dsl -dataset_file: str = parameters.dataset.format(dsl_name=dsl_name) -# ================================ -# Load constants specific to DSL -# ================================ -if dsl_name == CONALA: - from conala.conala import dsl, lexicon -else: - print(F.LIGHTRED_EX + "Unknown dsl:", dsl_name + F.RESET, file=sys.stderr) - sys.exit(1) -# ================================ -# Load dataset & Task Generator -# ================================ -# Load dataset -print(f"Loading {F.LIGHTCYAN_EX}{dataset_file}{F.RESET}...", end="") -with chrono.clock("dataset.load") as c: - full_dataset: Dataset[NLP] = Dataset.load(dataset_file) - print(f"done in{F.LIGHTYELLOW_EX}", c.elapsed_time(), f"s{F.RESET}") - - -def print_value(name: str, value: str) -> None: - print(f"{F.GREEN}{name}{F.RESET}: {F.LIGHTYELLOW_EX}{value}{F.RESET}") - - -def summary(*args: str) -> None: - all_type_requests = full_dataset.type_requests() - print( - f"{F.LIGHTYELLOW_EX}{len(full_dataset)} {F.GREEN}tasks{F.RESET}, {F.LIGHTYELLOW_EX}{len([task for task in full_dataset if task.solution]) / len(full_dataset):.1%}{F.RESET} of which have {F.GREEN}solutions{F.RESET}." - ) - print( - f"{F.LIGHTYELLOW_EX}{len(all_type_requests)} {F.GREEN}type requests{F.RESET} supported." - ) - print_value("Lexicon", f"[{min(lexicon)};{max(lexicon)}]") - - -def types(*args: str) -> None: - all_type_requests = full_dataset.type_requests() - total = len(full_dataset) - max_len = max([len(str(t)) for t in all_type_requests]) - for type_req in all_type_requests: - n = len([task for task in full_dataset if task.type_request == type_req]) - percent = f"{n / total:.1%}" - print_value(f"{type_req!s:<{max_len}}", f"({percent:>5}) {n}") - - -def cfg(*args: str) -> None: - max_depth = 4 - if args: - try: - max_depth = int(args[0]) - except: - pass - all_type_requests = full_dataset.type_requests() - print_value("Max Depth", max_depth) - max_len = max([len(str(t)) for t in all_type_requests]) - programs_no = { - t: f"{CFG.depth_constraint(dsl, t, max_depth).size():,}" - for t in all_type_requests - } - max_len_programs_no = max(len(s) for s in programs_no.values()) - print_value( - "{0:<{max_len}}".format("", max_len=max_len), - "{0:>{max_len_programs_no}} {1}".format( - "", - "", - max_len_programs_no=max_len_programs_no, - ), - ) - for type_req in all_type_requests: - cfg = CFG.depth_constraint(dsl, type_req, max_depth) - print_value( - f"{type_req!s:<{max_len}}", - f"{programs_no[type_req]:>{max_len_programs_no}} {len(cfg.rules)}", - ) - - -def task(*args: str) -> None: - try: - task_no = int(args[0]) - except: - print( - f"{F.LIGHTRED_EX}You must specify a task number in the range[0;{len(full_dataset) - 1}]!{F.RESET}" - ) - return - if task_no < 0 or task_no >= len(full_dataset): - print( - f"{F.LIGHTRED_EX}{task_no} is an invalid task number, it must be in the range [0;{len(full_dataset) - 1}]!{F.RESET}" - ) - return - task: Task[NLP] = full_dataset[task_no] - print_value(f"Name", task.metadata.get("name", "None")) - print_value("Type", task.type_request) - print_value("Solution", task.solution) - print_value("Intent", task.specification.intent) - print_value("Metadata", task.metadata) - - -def filter_tasks(*args: str) -> None: - if not args: - print( - F.LIGHTRED_EX - + "Invalid syntax: you must give a valid boolean python expression that only depend on a task parameter." - + F.RESET - ) - return - code = "[i for i, task in enumerate(full_dataset) if " + " ".join(args) + "]" - queried_tasks = eval(code) - if len(queried_tasks) == 0: - print(f"{F.LIGHTYELLOW_EX}No {F.GREEN}task{F.RESET} matched your query!") - elif len(queried_tasks) == 1: - print( - f"{F.GREEN}Task {F.LIGHTYELLOW_EX}n°{queried_tasks[0]}{F.RESET} matched your query!" - ) - else: - print( - f"{F.LIGHTYELLOW_EX}{len(queried_tasks)} {F.GREEN}tasks{F.RESET} matched your query:" - ) - print(queried_tasks) - - -COMMANDS = { - "summary": summary, - "types": types, - "task": task, - "cfg": cfg, - "lexicon": lambda *args: print(lexicon), - "filter": filter_tasks, -} - - -def __expand_name__(prefix: str) -> Optional[str]: - candidates = list(COMMANDS.keys()) - for i, l in enumerate(prefix): - candidates = [cand for cand in candidates if len(cand) > i and cand[i] == l] - if len(candidates) == 1: - return candidates[0] - for cand in candidates: - if len(cand) == len(prefix): - return cand - return None - - -def try_execute_command(cmd: str) -> None: - words = cmd.split(" ") - real_cmd = __expand_name__(words.pop(0)) - if real_cmd: - COMMANDS[real_cmd](*words) - else: - print( - "Available commands are:", - ", ".join([F.GREEN + x + F.RESET for x in COMMANDS.keys()]) + ".", - ) - - -print(f'Type "{F.GREEN}help{F.RESET}" for help.') -print( - f'Unambigous commands prefixes work (e.g: "{F.LIGHTGREEN_EX}h{F.RESET}" for "{F.GREEN}help{F.RESET}").' -) -while True: - try: - cmd = input(">").lower().strip() - except EOFError: - break - try_execute_command(cmd) diff --git a/examples/nlp/human.json b/examples/nlp/human.json deleted file mode 100644 index b440f217..00000000 --- a/examples/nlp/human.json +++ /dev/null @@ -1,6 +0,0 @@ -[ - { - "intent": "Sort a list `numbers`.", - "solution": "(SORT var0)" - } -] \ No newline at end of file diff --git a/examples/nlp/train_bert.py b/examples/nlp/train_bert.py deleted file mode 100644 index 3c960644..00000000 --- a/examples/nlp/train_bert.py +++ /dev/null @@ -1,142 +0,0 @@ -from deepcoder import dsl, evaluator, lexicon -from typing import List - - -import tqdm - -import torch -from torch import Tensor -import torch.nn as nn - -import numpy as np - -from synth import Dataset, Task -from synth.nn import ( - PrimitivePredictorLayer, - print_model_summary, -) -from synth.specification import NLP -from synth.nlp import NLPEncoder -from synth.syntax import ConcreteCFG, Function, Variable, Arrow -from synth.utils import chrono - -seed: int = 1 -cpu_only = True -batch_size = 2 - -torch.manual_seed(seed) -# ============================= -# Misc init -# ================================ -# Get device -device = "cuda" if not cpu_only and torch.cuda.is_available() else "cpu" -print("Using device:", device) - -# Dataset -dataset = [] -for P in dsl.list_primitives: - if "[" in P.primitive: - continue - if isinstance(P.type, Arrow) and len(P.type.arguments()) == 1: - dataset.append( - Task[NLP]( - P.type, - NLP(P.primitive.lower() + " the list `list`."), - Function(P, [Variable(0, P.type.arguments()[0])]), - ) - ) - - -full_dataset = Dataset(dataset) -# ================================ -# Neural Network creation -# ================================ -# Generate the CFG dictionnary -all_type_requests = full_dataset.type_requests() -print(f"{len(all_type_requests)} type requests supported.") -print(f"Lexicon: [{min(lexicon)};{max(lexicon)}]") - - -class MyPredictor(nn.Module): - def __init__(self, size: int) -> None: - super().__init__() - self.primitive_layer = PrimitivePredictorLayer(size, dsl, 0.2) - self.encoder = NLPEncoder() - input_size = self.encoder.embedding_size - self.rnn = nn.LSTM(input_size, size, 1, batch_first=True) - - self.end = nn.Sequential( - nn.Linear(size, size), - nn.ReLU(), - nn.Linear(size, size), - nn.ReLU(), - ) - - def forward(self, x: List[Task[NLP]]) -> Tensor: - xx = [self.encoder.encode(task) for task in x] - xxx = torch.stack(xx).squeeze(1) - y0, _ = self.rnn(xxx) - y = y0.data[:, -1, :] - return self.primitive_layer(self.end(y)) - - -predictor = MyPredictor(2000).to(device) -print_model_summary(predictor) -optim = torch.optim.AdamW(predictor.parameters(), 1e-3) - -dataset_index = 0 - - -@chrono.clock(prefix="train.do_batch") -def get_batch_of_tasks() -> List[Task[NLP]]: - global dataset_index - batch = full_dataset[dataset_index : dataset_index + batch_size] - dataset_index += batch_size - return batch - - -def do_batch(iter_number: int) -> None: - nb_batch_per_epoch = int(np.ceil(len(full_dataset) / batch_size)) - - batch = get_batch_of_tasks() - batch_programs = [task.solution for task in batch] - # Logging - with chrono.clock("train.do_batch.inference"): - batch_outputs: Tensor = predictor(batch) - # Gradient descent - with chrono.clock("train.do_batch.loss"): - optim.zero_grad() - with chrono.clock("train.do_batch.loss.compute"): - loss = predictor.primitive_layer.loss(batch_programs, batch_outputs) - with chrono.clock("train.do_batch.loss.backprop"): - loss.backward() - optim.step() - if (iter_number + 1) % nb_batch_per_epoch == 0: - print("Loss=", loss.item()) - task = batch[0] - print(task) - out = batch_outputs[0] - pcfg = predictor.primitive_layer.tensor2pcfg( - out, task.type_request, max_depth=2 - ).to_pcfg() - print(pcfg) - - -def do_epoch(j: int) -> int: - global dataset_index - dataset_index = 0 - nb_batch_per_epoch = int(np.ceil(len(full_dataset) / batch_size)) - i = j - for _ in tqdm.trange(nb_batch_per_epoch, desc="batchs"): - do_batch(i) - i += 1 - return i - - -def train() -> None: - j = 0 - for ep in tqdm.trange(100, desc="epochs"): - j = do_epoch(j) - - -train() diff --git a/examples/pbe/README.md b/examples/pbe/README.md index 05aa94f3..b3222b13 100644 --- a/examples/pbe/README.md +++ b/examples/pbe/README.md @@ -4,6 +4,10 @@ This folder contains ready to use scripts and files that you can leverage to rep +- [Scripts](#scripts) + - [Dataset Manipulation](#dataset-manipulation) + - [Model Training and Evaluation](#model-training-and-evaluation) + - [DSL Manipulation](#dsl-manipulation) - [DSLs](#dsls) - [Calculator](#calculator) - [Deepcoder](#deepcoder) @@ -16,19 +20,36 @@ This folder contains ready to use scripts and files that you can leverage to rep - [Example pipeline](#example-pipeline) -Here is an exhaustive list of available scripts, we recommend running them with -h to see the available options: +## Scripts -- The `dataset_generator.py` loads a dataset, reproduces the task distribution, and generate a new synthetic dataset from scratch. -- The `dataset_explorer.py` loads a dataset and will provide you with an interactive prompt to explore the dataset. Use `help` to see the list of commands in the interactive prompt. -- The `evaluate.py` loads a dataset, a model, and runs heap search on every task trying to find a correct solution to the task. -- The `plot_results.py` plot the results files created by ``evaluate.py``. -- The `model_trainer.py` loads a dataset then train a neural net to predict the probabilities of the grammar. Metrics are logged with [TensorBoard](https://www.tensorflow.org/tensorboard/) and a report of time spent is printed at the end of the script. -- The `dataset_improve.py` takes a dataset and a solution file (obtained with `evaluate.py`) and replace the solutions of the dataset by the ones found if they are shorter. -- The `dsl_analyser.py` loads a dataset and tries to find all redundant derivations at depth 2 such as `(MAP[/4] (MAP[*4] var0))` or `(LENGTH (SORT var0))` and produces constraints to forbid them using pruning. +### Dataset Manipulation + +You can **explore** a dataset with `dataset_explorer.py`. It will provide you with an interactive prompt to explore the dataset. Use `help` to see the list of commands in the interactive prompt. + +You can **generate synthetic datasets** based on a existing dataset, reproducing their distribution with: + +- `dataset_generator.py` +- `dataset_generator_unique.py` which has the constraint that programs must uniquely identifiable from the examples. This is the **recommended way** of generating a dataset. + +You can **improve solutions of a dataset** with `dataset_improve.py`. It takes a dataset and a solution file (obtained with `evaluate.py`) and replace the solutions of the dataset by the ones found if they are shorter. + +### Model Training and Evaluation + +You can **train a model** with `model_trainer.py`. It loads a dataset then train a neural net to predict the probabilities of the grammar. Metrics are logged with [TensorBoard](https://www.tensorflow.org/tensorboard/. + +You can **evaluate a model** with `evaluate.py`. It loads a dataset, a model, and runs our synthesis algorithm on every task trying to find a correct solution to the task. + +You can **plot the results** of `evaluate.py` with `plot_results.py` which is located in the parent folder. + +### DSL Manipulation + +You can **find equations automatically for a given DSL** with `dsl_equation_generator.py`. loads a dataset and tries to find all redundant derivations at depth 2 such as `(MAP[/4] (MAP[*4] var0))` or `(LENGTH (SORT var0))` and produces constraints to forbid them. + +You can **learn new primitives** with `dataset_learner.py`. It loads a dataset and try to learn a new primitive that would most help with expressing the dataset. ## DSLs -Here is an exhaustive list of available DSLs wit hthis specification. +Here is an exhaustive list of available DSLs with this specification. ### Calculator @@ -76,7 +97,7 @@ This is a dataset where there are positive and negative examples the goal is to ### Transductions -THis is a dataset of string manipulation, it contains per-task constants, no polymrphic types nor need lambdas. +This is a dataset of string manipulation, it contains per-task constants, no polymrphic types nor need lambdas. This is a dataset in the idea of the string manipulation dataset of FlashFill: > *S. Gulwani* Automating String Processing in Spreadsheets using Input-Output Examples. PoPL'11, January 26-28, 2011, Austin, Texas, USA. URL @@ -88,26 +109,32 @@ The SL files were converted and compressed in some JSON file that is provided in ## Example Pipeline Here is an example pipeline for a DSL. -This would first produce a ttrain and test dataset, then train a model, evaluate it then plot the results. +This would first produce a train and test dataset, then train a model, evaluate it then plot the results. ```bash #!/bin/env bash # =================================================================== # PARAMETERS # =================================================================== +DSL_NAME="transduction" +BASE_DATASET="./flashfill.pickle" SEED=2 +### TASK PARAMETERS +# maximum depth of programs +MAX_DEPTH=5 +# maximum number of examples per task +MAX_EXAMPLES=5 +### MODEL TRAINING PARAMETERS # size of training dataset TRAIN_SIZE=2500 -# useful only if test dataset != base dataset -TEST_SIZE=500 BATCH_SIZE=16 EPOCHS=2 +### EVALUATION PARAMETERS +TEST_DATASET="$BASE_DATASET" +# useful only if test dataset != base dataset +TEST_SIZE=500 # timeout in seconds for the evaluation TIMEOUT=60 - -DSL_NAME="transduction" -BASE_DATASET="./flashfill.pickle" -TEST_DATASET="$BASE_DATASET" # =================================================================== # CONSTANTS # =================================================================== @@ -117,7 +144,7 @@ MODEL_FILE="$EXPERIMENT_FOLDER/model.pt" # =================================================================== # MAIN CODE # =================================================================== -# Check deepcoder dataset exists +# Check base dataset exists if [ ! -f "$BASE_DATASET" ]; then echo "$BASE_DATASET is missing!" exit 1 @@ -127,7 +154,7 @@ mkdir -p $EXPERIMENT_FOLDER # Make test dataset if [ ! -f "$TEST_DATASET" ]; then echo "[Generation] Creating the test dataset." - python examples/pbe/dataset_generator.py --dsl $DSL_NAME --dataset $TEST_DATASET --seed $SEED --size $TEST_SIZE -o $TEST_DATASET + python examples/pbe/dataset_generator_unique.py --dsl $DSL_NAME --dataset $TEST_DATASET --seed $SEED --programs $TEST_SIZE --inputs 2 -o $TEST_DATASET --max-depth $MAX_DEPTH --max-examples $MAX_EXAMPLES if [ $? != "0" ]; then exit 2 fi @@ -136,7 +163,7 @@ fi # Make train dataset if [ ! -f "$TRAIN_DATASET" ]; then echo "[Generation] Creating the train dataset." - python examples/pbe/dataset_generator.py --dsl $DSL_NAME --dataset $TEST_DATASET --seed $SEED --size $TRAIN_SIZE -o $TRAIN_DATASET + python examples/pbe/dataset_generator_unique.py --dsl $DSL_NAME --dataset $TEST_DATASET --seed $SEED --programs $TRAIN_SIZE -o $TRAIN_DATASET --inputs 2 --max-depth $MAX_DEPTH --max-examples $MAX_EXAMPLES if [ $? != "0" ]; then exit 3 fi @@ -157,4 +184,4 @@ if [ $? != "0" ]; then fi # Plotting python examples/pbe/plot_results.py --dataset $TEST_DATASET --folder $EXPERIMENT_FOLDER -``` \ No newline at end of file +``` diff --git a/examples/pbe/analysis/dsl_analyzer.py b/examples/pbe/analysis/dsl_analyzer.py index 84c6955d..3e5f162b 100644 --- a/examples/pbe/analysis/dsl_analyzer.py +++ b/examples/pbe/analysis/dsl_analyzer.py @@ -54,9 +54,7 @@ def __count_programs__(solution: Program, pcfg: ProbDetGrammar) -> int: counters[str(sub_program)] = (counter, sub_program.depth()) found = True break - if ( - not found - ): # can happen when function signature does not correspond to pcfg (thus, cannot be found) + if not found: # can happen when function signature does not correspond to pcfg (thus, cannot be found) counters[str(sub_program)] = (MAX_TESTS, sub_program.depth()) return counters @@ -234,6 +232,7 @@ def forward(self, x: List[Task[PBE]]) -> Tensor: predictor.load_state_dict(torch.load(model_file)) predictor = predictor.to(device) predictor.eval() + # ================================ # Predict PCFG # ================================ diff --git a/examples/pbe/calculator/calculator.py b/examples/pbe/calculator/calculator.py index 17c1ebd5..165fbdcc 100644 --- a/examples/pbe/calculator/calculator.py +++ b/examples/pbe/calculator/calculator.py @@ -10,12 +10,21 @@ ) from synth.semantic import DSLEvaluator, Evaluator from synth.specification import PBE -from synth.syntax import DSL, INT, Arrow, PolymorphicType, PrimitiveType, BOOL +from synth.syntax import ( + DSL, + INT, + Arrow, + FixedPolymorphicType, + PrimitiveType, + BOOL, + FunctionType, + auto_type, +) from synth.task import Dataset # a type representing either an int or a float -type = PolymorphicType("int/float") FLOAT = PrimitiveType("float") +type = FixedPolymorphicType("int/float", INT | FLOAT) __semantics = { "+": lambda a: lambda b: round(a + b, 1), @@ -30,8 +39,8 @@ __primitive_types = { # int|float -> int|float -> int|float "+": Arrow(type, Arrow(type, type)), - "-": Arrow(type, Arrow(type, type)), - "int2float": Arrow(INT, FLOAT), + "-": FunctionType(type, type, type), + "int2float": auto_type("int->float"), "1": INT, "2": INT, "3": INT, @@ -39,12 +48,13 @@ # Short example of a forbidden patterns (if add1 and sub1 are defined in _semantics and _primitive_types) _forbidden_patterns = { - "add1": {"sub1"}, - "sub1": {"add1"}, + ("add1", 0): {"sub1"}, + ("sub1", 0): {"add1"}, } dsl = DSL(__primitive_types, forbidden_patterns=_forbidden_patterns) -evaluator = DSLEvaluator(__semantics) +dsl.instantiate_polymorphic_types() +evaluator = DSLEvaluator(dsl.instantiate_semantics(__semantics)) lexicon = [round(x, 1) for x in np.arange(-256, 256 + 1, 0.1)] @@ -55,9 +65,8 @@ def reproduce_calculator_dataset( seed: Optional[int] = None, int_bound: int = 1000, *args: Any, - **kwargs: Any + **kwargs: Any, ) -> Tuple[TaskGenerator, TList[int]]: - int_range: TList[int] = [int_bound, 0] int_range[1] = -int_range[0] @@ -112,5 +121,5 @@ def get_lexicon(start: None) -> TList[float]: get_lexicon, seed, *args, - **kwargs + **kwargs, ) diff --git a/examples/pbe/calculator/convert_calculator.py b/examples/pbe/calculator/convert_calculator.py index 62ec5cc2..1cb3871c 100644 --- a/examples/pbe/calculator/convert_calculator.py +++ b/examples/pbe/calculator/convert_calculator.py @@ -1,37 +1,19 @@ import json -from typing import Any, Callable, Dict, Tuple, List as TList +from typing import Any, Callable, Dict, List as TList import tqdm from synth import Task, Dataset, PBE, Example from synth.syntax import ( - INT, FunctionType, - List, - Type, - Arrow, - Function, - Primitive, Program, - Variable, - PrimitiveType, - PolymorphicType, + UnknownType, + guess_type, ) from calculator import dsl, evaluator, FLOAT -from synth.syntax.type_system import UnknownType, guess_type -# this dictionary contains the primitives as defined in the dsl -name2type: Dict[str, Type] = {p.primitive: p.type for p in dsl.list_primitives} -# this dictionary contains the instantiated primitives, that is to say after removal of polymorphic type -name2fulltype = {} dsl.instantiate_polymorphic_types(5) -for p in dsl.list_primitives: - if isinstance(p.type, Arrow): - name = str(p) + str(p.type.arguments()[0]) - else: - name = p.primitive - name2fulltype[name] = p.type def __convert__(load: Callable[[], Dataset[PBE]], name: str) -> None: @@ -42,7 +24,10 @@ def __convert__(load: Callable[[], Dataset[PBE]], name: str) -> None: # Integrity check for task in tqdm.tqdm(tasks, desc="integrity check"): for ex in task.specification.examples: - assert evaluator.eval(task.solution, ex.inputs) == ex.output + obt = evaluator.eval(task.solution, ex.inputs) + assert ( + obt == ex.output + ), f"failed on {task.solution} inputs:{ex.inputs} got:{obt} target:{ex.output}" def convert_calculator( @@ -61,13 +46,13 @@ def load() -> Dataset[PBE]: args_types = [guess_type(arg) for arg in inputs[0]] + [ guess_type(outputs[0]) ] - # guess_type doesn't recognise FLOAT but since it is the only type not recognised we know that Unknown TYpe is acutally FLOAT + # guess_type doesn't recognise FLOAT but since it is the only type not recognised we know that Unknown Type is acutally FLOAT args_types = [ at if not isinstance(at, UnknownType) else FLOAT for at in args_types ] type_request = FunctionType(*args_types) - prog = dsl.parse_program(name, type_request) + prog: Program = dsl.parse_program(name, type_request) examples = [ Example(inp, out) for inp, out in zip(inputs, outputs) @@ -83,47 +68,6 @@ def load() -> Dataset[PBE]: __convert__(load, output_file) -""" -Parser of program stored in the json file, for a given task. -s: program attribute represented as a string -returns: Tuple[Program, Type] where program is the solution of type Type to the task -""" - - -def __calculator_str2prog(s: str) -> Tuple[Program, Type]: - parts = s.split("|") - stack: TList[Program] = [] - var: int = 0 - type_stack: TList[Type] = [] - for part in parts: - subparts = part.split(",") - name = subparts.pop(0) - # possible inputs, int or float in our case - if name == "INT": - stack.append(Variable(var, INT)) - var += 1 - type_stack.append(INT) - continue - if name == "FLOAT": - stack.append(Variable(var, FLOAT)) - var += 1 - type_stack.append(FLOAT) - continue - # primitives that serve as constants - if name in ["1", "2", "3"]: - primitive = Primitive(name, name2fulltype[name]) - stack.append(primitive) - else: # other primitives are functions, we want to add their type - targets = [int(x) for x in subparts] - arguments = [stack[x] for x in targets] - longname = name + str(arguments[-1].type) - primitive = Primitive(name, name2fulltype[longname]) - stack.append(Function(primitive, arguments)) - type_stack.append(stack[-1].type) - type_request = FunctionType(*type_stack) - return stack[-1], type_request - - if __name__ == "__main__": import argparse diff --git a/examples/pbe/convert_to_psl.py b/examples/pbe/convert_to_psl.py new file mode 100644 index 00000000..630ac3c3 --- /dev/null +++ b/examples/pbe/convert_to_psl.py @@ -0,0 +1,90 @@ +import argparse +import os.path as path +from typing import Union + +from dsl_loader import add_dsl_choice_arg, load_DSL +from dataset_loader import add_dataset_choice_arg, load_dataset + +from synth import Dataset, PBE +from synth.specification import PBEWithConstants + + +parser = argparse.ArgumentParser( + description="Convert a ProgSynth dataset to the PSL format" +) +add_dsl_choice_arg(parser) +add_dataset_choice_arg(parser) +parser.add_argument( + "dest_folder", + type=str, + help="destination folder", +) +parser.add_argument( + "logics", + type=str, + help="logics used", +) + +parameters = parser.parse_args() +dsl_name: str = parameters.dsl +dataset_file: str = parameters.dataset +dest_folder: str = parameters.dest_folder +logics: str = parameters.logics +# ================================ +# Load constants specific to DSL +# ================================ +dsl_module = load_DSL(dsl_name) +dsl = dsl_module.dsl +# ================================ +# Load dataset & Task Generator +# ================================ +# Load dataset +full_dataset: Dataset[Union[PBE, PBEWithConstants]] = load_dataset( + dsl_name, dataset_file +) +COMMENT_PREFIX = "#" + + +for i, task in enumerate(full_dataset.tasks): + spec = task.specification + name = task.metadata.get("name", f"{dsl_name}_{i}") + filepath = path.join(dest_folder, name + ".psl") + try: + fd = open(filepath, "w") + fd.close() + except: + filepath = path.join(dest_folder, f"{dsl_name}_{i}" + ".psl") + + with open(filepath, "w") as fd: + fd.write(f"(set-logic {logics})\n") + + fd.write(f"\n{COMMENT_PREFIX} Function Synthesis\n") + fd.write("(synth-fun f ") + for j, arg in enumerate(task.type_request.arguments()): + fd.write(f"(x{j+1} {arg}) ") + fd.write(f"{task.type_request.returns()})\n") + + fd.write(f"\n{COMMENT_PREFIX} PBE Examples\n") + for example in spec.examples: + inputs = " ".join(map(str, example.inputs)) + output = str(example.output) + fd.write(f"(constraint-pbe (f {inputs}) {output})\n") + + if isinstance(spec, PBEWithConstants): + constants = spec.constants + fd.write(f"\n{COMMENT_PREFIX} Constants\n") + for type, values in spec.constants.items(): + allowed = " ".join(map(str, values)) + fd.write(f"(define-const {type} {allowed})\n") + + fd.write("\n(check-progsynth)\n") + if task.solution is not None: + fd.write(f"\n(solution-pbe {task.solution})\n") + lines = [] + for name, val in task.metadata.items(): + if name == "name": + continue + lines.append(f"{COMMENT_PREFIX} {name}: {val}") + if lines: + fd.write(f"\n{COMMENT_PREFIX} Metadata:\n") + fd.writelines(lines) diff --git a/examples/pbe/dataset_explorer.py b/examples/pbe/dataset_explorer.py index 78b8f1ac..aae4c34d 100644 --- a/examples/pbe/dataset_explorer.py +++ b/examples/pbe/dataset_explorer.py @@ -3,31 +3,29 @@ from colorama import Fore as F from synth import Dataset, PBE -from synth.syntax import CFG +from synth.syntax import CFG, Program from synth.task import Task -from synth.utils import chrono from dsl_loader import add_dsl_choice_arg, load_DSL +from dataset_loader import add_dataset_choice_arg, load_dataset import argparse parser = argparse.ArgumentParser(description="Explore a dataset") add_dsl_choice_arg(parser) -parser.add_argument( - "--dataset", - type=str, - default="{dsl_name}.pickle", - help="dataset file (default: {dsl_name}.pickle)", -) +add_dataset_choice_arg(parser) + parameters = parser.parse_args() dsl_name: str = parameters.dsl -dataset_file: str = parameters.dataset.format(dsl_name=dsl_name) +dataset_file: str = parameters.dataset + + # ================================ # Load constants specific to DSL # ================================ -def pretty_print_solution(str: str): - return str +def pretty_print_solution(program: Program): + return "\n".join(program.pretty_print()) def pretty_print_inputs(str: str): @@ -44,10 +42,7 @@ def pretty_print_inputs(str: str): # Load dataset & Task Generator # ================================ # Load dataset -print(f"Loading {F.LIGHTCYAN_EX}{dataset_file}{F.RESET}...", end="") -with chrono.clock("dataset.load") as c: - full_dataset: Dataset[PBE] = Dataset.load(dataset_file) - print(f"done in{F.LIGHTYELLOW_EX}", c.elapsed_time(), f"s{F.RESET}") +full_dataset: Dataset = load_dataset(dsl_name, dataset_file) def print_value(name: str, value: str) -> None: @@ -86,7 +81,7 @@ def cfg(*args: str) -> None: print_value("Max Depth", max_depth) max_len = max([len(str(t)) for t in all_type_requests]) programs_no = { - t: f"{CFG.depth_constraint(dsl, t, max_depth).size():,}" + t: f"{CFG.depth_constraint(dsl, t, max_depth).programs():,}" for t in all_type_requests } max_len_programs_no = max(len(s) for s in programs_no.values()) @@ -122,7 +117,10 @@ def task(*args: str) -> None: task: Task[PBE] = full_dataset[task_no] print_value(f"Name", task.metadata.get("name", "None")) print_value("Type", task.type_request) - print_value("Solution", pretty_print_solution(task.solution)) + if task.solution is not None: + print_value("Solution", "\n" + pretty_print_solution(task.solution)) + else: + print(f"{F.LIGHTRED_EX}No Solution{F.RESET}") print_value("Examples", "") for example in task.specification.examples: print_value( diff --git a/examples/pbe/dataset_generator.py b/examples/pbe/dataset_generator.py index a4de1297..233697c1 100644 --- a/examples/pbe/dataset_generator.py +++ b/examples/pbe/dataset_generator.py @@ -1,8 +1,13 @@ import argparse + +import tqdm + +from dataset_loader import add_dataset_choice_arg, load_dataset from dsl_loader import add_dsl_choice_arg, load_DSL from synth import Dataset, PBE -from synth.utils import chrono, gen_take +from synth.utils import chrono +from synth.syntax import CFG DREAMCODER = "dreamcoder" REGEXP = "regexp" @@ -14,13 +19,7 @@ description="Generate a dataset copying the original distribution of another dataset" ) add_dsl_choice_arg(parser) - -parser.add_argument( - "--dataset", - type=str, - default="{dsl_name}.pickle", - help="dataset file (default: {dsl_name}.pickle)", -) +add_dataset_choice_arg(parser) parser.add_argument( "-o", "--output", @@ -44,15 +43,30 @@ default=False, help="does not try to generate unique tasks", ) +parser.add_argument( + "--constrained", + action="store_true", + default=False, + help="tries to add constraints of the DSL to the grammar", +) +parser.add_argument( + "-v", + "--verbose", + action="store_true", + default=False, + help="verbose generation", +) parameters = parser.parse_args() dsl_name: str = parameters.dsl -dataset_file: str = parameters.dataset.format(dsl_name=dsl_name) +dataset_file: str = parameters.dataset output_file: str = parameters.output seed: int = parameters.seed max_depth: int = parameters.max_depth gen_dataset_size: int = parameters.size uniform: bool = parameters.uniform no_unique: bool = parameters.no_unique +constrained: bool = parameters.constrained +verbose: bool = parameters.verbose # ================================ # Load constants specific to DSL # ================================ @@ -65,18 +79,18 @@ else: from synth.pbe.task_generator import reproduce_int_dataset as reproduce_dataset +constraints = [] +if hasattr(dsl_module, "constraints") and constrained: + constraints = dsl_module.constraints if dsl_name == DREAMCODER: max_list_length = 10 - # ================================ # Load dataset & Task Generator # ================================ # Load dataset -print(f"Loading {dataset_file}...", end="") -with chrono.clock("dataset.load") as c: - full_dataset: Dataset[PBE] = Dataset.load(dataset_file) - print("done in", c.elapsed_time(), "s") +full_dataset: Dataset[PBE] = load_dataset(dsl_name, dataset_file) + # Reproduce dataset distribution print("Reproducing dataset...", end="", flush=True) with chrono.clock("dataset.reproduce") as c: @@ -88,7 +102,15 @@ max_list_length=max_list_length, default_max_depth=max_depth, uniform_pgrammar=uniform, + constraints=constraints, + verbose=verbose, ) + cfgs = task_generator.type2pgrammar + if constrained: + cfgs = { + t: CFG.depth_constraint(dsl, t, max_depth, min_variable_depth=0) + for t in task_generator.type2pgrammar + } print("done in", c.elapsed_time(), "s") # Add some exceptions that are ignored during task generation task_generator.skip_exceptions.add(TypeError) @@ -96,8 +118,18 @@ task_generator.verbose = True print("Generating dataset...", gen_dataset_size, end="", flush=True) with chrono.clock("dataset.generate") as c: + tasks = [] + pbar = tqdm.tqdm(total=gen_dataset_size, desc="tasks generated") + for task in task_generator.generator(): + if constrained and task.solution not in cfgs[task.type_request]: + continue + pbar.update(1) + tasks.append(task) + if len(tasks) == gen_dataset_size: + break + pbar.close() gen_dataset = Dataset( - gen_take(task_generator.generator(), gen_dataset_size, progress=True), + tasks, { "seed": seed, "max_depth": max_depth, diff --git a/examples/pbe/dataset_generator_unique.py b/examples/pbe/dataset_generator_unique.py new file mode 100644 index 00000000..34c6dd04 --- /dev/null +++ b/examples/pbe/dataset_generator_unique.py @@ -0,0 +1,343 @@ +import argparse +from collections import defaultdict +from typing import Any, Callable, Dict, Set, Tuple, List, Union + +import tqdm + +from dataset_loader import add_dataset_choice_arg, load_dataset +from dsl_loader import add_dsl_choice_arg, load_DSL + +from synth import Dataset, PBE +from synth.pbe.task_generator import TaskGenerator +from synth.semantic import DSLEvaluator +from synth.utils import chrono +from synth.syntax import CFG, Type, Program + +DREAMCODER = "dreamcoder" +REGEXP = "regexp" +CALCULATOR = "calculator" +TRANSDUCTION = "transduction" + + +parser = argparse.ArgumentParser( + description="Generate a dataset copying the original distribution of another dataset" +) +add_dsl_choice_arg(parser) +add_dataset_choice_arg(parser) +parser.add_argument( + "-o", + "--output", + type=str, + default="dataset.pickle", + help="output file (default: dataset.pickle)", +) +parser.add_argument("-s", "--seed", type=int, default=0, help="seed (default: 0)") +parser.add_argument( + "--programs", type=int, default=50, help="generated programs (default: 100)" +) +parser.add_argument( + "--inputs", type=int, default=1, help="generated inputs (default: 1)" +) +parser.add_argument( + "--max-depth", type=int, default=5, help="solutions max depth (default: 5)" +) +parser.add_argument( + "--max-examples", + type=int, + default=5, + help="max number of examples per task (default: 5)", +) +parser.add_argument( + "--test-examples", + type=int, + default=0, + help="number of test examples per task (default: 0)", +) +parser.add_argument( + "--uniform", action="store_true", default=False, help="use uniform PCFGs" +) +parser.add_argument( + "--constrained", + action="store_true", + default=False, + help="tries to add constraints of the DSL to the grammar", +) +parser.add_argument( + "-v", + "--verbose", + action="store_true", + default=False, + help="verbose generation", +) +parameters = parser.parse_args() +dsl_name: str = parameters.dsl +dataset_file: str = parameters.dataset +output_file: str = parameters.output +seed: int = parameters.seed +max_depth: int = parameters.max_depth +max_examples: int = parameters.max_examples +test_examples: int = parameters.test_examples +nb_programs: int = parameters.programs +nb_inputs: int = parameters.inputs +uniform: bool = parameters.uniform +constrained: bool = parameters.constrained +verbose: bool = parameters.verbose + +if constrained and not uniform: + raise NotImplementedError( + "Constrained grammars when uniform=False is currently not implemented!" + ) + +# ================================ +# Load constants specific to DSL +# ================================ +max_list_length = None +dsl_module = load_DSL(dsl_name) +dsl, evaluator, lexicon = dsl_module.dsl, dsl_module.evaluator, dsl_module.lexicon + +if hasattr(dsl_module, "reproduce_dataset"): + reproduce_dataset = dsl_module.reproduce_dataset +else: + from synth.pbe.task_generator import reproduce_int_dataset as reproduce_dataset + +constraints = [] +if hasattr(dsl_module, "constraints") and constrained: + constraints = dsl_module.constraints + +if dsl_name == DREAMCODER: + max_list_length = 10 + +# ================================ +# Load dataset & Task Generator +# ================================ +# Load dataset +full_dataset: Dataset[PBE] = load_dataset(dsl_name, dataset_file) + +# Reproduce dataset distribution +print("Reproducing dataset...", end="", flush=True) +with chrono.clock("dataset.reproduce") as c: + task_generator, lexicon = reproduce_dataset( + full_dataset, + dsl, + evaluator, + seed, + max_list_length=max_list_length, + default_max_depth=max_depth, + uniform_pgrammar=uniform, + constraints=constraints, + verbose=verbose, + ) + cfgs = task_generator.type2pgrammar + if constrained: + cfgs = { + t: CFG.depth_constraint(dsl, t, max_depth, min_variable_depth=0) + for t in task_generator.type2pgrammar + } + print("done in", c.elapsed_time(), "s") +# Add some exceptions that are ignored during task generation +task_generator.skip_exceptions.add(TypeError) +task_generator.uniques = True +task_generator.verbose = True + + +def generate_programs_and_samples_for( + tr: Type, + nb_programs: int, + nb_inputs: int, + task_generator: TaskGenerator, + threshold: int = 1000, +): + # 3 Phases algorithm to generate m programs + # Phase 1 generate n programs + # Phase 2 generate k examples to differentiate as much as possible programs in n + # Phase 3 try to generate new programs to get n as close as possible to m + # Phase 1 + programs = set() + for _ in tqdm.trange(nb_programs * 2, desc="1: generation"): + prog, unique = task_generator.generate_program(tr) + while not unique: + prog, unique = task_generator.generate_program(tr) + if isinstance(task_generator.evaluator, DSLEvaluator): + prog = task_generator.evaluator.compress(prog) + programs.add(prog) + # Phase 2 + samples, equiv = generate_samples_for( + programs, + lambda: task_generator.sample_input(tr.arguments()), + task_generator.eval_input, + task_generator.evaluator.clear_cache, + examples=max_examples, + threshold=threshold, + ) + # Phase 3 + pbar = tqdm.tqdm(total=nb_programs - len(equiv), desc="3: improvement") + tries = 0 + while len(equiv) < nb_programs: + prog, unique = task_generator.generate_program(tr) + while not unique: + prog, unique = task_generator.generate_program(tr) + if isinstance(task_generator.evaluator, DSLEvaluator): + prog = task_generator.evaluator.compress(prog) + # Compute semantic hash + cl = None + has_none = False + for x in samples: + o = task_generator.eval_input(prog, x) + if o is None: + has_none = True + break + if isinstance(o, List): + o = tuple(o) + cl = (o, cl) + # Check + if cl not in equiv and not has_none: + equiv[cl].append(prog) + tries = 0 + pbar.update(1) + else: + tries += 1 + if tries > 1000: + break + pbar.close() + rel_programs = [ + min([(p.size(), p) for p in v], key=lambda x: x[0])[1] for v in equiv.values() + ] + out = [samples] + if nb_inputs > 1: + for _ in tqdm.trange(nb_inputs - 1, desc="4: additional examples"): + samples, equiv = generate_samples_for( + rel_programs, + lambda: task_generator.sample_input(tr.arguments()), + task_generator.eval_input, + task_generator.evaluator.clear_cache, + examples=max_examples, + threshold=-threshold, + ) + if len(samples) == 0: + samples, equiv = generate_samples_for( + rel_programs, + lambda: task_generator.sample_input(tr.arguments()), + task_generator.eval_input, + task_generator.evaluator.clear_cache, + examples=max_examples, + threshold=-threshold, + ) + if len(samples) > 0: + out.append(samples) + else: + out.append(samples) + + if test_examples > 0: + out = [ + x + + [ + task_generator.sample_input(tr.arguments()) + for _ in range(test_examples) + ] + for x in out + ] + del task_generator.type2pgrammar[tr] + return out, rel_programs + + +# +def generate_samples_for( + programs: Union[List[Program], Set[Program]], + input_sampler: Callable[[], Any], + eval_prog: Callable[[Program, Any], Any], + clear_cache: Callable, + threshold: int = 1000, + examples: int = 5, +) -> Tuple[List, Dict[Any, List[Program]]]: + samples = [] + equiv_classes = {None: programs} + nb_examples = 0 + nb_tested = 0 + pbar = tqdm.tqdm(total=examples * abs(threshold), desc="2: sem. unicity") + best = None + best_score = 0 + while nb_examples < examples: + next_equiv_classes = defaultdict(list) + clear_cache() + thres_reached = nb_tested * nb_tested > threshold * threshold + ui = best if thres_reached and best is not None else input_sampler() + failed_ratio = 0 + for cl, prog in equiv_classes.items(): + for p in prog: + o = eval_prog(p, ui) + if not task_generator.output_validator(o): + failed_ratio += 1 + if isinstance(o, List): + o = tuple(o) + next_equiv_classes[(o, cl)].append(p) + ratio = len(programs) / len(equiv_classes) + if len(next_equiv_classes) > best_score and failed_ratio / len(programs) < 0.2: + best = ui + best_score = len(next_equiv_classes) + # Early stopping if no new examples is interesting + if thres_reached and best_score == len(equiv_classes): + break + # If timeout or good example + if thres_reached or ( + threshold > 0 + and len(next_equiv_classes) * (ratio ** (examples - nb_examples)) + >= len(programs) / 2 + ): + nb_examples += 1 + nb_tested = 0 + pbar.update(1) + equiv_classes = next_equiv_classes + samples.append(ui) + best_score = len(next_equiv_classes) + pbar.n = nb_examples * abs(threshold) + pbar.refresh() + pbar.set_postfix_str( + f"{len(equiv_classes)}->{best_score} | {best_score/len(programs):.0%}" + ) + pbar.update(1) + nb_tested += 1 + clear_cache() + pbar.close() + return samples, equiv_classes + + +# +print("Computing task type distribution...", end="", flush=True) +programs_by_tr = defaultdict(int) +with chrono.clock("dataset.generate.distribution") as c: + for i in tqdm.trange(nb_programs): + tr = task_generator.generate_type_request() + programs_by_tr[tr] += 1 + print("done in", c.elapsed_time(), "s") +tasks = [] + +print("Generating tasks by type...", flush=True) +for tr, count in programs_by_tr.items(): + print("\t", tr) + list_samples, programs = generate_programs_and_samples_for( + tr, count, nb_inputs, task_generator + ) + for samples in list_samples: + for program in programs: + tasks.append( + task_generator.make_task( + tr, + program, + samples, + [task_generator.eval_input(program, x) for x in samples], + test_examples=test_examples, + ) + ) +gen_dataset = Dataset(tasks) +print("Saving dataset...", end="", flush=True) +with chrono.clock("dataset.save") as c: + gen_dataset.save(output_file) + print("done in", c.elapsed_time(), "s") + +# ================================ +# Print some stats +# ================================ +# Generate the CFG dictionnary +all_type_requests = gen_dataset.type_requests() +print(f"{len(all_type_requests)} type requests supported.") +print(f"Lexicon: [{min(lexicon)};{max(lexicon)}]") diff --git a/examples/pbe/dataset_improve.py b/examples/pbe/dataset_improve.py index 3a97502d..ea44295f 100644 --- a/examples/pbe/dataset_improve.py +++ b/examples/pbe/dataset_improve.py @@ -2,6 +2,7 @@ import csv from dsl_loader import add_dsl_choice_arg, load_DSL +from dataset_loader import add_dataset_choice_arg, load_dataset from synth import Dataset, PBE from synth.utils import chrono @@ -11,12 +12,7 @@ description="Generate a new dataset by replacing solutions found if they are shorter than the original's" ) add_dsl_choice_arg(parser) -parser.add_argument( - "--dataset", - type=str, - default="{dsl_name}.pickle", - help="dataset file (default: {dsl_name}.pickle)", -) +add_dataset_choice_arg(parser) parser.add_argument( "-s", "--solution", @@ -26,7 +22,7 @@ parameters = parser.parse_args() dsl_name: str = parameters.dsl -dataset_file: str = parameters.dataset.format(dsl_name=dsl_name) +dataset_file: str = parameters.dataset solution_file: str = parameters.solution # ================================ # Load constants specific to DSL @@ -37,11 +33,7 @@ # Load dataset & Task Generator # ================================ # Load dataset -print(f"Loading {dataset_file}...", end="") -with chrono.clock("dataset.load") as c: - full_dataset: Dataset[PBE] = Dataset.load(dataset_file) - print("done in", c.elapsed_time(), "s") - +full_dataset: Dataset[PBE] = load_dataset(dsl_name, dataset_file) print("Loading solutions...", end="", flush=True) with chrono.clock("solutions.load") as c: @@ -63,8 +55,8 @@ task.solution = dsl.parse_program(new_sol, task.type_request) continue size = new_sol.count(" ") + 1 - if size < task.solution.length(): - saved += task.solution.length() - size + if size < task.solution.size(): + saved += task.solution.size() - size task.solution = dsl.parse_program(new_sol, task.type_request) replaced += 1 diff --git a/examples/pbe/dataset_learner.py b/examples/pbe/dataset_learner.py new file mode 100644 index 00000000..3409e733 --- /dev/null +++ b/examples/pbe/dataset_learner.py @@ -0,0 +1,48 @@ +from synth import Dataset, PBE +from synth.utils import chrono +from synth.library import learn, make_score_probabilistic, score_description + +from dsl_loader import add_dsl_choice_arg, load_DSL +from dataset_loader import add_dataset_choice_arg, load_dataset + +import argparse + +parser = argparse.ArgumentParser(description="Learn a new primitive based on a dataset") +add_dsl_choice_arg(parser) +add_dataset_choice_arg(parser) +parser.add_argument( + "--probabilistic", + action="store_true", + help="Maximise probability instead of reducing description size", +) +parameters = parser.parse_args() +dsl_name: str = parameters.dsl +proba: bool = parameters.probabilistic +dataset_file: str = parameters.dataset +# ================================ +# Load constants specific to DSL +# ================================ +dsl_module = load_DSL(dsl_name) +dsl, evaluator, lexicon = dsl_module.dsl, dsl_module.evaluator, dsl_module.lexicon +# ================================ +# Load dataset +# ================================ +# Load dataset +print(f"Loading {dataset_file}...", end="") +with chrono.clock("dataset.load") as c: + full_dataset: Dataset[PBE] = Dataset.load(dataset_file) + print("done in", c.elapsed_time(), "s") + +programs = [t.solution for t in full_dataset if t.solution is not None] +score_fn = make_score_probabilistic(programs, False) if proba else score_description +score, prog = learn(programs, score_fn, progress=True) +if proba: + print(f"Best program is {prog} which bumps the programs set's log prob to:{score}.") +else: + print(f"Best program is {prog} which reduces description size by:{score}.") +# print(f"This would reduce the size by {(size - 1) * occs}.") +# score, prog = learn( +# [t.solution for t in full_dataset if t.solution is not None], progress=True +# ) +# # print(f"Found {occs} occurences of {prog}.") +# print(f"This would reduce the size by {score}.") diff --git a/examples/pbe/dataset_loader.py b/examples/pbe/dataset_loader.py new file mode 100644 index 00000000..2cae8d44 --- /dev/null +++ b/examples/pbe/dataset_loader.py @@ -0,0 +1,36 @@ +from argparse import ArgumentParser +import pickle + +from colorama import Fore as F + +from synth import Dataset +from synth.utils import chrono + + +class DatasetUnpickler(pickle.Unpickler): + def find_class(self, module, name): + try: + return super().find_class(module, name) + except: + return super().find_class(module + "." + module, name) + + +def add_dataset_choice_arg(parser: ArgumentParser) -> None: + parser.add_argument( + "-d", + "--dataset", + type=str, + default="{dsl_name}.pickle", + help="the dataset file to load (default: {dsl_name}.pickle)", + ) + + +def load_dataset(dsl_name: str, dataset_file: str, verbose: bool = True) -> Dataset: + dataset_file = dataset_file.format(dsl_name=dsl_name) + if verbose: + print(f"Loading {F.LIGHTCYAN_EX}{dataset_file}{F.RESET}...", end="") + with chrono.clock("dataset.load") as c: + full_dataset: Dataset = Dataset.load(dataset_file, DatasetUnpickler) + if verbose: + print(f"done in {c.elapsed_time():.2}s") + return full_dataset diff --git a/examples/pbe/deepcoder/deepcoder.py b/examples/pbe/deepcoder/deepcoder.py index e313f708..f0a89bac 100644 --- a/examples/pbe/deepcoder/deepcoder.py +++ b/examples/pbe/deepcoder/deepcoder.py @@ -398,7 +398,7 @@ def aux(l): } dsl = DSL(__primitive_types, __forbidden_patterns) dsl_raw = DSL(__primitive_types) -evaluator = DSLEvaluator(__semantics) +evaluator = DSLEvaluator(dsl.instantiate_semantics(__semantics)) evaluator.skip_exceptions.add(OverflowError) lexicon = list(range(-256, 256 + 1)) diff --git a/examples/pbe/dreamcoder/convert_dreamcoder.py b/examples/pbe/dreamcoder/convert_dreamcoder.py index 8808ca18..a3924c11 100644 --- a/examples/pbe/dreamcoder/convert_dreamcoder.py +++ b/examples/pbe/dreamcoder/convert_dreamcoder.py @@ -23,7 +23,6 @@ def load() -> Dataset[PBE]: with open(file, "rb") as fd: li: List[Dict[str, Any]] = json.load(fd) for task_dict in tqdm.tqdm(li, desc="converting"): - examples = [ Example([dico["i"]], dico["o"]) for dico in task_dict["examples"] ] diff --git a/examples/pbe/dreamcoder/dreamcoder.py b/examples/pbe/dreamcoder/dreamcoder.py index 65c6e813..96622ee9 100644 --- a/examples/pbe/dreamcoder/dreamcoder.py +++ b/examples/pbe/dreamcoder/dreamcoder.py @@ -346,9 +346,30 @@ def _miter(k, f, x): "5": INT, } +constraints = [ + "(+ ^+,0 ^*,0)", + "(length ^range,cdr,map,cons)", + "(* ^*,1,0 ^+,2,1,0)", + "(cdr ^cons,map,filter)", + "(empty? ^range,map)", + "(- _ ^0)", + "(is-mod ^0,1 _)", + "(mod ^0,1 _)", + "(max ^max _)", + "(min ^min _)", + "(not ^not)", + "(index ^0 _)", + "(if ^not _ _)", + "(range ^0)", + "(car ^range)", + "(le? ^- _)", + "(gt? ^- _)", +] + dsl = DSL(__primitive_types__) -evaluator = DSLEvaluator(__semantics__) +dsl.instantiate_polymorphic_types(1) +evaluator = DSLEvaluator(dsl.instantiate_semantics(__semantics__)) evaluator.skip_exceptions.add(ValueError) evaluator.skip_exceptions.add(IndexError) evaluator.skip_exceptions.add(TypeError) diff --git a/examples/pbe/dsl_equation_generator.py b/examples/pbe/dsl_equation_generator.py new file mode 100644 index 00000000..2b09aeb1 --- /dev/null +++ b/examples/pbe/dsl_equation_generator.py @@ -0,0 +1,403 @@ +from collections import defaultdict +import json +from typing import Any, Callable, Dict, Generator, List, Set, Tuple, TypeVar +import argparse + +import tqdm +import numpy as np +from colorama import Fore as F + +from dataset_loader import add_dataset_choice_arg, load_dataset +from dsl_loader import add_dsl_choice_arg, load_DSL +from equivalence_classes_to_filter import equivalence_classes_to_filters + + +from synth import Dataset, PBE +from synth.generation.sampler import Sampler +from synth.pbe import reproduce_dataset +from synth.filter import Filter +from synth.specification import PBEWithConstants +from synth.syntax import ( + CFG, + DetGrammar, + ProbDetGrammar, + bps_enumerate_prob_grammar as enumerate_prob_grammar, + Function, + Primitive, + Program, + Variable, + Type, + ProgramEnumerator, +) +from synth.utils import chrono + + +parser = argparse.ArgumentParser( + description="Generate a dataset copying the original distribution of another dataset" +) +add_dsl_choice_arg(parser) +add_dataset_choice_arg(parser) +parser.add_argument( + "--no-reproduce", + action="store_true", + default=False, + help="instead of trying to sample new inputs, scrap inputs from the dataset", +) +parser.add_argument("-s", "--seed", type=int, default=0, help="seed (default: 0)") +parser.add_argument( + "--n", type=int, default=500, help="number of examples to be sampled (default: 500)" +) +parser.add_argument( + "--max-depth", + type=int, + default=2, + help="max depth of programs to check for (default: 2)", +) + + +parameters = parser.parse_args() +dsl_name: str = parameters.dsl +dataset_file: str = parameters.dataset +input_checks: int = parameters.n +max_depth: int = parameters.max_depth +no_reproduce: bool = parameters.no_reproduce +seed: int = parameters.seed +# ================================ +# Initialisation +# ================================ +dsl_module = load_DSL(dsl_name) +dsl, evaluator = dsl_module.dsl, dsl_module.evaluator + +# Load dataset +full_dataset: Dataset[PBE] = load_dataset(dsl_name, dataset_file) + +our_eval = lambda *args: evaluator.eval(*args) + + +# ================================ +# Produce data samplers +# ================================ +if not no_reproduce: + if hasattr(dsl_module, "reproduce_dataset"): + reproduce_dataset = dsl_module.reproduce_dataset + else: + from synth.pbe.task_generator import reproduce_int_dataset as reproduce_dataset + + # Reproduce dataset distribution + print("Reproducing dataset...", end="", flush=True) + with chrono.clock("dataset.reproduce") as c: + task_generator, _ = reproduce_dataset( + full_dataset, + dsl, + evaluator, + 0, + default_max_depth=max_depth, + uniform_pgrammar=True, + ) + print("done in", c.elapsed_time(), "s") + # We only get a task generator for the input generator + input_sampler = task_generator.input_generator +else: + inputs_from_type = defaultdict(list) + for task in full_dataset: + for ex in task.specification.examples: + for inp, arg in zip(ex.inputs, task.type_request.arguments()): + inputs_from_type[arg].append(inp) + # Manage input/output constants + if isinstance(task.specification, PBEWithConstants): + for ct, values in task.specification.constants: + inputs_from_type[ct] += values + + class SamplesSampler(Sampler): + def __init__(self, samples: Dict[Type, List[Any]], seed: int) -> None: + self._gen = np.random.default_rng(seed=seed) + self.samples = samples + + def sample(self, type: Type, **kwargs: Any) -> Any: + return self._gen.choice(self.samples[type]) + + input_sampler = SamplesSampler(inputs_from_type, seed=seed) + + +# ========================================================================================= +# Equivalence classes +# ========================================================================================= + +equivalence_classes = defaultdict(dict) +commutatives = [] +constants = [] +identities = [] +n_equiv_classes = 0 + + +def new_equivalence_class(program: Program) -> None: + global n_equiv_classes + equivalence_classes[program] = n_equiv_classes + n_equiv_classes += 1 + + +def merge_equivalence_classes(program: Program, representative: Program) -> None: + equivalence_classes[program] = equivalence_classes[representative] + + +def get_equivalence_class(num: int) -> Set[Program]: + return {p for p, v in equivalence_classes.items() if v == num} + + +# ========================================================================================= +# UTILS +# ========================================================================================= + +T = TypeVar("T") + + +def produce_all_variants(possibles: List[List[T]]) -> Generator[List[T], None, None]: + # Enumerate all combinations + n = len(possibles) + maxi = [len(possibles[i]) for i in range(n)] + current = [0 for _ in range(n)] + while current[0] < maxi[0]: + yield [possibles[i][j] for i, j in enumerate(current)] + # Next combination + i = n - 1 + current[i] += 1 + while i > 0 and current[i] >= maxi[i]: + current[i] = 0 + i -= 1 + current[i] += 1 + + +# ========================================================================================= +# ========================================================================================= + +sampled_inputs = {} +all_solutions: Dict[Type, Dict[Program, List]] = defaultdict(dict) +programs_done = set() +forbidden_types = set() + + +def init_base_primitives() -> None: + """ + Init sampled data types and create signatures for all base primitives + """ + primitives: List[Primitive] = dsl.list_primitives + # Check forbidden types + all_types = set() + for primitive in primitives: + all_types |= primitive.type.decompose_type()[0] + for arg_type in all_types: + try: + input_sampler.sample(type=arg_type) + except: + forbidden_types.add(arg_type) + # Pre Sample Inputs + Pre Execute base primitives + for primitive in primitives: + arguments = primitive.type.arguments() + if len(arguments) == 0 or any(arg in forbidden_types for arg in arguments): + continue + if primitive.type not in sampled_inputs: + sampled_inputs[primitive.type] = [ + [input_sampler.sample(type=arg) for arg in arguments] + for _ in range(input_checks) + ] + inputs = sampled_inputs[primitive.type] + base_program = Function( + primitive, + [Variable(i, arg_type) for i, arg_type in enumerate(arguments)], + ) + programs_done.add(base_program) + solutions = [our_eval(base_program, inp) for inp in inputs] + all_solutions[base_program.type.returns()][base_program] = solutions + new_equivalence_class(base_program) + + +def check_program( + program: Program, inputs: List, all_sol: Dict[Program, List] +) -> Tuple[bool, bool, Set[Program], List[Any]]: + is_constant = True + is_list_constant = True + my_outputs = [] + candidates = set(all_sol.keys()) + is_identity = [len(program.used_variables()) == 1 for _ in program.type.arguments()] + for i, inp in enumerate(inputs): + out = our_eval(program, inp) + # Update candidates + candidates = {c for c in candidates if all_sol[c][i] == out} + is_identity = [x and out == inp[i] for i, x in enumerate(is_identity)] + if is_constant and len(my_outputs) > 0 and my_outputs[-1] != out: + is_constant = False + is_list_constant = ( + is_list_constant + and isinstance(out, List) + and all(x == out[0] for x in out) # list with only one same element + and ( + len(my_outputs) == 0 # no input so far + or len(out) == 0 # out is empty list + or all( + len(x) == 0 for x in my_outputs + ) # all elements previously in my_outputs are empty + or [x for x in my_outputs if len(x) > 0][0][0] + == out[0] # sole value of out == previous sole value of my_outputs + ) + ) + my_outputs.append(out) + return is_constant or is_list_constant, any(is_identity), candidates, my_outputs + + +def check_symmetries() -> None: + """ + Try to find symmetries (commutativity) + """ + iterable = tqdm.tqdm(dsl.list_primitives) + for primitive in iterable: + arguments = primitive.type.arguments() + if len(arguments) == 0 or any(arg in forbidden_types for arg in arguments): + continue + iterable.set_postfix_str(primitive.primitive) + + base_program = Function( + primitive, + [Variable(i, arg_type) for i, arg_type in enumerate(arguments)], + ) + inputs = sampled_inputs[primitive.type] + all_sol = all_solutions[base_program.type.returns()] + + # ======================== + # Symmetry+Identity part + # ======================== + # Fill arguments per type + arguments_per_type: Dict[Type, List[Variable]] = {} + for i, arg_type in enumerate(arguments): + if arg_type not in arguments_per_type: + arguments_per_type[arg_type] = [] + arguments_per_type[arg_type].append(Variable(i, arg_type)) + # Enumerate all combinations + for args in produce_all_variants([arguments_per_type[t] for t in arguments]): + current_prog = Function( + primitive, + args, + ) + if current_prog in programs_done: + continue + programs_done.add(current_prog) + is_constant, is_identity, candidates, my_outputs = check_program( + current_prog, inputs, all_sol + ) + is_symmetric = base_program in candidates + if is_identity: + identities.append(current_prog) + elif is_constant: + constants.append(current_prog) + elif is_symmetric: + commutatives.append(current_prog) + merge_equivalence_classes(current_prog, base_program) + else: + new_equivalence_class(base_program) + all_sol[current_prog] = my_outputs + + +def check_equivalent() -> None: + ftypes = tqdm.tqdm(sampled_inputs.keys()) + for ftype in ftypes: + cfg = CFG.depth_constraint(dsl, ftype, max_depth + 1) + + inputs = sampled_inputs[ftype] + all_sol = all_solutions[ftype.returns()] + + cfg_size = cfg.programs() + ftypes.set_postfix_str(f"{F.GREEN}{0 / cfg_size:.0%}{F.RESET}") + + # ======================== + # Check all programs starting with max depth + # ======================== + for done, program in enumerate(get_enumerator(cfg)): + if program in programs_done: + continue + is_constant, is_identity, candidates, my_outputs = check_program( + program, inputs, all_sol + ) + if is_identity: + identities.append(program) + elif is_constant: + constants.append(program) + elif len(candidates) > 0: + merge_equivalence_classes(program, list(candidates)[0]) + else: + new_equivalence_class(program) + all_sol[program] = my_outputs + ftypes.set_postfix_str(f"{F.GREEN}{done / cfg_size:.0%}{F.RESET}") + ftypes.close() + + +def get_equivalence_classes() -> List[Set[Program]]: + classes = [get_equivalence_class(i) for i in range(n_equiv_classes)] + classes.append(identities + constants + [Variable(0)]) + classes = [l for l in classes if len(l) > 1] + return classes + + +def update_filter( + verbose: bool = True, +) -> Tuple[Callable[[Type], Filter[Program]], Dict[str, float]]: + classes = get_equivalence_classes() + if len(classes) == 0: + return lambda x: None, {} + if verbose: + print( + f"\tcurrently found {F.YELLOW}{len(classes)}{F.RESET} equivalence classes" + ) + builder = equivalence_classes_to_filters(commutatives, classes, dsl) + if verbose: + print( + f"\tfound {F.YELLOW}{builder.stats['constraints.successes']}{F.RESET} ({F.YELLOW}{builder.stats['constraints.successes']/builder.stats['constraints.total']:.1%}{F.RESET}) constraints" + ) + return (lambda t: builder.get_filter(t, set())), builder.stats + + +def get_enumerator(cfg: DetGrammar) -> ProgramEnumerator: + pcfg = ProbDetGrammar.uniform(cfg) + enumerator = enumerate_prob_grammar(pcfg) + enumerator.filter = update_filter(False)[0](pcfg.type_request) + return enumerator + + +def reduced_explosion() -> Tuple[float, float]: + stats = update_filter(False)[1] + ratio_added = stats["constraints.successes"] / stats["constraints.total"] + ratio_size = stats["dfta.size.final"] / stats["dfta.size.initial"] + return ratio_added, ratio_size + + +init_base_primitives() +check_symmetries() +check_equivalent() + +print(f"Cache hit rate: {evaluator.cache_hit_rate:.1%}") +print() + +classes = get_equivalence_classes() +with open(f"equivalent_classes_{dsl_name}.json", "w") as fd: + my_list = [list(map(str, l)) for l in classes] + json.dump( + { + "classes": my_list, + "commutatives": list(map(str, commutatives)), + "identities": list(map(str, identities)), + "constants": list(map(str, constants)), + }, + fd, + ) + + +print(f"Data saved to {F.GREEN}equivalent_classes_{dsl_name}.json{F.RESET}.") + +print(f"Found {F.GREEN}{len(classes)}{F.RESET} equivalence classes.") +print( + f"Found {F.GREEN}{len(identities)}{F.RESET} programs that were the identify function." +) +print(f"Found {F.GREEN}{len(constants)}{F.RESET} programs that were a constant.") +print(f"Found {F.GREEN}{len(commutatives)}{F.RESET} instances of commutativity.") +print() +r1, r2 = reduced_explosion() +print(f"converted {F.GREEN}{r1:.1%}{F.RESET} of constraints found.") +print(f"reduced combinatorial explosion to {F.GREEN}{r2:.1%}{F.RESET} of original.") diff --git a/examples/pbe/dsl_loader.py b/examples/pbe/dsl_loader.py index da191e44..5f1e5c2c 100644 --- a/examples/pbe/dsl_loader.py +++ b/examples/pbe/dsl_loader.py @@ -1,12 +1,14 @@ """ Module to change to add your own DSL easily in all scripts. -Some constants may need to be chnaged directly in the script. +Some constants may need to be changed directly in the script. """ + from argparse import ArgumentParser -import importlib from types import SimpleNamespace from typing import Callable, Dict, Iterable, List, Optional, Tuple, Union +from synth.utils.import_utils import import_file_function + def __base_loader( name: str, @@ -18,18 +20,7 @@ def __base_loader( where "X, Y, ..." are elements of keys """ - def loader(fully_load: bool = True) -> Optional[SimpleNamespace]: - if not fully_load: - return importlib.util.find_spec(name) - - module = importlib.import_module(name) - out = {} - for key in keys: - get, set = key if isinstance(key, tuple) else (key, key) - out[set] = module.__getattribute__(get) - return SimpleNamespace(**out) - - return loader + return import_file_function(name, keys, ["pbe", "examples.pbe"]) # ====================================================================================== @@ -37,17 +28,28 @@ def loader(fully_load: bool = True) -> Optional[SimpleNamespace]: # when the parameter to your loading_func is: # - False, we just want to check we CAN import # - True, we want to import everything and return it in a NameSpace +# The easiest way is to use the __base_loader(path without the .py to your python file defining the DSL, list of str) +# the second argument is the list of global variables that you want to make available; +# instead of a str you can also use a tuple to rename (name of the var in your file, name under which to make it available) +# # /!\ Convention for names in your namespace: # dsl: DSL - the actual DSL # evaluator: Evaluator - the DSL's evaluator # lexicon: List - the DSL's lexicon +# /!\ all of the following are optional: +# constraints: List[str] - the list of constraints for sharpening +# reproduce_dataset: Callable - synth.pbe.task_generator.reproduce_int_dataset like function +# pretty_print_inputs: Callable[[List[Any]], str] - a function to change the default format to print the inputs to an example in a task +# pretty_print_solution: Callable[[Any], str] - a function to change the default format to print the solution to a task # ======================================================================================= __dsl_funcs: Dict[str, Callable[[bool], Optional[SimpleNamespace]]] = { "deepcoder": __base_loader("deepcoder.deepcoder"), "deepcoder.raw": __base_loader( "deepcoder.deepcoder", [("dsl_raw", "dsl"), "evaluator", "lexicon"] ), - "dreamcoder": __base_loader("dreamcoder.dreamcoder"), + "dreamcoder": __base_loader( + "dreamcoder.dreamcoder", ["dsl", "evaluator", "lexicon", "constraints"] + ), "regexp": __base_loader( "regexp.regexp", [ @@ -55,7 +57,7 @@ def loader(fully_load: bool = True) -> Optional[SimpleNamespace]: "evaluator", "lexicon", "pretty_print_inputs", - "pretty_print_inputs", + "pretty_print_solution", ("reproduce_regexp_dataset", "reproduce_dataset"), ], ), @@ -66,6 +68,7 @@ def loader(fully_load: bool = True) -> Optional[SimpleNamespace]: "evaluator", "lexicon", "constant_types", + "constraints", ("reproduce_transduction_dataset", "reproduce_dataset"), ], ), @@ -78,6 +81,16 @@ def loader(fully_load: bool = True) -> Optional[SimpleNamespace]: ("reproduce_calculator_dataset", "reproduce_dataset"), ], ), + "karel": __base_loader( + "karel.karel", + [ + "dsl", + "evaluator", + "lexicon", + "constraints", + "pretty_print_inputs", + ], + ), } # ======================================================================================= # Nothing to change after this diff --git a/examples/pbe/equivalence_classes_to_filter.py b/examples/pbe/equivalence_classes_to_filter.py new file mode 100644 index 00000000..c1e84b51 --- /dev/null +++ b/examples/pbe/equivalence_classes_to_filter.py @@ -0,0 +1,385 @@ +from typing import List, Dict, Optional, Set, Tuple +import itertools +import copy + +import tqdm +from colorama import Fore as F + +from synth.syntax import ( + Program, + Constant, + Primitive, + Variable, + Function, + DSL, + Type, + DFTA, + UnknownType, +) +from synth.filter import DFTAFilter, LocalStatelessFilter, Filter +from synth.syntax.grammars.grammar import DerivableProgram + +uk = UnknownType() + + +class FiltersBuilder: + def __init__( + self, dfta: DFTA[Tuple[Type, DerivableProgram], DerivableProgram] + ) -> None: + self.dfta = dfta + self.equal_parameters_reject: Set[ + Tuple[DerivableProgram, Tuple[Tuple[int, int], ...]] + ] = set() + self.stats = { + "dfta.size.initial": self.dfta.size(), + "dfta.size.final": self.dfta.size(), + "constraints.total": 0, + "constraints.successes": 0, + } + + def add_commutativity_constraint(self, program: Program) -> bool: + assert program.depth() == 2, f"{program}: depth={program.depth()}" + assert isinstance(program, Function), f"{program}: type={type(program)}" + self.stats["constraints.total"] += 1 + swapped_indices = [] + for i, arg in enumerate(program.arguments): + assert isinstance(arg, Variable) + if i != arg.variable: + swapped_indices.append(i) + if len(swapped_indices) > 2: + return False + x, y = min(swapped_indices), max(swapped_indices) + fun = program.function + relevant = [] + for state in self.dfta.rules: + if state[0] == fun: + x_arg = state[1][x] + y_arg = state[1][y] + if hash(x_arg[1]) < hash(y_arg[1]): + relevant.append(state) + for state in relevant: + del self.dfta.rules[state] + self.stats["constraints.successes"] += 1 + return True + + def __simple_constraint(self, program: Program) -> bool: + vars = 0 + for p in program.depth_first_iter(): + if isinstance(p, Variable): + vars += 1 + if vars > 1: + return False + fun = program.function + relevant = [] + for state in self.dfta.rules: + if state[0] == fun: + success = True + for arg, sarg in zip(program.arguments, state[1]): + if isinstance(arg, Function) and arg.function != sarg[1]: + success = False + break + if success: + relevant.append(state) + for state in relevant: + del self.dfta.rules[state] + return True + + def __program_to_stateless_constraint(self, program: Program) -> bool: + if program.depth() == 2 and isinstance(program, Function): + diff = [] + for i, arg in enumerate(program.arguments): + if not isinstance(arg, Variable): + return False + if i != arg.variable: + diff.append((min(i, arg.variable), max(i, arg.variable))) + self.equal_parameters_reject.add((program.function, tuple(diff))) + return True + return False + + def forbid_program(self, program: Program) -> bool: + self.stats["constraints.total"] += 1 + if program.depth() > 3 or not isinstance(program, Function): + return False + out = self.__simple_constraint( + program + ) or self.__program_to_stateless_constraint(program) + if out: + self.stats["constraints.successes"] += 1 + return out + + def add_equivalence_class(self, programs: List[Program]) -> int: + # Find representative + representative = programs[0] + for p in programs: + if p.size() < representative.size() or ( + p.size() == representative.size() and p.depth() < representative.depth() + ): + representative = p + # Remove representative + eq_class = [p for p in programs if p != representative] + added = 0 + for p in eq_class: + added += self.forbid_program(p) + return added + + def compress(self): + self.dfta.__remove_unreachable__() + self.stats["dfta.size.final"] = self.dfta.size() + + def get_filter( + self, + type_request: Type, + constant_types: Set[Type], + ) -> Filter[Program]: + # DFTA part + r = copy.deepcopy(self.dfta.rules) + dst = r[(Variable(0, uk), tuple())] + for i, arg_type in enumerate(type_request.arguments()): + r[(Variable(i, arg_type), tuple())] = dst + for cst_type in constant_types: + r[(Constant(cst_type), tuple())] = dst + x = DFTAFilter(DFTA(r, set())) + + if len(self.equal_parameters_reject) == 0: + return x + + # Local Stateless Part + def make_equal_filter(to_look): + return lambda *args: all(args[i] == args[j] for i, j in to_look) + + def make_or(f, g): + return lambda *args: g(*args) or f(*args) + + should_reject = {} + for p, to_look in self.equal_parameters_reject: + f = make_equal_filter(to_look) + key = p.primitive + if key in should_reject: + old = should_reject[key] + should_reject[key] = make_or(old, f) + else: + should_reject[key] = f + filter = LocalStatelessFilter(should_reject) + return filter.intersection(x) + + def to_code(self, commented: bool = False) -> str: + states = list(set(self.dfta.rules.values())) + state2index = {s: i for i, s in enumerate(states)} + letters = list(self.dfta.alphabet) + prim2index = {p: i for i, p in enumerate(letters)} + types_list = list(set(p.type for p in letters).union(set(s[0] for s in states))) + type2index = {p: i for i, p in enumerate(types_list)} + + def state2code( + x: Tuple[Type, DerivableProgram], compressed: bool = False + ) -> str: + if compressed: + return f"__states[{state2index[x]}]" + return f"(__types[{type2index[x[0]]}], __primitives[{prim2index[x[1]]}])" + + out = "" + out += "from synth.syntax import Type, Primitive, Variable, Constant, Program, auto_type, UnknownType, DFTA\n" + out += "from synth.filter import DFTAFilter, Filter, LocalStatelessFilter\n" + out += "from typing import Set\n\n" + # DFTA PART + out += "__types = [" + ",".join(map(type_to_code, types_list)) + "]\n" + out += ( + "__primitives = [" + + ",".join(map(lambda l: derivable_program_to_code(l, type2index), letters)) + + "]\n" + ) + out += "__states = [" + ",".join(map(state2code, states)) + "]\n" + out += "__rules = {\n" + for state, dst in self.dfta.rules.items(): + dst_code = state2code(dst, True) + state_code = f"(__primitives[{prim2index[state[0]]}], " + if len(state[1]) > 0: + if len(state[1]) == 1: + state_code += "(" + state2code(state[1][0], True) + ",)" + else: + state_code += ( + "(" + + ",".join(map(lambda p: state2code(p, True), list(state[1]))) + + ")" + ) + else: + state_code += "tuple()" + out += f"\t{state_code}): {dst_code},\n" + if commented: + out += f"#\t{state} -> {dst}\n" + + out += "}\n\n" + # LOCAL STATELESS PART + if len(self.equal_parameters_reject) > 0: + out += "__should_reject = {\n" + # build dict + should_reject = {} + for p, to_look in self.equal_parameters_reject: + key = p.primitive + code = f"all(args[i] == args[j] for i,j in {to_look})" + if key in should_reject: + old = should_reject[key] + should_reject[key] = old + " or " + code + else: + should_reject[key] = code + # print it + for p, code in should_reject.items(): + out += f'\t"{p}": lambda *args: {code},\n' + out += "}\n\n" + # GETTER FUNCTION + out += "def get_filter(type_request: Type, constant_types: Set[Type]) -> Filter[Program]:\n" + out += "\timport copy\n" + out += "\tr = copy.deepcopy(__rules)\n" + out += "\tfor i, arg_type in enumerate(type_request.arguments()):\n" + out += f"\t\tr[(Variable(i, arg_type), tuple())] = __states[{state2index[(uk, Variable(0, uk))]}]\n" + out += "\tfor cst_type in constant_types:\n" + out += f"\t\tr[(Constant(cst_type), tuple())] = __states[{state2index[(uk, Variable(0, uk))]}]\n" + out += "\tx: Filter[Program] = DFTAFilter(DFTA(r, set()))\n" + if len(self.equal_parameters_reject) > 0: + out += "\ty = LocalStatelessFilter(__should_reject)\n" + out += "\tx = x.intersection(y)\n" + out += "\treturn x\n" + return out + + @staticmethod + def from_dsl(dsl: DSL) -> "FiltersBuilder": + primitives = dsl.list_primitives + primitive2state: Dict[Primitive, Tuple[Type, Primitive]] = {} + rules: Dict[ + Tuple[DerivableProgram, Tuple[Tuple[Type, DerivableProgram], ...]], + Tuple[Type, DerivableProgram], + ] = {} + for primitive in primitives: + primitive2state[primitive] = (primitive.type.returns(), primitive) + for primitive in primitives: + args_possibles = [] + for arg_type in primitive.type.arguments(): + args_possibles.append( + [ + primitive2state[p] + for p in primitive2state.keys() + if p.type.returns() == arg_type + ] + + [(uk, Variable(0, uk))] + ) + for arg_comb in itertools.product(*args_possibles): + rules[(primitive, tuple(arg_comb))] = primitive2state[primitive] + rules[(Variable(0, uk), tuple())] = (uk, Variable(0, uk)) + return FiltersBuilder(DFTA(rules, set())) + + +def equivalence_classes_to_filters( + commutatives: List[Program], + eq_classes: List[Set[Program]], + dsl: DSL, + progress: bool = True, +) -> FiltersBuilder: + added = 0 + pbar = tqdm.tqdm(eq_classes) if progress else eq_classes + total = 0 + builder = FiltersBuilder.from_dsl(dsl) + for program in commutatives: + added += builder.add_commutativity_constraint(program) + total += 1 + for eq_class in pbar: + total += len(eq_class) - 1 + this_class = list(eq_class) + added += builder.add_equivalence_class(this_class) + if progress: + pbar.set_postfix_str( + f"{F.CYAN}{added}{F.RESET}/{total} constraints ({F.GREEN}{added/total:.1%}{F.RESET})" + ) + builder.compress() + + return builder + + +def type_to_code(type: Type) -> str: + if isinstance(type, UnknownType): + return "UnknownType()" + return f'auto_type("{type}")' + + +def derivable_program_to_code( + program: DerivableProgram, type2index: Optional[Dict[Type, int]] = None +) -> str: + type_part = ( + type_to_code(program.type) + if type2index is None + else f"__types[{type2index[program.type]}]" + ) + if isinstance(program, Primitive): + return f'Primitive("{program.primitive}", {type_part})' + elif isinstance(program, Variable): + return f"Variable({program.variable}, {type_part})" + assert False, "not implemented" + + +if __name__ == "__main__": + import argparse + import json + import dsl_loader + + parser = argparse.ArgumentParser( + description="Transform a JSON file of equivalence classes into constraints" + ) + dsl_loader.add_dsl_choice_arg(parser) + parser.add_argument( + "data", + type=str, + help="JSON file containing the equivalence classes", + ) + parser.add_argument( + "-o", + "--output", + type=str, + default="dfta_filter_{dsl}.py", + help="Output python file containing the filter", + ) + parser.add_argument( + "-v", + "--verbose", + action="store_true", + default=False, + help="verbose mode", + ) + parser.add_argument( + "-c", + "--comment", + action="store_true", + default=False, + help="comment the output automaton", + ) + parameters = parser.parse_args() + data_file: str = parameters.data + verbose: bool = parameters.verbose + comment: bool = parameters.comment + dsl_module = dsl_loader.load_DSL(parameters.dsl) + output_file: str = parameters.output.format(dsl=parameters.dsl) + + dsl: DSL = dsl_module.dsl + + with open(data_file) as fd: + dico = json.load(fd) + classes = dico["classes"] + commutatives = list( + map(lambda p: dsl.auto_parse_program(p), dico["commutatives"]) + ) + classes = [ + list(map(lambda p: dsl.auto_parse_program(p), eq_class)) + for eq_class in classes + ] + if verbose: + print(f"found {F.CYAN}{len(classes)}{F.RESET} equivalence classes") + builder = equivalence_classes_to_filters(commutatives, classes, dsl) + if verbose: + stats = builder.stats + print( + f"found {F.CYAN}{stats['constraints.successes']}{F.RESET} ({F.GREEN}{stats['constraints.successes'] / stats['constraints.total']:.1%}{F.RESET}) constraints" + ) + print( + f"reduced size to {F.GREEN}{stats['dfta.size.final']/stats['dfta.size.initial']:.1%}{F.RESET} of original size" + ) + with open(output_file, "w") as fd: + fd.write(builder.to_code(commented=comment)) + print(f"Saved to {output_file}!") diff --git a/examples/pbe/evaluate.py b/examples/pbe/evaluate_deprecated.py similarity index 67% rename from examples/pbe/evaluate.py rename to examples/pbe/evaluate_deprecated.py index 77679287..a9b4f1ae 100644 --- a/examples/pbe/evaluate.py +++ b/examples/pbe/evaluate_deprecated.py @@ -2,60 +2,50 @@ from collections import defaultdict import os import sys -from typing import Callable, Iterable, List, Optional, Tuple +from typing import Callable, Iterable, List, Optional, Tuple, Union import csv -import pickle import tqdm import torch -from torch import Tensor -import torch.nn as nn -from torch.nn.utils.rnn import PackedSequence +from dataset_loader import add_dataset_choice_arg, load_dataset from dsl_loader import add_dsl_choice_arg, load_DSL -from examples.pbe.transduction.knowledge_graph.kg_path_finder import ( - build_wrapper, - choose_best_path, - find_paths_from_level, +from model_loader import ( + add_model_choice_arg, + instantiate_predictor, ) -from examples.pbe.transduction.knowledge_graph.preprocess_tasks import sketch + from synth import Dataset, PBE, Task -from synth.nn import ( - GrammarPredictorLayer, - Task2Tensor, - abstractions, - free_pytorch_memory, -) -from synth.pbe import IOEncoder +from synth.nn import free_pytorch_memory +from synth.filter import add_dfta_constraints from synth.semantic import DSLEvaluator from synth.semantic.evaluator import DSLEvaluatorWithConstant from synth.specification import Example, PBEWithConstants from synth.syntax import ( CFG, + UCFG, ProbDetGrammar, + ProbUGrammar, enumerate_prob_grammar, + enumerate_prob_u_grammar, enumerate_bucket_prob_grammar, + enumerate_bucket_prob_u_grammar, DSL, Program, ) -from synth.syntax.grammars.heap_search import HSEnumerator +from synth.syntax.grammars.enumeration.heap_search import HSEnumerator from synth.syntax.program import Function, Primitive, Variable from synth.syntax.type_system import STRING, Arrow -from synth.utils import chrono +from synth.utils import chrono, load_object, save_object + import argparse parser = argparse.ArgumentParser(description="Evaluate model prediction") parser.add_argument("-m", "--model", default="", type=str, help="model file") -parser.add_argument( - "-d", - "--dataset", - type=str, - default="{dsl_name}.pickle", - help="dataset (default: {dsl_name}}.pickle)", -) +add_dataset_choice_arg(parser) parser.add_argument( "-s", "--search", @@ -63,31 +53,27 @@ default="heap_search", help="enumeration algorithm (default: heap_search)", ) -add_dsl_choice_arg(parser) parser.add_argument( - "-o", "--output", type=str, default="./", help="output folder (default: './')" + "--method", + type=str, + default="base", + help="used method (default: base)", ) -gg = parser.add_argument_group("model parameters") -gg.add_argument( - "-v", - "--var-prob", - type=float, - default=0.2, - help="variable probability (default: .2)", +parser.add_argument( + "--predict", + action="store_true", + help="only do the PCFG prediction part", ) -gg.add_argument( - "-ed", - "--encoding-dimension", - type=int, - default=512, - help="encoding dimension (default: 512)", +add_dsl_choice_arg(parser) +add_model_choice_arg(parser) +parser.add_argument( + "-o", "--output", type=str, default="./", help="output folder (default: './')" ) -gg.add_argument( - "-hd", - "--hidden-size", - type=int, - default=512, - help="hidden layer size (default: 512)", +parser.add_argument( + "--support", + type=str, + default=None, + help="train dataset to get the set of supported type requests", ) g = parser.add_argument_group("pcfg prediction parameter") g.add_argument( @@ -104,15 +90,18 @@ parameters = parser.parse_args() dsl_name: str = parameters.dsl -dataset_file: str = parameters.dataset.format(dsl_name=dsl_name) +dataset_file: str = parameters.dataset search_algo: str = parameters.search +method: str = parameters.method output_folder: str = parameters.output model_file: str = parameters.model -variable_probability: float = parameters.var_prob -encoding_dimension: int = parameters.encoding_dimension -hidden_size: int = parameters.hidden_size task_timeout: float = parameters.timeout batch_size: int = parameters.batch_size +constrained: bool = parameters.constrained +predict_only: bool = parameters.predict +support: Optional[str] = ( + None if not parameters.support else parameters.support.format(dsl_name=dsl_name) +) if not os.path.exists(model_file) or not os.path.isfile(model_file): @@ -121,11 +110,21 @@ elif not os.path.exists(dataset_file) or not os.path.isfile(dataset_file): print("Dataset must be a valid dataset file!", file=sys.stderr) sys.exit(1) - +elif support is not None and ( + not os.path.exists(support) or not os.path.isfile(support) +): + print("Support dataset must be a valid dataset file!", file=sys.stderr) + sys.exit(1) if search_algo == "heap_search": - custom_enumerate = enumerate_prob_grammar + custom_enumerate = ( + enumerate_prob_grammar if not constrained else enumerate_prob_u_grammar + ) elif search_algo == "bucket_search": - custom_enumerate = lambda x: enumerate_bucket_prob_grammar(x, 3) + custom_enumerate = ( + lambda x: enumerate_bucket_prob_grammar(x, 3) + if not constrained + else enumerate_bucket_prob_u_grammar(x, 3) + ) # TODO: add parameter for bucket_search size else: print( @@ -141,24 +140,25 @@ ) dataset_name = dataset_file[start_index : dataset_file.index(".", start_index)] +supported_type_requests = Dataset.load(support).type_requests() if support else None + # ================================ # Load constants specific to dataset # ================================ -def load_dataset() -> Tuple[ - Dataset[PBE], DSL, DSLEvaluatorWithConstant, List[int], str -]: +def load_dsl_and_dataset() -> ( + Tuple[Dataset[PBE], DSL, DSLEvaluatorWithConstant, List[int], str, List[str]] +): dsl_module = load_DSL(dsl_name) dsl, evaluator, lexicon = dsl_module.dsl, dsl_module.evaluator, dsl_module.lexicon + constraints = [] + if constrained and hasattr(dsl_module, "constraints"): + constraints = dsl_module.constraints # ================================ # Load dataset # ================================ - # Load dataset - print(f"Loading {dataset_file}...", end="") - with chrono.clock("dataset.load") as c: - full_dataset = Dataset.load(dataset_file) - print("done in", c.elapsed_time(), "s") + full_dataset = load_dataset(dsl_name, dataset_file) start_index = ( 0 @@ -166,14 +166,14 @@ def load_dataset() -> Tuple[ else (len(model_file) - model_file[::-1].index(os.path.sep)) ) model_name = model_file[start_index : model_file.index(".", start_index)] - return full_dataset, dsl, evaluator, lexicon, model_name + return full_dataset, dsl, evaluator, lexicon, model_name, constraints # Produce PCFGS ========================================================== @torch.no_grad() def produce_pcfgs( - full_dataset: Dataset[PBE], dsl: DSL, lexicon: List[int] -) -> List[CFG]: + full_dataset: Dataset[PBE], dsl: DSL, lexicon: List[int], constraints: List[str] +) -> Union[List[ProbDetGrammar], List[ProbUGrammar]]: # ================================ # Load already done PCFGs # ================================ @@ -185,17 +185,22 @@ def produce_pcfgs( ) model_name = model_file[start_index : model_file.index(".", start_index)] file = os.path.join(dir, f"pcfgs_{dataset_name}_{model_name}.pickle") - pcfgs: List[ProbDetGrammar] = [] + pcfgs: Union[List[ProbDetGrammar], List[ProbUGrammar]] = [] if os.path.exists(file): - with open(file, "rb") as fd: - pcfgs = pickle.load(fd) + pcfgs = load_object(file) tasks = full_dataset.tasks + tasks = [ + t + for t in tasks + if supported_type_requests is None or t.type_request in supported_type_requests + ] done = len(pcfgs) # ================================ # Skip if possible # ================================ if done >= len(tasks): return pcfgs + # Get device device = "cuda" if torch.cuda.is_available() else "cpu" print("Using device:", device) @@ -203,54 +208,46 @@ def produce_pcfgs( # Neural Network creation # ================================ # Generate the CFG dictionnary - all_type_requests = full_dataset.type_requests() + all_type_requests = ( + full_dataset.type_requests() if support is None else supported_type_requests + ) if all(task.solution is not None for task in full_dataset): max_depth = max(task.solution.depth() for task in full_dataset) else: - max_depth = 10 # TODO: set as parameter + max_depth = 5 # TODO: set as parameter + cfgs = [ - CFG.depth_constraint(dsl, t, max_depth, min_variable_depth=0) + CFG.depth_constraint( + dsl, + t, + max_depth, + upper_bound_type_size=10, + constant_types=set(), + min_variable_depth=0, + ) for t in all_type_requests ] + cfgs = [ + UCFG.from_DFTA_with_ngrams( + add_dfta_constraints(cfg, constraints, progress=False), 2 + ) + if constrained + else cfg + for cfg in cfgs + ] - class MyPredictor(nn.Module): - def __init__(self, size: int) -> None: - super().__init__() - self.bigram_layer = GrammarPredictorLayer( - size, - cfgs, - abstractions.cfg_bigram_without_depth_and_equi_prim, - variable_probability, - ) - - encoder = IOEncoder(encoding_dimension, lexicon) - self.packer = Task2Tensor( - encoder, nn.Embedding(len(encoder.lexicon), size), size, device=device - ) - self.rnn = nn.LSTM(size, size, 1) - self.end = nn.Sequential( - nn.Linear(size, size), - nn.ReLU(), - nn.Linear(size, size), - nn.ReLU(), - ) - - def forward(self, x: List[Task[PBE]]) -> Tensor: - seq: PackedSequence = self.packer(x) - _, (y, _) = self.rnn(seq) - y: Tensor = y.squeeze(0) - return self.bigram_layer(self.end(y)) - - predictor = MyPredictor(hidden_size) - predictor.load_state_dict(torch.load(model_file)) + predictor = instantiate_predictor(parameters, cfgs, lexicon) + predictor.load_state_dict(torch.load(model_file, map_location=device)) predictor = predictor.to(device) predictor.eval() + # ================================ # Predict PCFG # ================================ def save_pcfgs() -> None: - with open(file, "wb") as fd: - pickle.dump(pcfgs, fd) + print("Saving PCFGs...", end="") + save_object(file, pcfgs) + print("done!") atexit.register(save_pcfgs) @@ -263,15 +260,17 @@ def save_pcfgs() -> None: batch_outputs = predictor(batch) for task, tensor in zip(batch, batch_outputs): + obj = predictor.bigram_layer.tensor2log_prob_grammar( + tensor, task.type_request + ) pcfgs.append( - predictor.bigram_layer.tensor2log_prob_grammar( - tensor, task.type_request - ).to_prob_det_grammar() + obj.to_prob_u_grammar() if constrained else obj.to_prob_det_grammar() ) pbar.close() - with open(file, "wb") as fd: - pickle.dump(pcfgs, fd) + save_pcfgs() atexit.unregister(save_pcfgs) + if predict_only: + return pcfgs del predictor free_pytorch_memory() return pcfgs @@ -296,21 +295,25 @@ def save(trace: Iterable) -> None: def enumerative_search( dataset: Dataset[PBE], evaluator: DSLEvaluatorWithConstant, - pcfgs: List[ProbDetGrammar], + pcfgs: Union[List[ProbDetGrammar], List[ProbUGrammar]], trace: List[Tuple[bool, float]], method: Callable[ - [DSLEvaluatorWithConstant, Task[PBE], ProbDetGrammar], + [DSLEvaluatorWithConstant, Task[PBE], Union[ProbDetGrammar, ProbUGrammar]], Tuple[bool, float, int, Optional[Program]], ], - custom_enumerate: Callable[[ProbDetGrammar], HSEnumerator], + custom_enumerate: Callable[[Union[ProbDetGrammar, ProbUGrammar]], HSEnumerator], ) -> None: - start = len(trace) pbar = tqdm.tqdm(total=len(pcfgs) - start, desc="Tasks", smoothing=0) i = 0 solved = 0 total = 0 - for task, pcfg in zip(dataset.tasks[start:], pcfgs[start:]): + tasks = [ + t + for t in dataset.tasks + if supported_type_requests is None or t.type_request in supported_type_requests + ] + for task, pcfg in zip(tasks[start:], pcfgs[start:]): total += 1 try: out = method(evaluator, task, pcfg, custom_enumerate) @@ -334,13 +337,12 @@ def enumerative_search( def base( evaluator: DSLEvaluator, task: Task[PBE], - pcfg: ProbDetGrammar, - custom_enumerate: Callable[[ProbDetGrammar], HSEnumerator], + pcfg: Union[ProbDetGrammar, ProbUGrammar], + custom_enumerate: Callable[[Union[ProbDetGrammar, ProbUGrammar]], HSEnumerator], ) -> Tuple[bool, float, int, Optional[Program]]: time = 0.0 programs = 0 with chrono.clock("search.base") as c: - for program in custom_enumerate(pcfg): time = c.elapsed_time() if time >= task_timeout: @@ -362,11 +364,85 @@ def base( return (False, time, programs, None, None) +def semantic_base( + evaluator: DSLEvaluator, + task: Task[PBE], + pcfg: Union[ProbDetGrammar, ProbUGrammar], + custom_enumerate: Callable[[Union[ProbDetGrammar, ProbUGrammar]], HSEnumerator], +) -> Tuple[bool, float, int, Optional[Program], Optional[float]]: + time = 0.0 + programs = 0 + with chrono.clock("search.semantic_base") as c: + enumerator = custom_enumerate(pcfg) + for program in enumerator: + time = c.elapsed_time() + if time >= task_timeout: + return (False, time, programs, None, None) + programs += 1 + failed = False + for ex in task.specification.examples: + out = evaluator.eval(program, ex.inputs) + failed = failed or out != ex.output + if not failed: + return ( + True, + c.elapsed_time(), + programs, + program, + pcfg.probability(program), + ) + return (False, time, programs, None, None) + + +def semantic_equivalence( + evaluator: DSLEvaluator, + task: Task[PBE], + pcfg: Union[ProbDetGrammar, ProbUGrammar], + custom_enumerate: Callable[[Union[ProbDetGrammar, ProbUGrammar]], HSEnumerator], +) -> Tuple[bool, float, int, Optional[Program], Optional[float]]: + time = 0.0 + programs = 0 + with chrono.clock("search.semantic_equivalence") as c: + results = {} + enumerator = custom_enumerate(pcfg) + merged = 0 + for program in enumerator: + time = c.elapsed_time() + if time >= task_timeout: + return (False, time, programs, None, None) + programs += 1 + failed = False + outputs = None + for ex in task.specification.examples: + out = evaluator.eval(program, ex.inputs) + failed |= out != ex.output + if isinstance(out, list): + outputs = (outputs, tuple(out)) + else: + outputs = (outputs, out) + if not failed: + return ( + True, + c.elapsed_time(), + programs, + program, + merged, + ) + else: + original = results.get(outputs) + if original is not None: + enumerator.merge_program(original, program) + merged += 1 + else: + results[outputs] = program + return (False, time, programs, None, merged) + + def constants_injector( evaluator: DSLEvaluatorWithConstant, task: Task[PBEWithConstants], - pcfg: ProbDetGrammar, - custom_enumerate: Callable[[ProbDetGrammar], HSEnumerator], + pcfg: Union[ProbDetGrammar, ProbUGrammar], + custom_enumerate: Callable[[Union[ProbDetGrammar, ProbUGrammar]], HSEnumerator], ) -> Tuple[bool, float, int, Optional[Program]]: time = 0.0 programs = 0 @@ -380,7 +456,6 @@ def constants_injector( # if program == None: # return (False, time, programs, None, None) with chrono.clock("search.constant_injector") as c: - # print("\n-----------------------") # print(name) for program in custom_enumerate(pcfg): @@ -422,9 +497,16 @@ def constants_injector( def sketched_base( evaluator: DSLEvaluator, task: Task[PBE], - pcfg: ProbDetGrammar, - custom_enumerate: Callable[[ProbDetGrammar], HSEnumerator], + pcfg: Union[ProbDetGrammar, ProbUGrammar], + custom_enumerate: Callable[[Union[ProbDetGrammar, ProbUGrammar]], HSEnumerator], ) -> Tuple[bool, float, int, Optional[Program]]: + from examples.pbe.transduction.knowledge_graph.kg_path_finder import ( + build_wrapper, + choose_best_path, + find_paths_from_level, + ) + from examples.pbe.transduction.knowledge_graph.preprocess_tasks import sketch + programs = 0 global task_timeout if task.metadata.get("constants", None) is not None: @@ -599,37 +681,46 @@ def sketched_base( # Main ==================================================================== if __name__ == "__main__": - full_dataset, dsl, evaluator, lexicon, model_name = load_dataset() - method = sketched_base - name = "sketched_base" - # if isinstance(evaluator, DSLEvaluatorWithConstant): - # method = constants_injector - # name = "constants_injector" - - pcfgs = produce_pcfgs(full_dataset, dsl, lexicon) - file = os.path.join( - output_folder, f"{dataset_name}_{model_name}_{search_algo}_{name}.csv" - ) - trace = [] - if os.path.exists(file): - with open(file, "r") as fd: - reader = csv.reader(fd) - trace = [tuple(row) for row in reader] - trace.pop(0) - print( - "\tLoaded", - len(trace), - "/", - len(full_dataset), - "(", - int(len(trace) * 100 / len(full_dataset)), - "%)", - ) - try: + ( + full_dataset, + dsl, + evaluator, + lexicon, + model_name, + constraints, + ) = load_dsl_and_dataset() + + METHODS = { + "base": base, + "sem.equiv": semantic_equivalence, + "sem.base": semantic_base, + "wikicoder": sketched_base, + "constant": constants_injector, + } + method_fn = METHODS[method] + + pcfgs = produce_pcfgs(full_dataset, dsl, lexicon, constraints) + if not predict_only: + file = os.path.join( + output_folder, f"{dataset_name}_{model_name}_{search_algo}_{method}.csv" + ) + trace = [] + if os.path.exists(file): + with open(file, "r") as fd: + reader = csv.reader(fd) + trace = [tuple(row) for row in reader] + trace.pop(0) + print( + "\tLoaded", + len(trace), + "/", + len(full_dataset), + "(", + int(len(trace) * 100 / len(full_dataset)), + "%)", + ) enumerative_search( - full_dataset, evaluator, pcfgs, trace, method, custom_enumerate + full_dataset, evaluator, pcfgs, trace, method_fn, custom_enumerate ) - except Exception as e: - print(e) - save(trace) - print("csv file was saved as:", file) + save(trace) + print("csv file was saved as:", file) diff --git a/examples/pbe/karel/karel.py b/examples/pbe/karel/karel.py new file mode 100644 index 00000000..6784ad9a --- /dev/null +++ b/examples/pbe/karel/karel.py @@ -0,0 +1,199 @@ +from typing import List +from synth.syntax import ( + DSL, + auto_type, +) +from synth.semantic import DSLEvaluator + +import numpy as np + +import matplotlib.pyplot as plt + + +class KarelWorld: + DIRECTION_TOP = 0 + DIRECTION_LEFT = 1 + DIRECTION_BOTTOM = 2 + DIRECTION_RIGHT = 3 + + def __init__(self, width: int, height: int) -> None: + self.grid = np.zeros((width, height)) + self.markers = np.zeros_like(self.grid) + self.reset() + + def reset(self) -> None: + self.karel = (0, 0) + self.current_markers = self.markers.copy() + self.direction = self.DIRECTION_RIGHT + + def isFrontClear(self) -> bool: + x, y = self.karel + width, height = self.grid.shape + new = self.karel + if self.direction == self.DIRECTION_BOTTOM: + new = (x, y + 1) + elif self.direction == self.DIRECTION_LEFT: + new = (x - 1, y) + elif self.direction == self.DIRECTION_TOP: + new = (x, y - 1) + else: + new = (x + 1, y) + if min(new) < 0 or new[0] >= width or new[1] >= height: + return False + return self.grid[new] <= 0 + + def act(self, command: str) -> "KarelWorld": + if command == "move": + if not self.isFrontClear(): + return self + x, y = self.karel + if self.direction == self.DIRECTION_BOTTOM: + self.karel = (x, y + 1) + elif self.direction == self.DIRECTION_LEFT: + self.karel = (x - 1, y) + elif self.direction == self.DIRECTION_TOP: + self.karel = (x, y - 1) + else: + self.karel = (x + 1, y) + elif command == "turnLeft": + self.direction -= 1 + if self.direction < 0: + self.direction = 3 + elif command == "turnRight": + self.direction += 1 + if self.direction > 3: + self.direction = 0 + elif command == "putMarker": + self.current_markers[self.karel] = 1 + elif command == "pickMarker": + self.current_markers[self.karel] = 0 + else: + raise Exception(f"invalid command:{command}") + return self + + def eval(self, cond: str) -> bool: + if cond == "frontIsClear": + return self.isFrontClear() + elif cond == "leftIsClear": + self.act("turnLeft") + isClear = self.isFrontClear() + self.act("turnRight") + return isClear + elif cond == "rightIsClear": + self.act("turnRight") + isClear = self.isFrontClear() + self.act("turnLeft") + return isClear + elif cond == "markersPresent": + return self.markers[self.karel] + elif cond == "noMarkersPresent": + return not self.markers[self.karel] + raise Exception(f"invalid cond:{cond}") + + def state(self) -> tuple: + out = self.markers * 2 + self.grid + self.current_markers * 4 + out[self.karel] += 8 + return tuple(tuple(x) for x in out) + + def show(self) -> None: + plt.figure() + + # Draw Karel + x, y = self.karel + xs = [x, x + 1 / 2, x + 1] + ys = [y, y + 1, y] + if self.direction == self.DIRECTION_RIGHT: + xs = [x, x + 1, x] + ys = [y, y + 1 / 2, y + 1] + elif self.direction == self.DIRECTION_LEFT: + xs = [x + 1, x, x + 1] + ys = [y, y + 1 / 2, y + 1] + elif self.direction == self.DIRECTION_TOP: + xs = [x, x + 1 / 2, x + 1] + ys = [y + 1, y, y + 1] + plt.fill(xs, ys, "blue") + # Draw Grid + for x in range(self.grid.shape[0]): + for y in range(self.grid.shape[1]): + if self.grid[x, y] > 0: + plt.fill([x, x, x + 1, x + 1], [y + 1, y, y, y + 1], "g") + elif self.current_markers[x, y] > 0: + plt.scatter([x + 1 / 2], [y + 1 / 2], color="r", marker="D", s=12) + plt.xlim(0, self.grid.shape[0]) + plt.ylim(0, self.grid.shape[1]) + plt.xticks(list(range(self.grid.shape[0] + 1))) + plt.yticks(list(range(self.grid.shape[1] + 1))) + plt.grid() + plt.show() + + +__syntax = auto_type( + { + "then": "(world -> world) -> (world -> world) -> (world -> world)", + "move": "world -> world", + "turnRight": "world -> world", + "turnLeft": "world -> world", + "pickMarker": "world -> world", + "putMarker": "world -> world", + "frontIsClear": "world -> bool", + "leftIsClear": "world -> bool", + "rightIsClear": "world -> bool", + "markersPresent": "world -> bool", + "noMarkersPresent": "world -> bool", + "not": "'a [bool | (world -> bool)] -> 'a [bool | (world -> bool)]", + "ite": "bool -> (world -> world) -> (world -> world) -> (world -> world)", + "repeat": "int -> (world -> world) -> (world -> world)", + "while": "(world -> bool) -> (world -> world) -> (world -> world)", + } +) + + +def __while(w: KarelWorld, c, s) -> KarelWorld: + n = 0 + while c(w) and n < 10000: + w = s(w) + n += 1 + return w + + +__semantics = { + "then": lambda f: lambda g: lambda z: f(g(z)), + "move": lambda w: w.act("move"), + "turnRight": lambda w: w.act("turnRight"), + "turnLeft": lambda w: w.act("turnLeft"), + "pickMarker": lambda w: w.act("pickMarker"), + "putMarker": lambda w: w.act("putMarker"), + "frontIsClear": lambda w: w.eval("frontIsClear"), + "leftIsClear": lambda w: w.eval("leftIsClear"), + "rightIsClear": lambda w: w.eval("rightIsClear"), + "markersPresent": lambda w: w.eval("markersPresent"), + "noMarkersPresent": lambda w: w.eval("noMarkersPresent"), + "not": lambda c: not c if isinstance(c, bool) else lambda w: not c(w), + "if": lambda c: lambda ifblock: lambda elseblock: ifblock if c else elseblock, + "repeat": lambda n: lambda s: lambda w: [s(w) for _ in range(n)][-1], + "while": lambda c: lambda s: lambda w: __while(w, c, s), +} +# Add constants +for i in range(3, 10): + __syntax[str(i)] = auto_type("int") + __semantics[str(i)] = i + +__forbidden_patterns = {("not", 0): {"not", "markersPresent"}, ("then", 0): {"then"}} + +dsl = DSL(__syntax, __forbidden_patterns) +dsl.instantiate_polymorphic_types(3) +evaluator = DSLEvaluator(dsl.instantiate_semantics(__semantics)) +lexicon = [] + + +constraints = [ + "then ^then _", + "while _ ^while,repeat", + "repeat _ ^while,repeat", +] + + +def pretty_print_inputs(inputs: List[KarelWorld]) -> str: + world = inputs[0] + world.show() + return "shown" diff --git a/examples/pbe/dsl_analyser.py b/examples/pbe/karel/karel_equation_generator.py similarity index 75% rename from examples/pbe/dsl_analyser.py rename to examples/pbe/karel/karel_equation_generator.py index b480bdf4..caee7c96 100644 --- a/examples/pbe/dsl_analyser.py +++ b/examples/pbe/karel/karel_equation_generator.py @@ -1,151 +1,107 @@ from collections import defaultdict import json -from typing import Any, Dict, Generator, List, Set, Tuple, TypeVar +from typing import Any, Dict, Generator, List, Optional, Set, Tuple, TypeVar import copy import argparse import tqdm import numpy as np -from dsl_loader import add_dsl_choice_arg, load_DSL - -from synth import Dataset, PBE -from synth.generation.sampler import Sampler -from synth.pbe import reproduce_dataset -from synth.pruning import UseAllVariablesPruner -from synth.semantic.evaluator import DSLEvaluatorWithConstant -from synth.specification import PBEWithConstants +from synth.generation.sampler import ( + LexiconSampler, + RequestSampler, + Sampler, + UnionSampler, +) +from synth.filter import UseAllVariablesPruner from synth.syntax import ( CFG, ProbDetGrammar, - enumerate_prob_grammar, - PrimitiveType, + bps_enumerate_prob_grammar as enumerate_prob_grammar, Function, Primitive, Program, Variable, Arrow, Type, + auto_type, ) -from synth.pruning.type_constraints.utils import SYMBOL_ANYTHING, SYMBOL_FORBIDDEN -from synth.utils import chrono +from karel import dsl, evaluator, KarelWorld +from karel_task_generator import random_world -parser = argparse.ArgumentParser( - description="Generate a dataset copying the original distribution of another dataset" -) -add_dsl_choice_arg(parser) -parser.add_argument( - "--dataset", - type=str, - default="{dsl_name}.pickle", - help="dataset file (default: {dsl_name}.pickle)", -) -parser.add_argument( - "--no-reproduce", - action="store_true", - default=False, - help="instead of trying to sample new inputs, scrap inputs from the dataset", -) +parser = argparse.ArgumentParser(description="Generate equations for Karel") parser.add_argument("-s", "--seed", type=int, default=0, help="seed (default: 0)") parser.add_argument( - "--n", type=int, default=500, help="number of examples to be sampled (default: 500)" + "--n", type=int, default=50, help="number of examples to be sampled (default: 50)" ) parser.add_argument( "--max-depth", type=int, - default=2, - help="max depth of programs to check for (default: 2)", + default=5, + help="max depth of programs to check for (default: 5)", ) parameters = parser.parse_args() -dsl_name: str = parameters.dsl -dataset_file: str = parameters.dataset.format(dsl_name=dsl_name) +dsl_name: str = "karel" input_checks: int = parameters.n max_depth: int = parameters.max_depth -no_reproduce: bool = parameters.no_reproduce seed: int = parameters.seed # ================================ -# Constants -# ================================ -DREAMCODER = "dreamcoder" -REGEXP = "regexp" -CALCULATOR = "calculator" -TRANSDUCTION = "transduction" -# ================================ # Initialisation # ================================ -dsl_module = load_DSL(dsl_name) -dsl, evaluator = dsl_module.dsl, dsl_module.evaluator - -# Load dataset -print(f"Loading {dataset_file}...", end="") -with chrono.clock("dataset.load") as c: - full_dataset: Dataset[PBE] = Dataset.load(dataset_file) - print("done in", c.elapsed_time(), "s") our_eval = lambda *args: evaluator.eval(*args) -if not no_reproduce: - if hasattr(dsl_module, "reproduce_dataset"): - reproduce_dataset = dsl_module.reproduce_dataset - else: - from synth.pbe.task_generator import reproduce_int_dataset as reproduce_dataset - - # Reproduce dataset distribution - print("Reproducing dataset...", end="", flush=True) - with chrono.clock("dataset.reproduce") as c: - task_generator, _ = reproduce_dataset( - full_dataset, - dsl, - evaluator, - 0, - default_max_depth=max_depth, - uniform_pgrammar=True, - ) - print("done in", c.elapsed_time(), "s") - # We only get a task generator for the input generator - input_sampler = task_generator.input_generator -else: - inputs_from_type = defaultdict(list) - CSTE_IN = PrimitiveType("CST_STR_INPUT") - CSTE_OUT = PrimitiveType("CST_STR_OUTPUT") - for task in full_dataset: - for ex in task.specification.examples: - for inp, arg in zip(ex.inputs, task.type_request.arguments()): - inputs_from_type[arg].append(inp) - # Manage input/output constants - if isinstance(task.specification, PBEWithConstants): - for c_in in task.specification.constants_in: - inputs_from_type[CSTE_IN].append(c_in) - for c_out in task.specification.constants_out: - inputs_from_type[CSTE_OUT].append(c_out) - - class SamplesSampler(Sampler): - def __init__(self, samples: Dict[Type, List[Any]], seed: int) -> None: - self._gen = np.random.default_rng(seed=seed) - self.samples = samples - - def sample(self, type: Type, **kwargs: Any) -> Any: - return self._gen.choice(self.samples[type]) - - input_sampler = SamplesSampler(inputs_from_type, seed=seed) - - if isinstance(evaluator, DSLEvaluatorWithConstant): - cstes_mapper = {} - - def the_our_eval(prog: Program, inp: List[Any]) -> Any: - key = tuple(inp) - if key not in cstes_mapper: - cstes_mapper[key] = input_sampler.sample(CSTE_IN), input_sampler.sample( - CSTE_OUT - ) - c_in, c_out = cstes_mapper[key] - return evaluator.eval_with_constant(prog, inp, c_in, c_out) - - our_eval = the_our_eval + +class GridSampler(RequestSampler[KarelWorld]): + def __init__(self, seed: Optional[int] = None): + self.rng = np.random.default_rng(seed) + + def sample_for(self, type: Type, **kwargs: Any) -> KarelWorld: + return random_world(10, 10, self.rng) + + +basic_samplers: Dict[Type, Sampler] = {auto_type("world"): GridSampler(seed)} +for this_type in [auto_type("stmt"), auto_type("cond"), auto_type("int")]: + elements = [] + for el in dsl.list_primitives: + if el.type == this_type: + elements.append(evaluator.semantics[el.primitive]) + basic_samplers[this_type] = LexiconSampler(elements, seed=seed) + +# t = auto_type("stmt") +# stmt_lex = [] +# comp: List[Primitive] = [] +# for el in dsl.list_primitives: +# if el.type == t: +# stmt_lex.append(evaluator.semantics[el.primitive]) +# elif el.type.returns() == t: +# comp.append(el) + +# for _ in range(1): +# print("level:", _, "statements:", len(stmt_lex)) +# current_gen = [] +# for f in comp: +# candidates = [] +# for arg_t in f.type.arguments(): +# if arg_t == t: +# candidates.append(stmt_lex) +# else: +# candidates.append(basic_samplers[arg_t].lexicon) +# for poss in product(*candidates): +# fun = evaluator.semantics[f.primitive] +# args = list(poss) +# value = fun +# while args: +# value = value(args.pop(0)) +# current_gen.append(value) + +# stmt_lex += current_gen +# basic_samplers[t] = LexiconSampler(stmt_lex, seed=seed) +input_sampler = UnionSampler(basic_samplers) # ================================ # Load dataset & Task Generator @@ -304,8 +260,10 @@ def init_base_primitives() -> None: for arg_type in all_types: try: input_sampler.sample(type=arg_type) - except: + except Exception as e: forbidden_types.add(arg_type) + # raise e + print("Some types could not be sampled:", forbidden_types) # Pre Sample Inputs + Pre Execute base primitives for primitive in primitives: arguments = primitive.type.arguments() @@ -385,7 +343,8 @@ def check_equivalent() -> None: inputs = sampled_inputs[ftype] all_sol = all_solutions[ftype.returns()] - cfg_size = cfg.size() + cfg_size = cfg.programs() + print("programs:", cfg_size) ftypes.set_postfix_str(f"{0 / cfg_size:.0%}") # ======================== @@ -394,6 +353,7 @@ def check_equivalent() -> None: for done, program in enumerate(enumerate_prob_grammar(pcfg)): if program in programs_done or not simpler_pruner.accept((ftype, program)): continue + ftypes.set_postfix_str(f"{done}/{cfg_size} | {done / cfg_size:.0%}") is_constant = True my_outputs = [] candidates = set(all_sol.keys()) @@ -417,8 +377,8 @@ def check_equivalent() -> None: else: new_equivalence_class(program) all_sol[program] = my_outputs - if done & 256 == 0: - ftypes.set_postfix_str(f"{done / cfg_size:.0%}") + # if done & 256 == 0: + # ftypes.set_postfix_str(f"{done / cfg_size:.0%}") def check_constants() -> None: @@ -455,10 +415,15 @@ def exploit_symmetries() -> None: syntaxic_restrictions[(P.primitive, i)] |= sym_types[arg] +print("Primitives:") init_base_primitives() +print("Symmetries:") check_symmetries() +print("Equivalents:") check_equivalent() +print("Constants:") check_constants() +print("Exploiting symmetries:") exploit_symmetries() print(f"Cache hit rate: {evaluator.cache_hit_rate:.1%}") diff --git a/examples/pbe/karel/karel_task_generator.py b/examples/pbe/karel/karel_task_generator.py new file mode 100644 index 00000000..767cbde0 --- /dev/null +++ b/examples/pbe/karel/karel_task_generator.py @@ -0,0 +1,145 @@ +from typing import Optional +from karel import KarelWorld + +import numpy as np + + +def random_world( + width: int, height: int, rng: Optional[np.random.Generator] = None +) -> KarelWorld: + world = KarelWorld(width, height) + gen = rng or np.random.default_rng() + world.grid[gen.random((width, height)) > 0.8] = 1 + world.grid[world.karel] = 0 + world.markers = (gen.random((width, height)) > 0.7).astype(int) + world.markers[world.grid > 0] = 0 + return world + + +if __name__ == "__main__": + import argparse + import tqdm + + from synth import Dataset, PBE, Task, Example + from synth.utils import chrono + from synth.syntax import CFG, ProbDetGrammar, auto_type + + from karel import dsl, evaluator + + parser = argparse.ArgumentParser(description="Generate a karel dataset") + parser.add_argument( + "-o", + "--output", + type=str, + default="karel.pickle", + help="output file (default: karel.pickle)", + ) + parser.add_argument("-s", "--seed", type=int, default=0, help="seed (default: 0)") + parser.add_argument( + "-w", "--width", type=int, default=10, help="grid width (default: 10)" + ) + parser.add_argument( + "--height", type=int, default=10, help="grid height (default: 10)" + ) + parser.add_argument( + "-g", + "--grids", + type=int, + default=3, + help="number of grids per task (default: 3)", + ) + parser.add_argument( + "--size", type=int, default=100, help="generated dataset size (default: 100)" + ) + parser.add_argument( + "--max-operations", + type=int, + default=5, + help="solutions max operations (default: 5)", + ) + parser.add_argument( + "--uniform", action="store_true", default=False, help="use uniform PCFGs" + ) + parser.add_argument( + "-v", + "--verbose", + action="store_true", + default=False, + help="verbose generation", + ) + parameters = parser.parse_args() + output_file: str = parameters.output + seed: int = parameters.seed + grids: int = parameters.grids + width: int = parameters.width + height: int = parameters.height + max_depth: int = parameters.max_operations + gen_dataset_size: int = parameters.size + uniform: bool = parameters.uniform + verbose: bool = parameters.verbose + # ================================ + # Task Generator + # ================================ + tr = auto_type("world -> result") + print("Generating dataset...", gen_dataset_size, end="", flush=True) + with chrono.clock("dataset.generate") as c: + cfg = CFG.depth_constraint(dsl, tr, max_depth) + if uniform: + pcfg = ProbDetGrammar.uniform(cfg) + else: + print( + "This has yet to be implemented, falling back to uniform probabilistic grammar." + ) + pcfg = ProbDetGrammar.uniform(cfg) + pass # TODO + pcfg.init_sampling(seed) + rng = np.random.default_rng(seed) + tasks = [] + pbar = tqdm.tqdm(total=gen_dataset_size, desc="tasks generated") + generated = set() + for __ in range(gen_dataset_size): + worlds = [random_world(width, height, rng) for _ in range(grids)] + program = pcfg.sample_program() + i = 0 + while program in generated: + program = pcfg.sample_program() + i += 1 + assert ( + i < 10000 + ), f"Grammar is likely too shallow to only generate unique programs" + + task = Task( + tr, + PBE( + [ + Example([worlds[i]], evaluator.eval(program, [worlds[i]])) + for i in range(grids) + ] + ), + program, + ) + pbar.update(1) + tasks.append(task) + if len(tasks) == gen_dataset_size: + break + pbar.close() + gen_dataset = Dataset( + tasks, + { + "seed": seed, + "max_depth": max_depth, + "dsl": "karel", + }, + ) + print("done in", c.elapsed_time(), "s") + print("Saving dataset...", end="", flush=True) + with chrono.clock("dataset.save") as c: + gen_dataset.save(output_file) + print("done in", c.elapsed_time(), "s") + + # ================================ + # Print some stats + # ================================ + # Generate the CFG dictionnary + all_type_requests = gen_dataset.type_requests() + print(f"{len(all_type_requests)} type requests supported.") diff --git a/examples/pbe/model_embeddings_visualizer.py b/examples/pbe/model_embeddings_visualizer.py new file mode 100644 index 00000000..a97cc49d --- /dev/null +++ b/examples/pbe/model_embeddings_visualizer.py @@ -0,0 +1,94 @@ +import torch +from torch import Tensor +from torch.utils.tensorboard.writer import SummaryWriter + + +from dataset_loader import add_dataset_choice_arg, load_dataset +from dsl_loader import add_dsl_choice_arg, load_DSL +from model_loader import ( + add_model_choice_arg, + instantiate_predictor, +) + + +from synth.nn import print_model_summary +from synth.syntax import CFG, UCFG +from synth.filter import add_dfta_constraints +from synth.pbe.io_encoder import IOEncoder + + +import argparse + + +parser = argparse.ArgumentParser(description="Visualize model") +parser.add_argument("-m", "--model", default="", type=str, help="model file") +add_dataset_choice_arg(parser) +add_dsl_choice_arg(parser) +add_model_choice_arg(parser) + +parameters = parser.parse_args() +dsl_name: str = parameters.dsl +dataset_file: str = parameters.dataset +cpu_only: bool = parameters.cpu +model_file: str = parameters.model +constrained: bool = parameters.constrained +# Get device +device = "cuda" if not cpu_only and torch.cuda.is_available() else "cpu" +print("Using device:", device) +# Load DSL ================================================================ +dsl_module = load_DSL(dsl_name) +dsl, lexicon = dsl_module.dsl, dsl_module.lexicon +constraints = getattr(dsl_module, "constraints", []) +constant_types = getattr(dsl_module, "constant_types", set()) +# Load Dataset ============================================================ +full_dataset = load_dataset(dsl_name, dataset_file) +# Load CFGs =============================================================== +all_type_requests = full_dataset.type_requests() + +if all(task.solution is not None for task in full_dataset): + max_depth = max(task.solution.depth() for task in full_dataset) +else: + max_depth = 5 # TODO: set as parameter +cfgs = [ + CFG.depth_constraint( + dsl, + t, + max_depth, + upper_bound_type_size=10, + constant_types=constant_types, + min_variable_depth=0, + ) + for t in all_type_requests +] +cfgs = [ + UCFG.from_DFTA_with_ngrams( + add_dfta_constraints(cfg, constraints, progress=False), 2 + ) + if constrained + else cfg + for cfg in cfgs +] + +writer = SummaryWriter(comment=f"model_vizualizer_{model_file}") +# Load Model ============================================================== +predictor = instantiate_predictor(parameters, cfgs, lexicon) +predictor.load_state_dict(torch.load(model_file, map_location=device)) +predictor = predictor.to(device) +predictor.eval() +print_model_summary(predictor) +# Plot embeddings ========================================================= +print("Generating embeddings data:") +encoder = predictor.packer.encoder +embedder = predictor.packer.embedder +# For now this part assumes isinstance(encoder, IOEncoder) +assert isinstance(encoder, IOEncoder) +encoded = [] +for l in lexicon: + encoder.__encode_element__(l, encoded) +# Built as a Tensor +res = torch.LongTensor(encoded).to(device).reshape((-1, 1)) +output: Tensor = embedder(res).squeeze() +writer.add_embedding(output, metadata=lexicon) +# END ==================================================================== +print("Additional model data can now be viewed with TensorBoard!") +writer.close() diff --git a/examples/pbe/model_loader.py b/examples/pbe/model_loader.py new file mode 100644 index 00000000..da96e22b --- /dev/null +++ b/examples/pbe/model_loader.py @@ -0,0 +1,134 @@ +from argparse import ArgumentParser, Namespace +from typing import List, Union + +import torch +from torch import Tensor +import torch.nn as nn +from torch.nn.utils.rnn import PackedSequence + +from synth import PBE, Task +from synth.nn import ( + DetGrammarPredictorLayer, + UGrammarPredictorLayer, + abstractions, + Task2Tensor, +) +from synth.pbe import IOEncoder +from synth.syntax import UCFG, TTCFG +from synth.syntax.grammars.cfg import CFG + + +class MyPredictor(nn.Module): + def __init__( + self, + size: int, + constrained: bool, + cfgs: Union[List[TTCFG], List[UCFG]], + variable_probability: float, + encoding_dimension: int, + device: str, + lexicon, + ) -> None: + super().__init__() + layer = UGrammarPredictorLayer if constrained else DetGrammarPredictorLayer + abstraction = ( + abstractions.ucfg_bigram + if constrained + else abstractions.cfg_bigram_without_depth + ) + self.bigram_layer = layer( + size, + cfgs, + abstraction, + variable_probability, + ) + encoder = IOEncoder(encoding_dimension, lexicon) + self.packer = Task2Tensor( + encoder, nn.Embedding(len(encoder.lexicon), size), size, device=device + ) + self.rnn = nn.LSTM(size, size, 1) + self.end = nn.Sequential( + nn.Linear(size, size), + nn.ReLU(), + nn.Linear(size, size), + nn.ReLU(), + ) + + def forward(self, x: List[Task[PBE]]) -> Tensor: + seq: PackedSequence = self.packer(x) + _, (y, _) = self.rnn(seq) + y: Tensor = y.squeeze(0) + return self.bigram_layer(self.end(y)) + + +def instantiate_predictor( + parameters: Namespace, cfgs: Union[List[CFG], List[UCFG]], lexicon: List +) -> MyPredictor: + variable_probability: float = parameters.var_prob + encoding_dimension: int = parameters.encoding_dimension + hidden_size: int = parameters.hidden_size + cpu_only: bool = parameters.cpu + constrained: bool = parameters.constrained + device = "cuda" if not cpu_only and torch.cuda.is_available() else "cpu" + + return MyPredictor( + hidden_size, + constrained, + cfgs, + variable_probability, + encoding_dimension, + device, + lexicon, + ).to(device) + + +def add_model_choice_arg(parser: ArgumentParser) -> None: + gg = parser.add_argument_group("model parameters") + gg.add_argument( + "-v", + "--var-prob", + type=float, + default=0.2, + help="variable probability (default: .2)", + ) + gg.add_argument( + "-ed", + "--encoding-dimension", + type=int, + default=512, + help="encoding dimension (default: 512)", + ) + gg.add_argument( + "-hd", + "--hidden-size", + type=int, + default=512, + help="hidden layer size (default: 512)", + ) + gg.add_argument( + "--cpu", + action="store_true", + default=False, + help="do not try to run things on cuda", + ) + gg = parser.add_argument_group("grammar parameters") + gg.add_argument( + "--constrained", + action="store_true", + default=False, + help="use unambigous grammar to include constraints in the grammar if available", + ) + + gg.add_argument( + "--max-depth", + type=int, + default=5, + help="maximum depth of grammars used (-1 for infinite, default: 5)", + ) + gg.add_argument( + "--ngram", + type=int, + default=2, + choices=[1, 2], + help="ngram used by grammars (default: 2)", + ) diff --git a/examples/pbe/model_prediction.py b/examples/pbe/model_prediction.py new file mode 100644 index 00000000..1c4154e6 --- /dev/null +++ b/examples/pbe/model_prediction.py @@ -0,0 +1,236 @@ +import argparse +import atexit +import os +import sys +from typing import List, Optional, Set, Tuple, Union + +import tqdm + +import torch + +from dataset_loader import add_dataset_choice_arg, load_dataset +from dsl_loader import add_dsl_choice_arg, load_DSL +from model_loader import ( + add_model_choice_arg, + instantiate_predictor, +) + + +from synth import Dataset, PBE +from synth.filter import add_dfta_constraints +from synth.syntax import CFG, UCFG, ProbDetGrammar, ProbUGrammar, DSL, Type +from synth.utils import load_object, save_object +from synth.utils.data_storage import legacy_save_object, legacy_load_object + + +parser = argparse.ArgumentParser( + description="Predict Probabilistic grammars for a dataset" +) +add_dsl_choice_arg(parser) +add_dataset_choice_arg(parser) +parser.add_argument("-m", "--model", default="", type=str, help="model file") +add_model_choice_arg(parser) +parser.add_argument( + "--support", + type=str, + default=None, + help="train dataset to get the set of supported type requests", +) +g = parser.add_argument_group("pcfg prediction parameter") +g.add_argument( + "-b", + "--batch-size", + type=int, + default=16, + help="batch size to compute PCFGs (default: 16)", +) + + +parameters = parser.parse_args() +dsl_name: str = parameters.dsl +dataset_file: str = parameters.dataset +model_file: str = parameters.model +batch_size: int = parameters.batch_size +constrained: bool = parameters.constrained +max_depth: int = parameters.max_depth +ngram: int = parameters.ngram +support: Optional[str] = ( + None if not parameters.support else parameters.support.format(dsl_name=dsl_name) +) + + +if not os.path.exists(model_file) or not os.path.isfile(model_file): + print("Model must be a valid model file!", file=sys.stderr) + sys.exit(1) +elif not os.path.exists(dataset_file) or not os.path.isfile(dataset_file): + print("Dataset must be a valid dataset file!", file=sys.stderr) + sys.exit(1) +elif support is not None and ( + not os.path.exists(support) or not os.path.isfile(support) +): + print("Support dataset must be a valid dataset file!", file=sys.stderr) + sys.exit(1) + + +start_index = ( + 0 + if not os.path.sep in dataset_file + else (len(dataset_file) - dataset_file[::-1].index(os.path.sep)) +) +dataset_name = dataset_file[start_index : dataset_file.index(".", start_index)] + +supported_type_requests = Dataset.load(support).type_requests() if support else None + +# ================================ +# Load constants specific to dataset +# ================================ + + +def load_dsl_and_dataset() -> ( + Tuple[Dataset[PBE], DSL, List[int], str, List[str], Set[Type]] +): + dsl_module = load_DSL(dsl_name) + dsl, lexicon = dsl_module.dsl, dsl_module.lexicon + constant_types: Set[Type] = set() + constraints = [] + if constrained and hasattr(dsl_module, "constraints"): + constraints = dsl_module.constraints + if hasattr(dsl_module, "constant_types"): + constant_types = dsl_module.constant_types + # ================================ + # Load dataset + # ================================ + full_dataset = load_dataset(dsl_name, dataset_file) + + start_index = ( + 0 + if not os.path.sep in model_file + else (len(model_file) - model_file[::-1].index(os.path.sep)) + ) + model_name = model_file[start_index : model_file.index(".", start_index)] + return full_dataset, dsl, lexicon, model_name, constraints, constant_types + + +# Produce PCFGS ========================================================== +@torch.no_grad() +def produce_pcfgs( + full_dataset: Dataset[PBE], + dsl: DSL, + lexicon: List[int], + constraints: List[str], + constant_types: Set[Type], +) -> Union[List[ProbDetGrammar], List[ProbUGrammar]]: + # ================================ + # Load already done PCFGs + # ================================ + dir = os.path.realpath(os.path.dirname(model_file)) + start_index = ( + 0 + if not os.path.sep in model_file + else (len(model_file) - model_file[::-1].index(os.path.sep)) + ) + model_name = model_file[start_index : model_file.index(".", start_index)] + file = os.path.join(dir, f"pcfgs_{dataset_name}_{model_name}.pickle") + pcfgs: Union[List[ProbDetGrammar], List[ProbUGrammar]] = [] + if os.path.exists(file): + if constrained: + pcfgs = legacy_load_object(file) + else: + pcfgs = load_object(file) + tasks = full_dataset.tasks + tasks = [ + t + for t in tasks + if supported_type_requests is None or t.type_request in supported_type_requests + ] + done = len(pcfgs) + # ================================ + # Skip if possible + # ================================ + if done >= len(tasks): + return pcfgs + + # Get device + device = "cuda" if torch.cuda.is_available() else "cpu" + print("Using device:", device) + # ================================ + # Neural Network creation + # ================================ + # Generate the CFG dictionnary + all_type_requests = ( + full_dataset.type_requests() if support is None else supported_type_requests + ) + + cfgs = [ + CFG.depth_constraint( + dsl, + t, + max_depth, + constant_types=constant_types, + min_variable_depth=0, + n_gram=ngram, + ) + for t in all_type_requests + ] + cfgs = [ + UCFG.from_DFTA_with_ngrams( + add_dfta_constraints(cfg, constraints, progress=False), ngram + ) + if constrained + else cfg + for cfg in cfgs + ] + + predictor = instantiate_predictor(parameters, cfgs, lexicon) + predictor.load_state_dict(torch.load(model_file, map_location=device)) + predictor = predictor.to(device) + predictor.eval() + + # ================================ + # Predict PCFG + # ================================ + def save_pcfgs() -> None: + print("Saving PCFGs...", end="") + if constrained: + legacy_save_object(file, pcfgs) + else: + save_object(file, pcfgs, compress_level=9) + print("done!") + + atexit.register(save_pcfgs) + + pbar = tqdm.tqdm(total=len(tasks) - done, desc="PCFG prediction") + while done < len(tasks): + end = min(len(tasks), done + batch_size) + batch = tasks[done:end] + pbar.update(end - done) + done = end + batch_outputs = predictor(batch) + + for task, tensor in zip(batch, batch_outputs): + obj = predictor.bigram_layer.tensor2log_prob_grammar( + tensor, task.type_request + ) + pcfgs.append( + obj.to_prob_u_grammar() if constrained else obj.to_prob_det_grammar() + ) + pbar.close() + save_pcfgs() + atexit.unregister(save_pcfgs) + + return pcfgs + + +# Main ==================================================================== + +if __name__ == "__main__": + ( + full_dataset, + dsl, + lexicon, + model_name, + constraints, + constant_types, + ) = load_dsl_and_dataset() + + pcfgs = produce_pcfgs(full_dataset, dsl, lexicon, constraints, constant_types) diff --git a/examples/pbe/model_trainer.py b/examples/pbe/model_trainer.py index 48e24745..cae9f750 100644 --- a/examples/pbe/model_trainer.py +++ b/examples/pbe/model_trainer.py @@ -1,5 +1,4 @@ from typing import List -import atexit import sys import os import random @@ -8,28 +7,25 @@ import torch from torch import Tensor -import torch.nn as nn -from torch.nn.utils.rnn import PackedSequence from torch.utils.tensorboard import SummaryWriter import numpy as np +from dataset_loader import add_dataset_choice_arg, load_dataset from dsl_loader import add_dsl_choice_arg, load_DSL +from model_loader import ( + add_model_choice_arg, + instantiate_predictor, +) from synth import Dataset, PBE, Task -from synth.nn import ( - GrammarPredictorLayer, - abstractions, - Task2Tensor, - print_model_summary, -) -from synth.pbe import IOEncoder -from synth.syntax import CFG +from synth.nn import print_model_summary +from synth.syntax import CFG, UCFG from synth.utils import chrono +from synth.filter import add_dfta_constraints DREAMCODER = "dreamcoder" -DEEPCODER = "deepcoder" REGEXP = "regexp" CALCULATOR = "calculator" TRANSDUCTION = "transduction" @@ -38,27 +34,17 @@ import argparse parser = argparse.ArgumentParser(description="Evaluate model prediction") -parser.add_argument( - "-d", - "--dataset", - type=str, - default="{dsl_name}.pickle", - help="dataset (default: {dsl_name}}.pickle)", -) +add_dataset_choice_arg(parser) add_dsl_choice_arg(parser) +add_model_choice_arg(parser) parser.add_argument( "-o", "--output", type=str, - default="model.pt", - help="output file (default: model.pt)", -) -parser.add_argument( - "--cpu", - action="store_true", - default=False, - help="do not try to run things on cuda", + default="seed_{seed}_model.pt", + help="model file name, should respect format 'seed_X_Y' where X is the seed and Y is the name of the model (default: seed_{seed}_model.pt)", ) + parser.add_argument( "--no-clean", action="store_true", @@ -71,28 +57,6 @@ default=False, help="do not produce stats increasing speed", ) -gg = parser.add_argument_group("model parameters") -gg.add_argument( - "-v", - "--var-prob", - type=float, - default=0.2, - help="variable probability (default: .2)", -) -gg.add_argument( - "-ed", - "--encoding-dimension", - type=int, - default=512, - help="encoding dimension (default: 512)", -) -gg.add_argument( - "-hd", - "--hidden-size", - type=int, - default=512, - help="hidden layer size (default: 512)", -) g = parser.add_argument_group("training parameters") g.add_argument( "-b", @@ -132,21 +96,20 @@ parameters = parser.parse_args() dsl_name: str = parameters.dsl -dataset_file: str = parameters.dataset.format(dsl_name=dsl_name) -output_file: str = parameters.output -variable_probability: float = parameters.var_prob +dataset_file: str = parameters.dataset +seed: int = parameters.seed +output_file: str = parameters.output.format(seed=seed) batch_size: int = parameters.batch_size epochs: int = parameters.epochs lr: float = parameters.learning_rate weight_decay: float = parameters.weight_decay -seed: int = parameters.seed -encoding_dimension: int = parameters.encoding_dimension -hidden_size: int = parameters.hidden_size cpu_only: bool = parameters.cpu no_clean: bool = parameters.no_clean no_shuffle: bool = parameters.no_shuffle no_stats: bool = parameters.no_stats -should_generate_dataset: bool = False +constrained: bool = parameters.constrained +max_depth: int = parameters.max_depth +ngram: int = parameters.ngram random.seed(seed) torch.manual_seed(seed) @@ -154,7 +117,6 @@ # Load constants specific to DSL # ================================ max_list_length = None -upper_bound_type_size = 10 dsl_constant_types = set() dsl_module = load_DSL(dsl_name) dsl, evaluator, lexicon = dsl_module.dsl, dsl_module.evaluator, dsl_module.lexicon @@ -162,13 +124,15 @@ max_list_length = 10 elif dsl_name == REGEXP: max_list_length = 10 -elif dsl_name == CALCULATOR: - upper_bound_type_size = 5 -elif dsl_name == TRANSDUCTION: - upper_bound_type_size = 5 if hasattr(dsl_module, "constant_types"): dsl_constant_types = dsl_module.constant_types +constraints = [] +if hasattr(dsl_module, "constraints"): + constraints = dsl_module.constraints +else: + constrained = False + # ================================ # Load dataset & Task Generator # ================================ @@ -178,10 +142,7 @@ sys.exit(1) # Load dataset -print(f"Loading {dataset_file}...", end="") -with chrono.clock("dataset.load") as c: - full_dataset: Dataset[PBE] = Dataset.load(dataset_file) - print("done in", c.elapsed_time(), "s") +full_dataset: Dataset[PBE] = load_dataset(dsl_name, dataset_file) # ================================ @@ -197,54 +158,32 @@ # ================================ # Generate the CFG dictionnary all_type_requests = full_dataset.type_requests() -if all(task.solution is not None for task in full_dataset): - max_depth = max(task.solution.depth() for task in full_dataset) -else: - max_depth = 15 # TODO: set as parameter +print("max depth:", max_depth) cfgs = [ CFG.depth_constraint( dsl, t, max_depth, - upper_bound_type_size=upper_bound_type_size, constant_types=dsl_constant_types, + min_variable_depth=0, + n_gram=ngram, ) for t in all_type_requests ] -type2cfg = {cfg.type_request: cfg for cfg in cfgs} +type2cfg = { + cfg.type_request: UCFG.from_DFTA_with_ngrams( + add_dfta_constraints(cfg, constraints, progress=False), ngram + ) + if constrained + else cfg + for cfg in cfgs +} +cfgs = list(type2cfg.values()) print(f"{len(all_type_requests)} type requests supported.") print(f"Lexicon: [{min(lexicon)};{max(lexicon)}]") -class MyPredictor(nn.Module): - def __init__(self, size: int) -> None: - super().__init__() - self.bigram_layer = GrammarPredictorLayer( - size, - cfgs, - abstractions.cfg_bigram_without_depth_and_equi_prim, - variable_probability, - ) - encoder = IOEncoder(encoding_dimension, lexicon) - self.packer = Task2Tensor( - encoder, nn.Embedding(len(encoder.lexicon), size), size, device=device - ) - self.rnn = nn.LSTM(size, size, 1) - self.end = nn.Sequential( - nn.Linear(size, size), - nn.ReLU(), - nn.Linear(size, size), - nn.ReLU(), - ) - - def forward(self, x: List[Task[PBE]]) -> Tensor: - seq: PackedSequence = self.packer(x) - _, (y, _) = self.rnn(seq) - y: Tensor = y.squeeze(0) - return self.bigram_layer(self.end(y)) - - -predictor = MyPredictor(hidden_size).to(device) +predictor = instantiate_predictor(parameters, cfgs, lexicon) print_model_summary(predictor) optim = torch.optim.AdamW(predictor.parameters(), lr, weight_decay=weight_decay) scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optim, "min") @@ -272,20 +211,16 @@ def do_batch(iter_number: int) -> None: writer.add_scalar( "program/depth", np.mean([p.depth() for p in batch_programs]), iter_number ) - mean_length = np.mean([p.length() for p in batch_programs]) + mean_length = np.mean([p.size() for p in batch_programs]) writer.add_scalar("program/length", mean_length, iter_number) with chrono.clock("train.do_batch.inference"): batch_outputs: Tensor = predictor(batch) - with chrono.clock("train.do_batch.embed"): - batch_programs = [ - type2cfg[task.type_request].embed(task.solution) for task in batch - ] # Gradient descent with chrono.clock("train.do_batch.loss"): optim.zero_grad() with chrono.clock("train.do_batch.loss.compute"): - loss = predictor.bigram_layer.loss_cross_entropy( + loss = predictor.bigram_layer.loss_mse( batch_programs, batch_tr, batch_outputs ) with chrono.clock("train.do_batch.loss.backprop"): @@ -333,30 +268,6 @@ def train() -> None: torch.save(predictor.state_dict(), f"{output_file}_epoch{ep}.tmp") -# Save on exit -def on_exit(): - writer.add_hparams( - { - "Learning rate": lr, - "Weight Decay": weight_decay, - "Batch Size": batch_size, - "Epochs": epochs, - "Variable Probability": variable_probability, - }, - {}, - ) - writer.flush() - writer.close() - print( - chrono.summary( - time_formatter=lambda t: f"{int(t*1000)}ms" if not np.isnan(t) else "nan" - ) - ) - - -atexit.register(on_exit) - - train() torch.save(predictor.state_dict(), output_file) if not no_clean: diff --git a/examples/pbe/plot_results.py b/examples/pbe/plot_results.py deleted file mode 100644 index 4ea61dd4..00000000 --- a/examples/pbe/plot_results.py +++ /dev/null @@ -1,127 +0,0 @@ -from glob import glob -import os -import numpy as np -import matplotlib.pyplot as plt -import pltpublish as pub -import csv -import argparse - -parser = argparse.ArgumentParser(description="Plot results") -parser.add_argument( - "-d", - "--dataset", - type=str, - default="dataset.pickle", - help="dataset (default: dataset.pickle)", -) -parser.add_argument( - "--folder", - type=str, - default="./", - help="folder in which to look for CSV files (default: './')", -) -parser.add_argument( - "--no-show", - action="store_true", - default=False, - help="just save the image does not show it", -) -parser.add_argument( - "--no-sort", - action="store_true", - default=False, - help="does not sort tasks by time taken", -) -parser.add_argument( - "--no-programs", - action="store_true", - default=False, - help="does not show programs wrt tasks", -) - -parameters = parser.parse_args() -dataset_file: str = parameters.dataset -output_folder: str = parameters.folder -no_show: bool = parameters.no_show -no_sort: bool = parameters.no_sort -no_progs: bool = parameters.no_programs - -start_index = ( - 0 - if not os.path.sep in dataset_file - else (len(dataset_file) - dataset_file[::-1].index(os.path.sep)) -) -dataset_name = dataset_file[start_index : dataset_file.index(".", start_index)] - -pub.setup() -if not no_progs: - ax1 = plt.subplot(1, 2, 1) -else: - ax1 = plt.subplot(1, 1, 1) -plt.xlabel("Time (in s)") -plt.ylabel("Tasks Completed") -plt.grid() -if not no_progs: - ax2 = plt.subplot(1, 2, 2) - plt.xlabel("# Programs") - plt.ylabel("Tasks Completed") - plt.grid() -max_time, max_programs = 0, 0 -max_tasks = 0 -for file in glob(os.path.join(output_folder, "*.csv")): - file = os.path.relpath(file, output_folder) - if not file.startswith(dataset_name): - continue - name = file[len(dataset_name) : -4] - if "_" not in name: - continue - name = name[name.index("_") + 1 :].replace("_", " ") - trace = [] - with open(os.path.join(output_folder, file), "r") as fd: - reader = csv.reader(fd) - trace = [tuple(row) for row in reader] - trace.pop(0) - trace = [(row[0] == "True", float(row[1]), int(row[2])) for row in trace] - max_tasks = max(len(trace), max_tasks) - # Plot tasks wrt time - trace_time = trace if no_sort else sorted(trace, key=lambda x: x[1]) - cum_sol1, cum_time = np.cumsum([row[0] for row in trace_time]), np.cumsum( - [row[1] for row in trace_time] - ) - max_time = max(max_time, cum_time[-1]) - ax1.plot(cum_time, cum_sol1, label=name.capitalize()) - # Plot tasks wrt programs - trace_programs = trace if no_sort else sorted(trace, key=lambda x: x[2]) - cum_sol2, cum_programs = np.cumsum([row[0] for row in trace_programs]), np.cumsum( - [row[2] for row in trace_programs] - ) - max_programs = max(max_programs, cum_programs[-1]) - if not no_progs: - ax2.plot(cum_programs, cum_sol2, label=name.capitalize()) - print(name, "solved", cum_sol2[-1], "/", len(trace)) -ax1.hlines( - [max_tasks], - xmin=0, - xmax=max_time, - label="All tasks", - color="k", - linestyles="dashed", -) -ax1.set_xlim(0, max_time) -ax1.set_ylim(0, max_tasks + 10) -if not no_progs: - ax2.hlines( - [max_tasks], - xmin=0, - xmax=max_programs, - label="All tasks", - color="k", - linestyles="dashed", - ) - ax2.set_xlim(0, max_programs) - ax2.set_ylim(0, max_tasks + 10) - ax2.legend() -ax1.legend() -pub.save_fig(os.path.join(output_folder, "results.png")) -if not no_show: - plt.show() diff --git a/examples/pbe/quantum_circuits/quantum.py b/examples/pbe/quantum_circuits/quantum.py new file mode 100644 index 00000000..fa9f6637 --- /dev/null +++ b/examples/pbe/quantum_circuits/quantum.py @@ -0,0 +1,77 @@ +from typing import Any, Dict +import numpy as np + +from synth.syntax import ( + PolymorphicType, + Arrow, + INT, + BOOL, + List, + DSL, + auto_type, + Program, +) +from synth.semantic import DSLEvaluator + + +import qiskit as qk +from synth.syntax.program import Primitive + +__syntax = auto_type( + { + "H": "circuit -> int -> circuit", + "T": "circuit -> int -> circuit", + "Tdg": "circuit -> int -> circuit", + "CNOT": "circuit -> int -> int -> circuit", + } +) + + +__semantics = { + "H": lambda QT: lambda q1: QT if QT.circuit.h(QT.q(q1)) is not None else QT, + "T": lambda QT: lambda q1: QT if QT.circuit.t(QT.q(q1)) is not None else QT, + "Tdg": lambda QT: lambda q1: QT if QT.circuit.tdg(QT.q(q1)) is not None else QT, + "CNOT": lambda QT: lambda q1: lambda q2: QT + if QT.circuit.cnot(QT.q(q1), QT.q(q2)) is not None + else QT, +} + + +class QiskitTester: + def __init__(self, n_qubits: int): + self.n_qubits = n_qubits + self.unitary_matrix = None + self.qreg_q = qk.QuantumRegister(self.n_qubits, "q") + self.circuit = qk.QuantumCircuit(self.qreg_q) + + def q(self, q_num: int) -> int: + return self.n_qubits - 1 - q_num + + def __enter__(self): + return self + + def __exit__(self, *args: Any, **kwargs: Any) -> None: + pass + + def __str__(self) -> str: + return self.circuit.__str__() + + def execute(self, backend: qk.AerWrapper) -> np.ndarray: + return np.array(qk.execute(self.circuit, backend).result().get_unitary()).T + + +class QuantumCircuitEvaluator(DSLEvaluator): + def __init__(self, semantics: Dict[Primitive, Any], nqbits: int = 3) -> None: + super().__init__(semantics, False) + self.nqbits = nqbits + self.backend = qk.Aer.get_backend("unitary_simulator") + + def eval(self, program: Program, input: List) -> Any: + with QiskitTester(self.nqbits) as QT: + super().eval(program, [QT] + input) + + return QT.execute(self.backend) + + +dsl = DSL(__syntax) +evaluator = QuantumCircuitEvaluator(dsl.instantiate_semantics(__semantics)) diff --git a/examples/pbe/quantum_circuits/quantum_tasks_generator.py b/examples/pbe/quantum_circuits/quantum_tasks_generator.py new file mode 100644 index 00000000..71d32fdf --- /dev/null +++ b/examples/pbe/quantum_circuits/quantum_tasks_generator.py @@ -0,0 +1,395 @@ +from typing import ( + Any, + Dict, + Set, + Tuple, + TypeVar, +) +from itertools import product + +import numpy as np + +from synth.syntax import ( + List, + DSL, + auto_type, + Program, + CFG, + UCFG, + Type, + ProbUGrammar, + enumerate_prob_u_grammar, + DFTA, +) +from synth.semantic import DSLEvaluator +from synth.syntax.grammars.det_grammar import DerivableProgram +from synth import Task, PBE, Dataset + +import tqdm + +from quantum import QiskitTester + +import qiskit as qk +from qiskit.transpiler.passes import SolovayKitaev + + +__syntax = auto_type( + { + "H": "circuit -> int -> circuit", + "T": "circuit -> int -> circuit", + "Tdg": "circuit -> int -> circuit", + "CNOT": "circuit -> int -> int -> circuit", + "I": "circuit -> int -> circuit", + "S": "circuit -> int -> circuit", + "X": "circuit -> int -> circuit", + "Y": "circuit -> int -> circuit", + "Z": "circuit -> int -> circuit", + "SX": "circuit -> int -> circuit", + "SXdg": "circuit -> int -> circuit", + "CY": "circuit -> int -> int -> circuit", + "CZ": "circuit -> int -> int -> circuit", + "CS": "circuit -> int -> int -> circuit", + "CH": "circuit -> int -> int -> circuit", + "SWAP": "circuit -> int -> int -> circuit", + "iSWAP": "circuit -> int -> int -> circuit", + } +) + +# Trick to always return QT while running statement X: +# QT if X is not None else QT +__semantics = { + "H": lambda QT: lambda q1: QT if QT.circuit.h(QT.q(q1)) is not None else QT, + "T": lambda QT: lambda q1: QT if QT.circuit.t(QT.q(q1)) is not None else QT, + "Tdg": lambda QT: lambda q1: QT if QT.circuit.tdg(QT.q(q1)) is not None else QT, + "CNOT": lambda QT: lambda q1: lambda q2: QT + if QT.circuit.cnot(QT.q(q1), QT.q(q2)) is not None + else QT, + "I": lambda QT: lambda q1: QT if QT.circuit.id(QT.q(q1)) is not None else QT, + "S": lambda QT: lambda q1: QT if QT.circuit.s(QT.q(q1)) is not None else QT, + "X": lambda QT: lambda q1: QT if QT.circuit.x(QT.q(q1)) is not None else QT, + "Y": lambda QT: lambda q1: QT if QT.circuit.y(QT.q(q1)) is not None else QT, + "Z": lambda QT: lambda q1: QT if QT.circuit.z(QT.q(q1)) is not None else QT, + "SX": lambda QT: lambda q1: QT if QT.circuit.sx(QT.q(q1)) is not None else QT, + "SXdg": lambda QT: lambda q1: QT if QT.circuit.sxdg(QT.q(q1)) is not None else QT, + "CY": lambda QT: lambda q1: lambda q2: QT + if QT.circuit.cy(QT.q(q1), QT.q(q2)) is not None + else QT, + "CZ": lambda QT: lambda q1: lambda q2: QT + if QT.circuit.cz(QT.q(q1), QT.q(q2)) is not None + else QT, + "CS": lambda QT: lambda q1: lambda q2: QT + if QT.circuit.append(qk.circuit.library.SGate().control(1), (QT.q(q1), QT.q(q2))) + is not None + else QT, + "CH": lambda QT: lambda q1: lambda q2: QT + if QT.circuit.ch(QT.q(q1), QT.q(q2)) is not None + else QT, + "SWAP": lambda QT: lambda q1: lambda q2: QT + if QT.circuit.swap(QT.q(q1), QT.q(q2)) is not None + else QT, + "iSWAP": lambda QT: lambda q1: lambda q2: QT + if QT.circuit.iswap(QT.q(q1), QT.q(q2)) is not None + else QT, +} + +# Is this important? +with QiskitTester(1) as QT: + QT.circuit.t(0) + QT.circuit.t(0) +qk.circuit.equivalence_library.StandardEquivalenceLibrary.add_equivalence( + qk.circuit.library.SGate(), QT.circuit +) + +with QiskitTester(1) as QT: + QT.circuit.tdg(0) + QT.circuit.tdg(0) +qk.circuit.equivalence_library.StandardEquivalenceLibrary.add_equivalence( + qk.circuit.library.SdgGate(), QT.circuit +) + + +class ParametricSubstitution(qk.transpiler.TransformationPass): + def run(self, dag): + # iterate over all operations + for node in dag.op_nodes(): + # print(node.op.name, node.op.params) + # if we hit a RYY or RZZ gate replace it + + if node.op.name in ["cp"]: + replacement = qk.QuantumCircuit(2) + replacement.p(node.op.params[0] / 2, 0) + replacement.cx(0, 1) + replacement.p(-node.op.params[0] / 2, 1) + replacement.cx(0, 1) + replacement.p(node.op.params[0] / 2, 1) + + # replace the node with our new decomposition + dag.substitute_node_with_dag( + node, qk.converters.circuit_to_dag(replacement) + ) + + if node.op.name in ["p"] and node.op.params[0] == np.pi / 2: + # calculate the replacement + replacement = qk.QuantumCircuit(1) + replacement.s([0]) + + # replace the node with our new decomposition + dag.substitute_node_with_dag( + node, qk.converters.circuit_to_dag(replacement) + ) + + elif node.op.name in ["p"] and node.op.params[0] == 3 * np.pi / 2: + # calculate the replacement + replacement = qk.QuantumCircuit(1) + replacement.tdg([0]) + replacement.tdg([0]) + + # replace the node with our new decomposition + dag.substitute_node_with_dag( + node, qk.converters.circuit_to_dag(replacement) + ) + + elif node.op.name in ["p"] and node.op.params[0] == 5 * np.pi / 2: + # calculate the replacement + replacement = qk.QuantumCircuit(1) + replacement.t([0]) + replacement.t([0]) + + # replace the node with our new decomposition + dag.substitute_node_with_dag( + node, qk.converters.circuit_to_dag(replacement) + ) + + return dag + + +def decompose( + circuit: qk.QuantumCircuit, + backend: qk.AerWrapper, + pm: qk.transpiler.PassManager, + skd: SolovayKitaev, +) -> qk.QuantumCircuit: + transpiled = qk.transpile(circuit, backend) + circuit2 = pm.run(transpiled) + discretized = skd(circuit2) + return qk.transpile(discretized, backend, ["h", "cx", "t", "tdg"]) + + +def circuit_to_program(circuit: qk.QuantumCircuit, dsl: DSL, tr: Type) -> Program: + program = "var0" + for inst in circuit.data: + name: str = inst.operation.name + program = f"({name.capitalize()} {program}" + for qbit in inst.qubits: + index, registers = circuit.find_bit(qbit) + register, idx_in_reg = registers[0] + program += f" {register.size-index - 1}" + program += ")" + return dsl.parse_program(program.replace("Cx", "CNOT"), tr) + + +class PartialQuantumCircuitEvaluator(DSLEvaluator): + def __init__(self, semantics: Dict[str, Any], nqbits: int = 3) -> None: + super().__init__(semantics, False) + self.nqbits = nqbits + self.backend = qk.Aer.get_backend("unitary_simulator") + + def eval(self, program: Program, input: List) -> Any: + with QiskitTester(self.nqbits) as QT: + super().eval(program, [QT] + input) + + return QT.circuit + + +U = TypeVar("U") +V = TypeVar("V") + + +def __cfg2dfta__( + grammar: CFG, +) -> DFTA[Tuple[Type, int], DerivableProgram]: + StateT = Tuple[Type, int] + dfta_rules: Dict[Tuple[DerivableProgram, Tuple[StateT, ...]], StateT] = {} + max_depth = grammar.max_program_depth() + all_cases: Dict[ + Tuple[int, Tuple[Type, ...]], Set[Tuple[Tuple[Type, int], ...]] + ] = {} + for S in grammar.rules: + for P in grammar.rules[S]: + args = grammar.rules[S][P][0] + if len(args) == 0: + dfta_rules[(P, ())] = (P.type, 0) + else: + key = (len(args), tuple([arg[0] for arg in args])) + if key not in all_cases: + all_cases[key] = set( + [ + tuple(x) + for x in product( + *[ + [(arg[0], j) for j in range(max_depth)] + for arg in args + ] + ) + ] + ) + for nargs in all_cases[key]: + new_depth = max(i for _, i in nargs) + 1 + if new_depth >= max_depth: + continue + dfta_rules[(P, nargs)] = ( + S[0], + new_depth, + ) + r = grammar.type_request.returns() + dfta = DFTA(dfta_rules, {(r, x) for x in range(max_depth)}) + dfta.reduce() + return dfta + + +def __generate_syntax__( + nqbits: int, max_operations: int, verbose: bool +) -> Tuple[PartialQuantumCircuitEvaluator, DSL, ProbUGrammar]: + tr = auto_type("circuit -> circuit") + type_int = auto_type("int") + if verbose: + print("Augmenting DSL...", end="") + # Make copies to have no side-effects + syntax = {x: y for x, y in __syntax.items()} + semantics = {x: y for x, y in __semantics.items()} + # Add constants for qbits + for n in range(nqbits): + syntax[str(n)] = type_int + semantics[str(n)] = n + # DSL + Evaluator + dsl = DSL(syntax) + evaluator = PartialQuantumCircuitEvaluator(semantics, nqbits) + if verbose: + print("done!\nGenerating grammar...", end="") + # PCFG + cfg = CFG.depth_constraint(dsl, tr, max_operations) + + dfta = __cfg2dfta__(cfg) + if verbose: + print("done!\nFixing grammar...", end="") + depths = {y for x, y in dfta.states if x == type_int} + for depth in depths: + for n in range(nqbits): + s = (auto_type("int" + str(n)), depth) + dfta.states.add(s) + dfta.rules[(dsl.get_primitive(str(n)), ())] = s + for (P, args), dst in list(dfta.rules.items()): + if len(args) == 3: + del dfta.rules[(P, args)] + for n1 in range(nqbits): + for n2 in range(nqbits): + if n1 == n2: + continue + n_args = ( + args[0], + (auto_type("int" + str(n1)), args[1][1]), + (auto_type("int" + str(n2)), args[2][1]), + ) + dfta.rules[(P, n_args)] = dst + elif len(args) == 2: + del dfta.rules[(P, args)] + for n1 in range(nqbits): + n_args = ( + args[0], + (auto_type("int" + str(n1)), args[1][1]), + ) + dfta.rules[(P, n_args)] = dst + if verbose: + print("done!\nConverting to probabilistic grammar...", end="") + pcfg = ProbUGrammar.uniform(UCFG.from_DFTA_with_ngrams(dfta, 2)) + if verbose: + print("done!") + return evaluator, dsl, pcfg + + +def generate_tasks( + nqbits: int = 3, n_tasks: int = 1000, max_operations: int = 5, verbose: bool = False +) -> Dataset[PBE]: + evaluator, dsl, pcfg = __generate_syntax__(nqbits, max_operations, verbose) + tr = pcfg.type_request + if verbose: + print("Loading quantum backend...", end="") + + backend = qk.Aer.get_backend("unitary_simulator") + skd = SolovayKitaev() + pm = qk.transpiler.PassManager() + pm.append(ParametricSubstitution()) + if verbose: + print("done!") + + tasks = [] + pbar = None + if verbose: + pbar = tqdm.tqdm(total=n_tasks, desc="Task Generation") + for program in enumerate_prob_u_grammar(pcfg): + # print("Evaluating:", str(program)) + complex_circuit = evaluator.eval(program, []) + try: + base_circuit = decompose(complex_circuit, backend, pm, skd) + except qk.transpiler.exceptions.TranspilerError: + continue + task = Task[PBE](tr, PBE([]), circuit_to_program(base_circuit, dsl, tr)) + tasks.append(task) + if pbar: + pbar.update(1) + if len(tasks) >= n_tasks: + break + if pbar: + pbar.close() + return Dataset(tasks, {"generated:": True}) + + +if __name__ == "__main__": + import argparse + + from synth.utils import chrono + + parser = argparse.ArgumentParser(description="Generate a quantum dataset") + parser.add_argument( + "-o", + "--output", + type=str, + default="quantum.pickle", + help="output file (default: quantum.pickle)", + ) + parser.add_argument("-q", "--qbits", type=int, default=3, help="qbits (default: 3)") + parser.add_argument( + "--size", type=int, default=100, help="generated dataset size (default: 100)" + ) + parser.add_argument( + "--max-operations", + type=int, + default=5, + help="solutions max operations (default: 5)", + ) + parser.add_argument( + "-v", + "--verbose", + action="store_true", + default=False, + help="verbose generation", + ) + parameters = parser.parse_args() + output_file: str = parameters.output + qbits: int = parameters.qbits + max_depth: int = parameters.max_operations + gen_dataset_size: int = parameters.size + verbose: bool = parameters.verbose + # ================================ + # Task Generator + # ================================ + print("Generating dataset...", gen_dataset_size, end="", flush=True) + with chrono.clock("dataset.generate") as c: + gen_dataset = generate_tasks( + qbits, gen_dataset_size, max_depth, verbose=verbose + ) + print("done in", c.elapsed_time(), "s") + print("Saving dataset...", end="", flush=True) + with chrono.clock("dataset.save") as c: + gen_dataset.save(output_file) + print("done in", c.elapsed_time(), "s") diff --git a/examples/pbe/regexp/regexp.py b/examples/pbe/regexp/regexp.py index c48f5618..11876144 100644 --- a/examples/pbe/regexp/regexp.py +++ b/examples/pbe/regexp/regexp.py @@ -99,7 +99,7 @@ def __eval__(x, reg): } dsl = DSL(__primitive_types, __forbidden_patterns) -evaluator = DSLEvaluator(__semantics) +evaluator = DSLEvaluator(dsl.instantiate_semantics(__semantics)) evaluator.skip_exceptions.add(re.error) lexicon = list([chr(i) for i in range(32, 126)]) regexp_evaluator = RegexpEvaluator(__semantics) diff --git a/examples/pbe/regexp/task_generator_regexp.py b/examples/pbe/regexp/task_generator_regexp.py index f2cb777c..0db76e7a 100644 --- a/examples/pbe/regexp/task_generator_regexp.py +++ b/examples/pbe/regexp/task_generator_regexp.py @@ -42,12 +42,12 @@ def __generate_program__(self, type_request: Type) -> Tuple[Program, bool]: """ (program, is_unique) """ - program, is_unique = super().__generate_program__(type_request) + program, is_unique = super().generate_program(type_request) self.__successful_tries = 0 self.__regexp = "".join(program.__str__().split("(")[2:]).split(" ") return program, is_unique - def __sample_input__(self, arguments: TList[Type]) -> TList: + def sample_input(self, arguments: TList[Type]) -> TList: new_input = [""] # true examples if self.__successful_tries < 3: @@ -76,7 +76,7 @@ def __sample_input__(self, arguments: TList[Type]) -> TList: ] return new_input - def __eval_input__(self, solution: Program, new_input: TList) -> Any: + def eval_input(self, solution: Program, new_input: TList) -> Any: try: output = self.evaluator.eval(solution, new_input) if self.__successful_tries < 3 and output: @@ -95,9 +95,8 @@ def reproduce_regexp_dataset( evaluator: Evaluator, seed: Optional[int] = None, *args: Any, - **kwargs: Any + **kwargs: Any, ) -> Tuple[RegexpTaskGenerator, TList[int]]: - str_lexicon = list([chr(i) for i in range(32, 126)]) n_lexicon = [chr(i) for i in range(48, 58)] u_lexicon = [chr(i) for i in range(65, 91)] @@ -127,7 +126,7 @@ def reproduce_regexp_dataset( lambda _: str_lexicon, seed, *args, - **kwargs + **kwargs, ) return ( @@ -140,7 +139,7 @@ def reproduce_regexp_dataset( task_generator.output_validator, task_generator.max_tries, task_generator.uniques, - task_generator.verbose, + verbose=task_generator.verbose, ), str_lexicon, ) diff --git a/examples/pbe/regexp/type_regex.py b/examples/pbe/regexp/type_regex.py index f4ac9d69..f00d5daa 100644 --- a/examples/pbe/regexp/type_regex.py +++ b/examples/pbe/regexp/type_regex.py @@ -2,6 +2,7 @@ This file provides useful structures in order to treat with regular expressions (regex). Inspired by pregex project of github user Hieuzest: https://github.com/Hieuzest/pregex/blob/master/pregex.py """ + from typing import Any, Callable, Dict, List, Tuple, Optional, Match from abc import ABC, abstractmethod from functools import lru_cache @@ -224,9 +225,9 @@ def __init__( self._match = match self._pattern = pattern self._flags = flags - self._repeat_map: Dict[ - str, Tuple[str, CompiledPattern] - ] = self._pattern._kwargs["_repeat_map"] + self._repeat_map: Dict[str, Tuple[str, CompiledPattern]] = ( + self._pattern._kwargs["_repeat_map"] + ) self._require_post: Dict[str, Callable] = self._pattern._kwargs["_require_post"] @property diff --git a/examples/pbe/solve.py b/examples/pbe/solve.py new file mode 100644 index 00000000..de3c8b61 --- /dev/null +++ b/examples/pbe/solve.py @@ -0,0 +1,358 @@ +import os +import sys +from typing import Any, Callable, Iterable, List, Optional, Set, Tuple, Union +import csv + +import tqdm + +from dataset_loader import add_dataset_choice_arg, load_dataset +from dsl_loader import add_dsl_choice_arg, load_DSL + + +from synth import Dataset, PBE +from synth.filter.filter import Filter +from synth.semantic.evaluator import DSLEvaluator +from synth.specification import PBEWithConstants +from synth.syntax import ( + ProbDetGrammar, + ProbUGrammar, + DSL, + hs_enumerate_prob_grammar, + bs_enumerate_prob_grammar, + bps_enumerate_prob_grammar, + hs_enumerate_prob_u_grammar, + hs_enumerate_bucket_prob_grammar, + hs_enumerate_bucket_prob_u_grammar, + cd_enumerate_prob_grammar, + as_enumerate_prob_grammar, + ProgramEnumerator, + Type, + CFG, +) +from synth.filter import DFTAFilter, ObsEqFilter +from synth.filter.constraints import add_dfta_constraints +from synth.syntax.program import Program +from synth.task import Task +from synth.utils import load_object +from synth.utils.data_storage import legacy_load_object +from synth.utils.import_utils import import_file_function +from synth.pbe.solvers import ( + NaivePBESolver, + PBESolver, + CutoffPBESolver, + RestartPBESolver, +) + + +import argparse + + +SOLVERS = {solver.name(): solver for solver in [NaivePBESolver, CutoffPBESolver]} +base_solvers = {x: y for x, y in SOLVERS.items()} +for meta_solver in [RestartPBESolver]: + for name, solver in base_solvers.items(): + SOLVERS[f"{meta_solver.name()}.{name}"] = lambda *args, **kwargs: meta_solver( + *args, solver_builder=solver, **kwargs + ) + +SEARCH_ALGOS = { + "cd_search": (lambda x: cd_enumerate_prob_grammar(x, 20), None), + "beap_search": (bps_enumerate_prob_grammar, None), + "heap_search": (hs_enumerate_prob_grammar, hs_enumerate_prob_u_grammar), + "bucket_search": ( + lambda x: hs_enumerate_bucket_prob_grammar(x, 3), + lambda x: hs_enumerate_bucket_prob_u_grammar(x, 3), + ), + "bee_search": (bs_enumerate_prob_grammar, None), + "a_star": (as_enumerate_prob_grammar, None), +} + +PRUNING = {"dfta", "obs-eq"} + +parser = argparse.ArgumentParser( + description="Solve program synthesis tasks", fromfile_prefix_chars="@" +) +add_dsl_choice_arg(parser) +add_dataset_choice_arg(parser) +parser.add_argument( + "--pcfg", type=str, default=None, help="files containing the predicted PCFGs" +) +parser.add_argument( + "-s", + "--search", + choices=SEARCH_ALGOS.keys(), + default=list(SEARCH_ALGOS.keys())[0], + help=f"enumeration algorithm (default: {list(SEARCH_ALGOS.keys())[0]})", +) +parser.add_argument( + "--solver", + choices=list(SOLVERS.keys()), + default="naive", + help=f"used solver (default: naive)", +) +parser.add_argument( + "-o", "--output", type=str, default="./", help="output folder (default: './')" +) +parser.add_argument( + "--constrained", + action="store_true", + default=False, + help="use unambigous grammar to include constraints in the grammar if available", +) +parser.add_argument( + "--support", + type=str, + default=None, + help="train dataset to get the set of supported type requests", +) +parser.add_argument( + "-t", "--timeout", type=float, default=300, help="task timeout in s (default: 300)" +) + +parser.add_argument( + "-p", + "--pruning", + nargs="*", + choices=list(x for x in PRUNING), + help="runtime pruning", +) +parser.add_argument( + "--filter", + nargs="*", + type=str, + help="load the given files and call their get_filter functions to get a Filter[Program]", +) + +parameters = parser.parse_args() +dsl_name: str = parameters.dsl +dataset_file: str = parameters.dataset +pcfg_file: Optional[str] = parameters.pcfg +search_algo: str = parameters.search +method: Callable[[Any], PBESolver] = SOLVERS[parameters.solver] +output_folder: str = parameters.output +task_timeout: float = parameters.timeout +constrained: bool = parameters.constrained +support: Optional[str] = ( + None if not parameters.support else parameters.support.format(dsl_name=dsl_name) +) +pruning: List[str] = parameters.pruning or [] +filter_files: List[str] = parameters.filter or [] + +if not os.path.exists(dataset_file) or not os.path.isfile(dataset_file): + print("Dataset must be a valid dataset file!", file=sys.stderr) + sys.exit(1) +elif support is not None and ( + not os.path.exists(support) or not os.path.isfile(support) +): + print("Support dataset must be a valid dataset file!", file=sys.stderr) + sys.exit(1) + +det_search, u_search = SEARCH_ALGOS[search_algo] +custom_enumerate = u_search if constrained else det_search +if custom_enumerate is None: + txt = "det-CFG" if not constrained else "UCFG" + print( + f"search algorithm {search_algo} does not support enumeration for {txt}!", + file=sys.stderr, + ) + sys.exit(1) + +start_index = ( + 0 + if not os.path.sep in dataset_file + else (len(dataset_file) - dataset_file[::-1].index(os.path.sep)) +) +dataset_name = dataset_file[start_index : dataset_file.index(".", start_index)] + +supported_type_requests = Dataset.load(support).type_requests() if support else None + +# ================================ +# Load dftas files +# ================================ +filter_pot_funs = [ + import_file_function(file[:-3].replace("/", "."), ["get_filter"])() + for file in filter_files +] +filter_funs = [x.get_filter for x in filter_pot_funs if x is not None] + +# ================================ +# Load constants specific to dataset +# ================================ + + +def load_dsl_and_dataset() -> ( + Tuple[Dataset[PBE], DSL, DSLEvaluator, List[str], Set[Type]] +): + dsl_module = load_DSL(dsl_name) + dsl, evaluator = dsl_module.dsl, dsl_module.evaluator + # ================================ + # Load dataset + # ================================ + full_dataset = load_dataset(dsl_name, dataset_file) + + return ( + full_dataset, + dsl, + evaluator, + getattr(dsl_module, "constraints", []), + getattr(dsl_module, "constant_types", set()), + ) + + +# Produce PCFGS ========================================================== + + +def save(trace: Iterable, file: str) -> None: + with open(file, "w") as fd: + writer = csv.writer(fd) + writer.writerows(trace) + + +# Enumeration methods ===================================================== +def setup_filters( + task: Task[PBE], constant_types: Set[Type] +) -> Optional[Filter[Program]]: + out = None + # Dynamic DFTA filters + filters = [f(task.type_request, constant_types) for f in filter_funs] + for filter in filters: + out = filter if out is None else out.intersection(filter) + if "dfta" in pruning: + base_grammar = CFG.infinite( + dsl, task.type_request, constant_types=constant_types + ) + filter = DFTAFilter( + add_dfta_constraints(base_grammar, constraints, progress=False) + ) + out = filter if out is None else out.intersection(filter) + if "obs-eq" in pruning: + filter = ObsEqFilter( + solver.evaluator, [ex.inputs for ex in task.specification.examples] + ) + out = filter if out is None else out.intersection(filter) + return out + + +def enumerative_search( + dataset: Dataset[PBE], + evaluator: DSLEvaluator, + pcfgs: Union[List[ProbDetGrammar], List[ProbUGrammar]], + trace: List[Tuple[bool, float]], + solver: PBESolver, + custom_enumerate: Callable[ + [Union[ProbDetGrammar, ProbUGrammar]], ProgramEnumerator + ], + save_file: str, + constant_types: Set[Type], +) -> None: + start = max(0, len(trace) - 1) + pbar = tqdm.tqdm(total=len(pcfgs) - start, desc="Tasks", smoothing=0) + i = 0 + solved = 0 + total = 0 + tasks = [ + t + for t in dataset.tasks + if supported_type_requests is None or t.type_request in supported_type_requests + ] + stats_name = solver.available_stats() + if start == 0: + trace.append(["solved", "solution"] + stats_name) + for task, pcfg in zip(tasks[start:], pcfgs[start:]): + if task.metadata.get("name", None) is not None: + pbar.set_description_str(task.metadata["name"]) + total += 1 + task_solved = False + solution = None + if isinstance(task.specification, PBEWithConstants): + pcfg = pcfg.instantiate_constants(task.specification.constants) + try: + enumerator = custom_enumerate(pcfg) + enumerator.filter = setup_filters(task, constant_types) + sol_generator = solver.solve(task, enumerator, timeout=task_timeout) + solution = next(sol_generator) + sol_generator.send(True) + task_solved = True + solved += 1 + except KeyboardInterrupt: + break + except StopIteration: + pass + out = [task_solved, solution] + [solver.get_stats(name) for name in stats_name] + solver.reset_stats() + trace.append(out) + pbar.update(1) + evaluator.clear_cache() + # print("Cache hit:", evaluator.cache_hit_rate) + # print("Programs tried:", trace[len(trace) - 1][2]) + if i % 10 == 0: + pbar.set_postfix_str("Saving...") + save(trace, save_file) + pbar.set_postfix_str(f"Solved {solved}/{total}") + + pbar.close() + + +def load_pcfgs( + pcfg_file: Optional[str], +) -> Union[List[ProbDetGrammar], List[ProbUGrammar]]: + if pcfg_file is not None: + if constrained: + return legacy_load_object(pcfg_file) + else: + return load_object(pcfg_file) + pcfgs = [] + for task in full_dataset: + constant_types = set() + if isinstance(task.specification, PBEWithConstants): + constant_types = set(task.specification.constants.keys()) + g = CFG.infinite(dsl, task.type_request, 1, constant_types=constant_types) + pcfgs.append(ProbDetGrammar.uniform(g)) + return pcfgs + + +# Main ==================================================================== + +if __name__ == "__main__": + (full_dataset, dsl, evaluator, constraints, constant_types) = load_dsl_and_dataset() + + solver: PBESolver = method(evaluator=evaluator) + + pcfgs = load_pcfgs(pcfg_file) + if pcfg_file is None: + model_name = "uniform" + else: + model_name = os.path.split(pcfg_file)[1][ + len(f"pcfgs_{dataset_name}_") : -len(".pickle") + ] + pruning_suffix = "_".join(pruning) + file = os.path.join( + output_folder, + f"{dataset_name}_{search_algo}_{model_name}_{solver.full_name()}{pruning_suffix}.csv", + ) + trace = [] + if os.path.exists(file): + with open(file, "r") as fd: + reader = csv.reader(fd) + trace = [tuple(row) for row in reader] + print( + "\tLoaded", + len(trace) - 1, + "/", + len(full_dataset), + "(", + int((len(trace) - 1) * 100 / len(full_dataset)), + "%)", + ) + enumerative_search( + full_dataset, + evaluator, + pcfgs, + trace, + solver, + custom_enumerate, + file, + constant_types, + ) + save(trace, file) + print("csv file was saved as:", file) diff --git a/examples/pbe/transduction/dataset/convert_flashfill.py b/examples/pbe/transduction/dataset/convert_flashfill.py index 25bc4ea6..02cd6733 100644 --- a/examples/pbe/transduction/dataset/convert_flashfill.py +++ b/examples/pbe/transduction/dataset/convert_flashfill.py @@ -17,7 +17,12 @@ Arrow, ) from examples.pbe.regexp.type_regex import REGEXP +from examples.pbe.transduction.transduction import dsl, evaluator + import argparse +from synth.syntax.program import Constant + +from synth.syntax.type_helper import auto_type TRANSDUCTION = "transduction" @@ -34,26 +39,18 @@ help="Source JSON transduction file to be converted", ) -argument_parser.add_argument( - "--dsl", - type=str, - default=TRANSDUCTION, - choices=[TRANSDUCTION, TRANSDUCTION_OLD], - help="dsl (default: transduction)", -) parsed_parameters = argument_parser.parse_args() -dsl = parsed_parameters.dsl -if dsl == TRANSDUCTION: - from examples.pbe.transduction.transduction import dsl, evaluator -elif dsl == TRANSDUCTION_OLD: - from examples.pbe.transduction.transduction_old import dsl, evaluator -else: - print("DSL not recognized.") - exit() + name2type = {p.primitive: p.type for p in dsl.list_primitives} +name_converter = { + "tail_cst": "split_snd_cst", + "head_cst": "split_first_cst", + "head": "split_first", +} + def lcs(u, v): # t[(n,m)] = length of longest common string ending at first @@ -493,10 +490,16 @@ def __flashfill_str2prog__(s: str) -> Tuple[Program, Type]: var += 1 type_stack.append(STRING) continue + if name in name_converter: + name = name_converter[name] # primitives that serve as constants - if name in ["cste_in", "cste_out", "W", "$", ".", "epsilon"]: - primitive = Primitive(name, name2type[name]) - stack.append(primitive) + if name in ["cst_in", "cst_out", "W", "$", ".", "epsilon"]: + if name.startswith("cst_"): + t = CST_IN if name == "cst_in" else CST_OUT + stack.append(Constant(t)) + else: + primitive = Primitive(name, name2type[name]) + stack.append(primitive) else: # other primitives are functions, we want to add their type targets = [int(x) for x in subparts] arguments = [stack[x] for x in targets] @@ -507,6 +510,10 @@ def __flashfill_str2prog__(s: str) -> Tuple[Program, Type]: return stack[-1], type_request +CST_IN = auto_type("CST_STR_INPUT") +CST_OUT = auto_type("CST_STR_OUTPUT") + + def __convert__(load: Callable[[], Dataset[PBEWithConstants]], name: str) -> None: tasks = load() tasks.save(name) @@ -517,32 +524,25 @@ def __convert__(load: Callable[[], Dataset[PBEWithConstants]], name: str) -> Non if task.solution is None: print("Unsolved task: ", task.metadata["name"]) continue - for ex in task.specification.examples: - found = False - constants_in = task.specification.constants_in - constants_in.append("") - constants_out = task.specification.constants_out - constants_out.append("") - for cons_in in constants_in: - for cons_out in constants_out: - if found: - break - # print(evaluator.eval_with_constant(task.solution, ex.inputs, cons_in, cons_out), " vs ", ex.output) - found = ( - evaluator.eval_with_constant( - task.solution, ex.inputs, cons_in, cons_out - ) - == ex.output - ) - - if not found: - print(task.metadata["name"], "FAILED") - assert found + failed = True + for instance in task.solution.all_constants_instantiation( + task.specification.constants + ): + found = True + for ex in task.specification.examples: + found &= evaluator.eval(instance, ex.inputs) == ex.output + if not found: + break + failed &= not found + if failed: + print(task.metadata["name"], "FAILED:", task.solution) + print("\tconstants:", task.specification.constants) + assert not failed def convert_flashfill(input: str, name2type: Dict[str, Any]): def load(): - tasks: TList[Task[PBE]] = [] + tasks: TList[Task[PBEWithConstants]] = [] with open(input, "r") as fd: raw_tasks: TList[Dict[str, Any]] = json.load(fd) for raw_task in tqdm.tqdm(raw_tasks, desc="converting"): @@ -565,7 +565,9 @@ def load(): tasks.append( Task[PBEWithConstants]( type_request, - PBEWithConstants(examples, constants_in, constants_out), + PBEWithConstants( + examples, {CST_IN: constants_in, CST_OUT: constants_out} + ), prog, {"name": name}, ) diff --git a/examples/pbe/transduction/dataset/flashfill.json b/examples/pbe/transduction/dataset/flashfill.json index 9bfe9a0f..dd5509f7 100644 --- a/examples/pbe/transduction/dataset/flashfill.json +++ b/examples/pbe/transduction/dataset/flashfill.json @@ -1,6 +1,6 @@ [ { - "program": "STRING|cste_in|tail_cste,0,1", + "program": "STRING|cst_in|tail_cst,0,1", "metadata": { "name": "drop first word delimited by '.'" }, @@ -37,7 +37,7 @@ "constants_out": [] }, { - "program": "STRING|cste_in|head_cste,0,1", + "program": "STRING|cst_in|head_cst,0,1", "metadata": { "name": "nth (n=0) word delimited by '.'" }, @@ -73,7 +73,7 @@ "constants_out": [] }, { - "program": "STRING|cste_in|match_cste,0,1|concat,0,2|tail_cste,3,1|head_cste,4,1", + "program": "STRING|cst_in|match_cst,0,1|concat,0,2|tail_cst,3,1|head_cst,4,1", "metadata": { "name": "nth (n=1) word delimited by '.'" }, @@ -109,7 +109,7 @@ "constants_out": [] }, { - "program": "STRING|cste_in|except_end,1|match,0,2", + "program": "STRING|cst_in|except_end,1|match,0,2", "metadata": { "name": "nth (n=-1) word delimited by '.'" }, @@ -145,7 +145,7 @@ "constants_out": [] }, { - "program": "STRING|cste_in|tail_cste,0,1", + "program": "STRING|cst_in|tail_cst,0,1", "metadata": { "name": "drop first word delimited by ','" }, @@ -181,7 +181,7 @@ "constants_out": [] }, { - "program": "STRING|cste_in|head_cste,0,1", + "program": "STRING|cst_in|head_cst,0,1", "metadata": { "name": "nth (n=0) word delimited by ','" }, @@ -217,7 +217,7 @@ "constants_out": [] }, { - "program": "STRING|cste_in|match_cste,0,1|concat,0,2|tail_cste,3,1|head_cste,4,1", + "program": "STRING|cst_in|match_cst,0,1|concat,0,2|tail_cst,3,1|head_cst,4,1", "metadata": { "name": "nth (n=1) word delimited by ','" }, @@ -253,7 +253,7 @@ "constants_out": [] }, { - "program": "STRING|cste_in|except_end,1|match,0,2", + "program": "STRING|cst_in|except_end,1|match,0,2", "metadata": { "name": "nth (n=-1) word delimited by ','" }, @@ -289,7 +289,7 @@ "constants_out": [] }, { - "program": "STRING|cste_in|tail_cste,0,1", + "program": "STRING|cst_in|tail_cst,0,1", "metadata": { "name": "drop first word delimited by ' '" }, @@ -325,7 +325,7 @@ "constants_out": [] }, { - "program": "STRING|cste_in|head_cste,0,1", + "program": "STRING|cst_in|head_cst,0,1", "metadata": { "name": "nth (n=0) word delimited by ' '" }, @@ -361,7 +361,7 @@ "constants_out": [] }, { - "program": "STRING|cste_in|match_cste,0,1|concat,0,2|tail_cste,3,1|head_cste,4,1", + "program": "STRING|cst_in|match_cst,0,1|concat,0,2|tail_cst,3,1|head_cst,4,1", "metadata": { "name": "nth (n=1) word delimited by ' '" }, @@ -397,7 +397,7 @@ "constants_out": [] }, { - "program": "STRING|cste_in|except_end,1|match,0,2", + "program": "STRING|cst_in|except_end,1|match,0,2", "metadata": { "name": "nth (n=-1) word delimited by ' '" }, @@ -433,7 +433,7 @@ "constants_out": [] }, { - "program": "STRING|cste_in|tail_cste,0,1", + "program": "STRING|cst_in|tail_cst,0,1", "metadata": { "name": "drop first word delimited by '('" }, @@ -469,7 +469,7 @@ "constants_out": [] }, { - "program": "STRING|cste_in|head_cste,0,1", + "program": "STRING|cst_in|head_cst,0,1", "metadata": { "name": "nth (n=0) word delimited by '('" }, @@ -505,7 +505,7 @@ "constants_out": [] }, { - "program": "STRING|cste_in|match_cste,0,1|concat,0,2|tail_cste,3,1|head_cste,4,1", + "program": "STRING|cst_in|match_cst,0,1|concat,0,2|tail_cst,3,1|head_cst,4,1", "metadata": { "name": "nth (n=1) word delimited by '('" }, @@ -541,7 +541,7 @@ "constants_out": [] }, { - "program": "STRING|cste_in|except_end,1|match,0,2", + "program": "STRING|cst_in|except_end,1|match,0,2", "metadata": { "name": "nth (n=-1) word delimited by '('" }, @@ -577,7 +577,7 @@ "constants_out": [] }, { - "program": "STRING|cste_in|tail_cste,0,1", + "program": "STRING|cst_in|tail_cst,0,1", "metadata": { "name": "drop first word delimited by ')'" }, @@ -613,7 +613,7 @@ "constants_out": [] }, { - "program": "STRING|cste_in|head_cste,0,1", + "program": "STRING|cst_in|head_cst,0,1", "metadata": { "name": "nth (n=0) word delimited by ')'" }, @@ -649,7 +649,7 @@ "constants_out": [] }, { - "program": "STRING|cste_in|match_cste,0,1|concat,0,2|tail_cste,3,1|head_cste,4,1", + "program": "STRING|cst_in|match_cst,0,1|concat,0,2|tail_cst,3,1|head_cst,4,1", "metadata": { "name": "nth (n=1) word delimited by ')'" }, @@ -685,7 +685,7 @@ "constants_out": [] }, { - "program": "STRING|cste_in|except_end,1|match,0,2", + "program": "STRING|cst_in|except_end,1|match,0,2", "metadata": { "name": "nth (n=-1) word delimited by ')'" }, @@ -721,7 +721,7 @@ "constants_out": [] }, { - "program": "STRING|cste_in|tail_cste,0,1", + "program": "STRING|cst_in|tail_cst,0,1", "metadata": { "name": "drop first word delimited by '-'" }, @@ -757,7 +757,7 @@ "constants_out": [] }, { - "program": "STRING|cste_in|head_cste,0,1", + "program": "STRING|cst_in|head_cst,0,1", "metadata": { "name": "nth (n=0) word delimited by '-'" }, @@ -793,7 +793,7 @@ "constants_out": [] }, { - "program": "STRING|cste_in|match_cste,0,1|concat,0,2|tail_cste,3,1|head_cste,4,1", + "program": "STRING|cst_in|match_cst,0,1|concat,0,2|tail_cst,3,1|head_cst,4,1", "metadata": { "name": "nth (n=1) word delimited by '-'" }, @@ -829,7 +829,7 @@ "constants_out": [] }, { - "program": "STRING|cste_in|except_end,1|match,0,2", + "program": "STRING|cst_in|except_end,1|match,0,2", "metadata": { "name": "nth (n=-1) word delimited by '-'" }, @@ -865,7 +865,7 @@ "constants_out": [] }, { - "program": "STRING|STRING|cste_out|concat_cste,0,2|concat,3,1", + "program": "STRING|STRING|cst_out|concat_cst,0,2|concat,3,1", "metadata": { "name": "Append two words delimited by '.'" }, @@ -905,7 +905,7 @@ ] }, { - "program": "STRING|STRING|cste_out|concat_cste,0,2|concat,3,1", + "program": "STRING|STRING|cst_out|concat_cst,0,2|concat,3,1", "metadata": { "name": "Append two words delimited by ','" }, @@ -945,7 +945,7 @@ ] }, { - "program": "STRING|STRING|cste_out|concat_cste,0,2|concat,3,1", + "program": "STRING|STRING|cst_out|concat_cst,0,2|concat,3,1", "metadata": { "name": "Append two words delimited by ' '" }, @@ -985,7 +985,7 @@ ] }, { - "program": "STRING|STRING|cste_out|concat_cste,0,2|concat,3,1", + "program": "STRING|STRING|cst_out|concat_cst,0,2|concat,3,1", "metadata": { "name": "Append two words delimited by '('" }, @@ -1025,7 +1025,7 @@ ] }, { - "program": "STRING|STRING|cste_out|concat_cste,0,2|concat,3,1", + "program": "STRING|STRING|cst_out|concat_cst,0,2|concat,3,1", "metadata": { "name": "Append two words delimited by ')'" }, @@ -1065,7 +1065,7 @@ ] }, { - "program": "STRING|STRING|cste_out|concat_cste,0,2|concat,3,1", + "program": "STRING|STRING|cst_out|concat_cst,0,2|concat,3,1", "metadata": { "name": "Append two words delimited by '-'" }, @@ -1627,7 +1627,7 @@ "constants_out": [] }, { - "program": "STRING|.|cste_out|match,0,1|concat_cste,3,2", + "program": "STRING|.|cst_out|match,0,1|concat_cst,3,2", "metadata": { "name": "Take first character and append '.'" }, @@ -1663,7 +1663,7 @@ ] }, { - "program": "STRING|.|cste_out|match,0,1|concat_cste,3,2", + "program": "STRING|.|cst_out|match,0,1|concat_cst,3,2", "metadata": { "name": "Take first character and append ','" }, @@ -1699,7 +1699,7 @@ ] }, { - "program": "STRING|.|cste_out|match,0,1|concat_cste,3,2", + "program": "STRING|.|cst_out|match,0,1|concat_cst,3,2", "metadata": { "name": "Take first character and append ' '" }, @@ -1735,7 +1735,7 @@ ] }, { - "program": "STRING|.|cste_out|match,0,1|concat_cste,3,2", + "program": "STRING|.|cst_out|match,0,1|concat_cst,3,2", "metadata": { "name": "Take first character and append '('" }, @@ -1771,7 +1771,7 @@ ] }, { - "program": "STRING|.|cste_out|match,0,1|concat_cste,3,2", + "program": "STRING|.|cst_out|match,0,1|concat_cst,3,2", "metadata": { "name": "Take first character and append ')'" }, @@ -1807,7 +1807,7 @@ ] }, { - "program": "STRING|.|cste_out|match,0,1|concat_cste,3,2", + "program": "STRING|.|cst_out|match,0,1|concat_cst,3,2", "metadata": { "name": "Take first character and append '-'" }, @@ -1843,7 +1843,7 @@ ] }, { - "program": "STRING|STRING|.|cste_out|match,0,2|concat_cste,4,3|match,1,2|concat,5,6|concat_cste,7,3", + "program": "STRING|STRING|.|cst_out|match,0,2|concat_cst,4,3|match,1,2|concat,5,6|concat_cst,7,3", "metadata": { "name": "Abbreviate separate words (I)" }, @@ -1883,7 +1883,7 @@ ] }, { - "program": "STRING|.|cste_in|cste_out|match,0,1|tail_cste,0,2|match,5,1|concat_cste,4,3|concat,7,6|concat_cste,8,3", + "program": "STRING|.|cst_in|cst_out|match,0,1|tail_cst,0,2|match,5,1|concat_cst,4,3|concat,7,6|concat_cst,8,3", "metadata": { "name": "Abbreviate words separated by '.'" }, @@ -1921,7 +1921,7 @@ ] }, { - "program": "STRING|STRING|.|cste_out|match,0,2|concat_cste,4,3|match,1,2|concat,5,6|concat_cste,7,3", + "program": "STRING|STRING|.|cst_out|match,0,2|concat_cst,4,3|match,1,2|concat,5,6|concat_cst,7,3", "metadata": { "name": "Abbreviate separate words (II)" }, @@ -1961,7 +1961,7 @@ ] }, { - "program": "STRING|.|cste_in|cste_out|match,0,1|tail_cste,0,2|match,5,1|concat_cste,4,3|concat,7,6|concat_cste,8,3", + "program": "STRING|.|cst_in|cst_out|match,0,1|tail_cst,0,2|match,5,1|concat_cst,4,3|concat,7,6|concat_cst,8,3", "metadata": { "name": "Abbreviate words separated by ','" }, @@ -1999,7 +1999,7 @@ ] }, { - "program": "STRING|STRING|.|cste_out|match,0,2|concat_cste,4,3|match,1,2|concat,5,6|concat_cste,7,3", + "program": "STRING|STRING|.|cst_out|match,0,2|concat_cst,4,3|match,1,2|concat,5,6|concat_cst,7,3", "metadata": { "name": "Abbreviate separate words (III)" }, @@ -2039,7 +2039,7 @@ ] }, { - "program": "STRING|.|cste_in|cste_out|match,0,1|tail_cste,0,2|match,5,1|concat_cste,4,3|concat,7,6|concat_cste,8,3", + "program": "STRING|.|cst_in|cst_out|match,0,1|tail_cst,0,2|match,5,1|concat_cst,4,3|concat,7,6|concat_cst,8,3", "metadata": { "name": "Abbreviate words separated by ' '" }, @@ -2077,7 +2077,7 @@ ] }, { - "program": "STRING|STRING|.|cste_out|match,0,2|concat_cste,4,3|match,1,2|concat,5,6|concat_cste,7,3", + "program": "STRING|STRING|.|cst_out|match,0,2|concat_cst,4,3|match,1,2|concat,5,6|concat_cst,7,3", "metadata": { "name": "Abbreviate separate words (IIII)" }, @@ -2117,7 +2117,7 @@ ] }, { - "program": "STRING|.|cste_in|cste_out|match,0,1|tail_cste,0,2|match,5,1|concat_cste,4,3|concat,7,6|concat_cste,8,3", + "program": "STRING|.|cst_in|cst_out|match,0,1|tail_cst,0,2|match,5,1|concat_cst,4,3|concat,7,6|concat_cst,8,3", "metadata": { "name": "Abbreviate words separated by '('" }, @@ -2155,7 +2155,7 @@ ] }, { - "program": "STRING|STRING|.|cste_out|match,0,2|concat_cste,4,3|match,1,2|concat,5,6|concat_cste,7,3", + "program": "STRING|STRING|.|cst_out|match,0,2|concat_cst,4,3|match,1,2|concat,5,6|concat_cst,7,3", "metadata": { "name": "Abbreviate separate words (IIIII)" }, @@ -2195,7 +2195,7 @@ ] }, { - "program": "STRING|.|cste_in|cste_out|match,0,1|tail_cste,0,2|match,5,1|concat_cste,4,3|concat,7,6|concat_cste,8,3", + "program": "STRING|.|cst_in|cst_out|match,0,1|tail_cst,0,2|match,5,1|concat_cst,4,3|concat,7,6|concat_cst,8,3", "metadata": { "name": "Abbreviate words separated by ')'" }, @@ -2233,7 +2233,7 @@ ] }, { - "program": "STRING|STRING|.|cste_out|match,0,2|concat_cste,4,3|match,1,2|concat,5,6|concat_cste,7,3", + "program": "STRING|STRING|.|cst_out|match,0,2|concat_cst,4,3|match,1,2|concat,5,6|concat_cst,7,3", "metadata": { "name": "Abbreviate separate words (IIIIII)" }, @@ -2273,7 +2273,7 @@ ] }, { - "program": "STRING|.|cste_in|cste_out|match,0,1|tail_cste,0,2|match,5,1|concat_cste,4,3|concat,7,6|concat_cste,8,3", + "program": "STRING|.|cst_in|cst_out|match,0,1|tail_cst,0,2|match,5,1|concat_cst,4,3|concat,7,6|concat_cst,8,3", "metadata": { "name": "Abbreviate words separated by '-'" }, @@ -2539,7 +2539,7 @@ "constants_out": [] }, { - "program": "STRING|.|cste_out|head,0,1|concat_cste,3,2|concat,4,0", + "program": "STRING|.|cst_out|head,0,1|concat_cst,3,2|concat,4,0", "metadata": { "name": "Prepend 'UCLA'" }, @@ -2575,7 +2575,7 @@ ] }, { - "program": "STRING|cste_out|concat_cste,0,1", + "program": "STRING|cst_out|concat_cst,0,1", "metadata": { "name": "Append '984'" }, @@ -2611,7 +2611,7 @@ ] }, { - "program": "STRING|.|cste_in|cste_out|head,0,1|except,2|match,0,5|concat_cste,4,3|concat,7,6", + "program": "STRING|.|cst_in|cst_out|head,0,1|except,2|match,0,5|concat_cst,4,3|concat,7,6", "metadata": { "name": "Prepend 'Elias' to first word" }, @@ -2649,7 +2649,7 @@ ] }, { - "program": "STRING|.|cste_out|head,0,1|concat_cste,3,2|concat,4,0", + "program": "STRING|.|cst_out|head,0,1|concat_cst,3,2|concat,4,0", "metadata": { "name": "Prepend 'Santa'" }, @@ -2685,7 +2685,7 @@ ] }, { - "program": "STRING|cste_out|concat_cste,0,1", + "program": "STRING|cst_out|concat_cst,0,1", "metadata": { "name": "Append 'Kimberley'" }, @@ -2721,7 +2721,7 @@ ] }, { - "program": "STRING|.|cste_in|cste_out|head,0,1|except,2|match,0,5|concat_cste,4,3|concat,7,6", + "program": "STRING|.|cst_in|cst_out|head,0,1|except,2|match,0,5|concat_cst,4,3|concat,7,6", "metadata": { "name": "Prepend '196' to first word" }, @@ -2759,7 +2759,7 @@ ] }, { - "program": "STRING|.|cste_out|head,0,1|concat_cste,3,2|concat,4,0", + "program": "STRING|.|cst_out|head,0,1|concat_cst,3,2|concat,4,0", "metadata": { "name": "Prepend '+65'" }, @@ -2795,7 +2795,7 @@ ] }, { - "program": "STRING|cste_out|concat_cste,0,1", + "program": "STRING|cst_out|concat_cst,0,1", "metadata": { "name": "Append 'Jani'" }, @@ -2831,7 +2831,7 @@ ] }, { - "program": "STRING|.|cste_in|cste_out|head,0,1|except,2|match,0,5|concat_cste,4,3|concat,7,6", + "program": "STRING|.|cst_in|cst_out|head,0,1|except,2|match,0,5|concat_cst,4,3|concat,7,6", "metadata": { "name": "Prepend 'Edison' to first word" }, @@ -2869,7 +2869,7 @@ ] }, { - "program": "STRING|.|cste_out|head,0,1|concat_cste,3,2|concat,4,0", + "program": "STRING|.|cst_out|head,0,1|concat_cst,3,2|concat,4,0", "metadata": { "name": "Prepend '+115'" }, @@ -2905,7 +2905,7 @@ ] }, { - "program": "STRING|cste_out|concat_cste,0,1", + "program": "STRING|cst_out|concat_cst,0,1", "metadata": { "name": "Append 'Reily'" }, @@ -2941,7 +2941,7 @@ ] }, { - "program": "STRING|.|cste_in|cste_out|head,0,1|except,2|match,0,5|concat_cste,4,3|concat,7,6", + "program": "STRING|.|cst_in|cst_out|head,0,1|except,2|match,0,5|concat_cst,4,3|concat,7,6", "metadata": { "name": "Prepend 'Bradford' to first word" }, @@ -2979,7 +2979,7 @@ ] }, { - "program": "STRING|.|cste_out|head,0,1|concat_cste,3,2|concat,4,0", + "program": "STRING|.|cst_out|head,0,1|concat_cst,3,2|concat,4,0", "metadata": { "name": "Prepend '+132'" }, @@ -3015,7 +3015,7 @@ ] }, { - "program": "STRING|cste_out|concat_cste,0,1", + "program": "STRING|cst_out|concat_cst,0,1", "metadata": { "name": "Append '843'" }, @@ -3051,7 +3051,7 @@ ] }, { - "program": "STRING|.|cste_in|cste_out|head,0,1|except,2|match,0,5|concat_cste,4,3|concat,7,6", + "program": "STRING|.|cst_in|cst_out|head,0,1|except,2|match,0,5|concat_cst,4,3|concat,7,6", "metadata": { "name": "Prepend 'Cornell' to first word" }, @@ -3089,7 +3089,7 @@ ] }, { - "program": "STRING|.|cste_out|head,0,1|concat_cste,3,2|concat,4,0", + "program": "STRING|.|cst_out|head,0,1|concat_cst,3,2|concat,4,0", "metadata": { "name": "Prepend 'Park'" }, @@ -3125,7 +3125,7 @@ ] }, { - "program": "STRING|cste_out|concat_cste,0,1", + "program": "STRING|cst_out|concat_cst,0,1", "metadata": { "name": "Append 'Haven'" }, @@ -3161,7 +3161,7 @@ ] }, { - "program": "STRING|.|cste_in|cste_out|head,0,1|except,2|match,0,5|concat_cste,4,3|concat,7,6", + "program": "STRING|.|cst_in|cst_out|head,0,1|except,2|match,0,5|concat_cst,4,3|concat,7,6", "metadata": { "name": "Prepend '487' to first word" }, @@ -3462,7 +3462,7 @@ ] }, { - "program": "STRING|cste_out|concat_if,0,1", + "program": "STRING|cst_out|concat_if,0,1", "metadata": { "name": "ensure suffix `Montiel`" }, @@ -3501,7 +3501,7 @@ ] }, { - "program": "STRING|cste_out|concat_if,0,1", + "program": "STRING|cst_out|concat_if,0,1", "metadata": { "name": "ensure suffix `421`" }, @@ -3540,7 +3540,7 @@ ] }, { - "program": "STRING|cste_out|concat_if,0,1", + "program": "STRING|cst_out|concat_if,0,1", "metadata": { "name": "ensure suffix `Phillip`" }, @@ -3579,7 +3579,7 @@ ] }, { - "program": "STRING|cste_out|concat_if,0,1", + "program": "STRING|cst_out|concat_if,0,1", "metadata": { "name": "ensure suffix `Salley`" }, @@ -3618,7 +3618,7 @@ ] }, { - "program": "STRING|cste_out|concat_if,0,1", + "program": "STRING|cst_out|concat_if,0,1", "metadata": { "name": "ensure suffix `Santa`" }, @@ -3657,7 +3657,7 @@ ] }, { - "program": "STRING|cste_out|concat_if,0,1", + "program": "STRING|cst_out|concat_if,0,1", "metadata": { "name": "ensure suffix `Acura125`" }, @@ -3696,7 +3696,7 @@ ] }, { - "program": "STRING|cste_out|concat_if,0,1", + "program": "STRING|cst_out|concat_if,0,1", "metadata": { "name": "ensure suffix `Madelaine`" }, diff --git a/examples/pbe/transduction/knowledge_graph/preprocess_tasks.py b/examples/pbe/transduction/knowledge_graph/preprocess_tasks.py index 30012146..6fa9614a 100644 --- a/examples/pbe/transduction/knowledge_graph/preprocess_tasks.py +++ b/examples/pbe/transduction/knowledge_graph/preprocess_tasks.py @@ -33,7 +33,6 @@ def find_constants( my_indices: Optional[List[int]] = None, memory: Optional[Dict[Tuple[int, ...], List[Union[str, None]]]] = None, ) -> List[Union[str, None]]: - indices = my_indices or [0 for _ in strings] start = indices[0] iterator = list(range(len(strings))) diff --git a/examples/pbe/transduction/task_generator_transduction.py b/examples/pbe/transduction/task_generator_transduction.py index b5a472ad..bdf9c2ee 100644 --- a/examples/pbe/transduction/task_generator_transduction.py +++ b/examples/pbe/transduction/task_generator_transduction.py @@ -16,11 +16,12 @@ reproduce_dataset, ) from synth.syntax.program import Program +from synth.syntax.type_helper import auto_type from synth.syntax.type_system import List, Type from synth.task import Dataset, Task from synth.specification import PBE, Example, PBEWithConstants -from synth.semantic.evaluator import Evaluator +from synth.semantic.evaluator import DSLEvaluator from synth.syntax import ( STRING, DSL, @@ -30,34 +31,29 @@ UnionSampler, ) +CST_IN = auto_type("CST_STR_INPUT") +CST_OUT = auto_type("CST_STR_OUTPUT") + class TransductionTaskGenerator(TaskGenerator): - def __generate_program__(self, type_request: Type) -> Tuple[Program, bool]: - """ - (program, is_unique) - """ - program, is_unique = super().__generate_program__(type_request) - self.__constants = super().__sample_input__([STRING, STRING]) + def generate_program(self, type_request: Type) -> Tuple[Program, bool]: + program, is_unique = super().generate_program(type_request) + self.__constants = super().sample_input([STRING, STRING]) return program, is_unique - def __sample_input__(self, arguments: TList[Type]) -> TList: - arguments = super().__sample_input__(arguments) - return self.__constants + arguments - - def __make_task__( + def make_task( self, type_request: Type, solution: Program, inputs: TList, outputs: TList, - **kwargs: Any + **kwargs: Any, ) -> Task[PBEWithConstants]: return Task( type_request, PBEWithConstants( - [Example(inp[2:], out) for inp, out in zip(inputs, outputs)], - [self.__constants[0]], - [self.__constants[1]], + [Example(inp, out) for inp, out in zip(inputs, outputs)], + {CST_IN: [self.__constants[0]], CST_OUT: [self.__constants[1]]}, ), solution, {"generated": True, **kwargs}, @@ -67,10 +63,10 @@ def __make_task__( def reproduce_transduction_dataset( dataset: Dataset[PBE], dsl: DSL, - evaluator: Evaluator, + evaluator: DSLEvaluator, seed: Optional[int] = None, *args: Any, - **kwargs: Any + **kwargs: Any, ) -> Tuple[TaskGenerator, TList[int]]: def analyser(start: None, elment: Any) -> None: pass @@ -132,6 +128,7 @@ def get_sampler(start: None) -> UnionSampler: } ) + str_bank = str_lexicon + regexp_symbols task_generator, str_lexicon = reproduce_dataset( dataset, dsl, @@ -139,13 +136,12 @@ def get_sampler(start: None) -> UnionSampler: None, lambda _, __: None, get_sampler, - lambda _, max_list_length: basic_output_validator( - {str: str_lexicon + regexp_symbols}, max_list_length - ), + lambda _, max_list_length: lambda x: x is not None + and all(xi in str_bank for xi in x), lambda _: str_lexicon + regexp_symbols, seed, *args, - **kwargs + **kwargs, ) generator = TransductionTaskGenerator( @@ -157,7 +153,7 @@ def get_sampler(start: None) -> UnionSampler: task_generator.output_validator, task_generator.max_tries, task_generator.uniques, - task_generator.verbose, + verbose=task_generator.verbose, ) generator.skip_exceptions.add(ValueError) diff --git a/examples/pbe/transduction/transduction.py b/examples/pbe/transduction/transduction.py index f27e9c6c..946c292d 100644 --- a/examples/pbe/transduction/transduction.py +++ b/examples/pbe/transduction/transduction.py @@ -1,11 +1,10 @@ -from typing import Set import re from examples.pbe.transduction.task_generator_transduction import ( reproduce_transduction_dataset, ) -from synth.semantic.evaluator import DSLEvaluatorWithConstant +from synth.semantic.evaluator import DSLEvaluator from synth.syntax import ( DSL, Arrow, @@ -20,8 +19,8 @@ regex_search, ) -CSTE_IN = PrimitiveType("CST_STR_INPUT") -CSTE_OUT = PrimitiveType("CST_STR_OUTPUT") +CST_IN = PrimitiveType("CST_STR_INPUT") +CST_OUT = PrimitiveType("CST_STR_OUTPUT") def __concat__(x, y): @@ -36,14 +35,14 @@ def __concat_if__(x, y): return "" + x + y -def __head__(x: str, regexp: str): +def __split_first__(x: str, regexp: str): sbstr = regex_search(Raw(get_regexp(regexp)), x, flags=re.ASCII) if sbstr == None: return "" return x.split(sbstr.match.group(), 1)[0] -def __tail__(x: str, regexp: str): +def __split_snd__(x: str, regexp: str): sbstr = regex_search(Raw(get_regexp(regexp)), x, flags=re.ASCII) if sbstr == None: return "" @@ -58,7 +57,7 @@ def __match__(x: str, regexp: str): # untreated matching, done for constant text inputs (e.g. "." will be considered as a point instead of any char) -def __head_text__(x: str, text: str): +def __split_first_cst__(x: str, text: str): regexp = "(\\" + text + ")" sbstr = regex_search(Raw(regexp), x, flags=re.ASCII) if sbstr == None: @@ -66,7 +65,7 @@ def __head_text__(x: str, text: str): return x.split(sbstr.match.group(), 1)[0] -def __tail_text__(x: str, text: str): +def __split_snd_cst__(x: str, text: str): regexp = "(\\" + text + ")" sbstr = regex_search(Raw(regexp), x, flags=re.ASCII) if sbstr == None: @@ -74,7 +73,7 @@ def __tail_text__(x: str, text: str): return x.split(sbstr.match.group(), 1)[1] -def __match_text__(x: str, text: str): +def __match_cst__(x: str, text: str): regexp = "(\\" + text + ")" sbstr = regex_search(Raw(regexp), x, flags=re.ASCII) if sbstr == None: @@ -88,17 +87,15 @@ def __compose__(x, y): __semantics = { "concat": lambda x: lambda y: __concat__(x, y), - "concat_cste": lambda x: lambda y: __concat__(x, y), + "concat_cst": lambda x: lambda y: __concat__(x, y), "concat_if": lambda x: lambda y: __concat_if__(x, y), - "head": lambda x: lambda regexp: __head__(x, regexp), - "tail": lambda x: lambda regexp: __tail__(x, regexp), + "split_first": lambda x: lambda regexp: __split_first__(x, regexp), + "split_snd": lambda x: lambda regexp: __split_snd__(x, regexp), "match": lambda x: lambda regexp: __match__(x, regexp), - "head_cste": lambda x: lambda text: __head_text__(x, text), - "tail_cste": lambda x: lambda text: __tail_text__(x, text), - "match_cste": lambda x: lambda text: __match_text__(x, text), + "split_first_cst": lambda x: lambda text: __split_first_cst__(x, text), + "split_snd_cst": lambda x: lambda text: __split_snd_cst__(x, text), + "match_cst": lambda x: lambda text: __match_cst__(x, text), "compose": lambda x: lambda y: __compose__(x, y), - "cste_in": lambda x: x, - "cste_out": lambda x: x, "$": "$", ".": ".", "except": lambda x: "([^" + x + "]+", @@ -107,29 +104,29 @@ def __compose__(x, y): __primitive_types = { "concat": Arrow(STRING, Arrow(STRING, STRING)), - "concat_cste": Arrow(STRING, Arrow(CSTE_OUT, STRING)), - "concat_if": Arrow(STRING, Arrow(CSTE_OUT, STRING)), - "head": Arrow(STRING, Arrow(REGEXP, STRING)), - "tail": Arrow(STRING, Arrow(REGEXP, STRING)), + "concat_cst": Arrow(STRING, Arrow(CST_OUT, STRING)), + "concat_if": Arrow(STRING, Arrow(CST_OUT, STRING)), + "split_first": Arrow(STRING, Arrow(REGEXP, STRING)), + "split_snd": Arrow(STRING, Arrow(REGEXP, STRING)), "match": Arrow(STRING, Arrow(REGEXP, STRING)), - "head_cste": Arrow(STRING, Arrow(CSTE_IN, STRING)), - "tail_cste": Arrow(STRING, Arrow(CSTE_IN, STRING)), - "match_cste": Arrow(STRING, Arrow(CSTE_IN, STRING)), + "split_first_cst": Arrow(STRING, Arrow(CST_IN, STRING)), + "split_snd_cst": Arrow(STRING, Arrow(CST_IN, STRING)), + "match_cst": Arrow(STRING, Arrow(CST_IN, STRING)), "compose": Arrow(REGEXP, Arrow(REGEXP, REGEXP)), - "cste_in": CSTE_IN, - "cste_out": CSTE_OUT, "$": REGEXP, ".": REGEXP, - "except": Arrow(CSTE_IN, REGEXP), - "except_end": Arrow(CSTE_IN, REGEXP), + "except": Arrow(CST_IN, REGEXP), + "except_end": Arrow(CST_IN, REGEXP), } __forbidden_patterns = {} dsl = DSL(__primitive_types, __forbidden_patterns) -constant_types: Set[PrimitiveType] = set() -constant_types.add(CSTE_IN) -constant_types.add(CSTE_OUT) -evaluator = DSLEvaluatorWithConstant(__semantics, constant_types) +constant_types = {CST_IN, CST_OUT} +evaluator = DSLEvaluator(dsl.instantiate_semantics(__semantics)) evaluator.skip_exceptions.add(re.error) lexicon = list([chr(i) for i in range(32, 126)]) +constraints = [ + "concat ^concat _", + "compose ^compose _", +] diff --git a/examples/plot_enumeration_results.py b/examples/plot_enumeration_results.py new file mode 100644 index 00000000..2cac04e2 --- /dev/null +++ b/examples/plot_enumeration_results.py @@ -0,0 +1,123 @@ +from collections import OrderedDict, defaultdict +from typing import Dict, List +import matplotlib.pyplot as plt +import pltpublish as pub +import csv + +from plot_helper import ( + plot_y_wrt_x, + make_plot_wrapper, +) + + +__DATA__ = { + "time": (0, "Time (in s)"), + "programs": (1, "Programs Enumerated"), + "queued": (2, "Queue Size"), + "banked": (3, "Programs in Banks"), + "non_terminals": (4, "Non Terminals in Grammar"), + "rules": (5, "Derivation Rules in Grammar"), +} + + +def load_data(output_file: str, verbose: bool = False) -> Dict[str, Dict[int, List]]: + # Dict[name, data] + methods = {} + + # filename should end with a specific pattern + name = output_file[:-4] + if not (name.endswith("_detailed") or name.endswith("_growth")): + if verbose: + print(f"filename:{output_file} does not seem valid!") + return {} + trace = [] + with open(output_file, "r") as fd: + reader = csv.reader(fd) + trace = [tuple(row) for row in reader] + # Pop columns names + columns = {name: ind for ind, name in enumerate(trace.pop(0))} + indices = [ + columns["search"], + columns["time"], + columns["programs"], + columns["queue"], + columns["bank"], + columns["non_terminals"], + columns["derivation_rules"], + columns.get("seed", -1), + ] + data = [tuple(row[k] if k >= 0 else 0 for k in indices) for row in trace] + if len(data) == 0: + if verbose: + print(f"filename:{output_file} is empty!") + return {} + agg = defaultdict(dict) + for row in data: + seed = int(row[-1]) + if seed not in agg[row[0]]: + agg[row[0]][seed] = [] + agg[row[0]][seed].append(row[1:-1]) + for name, data in agg.items(): + name = name.replace("_", " ") + if name not in methods: + methods[name] = {} + # Save data for method + for seed, vals in data.items(): + methods[name][seed] = [tuple(float(x) for x in row) for row in vals] + # Backend support onl yseeded data so we register every data as seed 1 + return methods + + +# Generate all possible combinations +__PLOTS__ = {} +for ydata in list(__DATA__.keys()): + for xdata in list(__DATA__.keys()): + if xdata == ydata: + continue + __PLOTS__[f"{ydata}_wrt_{xdata}"] = make_plot_wrapper( + plot_y_wrt_x, + __DATA__[xdata], + __DATA__[ydata], + cumulative=False, + logy=xdata == "non_terminals", + ) + +if __name__ == "__main__": + import argparse + import sys + + parser = argparse.ArgumentParser(description="Plot results") + parser.add_argument( + "file", + type=str, + help="data file to load", + ) + parser.add_argument( + "-v", + "--verbose", + action="store_true", + default=False, + help="verbose mode", + ) + parser.add_argument("plots", nargs="+", choices=list(__PLOTS__.keys())) + parameters = parser.parse_args() + output_file: str = parameters.file + verbose: bool = parameters.verbose + plots: List[str] = parameters.plots + + # Load data + pub.setup() + methods = load_data(output_file, verbose) + # Check we have at least one file + if len(methods) == 0: + print("Error: no performance file was found!", file=sys.stderr) + sys.exit(1) + # Order by name so that it is always the same color for the same methods if diff. DSL + ordered_methods = OrderedDict() + for met in sorted(methods.keys()): + ordered_methods[met] = methods[met] + # Plotting + for count, to_plot in enumerate(plots): + ax = plt.subplot(1, len(plots), count + 1) + __PLOTS__[to_plot](ax, ordered_methods) + plt.show() diff --git a/examples/plot_helper.py b/examples/plot_helper.py new file mode 100644 index 00000000..1c68300f --- /dev/null +++ b/examples/plot_helper.py @@ -0,0 +1,295 @@ +import numpy as np + +import matplotlib.pyplot as plt + +from typing import List, Optional, Dict, Tuple + + +def plot_with_incertitude( + ax: plt.Axes, + x: List[np.ndarray], + y: List[np.ndarray], + label: str, + std_factor: float = 1.96, + miny: Optional[float] = None, + maxy: Optional[float] = None, + cumulative: bool = True, + n_points: int = 50, +) -> None: + max_len = max(len(xi) for xi in x) + X = np.array([xi for xi in x if len(xi) == max_len]) + Y = np.array([yi for yi in y if len(yi) == max_len]) + + x_min = np.min(X) + x_max = np.max(X) + if x_max == x_min: + return + if cumulative: + x_mean = np.cumsum(np.mean(X, axis=0)) + x_min = np.min(x_mean) + x_max = np.max(x_mean) + step = (x_max - x_min) / n_points + target_x = ( + np.arange(x_min, x_max + step / 2, step=step) + if len(X.shape) > 1 or len(X) > n_points + else X.reshape(-1) + ) + + y_mean = np.cumsum(np.mean(Y, axis=0)) + mean = np.interp(target_x, x_mean, y_mean) + + y_var = np.cumsum(np.var(Y, axis=0)) + y_var = np.interp(target_x, x_mean, y_var) + std = std_factor * np.sqrt(y_var) + else: + step = (x_max - x_min) / n_points + target_x = ( + np.arange(x_min, x_max + step / 2, step=step) + if len(X.shape) > 1 or len(X) > n_points + else X.reshape(-1) + ) + data = [] + for xi, yi in zip(X, Y): + nyi = np.interp(target_x, xi, yi) + data.append(nyi) + # Compute distribution + Y = np.array(data) + mean = np.mean(Y, axis=0) + std = std_factor * np.std(Y, axis=0) + + p = ax.plot(target_x, mean, label=label) + color = p[0].get_color() + upper = mean + std + if maxy is not None: + upper = np.minimum(upper, maxy) + lower = mean - std + if miny is not None: + lower = np.maximum(lower, miny) + ax.fill_between(target_x, lower, upper, color=color, alpha=0.5) + + +def make_plot_wrapper(func, *args, **kwargs) -> None: + def f(ax: plt.Axes, methods: Dict[str, Dict[int, List]]) -> None: + return func(ax, methods, *args, **kwargs) + + return f + + +def plot_y_wrt_x( + ax: plt.Axes, + methods: Dict[str, Dict[int, List]], + x_data: Tuple[int, str], + y_data: Tuple[int, str], + cumulative: bool = True, + logx: bool = False, + logy: bool = False, + xlim: Tuple[Optional[int], Optional[int]] = (0, None), + ylim: Tuple[Optional[int], Optional[int]] = (0, None), + hline_at_length: bool = False, + vline_at_length: bool = False, +) -> None: + # Plot data with incertitude + a_index, a_name = y_data + b_index, b_name = x_data + max_a = 0 + max_b = 0 + data_length = 0 + for method, seeds_dico in methods.items(): + seeds = list(seeds_dico.keys()) + data = [ + [(elems[b_index], elems[a_index]) for elems in seeds_dico[seed]] + for seed in seeds + ] + data_length = max(data_length, len(data[0])) + + xdata = [[x[0] for x in seed_data] for seed_data in data] + ydata = [[x[1] for x in seed_data] for seed_data in data] + plot_with_incertitude( + ax, + xdata, + ydata, + method.capitalize(), + miny=0, + maxy=data_length if hline_at_length else None, + cumulative=cumulative, + ) + max_a = max(max(np.max(yi) for yi in ydata), max_a) + max_b = max(max(np.max(xi) for xi in xdata), max_b) + if cumulative: + max_a = max(max(np.sum(yi) for yi in ydata), max_a) + max_b = max(max(np.sum(xi) for xi in xdata), max_b) + ax.set_xlabel(b_name) + ax.set_ylabel(a_name) + if hline_at_length: + ax.hlines( + [data_length], + xmin=0, + xmax=(xlim[1] or max_b), + label=f"All {a_name}", + color="k", + linestyles="dashed", + ) + if vline_at_length: + ax.vlines( + [data_length], + ymin=0, + ymax=(xlim[1] or max_a), + label=f"All {b_name}", + color="k", + linestyles="dashed", + ) + if logx: + ax.set_xscale("log") + else: + ax.set_xlim(xlim[0], xlim[1]) + if logy: + ax.set_yscale("log") + else: + ax.set_ylim(ylim[0], ylim[1]) + ax.grid() + ax.legend() + + +def get_rank_matrix( + methods: Dict[str, Dict[int, List]], yindex: int, maximize: bool +) -> np.ndarray: + method_names = list(methods.keys()) + task_len = len(list(list(methods.values())[0].values())[0]) + for val in methods.values(): + task_len = max(max(len(x) for x in val.values()), task_len) + seeds = set(list(methods.values())[0].keys()) + for val in methods.values(): + local_seeds = set(x for x, y in val.items() if len(y) == task_len) + seeds &= local_seeds + rank_matrix = np.ndarray((len(methods), task_len, len(methods)), dtype=float) + data = np.ndarray((len(methods), len(seeds)), dtype=float) + rng = np.random.default_rng(1) + for task_no in range(task_len): + for i, method in enumerate(method_names): + for j, seed in enumerate(seeds): + data[i, j] = methods[method][seed][task_no][yindex] + if maximize: + data = -data + rand_x = rng.random(size=data.shape) + # This is done to randomly break ties. + # Last key is the primary key, + indices = np.lexsort((rand_x, data), axis=0) + for i, method in enumerate(method_names): + rank_matrix[i, task_no] = [ + np.sum(indices[i] == rank) / len(seeds) for rank in range(len(methods)) + ] + return rank_matrix + + +def __ready_for_stacked_dist_plot__(ax: plt.Axes) -> None: + ax.spines["top"].set_visible(False) + ax.spines["right"].set_visible(False) + ax.spines["bottom"].set_visible(False) + ax.spines["left"].set_visible(False) + ax.tick_params( + axis="both", + which="both", + bottom=False, + top=False, + left=True, + right=False, + labeltop=False, + labelbottom=True, + labelleft=True, + labelright=False, + ) + + ax.legend(fancybox=True, fontsize="large") + + +def plot_rank_by( + ax: plt.Axes, + methods: Dict[str, Dict[int, List]], + y_data: Tuple[int, str], + maximize: bool = True, +) -> None: + width = 1.0 + a_index, a_name = y_data + rank_matrix = get_rank_matrix(methods, a_index, maximize) + labels = list(range(1, len(methods) + 1)) + mean_ranks = np.mean(rank_matrix, axis=-2) + bottom = np.zeros_like(mean_ranks[0]) + for i, key in enumerate(methods.keys()): + label = key + bars = ax.bar( + labels, + mean_ranks[i], + width, + label=label, + bottom=bottom, + alpha=0.9, + linewidth=1, + edgecolor="white", + ) + ax.bar_label(bars, labels=[f"{x:.1%}" for x in mean_ranks[i]]) + bottom += mean_ranks[i] + + ax.set_ylabel("Fraction (in %)", size="large") + yticks = np.array(range(0, 101, 20)) + ax.set_yticklabels(yticks) + ax.set_yticks(yticks * 0.01) + ax.set_xlabel("Ranking", size="large") + ax.set_xticks(labels) + ax.set_xticklabels(labels) + __ready_for_stacked_dist_plot__(ax) + word = "Most" if maximize else "Least" + ax.set_title(f"{word} {a_name}") + + +def plot_dist( + ax: plt.Axes, + methods: Dict[str, Dict[int, List]], + y_data: Tuple[int, str], + x_axis_name: str, +) -> None: + width = 1.0 + data_length = 0 + a_index, a_name = y_data + max_a = max( + max(max([y[a_index] for y in x]) for x in seed_dico.values()) + for seed_dico in methods.values() + ) + bottom = None + nbins = 5 + bins = [max_a] + while len(bins) <= nbins: + bins.insert(0, np.sqrt(bins[0] + 1)) + for i in range(nbins): + if bins[i + 1] < 2 * bins[i]: + bins[i + 1] = 2 * bins[i] + x_bar = list(range(1, nbins + 1)) + for method, seeds_dico in methods.items(): + hists = [] + for seed, raw_data in seeds_dico.items(): + data = [x[a_index] for x in raw_data] + data_length = max(data_length, len(data)) + hist, edges = np.histogram( + data, bins=bins, range=(1e-3, max_a), density=False + ) + hists.append(hist) + true_hist = np.mean(hists, axis=0) / data_length + if bottom is None: + bottom = np.zeros_like(true_hist) + label = method + bars = ax.bar( + x_bar, + true_hist, + width, + label=label, + bottom=bottom, + alpha=0.9, + linewidth=1, + edgecolor="white", + ) + ax.bar_label(bars, labels=[f"{x:.1%}" for x in true_hist]) + bottom += true_hist + __ready_for_stacked_dist_plot__(ax) + ax.set_yticklabels([]) + ax.set_xlabel(a_name, size="large") + ax.set_xticklabels(map(lambda x: f"<{x:.0f}", edges)) + ax.set_title(f"Distribution of {a_name} per {x_axis_name}") diff --git a/examples/plot_solve_results.py b/examples/plot_solve_results.py new file mode 100644 index 00000000..f19abade --- /dev/null +++ b/examples/plot_solve_results.py @@ -0,0 +1,325 @@ +from glob import glob +import os +from collections import OrderedDict +from typing import Dict, List, Tuple +import numpy as np +import matplotlib.pyplot as plt +import pltpublish as pub +import csv +from colorama import Fore as F + + +from plot_helper import ( + plot_y_wrt_x, + make_plot_wrapper, + plot_dist, + plot_rank_by, +) + + +def load_data( + dataset_name: str, output_folder: str, verbose: bool = False +) -> Tuple[Dict[str, Dict[int, List]], float]: + # Dict[name, Dict[seed, data]] + methods = {} + timeout = 1e99 + all_search = set() + all_solver = set() + + summary = {} + max_len = 0 + + for file in glob(os.path.join(output_folder, "*.csv")): + filename = os.path.relpath(file, output_folder) + if verbose: + print("found csv file:", file) + # filename should be {dataset_name}_seed_{seed}_{name}.csv + if not filename.startswith(dataset_name): + if verbose: + print(f"\tskipped: does not start with {dataset_name}") + continue + name = filename[len(dataset_name) : -4] + seed_text = "_seed_" + if seed_text not in name: + seed_text = "_uniform_" + if seed_text not in name: + if verbose: + print(f"\tskipped: does not contain _seed_ nor _uniform_") + continue + search = name[1 : name.index(seed_text)].replace("_", " ") + all_search.add(search) + name = name[name.index(seed_text) + len(seed_text) :] + seed = int(name[: name.index("_")]) if "seed" in seed_text else 0 + solver = name + if "_" in solver: + solver = solver[solver.index("_") + 1 :].replace("_", " ") + all_solver.add(solver) + name = search + " " + solver + if name not in methods: + methods[name] = {} + if seed in methods[name]: + print(f"Warning: duplicate seed {seed} for method {name}!") + continue + # Load data from the file + trace = [] + with open(os.path.join(output_folder, filename), "r") as fd: + reader = csv.reader(fd) + trace = [tuple(row) for row in reader] + # Pop columns names + columns = {name: ind for ind, name in enumerate(trace.pop(0))} + indices = [ + columns["solution"], + columns["time"], + columns["programs"], + columns.get("merged", -1), + columns.get("restarts", -1), + ] + data = [tuple(row[k] if k >= 0 else 0 for k in indices) for row in trace] + # Type conversion (success, time, num_of_programs) + trace = [ + ( + len(row[0]) > 1, + float(row[1]), + int(row[2]), + int(row[3]), + int(row[4]), + ) + for row in data + ] + for x in trace: + if x[0] == 0: + timeout = min(x[1], timeout) + if len(trace) == 0: + if verbose: + print(f"\tskipped: no data") + continue + # Save data for method + methods[name][seed] = trace + # Save summary data + if seed not in summary: + summary[seed] = {} + solved = sum(x[0] for x in trace) + summary[seed][name] = (solved, len(trace)) + max_len = max(max_len, len(trace)) + if verbose: + print( + f"{name} (seed={seed}) solved", + solved, + "/", + len(trace), + ) + to_replace = "" + if len(all_search) == 1 and len(all_solver) > 1: + search = list(all_search)[0] + to_replace = search + methods = { + k.replace(search, "").strip(" ").capitalize(): v for k, v in methods.items() + } + elif len(all_solver) == 1 and len(all_search) > 1: + solver = list(all_solver)[0] + to_replace = solver + methods = { + k.replace(solver, "").strip(" ").capitalize(): v for k, v in methods.items() + } + for seed in sorted(summary): + finished = sum( + 1 for solved, total in summary[seed].values() if total == max_len + ) + print(f"seed {F.LIGHTBLUE_EX}{seed}{F.RESET} ({finished}/{len(summary[seed])})") + for name, (solved, total) in sorted(summary[seed].items()): + if len(to_replace) > 0: + name = name.replace(to_replace, "").strip() + print( + f"\t{F.GREEN}{name}{F.RESET} solved {F.YELLOW}{solved}{F.RESET}/{total} ({F.YELLOW}{solved/total:.1%}{F.RESET}) tasks" + ) + return methods, timeout + + +def make_filter_wrapper(func, *args) -> None: + def f(task_index: int, methods: Dict[str, Dict[int, List]], timeout: float) -> bool: + return func(task_index, methods, timeout, *args) + + return f + + +def timeout_filter( + task_index: int, + methods: Dict[str, Dict[int, List]], + timeout: float, + nbr_timeouts: int, +) -> bool: + timeouts = 0 + for method, seeds_dico in methods.items(): + if nbr_timeouts == -1: + nbr_timeouts = len(methods) * len(seeds_dico) + for seed, data in seeds_dico.items(): + if task_index >= len(data): + nbr_timeouts -= 1 + continue + timeouts += 1 - data[task_index][0] + if timeouts > nbr_timeouts: + return False + + return True + + +def time_filter( + task_index: int, + methods: Dict[str, Dict[int, List]], + timeout: float, + ratio: float, + aggregator, +) -> bool: + all_times = [] + for method, seeds_dico in methods.items(): + for seed, data in seeds_dico.items(): + all_times.append(data[task_index][1]) + return aggregator(all_times) >= ratio * timeout + + +def reverse_filter(func): + def f( + task_index: int, methods: Dict[str, Dict[int, List]], timeout: float, *args + ) -> bool: + return not func(task_index, methods, timeout, *args) + + return f + + +__FILTERS__ = { + "timeouts.none": make_filter_wrapper(timeout_filter, 1), + "solve>=1": make_filter_wrapper(timeout_filter, -1), +} + +for ratio in [0.25, 0.5, 0.75]: + for name, aggr in [ + ("fastest", np.min), + ("mean", np.mean), + ("median", np.median), + ("slowest", np.max), + ]: + __FILTERS__[f"time.{name}>={ratio:.0%}"] = make_filter_wrapper( + time_filter, ratio, aggr + ) + +for key in list(__FILTERS__.keys()): + __FILTERS__[f"not.{key}"] = reverse_filter(__FILTERS__[key]) + + +def filter( + methods: Dict[str, Dict[int, List]], filter_name: str, timeout: float +) -> Dict[str, Dict[int, List]]: + fun = __FILTERS__[filter_name] + task_len = len(list(list(methods.values())[0].values())[0]) + should_keep = [fun(i, methods, timeout) for i in range(task_len)] + return { + m: { + s: [x for i, x in enumerate(data) if should_keep[i]] + for s, data in val.items() + if len(data) == task_len + } + for m, val in methods.items() + } + + +__DATA__ = { + "tasks": (0, "Tasks completed"), + "time": (1, "Time (in s)"), + "programs": (2, "Programs Enumerated"), + "merges": (3, "Programs Merged"), + "restarts": (4, "Restarts"), +} + + +# Generate all possible combinations +__PLOTS__ = {} +for ydata in list(__DATA__.keys()): + for xdata in list(__DATA__.keys()): + if xdata == ydata: + continue + __PLOTS__[f"{ydata}_wrt_{xdata}"] = make_plot_wrapper( + plot_y_wrt_x, + __DATA__[xdata], + __DATA__[ydata], + hline_at_length=ydata == "tasks", + vline_at_length=xdata == "tasks", + ) + if ydata != "tasks": + __PLOTS__[f"rank_by_{ydata}"] = make_plot_wrapper( + plot_rank_by, __DATA__[ydata], maximize=ydata == "tasks" + ) + __PLOTS__[f"dist_{ydata}_by_task"] = make_plot_wrapper( + plot_dist, __DATA__[ydata], "tasks" + ) + + +if __name__ == "__main__": + import argparse + import sys + + parser = argparse.ArgumentParser(description="Plot results") + parser.add_argument( + "-d", + "--dataset", + type=str, + default="dataset.pickle", + help="dataset (default: dataset.pickle)", + ) + parser.add_argument( + "--folder", + type=str, + default="./", + help="folder in which to look for CSV files (default: './')", + ) + parser.add_argument( + "-v", + "--verbose", + action="store_true", + default=False, + help="verbose mode", + ) + parser.add_argument( + "--filter", + type=str, + nargs="*", + choices=list(__FILTERS__.keys()), + help="filter tasks (keep data based on the filter)", + ) + parser.add_argument("plots", nargs="+", choices=list(__PLOTS__.keys())) + parameters = parser.parse_args() + dataset_file: str = parameters.dataset + output_folder: str = parameters.folder + verbose: bool = parameters.verbose + plots: List[str] = parameters.plots + filters: List[str] = parameters.filter or [] + + # Initial Setup + start_index = ( + 0 + if not os.path.sep in dataset_file + else (len(dataset_file) - dataset_file[::-1].index(os.path.sep)) + ) + dataset_name = dataset_file[start_index : dataset_file.index(".", start_index)] + # Load data + pub.setup() + methods, timeout = load_data(dataset_name, output_folder, verbose) + # Check we have at least one file + if len(methods) == 0: + print(f"{F.RED}Error: no performance file was found!{F.RESET}", file=sys.stderr) + sys.exit(1) + for filter_name in filters: + methods = filter(methods, filter_name, timeout) + # Check we did not remove everything + task_len = len(list(list(methods.values())[0].values())[0]) + if task_len == 0: + print(f"{F.RED}Error: filters left no tasks!{F.RESET}", file=sys.stderr) + sys.exit(1) + # Order by name so that it is always the same color for the same methods if diff. DSL + ordered_methods = OrderedDict() + for met in sorted(methods.keys()): + ordered_methods[met] = methods[met] + # Plotting + for count, to_plot in enumerate(plots): + ax = plt.subplot(1, len(plots), count + 1) + __PLOTS__[to_plot](ax, ordered_methods) + plt.show() diff --git a/experiments/deepcoder.sh b/experiments/deepcoder.sh deleted file mode 100755 index 2270edac..00000000 --- a/experiments/deepcoder.sh +++ /dev/null @@ -1,77 +0,0 @@ -#!/bin/env bash -# =================================================================== -# PARAMETERS -# =================================================================== -SEED=2 -# size of training dataset -TRAIN_SIZE=2500 -TEST_SIZE=500 -BATCH_SIZE=16 -EPOCHS=2 -# timeout in seconds -TIMEOUT=60 -# =================================================================== -# CONSTANTS -# =================================================================== -EXPERIMENT_FOLDER="./deepcoder_experiment/" -DEEPCODER_DATASET="./deepcoder.pickle" -TEST_DATASET="$EXPERIMENT_FOLDER/test.pickle" -TRAIN_DATASET="$EXPERIMENT_FOLDER/train.pickle" -MODEL_RAW_FILE="$EXPERIMENT_FOLDER/model_raw.pt" -MODEL_PRUNED_FILE="$EXPERIMENT_FOLDER/model_pruned.pt" -# =================================================================== -# MAIN CODE -# =================================================================== -# Check deepcoder dataset exists -if [ ! -f "$DEEPCODER_DATASET" ]; then - echo "$DEEPCODER_DATASET is missing!" - exit -fi -# Check experiment folder exists and create it -mkdir -p $EXPERIMENT_FOLDER -# Make test dataset -if [ ! -f "$TEST_DATASET" ]; then - echo "[Generation] Creating the test dataset." - python examples/pbe/dataset_generator.py --dsl deepcoder --dataset $DEEPCODER_DATASET --seed $SEED --size $TEST_SIZE -o $TEST_DATASET --uniform - if [ $? != "0" ]; then - exit 1 - fi -fi -# Make train dataset -if [ ! -f "$TRAIN_DATASET" ]; then - echo "[Generation] Creating the train dataset." - python examples/pbe/dataset_generator.py --dsl deepcoder --dataset $TEST_DATASET --seed $SEED --size $TRAIN_SIZE -o $TRAIN_DATASET --uniform - if [ $? != "0" ]; then - exit 1 - fi -fi -# Train raw model -if [ ! -f "$MODEL_RAW_FILE" ]; then - echo "[Training] Creating the model from the raw DSL." - python examples/pbe/model_trainer.py --dsl deepcoder.raw --dataset $TRAIN_DATASET --seed $SEED --b $BATCH_SIZE -o $MODEL_RAW_FILE -e $EPOCHS - if [ $? != "0" ]; then - exit 2 - fi -fi -# Train pruned model -if [ ! -f "$MODEL_PRUNED_FILE" ]; then - echo "[Training] Creating the model from the pruned DSL." - python examples/pbe/model_trainer.py --dsl deepcoder --dataset $TRAIN_DATASET --seed $SEED --b $BATCH_SIZE -o $MODEL_PRUNED_FILE -e $EPOCHS - if [ $? != "0" ]; then - exit 3 - fi -fi -# Train raw model -echo "[Evaluation] Evaluating the model from the raw DSL." -python examples/pbe/evaluate.py --dsl deepcoder.raw --dataset $TEST_DATASET --b $BATCH_SIZE --model $MODEL_RAW_FILE -o $EXPERIMENT_FOLDER -t $TIMEOUT -if [ $? != "0" ]; then - exit 4 -fi -# Train pruned model -echo "[Evaluation] Evaluating the model from the pruned DSL." -python examples/pbe/evaluate.py --dsl deepcoder --dataset $TEST_DATASET --b $BATCH_SIZE --model $MODEL_PRUNED_FILE -o $EXPERIMENT_FOLDER -t $TIMEOUT -if [ $? != "0" ]; then - exit 5 -fi -# Plotting -python examples/pbe/plot_results.py --dataset $TEST_DATASET --folder $EXPERIMENT_FOLDER \ No newline at end of file diff --git a/experiments/test_performance.sh b/experiments/test_performance.sh deleted file mode 100755 index bd3ad046..00000000 --- a/experiments/test_performance.sh +++ /dev/null @@ -1,66 +0,0 @@ -#!/bin/env bash -# =================================================================== -# PARAMETERS -# =================================================================== -SEED=2 -# size of training dataset -TRAIN_SIZE=2500 -# useful only if test dataset != base dataset -TEST_SIZE=500 -BATCH_SIZE=16 -EPOCHS=2 -# timeout in seconds -TIMEOUT=60 - -DSL_NAME="transduction" -BASE_DATASET="./flashfill.pickle" -TEST_DATASET="$BASE_DATASET" -# =================================================================== -# CONSTANTS -# =================================================================== -EXPERIMENT_FOLDER="./${DSL_NAME}_experiment/" -TRAIN_DATASET="$EXPERIMENT_FOLDER/train.pickle" -MODEL_FILE="$EXPERIMENT_FOLDER/model.pt" -# =================================================================== -# MAIN CODE -# =================================================================== -# Check deepcoder dataset exists -if [ ! -f "$BASE_DATASET" ]; then - echo "$BASE_DATASET is missing!" - exit -fi -# Check experiment folder exists and create it -mkdir -p $EXPERIMENT_FOLDER -# Make test dataset -if [ ! -f "$TEST_DATASET" ]; then - echo "[Generation] Creating the test dataset." - python examples/pbe/dataset_generator.py --dsl $DSL_NAME --dataset $TEST_DATASET --seed $SEED --size $TEST_SIZE -o $TEST_DATASET --uniform - if [ $? != "0" ]; then - exit 1 - fi -fi - -# Make train dataset -if [ ! -f "$TRAIN_DATASET" ]; then - echo "[Generation] Creating the train dataset." - python examples/pbe/dataset_generator.py --dsl $DSL_NAME --dataset $TEST_DATASET --seed $SEED --size $TRAIN_SIZE -o $TRAIN_DATASET --uniform - if [ $? != "0" ]; then - exit 1 - fi -fi -# Train model -if [ ! -f "$MODEL_FILE" ]; then - echo "[Training] Creating the model." - python examples/pbe/model_trainer.py --dsl $DSL_NAME --dataset $TRAIN_DATASET --seed $SEED --b $BATCH_SIZE -o $MODEL_FILE -e $EPOCHS - if [ $? != "0" ]; then - exit 2 - fi -fi -# Eval model -echo "[Evaluation] Evaluating the model." -python examples/pbe/evaluate.py --dsl $DSL_NAME --dataset $TEST_DATASET --b $BATCH_SIZE --model $MODEL_FILE -o $EXPERIMENT_FOLDER -t $TIMEOUT -if [ $? != "0" ]; then - exit 4 -fi -# Plotting -python examples/pbe/plot_results.py --dataset $TEST_DATASET --folder $EXPERIMENT_FOLDER \ No newline at end of file diff --git a/poetry.lock b/poetry.lock deleted file mode 100644 index 2da12b76..00000000 --- a/poetry.lock +++ /dev/null @@ -1,817 +0,0 @@ -[[package]] -name = "atomicwrites" -version = "1.4.0" -description = "Atomic file writes." -category = "dev" -optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" - -[[package]] -name = "attrs" -version = "21.4.0" -description = "Classes Without Boilerplate" -category = "dev" -optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" - -[package.extras] -dev = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "mypy", "pytest-mypy-plugins", "zope.interface", "furo", "sphinx", "sphinx-notfound-page", "pre-commit", "cloudpickle"] -docs = ["furo", "sphinx", "zope.interface", "sphinx-notfound-page"] -tests = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "mypy", "pytest-mypy-plugins", "zope.interface", "cloudpickle"] -tests_no_zope = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "mypy", "pytest-mypy-plugins", "cloudpickle"] - -[[package]] -name = "black" -version = "22.3.0" -description = "The uncompromising code formatter." -category = "dev" -optional = false -python-versions = ">=3.6.2" - -[package.dependencies] -click = ">=8.0.0" -mypy-extensions = ">=0.4.3" -pathspec = ">=0.9.0" -platformdirs = ">=2" -tomli = {version = ">=1.1.0", markers = "python_version < \"3.11\""} -typed-ast = {version = ">=1.4.2", markers = "python_version < \"3.8\" and implementation_name == \"cpython\""} -typing-extensions = {version = ">=3.10.0.0", markers = "python_version < \"3.10\""} - -[package.extras] -colorama = ["colorama (>=0.4.3)"] -d = ["aiohttp (>=3.7.4)"] -jupyter = ["ipython (>=7.8.0)", "tokenize-rt (>=3.2.0)"] -uvloop = ["uvloop (>=0.15.2)"] - -[[package]] -name = "click" -version = "8.1.3" -description = "Composable command line interface toolkit" -category = "dev" -optional = false -python-versions = ">=3.7" - -[package.dependencies] -colorama = {version = "*", markers = "platform_system == \"Windows\""} -importlib-metadata = {version = "*", markers = "python_version < \"3.8\""} - -[[package]] -name = "colorama" -version = "0.4.4" -description = "Cross-platform colored terminal text." -category = "main" -optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" - -[[package]] -name = "cycler" -version = "0.11.0" -description = "Composable style cycles" -category = "main" -optional = false -python-versions = ">=3.6" - -[[package]] -name = "cython" -version = "0.29.30" -description = "The Cython compiler for writing C extensions for the Python language." -category = "main" -optional = false -python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" - -[[package]] -name = "fonttools" -version = "4.33.3" -description = "Tools to manipulate font files" -category = "main" -optional = false -python-versions = ">=3.7" - -[package.extras] -all = ["fs (>=2.2.0,<3)", "lxml (>=4.0,<5)", "zopfli (>=0.1.4)", "lz4 (>=1.7.4.2)", "matplotlib", "sympy", "skia-pathops (>=0.5.0)", "uharfbuzz (>=0.23.0)", "brotlicffi (>=0.8.0)", "scipy", "brotli (>=1.0.1)", "munkres", "unicodedata2 (>=14.0.0)", "xattr"] -graphite = ["lz4 (>=1.7.4.2)"] -interpolatable = ["scipy", "munkres"] -lxml = ["lxml (>=4.0,<5)"] -pathops = ["skia-pathops (>=0.5.0)"] -plot = ["matplotlib"] -repacker = ["uharfbuzz (>=0.23.0)"] -symfont = ["sympy"] -type1 = ["xattr"] -ufo = ["fs (>=2.2.0,<3)"] -unicode = ["unicodedata2 (>=14.0.0)"] -woff = ["zopfli (>=0.1.4)", "brotlicffi (>=0.8.0)", "brotli (>=1.0.1)"] - -[[package]] -name = "importlib-metadata" -version = "4.11.4" -description = "Read metadata from Python packages" -category = "dev" -optional = false -python-versions = ">=3.7" - -[package.dependencies] -typing-extensions = {version = ">=3.6.4", markers = "python_version < \"3.8\""} -zipp = ">=0.5" - -[package.extras] -docs = ["sphinx", "jaraco.packaging (>=9)", "rst.linker (>=1.9)"] -perf = ["ipython"] -testing = ["pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytest-cov", "pytest-enabler (>=1.0.1)", "packaging", "pyfakefs", "flufl.flake8", "pytest-perf (>=0.9.2)", "pytest-black (>=0.3.7)", "pytest-mypy (>=0.9.1)", "importlib-resources (>=1.3)"] - -[[package]] -name = "iniconfig" -version = "1.1.1" -description = "iniconfig: brain-dead simple config-ini parsing" -category = "dev" -optional = false -python-versions = "*" - -[[package]] -name = "kiwisolver" -version = "1.4.2" -description = "A fast implementation of the Cassowary constraint solver" -category = "main" -optional = false -python-versions = ">=3.7" - -[package.dependencies] -typing-extensions = {version = "*", markers = "python_version < \"3.8\""} - -[[package]] -name = "matplotlib" -version = "3.5.2" -description = "Python plotting package" -category = "main" -optional = false -python-versions = ">=3.7" - -[package.dependencies] -cycler = ">=0.10" -fonttools = ">=4.22.0" -kiwisolver = ">=1.0.1" -numpy = ">=1.17" -packaging = ">=20.0" -pillow = ">=6.2.0" -pyparsing = ">=2.2.1" -python-dateutil = ">=2.7" -setuptools_scm = ">=4" - -[[package]] -name = "mypy" -version = "0.961" -description = "Optional static typing for Python" -category = "dev" -optional = false -python-versions = ">=3.6" - -[package.dependencies] -mypy-extensions = ">=0.4.3" -tomli = {version = ">=1.1.0", markers = "python_version < \"3.11\""} -typed-ast = {version = ">=1.4.0,<2", markers = "python_version < \"3.8\""} -typing-extensions = ">=3.10" - -[package.extras] -dmypy = ["psutil (>=4.0)"] -python2 = ["typed-ast (>=1.4.0,<2)"] -reports = ["lxml"] - -[[package]] -name = "mypy-extensions" -version = "0.4.3" -description = "Experimental type system extensions for programs checked with the mypy typechecker." -category = "dev" -optional = false -python-versions = "*" - -[[package]] -name = "numpy" -version = "1.21.1" -description = "NumPy is the fundamental package for array computing with Python." -category = "main" -optional = false -python-versions = ">=3.7" - -[[package]] -name = "packaging" -version = "21.3" -description = "Core utilities for Python packages" -category = "main" -optional = false -python-versions = ">=3.6" - -[package.dependencies] -pyparsing = ">=2.0.2,<3.0.5 || >3.0.5" - -[[package]] -name = "pathspec" -version = "0.9.0" -description = "Utility library for gitignore style pattern matching of file paths." -category = "dev" -optional = false -python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,>=2.7" - -[[package]] -name = "pillow" -version = "9.1.1" -description = "Python Imaging Library (Fork)" -category = "main" -optional = false -python-versions = ">=3.7" - -[package.extras] -docs = ["olefile", "sphinx (>=2.4)", "sphinx-copybutton", "sphinx-issues (>=3.0.1)", "sphinx-removed-in", "sphinx-rtd-theme (>=1.0)", "sphinxext-opengraph"] -tests = ["check-manifest", "coverage", "defusedxml", "markdown2", "olefile", "packaging", "pyroma", "pytest", "pytest-cov", "pytest-timeout"] - -[[package]] -name = "platformdirs" -version = "2.5.2" -description = "A small Python module for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." -category = "dev" -optional = false -python-versions = ">=3.7" - -[package.extras] -docs = ["furo (>=2021.7.5b38)", "proselint (>=0.10.2)", "sphinx-autodoc-typehints (>=1.12)", "sphinx (>=4)"] -test = ["appdirs (==1.4.4)", "pytest-cov (>=2.7)", "pytest-mock (>=3.6)", "pytest (>=6)"] - -[[package]] -name = "pltpublish" -version = "1.0.0" -description = "Utility package that takes care of configuring Matplotlib for publication-ready figures!" -category = "main" -optional = false -python-versions = ">=3.7.1" - -[package.dependencies] -matplotlib = ">=3.0" - -[[package]] -name = "pluggy" -version = "1.0.0" -description = "plugin and hook calling mechanisms for python" -category = "dev" -optional = false -python-versions = ">=3.6" - -[package.dependencies] -importlib-metadata = {version = ">=0.12", markers = "python_version < \"3.8\""} - -[package.extras] -dev = ["pre-commit", "tox"] -testing = ["pytest", "pytest-benchmark"] - -[[package]] -name = "py" -version = "1.11.0" -description = "library with cross-python path, ini-parsing, io, code, log facilities" -category = "dev" -optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" - -[[package]] -name = "pyparsing" -version = "3.0.9" -description = "pyparsing module - Classes and methods to define and execute parsing grammars" -category = "main" -optional = false -python-versions = ">=3.6.8" - -[package.extras] -diagrams = ["railroad-diagrams", "jinja2"] - -[[package]] -name = "pytest" -version = "7.1.2" -description = "pytest: simple powerful testing with Python" -category = "dev" -optional = false -python-versions = ">=3.7" - -[package.dependencies] -atomicwrites = {version = ">=1.0", markers = "sys_platform == \"win32\""} -attrs = ">=19.2.0" -colorama = {version = "*", markers = "sys_platform == \"win32\""} -importlib-metadata = {version = ">=0.12", markers = "python_version < \"3.8\""} -iniconfig = "*" -packaging = "*" -pluggy = ">=0.12,<2.0" -py = ">=1.8.2" -tomli = ">=1.0.0" - -[package.extras] -testing = ["argcomplete", "hypothesis (>=3.56)", "mock", "nose", "pygments (>=2.7.2)", "requests", "xmlschema"] - -[[package]] -name = "python-dateutil" -version = "2.8.2" -description = "Extensions to the standard Python datetime module" -category = "main" -optional = false -python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" - -[package.dependencies] -six = ">=1.5" - -[[package]] -name = "setuptools-scm" -version = "6.4.2" -description = "the blessed package to manage your versions by scm tags" -category = "main" -optional = false -python-versions = ">=3.6" - -[package.dependencies] -packaging = ">=20.0" -tomli = ">=1.0.0" - -[package.extras] -test = ["pytest (>=6.2)", "virtualenv (>20)"] -toml = ["setuptools (>=42)"] - -[[package]] -name = "six" -version = "1.16.0" -description = "Python 2 and 3 compatibility utilities" -category = "main" -optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" - -[[package]] -name = "tomli" -version = "2.0.1" -description = "A lil' TOML parser" -category = "main" -optional = false -python-versions = ">=3.7" - -[[package]] -name = "torch" -version = "1.11.0" -description = "Tensors and Dynamic neural networks in Python with strong GPU acceleration" -category = "main" -optional = false -python-versions = ">=3.7.0" - -[package.dependencies] -typing-extensions = "*" - -[[package]] -name = "tqdm" -version = "4.64.0" -description = "Fast, Extensible Progress Meter" -category = "main" -optional = false -python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,>=2.7" - -[package.dependencies] -colorama = {version = "*", markers = "platform_system == \"Windows\""} - -[package.extras] -dev = ["py-make (>=0.1.0)", "twine", "wheel"] -notebook = ["ipywidgets (>=6)"] -slack = ["slack-sdk"] -telegram = ["requests"] - -[[package]] -name = "typed-ast" -version = "1.5.4" -description = "a fork of Python 2 and 3 ast modules with type comment support" -category = "dev" -optional = false -python-versions = ">=3.6" - -[[package]] -name = "typing-extensions" -version = "4.2.0" -description = "Backported and Experimental Type Hints for Python 3.7+" -category = "main" -optional = false -python-versions = ">=3.7" - -[[package]] -name = "vose" -version = "0.0.2" -description = "" -category = "main" -optional = false -python-versions = ">=3.6" -develop = false - -[package.dependencies] -numpy = "*" - -[package.extras] -dev = ["pytest", "scipy"] - -[package.source] -type = "git" -url = "https://github.com/Theomat/vose.git" -reference = "master" -resolved_reference = "bc7d45899a657ff99f68704d72617fad841f3508" - -[[package]] -name = "zipp" -version = "3.8.0" -description = "Backport of pathlib-compatible object wrapper for zip files" -category = "dev" -optional = false -python-versions = ">=3.7" - -[package.extras] -docs = ["sphinx", "jaraco.packaging (>=9)", "rst.linker (>=1.9)"] -testing = ["pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytest-cov", "pytest-enabler (>=1.0.1)", "jaraco.itertools", "func-timeout", "pytest-black (>=0.3.7)", "pytest-mypy (>=0.9.1)"] - -[metadata] -lock-version = "1.1" -python-versions = ">=3.7.1" -content-hash = "1631d70ea19cb84928e911891fa8dcd012f6404614c16b94d6bf83bbce0b4e20" - -[metadata.files] -atomicwrites = [ - {file = "atomicwrites-1.4.0-py2.py3-none-any.whl", hash = "sha256:6d1784dea7c0c8d4a5172b6c620f40b6e4cbfdf96d783691f2e1302a7b88e197"}, - {file = "atomicwrites-1.4.0.tar.gz", hash = "sha256:ae70396ad1a434f9c7046fd2dd196fc04b12f9e91ffb859164193be8b6168a7a"}, -] -attrs = [ - {file = "attrs-21.4.0-py2.py3-none-any.whl", hash = "sha256:2d27e3784d7a565d36ab851fe94887c5eccd6a463168875832a1be79c82828b4"}, - {file = "attrs-21.4.0.tar.gz", hash = "sha256:626ba8234211db98e869df76230a137c4c40a12d72445c45d5f5b716f076e2fd"}, -] -black = [ - {file = "black-22.3.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:2497f9c2386572e28921fa8bec7be3e51de6801f7459dffd6e62492531c47e09"}, - {file = "black-22.3.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:5795a0375eb87bfe902e80e0c8cfaedf8af4d49694d69161e5bd3206c18618bb"}, - {file = "black-22.3.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e3556168e2e5c49629f7b0f377070240bd5511e45e25a4497bb0073d9dda776a"}, - {file = "black-22.3.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:67c8301ec94e3bcc8906740fe071391bce40a862b7be0b86fb5382beefecd968"}, - {file = "black-22.3.0-cp310-cp310-win_amd64.whl", hash = "sha256:fd57160949179ec517d32ac2ac898b5f20d68ed1a9c977346efbac9c2f1e779d"}, - {file = "black-22.3.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:cc1e1de68c8e5444e8f94c3670bb48a2beef0e91dddfd4fcc29595ebd90bb9ce"}, - {file = "black-22.3.0-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6d2fc92002d44746d3e7db7cf9313cf4452f43e9ea77a2c939defce3b10b5c82"}, - {file = "black-22.3.0-cp36-cp36m-win_amd64.whl", hash = "sha256:a6342964b43a99dbc72f72812bf88cad8f0217ae9acb47c0d4f141a6416d2d7b"}, - {file = "black-22.3.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:328efc0cc70ccb23429d6be184a15ce613f676bdfc85e5fe8ea2a9354b4e9015"}, - {file = "black-22.3.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:06f9d8846f2340dfac80ceb20200ea5d1b3f181dd0556b47af4e8e0b24fa0a6b"}, - {file = "black-22.3.0-cp37-cp37m-win_amd64.whl", hash = "sha256:ad4efa5fad66b903b4a5f96d91461d90b9507a812b3c5de657d544215bb7877a"}, - {file = "black-22.3.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:e8477ec6bbfe0312c128e74644ac8a02ca06bcdb8982d4ee06f209be28cdf163"}, - {file = "black-22.3.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:637a4014c63fbf42a692d22b55d8ad6968a946b4a6ebc385c5505d9625b6a464"}, - {file = "black-22.3.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:863714200ada56cbc366dc9ae5291ceb936573155f8bf8e9de92aef51f3ad0f0"}, - {file = "black-22.3.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:10dbe6e6d2988049b4655b2b739f98785a884d4d6b85bc35133a8fb9a2233176"}, - {file = "black-22.3.0-cp38-cp38-win_amd64.whl", hash = "sha256:cee3e11161dde1b2a33a904b850b0899e0424cc331b7295f2a9698e79f9a69a0"}, - {file = "black-22.3.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:5891ef8abc06576985de8fa88e95ab70641de6c1fca97e2a15820a9b69e51b20"}, - {file = "black-22.3.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:30d78ba6bf080eeaf0b7b875d924b15cd46fec5fd044ddfbad38c8ea9171043a"}, - {file = "black-22.3.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:ee8f1f7228cce7dffc2b464f07ce769f478968bfb3dd1254a4c2eeed84928aad"}, - {file = "black-22.3.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6ee227b696ca60dd1c507be80a6bc849a5a6ab57ac7352aad1ffec9e8b805f21"}, - {file = "black-22.3.0-cp39-cp39-win_amd64.whl", hash = "sha256:9b542ced1ec0ceeff5b37d69838106a6348e60db7b8fdd245294dc1d26136265"}, - {file = "black-22.3.0-py3-none-any.whl", hash = "sha256:bc58025940a896d7e5356952228b68f793cf5fcb342be703c3a2669a1488cb72"}, - {file = "black-22.3.0.tar.gz", hash = "sha256:35020b8886c022ced9282b51b5a875b6d1ab0c387b31a065b84db7c33085ca79"}, -] -click = [ - {file = "click-8.1.3-py3-none-any.whl", hash = "sha256:bb4d8133cb15a609f44e8213d9b391b0809795062913b383c62be0ee95b1db48"}, - {file = "click-8.1.3.tar.gz", hash = "sha256:7682dc8afb30297001674575ea00d1814d808d6a36af415a82bd481d37ba7b8e"}, -] -colorama = [ - {file = "colorama-0.4.4-py2.py3-none-any.whl", hash = "sha256:9f47eda37229f68eee03b24b9748937c7dc3868f906e8ba69fbcbdd3bc5dc3e2"}, - {file = "colorama-0.4.4.tar.gz", hash = "sha256:5941b2b48a20143d2267e95b1c2a7603ce057ee39fd88e7329b0c292aa16869b"}, -] -cycler = [ - {file = "cycler-0.11.0-py3-none-any.whl", hash = "sha256:3a27e95f763a428a739d2add979fa7494c912a32c17c4c38c4d5f082cad165a3"}, - {file = "cycler-0.11.0.tar.gz", hash = "sha256:9c87405839a19696e837b3b818fed3f5f69f16f1eec1a1ad77e043dcea9c772f"}, -] -cython = [ - {file = "Cython-0.29.30-cp27-cp27m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:e5cb144728a335d7a7fd0a61dff6abb7a9aeff9acd46d50b886b7d9a95bb7311"}, - {file = "Cython-0.29.30-cp27-cp27m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:d52d5733dcb144deca8985f0a197c19cf71e6bd6bd9d8034f3f67b2dea68d12b"}, - {file = "Cython-0.29.30-cp27-cp27mu-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:0cd6c932e945af15ae4ddcf8fdc0532bda48784c92ed0a53cf4fae897067ccd1"}, - {file = "Cython-0.29.30-cp27-cp27mu-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:a30092c6e2d24255fbfe0525f9a750554f96a263ed986d12ac3c9f7d9a85a424"}, - {file = "Cython-0.29.30-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_24_aarch64.whl", hash = "sha256:abcaf99f90cddc0f53600613eaafc81d27c4ac0671f0df8bce5466d4e86d54a1"}, - {file = "Cython-0.29.30-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:9826981308802c61a76f967875b31b7c683b7fc369eabaa6cbc22efeb12c90e8"}, - {file = "Cython-0.29.30-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_24_i686.whl", hash = "sha256:d166d9f853db436f5e10733a9bd615699ddb4238feadcbdf5ae50dc0b18b18f5"}, - {file = "Cython-0.29.30-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:0b83a342a071c4f14e7410568e0c0bd95e2f20c0b32944e3a721649a1357fda4"}, - {file = "Cython-0.29.30-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_24_aarch64.whl", hash = "sha256:ffa8c09617833ff0824aa7926fa4fa9d2ec3929c67168e89105f276b7f36a63e"}, - {file = "Cython-0.29.30-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:6b389a94b42909ff56d3491fde7c44802053a103701a7d210dcdd449a5b4f7b4"}, - {file = "Cython-0.29.30-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_24_i686.whl", hash = "sha256:7eff71c39b98078deaad1d1bdbf10864d234e2ab5d5257e980a6926a8523f697"}, - {file = "Cython-0.29.30-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:8e08f18d249b9b65e272a5a60f3360a8922c4c149036b98fc821fe1afad5bdae"}, - {file = "Cython-0.29.30-cp35-cp35m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:3993aafd68a7311ef94e00e44a137f6a50a69af0575ebcc8a0a074ad4152a2b2"}, - {file = "Cython-0.29.30-cp35-cp35m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:5c7cfd908efc77306ddd41ef07f5a7a352c9205ced5c1e00a0e5ece4391707c4"}, - {file = "Cython-0.29.30-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_24_aarch64.whl", hash = "sha256:e605635a92ae862cb46d84d1d6883324518f9aaff4a71cede6d61df20b6a410c"}, - {file = "Cython-0.29.30-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:786ee7b0cdb508b6de64c0f1f9c74f207186dfafad1ef938f25b7494cc481a80"}, - {file = "Cython-0.29.30-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_24_i686.whl", hash = "sha256:1e078943bbde703ca08d43e719480eb8b187d9023cbd91798619f5b5e18d0d71"}, - {file = "Cython-0.29.30-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:5183356c756b56c2df12d96300d602e47ffb89943c5a0bded66faca5d3da7be0"}, - {file = "Cython-0.29.30-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:e36755e71fd20eceb410cc441b7f2586654c2edb013f4663842fdaf60b96c1ca"}, - {file = "Cython-0.29.30-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:e29d3487f357108b711f2f29319811d92166643d29aec1b8e063aad46a346775"}, - {file = "Cython-0.29.30-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_24_aarch64.whl", hash = "sha256:5a8a3709ad9343a1dc02b8ec9cf6bb284be248d2c64af85464d9c3525eec74a5"}, - {file = "Cython-0.29.30-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:b17639b6a155abaa61a89f6f1323fb57b138d0529911ca03978d594945d062ba"}, - {file = "Cython-0.29.30-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_24_i686.whl", hash = "sha256:9462e9cf284d9b1d2c5b53d62188e3c09cc5c7a0018ba349d99b73cf930238de"}, - {file = "Cython-0.29.30-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:58d2b734250c1093bc69c1c3a6f5736493b9f8b34eb765f0a28a4a09468c0b00"}, - {file = "Cython-0.29.30-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:28db751e2d8365b39664d9cb62dc1668688b8fcc5b954e9ca9d20e0b8e03d8b0"}, - {file = "Cython-0.29.30-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:5f2dae7dd56860018d5fd5032a71f11fdc224020932b463d0511a1536f27df85"}, - {file = "Cython-0.29.30-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_24_aarch64.whl", hash = "sha256:d0859a958e0155b6ae4dee04170ccfac2c3d613a7e3bee8749614530b9e3b4a4"}, - {file = "Cython-0.29.30-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:d0f34b44078e3e0b2f1be2b99044619b37127128e7d55c54bbd2438adcaf31d3"}, - {file = "Cython-0.29.30-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_24_i686.whl", hash = "sha256:80a7255ad84620f53235c0720cdee2bc7431d9e3db7b3742823a606c329eb539"}, - {file = "Cython-0.29.30-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:3d0239c7a22a0f3fb1deec75cab0078eba4dd17868aa992a54a178851e0c8684"}, - {file = "Cython-0.29.30-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:c299c5b250ae9f81c38200441b6f1d023aeee9d8e7f61c04001c7437181ccb06"}, - {file = "Cython-0.29.30-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:019d330ac580b2ca4a457c464ac0b8c35009d820ef5d09f328d6e31a10e1ce89"}, - {file = "Cython-0.29.30-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_24_aarch64.whl", hash = "sha256:71fd1d910aced510c001936667fc7f2901c49b2ca7a2ad67358979c94a7f42ac"}, - {file = "Cython-0.29.30-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:60d370c33d56077d30e5f425026e58c2559e93b4784106f61581cf54071f6270"}, - {file = "Cython-0.29.30-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_24_i686.whl", hash = "sha256:20778297c8bcba201ca122a2f792a9899d6e64c68a92363dd7eb24306d54d7ce"}, - {file = "Cython-0.29.30-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:9f1fe924c920b699af27aefebd722df4cfbb85206291623cd37d1a7ddfd57792"}, - {file = "Cython-0.29.30-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:c79685dd4631a188e2385dc6a232896c7b67ea2e3e5f8b5555b4b743f475d6d7"}, - {file = "Cython-0.29.30-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:88c5e2f92f16cd999ddfc43d572639679e8a057587088e627e98118e46a803e6"}, - {file = "Cython-0.29.30-py2.py3-none-any.whl", hash = "sha256:acb72e0b42079862cf2f894964b41f261e941e75677e902c5f4304b3eb00af33"}, - {file = "Cython-0.29.30.tar.gz", hash = "sha256:2235b62da8fe6fa8b99422c8e583f2fb95e143867d337b5c75e4b9a1a865f9e3"}, -] -fonttools = [ - {file = "fonttools-4.33.3-py3-none-any.whl", hash = "sha256:f829c579a8678fa939a1d9e9894d01941db869de44390adb49ce67055a06cc2a"}, - {file = "fonttools-4.33.3.zip", hash = "sha256:c0fdcfa8ceebd7c1b2021240bd46ef77aa8e7408cf10434be55df52384865f8e"}, -] -importlib-metadata = [ - {file = "importlib_metadata-4.11.4-py3-none-any.whl", hash = "sha256:c58c8eb8a762858f49e18436ff552e83914778e50e9d2f1660535ffb364552ec"}, - {file = "importlib_metadata-4.11.4.tar.gz", hash = "sha256:5d26852efe48c0a32b0509ffbc583fda1a2266545a78d104a6f4aff3db17d700"}, -] -iniconfig = [ - {file = "iniconfig-1.1.1-py2.py3-none-any.whl", hash = "sha256:011e24c64b7f47f6ebd835bb12a743f2fbe9a26d4cecaa7f53bc4f35ee9da8b3"}, - {file = "iniconfig-1.1.1.tar.gz", hash = "sha256:bc3af051d7d14b2ee5ef9969666def0cd1a000e121eaea580d4a313df4b37f32"}, -] -kiwisolver = [ - {file = "kiwisolver-1.4.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:6e395ece147f0692ca7cdb05a028d31b83b72c369f7b4a2c1798f4b96af1e3d8"}, - {file = "kiwisolver-1.4.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:0b7f50a1a25361da3440f07c58cd1d79957c2244209e4f166990e770256b6b0b"}, - {file = "kiwisolver-1.4.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:3c032c41ae4c3a321b43a3650e6ecc7406b99ff3e5279f24c9b310f41bc98479"}, - {file = "kiwisolver-1.4.2-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:1dcade8f6fe12a2bb4efe2cbe22116556e3b6899728d3b2a0d3b367db323eacc"}, - {file = "kiwisolver-1.4.2-cp310-cp310-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:0e45e780a74416ef2f173189ef4387e44b5494f45e290bcb1f03735faa6779bf"}, - {file = "kiwisolver-1.4.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9d2bb56309fb75a811d81ed55fbe2208aa77a3a09ff5f546ca95e7bb5fac6eff"}, - {file = "kiwisolver-1.4.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:69b2d6c12f2ad5f55104a36a356192cfb680c049fe5e7c1f6620fc37f119cdc2"}, - {file = "kiwisolver-1.4.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:262c248c60f22c2b547683ad521e8a3db5909c71f679b93876921549107a0c24"}, - {file = "kiwisolver-1.4.2-cp310-cp310-win32.whl", hash = "sha256:1008346a7741620ab9cc6c96e8ad9b46f7a74ce839dbb8805ddf6b119d5fc6c2"}, - {file = "kiwisolver-1.4.2-cp310-cp310-win_amd64.whl", hash = "sha256:6ece2e12e4b57bc5646b354f436416cd2a6f090c1dadcd92b0ca4542190d7190"}, - {file = "kiwisolver-1.4.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:b978afdb913ca953cf128d57181da2e8798e8b6153be866ae2a9c446c6162f40"}, - {file = "kiwisolver-1.4.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7f88c4b8e449908eeddb3bbd4242bd4dc2c7a15a7aa44bb33df893203f02dc2d"}, - {file = "kiwisolver-1.4.2-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e348f1904a4fab4153407f7ccc27e43b2a139752e8acf12e6640ba683093dd96"}, - {file = "kiwisolver-1.4.2-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c839bf28e45d7ddad4ae8f986928dbf5a6d42ff79760d54ec8ada8fb263e097c"}, - {file = "kiwisolver-1.4.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:8ae5a071185f1a93777c79a9a1e67ac46544d4607f18d07131eece08d415083a"}, - {file = "kiwisolver-1.4.2-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:c222f91a45da9e01a9bc4f760727ae49050f8e8345c4ff6525495f7a164c8973"}, - {file = "kiwisolver-1.4.2-cp37-cp37m-win32.whl", hash = "sha256:a4e8f072db1d6fb7a7cc05a6dbef8442c93001f4bb604f1081d8c2db3ca97159"}, - {file = "kiwisolver-1.4.2-cp37-cp37m-win_amd64.whl", hash = "sha256:be9a650890fb60393e60aacb65878c4a38bb334720aa5ecb1c13d0dac54dd73b"}, - {file = "kiwisolver-1.4.2-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:8ec2e55bf31b43aabe32089125dca3b46fdfe9f50afbf0756ae11e14c97b80ca"}, - {file = "kiwisolver-1.4.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1d1078ba770d6165abed3d9a1be1f9e79b61515de1dd00d942fa53bba79f01ae"}, - {file = "kiwisolver-1.4.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:cbb5eb4a2ea1ffec26268d49766cafa8f957fe5c1b41ad00733763fae77f9436"}, - {file = "kiwisolver-1.4.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2e6cda72db409eefad6b021e8a4f964965a629f577812afc7860c69df7bdb84a"}, - {file = "kiwisolver-1.4.2-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b1605c7c38cc6a85212dfd6a641f3905a33412e49f7c003f35f9ac6d71f67720"}, - {file = "kiwisolver-1.4.2-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:81237957b15469ea9151ec8ca08ce05656090ffabc476a752ef5ad7e2644c526"}, - {file = "kiwisolver-1.4.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:240009fdf4fa87844f805e23f48995537a8cb8f8c361e35fda6b5ac97fcb906f"}, - {file = "kiwisolver-1.4.2-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:240c2d51d098395c012ddbcb9bd7b3ba5de412a1d11840698859f51d0e643c4f"}, - {file = "kiwisolver-1.4.2-cp38-cp38-win32.whl", hash = "sha256:8b6086aa6936865962b2cee0e7aaecf01ab6778ce099288354a7229b4d9f1408"}, - {file = "kiwisolver-1.4.2-cp38-cp38-win_amd64.whl", hash = "sha256:0d98dca86f77b851350c250f0149aa5852b36572514d20feeadd3c6b1efe38d0"}, - {file = "kiwisolver-1.4.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:91eb4916271655dfe3a952249cb37a5c00b6ba68b4417ee15af9ba549b5ba61d"}, - {file = "kiwisolver-1.4.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:fa4d97d7d2b2c082e67907c0b8d9f31b85aa5d3ba0d33096b7116f03f8061261"}, - {file = "kiwisolver-1.4.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:71469b5845b9876b8d3d252e201bef6f47bf7456804d2fbe9a1d6e19e78a1e65"}, - {file = "kiwisolver-1.4.2-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:8ff3033e43e7ca1389ee59fb7ecb8303abb8713c008a1da49b00869e92e3dd7c"}, - {file = "kiwisolver-1.4.2-cp39-cp39-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:89b57c2984f4464840e4b768affeff6b6809c6150d1166938ade3e22fbe22db8"}, - {file = "kiwisolver-1.4.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ffbdb9a96c536f0405895b5e21ee39ec579cb0ed97bdbd169ae2b55f41d73219"}, - {file = "kiwisolver-1.4.2-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8a830a03970c462d1a2311c90e05679da56d3bd8e78a4ba9985cb78ef7836c9f"}, - {file = "kiwisolver-1.4.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f74f2a13af201559e3d32b9ddfc303c94ae63d63d7f4326d06ce6fe67e7a8255"}, - {file = "kiwisolver-1.4.2-cp39-cp39-win32.whl", hash = "sha256:e677cc3626287f343de751e11b1e8a5b915a6ac897e8aecdbc996cd34de753a0"}, - {file = "kiwisolver-1.4.2-cp39-cp39-win_amd64.whl", hash = "sha256:b3e251e5c38ac623c5d786adb21477f018712f8c6fa54781bd38aa1c60b60fc2"}, - {file = "kiwisolver-1.4.2-pp37-pypy37_pp73-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:0c380bb5ae20d829c1a5473cfcae64267b73aaa4060adc091f6df1743784aae0"}, - {file = "kiwisolver-1.4.2-pp37-pypy37_pp73-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:484f2a5f0307bc944bc79db235f41048bae4106ffa764168a068d88b644b305d"}, - {file = "kiwisolver-1.4.2-pp37-pypy37_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0e8afdf533b613122e4bbaf3c1e42c2a5e9e2d1dd3a0a017749a7658757cb377"}, - {file = "kiwisolver-1.4.2-pp37-pypy37_pp73-win_amd64.whl", hash = "sha256:42f6ef9b640deb6f7d438e0a371aedd8bef6ddfde30683491b2e6f568b4e884e"}, - {file = "kiwisolver-1.4.2.tar.gz", hash = "sha256:7f606d91b8a8816be476513a77fd30abe66227039bd6f8b406c348cb0247dcc9"}, -] -matplotlib = [ - {file = "matplotlib-3.5.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:03bbb3f5f78836855e127b5dab228d99551ad0642918ccbf3067fcd52ac7ac5e"}, - {file = "matplotlib-3.5.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:49a5938ed6ef9dda560f26ea930a2baae11ea99e1c2080c8714341ecfda72a89"}, - {file = "matplotlib-3.5.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:77157be0fc4469cbfb901270c205e7d8adb3607af23cef8bd11419600647ceed"}, - {file = "matplotlib-3.5.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5844cea45d804174bf0fac219b4ab50774e504bef477fc10f8f730ce2d623441"}, - {file = "matplotlib-3.5.2-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c87973ddec10812bddc6c286b88fdd654a666080fbe846a1f7a3b4ba7b11ab78"}, - {file = "matplotlib-3.5.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4a05f2b37222319753a5d43c0a4fd97ed4ff15ab502113e3f2625c26728040cf"}, - {file = "matplotlib-3.5.2-cp310-cp310-win32.whl", hash = "sha256:9776e1a10636ee5f06ca8efe0122c6de57ffe7e8c843e0fb6e001e9d9256ec95"}, - {file = "matplotlib-3.5.2-cp310-cp310-win_amd64.whl", hash = "sha256:b4fedaa5a9aa9ce14001541812849ed1713112651295fdddd640ea6620e6cf98"}, - {file = "matplotlib-3.5.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:ee175a571e692fc8ae8e41ac353c0e07259113f4cb063b0ec769eff9717e84bb"}, - {file = "matplotlib-3.5.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2e8bda1088b941ead50caabd682601bece983cadb2283cafff56e8fcddbf7d7f"}, - {file = "matplotlib-3.5.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:9480842d5aadb6e754f0b8f4ebeb73065ac8be1855baa93cd082e46e770591e9"}, - {file = "matplotlib-3.5.2-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:6c623b355d605a81c661546af7f24414165a8a2022cddbe7380a31a4170fa2e9"}, - {file = "matplotlib-3.5.2-cp37-cp37m-win32.whl", hash = "sha256:a91426ae910819383d337ba0dc7971c7cefdaa38599868476d94389a329e599b"}, - {file = "matplotlib-3.5.2-cp37-cp37m-win_amd64.whl", hash = "sha256:c4b82c2ae6d305fcbeb0eb9c93df2602ebd2f174f6e8c8a5d92f9445baa0c1d3"}, - {file = "matplotlib-3.5.2-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:ebc27ad11df3c1661f4677a7762e57a8a91dd41b466c3605e90717c9a5f90c82"}, - {file = "matplotlib-3.5.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:5a32ea6e12e80dedaca2d4795d9ed40f97bfa56e6011e14f31502fdd528b9c89"}, - {file = "matplotlib-3.5.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:2a0967d4156adbd0d46db06bc1a877f0370bce28d10206a5071f9ecd6dc60b79"}, - {file = "matplotlib-3.5.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e2b696699386766ef171a259d72b203a3c75d99d03ec383b97fc2054f52e15cf"}, - {file = "matplotlib-3.5.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:7f409716119fa39b03da3d9602bd9b41142fab7a0568758cd136cd80b1bf36c8"}, - {file = "matplotlib-3.5.2-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:b8d3f4e71e26307e8c120b72c16671d70c5cd08ae412355c11254aa8254fb87f"}, - {file = "matplotlib-3.5.2-cp38-cp38-win32.whl", hash = "sha256:b6c63cd01cad0ea8704f1fd586e9dc5777ccedcd42f63cbbaa3eae8dd41172a1"}, - {file = "matplotlib-3.5.2-cp38-cp38-win_amd64.whl", hash = "sha256:75c406c527a3aa07638689586343f4b344fcc7ab1f79c396699eb550cd2b91f7"}, - {file = "matplotlib-3.5.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:4a44cdfdb9d1b2f18b1e7d315eb3843abb097869cd1ef89cfce6a488cd1b5182"}, - {file = "matplotlib-3.5.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:3d8e129af95b156b41cb3be0d9a7512cc6d73e2b2109f82108f566dbabdbf377"}, - {file = "matplotlib-3.5.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:364e6bca34edc10a96aa3b1d7cd76eb2eea19a4097198c1b19e89bee47ed5781"}, - {file = "matplotlib-3.5.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ea75df8e567743207e2b479ba3d8843537be1c146d4b1e3e395319a4e1a77fe9"}, - {file = "matplotlib-3.5.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:44c6436868186564450df8fd2fc20ed9daaef5caad699aa04069e87099f9b5a8"}, - {file = "matplotlib-3.5.2-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:7d7705022df2c42bb02937a2a824f4ec3cca915700dd80dc23916af47ff05f1a"}, - {file = "matplotlib-3.5.2-cp39-cp39-win32.whl", hash = "sha256:ee0b8e586ac07f83bb2950717e66cb305e2859baf6f00a9c39cc576e0ce9629c"}, - {file = "matplotlib-3.5.2-cp39-cp39-win_amd64.whl", hash = "sha256:c772264631e5ae61f0bd41313bbe48e1b9bcc95b974033e1118c9caa1a84d5c6"}, - {file = "matplotlib-3.5.2-pp37-pypy37_pp73-macosx_10_9_x86_64.whl", hash = "sha256:751d3815b555dcd6187ad35b21736dc12ce6925fc3fa363bbc6dc0f86f16484f"}, - {file = "matplotlib-3.5.2-pp37-pypy37_pp73-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:31fbc2af27ebb820763f077ec7adc79b5a031c2f3f7af446bd7909674cd59460"}, - {file = "matplotlib-3.5.2-pp37-pypy37_pp73-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:4fa28ca76ac5c2b2d54bc058b3dad8e22ee85d26d1ee1b116a6fd4d2277b6a04"}, - {file = "matplotlib-3.5.2-pp37-pypy37_pp73-win_amd64.whl", hash = "sha256:24173c23d1bcbaed5bf47b8785d27933a1ac26a5d772200a0f3e0e38f471b001"}, - {file = "matplotlib-3.5.2.tar.gz", hash = "sha256:48cf850ce14fa18067f2d9e0d646763681948487a8080ec0af2686468b4607a2"}, -] -mypy = [ - {file = "mypy-0.961-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:697540876638ce349b01b6786bc6094ccdaba88af446a9abb967293ce6eaa2b0"}, - {file = "mypy-0.961-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:b117650592e1782819829605a193360a08aa99f1fc23d1d71e1a75a142dc7e15"}, - {file = "mypy-0.961-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:bdd5ca340beffb8c44cb9dc26697628d1b88c6bddf5c2f6eb308c46f269bb6f3"}, - {file = "mypy-0.961-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:3e09f1f983a71d0672bbc97ae33ee3709d10c779beb613febc36805a6e28bb4e"}, - {file = "mypy-0.961-cp310-cp310-win_amd64.whl", hash = "sha256:e999229b9f3198c0c880d5e269f9f8129c8862451ce53a011326cad38b9ccd24"}, - {file = "mypy-0.961-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:b24be97351084b11582fef18d79004b3e4db572219deee0212078f7cf6352723"}, - {file = "mypy-0.961-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:f4a21d01fc0ba4e31d82f0fff195682e29f9401a8bdb7173891070eb260aeb3b"}, - {file = "mypy-0.961-cp36-cp36m-win_amd64.whl", hash = "sha256:439c726a3b3da7ca84a0199a8ab444cd8896d95012c4a6c4a0d808e3147abf5d"}, - {file = "mypy-0.961-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:5a0b53747f713f490affdceef835d8f0cb7285187a6a44c33821b6d1f46ed813"}, - {file = "mypy-0.961-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:0e9f70df36405c25cc530a86eeda1e0867863d9471fe76d1273c783df3d35c2e"}, - {file = "mypy-0.961-cp37-cp37m-win_amd64.whl", hash = "sha256:b88f784e9e35dcaa075519096dc947a388319cb86811b6af621e3523980f1c8a"}, - {file = "mypy-0.961-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:d5aaf1edaa7692490f72bdb9fbd941fbf2e201713523bdb3f4038be0af8846c6"}, - {file = "mypy-0.961-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:9f5f5a74085d9a81a1f9c78081d60a0040c3efb3f28e5c9912b900adf59a16e6"}, - {file = "mypy-0.961-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:f4b794db44168a4fc886e3450201365c9526a522c46ba089b55e1f11c163750d"}, - {file = "mypy-0.961-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:64759a273d590040a592e0f4186539858c948302c653c2eac840c7a3cd29e51b"}, - {file = "mypy-0.961-cp38-cp38-win_amd64.whl", hash = "sha256:63e85a03770ebf403291ec50097954cc5caf2a9205c888ce3a61bd3f82e17569"}, - {file = "mypy-0.961-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:5f1332964963d4832a94bebc10f13d3279be3ce8f6c64da563d6ee6e2eeda932"}, - {file = "mypy-0.961-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:006be38474216b833eca29ff6b73e143386f352e10e9c2fbe76aa8549e5554f5"}, - {file = "mypy-0.961-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:9940e6916ed9371809b35b2154baf1f684acba935cd09928952310fbddaba648"}, - {file = "mypy-0.961-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:a5ea0875a049de1b63b972456542f04643daf320d27dc592d7c3d9cd5d9bf950"}, - {file = "mypy-0.961-cp39-cp39-win_amd64.whl", hash = "sha256:1ece702f29270ec6af25db8cf6185c04c02311c6bb21a69f423d40e527b75c56"}, - {file = "mypy-0.961-py3-none-any.whl", hash = "sha256:03c6cc893e7563e7b2949b969e63f02c000b32502a1b4d1314cabe391aa87d66"}, - {file = "mypy-0.961.tar.gz", hash = "sha256:f730d56cb924d371c26b8eaddeea3cc07d78ff51c521c6d04899ac6904b75492"}, -] -mypy-extensions = [ - {file = "mypy_extensions-0.4.3-py2.py3-none-any.whl", hash = "sha256:090fedd75945a69ae91ce1303b5824f428daf5a028d2f6ab8a299250a846f15d"}, - {file = "mypy_extensions-0.4.3.tar.gz", hash = "sha256:2d82818f5bb3e369420cb3c4060a7970edba416647068eb4c5343488a6c604a8"}, -] -numpy = [ - {file = "numpy-1.21.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:38e8648f9449a549a7dfe8d8755a5979b45b3538520d1e735637ef28e8c2dc50"}, - {file = "numpy-1.21.1-cp37-cp37m-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:fd7d7409fa643a91d0a05c7554dd68aa9c9bb16e186f6ccfe40d6e003156e33a"}, - {file = "numpy-1.21.1-cp37-cp37m-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:a75b4498b1e93d8b700282dc8e655b8bd559c0904b3910b144646dbbbc03e062"}, - {file = "numpy-1.21.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1412aa0aec3e00bc23fbb8664d76552b4efde98fb71f60737c83efbac24112f1"}, - {file = "numpy-1.21.1-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:e46ceaff65609b5399163de5893d8f2a82d3c77d5e56d976c8b5fb01faa6b671"}, - {file = "numpy-1.21.1-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:c6a2324085dd52f96498419ba95b5777e40b6bcbc20088fddb9e8cbb58885e8e"}, - {file = "numpy-1.21.1-cp37-cp37m-win32.whl", hash = "sha256:73101b2a1fef16602696d133db402a7e7586654682244344b8329cdcbbb82172"}, - {file = "numpy-1.21.1-cp37-cp37m-win_amd64.whl", hash = "sha256:7a708a79c9a9d26904d1cca8d383bf869edf6f8e7650d85dbc77b041e8c5a0f8"}, - {file = "numpy-1.21.1-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:95b995d0c413f5d0428b3f880e8fe1660ff9396dcd1f9eedbc311f37b5652e16"}, - {file = "numpy-1.21.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:635e6bd31c9fb3d475c8f44a089569070d10a9ef18ed13738b03049280281267"}, - {file = "numpy-1.21.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:4a3d5fb89bfe21be2ef47c0614b9c9c707b7362386c9a3ff1feae63e0267ccb6"}, - {file = "numpy-1.21.1-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:8a326af80e86d0e9ce92bcc1e65c8ff88297de4fa14ee936cb2293d414c9ec63"}, - {file = "numpy-1.21.1-cp38-cp38-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:791492091744b0fe390a6ce85cc1bf5149968ac7d5f0477288f78c89b385d9af"}, - {file = "numpy-1.21.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0318c465786c1f63ac05d7c4dbcecd4d2d7e13f0959b01b534ea1e92202235c5"}, - {file = "numpy-1.21.1-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:9a513bd9c1551894ee3d31369f9b07460ef223694098cf27d399513415855b68"}, - {file = "numpy-1.21.1-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:91c6f5fc58df1e0a3cc0c3a717bb3308ff850abdaa6d2d802573ee2b11f674a8"}, - {file = "numpy-1.21.1-cp38-cp38-win32.whl", hash = "sha256:978010b68e17150db8765355d1ccdd450f9fc916824e8c4e35ee620590e234cd"}, - {file = "numpy-1.21.1-cp38-cp38-win_amd64.whl", hash = "sha256:9749a40a5b22333467f02fe11edc98f022133ee1bfa8ab99bda5e5437b831214"}, - {file = "numpy-1.21.1-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:d7a4aeac3b94af92a9373d6e77b37691b86411f9745190d2c351f410ab3a791f"}, - {file = "numpy-1.21.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:d9e7912a56108aba9b31df688a4c4f5cb0d9d3787386b87d504762b6754fbb1b"}, - {file = "numpy-1.21.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:25b40b98ebdd272bc3020935427a4530b7d60dfbe1ab9381a39147834e985eac"}, - {file = "numpy-1.21.1-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:8a92c5aea763d14ba9d6475803fc7904bda7decc2a0a68153f587ad82941fec1"}, - {file = "numpy-1.21.1-cp39-cp39-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:05a0f648eb28bae4bcb204e6fd14603de2908de982e761a2fc78efe0f19e96e1"}, - {file = "numpy-1.21.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f01f28075a92eede918b965e86e8f0ba7b7797a95aa8d35e1cc8821f5fc3ad6a"}, - {file = "numpy-1.21.1-cp39-cp39-win32.whl", hash = "sha256:88c0b89ad1cc24a5efbb99ff9ab5db0f9a86e9cc50240177a571fbe9c2860ac2"}, - {file = "numpy-1.21.1-cp39-cp39-win_amd64.whl", hash = "sha256:01721eefe70544d548425a07c80be8377096a54118070b8a62476866d5208e33"}, - {file = "numpy-1.21.1-pp37-pypy37_pp73-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:2d4d1de6e6fb3d28781c73fbde702ac97f03d79e4ffd6598b880b2d95d62ead4"}, - {file = "numpy-1.21.1.zip", hash = "sha256:dff4af63638afcc57a3dfb9e4b26d434a7a602d225b42d746ea7fe2edf1342fd"}, -] -packaging = [ - {file = "packaging-21.3-py3-none-any.whl", hash = "sha256:ef103e05f519cdc783ae24ea4e2e0f508a9c99b2d4969652eed6a2e1ea5bd522"}, - {file = "packaging-21.3.tar.gz", hash = "sha256:dd47c42927d89ab911e606518907cc2d3a1f38bbd026385970643f9c5b8ecfeb"}, -] -pathspec = [ - {file = "pathspec-0.9.0-py2.py3-none-any.whl", hash = "sha256:7d15c4ddb0b5c802d161efc417ec1a2558ea2653c2e8ad9c19098201dc1c993a"}, - {file = "pathspec-0.9.0.tar.gz", hash = "sha256:e564499435a2673d586f6b2130bb5b95f04a3ba06f81b8f895b651a3c76aabb1"}, -] -pillow = [ - {file = "Pillow-9.1.1-cp310-cp310-macosx_10_10_x86_64.whl", hash = "sha256:42dfefbef90eb67c10c45a73a9bc1599d4dac920f7dfcbf4ec6b80cb620757fe"}, - {file = "Pillow-9.1.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:ffde4c6fabb52891d81606411cbfaf77756e3b561b566efd270b3ed3791fde4e"}, - {file = "Pillow-9.1.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9c857532c719fb30fafabd2371ce9b7031812ff3889d75273827633bca0c4602"}, - {file = "Pillow-9.1.1-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:59789a7d06c742e9d13b883d5e3569188c16acb02eeed2510fd3bfdbc1bd1530"}, - {file = "Pillow-9.1.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4d45dbe4b21a9679c3e8b3f7f4f42a45a7d3ddff8a4a16109dff0e1da30a35b2"}, - {file = "Pillow-9.1.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:e9ed59d1b6ee837f4515b9584f3d26cf0388b742a11ecdae0d9237a94505d03a"}, - {file = "Pillow-9.1.1-cp310-cp310-win32.whl", hash = "sha256:b3fe2ff1e1715d4475d7e2c3e8dabd7c025f4410f79513b4ff2de3d51ce0fa9c"}, - {file = "Pillow-9.1.1-cp310-cp310-win_amd64.whl", hash = "sha256:5b650dbbc0969a4e226d98a0b440c2f07a850896aed9266b6fedc0f7e7834108"}, - {file = "Pillow-9.1.1-cp37-cp37m-macosx_10_10_x86_64.whl", hash = "sha256:0b4d5ad2cd3a1f0d1df882d926b37dbb2ab6c823ae21d041b46910c8f8cd844b"}, - {file = "Pillow-9.1.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9370d6744d379f2de5d7fa95cdbd3a4d92f0b0ef29609b4b1687f16bc197063d"}, - {file = "Pillow-9.1.1-cp37-cp37m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b761727ed7d593e49671d1827044b942dd2f4caae6e51bab144d4accf8244a84"}, - {file = "Pillow-9.1.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8a66fe50386162df2da701b3722781cbe90ce043e7d53c1fd6bd801bca6b48d4"}, - {file = "Pillow-9.1.1-cp37-cp37m-win32.whl", hash = "sha256:2b291cab8a888658d72b575a03e340509b6b050b62db1f5539dd5cd18fd50578"}, - {file = "Pillow-9.1.1-cp37-cp37m-win_amd64.whl", hash = "sha256:1d4331aeb12f6b3791911a6da82de72257a99ad99726ed6b63f481c0184b6fb9"}, - {file = "Pillow-9.1.1-cp38-cp38-macosx_10_10_x86_64.whl", hash = "sha256:8844217cdf66eabe39567118f229e275f0727e9195635a15e0e4b9227458daaf"}, - {file = "Pillow-9.1.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:b6617221ff08fbd3b7a811950b5c3f9367f6e941b86259843eab77c8e3d2b56b"}, - {file = "Pillow-9.1.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:20d514c989fa28e73a5adbddd7a171afa5824710d0ab06d4e1234195d2a2e546"}, - {file = "Pillow-9.1.1-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:088df396b047477dd1bbc7de6e22f58400dae2f21310d9e2ec2933b2ef7dfa4f"}, - {file = "Pillow-9.1.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:53c27bd452e0f1bc4bfed07ceb235663a1df7c74df08e37fd6b03eb89454946a"}, - {file = "Pillow-9.1.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:3f6c1716c473ebd1649663bf3b42702d0d53e27af8b64642be0dd3598c761fb1"}, - {file = "Pillow-9.1.1-cp38-cp38-win32.whl", hash = "sha256:c67db410508b9de9c4694c57ed754b65a460e4812126e87f5052ecf23a011a54"}, - {file = "Pillow-9.1.1-cp38-cp38-win_amd64.whl", hash = "sha256:f054b020c4d7e9786ae0404278ea318768eb123403b18453e28e47cdb7a0a4bf"}, - {file = "Pillow-9.1.1-cp39-cp39-macosx_10_10_x86_64.whl", hash = "sha256:c17770a62a71718a74b7548098a74cd6880be16bcfff5f937f900ead90ca8e92"}, - {file = "Pillow-9.1.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:f3f6a6034140e9e17e9abc175fc7a266a6e63652028e157750bd98e804a8ed9a"}, - {file = "Pillow-9.1.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f372d0f08eff1475ef426344efe42493f71f377ec52237bf153c5713de987251"}, - {file = "Pillow-9.1.1-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:09e67ef6e430f90caa093528bd758b0616f8165e57ed8d8ce014ae32df6a831d"}, - {file = "Pillow-9.1.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:66daa16952d5bf0c9d5389c5e9df562922a59bd16d77e2a276e575d32e38afd1"}, - {file = "Pillow-9.1.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:d78ca526a559fb84faaaf84da2dd4addef5edb109db8b81677c0bb1aad342601"}, - {file = "Pillow-9.1.1-cp39-cp39-win32.whl", hash = "sha256:55e74faf8359ddda43fee01bffbc5bd99d96ea508d8a08c527099e84eb708f45"}, - {file = "Pillow-9.1.1-cp39-cp39-win_amd64.whl", hash = "sha256:7c150dbbb4a94ea4825d1e5f2c5501af7141ea95825fadd7829f9b11c97aaf6c"}, - {file = "Pillow-9.1.1-pp37-pypy37_pp73-macosx_10_10_x86_64.whl", hash = "sha256:769a7f131a2f43752455cc72f9f7a093c3ff3856bf976c5fb53a59d0ccc704f6"}, - {file = "Pillow-9.1.1-pp37-pypy37_pp73-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:488f3383cf5159907d48d32957ac6f9ea85ccdcc296c14eca1a4e396ecc32098"}, - {file = "Pillow-9.1.1-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0b525a356680022b0af53385944026d3486fc8c013638cf9900eb87c866afb4c"}, - {file = "Pillow-9.1.1-pp38-pypy38_pp73-macosx_10_10_x86_64.whl", hash = "sha256:6e760cf01259a1c0a50f3c845f9cad1af30577fd8b670339b1659c6d0e7a41dd"}, - {file = "Pillow-9.1.1-pp38-pypy38_pp73-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a4165205a13b16a29e1ac57efeee6be2dfd5b5408122d59ef2145bc3239fa340"}, - {file = "Pillow-9.1.1-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:937a54e5694684f74dcbf6e24cc453bfc5b33940216ddd8f4cd8f0f79167f765"}, - {file = "Pillow-9.1.1-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:baf3be0b9446a4083cc0c5bb9f9c964034be5374b5bc09757be89f5d2fa247b8"}, - {file = "Pillow-9.1.1.tar.gz", hash = "sha256:7502539939b53d7565f3d11d87c78e7ec900d3c72945d4ee0e2f250d598309a0"}, -] -platformdirs = [ - {file = "platformdirs-2.5.2-py3-none-any.whl", hash = "sha256:027d8e83a2d7de06bbac4e5ef7e023c02b863d7ea5d079477e722bb41ab25788"}, - {file = "platformdirs-2.5.2.tar.gz", hash = "sha256:58c8abb07dcb441e6ee4b11d8df0ac856038f944ab98b7be6b27b2a3c7feef19"}, -] -pltpublish = [ - {file = "pltpublish-1.0.0-py3-none-any.whl", hash = "sha256:cca8e88d272bdac81130a8fef64363a72982b7d816f1cca3f30727d05df1faa0"}, - {file = "pltpublish-1.0.0.tar.gz", hash = "sha256:b13b2e20fb524a6eb062f3af06aa3d8b33ac711db41bc30ce22ea89181d6c926"}, -] -pluggy = [ - {file = "pluggy-1.0.0-py2.py3-none-any.whl", hash = "sha256:74134bbf457f031a36d68416e1509f34bd5ccc019f0bcc952c7b909d06b37bd3"}, - {file = "pluggy-1.0.0.tar.gz", hash = "sha256:4224373bacce55f955a878bf9cfa763c1e360858e330072059e10bad68531159"}, -] -py = [ - {file = "py-1.11.0-py2.py3-none-any.whl", hash = "sha256:607c53218732647dff4acdfcd50cb62615cedf612e72d1724fb1a0cc6405b378"}, - {file = "py-1.11.0.tar.gz", hash = "sha256:51c75c4126074b472f746a24399ad32f6053d1b34b68d2fa41e558e6f4a98719"}, -] -pyparsing = [ - {file = "pyparsing-3.0.9-py3-none-any.whl", hash = "sha256:5026bae9a10eeaefb61dab2f09052b9f4307d44aee4eda64b309723d8d206bbc"}, - {file = "pyparsing-3.0.9.tar.gz", hash = "sha256:2b020ecf7d21b687f219b71ecad3631f644a47f01403fa1d1036b0c6416d70fb"}, -] -pytest = [ - {file = "pytest-7.1.2-py3-none-any.whl", hash = "sha256:13d0e3ccfc2b6e26be000cb6568c832ba67ba32e719443bfe725814d3c42433c"}, - {file = "pytest-7.1.2.tar.gz", hash = "sha256:a06a0425453864a270bc45e71f783330a7428defb4230fb5e6a731fde06ecd45"}, -] -python-dateutil = [ - {file = "python-dateutil-2.8.2.tar.gz", hash = "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86"}, - {file = "python_dateutil-2.8.2-py2.py3-none-any.whl", hash = "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"}, -] -setuptools-scm = [ - {file = "setuptools_scm-6.4.2-py3-none-any.whl", hash = "sha256:acea13255093849de7ccb11af9e1fb8bde7067783450cee9ef7a93139bddf6d4"}, - {file = "setuptools_scm-6.4.2.tar.gz", hash = "sha256:6833ac65c6ed9711a4d5d2266f8024cfa07c533a0e55f4c12f6eff280a5a9e30"}, -] -six = [ - {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, - {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, -] -tomli = [ - {file = "tomli-2.0.1-py3-none-any.whl", hash = "sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc"}, - {file = "tomli-2.0.1.tar.gz", hash = "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f"}, -] -torch = [ - {file = "torch-1.11.0-cp310-cp310-manylinux1_x86_64.whl", hash = "sha256:62052b50fffc29ca7afc0c04ef8206b6f1ca9d10629cb543077e12967e8d0398"}, - {file = "torch-1.11.0-cp310-cp310-manylinux2014_aarch64.whl", hash = "sha256:866bfba29ac98dec35d893d8e17eaec149d0ac7a53be7baae5c98069897db667"}, - {file = "torch-1.11.0-cp310-cp310-win_amd64.whl", hash = "sha256:951640fb8db308a59d9b510e7d1ad910aff92913323bbe4bc75435347ddd346d"}, - {file = "torch-1.11.0-cp310-none-macosx_10_9_x86_64.whl", hash = "sha256:5d77b5ece78fdafa5c7f42995ff9474399d22571cd6b2de21a5d666306a2ff8c"}, - {file = "torch-1.11.0-cp310-none-macosx_11_0_arm64.whl", hash = "sha256:b5a38682769b544c875ecc34bcb81fbad5c922139b61319aacffcfd8a32f528c"}, - {file = "torch-1.11.0-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:f82d77695a60626f2b7382d85bc566de8a6b3e50d32080755abc040db802e419"}, - {file = "torch-1.11.0-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:b96654d42566080a134e784705f33f8536b3b95b5dcde357ed7879b1692a5f78"}, - {file = "torch-1.11.0-cp37-cp37m-win_amd64.whl", hash = "sha256:8ee7c2e8d7f7020d5bfbc1bb91b9591044c26bbd0cee5e4f694cfd7ed8649260"}, - {file = "torch-1.11.0-cp37-none-macosx_10_9_x86_64.whl", hash = "sha256:6860b1d1bf0bb0b67a6bd47f85a0e4c825b518eea13b5d6101999dbbcbd5bc0c"}, - {file = "torch-1.11.0-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:4322aa29f50da7f404db06cdf30896ea67b09f673af4a985afc7162bc897864d"}, - {file = "torch-1.11.0-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:e4d2e0ddd652f30e94cff750220324ec45705d4ecc69658f773b3cb1c7a28dd0"}, - {file = "torch-1.11.0-cp38-cp38-win_amd64.whl", hash = "sha256:34ce5ea4d8d85da32cdbadb50d4585106901e9f8a3527991daa70c13a09de1f7"}, - {file = "torch-1.11.0-cp38-none-macosx_10_9_x86_64.whl", hash = "sha256:0ccc85cd06227a3edf809e2c795fd5762c3d4e8a38b5c9f744c6e7cf841361bb"}, - {file = "torch-1.11.0-cp38-none-macosx_11_0_arm64.whl", hash = "sha256:c1554e49d74f1b2c3e7202d77056ba2dd7465437585bac64062b580f714a44e9"}, - {file = "torch-1.11.0-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:58c7814502b1c129a650d7092033bbb0bbd64faf1a7941631aaa1aeaddc37570"}, - {file = "torch-1.11.0-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:831cf588f01dda9409e75576741d2823453990dee2983d670f2584b37a01adf7"}, - {file = "torch-1.11.0-cp39-cp39-win_amd64.whl", hash = "sha256:44a1d02fd20f827f0f36dc26fdcfc45e793806a6ad52769a22260655a77a4369"}, - {file = "torch-1.11.0-cp39-none-macosx_10_9_x86_64.whl", hash = "sha256:50fd9bf85c578c871c28f1cb0ace9dfc6024401c7f399b174fb0f370899f4454"}, - {file = "torch-1.11.0-cp39-none-macosx_11_0_arm64.whl", hash = "sha256:0e48af66ad755f0f9c5f2664028a414f57c49d6adc37e77e06fe0004da4edb61"}, -] -tqdm = [ - {file = "tqdm-4.64.0-py2.py3-none-any.whl", hash = "sha256:74a2cdefe14d11442cedf3ba4e21a3b84ff9a2dbdc6cfae2c34addb2a14a5ea6"}, - {file = "tqdm-4.64.0.tar.gz", hash = "sha256:40be55d30e200777a307a7585aee69e4eabb46b4ec6a4b4a5f2d9f11e7d5408d"}, -] -typed-ast = [ - {file = "typed_ast-1.5.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:669dd0c4167f6f2cd9f57041e03c3c2ebf9063d0757dc89f79ba1daa2bfca9d4"}, - {file = "typed_ast-1.5.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:211260621ab1cd7324e0798d6be953d00b74e0428382991adfddb352252f1d62"}, - {file = "typed_ast-1.5.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:267e3f78697a6c00c689c03db4876dd1efdfea2f251a5ad6555e82a26847b4ac"}, - {file = "typed_ast-1.5.4-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:c542eeda69212fa10a7ada75e668876fdec5f856cd3d06829e6aa64ad17c8dfe"}, - {file = "typed_ast-1.5.4-cp310-cp310-win_amd64.whl", hash = "sha256:a9916d2bb8865f973824fb47436fa45e1ebf2efd920f2b9f99342cb7fab93f72"}, - {file = "typed_ast-1.5.4-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:79b1e0869db7c830ba6a981d58711c88b6677506e648496b1f64ac7d15633aec"}, - {file = "typed_ast-1.5.4-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a94d55d142c9265f4ea46fab70977a1944ecae359ae867397757d836ea5a3f47"}, - {file = "typed_ast-1.5.4-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:183afdf0ec5b1b211724dfef3d2cad2d767cbefac291f24d69b00546c1837fb6"}, - {file = "typed_ast-1.5.4-cp36-cp36m-win_amd64.whl", hash = "sha256:639c5f0b21776605dd6c9dbe592d5228f021404dafd377e2b7ac046b0349b1a1"}, - {file = "typed_ast-1.5.4-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:cf4afcfac006ece570e32d6fa90ab74a17245b83dfd6655a6f68568098345ff6"}, - {file = "typed_ast-1.5.4-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ed855bbe3eb3715fca349c80174cfcfd699c2f9de574d40527b8429acae23a66"}, - {file = "typed_ast-1.5.4-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:6778e1b2f81dfc7bc58e4b259363b83d2e509a65198e85d5700dfae4c6c8ff1c"}, - {file = "typed_ast-1.5.4-cp37-cp37m-win_amd64.whl", hash = "sha256:0261195c2062caf107831e92a76764c81227dae162c4f75192c0d489faf751a2"}, - {file = "typed_ast-1.5.4-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:2efae9db7a8c05ad5547d522e7dbe62c83d838d3906a3716d1478b6c1d61388d"}, - {file = "typed_ast-1.5.4-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:7d5d014b7daa8b0bf2eaef684295acae12b036d79f54178b92a2b6a56f92278f"}, - {file = "typed_ast-1.5.4-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:370788a63915e82fd6f212865a596a0fefcbb7d408bbbb13dea723d971ed8bdc"}, - {file = "typed_ast-1.5.4-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:4e964b4ff86550a7a7d56345c7864b18f403f5bd7380edf44a3c1fb4ee7ac6c6"}, - {file = "typed_ast-1.5.4-cp38-cp38-win_amd64.whl", hash = "sha256:683407d92dc953c8a7347119596f0b0e6c55eb98ebebd9b23437501b28dcbb8e"}, - {file = "typed_ast-1.5.4-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:4879da6c9b73443f97e731b617184a596ac1235fe91f98d279a7af36c796da35"}, - {file = "typed_ast-1.5.4-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:3e123d878ba170397916557d31c8f589951e353cc95fb7f24f6bb69adc1a8a97"}, - {file = "typed_ast-1.5.4-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ebd9d7f80ccf7a82ac5f88c521115cc55d84e35bf8b446fcd7836eb6b98929a3"}, - {file = "typed_ast-1.5.4-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:98f80dee3c03455e92796b58b98ff6ca0b2a6f652120c263efdba4d6c5e58f72"}, - {file = "typed_ast-1.5.4-cp39-cp39-win_amd64.whl", hash = "sha256:0fdbcf2fef0ca421a3f5912555804296f0b0960f0418c440f5d6d3abb549f3e1"}, - {file = "typed_ast-1.5.4.tar.gz", hash = "sha256:39e21ceb7388e4bb37f4c679d72707ed46c2fbf2a5609b8b8ebc4b067d977df2"}, -] -typing-extensions = [ - {file = "typing_extensions-4.2.0-py3-none-any.whl", hash = "sha256:6657594ee297170d19f67d55c05852a874e7eb634f4f753dbd667855e07c1708"}, - {file = "typing_extensions-4.2.0.tar.gz", hash = "sha256:f1c24655a0da0d1b67f07e17a5e6b2a105894e6824b92096378bb3668ef02376"}, -] -vose = [] -zipp = [ - {file = "zipp-3.8.0-py3-none-any.whl", hash = "sha256:c4f6e5bbf48e74f7a38e7cc5b0480ff42b0ae5178957d564d18932525d5cf099"}, - {file = "zipp-3.8.0.tar.gz", hash = "sha256:56bf8aadb83c24db6c4b577e13de374ccfb67da2078beba1d037c17980bf43ad"}, -] diff --git a/pyproject.toml b/pyproject.toml index 505678c2..71ac3e88 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,28 +1,40 @@ -[tool.poetry] +[project] name = "synth" version = "0.1.0" description = "Automated Synthesis Framework" -authors = ["Théo Matricon ", "Nathanaël Fijalkow "] -license = "MIT" +authors = [ + { name = "Théo Matricon", email = "theomatricon@gmail.com" }, + { name = "Nathanaël Fijalkow", email = "nathanael.fijalkow@gmail.com" }, +] +requires-python = ">=3.10" readme = "README.md" -repository = "https://github.com/nathanael-fijalkow/AutoSynth" +license = "MIT" +dependencies = [ + "torch>=1.13.1", + "tensorboard>=0", + "cython>=0.29", + "numpy>=1.22", + "vose", + "colorama>=0.4.4", + "tqdm>=4.63.0", + "matplotlib>=3.5.1", + "pltpublish>=0.1.0", +] + +[project.urls] +Repository = "https://github.com/SynthesisLab/DeepSynth2" -[tool.poetry.dependencies] -python = ">=3.7.1" -torch = ">=1.9.0" -cython = ">=0.29" -numpy = ">=1.18" -vose = { git = "https://github.com/Theomat/vose.git"} -colorama = ">=0.4.4" -tqdm = ">=4.63.0" -matplotlib = ">=3.5.1" -pltpublish = ">=0.1.0" +[dependency-groups] +dev = [ + "ruff>= 0.4.3", + "pytest>=7.2.0", + "mypy>=0.910", +] -[tool.poetry.dev-dependencies] -black = "^22.3.0" -pytest = ">=6.2" -mypy = ">=0.910" +[tool.uv.sources] +vose = { git = "https://github.com/Theomat/vose.git" } +grape = { git = "https://github.com/SynthesisLab/grape.git" } [build-system] -requires = ["poetry-core>=1.0.0"] -build-backend = "poetry.core.masonry.api" \ No newline at end of file +requires = ["hatchling"] +build-backend = "hatchling.build" \ No newline at end of file diff --git a/synth/__init__.py b/synth/__init__.py index adb353da..fc801447 100644 --- a/synth/__init__.py +++ b/synth/__init__.py @@ -1,2 +1,3 @@ +# import torch from synth.task import Task, Dataset from synth.specification import TaskSpecification, PBE, NLP, NLPBE, Example diff --git a/synth/filter/__init__.py b/synth/filter/__init__.py new file mode 100644 index 00000000..d527870a --- /dev/null +++ b/synth/filter/__init__.py @@ -0,0 +1,15 @@ +""" +Module that contains anything relevant to pruning +""" + +from synth.filter.filter import Filter, UnionFilter, IntersectionFilter +from synth.filter.dfta_filter import DFTAFilter +from synth.filter.obs_eq_filter import ObsEqFilter +from synth.filter.local_stateless_filter import LocalStatelessFilter +from synth.filter.syntactic_filter import ( + UseAllVariablesFilter, + FunctionFilter, + SyntacticFilter, + SetFilter, +) +from synth.filter.constraints import add_constraints, add_dfta_constraints diff --git a/synth/filter/constraints/__init__.py b/synth/filter/constraints/__init__.py new file mode 100644 index 00000000..da2dace8 --- /dev/null +++ b/synth/filter/constraints/__init__.py @@ -0,0 +1,2 @@ +from synth.filter.constraints.ttcfg_constraints import add_constraints +from synth.filter.constraints.dfta_constraints import add_dfta_constraints diff --git a/synth/filter/constraints/dfta_constraints.py b/synth/filter/constraints/dfta_constraints.py new file mode 100644 index 00000000..930c6f0a --- /dev/null +++ b/synth/filter/constraints/dfta_constraints.py @@ -0,0 +1,358 @@ +from itertools import product +from typing import ( + Any, + Callable, + Dict, + Iterable, + List as TList, + Optional, + Set, + Tuple, + TypeVar, + Union, +) + +import tqdm +from synth.filter.constraints.parsing import ( + Token, + TokenAllow, + TokenAnything, + TokenAtLeast, + TokenAtMost, + TokenFunction, + TokenForceSubtree, + TokenForbidSubtree, + parse_specification, +) +from synth.syntax.automata.tree_automaton import DFTA +from synth.syntax.grammars.det_grammar import DerivableProgram +from synth.syntax.grammars.cfg import CFG +from synth.syntax.type_system import Type + + +U = TypeVar("U") +V = TypeVar("V") + + +def __cfg2dfta__( + grammar: CFG, +) -> DFTA[Tuple[Type, int], DerivableProgram]: + StateT = Tuple[Type, int] + dfta_rules: Dict[Tuple[DerivableProgram, Tuple[StateT, ...]], StateT] = {} + max_depth = grammar.max_program_depth() + all_cases: Dict[ + Tuple[int, Tuple[Type, ...]], Set[Tuple[Tuple[Type, int], ...]] + ] = {} + if max_depth == -1: + for S in grammar.rules: + for P in grammar.rules[S]: + args = grammar.rules[S][P][0] + if len(args) == 0: + dfta_rules[(P, ())] = (P.type, 0) + else: + key = (len(args), tuple([arg[0] for arg in args])) + if key not in all_cases: + all_cases[key] = set([tuple([(arg[0], 0) for arg in args])]) + for nargs in all_cases[key]: + dfta_rules[(P, nargs)] = ( + S[0], + 0, + ) + else: + for S in grammar.rules: + for P in grammar.rules[S]: + args = grammar.rules[S][P][0] + if len(args) == 0: + dfta_rules[(P, ())] = (P.type, 0) + else: + key = (len(args), tuple([arg[0] for arg in args])) + if key not in all_cases: + all_cases[key] = set( + [ + tuple(x) + for x in product( + *[ + [(arg[0], j) for j in range(max_depth)] + for arg in args + ] + ) + ] + ) + for nargs in all_cases[key]: + new_depth = max(i for _, i in nargs) + 1 + if new_depth >= max_depth: + continue + dfta_rules[(P, nargs)] = ( + S[0], + new_depth, + ) + r = grammar.type_request.returns() + dfta = DFTA( + dfta_rules, {(r, x) for x in range(max_depth)} if max_depth > 0 else {(r, 0)} + ) + dfta.reduce() + return dfta + + +def __augment__( + grammar: DFTA[Tuple[Type, U], DerivableProgram], +) -> DFTA[Tuple[Type, Tuple[U, int]], DerivableProgram]: + new_dfta = DFTA( + { + (P, tuple((arg[0], (arg[1], 0)) for arg in args)): (dst[0], (dst[1], 0)) + for (P, args), dst in grammar.rules.items() + }, + {(t, (q, 0)) for t, q in grammar.finals}, + ) + return new_dfta + + +def __get_tuple_val__(t: Tuple[U, int], index: int) -> int: + if index == -1: + return t[1] + return __get_tuple_val__(t[0], index + 1) # type: ignore + + +def __tuple_len__(t: Tuple[U, ...]) -> int: + if isinstance(t, tuple): + return 1 + __tuple_len__(t[0]) # type: ignore + return 1 + + +def __count__( + grammar: DFTA[Tuple[Type, U], DerivableProgram], + count: int, + to_count: TList[DerivableProgram], + at_most: bool, +) -> DFTA[Tuple[Type, Tuple[U, int]], DerivableProgram]: + dfta = __augment__(grammar) + maxi = count + (1 if at_most else 0) + # We duplicate rules in order to count + all_alternatives = lambda s: [(s[0], (s[1][0], i)) for i in range(maxi + 1)] + for (P, args), dst in list(dfta.rules.items()): + possibles = [all_alternatives(arg) for arg in args] + for new_args in product(*possibles): + total = sum(arg[1][1] for arg in new_args) + if P in to_count: + total += 1 + dfta.rules[(P, new_args)] = (dst[0], (dst[1][0], min(total, maxi))) + # We duplicate finals as well + for q in list(dfta.finals): + for new_q in all_alternatives(q): + dfta.finals.add(new_q) + return dfta + + +def __tag__( + grammar: DFTA[Tuple[Type, U], DerivableProgram], + check: Callable[ + [ + DerivableProgram, + Tuple[Tuple[Type, Tuple[U, int]], ...], + Tuple[Type, Tuple[U, int]], + ], + bool, + ], +) -> DFTA[Tuple[Type, Tuple[U, int]], DerivableProgram]: + out_grammar = __augment__(grammar) + tag_state = lambda s: (s[0], (s[1][0], 1)) + added = set() + # Whenever the pattern is correct we tag with 1 + for (P, p_args), p_dst in list(out_grammar.rules.items()): + if check(P, p_args, p_dst): + out_grammar.rules[(P, p_args)] = tag_state(p_dst) + added.add(p_dst) + # We also need to be able to consume the new tagged states like the others + for (P, p_args), p_dst in list(out_grammar.rules.items()): + if any(arg in added for arg in p_args): + possibles = [ + [arg] if arg not in added else [arg, tag_state(arg)] for arg in p_args + ] + for new_args in product(*possibles): + out_grammar.rules[(P, new_args)] = p_dst + for q in added.intersection(out_grammar.finals): + out_grammar.finals.add(tag_state(q)) + return out_grammar + + +def __filter__( + grammar: DFTA[Tuple[Type, U], DerivableProgram], + check: Callable[ + [ + DerivableProgram, + Tuple[Tuple[Type, Tuple[U, int]], ...], + Tuple[Type, Tuple[U, int]], + ], + bool, + ], +) -> DFTA[Tuple[Type, U], DerivableProgram]: + out_grammar: DFTA[Tuple[Type, U], DerivableProgram] = DFTA( + {}, {q for q in grammar.finals} + ) + # Whenever the pattern is correct we tag with 1 + for (P, p_args), p_dst in grammar.rules.items(): + if check(P, p_args, p_dst): # type: ignore + out_grammar.rules[(P, p_args)] = p_dst + out_grammar.finals = out_grammar.finals.intersection(out_grammar.rules.values()) + return out_grammar + + +def __match__( + args: Tuple[Tuple[Type, Tuple[U, int]], ...], + dst: Tuple[Type, Tuple[U, int]], + primitive_check: int, + indices: TList[int], + should_check: TList[bool], +) -> bool: + if __get_tuple_val__(dst[1], -primitive_check - 2) != 1: + return False + for arg, state_index, check in zip(args, indices, should_check): + if not check: + continue + if __get_tuple_val__(arg[1], -state_index - 2) != 1: + return False + + return True + + +def __process__( + grammar: DFTA[Tuple[Type, U], DerivableProgram], + token: Token, + local: bool, + level: int = 0, +) -> DFTA[Tuple[Type, Any], DerivableProgram]: + # print("\t" * level, "processing:", token) + if isinstance(token, TokenFunction): + has_check = [] + grammar = __process__(grammar, token.function, local, level + 1) + lengths = [__tuple_len__(list(grammar.finals)[0][1])] # type: ignore + for arg in token.args: + grammar = __process__(grammar, arg, local, level + 1) + cur_len = __tuple_len__(list(grammar.finals)[0][1]) # type: ignore + has_check.append(cur_len - lengths[-1] > 0) + lengths.append(cur_len) + + indices = [lengths[-1] - l for l in lengths] + primitive_check = indices.pop(0) + out_grammar = __tag__( + grammar, + lambda _, args, dst: __match__( + args, dst, primitive_check, indices, has_check + ), + ) + + elif isinstance(token, TokenAllow): + # We need to augment grammar to tell that this we detected this the correct thing + allowed_P = token.allowed + out_grammar = __tag__(grammar, lambda P, _, __: P in allowed_P) + + elif isinstance(token, (TokenAtMost, TokenAtLeast)): + out_grammar = __count__( + grammar, + token.count, + token.to_count, + isinstance(token, TokenAtMost), + ) + allowed = [] + if isinstance(token, TokenAtMost): + allowed = list(range(token.count + 1)) + else: + allowed = [token.count] + out_grammar = __tag__( + out_grammar, # type: ignore + lambda _, __, state: state[1][0][-1] in allowed, # type: ignore + ) + + elif isinstance(token, TokenForbidSubtree): + return __process__(grammar, TokenAtMost(token.forbidden, 0), local, level) + elif isinstance(token, TokenForceSubtree): + return __process__(grammar, TokenAtLeast(token.forced, 1), local, level) + elif isinstance(token, TokenAnything): + return grammar + else: + assert False, f"Not implemented token: {token}({type(token)}) [level={level}]" + + if level == 0: + if not local: + out_grammar.finals = {q for q in out_grammar.finals if q[1][-1] == 1} + else: + assert isinstance( + token, TokenFunction + ), f"Unsupported topmost token for local constraint" + out_grammar = __filter__( + out_grammar, + lambda P, _, dst: P not in token.function.allowed or dst[1][-1] == 1, + ) + # out_grammar.finals = out_grammar.finals.intersection(set(out_grammar.rules.keys())) + return out_grammar + + +def add_dfta_constraints( + current_grammar: Union[CFG, DFTA[Tuple[Type, Any], DerivableProgram]], + constraints: Iterable[str], + sketch: Optional[str] = None, + progress: bool = True, +) -> DFTA[Tuple[Type, Set], DerivableProgram]: + """ + Add constraints to the specified grammar. + + If sketch is True the constraints are for sketches otherwise they are pattern like. + If progress is set to True use a tqdm progress bar. + + """ + constraint_plus = [(int("var" in c), c) for c in constraints] + constraint_plus.sort(reverse=True) + parsed_constraints = [ + parse_specification(constraint, current_grammar) # type: ignore + for _, constraint in constraint_plus + ] + dfta = None + pbar = None + if progress: + pbar = tqdm.tqdm( + total=len(parsed_constraints) + int(sketch is not None), + desc="constraints", + smoothing=1, + ) + base = ( + __cfg2dfta__(current_grammar) + if isinstance(current_grammar, CFG) + else current_grammar + ) + for constraint in parsed_constraints: + # Skip empty allow since it means the primitive was not recognized + if isinstance(constraint, TokenAnything) or ( + isinstance(constraint, TokenFunction) + and len(constraint.function.allowed) == 0 + ): + if pbar: + pbar.update(1) + continue + a = __process__(base, constraint, True) + if dfta is None: + dfta = a + else: + a.reduce() + dfta = dfta.read_product(a.minimise()) + dfta.reduce() + dfta = dfta.minimise() # type: ignore + if pbar: + pbar.update(1) + if sketch is not None: + a = __process__( + base, + parse_specification(sketch, current_grammar), # type: ignore + False, + ) + if dfta is None: + dfta = a + else: + a.reduce() + dfta = dfta.read_product(a.minimise()) # type: ignore + if pbar: + pbar.update(1) + dfta.reduce() + dfta = dfta.minimise() # type: ignore + if pbar: + pbar.close() + return dfta or base diff --git a/synth/filter/constraints/parsing.py b/synth/filter/constraints/parsing.py new file mode 100644 index 00000000..c88f673a --- /dev/null +++ b/synth/filter/constraints/parsing.py @@ -0,0 +1,262 @@ +from dataclasses import dataclass +from typing import ( + Set, + Tuple, + List as TList, + Type, + TypeVar, + Union, +) + +from synth.syntax.automata.tree_automaton import DFTA +from synth.syntax.grammars.det_grammar import DerivableProgram +from synth.syntax.grammars.ttcfg import TTCFG +from synth.syntax.program import Primitive, Variable + +# ======================================================================================== +# SYMBOLS +# ======================================================================================== +SYMBOL_ANYTHING = "_" +SYMBOL_FORBIDDEN = "^" +SYMBOL_SEPARATOR = "," +SYMBOL_SUBTREE = ">" +SYMBOL_AGGREGATOR = "#" + + +# ======================================================================================== +# TOKENS +# ======================================================================================== +class Token: + def __repr__(self) -> str: + return str(self) + + +class TokenAnything(Token): + def __repr__(self) -> str: + return str(self) + + def __str__(self) -> str: + return "Any" + + def __eq__(self, o: object) -> bool: + return isinstance(o, TokenAnything) + + +@dataclass +class TokenAllow(Token): + allowed: TList[DerivableProgram] + + def __repr__(self) -> str: + return str(self) + + def __str__(self) -> str: + return f"Allow ({self.allowed})" + + def __eq__(self, o: object) -> bool: + return isinstance(o, TokenAllow) and o.allowed == self.allowed + + +@dataclass +class TokenForbidSubtree(Token): + forbidden: TList[DerivableProgram] + + def __repr__(self) -> str: + return str(self) + + def __str__(self) -> str: + return f"ForbidSubtree ({self.forbidden})" + + def __eq__(self, o: object) -> bool: + return isinstance(o, TokenForbidSubtree) and o.forbidden == self.forbidden + + +@dataclass +class TokenForceSubtree(Token): + forced: TList[DerivableProgram] + + def __repr__(self) -> str: + return str(self) + + def __str__(self) -> str: + return f"ForceSubtree ({self.forced})" + + def __eq__(self, o: object) -> bool: + return isinstance(o, TokenForceSubtree) and o.forced == self.forced + + +@dataclass +class TokenAtMost(Token): + to_count: TList[DerivableProgram] + count: int + + def __repr__(self) -> str: + return str(self) + + def __str__(self) -> str: + return f"AtMost {self.count} ({self.to_count})" + + def __eq__(self, o: object) -> bool: + return ( + isinstance(o, TokenAtMost) + and o.to_count == self.to_count + and o.count == self.count + ) + + +@dataclass +class TokenAtLeast(Token): + to_count: TList[DerivableProgram] + count: int + + def __repr__(self) -> str: + return str(self) + + def __str__(self) -> str: + return f"AtLeast {self.count} ({self.to_count})" + + def __eq__(self, o: object) -> bool: + return ( + isinstance(o, TokenAtLeast) + and o.to_count == self.to_count + and o.count == self.count + ) + + +@dataclass +class TokenFunction(Token): + function: TokenAllow + args: TList[Token] + + def __repr__(self) -> str: + return str(self) + + def __str__(self) -> str: + return f"Func f={self.function} args=({self.args})" + + def __eq__(self, o: object) -> bool: + return ( + isinstance(o, TokenFunction) + and o.function == self.function + and o.args == self.args + ) + + +# ======================================================================================== +# PARSING +# ======================================================================================== +def __next_level__(string: str, start: str, end: str) -> int: + level = 0 + for i, el in enumerate(string): + if el == start: + level += 1 + if el == end: + level -= 1 + if level == 0: + return i + return i + + +def __parse_next_word__(program: str) -> Tuple[str, int]: + if program[0] in ["("]: + end = __next_level__(program, "(", ")") + else: + end = program.index(" ") - 1 if " " in program else len(program) - 1 + return program[: end + 1], end + 2 + + +def __str_to_derivable_program__( + word: str, primitives_used: Set[Primitive], variables: TList[Variable] +) -> TList[DerivableProgram]: + all_primitives = sorted(primitives_used, key=lambda p: p.primitive, reverse=True) + if word == SYMBOL_ANYTHING: + out: TList[DerivableProgram] = all_primitives # type: ignore + out += variables + return out + word = word.strip("(){}") + allowed = set( + [word] if not SYMBOL_SEPARATOR in word else word.split(SYMBOL_SEPARATOR) + ) + primitives: TList[DerivableProgram] = [ + P for P in all_primitives if P.primitive in allowed + ] + svar = sorted(variables, key=lambda x: x.variable) + for el in allowed: + if el.startswith("var"): + varno = int(el[3:]) + primitives.append(svar[varno]) + return primitives + + +def __interpret_word__( + word: str, primitives_used: Set[Primitive], variables: TList[Variable] +) -> Token: + word = word.strip() + if word.startswith(SYMBOL_FORBIDDEN): + forbidden = set(word[1:].split(SYMBOL_SEPARATOR)) + out: TList[DerivableProgram] = [ + P for P in primitives_used if P.primitive not in forbidden + ] + out += [V for V in variables if str(V) not in forbidden] + if len(out) == len(primitives_used) + len(variables): + return TokenAnything() + return TokenAllow(out) + elif word.startswith(SYMBOL_SUBTREE): + cst: Union[Type[TokenForceSubtree], Type[TokenForbidSubtree]] = ( + TokenForceSubtree + ) + str_content = word[1:] + if str_content.startswith(SYMBOL_FORBIDDEN): + str_content = str_content[1:] + cst = TokenForbidSubtree + return cst( + __str_to_derivable_program__(str_content, primitives_used, variables) + ) + elif word == SYMBOL_ANYTHING: + return TokenAnything() + elif word.startswith(SYMBOL_AGGREGATOR): + word = word[1:].replace(" ", "") + end_index = max(word.find("<="), word.find(">=")) + most = word[end_index] == "<" + considered = word[:end_index] + content = __str_to_derivable_program__(considered, primitives_used, variables) + count = int(word[len(considered) + 2 :]) + if most: + return TokenAtMost(content, count) + else: + return TokenAtLeast(content, count) + return TokenAllow(__str_to_derivable_program__(word, primitives_used, variables)) + + +U = TypeVar("U") + + +def parse_specification( + spec: str, grammar: Union[TTCFG, DFTA[Tuple[Type, U], DerivableProgram]] +) -> Token: + primitives_used = set() + variables = [] + if isinstance(grammar, TTCFG): + primitives_used = grammar.primitives_used() + variables = grammar.variables() + else: + primitives_used = {x for x in grammar.alphabet if isinstance(x, Primitive)} + variables = list(x for x in grammar.alphabet if isinstance(x, Variable)) + spec = spec.replace("\n", "").strip(")(") + index = 0 + elements = [] + while index < len(spec): + spec = spec[index:] + word, index = __parse_next_word__(spec) + if word.startswith("("): + token = parse_specification(word, grammar) + else: + token = __interpret_word__(word, primitives_used, variables) + elements.append(token) + assert len(elements) > 0 + if isinstance(elements[0], TokenAllow): + first = elements.pop(0) + if all(isinstance(el, TokenAnything) for el in elements): + return TokenAnything() + return TokenFunction(first, elements) # type: ignore + assert len(elements) == 1 + return elements[0] diff --git a/synth/filter/constraints/ttcfg_constraints.py b/synth/filter/constraints/ttcfg_constraints.py new file mode 100644 index 00000000..067cc25f --- /dev/null +++ b/synth/filter/constraints/ttcfg_constraints.py @@ -0,0 +1,563 @@ +from collections import defaultdict +from dataclasses import dataclass, field +from typing import ( + Any, + Dict, + Generic, + Iterable, + List as TList, + Optional, + Set, + Tuple, + TypeVar, +) + +import tqdm +from synth.filter.constraints.parsing import ( + Token, + TokenAllow, + TokenAtLeast, + TokenAtMost, + TokenFunction, + TokenForceSubtree, + TokenForbidSubtree, + parse_specification, +) +from synth.syntax.automata.dfa import DFA +from synth.syntax.grammars.det_grammar import DerivableProgram +from synth.syntax.grammars.ttcfg import TTCFG +from synth.syntax.type_system import Type + + +# ======================================================================================== +# PARSING +# ======================================================================================== + + +U = TypeVar("U") +V = TypeVar("V") + +State = Tuple[Type, Tuple[Tuple[U, int], V]] +Info = TList[Tuple[Type, Tuple[U, int]]] +DerElment = Tuple[Type, Tuple[U, int]] + + +@dataclass +class ProcessState: + new_terminal_no: int = field(default=1) + duplicate_from: Dict[State, State] = field(default_factory=lambda: {}) + + +@dataclass(frozen=True) +class Path(Generic[U, V]): + predecessors: TList[ + Tuple[Tuple[Type, Tuple[Tuple[U, int], V]], DerivableProgram] + ] = field(default_factory=lambda: []) + + def __hash__(self) -> int: + return hash(tuple(self.predecessors)) + + def __str__(self) -> str: + if len(self) > 0: + end = f"->{self.predecessors[-1][1]}" + return "->".join([f"{S[1]}" for S, P in self.predecessors]) + end + return "-" + + def __repr__(self) -> str: + return self.__str__() + + def __len__(self) -> int: + return len(self.predecessors) + + def last(self) -> Tuple[Tuple[Type, Tuple[Tuple[U, int], V]], DerivableProgram]: + return self.predecessors[-1] + + def fix_last(self, S: Tuple[Type, Tuple[Tuple[U, int], V]]) -> None: + self.predecessors[-1] = (S, self.predecessors[-1][1]) + + def next( + self, S: Tuple[Type, Tuple[Tuple[U, int], V]], P: DerivableProgram + ) -> "Path[U, V]": + return Path(self.predecessors + [(S, P)]) + + +Save = Tuple[ + Tuple[Type, Tuple[Tuple[U, int], V]], TList[Tuple[DerivableProgram, int, V]], Info +] + + +def __make_save__( + grammar: TTCFG[Tuple[U, int], V], path: Path[U, V], S: State, info: Info +) -> Save: + hist: TList[Tuple[DerivableProgram, int, V]] = [] + tuples = path.predecessors[:] + nexts = [ctx for ctx, _ in path.predecessors[1:]] + [S] + save = (path.predecessors[0][0], hist, info) + while tuples: + S, P = tuples.pop(0) + nextS = nexts.pop(0) + der = (nextS[0], nextS[1][0]) + derlst, _ = grammar.rules[S][P] + found = False + for i, SS in enumerate(derlst): + if SS == der: + found = True + hist.append((P, i, nextS[1][1])) + break + assert found + return save + + +def __restore_save__( + grammar: TTCFG[Tuple[U, int], V], save: Save +) -> Tuple[Path[U, V], State, Info]: + start, hist, info = save + path: Path[U, V] = Path() + while hist: + P, i, v = hist.pop(0) + derlst, _ = grammar.rules[start][P] + der = derlst[i] + next_S = (der[0], (der[1], v)) + path = path.next(start, P) + start = next_S + return path, start, info + + +def __dfa_start_from_any__( + grammar: TTCFG[Tuple[U, int], V], + relevant: TList[Tuple[Path, Tuple[Type, Tuple[Tuple[U, int], V]], Info]], + state: ProcessState, +) -> Tuple[ + DFA[int, Tuple[Type, Tuple[Tuple[U, int], V]]], TList[Tuple[Path, State, Info]] +]: + # Create DFA that self loops + rules: Dict[int, Dict[Tuple[Type, Tuple[Tuple[U, int], V]], int]] = { + 0: {S: 0 for S in grammar.rules}, + 1: {}, + } + # Now for only the relevant states we will need to not self loops + relevant_cpy = relevant[:] + # Redirection is to fix the path towards S if it has been changed during iteration + redirections: Dict[State, State] = {} + # Already done avoids conflicts when there are multiple identical derivations where only the from state changes + already_done: Dict[Tuple[State, Tuple[Type, Tuple[U, int]]], State] = {} + relevant = [] + while relevant_cpy: + path, S, info = relevant_cpy.pop() + if len(path) > 0: + parent_S, parent_P = path.last() + if parent_S in redirections: + parent_S = redirections[parent_S] + path.fix_last(parent_S) + key = (parent_S, (S[0], S[1][0])) + else: + key = None + if key in already_done: + tmpS = already_done[key] + new_S = (tmpS[0], (tmpS[1][0], S[1][1])) + grammar.rules[new_S] = { + P: (grammar.rules[S][P][0][:], grammar.rules[S][P][1]) + for P in grammar.rules[S] + } + elif S in redirections: + new_S = redirections[S] + else: + new_S = __duplicate__(grammar, S, state) + if key is not None: + already_done[key] = new_S + redirections[S] = new_S + if len(path) > 0: + __redirect__(grammar, parent_S, parent_P, S, new_S) + else: + grammar.start = new_S + rules[0][new_S] = 1 + outcomes = grammar.possible_outcomes_after(new_S) + if len(path) > 0: + parent_S, parent_P = path.last() + derlst, _ = grammar.rules[parent_S][parent_P] + der = (new_S[0], new_S[1][0]) + + added, found = False, False + nextS = der + for SS in derlst: + if SS == der: + found = True + elif found: + nextS = SS + added = True + break + if not added: + nextS = info[0] + for v in outcomes: + rules[1][(nextS[0], (nextS[1], v))] = 0 + + relevant.append((path, new_S, info)) + return DFA(0, rules), relevant + + +def __count_dfa__( + grammar: TTCFG[Tuple[U, int], V], + dfa: DFA[int, Tuple[Type, Tuple[Tuple[U, int], V]]], + to_count: TList[DerivableProgram], + count: int, +) -> DFA[int, Tuple[Tuple[Type, Tuple[Tuple[U, int], V]], DerivableProgram]]: + all_primitives: TList[DerivableProgram] = list(grammar.primitives_used()) + all_primitives += grammar.variables() + + rules: Dict[int, Dict[DerivableProgram, int]] = {} + start_count = count + while count > 0: + rules[count] = { + P: count - 1 if P in to_count else count for P in all_primitives + } + count -= 1 + # count == 0 + rules[count] = {P: count for P in all_primitives if P not in to_count} + counter = DFA(start_count, rules).map_states(lambda i: i + len(dfa.states) - 1) + new_rules: Dict[ + int, Dict[Tuple[Tuple[Type, Tuple[Tuple[U, int], V]], DerivableProgram], int] + ] = {} + + for u in dfa.rules: + new_rules[u] = {} + for S in dfa.rules[u]: + for P in grammar.rules[S]: + new_rules[u][(S, P)] = 0 if dfa.rules[u][S] == 0 else counter.start + + for u in counter.rules: + new_rules[u] = {} + for S in grammar.rules: + for P in counter.rules[u]: + new_rules[u][(S, P)] = ( + 0 + if S in dfa.rules[1] and dfa.rules[1][S] == 0 + else counter.rules[u][P] + ) + return DFA(0, new_rules) + + +def __preprocess_grammar__(grammar: TTCFG[U, V]) -> TTCFG[Tuple[U, int], V]: + """ + Goes from U to Tuple[U, int] by just making tuples of (U, 0) + """ + new_rules: Dict[State, Dict[DerivableProgram, Tuple[TList[DerElment], V]]] = {} + for S in grammar.rules: + u, v = S[1] + SS = (S[0], ((u, 0), v)) + new_rules[SS] = {} + for P in grammar.rules[S]: + derlst, state = grammar.rules[S][P] + new_rules[SS][P] = ([(t, (d, 0)) for t, d in derlst], state) + return TTCFG( + (grammar.start[0], ((grammar.start[1][0], 0), grammar.start[1][1])), + new_rules, + clean=False, + ) + + +def __redirect__( + grammar: TTCFG[Tuple[U, int], V], + parent_S: State, + parent_P: DerivableProgram, + old: State, + new: State, +) -> None: + """ + Redirect parent_S -> parent_P : ...old... + to parent_S -> parent_P : ...new... + """ + # Redirect original derivation + derlst, state = grammar.rules[parent_S][parent_P] + old_element = (old[0], old[1][0]) + new_element = (new[0], new[1][0]) + new_derlst = [el if el != old_element else new_element for el in derlst] + grammar.rules[parent_S][parent_P] = new_derlst, state + + +def __duplicate__( + grammar: TTCFG[Tuple[U, int], V], S: State, state: ProcessState +) -> State: + """ + Duplicate S and return the copy new_S + """ + up, v = S[1] + new_S = (S[0], ((up[0], state.new_terminal_no), v)) + state.new_terminal_no += 1 + if S in state.duplicate_from: + # print("found origin using:", state.duplicate_from[S], "instead of", S) + # print( + # "\t", + # len(grammar.rules[state.duplicate_from[S]]), + # " instead of", + # len(grammar.rules[S]), + # ) + S = state.duplicate_from[S] + state.duplicate_from[new_S] = S + grammar.rules[new_S] = { + P: (grammar.rules[S][P][0][:], grammar.rules[S][P][1]) for P in grammar.rules[S] + } + return new_S + + +def __forbid_subtree__( + grammar: TTCFG[Tuple[U, int], V], + parent_S: State, + parent_P: DerivableProgram, + S: State, + forbidden: TList[DerivableProgram], + state: ProcessState, + done: Optional[Set[State]] = None, + info: Optional[TList[Tuple[Type, Tuple[U, int]]]] = None, +) -> None: + new_S = __duplicate__(grammar, S, state) + __redirect__(grammar, parent_S, parent_P, S, new_S) + done = done or set() + for P in sorted(grammar.rules[S].keys(), key=lambda P: str(P)): + if P in forbidden: + # Delete forbidden + del grammar.rules[new_S][P] + else: + # Recursive calls + info, nextS = grammar.derive(info or grammar.start_information(), S, P) + # Nothing to do + if nextS not in grammar.rules: + continue + __forbid_subtree__(grammar, S, P, nextS, forbidden, state, done, info) + + +def __process__( + grammar: TTCFG[Tuple[U, int], V], + token: Token, + sketch: bool, + relevant: Optional[TList[Tuple[Path, State, Info]]] = None, + level: int = 0, + state: Optional[ProcessState] = None, +) -> Tuple[ + TTCFG[Tuple[U, int], Any], + Dict[Path[U, V], TList[Set[V]]], + TList[Tuple[Path, State, Info]], +]: + assert not isinstance( + token, (TokenAtLeast, TokenForceSubtree) + ), "Unsupported constraint for TTCFG(safe, det)" + out_grammar: TTCFG[Tuple[U, int], Any] = grammar + state = state or ProcessState() + possible_new_states: Dict[Path[U, V], TList[Set[V]]] = defaultdict(list) + # print( + # "\t" * level, + # "processing:", + # token, + # "len(relevant)=", + # len(relevant) if relevant else 0, + # ) + # if relevant is not None: + # for path, S, info in relevant: + # print("\t" * level, " path:", path, "S:", S) + if isinstance(token, TokenFunction): + if relevant is None: + # Compute relevant depending on sketch or not + if sketch: + relevant = [(Path(), grammar.start, grammar.start_information())] + grammar, _, __ = __process__( + grammar, token.function, sketch, relevant, level, state + ) + relevant = [] + relevant.append((Path(), grammar.start, grammar.start_information())) + else: + relevant = [] + for S in grammar.rules: + for P in grammar.rules[S]: + if P in token.function.allowed: + relevant.append((Path(), S, grammar.start_information())) + break + else: + saves = [ + __make_save__(grammar, path, S, info) for path, S, info in relevant + ] + # So here we have correct paths + grammar, _, __ = __process__( + grammar, token.function, sketch, relevant, level, state + ) + # However we have restricted the possible functions so we renamed our paths + # We need to fix that + new_relevant = [__restore_save__(grammar, save) for save in saves] + + relevant = new_relevant + # Go from relevant to first argument context + arg_relevant: TList[Tuple[Path, State, Info]] = [] + # print("\t" * level, "building arg relevant") + for path, S, info in relevant: + # print("\t" * level, " path:", path) + for P in grammar.rules[S]: + if P in token.function.allowed: + new_path = path.next(S, P) + new_info, new_S = grammar.derive(info, S, P) + arg_relevant.append((new_path, new_S, new_info)) + + for argno, arg in enumerate(token.args): + grammar, possible_states, new_relevant = __process__( + grammar, arg, sketch, arg_relevant, level + 1, state + ) + if isinstance(arg, TokenAtMost): + arg_relevant = new_relevant + next_relevant: TList[Tuple[Path, State, Info]] = [] + for path, S, info in arg_relevant: + pS, pP = path.last() + derlst = grammar.rules[pS][pP][0] + if argno + 1 >= len(derlst): + continue + t, u = derlst[argno + 1] + for states_list in possible_states[path]: + for v in states_list: + new_S = (t, (u, v)) + next_relevant.append((path, new_S, info)) + arg_relevant = next_relevant + out_grammar = grammar + elif isinstance(token, TokenAllow): + assert relevant is not None + relevant_cpy = relevant[:] + # Redirection is to fix the path towards S if it has been changed during iteration + redirections: Dict[State, State] = {} + # Already done avoids conflicts when there are multiple identical derivations where only the from state changes + already_done: Dict[Tuple[State, Tuple[Type, Tuple[U, int]]], State] = {} + relevant = [] + while relevant_cpy: + path, S, info = relevant_cpy.pop() + if len(path) > 0: + parent_S, parent_P = path.last() + if parent_S in redirections: + parent_S = redirections[parent_S] + if parent_P not in grammar.rules[parent_S]: + # The path that we took was actually deleted by the constraints + continue + path.fix_last(parent_S) + # print( + # "\t" * (level + 1), "parent:", parent_S , "->", parent_P, "=>", S) + key = (parent_S, (S[0], S[1][0])) + else: + key = None + should_del = True + if key in already_done: + # print("\t" * (level + 1), "copy") + tmpS = already_done[key] + new_S = (tmpS[0], (tmpS[1][0], S[1][1])) + grammar.rules[new_S] = { + P: (grammar.rules[S][P][0][:], grammar.rules[S][P][1]) + for P in grammar.rules[S] + } + elif S in redirections: + # print("\t" * (level + 1), "redirection") + new_S = redirections[S] + should_del = False + else: + # print("\t" * (level + 1), "duplicate") + new_S = __duplicate__(grammar, S, state) + if key is not None: + already_done[key] = new_S + redirections[S] = new_S + if should_del: + # Delete forbidden + for P in list(grammar.rules[new_S].keys()): + if P not in token.allowed: + # print("\t" * (level + 3), "del:", new_S, "->", P) + del grammar.rules[new_S][P] + if len(path) > 0: + __redirect__(grammar, parent_S, parent_P, S, new_S) + else: + grammar.start = new_S + + # print("\t" * (level + 1), "from:", S, "to", new_S) + relevant.append((path, new_S, info)) + elif isinstance(token, TokenAtMost): + assert relevant is not None + # Create a DFA that recognises the path + detector, relevant = __dfa_start_from_any__(grammar, relevant, state) + # Create a DFA that counts primitives + final_dfa = __count_dfa__(grammar, detector, token.to_count, token.count) + out_grammar = grammar * final_dfa + # Relevant contains old paths since we augmented the type of the grammar + # We need to create the new relevant + new_relevant = [] + # easier to go from saves we only need to map the starting one + ssaves = [ + (path, __make_save__(grammar, path, S, info)) for path, S, info in relevant + ] + for or_path, save in ssaves: + start, hist, info = save + assert len(hist) == len(or_path) + # Let's hope info is useless and we can just not care + found = 0 + for SS in out_grammar.rules: + old_S = (SS[0], (SS[1][0], SS[1][1][0])) + if old_S == start: + try: + path, SSS, Sinfo = __restore_save__( + out_grammar, # type: ignore + (SS, hist[:], info), + ) + assert len(path) == len( + or_path + ), f"or_path={or_path} path={path}" + new_relevant.append((path, SSS, Sinfo)) + found += 1 + except KeyError: + pass + relevant = new_relevant + elif isinstance(token, TokenForbidSubtree): + assert relevant is not None + for path, S, info in relevant: + parent_S, parent_P = path.last() + __forbid_subtree__(grammar, parent_S, parent_P, S, token.forbidden, state) + # Compute valid possible new states + assert relevant is not None + for path, S, info in relevant: + all_new_states = grammar.possible_outcomes_after(S) + possible_new_states[path].append(all_new_states) + return out_grammar, possible_new_states, relevant + + +def add_constraints( + current_grammar: TTCFG[U, V], + constraints: Iterable[str], + sketch: Optional[str] = None, + progress: bool = True, +) -> TTCFG[Tuple[U, int], Any]: + """ + Add constraints to the specified grammar. + + If sketch is True the constraints are for sketches otherwise they are pattern like. + If progress is set to True use a tqdm progress bar. + + """ + constraint_plus = [(int("var" in c), c) for c in constraints] + constraint_plus.sort(reverse=True) + parsed_constraints = [ + parse_specification(constraint, current_grammar) + for _, constraint in constraint_plus + ] + preprocessed = __preprocess_grammar__(current_grammar) + + if progress: + pbar = tqdm.tqdm( + total=len(parsed_constraints) + int(sketch is not None), + desc="constraints", + smoothing=1, + ) + state = ProcessState() + for constraint in parsed_constraints: + preprocessed = __process__(preprocessed, constraint, False, state=state)[0] + if progress: + pbar.update(1) + if sketch is not None: + preprocessed = __process__( + preprocessed, + parse_specification(sketch, current_grammar), + True, + state=state, + )[0] + if progress: + pbar.update(1) + preprocessed.clean() + if progress: + pbar.close() + return preprocessed diff --git a/synth/filter/dfta_filter.py b/synth/filter/dfta_filter.py new file mode 100644 index 00000000..d5d86ea8 --- /dev/null +++ b/synth/filter/dfta_filter.py @@ -0,0 +1,50 @@ +from typing import Dict, Generic, TypeVar, Optional + +from synth.filter.filter import Filter +from synth.syntax.automata.tree_automaton import DFTA +from synth.syntax.grammars.grammar import DerivableProgram +from synth.syntax.program import Function, Program, Lambda + +V = TypeVar("V") + + +class DFTAFilter(Filter, Generic[V]): + """ + Filters out programs depending on the given DFTA. + + If accepting_dfta then rejects programs that are not in the language of the DFTA. + If not accepting_dfta, rejects programs that are in the language of the DFTA. + + """ + + def __init__( + self, dfta: DFTA[V, DerivableProgram], accepting_dfta: bool = True + ) -> None: + self.dfta = dfta + self._cache: Dict[Program, V] = {} + self.accepting_dfta = accepting_dfta + + def _get_prog_state(self, prog: Program) -> Optional[V]: + state = self._cache.get(prog, None) + if state is not None: + return state + if isinstance(prog, Function): + fun = prog.function + args = tuple(self._get_prog_state(arg) for arg in prog.arguments) + state = self.dfta.read(fun, args) # type: ignore + if state is not None: + self._cache[prog] = state + return state + elif isinstance(prog, Lambda): + assert False, "Not implemented" + else: + state = self.dfta.read(prog, ()) # type: ignore + if state is not None: + self._cache[prog] = state + return state + + def accept(self, obj: Program) -> bool: + return (self._get_prog_state(obj) is not None) == self.accepting_dfta + + def reset_cache(self) -> None: + self._cache.clear() diff --git a/synth/filter/filter.py b/synth/filter/filter.py new file mode 100644 index 00000000..a94b77b0 --- /dev/null +++ b/synth/filter/filter.py @@ -0,0 +1,79 @@ +from abc import ABC, abstractmethod +from typing import Generic, TypeVar + + +T = TypeVar("T") + + +class Filter(ABC, Generic[T]): + @abstractmethod + def accept(self, obj: T) -> bool: + """ + Accepts objects that should be kept. + """ + pass + + def reject(self, obj: T) -> bool: + """ + Rejects objects that should NOT be kept. + """ + return not self.accept(obj) + + def __and__(self, other: "Filter[T]") -> "IntersectionFilter[T]": + return self.intersection(other) + + def intersection(self, other: "Filter[T]") -> "IntersectionFilter[T]": + if isinstance(other, IntersectionFilter): + return other.intersection(self) + elif isinstance(self, IntersectionFilter): + if isinstance(other, IntersectionFilter): + return IntersectionFilter(*self.filters, *other.filters) + return IntersectionFilter(*self.filters, other) + else: + return IntersectionFilter(self, other) + + def __or__(self, other: "Filter[T]") -> "UnionFilter[T]": + return self.union(other) + + def union(self, other: "Filter[T]") -> "UnionFilter[T]": + if isinstance(other, UnionFilter): + return other.union(self) + elif isinstance(self, UnionFilter): + if isinstance(other, UnionFilter): + return UnionFilter(*self.filters, *other.filters) + return UnionFilter(*self.filters, other) + else: + return UnionFilter(self, other) + + def __neg__(self) -> "Filter[T]": + return self.complementary() + + def complementary(self) -> "Filter[T]": + return NegFilter(self) + + +class NegFilter(Filter, Generic[T]): + def __init__(self, filter: Filter[T]) -> None: + self.filter = filter + + def accept(self, obj: T) -> bool: + return not self.filter.accept(obj) + + def complementary(self) -> "Filter[T]": + return self.filter + + +class UnionFilter(Filter, Generic[T]): + def __init__(self, *filters: Filter[T]) -> None: + self.filters = list(filters) + + def accept(self, obj: T) -> bool: + return any(p.accept(obj) for p in self.filters) + + +class IntersectionFilter(Filter, Generic[T]): + def __init__(self, *filters: Filter[T]) -> None: + self.filters = list(filters) + + def accept(self, obj: T) -> bool: + return all(p.accept(obj) for p in self.filters) diff --git a/synth/filter/local_stateless_filter.py b/synth/filter/local_stateless_filter.py new file mode 100644 index 00000000..ca731bc8 --- /dev/null +++ b/synth/filter/local_stateless_filter.py @@ -0,0 +1,35 @@ +from typing import Callable, Dict, Generic, TypeVar + +from synth.filter.filter import Filter +from synth.syntax.program import Function, Program, Primitive + +V = TypeVar("V") + + +class LocalStatelessFilter(Filter, Generic[V]): + def __init__(self, should_reject: Dict[str, Callable]) -> None: + self.should_reject = should_reject + + def accept(self, program: Program) -> bool: + accepted = True + if isinstance(program, Function): + fun: Primitive = program.function # type: ignore + rejects = self.should_reject.get(fun.primitive, None) + accepted = rejects is None or not rejects(*program.arguments) + return accepted + + +def commutative_rejection(p1: Program, p2: Program) -> bool: + """ + Rejection filter to have unique programs for a commutative binary operator + """ + return hash(p1) <= hash(p2) + + +def reject_functions(p: Program, *function_names: str) -> bool: + """ + Rejects any function whose name is in the parameters + """ + if isinstance(p, Function): + return p.function.primitive in function_names # type: ignore + return False diff --git a/synth/filter/obs_eq_filter.py b/synth/filter/obs_eq_filter.py new file mode 100644 index 00000000..27446328 --- /dev/null +++ b/synth/filter/obs_eq_filter.py @@ -0,0 +1,39 @@ +from collections import defaultdict +from typing import Any, Dict, List, Tuple + +from synth.filter.filter import Filter +from synth.semantic.evaluator import Evaluator +from synth.syntax.program import Program +from synth.syntax.type_system import Type + + +class ObsEqFilter(Filter): + def __init__(self, evaluator: Evaluator, inputs_list: List[List[Any]]) -> None: + self.evaluator = evaluator + self.inputs_list = inputs_list + self._cache: Dict[Type, Dict[Tuple[Any, ...], Program]] = defaultdict(dict) + + def _eval(self, prog: Program) -> bool: + """ + Returns True iff the prog is unique wrt to outputs + """ + outputs = None + for inputs in self.inputs_list: + out = self.evaluator.eval(prog, inputs) + if out is None: + return False + elif isinstance(out, List): + out = tuple(out) + outputs = (outputs, out) + original = self._cache[prog.type].get(outputs) # type: ignore + if original is not None and hash(original) != hash(prog): + return False + else: + self._cache[prog.type][outputs] = prog # type: ignore + return True + + def accept(self, obj: Program) -> bool: + return self._eval(obj) + + def reset_cache(self) -> None: + self._cache.clear() diff --git a/synth/pruning/syntactic_pruner.py b/synth/filter/syntactic_filter.py similarity index 76% rename from synth/pruning/syntactic_pruner.py rename to synth/filter/syntactic_filter.py index b56d0f59..5cec5a16 100644 --- a/synth/pruning/syntactic_pruner.py +++ b/synth/filter/syntactic_filter.py @@ -1,23 +1,21 @@ -from typing import Any, Callable, Dict, List, Literal, Set, Tuple, Union -from dataclasses import dataclass, field +from typing import Callable, Dict, Set, Tuple -from synth.pruning.pruner import Pruner -from synth.syntax.dsl import DSL -from synth.syntax.program import Function, Primitive, Program, Variable +from synth.filter.filter import Filter +from synth.syntax.program import Function, Primitive, Program from synth.syntax.type_system import Arrow, Type -SyntacticPruner = Pruner[Tuple[Type, Program]] +SyntacticFilter = Filter[Tuple[Type, Program]] -class UseAllVariablesPruner(SyntacticPruner): +class UseAllVariablesFilter(SyntacticFilter): def __init__(self) -> None: super().__init__() self._cached_variables_set: Dict[Type, Set[int]] = {} def __get_var_set__(self, treq: Type) -> Set[int]: if treq not in self._cached_variables_set: - if isinstance(treq, Arrow): + if treq.is_instance(Arrow): self._cached_variables_set[treq] = set(range(len(treq.arguments()))) else: self._cached_variables_set[treq] = set() @@ -30,7 +28,7 @@ def accept(self, obj: Tuple[Type, Program]) -> bool: return prog.used_variables() == target -class FunctionPruner(SyntacticPruner): +class FunctionFilter(SyntacticFilter): def __init__(self, is_useless: Dict[str, Callable]) -> None: super().__init__() self.is_useless = is_useless @@ -50,7 +48,7 @@ def accept(self, obj: Tuple[Type, Program]) -> bool: return True -class SetPruner(SyntacticPruner): +class SetFilter(SyntacticFilter): def __init__(self, forbidden: Set[Program]) -> None: super().__init__() self.forbidden = forbidden diff --git a/synth/generation/__init__.py b/synth/generation/__init__.py index 89d4e99c..3db1c636 100644 --- a/synth/generation/__init__.py +++ b/synth/generation/__init__.py @@ -1,6 +1,7 @@ """ Module that contains anything relevant to the generation """ + from synth.generation.sampler import ( Sampler, RequestSampler, diff --git a/synth/generation/sampler.py b/synth/generation/sampler.py index 49816dff..0783a606 100644 --- a/synth/generation/sampler.py +++ b/synth/generation/sampler.py @@ -14,7 +14,7 @@ import copy import numpy as np -import vose +from synth.utils.vose_polyfill import Sampler as VoseSampler from synth.syntax.type_system import List, Type @@ -53,7 +53,7 @@ def __init__( filled_probabilities = probabilites else: filled_probabilities = [1 / len(self.lexicon) for _ in lexicon] - self.sampler = vose.Sampler(np.asarray(filled_probabilities), seed=seed) + self.sampler = VoseSampler(np.asarray(filled_probabilities), seed=seed) def sample(self, **kwargs: Any) -> U: index: int = self.sampler.sample() @@ -104,7 +104,7 @@ def __init__( if not isinstance(probabilities[0], tuple): correct_prob = [(i + 1, p) for i, p in enumerate(probabilities)] # type: ignore self._length_mapping = [n for n, _ in correct_prob] - self.sampler = vose.Sampler( + self.sampler = VoseSampler( np.array([p for _, p in correct_prob]), seed=seed ) @@ -117,14 +117,13 @@ def __gen_length__(self, type: Type) -> int: def sample_for(self, type: Type, **kwargs: Any) -> Union[TList, U]: assert self.max_depth < 0 or type.depth() <= self.max_depth - if isinstance(type, List): + if type.is_instance(List): + element_type: Type = type.types[0] # type: ignore sampler: Sampler = self - if not isinstance(type.element_type, List): + if not element_type.is_instance(List): sampler = self.element_sampler length: int = self.__gen_length__(type) - return [ - sampler.sample(type=type.element_type, **kwargs) for _ in range(length) - ] + return [sampler.sample(type=element_type, **kwargs) for _ in range(length)] else: return self.element_sampler.sample(type=type, **kwargs) diff --git a/synth/library/__init__.py b/synth/library/__init__.py new file mode 100644 index 00000000..f78a1ec7 --- /dev/null +++ b/synth/library/__init__.py @@ -0,0 +1 @@ +from synth.library.learning import learn, make_score_probabilistic, score_description diff --git a/synth/library/learning.py b/synth/library/learning.py new file mode 100644 index 00000000..85d367c0 --- /dev/null +++ b/synth/library/learning.py @@ -0,0 +1,483 @@ +from collections import defaultdict +from dataclasses import dataclass +from typing import Callable, Dict, Generator, List, Optional, Set, Tuple, Union + +import tqdm + +from synth.syntax.program import Function, Primitive, Program, Variable +from synth.syntax.type_system import Type + +import numpy as np + + +_Graph = Tuple[ + Dict[int, Program], + Dict[int, List[int]], + Dict[Union[Primitive, Variable], List[int]], + Dict[int, int], + Dict[int, int], +] + + +def __prim__(p: Program) -> Primitive: + if isinstance(p, Function): + return p.function # type: ignore + else: + return p # type: ignore + + +@dataclass +class _PartialTree: + occurences: List[int] + occurences_vertices: List[Set[int]] + structure: Dict[int, List[int]] + parents: Dict[int, Tuple[int, int]] + + def num_occurences(self) -> int: + return len(self.occurences) + + def size(self) -> int: + return len(self.structure) + + def partial_copy(self) -> "_PartialTree": + return _PartialTree( + self.occurences[:], + [], + {k: v[:] for k, v in self.structure.items()}, + {k: (v[0], v[1]) for k, v in self.parents.items()}, + ) + + def unique_repr(self, graph: _Graph, occurence_index: int) -> Tuple: + """ + Compute a unique hashable representation for set membership queries + """ + vertices, edges = graph[0], graph[1] + start = self.occurences[occurence_index] + todo: List[Tuple[Optional[int], Optional[int]]] = [(start, 0)] + out: Tuple = (None,) + while todo: + real_vertex, local_vertex = todo.pop() + if real_vertex is None or local_vertex is None: + out = (None, out) + continue + out = (str(__prim__(vertices[real_vertex])), out) + for i, local in enumerate(self.structure[local_vertex]): + if local >= 0: + todo.append((edges[real_vertex][i], local)) + else: + todo.append((None, None)) + return out + + def path(self, local_vertex: int) -> List[int]: + path = [] + current = local_vertex + while current != 0: + parent, index = self.parents[current] + path.append(index) + current = parent + return path + + def follow_path_in_occurence( + self, graph: _Graph, occurence_index: int, path: List[int] + ) -> int: + """ + Compute vertex index in occurence in graph following the path from the start + """ + return self.follow_path(graph, self.occurences[occurence_index], path) + + def follow_path(self, graph: _Graph, start: int, path: List[int]) -> int: + """ + Compute global end vertex from start following path + """ + edges = graph[1] + i = 0 + while i < len(path): + index = path[-i - 1] + local_edges = edges[start] + if len(local_edges) <= index: + return -1 + start = edges[start][index] + i += 1 + return start + + def all_vertices_for_occurence( + self, graph: _Graph, occurence_index: int + ) -> Set[int]: + edges = graph[1] + out: Set[int] = set() + stack: List[Tuple[int, int]] = [(self.occurences[occurence_index], 0)] + while stack: + glbl, lcl = stack.pop() + out.add(glbl) + outgoing = self.structure.get(lcl, []) + for i, el in enumerate(outgoing): + local_edges = edges[glbl] + if len(local_edges) <= i: + break + stack.append((local_edges[i], el)) + + return out + + def add_link( + self, + graph: _Graph, + local_parent: int, + local_child_no: int, + occurence_index: int, + ) -> Tuple[List[int], Program]: + # Add in structure + j = len(self.structure) + self.structure[local_parent][local_child_no] = j + self.parents[j] = (local_parent, local_child_no) + # Match with reality + path = self.path(j) + new_real_vertex = self.follow_path_in_occurence(graph, occurence_index, path) + if new_real_vertex == -1: + return [], graph[0][0] + vertices = graph[0] + program = vertices[new_real_vertex] + # Finish structure + self.structure[j] = [-1 for _ in program.type.arguments()] + return path, program + + def expansions( + self, graph: _Graph, done: Set[Tuple] + ) -> Generator["_PartialTree", None, None]: + vertices = graph[0] + for vertex, edges in self.structure.items(): + for i, edge in enumerate(edges): + if edge < 0: + for k in range(len(self.occurences)): + next = self.partial_copy() + # Add link + path, program = next.add_link(graph, vertex, i, k) + # If path is empty we have a type mismatch + if len(path) == 0: + continue + # Check unique + r = next.unique_repr(graph, k) + if r not in done: + done.add(r) + else: + continue + + # Update occurences + next_occurences = [] + to_add = [] + target_prim = __prim__(program) + for z in range(len(self.occurences)): + real_vertex = next.follow_path_in_occurence(graph, z, path) + if ( + real_vertex >= 0 + and __prim__(vertices[real_vertex]) == target_prim + ): + next_occurences.append(self.occurences[z]) + next.occurences_vertices.append( + self.occurences_vertices[z].copy() + ) + to_add.append(real_vertex) + + next.occurences = next_occurences + + for x in next.__disambiguity__(graph, to_add): + yield x + + def __disambiguity__( + self, graph: _Graph, to_add: List[int], start: int = 0 + ) -> Generator["_PartialTree", None, None]: + vertex2tree = graph[-2] + # Check for intersection between intersections + inside = [True for _ in self.occurences] + for i in range(start, len(self.occurences)): + if not inside[i]: + continue + tree_i = vertex2tree[self.occurences[i]] + for j in range(i + 1, len(self.occurences)): + if not inside[j]: + continue + if vertex2tree[self.occurences[j]] != tree_i: + continue + if ( + to_add[i] in self.occurences_vertices[j] + or to_add[j] in self.occurences_vertices[j] + ): + other = self.partial_copy() + other.occurences = [ + x + for z, x in enumerate(other.occurences) + if inside[z] and z != i + ] + other.occurences_vertices = [ + x + for z, x in enumerate(self.occurences_vertices) + if inside[z] and z != i + ] + for x in other.__disambiguity__(graph, to_add, i + 1): + yield x + inside[j] = False + # Finally add elements + for i in range(len(self.occurences)): + if inside[i]: + self.occurences_vertices[i].add(to_add[i]) + # Filter + self.occurences = [x for z, x in enumerate(self.occurences) if inside[z]] + self.occurences_vertices = [ + x for z, x in enumerate(self.occurences_vertices) if inside[z] + ] + + yield self + + def string(self, graph: _Graph) -> str: + vertices, edges = graph[0], graph[1] + out = "" + todo: List[Tuple[Optional[int], Optional[int]]] = [(self.occurences[0], 0)] + close_parenthesis: List[int] = [] + while todo: + real, current = todo.pop(0) + if current is None or real is None: + out += "_" + close_parenthesis[-1] -= 1 + if close_parenthesis[-1] == 0: + out += ")" + else: + out += " " + continue + fun = any(arg >= 0 for arg in self.structure[current]) + if fun: + out += "(" + close_parenthesis.append(len(self.structure[current]) + 1) + for i, arg in enumerate(self.structure[current]): + if arg >= 0: + todo.insert(i, (edges[real][i], arg)) + else: + todo.insert(i, (None, None)) + out += str(__prim__(vertices[real])) + if close_parenthesis: + close_parenthesis[-1] -= 1 + if close_parenthesis[-1] == 0: + out += ")" + else: + out += " " + else: + out += " " + + return out + + +def __initial_tree__(graph: _Graph, vertex: int) -> _PartialTree: + vertices, edges, primitive2indices, _, __ = graph + P: Function = vertices[vertex] # type: ignore + occurences = primitive2indices[P.function] # type: ignore + occurences_vertices: List[Set[int]] = [{v} for v in occurences] + return _PartialTree( + occurences, + occurences_vertices, + {0: [-1 for _ in edges[vertex]]}, + {}, + ) + + +def __find_best__( + graph: _Graph, + best_score: float, + done: Set[Tuple], + tree: _PartialTree, + score_function: Callable[[_Graph, _PartialTree], float], +) -> _PartialTree: + best = tree + previous = score_function(graph, tree) if tree.size() > 1 else -float("inf") + local_best_score = previous + for expansion in tree.expansions(graph, done): + # Use fact that score only increases then only decreases + if score_function(graph, expansion) <= previous: + continue + tree = __find_best__(graph, best_score, done, expansion, score_function) + if score_function(graph, tree) > local_best_score: + best = tree + return best + + +def __programs_to_graph__(programs: List[Program]) -> _Graph: + vertices: Dict[int, Program] = {} + edges: Dict[int, List[int]] = {} + primitive2indices: Dict[Union[Primitive, Variable], List[int]] = defaultdict(list) + vertex2tree: Dict[int, int] = {} + tree2start: Dict[int, int] = {} + + for tree_no, program in enumerate(programs): + tree2start[tree_no] = len(vertices) + args_indices: List[int] = [] + for el in program.depth_first_iter(): + vertex = len(vertices) + vertices[vertex] = el + vertex2tree[vertex] = tree_no + if isinstance(el, Function): + primitive2indices[el.function].append(vertex) # type: ignore + args_len = len(el.function.type.arguments()) - len(el.type.arguments()) + edges[vertex] = args_indices[-args_len:] + # Pop all consumed + the one for P.function which we did not consume + args_indices = args_indices[: -(args_len + 1)] + elif isinstance(el, (Primitive, Variable)): + edges[vertex] = [] + args_indices.append(vertex) + assert len(args_indices) == 1, f"args_indices:{args_indices}" + return vertices, edges, primitive2indices, vertex2tree, tree2start + + +def score_description(graph: _Graph, tree: _PartialTree) -> float: + return tree.num_occurences() * tree.size() + + +def make_score_probabilistic( + programs: List[Program], predict_vars: bool = True, var_prob: float = 0.2 +) -> Callable[[_Graph, _PartialTree], float]: + type2dict: Dict[Type, Dict[Tuple[str, str, int], int]] = defaultdict( + lambda: defaultdict(int) + ) + tree2program = {} + for tree_no, program in enumerate(programs): + tree2program[tree_no] = program + type2count = type2dict[program.type] + for P in program.depth_first_iter(): + if isinstance(P, Function): + primitive = __prim__(P).primitive + for arg_no, arg in enumerate(P.arguments): + type2count[(primitive, str(__prim__(arg)), arg_no)] += 1 + + lvar_prob = np.log(var_prob) + + def probability( + cur_type2dict: Dict[Type, Dict[Tuple[str, str, int], int]], + ) -> float: + total: Dict[Type, Dict[Tuple[str, int], int]] = {} + vars = defaultdict(set) + for t in cur_type2dict: + total[t] = defaultdict(int) + for (a, b, c), count in cur_type2dict[t].items(): + if not predict_vars and b.startswith("var"): + vars[t].add((a, c)) + continue + total[t][(a, c)] += count + + normed = { + t: { + (a, b, c): np.log(count / total[t][(a, c)]) + for (a, b, c), count in cur_type2dict[t].items() + if count > 0 and total[t][(a, c)] > 0 + } + for t in cur_type2dict + } + prob = 0 + for t in cur_type2dict: + for (a, b, c), p in normed[t].items(): + if not predict_vars and b.startswith("var"): + p = lvar_prob if total[t][(a, c)] > 0 else 0 + count = cur_type2dict[t][(a, b, c)] + prob += p * count + # print("key:", (a, b, c), "prob:", p, "*", count) + + return prob + + # print("ORIGINAL:", probability(type2dict)) + SPECIAL_STRING = "zefjpozjqfpokqzùofkepqozkfùpokqzefjùqzifjeùpoqzefùpoqkfeokqzofkùezùqofkeùozqkfoe" + + def score(graph: _Graph, tree: _PartialTree) -> float: + cur_type2dict = { + k: {kk: vv for kk, vv in v.items()} for k, v in type2dict.items() + } + vertices, edges, primitive2indices, vertex2tree, tree2start = graph + # print("TREE:", tree.string(graph)) + for k in range(len(tree.occurences)): + start_of_occ = tree.occurences[k] + tree_no = vertex2tree[start_of_occ] + program: Program = tree2program[tree_no] + type2count = cur_type2dict[program.type] + vertex = tree2start[tree_no] + args_indices: List[Tuple[int, bool]] = [] + belonging = tree.all_vertices_for_occurence(graph, k) + for el in program.depth_first_iter(): + belongs = vertex in belonging + if isinstance(el, Function): + args_len = len(el.arguments) + primitive = __prim__(el).primitive + my_args = args_indices[-args_len:] + # print("\t", el.arguments, my_args) + # print("\tcurrent:", el) + if belongs: + for arg_no in range(args_len): + arg = el.arguments[arg_no] + arg_s = str(__prim__(arg)) + if my_args[arg_no][1]: + key = (primitive, arg_s, arg_no) + # if key in type2count: + type2count[(primitive, arg_s, arg_no)] -= 1 + else: + key = (SPECIAL_STRING, arg_s, arg_no) + if key not in type2count: + type2count[key] = 0 + type2count[key] += 1 + elif not belongs and any(b for _, b in my_args): + for i, (_, b) in enumerate(my_args): + if b: + key = (primitive, SPECIAL_STRING, i) + if key not in type2count: + type2count[key] = 0 + type2count[key] += 1 + # Pop all consumed + the one for P.function which we did not consume + args_indices = args_indices[: -(args_len + 1)] + args_indices.append((vertex, belongs)) + vertex += 1 + + return probability(cur_type2dict) + + return score + + +def learn( + programs: List[Program], + score_function: Callable[[_Graph, _PartialTree], float] = score_description, + progress: bool = False, +) -> Tuple[float, str]: + """ + Learn a new primitive from the specified benchmark. + The default scoring function maximise the gain in description size of the dataset. + Suppose: + for all x, score_function(x) > -inf + score_function is computed in the worst case in polynomial time + + Return: + - cost of new primitive + - str description of new primitive + """ + + done: Set[Tuple] = set() + done_primitives: Set[Program] = set() + graph = __programs_to_graph__(programs) + vertices = graph[0] + best_score = -float("inf") + best = None + pbar = None + if progress: + pbar = tqdm.tqdm(total=len(vertices)) + for vertex in range(len(vertices)): + if pbar: + pbar.update(1) + p = vertices[vertex] + if not isinstance(p, Function): + continue + if __prim__(p) in done_primitives: + continue + done_primitives.add(__prim__(p)) + base_tree = __initial_tree__(graph, vertex) + r = base_tree.unique_repr(graph, 0) + tree = __find_best__(graph, best_score, done, base_tree, score_function) + done.add(r) + ts = score_function(graph, tree) + if ts > best_score: + best_score = ts + best = tree + if pbar: + pbar.set_postfix_str(f"{best.string(graph)} ({best_score})") + if best is not None: + return best_score, best.string(graph) + return 0.0, "" diff --git a/synth/nn/__init__.py b/synth/nn/__init__.py index 8fbcbd58..54b815b6 100644 --- a/synth/nn/__init__.py +++ b/synth/nn/__init__.py @@ -1,7 +1,9 @@ """ Module that contains anything relevant to neural networks """ -from synth.nn.grammar_predictor import GrammarPredictorLayer + +from synth.nn.det_grammar_predictor import DetGrammarPredictorLayer +from synth.nn.u_grammar_predictor import UGrammarPredictorLayer import synth.nn.abstractions as abstractions from synth.nn.utils import ( AutoPack, diff --git a/synth/nn/abstractions.py b/synth/nn/abstractions.py index 6bc62a34..c0225f75 100644 --- a/synth/nn/abstractions.py +++ b/synth/nn/abstractions.py @@ -1,38 +1,55 @@ -from typing import Any, Optional, Tuple +from typing import Any, Optional, Tuple, TypeVar from synth.syntax.grammars.cfg import CFGState, NoneType from synth.syntax.grammars.det_grammar import DerivableProgram +from synth.syntax.grammars.grammar import NGram from synth.syntax.program import Primitive from synth.syntax.type_system import Type +T = TypeVar("T") +S = TypeVar("S") -def cfg_bigram_without_depth( - ctx: Tuple[Type, Tuple[CFGState, NoneType]] + +def ucfg_bigram( + ctx: Tuple[Type, Tuple[NGram, T]], ) -> Optional[Tuple[DerivableProgram, int]]: """ - Abstract away a CFG into tuples of (parent, no_arg). - We lose depth information. + Abstract away a TTCFG into tuples of (parent, no_arg). + We lose any other information. """ - _, (state, __) = ctx - ngram, ___ = state + _, (ngram, __) = ctx + while not isinstance(ngram, NGram): + ngram = ngram[0] if len(ngram) > 0: return ngram.last() return None -def cfg_bigram_without_depth_and_equi_prim( - ctx: Tuple[Type, Tuple[CFGState, NoneType]] +def ttcfg_bigram( + ctx: Tuple[Type, Tuple[S, T]], ) -> Optional[Tuple[DerivableProgram, int]]: """ - Abstract away a CFG into tuples of (parent, no_arg) and merging together equivalent primitives. + Abstract away a TTCFG into tuples of (parent, no_arg). + We lose any other information. + """ + _, (ngram, __) = ctx + while not isinstance(ngram, NGram): + ngram = ngram[0] # type: ignore + if len(ngram) > 0: + return ngram.last() + return None + + +def cfg_bigram_without_depth( + ctx: Tuple[Type, Tuple[CFGState, NoneType]], +) -> Optional[Tuple[DerivableProgram, int]]: + """ + Abstract away a CFG into tuples of (parent, no_arg). We lose depth information. """ _, (state, __) = ctx ngram, ___ = state if len(ngram) > 0: - (P, i) = ngram.last() - if isinstance(P, Primitive) and "@" in P.primitive: - return Primitive(P.primitive[: P.primitive.find("@")], P.type), i - return P, i + return ngram.last() return None diff --git a/synth/nn/grammar_predictor.py b/synth/nn/det_grammar_predictor.py similarity index 88% rename from synth/nn/grammar_predictor.py rename to synth/nn/det_grammar_predictor.py index 36796705..94d0f335 100644 --- a/synth/nn/grammar_predictor.py +++ b/synth/nn/det_grammar_predictor.py @@ -58,7 +58,7 @@ def to_prob_det_grammar(self) -> ProbDetGrammar[U, V, W]: return ProbDetGrammar(self.grammar, probabilities) -class GrammarPredictorLayer(nn.Module, Generic[A, U, V, W]): +class DetGrammarPredictorLayer(nn.Module, Generic[A, U, V, W]): """ Parameters: @@ -75,7 +75,7 @@ def __init__( abstraction: Callable[[Tuple[Type, U]], A], variable_probability: float = 0.2, ): - super(GrammarPredictorLayer, self).__init__() + super(DetGrammarPredictorLayer, self).__init__() self.grammar_dictionary = { grammar.type_request: grammar for grammar in grammars @@ -157,6 +157,7 @@ def tensor2log_prob_grammar( # List of all variables derivable from S variables: List[Variable] = [] + constants: List[Constant] = [] # For each derivation parse probabilities for P in grammar.rules[S]: cpy_P = P @@ -168,11 +169,14 @@ def tensor2log_prob_grammar( variables.append(V) # All variables together have probability mass self.variable_probability # then the probability of selecting a variable is uniform + elif isinstance(P, Constant): + C: Constant = P # ensure typing + constants.append(C) else: continue # If there are variables we need to normalise total = sum(np.exp(tags[S][P].item()) for P in tags[S]) - if variables: + if variables or constants: var_probability = self.variable_probability if total > 0: # Normalise rest @@ -184,7 +188,7 @@ def tensor2log_prob_grammar( var_probability = 1 # Normalise variable probability normalised_variable_logprob: float = np.log( - var_probability / len(variables) + var_probability / (len(variables) + len(constants)) ) for P in variables: tags[S][P] = torch.tensor(normalised_variable_logprob).to(device) @@ -193,7 +197,9 @@ def tensor2log_prob_grammar( normalised_variable_logprob = np.log( np.exp(normalised_variable_logprob) - 1e-7 ) - else: + for P in constants: + tags[S][P] = torch.tensor(normalised_variable_logprob).to(device) + elif total > 0: # We still need to normalise probabilities # Since all derivations aren't possible to_add = np.log(1 / total) @@ -227,21 +233,25 @@ def __normalize__(self, src: Tensor, dst: Tensor) -> None: src[:, start : start + length], dim=-1 ) - def loss_cross_entropy( + def loss_mse( self, programs: Iterable[Program], type_requests: Iterable[Type], batch_outputs: Tensor, reduce: Optional[Callable[[Tensor], Tensor]] = torch.mean, ) -> Tensor: - target = torch.stack( - [ - self.encode(prog, tr, device=batch_outputs.device) - for prog, tr in zip(programs, type_requests) - ] + target = torch.log( + 1e-5 + + torch.stack( + [ + self.encode(prog, tr, device=batch_outputs.device) + for prog, tr in zip(programs, type_requests) + ] + ) ).to(device=batch_outputs.device) - # Since we already do LogSoftmax we only have to do NNL to get cross entropy - out = F.cross_entropy(batch_outputs, target) + dst = torch.zeros_like(batch_outputs) + self.__normalize__(batch_outputs, dst) + out = F.mse_loss(dst, target) if reduce: out = reduce(out) return out @@ -259,7 +269,7 @@ def loss_negative_log_prob( """ if length_normed: log_prob_list = [ - log_pgrammar.log_probability(p) / p.length() + log_pgrammar.log_probability(p) / p.size() for p, log_pgrammar in zip(programs, log_pgrammars) ] else: @@ -274,11 +284,11 @@ def loss_negative_log_prob( def __reduce_encoder__( - t: Tuple[GrammarPredictorLayer[A, U, V, W], Tensor], + t: Tuple[DetGrammarPredictorLayer[A, U, V, W], Tensor], S: Tuple[Type, U], P: DerivableProgram, _: V, -) -> Tuple[GrammarPredictorLayer[A, U, V, W], Tensor]: +) -> Tuple[DetGrammarPredictorLayer[A, U, V, W], Tensor]: if isinstance(P, Primitive): G, tensor = t start, __, symbol2index = G.abs2index[G.real2abs[S]] diff --git a/synth/nn/u_grammar_predictor.py b/synth/nn/u_grammar_predictor.py new file mode 100644 index 00000000..90e10b6f --- /dev/null +++ b/synth/nn/u_grammar_predictor.py @@ -0,0 +1,347 @@ +from collections import defaultdict +from typing import ( + Callable, + Dict, + Generic, + Iterable, + List, + Literal, + Set, + Tuple, + Optional, + TypeVar, + Union, +) + +import numpy as np + +import torch +import torch.nn as nn +import torch.nn.functional as F +from torch import Tensor + +from synth.syntax.grammars.tagged_u_grammar import TaggedUGrammar, ProbUGrammar +from synth.syntax.grammars.u_grammar import DerivableProgram, UGrammar +from synth.syntax.program import Constant, Primitive, Program, Variable +from synth.syntax.type_system import Type + +A = TypeVar("A") +U = TypeVar("U") +V = TypeVar("V") +W = TypeVar("W") + + +class TensorLogProbUGrammar(TaggedUGrammar[Tensor, U, V, W]): + """ + Special version to compute with Tensors + """ + + def __init__( + self, + grammar: UGrammar[U, V, W], + tags: Dict[Tuple[Type, U], Dict[DerivableProgram, Dict[V, Tensor]]], + start_tags: Dict[Tuple[Type, U], Tensor], + ): + super().__init__(grammar, tags, start_tags) + some_start = list(self.starts)[0] + some_P = list(self.tags[some_start].keys())[0] + some_V = list(self.tags[some_start][some_P].keys())[0] + self.device = self.tags[some_start][some_P][some_V].device + + def log_probability( + self, + program: Program, + start: Optional[Tuple[Type, U]] = None, + ) -> Tensor: + device = self.device + return self.reduce_derivations( + lambda current, S, P, V: current + self.tags[S][P][tuple(V)], # type: ignore + torch.zeros((1,)).to(device), + program, + start, + )[0] + + def to_prob_u_grammar(self) -> ProbUGrammar[U, V, W]: + probabilities = { + S: { + P: {key: np.exp(t_prob.item()) for key, t_prob in dico.items()} + for P, dico in self.tags[S].items() + } + for S in self.tags + } + start_probs = { + S: np.exp(t_prob.item()) for S, t_prob in self.start_tags.items() + } + out = ProbUGrammar(self.grammar, probabilities, start_probs) + out.normalise() + return out + + +class UGrammarPredictorLayer(nn.Module, Generic[A, U, V, W]): + """ + + Parameters: + ------------ + - input_size: int - the input size of the tensor to this layer + - grammars: Iterable[DetGrammar[U, V, W]] - the set of all supported grammars + - variable_probability: float = 0.2 - the probability mass of all variable at any given derivation level + """ + + def __init__( + self, + input_size: int, + grammars: Iterable[UGrammar[U, V, W]], + abstraction: Callable[[Tuple[Type, U]], A], + variable_probability: float = 0.2, + ): + super(UGrammarPredictorLayer, self).__init__() + + self.grammar_dictionary = { + grammar.type_request: grammar for grammar in grammars + } + self.variable_probability = variable_probability + + # Compute all pairs (A, P) where A is an abstraction of S + self.abs2real: Dict[A, Set[Tuple[Type, U]]] = defaultdict(set) + self.real2abs: Dict[Tuple[Type, U], A] = {} + + self.all_pairs: Dict[Optional[A], Set[Primitive]] = {} + self.all_starts_abs: List[A] = [] + for grammar in grammars: + for S in grammar.rules: + abstract = abstraction(S) + self.abs2real[abstract].add(S) + self.real2abs[S] = abstract + + key = abstract + if not key in self.all_pairs: + self.all_pairs[key] = set() + for P in grammar.rules[S]: + if not isinstance(P, (Variable, Constant)): + self.all_pairs[key].add(P) + for S in grammar.starts: + a = abstraction(S) + if a not in self.all_starts_abs: + self.all_starts_abs.append(a) + output_size = sum(len(self.all_pairs[S]) for S in self.all_pairs) + len( + self.all_starts_abs + ) + self.output_size = output_size + self.abs2index: Dict[ + Optional[A], + Tuple[int, int, Dict[Primitive, int]], + ] = {} + current_index = 0 + for okey, set_for_key in self.all_pairs.items(): + self.abs2index[okey] = ( + current_index, + len(set_for_key), + {P: i for i, P in enumerate(self.all_pairs[okey])}, + ) + current_index += len(set_for_key) + + self.log_probs_predictor = nn.Linear( + input_size, + output_size, + ) + + def forward(self, x: Tensor) -> Tensor: + """ + batch_IOs is a tensor of size + (batch_size, input_size) + + returns: (batch_size, self.output_size) + """ + y: Tensor = self.log_probs_predictor(x) + return y + + def tensor2log_prob_grammar( + self, + x: Tensor, + type_request: Type, + total_variable_order: bool = True, + ) -> TensorLogProbUGrammar[U, V, W]: + """ + + Parameters: + ------------ + - x: Tensor - the tensor to be transformed into a TensorLogProbUGrammar + - type_request: Type - the type request of the PUCFG + - total_variable_order: bool = True - reduce very slighlty (1e-7) some variable probabilities to ensure they are totally ordered in terms of probablities + + """ + device = x.device + self.__normalize__(x, x) + grammar = self.grammar_dictionary[type_request] + tags: Dict[Tuple[Type, U], Dict[DerivableProgram, Dict[V, Tensor]]] = {} + for S in grammar.rules: + tags[S] = {} + key = self.real2abs[S] + start, length, symbol2index = self.abs2index[key] + y = x[start : start + length] + + # List of all variables derivable from S + variables: List[Variable] = [] + constants: List[Constant] = [] + # For each derivation parse probabilities + for P in grammar.rules[S]: + cpy_P = P + tags[S][cpy_P] = {} + if isinstance(P, Primitive): + primitive_index = symbol2index[P] + for v in grammar.rules[S][P]: + kv = v + if isinstance(v, List): + kv = tuple(v) # type: ignore + tags[S][cpy_P][kv] = y[primitive_index] + elif isinstance(P, Variable): + V: Variable = P # ensure typing + variables.append(V) + # All variables together have probability mass self.variable_probability + # then the probability of selecting a variable is uniform + elif isinstance(P, Constant): + C: Constant = P # ensure typing + constants.append(C) + else: + continue + # If there are variables we need to normalise + total = sum( + sum(np.exp(t_prob.item()) for t_prob in dico.values()) + for P, dico in tags[S].items() + ) + if variables or constants: + var_probability = self.variable_probability + if total > 0: + # Normalise rest + to_add: float = np.log((1 - self.variable_probability) / total) + for P in tags[S]: + tags[S][P] = { + z: prob + to_add for z, prob in tags[S][P].items() + } + else: + # There are no other choices than variables + var_probability = 1 + # Normalise variable probability + normalised_variable_logprob: float = np.log( + var_probability / (len(variables) + len(constants)) + ) + for P in variables: + for v in grammar.rules[S][P]: + tags[S][P][tuple(v)] = torch.tensor( # type: ignore + normalised_variable_logprob + ).to(device) + # Trick to allow a total ordering on variables + if total_variable_order: + normalised_variable_logprob = np.log( + np.exp(normalised_variable_logprob) - 1e-7 + ) + for P in constants: + for v in grammar.rules[S][P]: + tags[S][P][tuple(v)] = torch.tensor( # type: ignore + normalised_variable_logprob + ).to(device) + else: + # We still need to normalise probabilities + # Since all derivations aren't possible + to_add = np.log(1 / total) + for P in tags[S]: + tags[S][P] = {z: prob + to_add for z, prob in tags[S][P].items()} + start_tags: Dict[Tuple[Type, U], Tensor] = {} + z = x[self.output_size - len(self.all_starts_abs) :] + for i, abs in enumerate(self.all_starts_abs): + all_S = self.abs2real[abs] + for S in all_S: + if S in grammar.starts: + start_tags[S] = z[i] + total = sum(np.exp(t_prob.item()) for t_prob in start_tags.values()) + to_add = np.log(1 / total) + start_tags = {S: t_prob + to_add for S, t_prob in start_tags.items()} + grammar = TensorLogProbUGrammar(grammar, tags, start_tags) + return grammar + + def encode( + self, + program: Program, + type_request: Type, + device: Union[torch.device, str, Literal[None]] = None, + ) -> Tensor: + out: Tensor = torch.zeros((self.output_size), device=device) + grammar = self.grammar_dictionary[type_request] + grammar.reduce_derivations(__reduce_encoder__, (self, out), program) + return out + + def __normalize__(self, src: Tensor, dst: Tensor) -> None: + # Normalize + if len(dst.shape) == 1: + for _, (start, length, _) in self.abs2index.items(): + dst[start : start + length] = F.log_softmax( + src[start : start + length], dim=-1 + ) + + else: + for _, (start, length, _) in self.abs2index.items(): + dst[:, start : start + length] = F.log_softmax( + src[:, start : start + length], dim=-1 + ) + + def loss_mse( + self, + programs: Iterable[Program], + type_requests: Iterable[Type], + batch_outputs: Tensor, + reduce: Optional[Callable[[Tensor], Tensor]] = torch.mean, + ) -> Tensor: + target = torch.log( + 1e-5 + + torch.stack( + [ + self.encode(prog, tr, device=batch_outputs.device) + for prog, tr in zip(programs, type_requests) + ] + ) + ).to(device=batch_outputs.device) + dst = torch.zeros_like(batch_outputs) + self.__normalize__(batch_outputs, dst) + out = F.mse_loss(dst, target) + if reduce: + out = reduce(out) + return out + + def loss_negative_log_prob( + self, + programs: Iterable[Program], + log_pgrammars: Iterable[TensorLogProbUGrammar[U, V, W]], + reduce: Optional[Callable[[Tensor], Tensor]] = torch.mean, + length_normed: bool = True, + ) -> Tensor: + """ + Computes the negative log prob of each solution program. + This works independently of the abstraction used. + """ + if length_normed: + log_prob_list = [ + log_pgrammar.log_probability(p) / p.size() + for p, log_pgrammar in zip(programs, log_pgrammars) + ] + else: + log_prob_list = [ + log_pgrammar.log_probability(p) + for p, log_pgrammar in zip(programs, log_pgrammars) + ] + out = -torch.stack(log_prob_list) + if reduce: + out = reduce(out) + return out + + +def __reduce_encoder__( + t: Tuple[UGrammarPredictorLayer[A, U, V, W], Tensor], + S: Tuple[Type, U], + P: DerivableProgram, + _: V, +) -> Tuple[UGrammarPredictorLayer[A, U, V, W], Tensor]: + if isinstance(P, Primitive): + G, tensor = t + start, __, symbol2index = G.abs2index[G.real2abs[S]] + tensor[start + symbol2index[P]] = 1 + return t diff --git a/synth/nn/utils.py b/synth/nn/utils.py index 26e7f0ec..adc3e21e 100644 --- a/synth/nn/utils.py +++ b/synth/nn/utils.py @@ -82,7 +82,7 @@ def __init__( self.embedder = embedder pad_symbol = 0 if hasattr(self.encoder, "pad_symbol"): - pad_symbol = self.encoder.pad_symbol # type: ignore + pad_symbol = self.encoder.pad_symbol self.packer = AutoPack(pad_symbol) self.embed_size = embed_size diff --git a/synth/pbe/__init__.py b/synth/pbe/__init__.py index de1b7ad0..daccbf44 100644 --- a/synth/pbe/__init__.py +++ b/synth/pbe/__init__.py @@ -1,6 +1,7 @@ """ -Module that contains anything relevant to the Programming By Example (PBE) framework +Module that contains anything relevant to the Programming By Example (PBE) framework """ + from synth.pbe.io_encoder import IOEncoder from synth.pbe.task_generator import ( TaskGenerator, diff --git a/synth/pbe/solvers/__init__.py b/synth/pbe/solvers/__init__.py new file mode 100644 index 00000000..20a1d6c0 --- /dev/null +++ b/synth/pbe/solvers/__init__.py @@ -0,0 +1,7 @@ +from synth.pbe.solvers.pbe_solver import ( + PBESolver, + NaivePBESolver, + CutoffPBESolver, + MetaPBESolver, +) +from synth.pbe.solvers.restart_pbe_solver import RestartPBESolver diff --git a/synth/pbe/solvers/pbe_solver.py b/synth/pbe/solvers/pbe_solver.py new file mode 100644 index 00000000..8f811d96 --- /dev/null +++ b/synth/pbe/solvers/pbe_solver.py @@ -0,0 +1,209 @@ +from abc import ABC, abstractmethod +from typing import Callable, Dict, Generator, List, Optional, Any + +from synth.semantic.evaluator import DSLEvaluator +from synth.specification import PBE +from synth.syntax.grammars.enumeration.program_enumerator import ProgramEnumerator +from synth.syntax.program import Program +from synth.task import Task +from synth.utils import chrono + + +class PBESolver(ABC): + def __init__(self, evaluator: DSLEvaluator, **kwargs: Any) -> None: + self.evaluator = evaluator + self._stats: Dict[str, Any] = {} + self._init_stats_() + + def _init_stats_(self) -> None: + self._stats["programs"] = 0 + self._stats["time"] = 0 + self._stats["program_probability"] = 0 + + @classmethod + @abstractmethod + def name(cls) -> str: + """ + Returns the name of the class of this solver. + """ + pass + + def full_name(self) -> str: + """ + Returns the name of this particular instance of the solver, it may contain additional information. + """ + return self.name() + + def reset_stats(self) -> None: + """ + Reset the statistics collected by this solver. + """ + self._stats = {} + self._init_stats_() + + def get_stats(self, name: str) -> Optional[Any]: + """ + Get a stat by name or None if it does not exist. + """ + return self._stats.get(name, None) + + def available_stats(self) -> List[str]: + """ + List the name of all currently avialable stats. + """ + return list(self._stats.keys()) + + def _init_task_solving_( + self, task: Task[PBE], enumerator: ProgramEnumerator[None], timeout: float = 60 + ) -> None: + self._programs = 0 + + def _close_task_solving_( + self, + task: Task[PBE], + enumerator: ProgramEnumerator[None], + time_used: float, + solution: bool, + last_program: Program, + ) -> None: + self._stats["time"] += time_used + self._stats["program_probability"] = enumerator.probability(last_program) + self._stats["programs"] += self._programs + + def solve( + self, task: Task[PBE], enumerator: ProgramEnumerator[None], timeout: float = 60 + ) -> Generator[Program, None, bool]: + """ + Solve the given task by enumerating programs with the given enumerator. + When the timeout is reached, this function returns. + When a program that satisfies the task has been found, yield it. + The calling function should then send True if and only if it accepts the solution. + If False is sent the search continues. + """ + with chrono.clock(f"solve.{self.name()}") as c: # type: ignore + self._init_task_solving_(task, enumerator, timeout) + try: + for program in enumerator: + time = c.elapsed_time() + if time >= timeout: + self._close_task_solving_( + task, enumerator, time, False, program + ) + return False + self._programs += 1 + if self._test_(task, program): + should_stop = yield program + if should_stop: + self._close_task_solving_( + task, enumerator, time, True, program + ) + return True + except StopIteration as e: + self._close_task_solving_(task, enumerator, time, False, program) + raise e + return False + + def _test_(self, task: Task[PBE], program: Program) -> bool: + """ + Return true iff program satisfies the specification given by the task. + Fills self._score with a score representing how close the program was to solve the task. + + POSTCOND: + 0 <= self._score <= 1 + """ + failed = False + success = 0 + for ex in task.specification.examples: + if self.evaluator.eval(program, ex.inputs) != ex.output: + failed = True + else: + success += 1 + self._score = success / len(task.specification.examples) + return not failed + + +class NaivePBESolver(PBESolver): + @classmethod + def name(cls) -> str: + return "naive" + + +class MetaPBESolver(PBESolver, ABC): + def __init__( + self, + evaluator: DSLEvaluator, + solver_builder: Callable[..., PBESolver] = NaivePBESolver, + **kwargs: Any, + ) -> None: + self.subsolver = solver_builder(evaluator, **kwargs) + self.evaluator = evaluator + self._stats: Dict[str, Any] = {} + self._init_stats_() + + def _init_stats_(self) -> None: + super()._init_stats_() + self.subsolver._init_stats_() + for name, val in self.subsolver._stats.items(): + if name not in self._stats: + self._stats[name] = val + + @classmethod + @abstractmethod + def name(cls) -> str: + pass + + def full_name(self) -> str: + """ + Returns the name of this particular instance of the solver, for meta solver it has the format .. + """ + return super().full_name() + "." + self.subsolver.full_name() + + def reset_stats(self) -> None: + super().reset_stats() + self.subsolver.reset_stats() + + def _init_task_solving_( + self, task: Task[PBE], enumerator: ProgramEnumerator[None], timeout: float = 60 + ) -> None: + super()._init_task_solving_(task, enumerator, timeout) + self.subsolver._init_task_solving_(task, enumerator, timeout) + + def _close_task_solving_( + self, + task: Task[PBE], + enumerator: ProgramEnumerator[None], + time_used: float, + solution: bool, + last_program: Program, + ) -> None: + self.subsolver._close_task_solving_( + task, enumerator, time_used, solution, last_program + ) + for name, val in self.subsolver._stats.items(): + self._stats[name] = val + super()._close_task_solving_( + task, enumerator, time_used, solution, last_program + ) + + def _test_(self, task: Task[PBE], program: Program) -> bool: + return self.subsolver._test_(task, program) + + +class CutoffPBESolver(PBESolver): + """ + A solver that fails a program on first example that fails. + """ + + @classmethod + def name(cls) -> str: + return "cutoff" + + def _test_(self, task: Task[PBE], program: Program) -> bool: + n = 0 + for ex in task.specification.examples: + if self.evaluator.eval(program, ex.inputs) != ex.output: + self._score = n / len(task.specification.examples) + return False + n += 1 + self._score = 1 + return True diff --git a/synth/pbe/solvers/restart_pbe_solver.py b/synth/pbe/solvers/restart_pbe_solver.py new file mode 100644 index 00000000..a512afd0 --- /dev/null +++ b/synth/pbe/solvers/restart_pbe_solver.py @@ -0,0 +1,116 @@ +from typing import Any, Callable, Generator, List, Tuple +from synth.semantic.evaluator import DSLEvaluator + + +from synth.specification import PBE +from synth.syntax.grammars.enumeration.program_enumerator import ProgramEnumerator +from synth.syntax.grammars.grammar import DerivableProgram +from synth.syntax.program import Program +from synth.syntax.type_system import Type +from synth.task import Task +from synth.utils import chrono +from synth.pbe.solvers.pbe_solver import MetaPBESolver, NaivePBESolver, PBESolver + + +class RestartPBESolver(MetaPBESolver): + def __init__( + self, + evaluator: DSLEvaluator, + solver_builder: Callable[..., PBESolver] = NaivePBESolver, + restart_criterion: Callable[["RestartPBESolver"], bool] = lambda self: len( + self._data + ) + - self._last_size + > 10000, + uniform_prior: float = 0.05, + **kwargs: Any, + ) -> None: + super().__init__(evaluator, solver_builder, **kwargs) + self.restart_criterion = restart_criterion + self._last_size: int = 0 + self.uniform_prior = uniform_prior + + def _init_stats_(self) -> None: + super()._init_stats_() + self._stats["restarts"] = 0 + + @classmethod + def name(cls) -> str: + return "restart" + + def _init_task_solving_( + self, task: Task[PBE], enumerator: ProgramEnumerator[None], timeout: float = 60 + ) -> None: + super()._init_task_solving_(task, enumerator, timeout) + self._restarts = 0 + self._data: List[Tuple[Program, float]] = [] + self._last_size = 0 + + def _close_task_solving_( + self, + task: Task[PBE], + enumerator: ProgramEnumerator[None], + time_used: float, + solution: bool, + last_program: Program, + ) -> None: + super()._close_task_solving_( + task, enumerator, time_used, solution, last_program + ) + self._stats["restarts"] += self._restarts + + def solve( + self, task: Task[PBE], enumerator: ProgramEnumerator[None], timeout: float = 60 + ) -> Generator[Program, None, bool]: + with chrono.clock(f"solve.{self.name()}.{self.subsolver.name()}") as c: # type: ignore + self._enumerator = enumerator + self._init_task_solving_(task, self._enumerator, timeout) + gen = self._enumerator.generator() + program = next(gen) + while program is not None: + time = c.elapsed_time() + if time >= timeout: + self._close_task_solving_( + task, self._enumerator, time, False, program + ) + return False + self._programs += 1 + if self._test_(task, program): + should_stop = yield program + if should_stop: + self._close_task_solving_( + task, self._enumerator, time, True, program + ) + return True + self._score = self.subsolver._score + # Saves data + if self._score > 0: + self._data.append((program, self._score)) + # If should restart + if self._should_restart_(): + self._restarts += 1 + self._enumerator = self._restart_(self._enumerator) + gen = self._enumerator.generator() + program = next(gen) + return False + + def _should_restart_(self) -> bool: + return self.restart_criterion(self) + + def _restart_(self, enumerator: ProgramEnumerator[None]) -> ProgramEnumerator[None]: + pcfg = enumerator.G * 0 # type: ignore + self._last_size = len(self._data) + + def reduce( + score: float, S: Tuple[Type, Any], P: DerivableProgram, prob: float + ) -> float: + pcfg.probabilities[S][P] += score + return score + + for program, score in self._data: + pcfg.reduce_derivations(reduce, score, program) + if self.uniform_prior > 0: + pcfg = pcfg + (pcfg.uniform(pcfg.grammar) * self.uniform_prior) + pcfg.normalise() + new_enumerator = enumerator.clone(pcfg) + return new_enumerator diff --git a/synth/pbe/task_generator.py b/synth/pbe/task_generator.py index a34dd959..f9f380a5 100644 --- a/synth/pbe/task_generator.py +++ b/synth/pbe/task_generator.py @@ -10,11 +10,15 @@ Tuple, Type as PythonType, TypeVar, + Union, ) from collections.abc import Container import numpy as np +from synth.filter.constraints.dfta_constraints import add_dfta_constraints +from synth.syntax.grammars.tagged_u_grammar import ProbUGrammar +from synth.syntax.grammars.u_cfg import UCFG from synth.task import Dataset, Task from synth.specification import PBE, Example from synth.semantic.evaluator import Evaluator @@ -39,7 +43,7 @@ def __init__( evaluator: Evaluator, gen_random_type_request: Sampler[Type], gen_random_sample_number: Sampler[int], - pgrammars: Iterable[ProbDetGrammar], + pgrammars: Iterable[Union[ProbUGrammar, ProbDetGrammar]], output_validator: Callable[[Any], bool], max_tries: int = 100, uniques: bool = False, @@ -65,9 +69,9 @@ def __init__( } self.generated_types: Dict[Type, int] = {t: 0 for t in self.type2pgrammar} - def __generate_program__(self, type_request: Type) -> Tuple[Program, bool]: + def generate_program(self, type_request: Type) -> Tuple[Program, bool]: """ - (program, is_unique) + Returns (program, is_unique) """ nargs: int = len(type_request.arguments()) solution: Program = self.type2pgrammar[type_request].sample_program() @@ -92,7 +96,7 @@ def __generate_program__(self, type_request: Type) -> Tuple[Program, bool]: best = solution return best, unique_tries < self.max_tries - def __generate_type_request__(self) -> Type: + def generate_type_request(self) -> Type: type_request = self.gen_random_type_request.sample() i = 0 while type_request in self._failed_types and i <= self.max_tries: @@ -102,10 +106,10 @@ def __generate_type_request__(self) -> Type: self.difficulty[type_request] = [0, 0] return type_request - def __sample_input__(self, arguments: TList[Type]) -> TList: + def sample_input(self, arguments: TList[Type]) -> TList: return [self.input_generator.sample(type=arg_type) for arg_type in arguments] - def __eval_input__(self, solution: Program, input: TList) -> Any: + def eval_input(self, solution: Program, input: TList) -> Any: try: return self.evaluator.eval(solution, input) except Exception as e: @@ -114,13 +118,13 @@ def __eval_input__(self, solution: Program, input: TList) -> Any: else: raise e - def __make_task__( + def make_task( self, type_request: Type, solution: Program, inputs: TList, outputs: TList, - **kwargs: Any + **kwargs: Any, ) -> Task[PBE]: return Task( type_request, @@ -132,11 +136,11 @@ def __make_task__( def generate_task(self) -> Task[PBE]: self._failed_types.clear() while True: - type_request = self.__generate_type_request__() + type_request = self.generate_type_request() arguments = type_request.arguments() # Generate correct program that makes use of all variables - solution, is_unique = self.__generate_program__(type_request) + solution, is_unique = self.generate_program(type_request) # Try to generate the required number of samples samples = self.gen_random_sample_number.sample(type=type_request) inputs: TList = [] @@ -147,8 +151,8 @@ def generate_task(self) -> Task[PBE]: inputs ) >= samples and tries < self.max_tries: tries += 1 - new_input = self.__sample_input__(arguments) - output = self.__eval_input__(solution, new_input) + new_input = self.sample_input(arguments) + output = self.eval_input(solution, new_input) if self.output_validator(output) and output not in outputs: inputs.append(new_input) @@ -167,14 +171,14 @@ def generate_task(self) -> Task[PBE]: self.generated_types[type_request] += 1 if self.uniques and is_unique: self.seen.add(solution) - elif self.verbose: + elif self.verbose and not self.uniques: print( "Generated a copy of an existing program for type request:", type_request, "program:", solution, ) - return self.__make_task__( + return self.make_task( type_request, solution, inputs, @@ -213,8 +217,8 @@ def reproduce_int_dataset( int_bound: int = 1000, default_max_depth: int = 5, max_list_length: Optional[int] = None, + **kwargs: Any, ) -> Tuple[TaskGenerator, TList[int]]: - int_range: TList[int] = [999999999, 0] int_range[1] = -int_range[0] @@ -255,6 +259,7 @@ def get_lexicon(start: None) -> TList[int]: max_tries, default_max_depth, max_list_length, + **kwargs, ) @@ -275,6 +280,8 @@ def reproduce_dataset( max_tries: int = 100, default_max_depth: int = 5, max_list_length: Optional[int] = None, + constraints: TList[str] = [], + **kwargs: Any, ) -> Tuple[TaskGenerator, TList]: """ @@ -301,8 +308,8 @@ def reproduce_dataset( def analyze(element: Any, type: Type, depth: int = 1) -> None: if depth > max_list_depth[0]: max_list_depth[0] = depth - if isinstance(type, List): - elt_type = type.element_type + if type.is_instance(List): + elt_type: Type = type.types[0] # type: ignore if len(element) > 0: __multi_discrete_distribution__(list_length, type, len(element)) for el in element: @@ -349,17 +356,33 @@ def analyze(element: Any, type: Type, depth: int = 1) -> None: if max_depth == -1: max_depth = default_max_depth if uniform_pgrammar: - pgrammars = { - ProbDetGrammar.uniform(CFG.depth_constraint(dsl, t, max_depth)) - for t in allowed_types - } + pgrammars: Set[Union[ProbUGrammar, ProbDetGrammar]] = set() + if constraints: + pgrammars = { + ProbUGrammar.uniform( + UCFG.from_DFTA_with_ngrams( + add_dfta_constraints( + CFG.depth_constraint(dsl, t, max_depth), + constraints, + progress=False, + ), + 2, + ) + ) + for t in allowed_types + } + else: + pgrammars = { + ProbDetGrammar.uniform(CFG.depth_constraint(dsl, t, max_depth)) + for t in allowed_types + } else: type2grammar = { t: CFG.depth_constraint(dsl, t, max_depth) for t in allowed_types } type2samples = { t: [ - type2grammar[t].embed(task.solution) + task.solution for task in dataset if task.solution and (t == task.type_request) ] @@ -395,6 +418,7 @@ def analyze(element: Any, type: Type, depth: int = 1) -> None: or -1, ), max_tries, + **kwargs, ), get_lexicon(out[0]), ) @@ -415,7 +439,6 @@ def __multi_discrete_to_gen__( seed: Optional[int] = None, maxi: Optional[int] = None, ) -> RequestSampler[int]: - choice_map: Dict[Type, TList[int]] = {k: list(v.keys()) for k, v in distr.items()} probs_map: Dict[Type, np.ndarray] = { k: np.array(list(v.values()), dtype=float) for k, v in distr.items() diff --git a/synth/pruning/__init__.py b/synth/pruning/__init__.py deleted file mode 100644 index d3ce0984..00000000 --- a/synth/pruning/__init__.py +++ /dev/null @@ -1,15 +0,0 @@ -""" -Module that contains anything relevant to pruning -""" -from synth.pruning.pruner import Pruner, UnionPruner -from synth.pruning.syntactic_pruner import ( - UseAllVariablesPruner, - FunctionPruner, - SyntacticPruner, - SetPruner, -) -from synth.pruning.type_constraints import ( - export_syntax_to_python, - produce_new_syntax_for_constraints, - produce_new_syntax_for_sketch, -) diff --git a/synth/pruning/pruner.py b/synth/pruning/pruner.py deleted file mode 100644 index 15d5f9a9..00000000 --- a/synth/pruning/pruner.py +++ /dev/null @@ -1,19 +0,0 @@ -from abc import ABC, abstractmethod -from typing import Generic, TypeVar - -T = TypeVar("T") -U = TypeVar("U") - - -class Pruner(ABC, Generic[T]): - @abstractmethod - def accept(self, obj: T) -> bool: - pass - - -class UnionPruner(Pruner, Generic[U]): - def __init__(self, *pruners: Pruner[U]) -> None: - self.pruners = list(pruners) - - def accept(self, obj: U) -> bool: - return all(p.accept(obj) for p in self.pruners) diff --git a/synth/pruning/type_constraints/__init__.py b/synth/pruning/type_constraints/__init__.py deleted file mode 100644 index 656f151a..00000000 --- a/synth/pruning/type_constraints/__init__.py +++ /dev/null @@ -1,9 +0,0 @@ -from synth.pruning.type_constraints.pattern_constraints import ( - produce_new_syntax_for_constraints, -) -from synth.pruning.type_constraints.sketch import ( - produce_new_syntax_for_sketch, -) -from synth.pruning.type_constraints.utils import ( - export_syntax_to_python, -) diff --git a/synth/pruning/type_constraints/pattern_constraints.py b/synth/pruning/type_constraints/pattern_constraints.py deleted file mode 100644 index 6a56a1eb..00000000 --- a/synth/pruning/type_constraints/pattern_constraints.py +++ /dev/null @@ -1,367 +0,0 @@ -from collections import defaultdict -from typing import Any, Optional, Dict, Iterable, Set, Tuple, List as TList - -import tqdm - -from synth.syntax import Type, Arrow, FunctionType -from synth.pruning.type_constraints.utils import ( - SYMBOL_DUPLICATA, - SYMBOL_VAR_EXPR, - Syntax, - map_type, - get_prefix, - parse_choices, - SYMBOL_SEPARATOR, - SYMBOL_FORBIDDEN, - SYMBOL_ANYTHING, - clean, - parse_specification, - producers_of_using, - types_produced_directly_by, -) - - -def __add_variable_constraint__( - content: str, - parent: str, - argno: int, - arg_type: Type, - syntax: Syntax, - nconstraints: Dict[str, int], - type_request: Optional[Arrow], - level: int = 0, -) -> Arrow: - assert type_request, "A type request is needed for variable constraints!" - content = content.strip(f"{SYMBOL_VAR_EXPR}()") - variables = set(map(int, parse_choices(content.replace("var", "")))) - var_types = set(type_request.arguments()[i] for i in variables) - varno2type = {no: type_request.arguments()[no] for no in variables} - # print("\t" * level, "Variables:", variables, "types:", var_types) - # Always assume there are other variables of the same types - - to_duplicate = producers_of_using(syntax, arg_type, var_types) - # add constants of same types - to_duplicate |= { - p - for p, ptype in syntax.syntax.items() - if not isinstance(ptype, Arrow) and ptype in var_types - } - types_to_duplicate = types_produced_directly_by(to_duplicate, syntax) - # print("\t" * level, "To duplicate:", to_duplicate) - # print("\t" * level, "Types to duplicate:", types_to_duplicate) - - # Compute the mapping of types - types_map = {} - # variables first - for var_type in sorted(var_types, key=str): - concerned = {i for i in variables if varno2type[i] == var_type} - if any(nconstraints[f"var{i}"] > 0 for i in concerned): - already_defined = [i for i in concerned if nconstraints[f"var{i}"] > 0] - assert len(already_defined) == 1 - types_map[var_type] = var_type - else: - types_map[var_type] = syntax.duplicate_type(var_type) - # rest - for dtype in sorted(types_to_duplicate.difference(var_types), key=str): - types_map[dtype] = syntax.duplicate_type(dtype) - - # Duplicate primitives - for primitive in sorted(to_duplicate, key=str): - syntax.duplicate_primitive(primitive, map_type(syntax[primitive], types_map)) - - # Add casts - for var_type in var_types: - syntax.add_cast(var_type, types_map[var_type]) - - # Fix parent type - args = syntax[parent].arguments() - args[argno] = map_type(args[argno], types_map) - syntax[parent] = FunctionType(*args, syntax[parent].returns()) - - # Fix type request - args = type_request.arguments() - for i in variables: - args[i] = map_type(args[i], types_map) - # Add constraints - nconstraints[f"var{i}"] += 1 - return FunctionType(*args, type_request.returns()) # type: ignore - - -def __add_primitive_constraint__( - content: str, - parent: str, - argno: int, - syntax: Syntax, - level: int = 0, -) -> str: - prim = content.strip("()") - equiv = [prim] - if SYMBOL_DUPLICATA not in prim: - equiv = sorted(syntax.equivalent_primitives(prim)) - for primitive in equiv: - ptype = syntax[primitive] - rtype = ptype.returns() - new_type_needed = any( - get_prefix(p) != get_prefix(prim) for p in syntax.producers_of(rtype) - ) - # If there are other ways to produce the same thing - if new_type_needed: - new_return_type = syntax.duplicate_type(rtype) - ntype = new_return_type - if isinstance(ptype, Arrow): - ntype = FunctionType(*ptype.arguments(), new_return_type) - # We do not need a duplicate - if SYMBOL_DUPLICATA in prim: - new_primitive = primitive - syntax[primitive] = ntype - else: - new_primitive = syntax.duplicate_primitive(primitive, ntype) - - if primitive == prim: - prim = new_primitive - # print("\t" * level, "Added:", new_primitive, ":", syntax[new_primitive]) - # Update parent signature - parent_type = syntax[parent] - assert isinstance(parent_type, Arrow) - old_types = parent_type.arguments() + [parent_type.returns()] - old_types[argno] = new_return_type - syntax[parent] = FunctionType(*old_types) - return prim - - -def __add_primitives_constraint__( - content: str, - parent: str, - argno: int, - syntax: Syntax, - level: int = 0, -) -> None: - primitives = parse_choices(content) - primitives = syntax.filter_out_forbidden(parent, argno, primitives) - if len(primitives) <= 0: - return - # print("\t" * level, "content:", content) - # print("\t" * level, "primitives:", primitives) - # 1) Simply do it for all primitives in the list - new_primitives = [ - __add_primitive_constraint__(p, parent, argno, syntax, level + 1) - for p in primitives - ] - # 2) Make them coherent - ttype = syntax[new_primitives[0]].returns() - for new_primitive in new_primitives: - ntype = syntax[new_primitive] - syntax[new_primitive] = ( - FunctionType(*ntype.arguments(), ttype) - if isinstance(ntype, Arrow) - else ttype - ) - # print("\t" * level, "\tCoherent:", new_primitive, ":", syntax[new_primitive]) - - # Update parent signature - parent_type = syntax[parent] - old_types = parent_type.arguments() + [parent_type.returns()] - old_types[argno] = ttype - syntax[parent] = FunctionType(*old_types) - # print("\t" * level, "parent:", parent, ":", syntax[parent]) - - # Now small thing to take into account - # if parent is the same as one of our primitive we need to fix the children - for p in new_primitives: - if get_prefix(p) == parent: - ptype = syntax[p] - old_types = ptype.arguments() + [ptype.returns()] - old_types[argno] = ttype - syntax[p] = FunctionType(*old_types) - # print("\t" * level, "\tRefix:", p, ":", syntax[p]) - - -def __add_forbidden_constraint__( - content: str, - parent: str, - argno: int, - syntax: Syntax, - *args: Any, - level: int = 0, - **kwargs: Any, -) -> None: - # print("\t" * level, "\tcontent:", content) - primitives = parse_choices(content[1:]) - primitives = syntax.filter_out_forbidden(parent, argno, primitives) - if len(primitives) == 0: - return - all_forbidden = set() - for p in primitives: - all_forbidden |= syntax.equivalent_primitives(p) - all_producers = syntax.producers_of(syntax[parent].arguments()[argno]) - remaining = all_producers - all_forbidden - # print("\t" * level, "\tallowed:", remaining) - - __add_primitives_constraint__( - SYMBOL_SEPARATOR.join(remaining), parent, argno, syntax, *args, **kwargs - ) - - -def __process__( - constraint: TList[str], - syntax: Syntax, - nconstraints: Dict[str, int], - parents: TList[str], - type_request: Optional[Arrow], - level: int = 0, -) -> Tuple[TList[str], Optional[Arrow]]: - # If one element then there is nothing to do. - if len(constraint) == 1: - return constraint, type_request - # If we have parents then we need to keep the original use of all of these primitives and make a copy of them that can only be used the right way - function = sorted(syntax.equivalent_primitives(constraint.pop(0))) - if len(parents) > 0: - function = [syntax.duplicate_primitive(f, syntax[f]) for f in function] - args = [] - # We need to process all arguments first - for arg in constraint: - new_el, type_request = __process__( - parse_specification(arg), - syntax, - nconstraints, - function, - type_request, - level + 1, - ) - args.append(new_el) - # If there are only stars there's nothing to do at our level - if all(len(arg) == 1 and arg[0] == SYMBOL_ANYTHING for arg in args): - return function, type_request - - # print("\t" * level, "functions:", function) - # print("\t" * level, "processing:", args) - for parent in function: - fun_tr = syntax[get_prefix(parent)] - assert isinstance(fun_tr, Arrow) - for argno, (eq_args, arg_type) in enumerate(zip(args, fun_tr.arguments())): - if len(eq_args) > 1: - __add_primitives_constraint__( - SYMBOL_SEPARATOR.join(eq_args), parent, argno, syntax, level - ) - else: - content: str = eq_args[0] - if content == SYMBOL_ANYTHING: - continue - elif content[0] == SYMBOL_VAR_EXPR: - type_request = __add_variable_constraint__( - content, - parent, - argno, - arg_type, - syntax, - nconstraints, - type_request, - level, - ) - elif content[0] == SYMBOL_FORBIDDEN: - __add_forbidden_constraint__(content, parent, argno, syntax, level) - else: - __add_primitives_constraint__(content, parent, argno, syntax, level) - - return function, type_request - - -def produce_new_syntax_for_constraints( - syntax: Dict[str, Type], - constraints: Iterable[str], - type_request: Optional[Arrow] = None, - forbidden: Optional[Dict[Tuple[str, int], Set[str]]] = None, - progress: bool = True, -) -> Tuple[Dict[str, Type], Optional[Arrow]]: - """ - Add type constraints on the specified syntax in order to enforce the given constraints. - - If no constraint depends on variables the type request is ignored. - if progress is set to True use a tqdm progress bar. - """ - new_syntax = Syntax({k: v for k, v in syntax.items()}, forbidden) - constraint_plus = [(int("var" in c), c) for c in constraints] - constraint_plus.sort(reverse=True) - parsed_constraints = [ - parse_specification(constraint) for _, constraint in constraint_plus - ] - - if progress: - pbar = tqdm.tqdm(total=len(parsed_constraints), desc="constraints", smoothing=1) - - for constraint in parsed_constraints: - _, type_request = __process__( - constraint, new_syntax, defaultdict(int), [], type_request - ) - if progress: - pbar.update(1) - pbar.set_postfix_str("cleaning...") - clean(new_syntax, type_request) - if progress: - pbar.set_postfix_str(f"+{len(new_syntax)/ len(syntax) - 1:.0%} DSL size") - if progress: - pbar.close() - return new_syntax.syntax, type_request - - -if __name__ == "__main__": - from synth.syntax import DSL, CFG, INT, FunctionType, ProbDetGrammar, List - from synth.pruning.type_constraints.utils import export_syntax_to_python - - # from examples.pbe.towers.towers_base import syntax, BLOCK - - # type_request = FunctionType(INT, INT, BLOCK) - - # patterns = [ - # "ifX $(var0) ifY,elifY", - # "ifY * 1x3,3x1", - # "elifY ifY EMPTY,elifY", - # "elifX ifX EMPTY,elifX", - # "not ^not,and", - # "and ^and *", - # "or ^or,and ^and", - # "+ ^+,0 ^0", - # "not ^not,and", - # "* ^*,0,1 ^0,1", - # "- * ^0", - # ] - - from examples.pbe.deepcoder.deepcoder import pruned_version, dsl as old_dsl # type: ignore - - type_request = FunctionType(List(INT), List(INT)) - - max_depth = 4 - # original_size = CFG.depth_constraint(DSL(syntax), type_request, max_depth).size() - original_size = CFG.depth_constraint(old_dsl, type_request, max_depth).size() - - dsl, _ = pruned_version(True) - - # Print - print(f"[PATTERNS] New syntax with {len(dsl.list_primitives)} primitives") - # for P in dsl.list_primitives: - # prim, type = P.primitive, P.type - # print("\t", prim, ":", type) - new_size = CFG.depth_constraint( - dsl, - type_request, - max_depth - # DSL(new_syntax), - # type_request, - # max_depth, - ).size() - pc = (original_size - new_size) / original_size - print( - f"Removed {original_size - new_size:.2E} ({pc:%}) programs at depth", max_depth - ) - print(f"New size {new_size:.2E} programs at depth", max_depth) - print("New TR:", type_request) - - # pcfg = ProbDetGrammar.uniform( - # CFG.from_dsl(DSL(new_syntax), type_request, max_depth) - # ) - # pcfg.init_sampling(2) - # for i in range(30): - # print(pcfg.sample_program()) - - # with open("deepcoder2.py", "w") as fd: - # fd.write(export_syntax_to_python(new_syntax)) diff --git a/synth/pruning/type_constraints/sketch.py b/synth/pruning/type_constraints/sketch.py deleted file mode 100644 index cc996d52..00000000 --- a/synth/pruning/type_constraints/sketch.py +++ /dev/null @@ -1,160 +0,0 @@ -from collections import defaultdict -from typing import Optional, Dict, Set, Tuple, List as TList - -from synth.syntax import Type, Arrow, FunctionType -from synth.pruning.type_constraints.utils import ( - SYMBOL_VAR_EXPR, - Syntax, - SYMBOL_SEPARATOR, - SYMBOL_FORBIDDEN, - SYMBOL_ANYTHING, - clean, - parse_specification, -) -from synth.pruning.type_constraints.pattern_constraints import ( - __process__ as __pat_process__, - __add_forbidden_constraint__, - __add_primitives_constraint__, - __add_variable_constraint__, -) - - -def __process__( - constraint: TList[str], - syntax: Syntax, - nconstraints: Dict[str, int], - type_request: Arrow, -) -> Tuple[TList[str], Arrow]: - # If one element then there is nothing to do. - if len(constraint) == 1: - return constraint, type_request - function = sorted(syntax.equivalent_primitives(constraint.pop(0))) - args = [] - - old_rtype = type_request.returns() - new_rtype = syntax.duplicate_type(type_request.returns()) - type_request = FunctionType(*type_request.arguments(), new_rtype) # type: ignore - - new_function = [] - for parent in function: - fun_tr = syntax[parent] - if fun_tr.returns() != old_rtype: - continue - parent = syntax.duplicate_primitive( - parent, FunctionType(*fun_tr.arguments(), new_rtype) - ) - fun_tr = syntax[parent] - new_function.append(parent) - function = new_function - - # We need to process all arguments first - for arg in constraint: - new_el, tr = __pat_process__( - parse_specification(arg), syntax, nconstraints, function, type_request, 1 - ) # - assert tr is not None - type_request = tr - args.append(new_el) - # If there are only stars there's nothing to do at our level - if all(len(arg) == 1 and arg[0] == SYMBOL_ANYTHING for arg in args): - return function, type_request - # print("\t" * level, "processing:", constraint) - for parent in function: - fun_tr = syntax[parent] - assert isinstance(fun_tr, Arrow) - for argno, (eq_args, arg_type) in enumerate(zip(args, fun_tr.arguments())): - if len(eq_args) > 1: - __add_primitives_constraint__( - SYMBOL_SEPARATOR.join(eq_args), parent, argno, syntax, 0 - ) - else: - content: str = eq_args[0] - if content == SYMBOL_ANYTHING: - continue - elif content[0] == SYMBOL_VAR_EXPR: - type_request = __add_variable_constraint__( - content, - parent, - argno, - arg_type, - syntax, - nconstraints, - type_request, - 0, - ) - elif content[0] == SYMBOL_FORBIDDEN: - __add_forbidden_constraint__(content, parent, argno, syntax, 0) - else: - __add_primitives_constraint__(content, parent, argno, syntax, 0) - - return function, type_request - - -def produce_new_syntax_for_sketch( - syntax: Dict[str, Type], - sketch: str, - type_request: Arrow, - forbidden: Optional[Dict[Tuple[str, int], Set[str]]] = None, -) -> Tuple[Dict[str, Type], Arrow]: - """ - Add type constraints on the specified syntax in order to enforce the given sketch. - """ - new_syntax = Syntax({k: v for k, v in syntax.items()}, forbidden) - sketch_spec = parse_specification(sketch) - _, type_request = __process__( - sketch_spec, new_syntax, defaultdict(int), type_request - ) - clean(new_syntax, type_request) - return new_syntax.syntax, type_request - - -if __name__ == "__main__": - from synth.syntax import DSL, CFG, INT, FunctionType, ProbDetGrammar, List - from synth.pruning.type_constraints.utils import export_syntax_to_python - - from examples.pbe.deepcoder.deepcoder import dsl, __primitive_types # type: ignore - - type_request = FunctionType(List(INT), List(INT)) - - max_depth = 6 - # original_size = CFG.depth_constraint(DSL(syntax), type_request, max_depth).size() - original_size = CFG.depth_constraint(dsl, type_request, max_depth).size() - - new_syntax, type_request = produce_new_syntax_for_sketch( - # __primitive_types, - # "(MAP[*2] (MAP[+1] _))", - # type_request - __primitive_types, - "(MAP[*2] (MAP[+1] (MAP[*2] _)))", - type_request, # type: ignore - ) - - # Print - print(f"[PATTERNS] New syntax with {len(dsl.list_primitives)} primitives") - for P in DSL(new_syntax).list_primitives: - prim, type = P.primitive, P.type - print("\t", prim, ":", type) - new_size = CFG.depth_constraint( - # dsl, - # type_request, - # max_depth - DSL(new_syntax), - type_request, - max_depth, - ).size() - pc = (original_size - new_size) / original_size - print( - f"Removed {original_size - new_size:.2E} ({pc:%}) programs at depth", max_depth - ) - print(f"New size {new_size:.2E} programs at depth", max_depth) - print("New TR:", type_request) - - pcfg = ProbDetGrammar.uniform( - CFG.depth_constraint(DSL(new_syntax), type_request, max_depth) - ) - pcfg.init_sampling(20) - for i in range(30): - print(pcfg.sample_program()) - - # with open("deepcoder2.py", "w") as fd: - # fd.write(export_syntax_to_python(new_syntax)) diff --git a/synth/pruning/type_constraints/utils.py b/synth/pruning/type_constraints/utils.py deleted file mode 100644 index 9fb1d651..00000000 --- a/synth/pruning/type_constraints/utils.py +++ /dev/null @@ -1,507 +0,0 @@ -from collections import defaultdict -from typing import Generator, List as TList, Optional, Tuple, Dict, Set, Iterable, Union - - -from synth.syntax import Type, Arrow, List, PrimitiveType, PolymorphicType, FunctionType - - -SYMBOL_ANYTHING = "_" -SYMBOL_VAR_EXPR = "$" -SYMBOL_FORBIDDEN = "^" -SYMBOL_DUPLICATA = "@" -SYMBOL_SEPARATOR = "," - -PREFIX_CAST = "cast#" -# ======================================================================================== -# PARSING -# ======================================================================================== -def parse_choices(expression: str) -> TList[str]: - expression = expression.replace(" ", "") - if "," in expression: - return [s for s in expression.split(SYMBOL_SEPARATOR) if len(s) > 0] - if len(expression) > 0: - return [expression] - return [] - - -def __next_level__(string: str, start: str, end: str) -> int: - level = 0 - for i, el in enumerate(string): - if el == start: - level += 1 - if el == end: - level -= 1 - if level == 0: - return i - return i - - -def __parse_next_word__(program: str) -> Tuple[str, int]: - if program[0] in [SYMBOL_VAR_EXPR, "("]: - end = __next_level__(program, "(", ")") - else: - end = program.index(" ") - 1 if " " in program else len(program) - 1 - return program[: end + 1], end + 2 - - -def parse_specification(spec: str) -> TList[str]: - spec = spec.replace("\n", "").strip(")(") - index = 0 - elements = [] - while index < len(spec): - spec = spec[index:] - word, index = __parse_next_word__(spec) - elements.append(word) - return elements - - -# ======================================================================================== -# TYPE PRODUCERS/CONSUMERS -# ======================================================================================== - - -class Syntax: - def __init__( - self, - type_constraints: Dict[str, Type], - forbidden: Optional[Dict[Tuple[str, int], Set[str]]] = None, - ) -> None: - self.syntax = type_constraints - self.forbidden_patterns = forbidden or {} - - self._new_types_index: Dict[str, int] = defaultdict(int) - for ttype in __all_types__(self.syntax): - name = str(ttype) - if "@" in name and not name.startswith(PREFIX_CAST): - id = int(name[name.index("@") + 1 :]) - self._new_types_index[get_prefix(name)] = id + 1 - - # Init producers by type - self.producers_by_type: Dict[Type, Set[str]] = defaultdict(set) - for prim, ptype in self.syntax.items(): - if isinstance(ptype, Arrow): - self.producers_by_type[ptype.returns()].add(prim) - else: - self.producers_by_type[ptype].add(prim) - - # Init equivalent primitives - self.equivalents: Dict[str, Set[str]] = defaultdict(set) - for prim in self.syntax: - self.equivalents[get_prefix(prim)].add(prim) - - def __getitem__(self, item: str) -> Type: - return self.syntax[item] - - def __contains__(self, item: str) -> bool: - return item in self.syntax - - def __len__(self) -> int: - return len(self.syntax) - - def __delitem__(self, item: str) -> None: - ptype = self.syntax[item] - rtype = ptype - if isinstance(ptype, Arrow): - rtype = ptype.returns() - self.producers_by_type[rtype].remove(item) - self.equivalents[get_prefix(item)].remove(item) - del self.syntax[item] - - def __setitem__(self, item: str, new_t: Type) -> None: - ptype = self.syntax[item] - rtype = ptype.returns() if isinstance(ptype, Arrow) else ptype - rtype_now = new_t.returns() if isinstance(new_t, Arrow) else new_t - if rtype != rtype_now: - self.producers_by_type[rtype].remove(item) - self.producers_by_type[rtype_now].add(item) - - self.syntax[item] = new_t - - def producers_of(self, rtype: Type) -> Set[str]: - return self.producers_by_type[rtype] - - def consumers_of(self, atype: Type) -> Generator[str, None, None]: - for prim, ptype in self.syntax.items(): - if isinstance(ptype, Arrow) and atype in ptype.arguments(): - yield prim - - def equivalent_primitives(self, name: str) -> Set[str]: - return self.equivalents[get_prefix(name)] - - def replace_type(self, old_t: Type, new_t: Type) -> None: - tmap = {old_t: new_t} - for P, ptype in self.syntax.items(): - self.syntax[P] = map_type(ptype, tmap) - if P in self.producers_by_type[old_t]: - self.producers_by_type[old_t].remove(P) - self.producers_by_type[new_t].add(P) - - def duplicate_primitive(self, primitive: str, ntype: Type) -> str: - new_name = __new_primitive_name__(primitive, self) - self.syntax[new_name] = ntype - self.equivalents[get_prefix(new_name)].add(new_name) - rtype = ntype - if isinstance(ntype, Arrow): - rtype = ntype.returns() - self.producers_by_type[rtype].add(new_name) - return new_name - - def __new_type_name__(self, name: str) -> str: - prefix = get_prefix(name) - id = self._new_types_index[prefix] - self._new_types_index[prefix] += 1 - return f"{prefix}{SYMBOL_DUPLICATA}{id}" - - def duplicate_type(self, base: Type) -> Type: - out: Optional[Type] = None - if isinstance(base, PrimitiveType): - out = PrimitiveType(self.__new_type_name__(base.type_name)) - elif isinstance(base, List): - out = List(self.duplicate_type(base.element_type)) - elif isinstance(base, PolymorphicType): - # Not sure how relevant this is - out = PolymorphicType(self.__new_type_name__(base.name)) - assert out is not None, f"Could not duplicate type:{base}" - return out - - def add_cast(self, from_type: Type, to: Type) -> None: - name = f"{PREFIX_CAST}{from_type}->{to}" - self.syntax[name] = Arrow(from_type, to) - self.producers_by_type[to].add(name) - - def filter_out_forbidden( - self, parent: str, argno: int, all_forbidden: Iterable[str] - ) -> TList[str]: - forbidden = self.forbidden_patterns.get((parent, argno), set()) - return [P for P in all_forbidden if get_prefix(P) not in forbidden] - - -def producers_of_using(syntax: Syntax, rtype: Type, consuming: Set[Type]) -> Set[str]: - """ - Return the list of producers of that can directly or indirectly consume any - """ - # Compute the list of all possible candidates from a type - candidates = set() - queue = [rtype] - types_dones = set() - while queue: - atype = queue.pop() - for prod in syntax.producers_of(atype): - if prod not in candidates: - candidates.add(prod) - ptype = syntax[prod] - if isinstance(ptype, Arrow): - for a in ptype.arguments(): - if a not in types_dones: - types_dones.add(a) - queue.append(a) - - # Compute all consumers - all_consumers: Set[str] = set() - for atype in consuming: - all_consumers |= set(syntax.consumers_of(atype)) - # Now we can go down - out: Set[str] = candidates.intersection(all_consumers) - current: TList[str] = list(out) - types_dones = {x for x in consuming} - while current: - p = current.pop() - ptype = syntax[p] - if isinstance(ptype, Arrow): - prtype = ptype.returns() - if prtype not in types_dones: - consumers = syntax.consumers_of(prtype) - for consumer in consumers: - if consumer in candidates: - current.append(consumer) - out.add(consumer) - types_dones.add(prtype) - - return out - - -def types_produced_directly_by(primitives: Iterable[str], syntax: Syntax) -> Set[Type]: - out = set() - for prim in primitives: - ptype = syntax[prim] - if isinstance(ptype, Arrow): - out.add(ptype.returns()) - else: - out.add(ptype) - return out - - -def types_used_by(primitives: Iterable[str], syntax: Syntax) -> Set[Type]: - """ - Return the set of types that can be produced or consumed directly with the given primitives, then add all producers of those types recursively. - - """ - out = set() - queue = [] - # Add all types from primitives - for prim in primitives: - if prim not in syntax: - continue - ptype = syntax[prim] - if isinstance(ptype, Arrow): - for atype in [ptype.returns()] + ptype.arguments(): - if atype not in out: - out.add(atype) - queue.append(atype) - else: - if ptype not in out: - out.add(ptype) - queue.append(ptype) - # Update list for all other types - while queue: - producers = syntax.producers_of(queue.pop()) - for prim in producers: - ptype = syntax[prim] - if isinstance(ptype, Arrow): - for atype in ptype.arguments(): - if atype not in out: - out.add(atype) - queue.append(atype) - return out - - -# ======================================================================================== -# MISC -# ======================================================================================== -def get_prefix(name: str) -> str: - if name.startswith(PREFIX_CAST): - return name - return ( - name if SYMBOL_DUPLICATA not in name else name[: name.index(SYMBOL_DUPLICATA)] - ) - - -def map_type(old_type: Type, map: Dict[Type, Type]) -> Type: - if old_type in map: - return map[old_type] - elif isinstance(old_type, List): - return List(map_type(old_type.element_type, map)) - elif isinstance(old_type, Arrow): - return FunctionType( - *[map_type(arg, map) for arg in old_type.arguments()], - map_type(old_type.returns(), map), - ) - return old_type - - -# ======================================================================================== -# DUPLICATE PRIMITIVE/TYPE -# ======================================================================================== - - -def __all_types__(syntax: Dict[str, Type]) -> Set[PrimitiveType]: - all_types: Set[PrimitiveType] = set() - for t in syntax.values(): - for tt in t.decompose_type()[0]: - all_types.add(tt) - return all_types - - -def __new_primitive_name__(primitive: str, syntax: Syntax) -> str: - i = 0 - primitive = get_prefix(primitive) - while f"{primitive}{SYMBOL_DUPLICATA}{i}" in syntax: - i += 1 - return f"{primitive}{SYMBOL_DUPLICATA}{i}" - - -# ======================================================================================== -# CLEANING -# ======================================================================================== - - -def __are_equivalent_types__(syntax: Syntax, t1: Type, t2: Type) -> bool: - # Two types are equivalent iff - # for all primitives P producing t1 there is an equivalent primitive producing t2 and vice versa - # an equivalent primitive is a primitive that has the same type request but for the produced type and the same name_prefix up to @ - t2_producers = syntax.producers_of(t2) - t1_producers = syntax.producers_of(t1) - marked = [False for _ in range(len(t2_producers))] - # t1 in t2 - for p1 in t1_producers: - found_match = False - for i, p2 in enumerate(t2_producers): - if get_prefix(p1) != get_prefix(p2): - continue - tp1 = syntax[p1] - tp2 = syntax[p2] - if isinstance(tp1, Arrow) and isinstance(tp2, Arrow): - found_match = tp1.arguments() == tp2.arguments() - if found_match: - marked[i] = True - break - if not isinstance(tp1, Arrow) and not isinstance(tp2, Arrow): - found_match = True - marked[i] = True - break - if not found_match: - return False - # t2 in t1 - for already_found, p1 in zip(marked, t2_producers): - if already_found: - continue - found_match = False - for p2 in t1_producers: - if get_prefix(p1) != get_prefix(p2): - continue - tp1 = syntax[p1] - tp2 = syntax[p2] - if isinstance(tp1, Arrow) and isinstance(tp2, Arrow): - found_match = tp1.arguments() == tp2.arguments() - if found_match: - break - if not isinstance(tp1, Arrow) and not isinstance(tp2, Arrow): - found_match = True - break - if not found_match: - return False - return True - - -def __merge_for__(syntax: Syntax, primitive: str) -> bool: - candidates = sorted(syntax.equivalent_primitives(primitive)) - if len(candidates) <= 1: - return False - - merged = False - # Handle terminal - if not isinstance(syntax[candidates[0]], Arrow): - # Delete those with no consumers - for P in candidates: - if not any(syntax.consumers_of(syntax[P])): - merged = True - del syntax[P] - # Merge those with same types - for i, P1 in enumerate(candidates): - if P1 not in syntax: - continue - for P2 in candidates[i + 1 :]: - if P2 not in syntax: - continue - if syntax[P1] == syntax[P2]: - del syntax[P2] - merged = True - - return merged - - -def clean( - syntax: Union[Syntax, Dict[str, Type]], type_request: Optional[Arrow] = None -) -> None: - """ - Try merging duplicates that were created. - """ - if not isinstance(syntax, Syntax): - syntax = Syntax(syntax) - assert isinstance(syntax, Syntax) - # Delete all primitives using a non-interesting type - all_primitives = set(syntax.equivalents.keys()) - interesting_types = types_used_by(all_primitives, syntax) - var_types = set() - if type_request and isinstance(type_request, Arrow): - var_types = set(type_request.arguments()) - var_types.add(type_request.returns()) - interesting_types |= var_types - deletable_candidates = set(syntax.syntax.keys()) - ntypes = len(interesting_types) - old_n = 0 - while ntypes > old_n: - old_n = ntypes - new_deletable = set() - for P in deletable_candidates: - ptype = syntax[P] - # P cannot be used - if any(tt not in interesting_types for tt in ptype.arguments()): - new_deletable.add(P) - else: - for tt in ptype.arguments(): - interesting_types.add(tt) - interesting_types.add(ptype.returns()) - deletable_candidates = new_deletable - ntypes = len(interesting_types) - # Now we are sure that these primitives can be deleted - for P in deletable_candidates: - del syntax[P] - - # Do one merge for each primitive (only useful if no type merge occurs) - for p in all_primitives: - __merge_for__(syntax, p) - - # Gather equivalent type (by name up to @) in groups - type_classes = {} - for t in interesting_types: - prefix = get_prefix(str(t)) - if prefix in type_classes: - continue - type_classes[prefix] = sorted( - [ - tt - for tt in interesting_types - if get_prefix(str(tt)) == prefix and tt not in var_types - ], - key=str, - ) - # Try merging types - merged_types = True - while merged_types: - merged_types = False - - for prefix, tclass in list(type_classes.items()): - next_gen: TList[Type] = tclass[:] - # Try to merge two types in equivalence class - for i, t1 in enumerate(tclass): - if t1 not in next_gen: - continue - for t2 in tclass[i + 1 :]: - if t2 not in next_gen: - continue - if __are_equivalent_types__(syntax, t1, t2): - syntax.replace_type(t2, t1) - next_gen.remove(t2) - for p in all_primitives: - __merge_for__(syntax, p) - merged_types = True - type_classes[prefix] = next_gen - - -# ======================================================================================== -# EXPORTING -# ======================================================================================== -def __export_type__(ptype: Type) -> str: - if isinstance(ptype, Arrow): - return f"Arrow({__export_type__(ptype.type_in)}, {__export_type__(ptype.type_out)})" - elif isinstance(ptype, PrimitiveType): - return ptype.type_name.replace("@", "_") - elif isinstance(ptype, List): - return f"List({__export_type__(ptype.element_type)})" - elif isinstance(ptype, PolymorphicType): - return f"PolymorphicType({ptype.name})" - assert False - - -def export_syntax_to_python(syntax: Dict[str, Type], varname: str = "syntax") -> str: - nsyntax = Syntax(syntax) - types_declaration = "" - types = types_used_by(nsyntax.syntax.keys(), nsyntax) - for ntype in types: - while isinstance(ntype, List): - ntype = ntype.element_type - if isinstance(ntype, PrimitiveType) and "@" in ntype.type_name: - types_declaration += ( - ntype.type_name.replace("@", "_") - + ' = PrimitiveType("' - + ntype.type_name - + '")' - + "\n" - ) - - out = f"{varname} = " + "{\n" - for prim, ptype in syntax.items(): - out += '\t"' + prim + '": ' + __export_type__(ptype) + ",\n" - out += "\n}" - return types_declaration + out diff --git a/synth/semantic/__init__.py b/synth/semantic/__init__.py index 71772554..a2d56233 100644 --- a/synth/semantic/__init__.py +++ b/synth/semantic/__init__.py @@ -1,4 +1,5 @@ """ Module that contains anything relevant to the semantic """ + from synth.semantic.evaluator import Evaluator, DSLEvaluator diff --git a/synth/semantic/evaluator.py b/synth/semantic/evaluator.py index 7e21d8a4..1046389c 100644 --- a/synth/semantic/evaluator.py +++ b/synth/semantic/evaluator.py @@ -1,8 +1,7 @@ from abc import ABC, abstractmethod -from typing import Any, Dict, Iterable, List, Set +from typing import Any, Dict, List, Set, Callable, Tuple -from synth.syntax.program import Function, Primitive, Program, Variable -from synth.syntax.type_system import PrimitiveType +from synth.syntax.program import Constant, Function, Primitive, Program, Variable, Type class Evaluator(ABC): @@ -25,26 +24,8 @@ def __tuplify__(element: Any) -> Any: return element -def auto_complete_semantics( - primitives: Iterable[str], semantics: Dict[str, Any] -) -> None: - """ - Copy the semantics for all primitives that are not semantically defined yet there are defined up to prefix before @. - Examples: - 1) and, and@0, and@1 - Defining only and and then autocompleting will give the same semantic to the 3 previous primitives - 2) or@0 - Since or is not defined semantically then or@0 is not either. - """ - for prim in primitives: - if "@" in prim and prim not in semantics: - prefix = prim[: prim.index("@")] - if prefix in semantics: - semantics[prim] = semantics[prefix] - - class DSLEvaluator(Evaluator): - def __init__(self, semantics: Dict[str, Any], use_cache: bool = True) -> None: + def __init__(self, semantics: Dict[Primitive, Any], use_cache: bool = True) -> None: super().__init__() self.semantics = semantics self.use_cache = use_cache @@ -54,10 +35,47 @@ def __init__(self, semantics: Dict[str, Any], use_cache: bool = True) -> None: # Statistics self._total_requests = 0 self._cache_hits = 0 + self._dsl_constants: Dict[Tuple[Type, Any], Primitive] = {} + for p, val in semantics.items(): + if len(p.type.arguments()) == 0: + self._dsl_constants[(p.type, __tuplify__(val))] = p + + def compress(self, program: Program, allow_constants: bool = True) -> Program: + """ + Return a semantically equivalent version of the program by evaluating constant expressions. + Note for data saving/loading purposes, partial applications are left untouched. + """ + if isinstance(program, Function): + args = [ + self.compress(p, allow_constants=allow_constants) + for p in program.arguments + ] + if len(program.type.returns().arguments()) == 0 and all( + not a.uses_variables() for a in args + ): + before = self.use_cache + self.use_cache = False + value = self.eval(program, []) + self.use_cache = before + # Cancel compression of callable + if isinstance(value, Callable): # type: ignore + return Function(program.function, args) + tval = __tuplify__(value) + rtype = program.type + if (rtype, tval) in self._dsl_constants: + return self._dsl_constants[(rtype, tval)] + if allow_constants: + return Constant(program.type.returns(), value, True) + else: + return Function(program.function, args) + else: + return Function(program.function, args) + else: + return program def eval(self, program: Program, input: List) -> Any: key = __tuplify__(input) - if key not in self._cache and self.use_cache: + if self.use_cache and key not in self._cache: self._cache[key] = {} evaluations: Dict[Program, Any] = self._cache[key] if self.use_cache else {} if program in evaluations: @@ -69,125 +87,28 @@ def eval(self, program: Program, input: List) -> Any: self._cache_hits += 1 continue if isinstance(sub_prog, Primitive): - evaluations[sub_prog] = self.semantics[sub_prog.primitive] - elif isinstance(sub_prog, Variable): - evaluations[sub_prog] = input[sub_prog.variable] - elif isinstance(sub_prog, Function): - fun = evaluations[sub_prog.function] - for arg in sub_prog.arguments: - fun = fun(evaluations[arg]) - evaluations[sub_prog] = fun - except Exception as e: - if type(e) in self.skip_exceptions: - evaluations[program] = None - return None - else: - raise e - - return evaluations[program] - - def clear_cache(self) -> None: - self._cache = {} - self._cons_cache = {} - - @property - def cache_hit_rate(self) -> float: - return self._cache_hits / self._total_requests - - -class DSLEvaluatorWithConstant(Evaluator): - def __init__( - self, - semantics: Dict[str, Any], - constant_types: Set[PrimitiveType], - use_cache: bool = True, - ) -> None: - super().__init__() - self.semantics = semantics - self.constant_types = constant_types - self.use_cache = use_cache - self._cache: Dict[Any, Dict[Program, Any]] = {} - self._cons_cache: Dict[Any, Dict[Program, Any]] = {} - self._invariant_cache: Dict[Program, Any] = {} - self.skip_exceptions: Set[Exception] = set() - # Statistics - self._total_requests = 0 - self._cache_hits = 0 - - def eval_with_constant( - self, program: Program, input: List, constant_in: str, constant_out: str - ) -> Any: - evaluations: Dict[Program, Any] = {} - if self.use_cache: - used_cons = False - for sub_prog in program.depth_first_iter(): - if ( - isinstance(sub_prog, Primitive) - and sub_prog.type in self.constant_types - ): - used_cons = True - break - if used_cons: - key = input.copy() - key.append(constant_in) - key.append(constant_out) - key = __tuplify__(key) - evaluations = self._cons_cache[key] if key in self._cons_cache else {} - else: - key = __tuplify__(input) - evaluations = self._cache[key] if key in self._cache else {} - - if program in evaluations: - return evaluations[program] - try: - for sub_prog in program.depth_first_iter(): - self._total_requests += 1 - if sub_prog in evaluations: - self._cache_hits += 1 - continue - if sub_prog.is_invariant(self.constant_types): - if sub_prog in self._invariant_cache: - self._cache_hits += 1 - evaluations[sub_prog] = self._invariant_cache[sub_prog] - continue - else: - self._invariant_cache[sub_prog] = None - if isinstance(sub_prog, Primitive): - if sub_prog.primitive == "cste_in": - evaluations[sub_prog] = constant_in - elif sub_prog.primitive == "cste_out": - evaluations[sub_prog] = constant_out - else: - evaluations[sub_prog] = self.semantics[sub_prog.primitive] + evaluations[sub_prog] = self.semantics[sub_prog] elif isinstance(sub_prog, Variable): evaluations[sub_prog] = input[sub_prog.variable] + elif isinstance(sub_prog, Constant): + evaluations[sub_prog] = sub_prog.value elif isinstance(sub_prog, Function): fun = evaluations[sub_prog.function] for arg in sub_prog.arguments: fun = fun(evaluations[arg]) evaluations[sub_prog] = fun - if sub_prog.is_invariant(self.constant_types): - self._invariant_cache[sub_prog] = evaluations[sub_prog] - except Exception as e: if type(e) in self.skip_exceptions: evaluations[program] = None return None else: - print(e) raise e return evaluations[program] - def eval(self, program: Program, input: List) -> Any: - if len(input) >= 3: - return self.eval_with_constant(program, input[2:], input[0], input[1]) - return self.eval_with_constant(program, input, "", "") - def clear_cache(self) -> None: self._cache = {} self._cons_cache = {} - self._invariant_cache = {} @property def cache_hit_rate(self) -> float: diff --git a/synth/specification.py b/synth/specification.py index 4fda15bd..265b6d7a 100644 --- a/synth/specification.py +++ b/synth/specification.py @@ -1,7 +1,8 @@ from dataclasses import dataclass -from typing import Any, Generic, List, Optional, TypeVar +from typing import Any, Dict, Generic, List, Optional, TypeVar -from synth.syntax.type_system import FunctionType, EmptyList, Type, guess_type +from synth.syntax.type_system import EmptyList, Type +from synth.syntax.type_helper import FunctionType, guess_type class TaskSpecification: @@ -52,8 +53,7 @@ class PBEWithConstants(PBE): Programming By Example (PBE) with constants specification """ - constants_in: List[Any] - constants_out: List[Any] + constants: Dict[Type, List[Any]] @dataclass diff --git a/synth/syntax/__init__.py b/synth/syntax/__init__.py index 25cc3e69..23cd051a 100644 --- a/synth/syntax/__init__.py +++ b/synth/syntax/__init__.py @@ -1,30 +1,55 @@ """ Module that contains anything relevant to the syntax """ + from synth.syntax.dsl import DSL -from synth.syntax.program import Primitive, Variable, Function, Lambda, Program +from synth.syntax.program import ( + Primitive, + Variable, + Function, + Lambda, + Program, + Constant, +) +from synth.syntax.type_helper import guess_type, FunctionType, auto_type from synth.syntax.type_system import ( Type, - FunctionType, - guess_type, match, PrimitiveType, PolymorphicType, + FixedPolymorphicType, + Generic, + TypeFunctor, + GenericFunctor, List, Arrow, + Sum, + UnknownType, INT, BOOL, STRING, + UNIT, ) +from synth.syntax.automata import DFA, DFTA from synth.syntax.grammars import ( CFG, - DFA, + UCFG, TTCFG, Grammar, DetGrammar, + UGrammar, ProbDetGrammar, + ProbUGrammar, TaggedDetGrammar, - enumerate_prob_grammar, - enumerate_bucket_prob_grammar, - # split, + TaggedUGrammar, + ProgramEnumerator, + bs_enumerate_prob_grammar, + bps_enumerate_prob_grammar, + hs_enumerate_prob_grammar, + hs_enumerate_prob_u_grammar, + hs_enumerate_bucket_prob_grammar, + hs_enumerate_bucket_prob_u_grammar, + cd_enumerate_prob_grammar, + as_enumerate_prob_grammar, + split, ) diff --git a/synth/syntax/automata/__init__.py b/synth/syntax/automata/__init__.py new file mode 100644 index 00000000..bd36a9aa --- /dev/null +++ b/synth/syntax/automata/__init__.py @@ -0,0 +1,2 @@ +from synth.syntax.automata.dfa import DFA +from synth.syntax.automata.tree_automaton import DFTA diff --git a/synth/syntax/automata/dfa.py b/synth/syntax/automata/dfa.py new file mode 100644 index 00000000..8d6c9dd8 --- /dev/null +++ b/synth/syntax/automata/dfa.py @@ -0,0 +1,110 @@ +from typing import Callable, Dict, Generic, Set, Tuple, TypeVar + + +U = TypeVar("U") +V = TypeVar("V") +W = TypeVar("W") +X = TypeVar("X") + + +class DFA(Generic[U, V]): + """ + Deterministic safe finite automaton. + states: U + alphabet: V + Reads V elements from states U. + If there is no transition from U reading V it means it is non accepting. (there are no final states) + """ + + def __init__(self, initial: U, rules: Dict[U, Dict[V, U]]) -> None: + self.start = initial + self.rules = rules + # Clean unreachable states + reachables = self.states + for u in list(self.rules.keys()): + if u not in reachables: + del self.rules[u] + else: + for P in list(self.rules[u].keys()): + if self.rules[u][P] not in reachables: + del self.rules[u][P] + + def __mul__(self, other: "DFA[W, X]") -> "DFA[Tuple[U, W], Tuple[V, X]]": + start = (self.start, other.start) + rules: Dict[Tuple[U, W], Dict[Tuple[V, X], Tuple[U, W]]] = {} + for S1 in self.rules: + for S2 in other.rules: + rules[(S1, S2)] = {} + for w1 in self.rules[S1]: + for w2 in other.rules[S2]: + rules[(S1, S2)][(w1, w2)] = ( + self.rules[S1][w1], + other.rules[S2][w2], + ) + return DFA(start, rules) + + def __str__(self) -> str: + s = f"Print a DFA\n" + s += "start: {}\n".format(self.start) + for S in reversed(self.rules): + s += "#\n {}\n".format(S) + for P in self.rules[S]: + out = self.rules[S][P] + s += "\t{} -> {}\n".format(P, out) + return s + + def __repr__(self) -> str: + return self.__str__() + + @property + def states(self) -> Set[U]: + """ + The set of reachables states. + """ + all = set() + frontier = [self.start] + while frontier: + state = frontier.pop() + for P in self.rules[state]: + new_state = self.rules[state][P] + if new_state not in all: + all.add(new_state) + frontier.append(new_state) + return all + + def can_read(self, start: U, word: V) -> bool: + return start in self.rules and word in self.rules[start] + + def read(self, start: U, word: V) -> U: + return self.rules[start][word] + + def map_states(self, f: Callable[[U], W]) -> "DFA[W, V]": + mapping = {s: f(s) for s in self.states} + dst_rules = { + mapping[S]: {P: mapping[self.rules[S][P]] for P in self.rules[S]} + for S in self.rules + } + return DFA(mapping[self.start], dst_rules) + + def then(self, other: "DFA[U, V]") -> "DFA[U, V]": + assert self.states.isdisjoint(other.states) + new_rules = { + S: {P: self.rules[S][P] for P in self.rules[S]} for S in self.rules + } + for S in other.rules: + new_rules[S] = {P: other.rules[S][P] for P in other.rules[S]} + return DFA(self.start, new_rules) + + def read_product(self, other: "DFA[W, V]") -> "DFA[Tuple[U, W], V]": + start = (self.start, other.start) + rules: Dict[Tuple[U, W], Dict[V, Tuple[U, W]]] = {} + for S1 in self.rules: + for S2 in other.rules: + rules[(S1, S2)] = {} + for v in self.rules[S1]: + if v in other.rules[S2]: + rules[(S1, S2)][v] = ( + self.rules[S1][v], + other.rules[S2][v], + ) + return DFA(start, rules) diff --git a/synth/syntax/automata/tree_automaton.py b/synth/syntax/automata/tree_automaton.py new file mode 100644 index 00000000..8304af79 --- /dev/null +++ b/synth/syntax/automata/tree_automaton.py @@ -0,0 +1,339 @@ +from collections import defaultdict +from itertools import product +from typing import ( + Callable, + Dict, + Generic, + List, + Literal, + Optional, + Set, + Tuple, + TypeVar, + Union, + overload, +) + +U = TypeVar("U") +V = TypeVar("V") +W = TypeVar("W") +X = TypeVar("X") + + +class DFTA(Generic[U, V]): + """ + Deterministic finite tree automaton. + states: U + alphabet: V + """ + + def __init__( + self, + rules: Dict[ + Tuple[ + V, + Tuple[U, ...], + ], + U, + ], + finals: Set[U], + ) -> None: + self.finals = finals + self.rules = rules + + def __mul__(self, other: "DFTA[W, X]") -> "DFTA[Tuple[U, W], Tuple[V, X]]": + finals = set() + rules: Dict[ + Tuple[ + Tuple[V, X], + Tuple[Tuple[U, W], ...], + ], + Tuple[U, W], + ] = {} + for (l1, args1), dst1 in self.rules.items(): + for (l2, args2), dst2 in other.rules.items(): + if len(args1) != len(args2): + continue + S = ((l1, l2), tuple(zip(args1, args2))) + if (l1, args1) in self.finals and (l2, args2) in other.finals: + finals.add((dst1, dst2)) + rules[S] = (dst1, dst2) + return DFTA(rules, finals) + + def size(self) -> int: + """ + Return the size of the DFTA which is the number of rules. + """ + return len(self.rules) + + @property + def states(self) -> Set[U]: + """ + The set of reachable states. + """ + reachable = set() + added = True + rules = defaultdict(list) + for (_, args), dst in self.rules.items(): + rules[dst].append(args) + while added: + added = False + for dst in list(rules.keys()): + for args in rules[dst]: + if all(s in reachable for s in args): + reachable.add(dst) + added = True + del rules[dst] + break + return reachable + + @property + def alphabet(self) -> Set[V]: + """ + The set of letters. + """ + alphabet = set() + for (letter, _), __ in self.rules.items(): + alphabet.add(letter) + return alphabet + + def read(self, letter: V, children: Tuple[U, ...]) -> Optional[U]: + return self.rules.get((letter, children), None) + + def __remove_unreachable__(self) -> None: + new_states = self.states + new_rules = { + (letter, args): dst + for (letter, args), dst in self.rules.items() + if dst in new_states and all(s in new_states for s in args) + } + self.rules = new_rules + self.finals = self.finals.intersection(new_states) + + def __remove_unproductive__(self) -> None: + removed = True + while removed: + removed = False + consumed: Set[U] = {q for q in self.finals} + for _, args in self.rules: + for arg in args: + consumed.add(arg) + for S, dst in list(self.rules.items()): + if dst not in consumed: + del self.rules[S] + removed = True + + def reduce(self) -> None: + """ + Removes unreachable states and unproductive states. + """ + self.__remove_unreachable__() + self.__remove_unproductive__() + + def read_product(self, other: "DFTA[W, V]") -> "DFTA[Tuple[U, W], V]": + """ + Read self and other + """ + rules: Dict[ + Tuple[ + V, + Tuple[Tuple[U, W], ...], + ], + Tuple[U, W], + ] = {} + # Update rules + for (l1, args1), dst1 in self.rules.items(): + for (l2, args2), dst2 in other.rules.items(): + if len(args1) != len(args2) or l1 != l2: + continue + S = (l1, tuple((a, b) for a, b in zip(args1, args2))) + rules[S] = (dst1, dst2) + # Update final states + finals = set() + for dst1 in self.finals: + for dst2 in other.finals: + finals.add((dst1, dst2)) + out = DFTA(rules, finals) + return out + + def read_union( + self, + other: "DFTA[W, V]", + fusion: Callable[[Optional[U], Optional[W]], X] = lambda x, y: (x, y), # type: ignore + ) -> "DFTA[X, V]": + """ + Read self or other. + """ + rules: Dict[ + Tuple[ + V, + Tuple[X, ...], + ], + X, + ] = {} + # Update rules + mapping_s = defaultdict(list) + mapping_o = defaultdict(list) + # Compute all alternatives + finals = set() + ostates = other.states + for a in self.states: + for b in ostates: + f = fusion(a, b) + mapping_o[b].append(f) + mapping_s[a].append(f) + mapping_s[a].append(fusion(a, None)) + if a in self.finals: + finals |= set(mapping_s[a]) + for b in ostates: + mapping_o[b].append(fusion(None, b)) + if b in other.finals: + finals |= set(mapping_o[b]) + + for (l1, args1), dst1 in self.rules.items(): + cases = [mapping_s[x] for x in args1] + new_dst = fusion(dst1, None) + for new_args in product(*cases): + rules[(l1, new_args)] = new_dst + + for (l2, args2), dst2 in other.rules.items(): + cases = [mapping_o[x] for x in args2] + new_dst = fusion(None, dst2) + for new_args in product(*cases): + rules[(l2, new_args)] = new_dst + for (l1, args1), dst1 in self.rules.items(): + for (l2, args2), dst2 in other.rules.items(): + if len(args1) != len(args2) or l1 != l2: + continue + S = (l1, tuple(fusion(a, b) for a, b in zip(args1, args2))) + rules[S] = fusion(dst1, dst2) + + out = DFTA(rules, finals) + out.reduce() + return out + + @overload + def minimise(self, mapping: Callable[[Tuple[U, ...]], W]) -> "DFTA[W, V]": + pass + + @overload + def minimise(self, mapping: Literal[None] = None) -> "DFTA[Tuple[U, ...], V]": + pass + + def minimise( + self, mapping: Union[Literal[None], Callable[[Tuple[U, ...]], W]] = None + ) -> "Union[DFTA[Tuple[U, ...], V], DFTA[W, V]]": + """ + Assumes this is a reduced DTFA + + Adapted algorithm from: + Brainerd, Walter S.. “The Minimalization of Tree Automata.” Inf. Control. 13 (1968): 484-491. + """ + # 1. Build consumer_of + consumer_of: Dict[ + U, + List[ + Tuple[ + Tuple[ + V, + Tuple[U, ...], + ], + int, + ] + ], + ] = {q: [] for q in self.states} + for l, args in self.rules: + for k, ik in enumerate(args): + consumer_of[ik].append(((l, args), k)) + # 2. Init equiv classes + state2cls: Dict[U, int] = { + q: 0 if q not in self.finals else 1 for q in self.states + } + cls2states: Dict[int, Tuple[U, ...]] = { + j: tuple({q for q, i in state2cls.items() if i == j}) for j in [0, 1] + } + + n = 1 + finished = False + + # Routine + def are_equivalent(a: U, b: U) -> bool: + for S, k in consumer_of[a]: + P, args = S + # Replacing a at index k with b + newS = (P, tuple([p if j != k else b for j, p in enumerate(args)])) + # Check that rules[S] and rules[newS] go into the same equi. class + dst_cls = state2cls[self.rules[S]] + out = self.rules.get(newS) + if out is None or state2cls[out] != dst_cls: + return False + # Symmetry with b + for S, k in consumer_of[b]: + P, args = S + dst_cls = state2cls[self.rules[S]] + newS = (P, tuple([p if j != k else a for j, p in enumerate(args)])) + out = self.rules.get(newS) + if out is None or state2cls[out] != dst_cls: + return False + return True + + # 3. Main loop + while not finished: + finished = True + # For each equivalence class + for i in range(n + 1): + cls = list(cls2states[i]) + while cls: + new_cls = [] + representative = cls.pop() + new_cls.append(representative) + next_cls = [] + for q in cls: + if are_equivalent(representative, q): + new_cls.append(q) + else: + next_cls.append(q) + cls = next_cls + if len(cls) != 0: + # Create new equivalence class + n += 1 + for q in new_cls: + state2cls[q] = n + cls2states[n] = tuple(new_cls) + finished = False + else: + # new_cls (now) has NOT changed from cls (previous iter.), they are the same + # thus we just need to re-set it (because there might have been multiple iterations) + # i is a free slot since other classes are added at the end + cls2states[i] = tuple(new_cls) + + f = mapping or (lambda x: x) # type: ignore + new_rules = {} + for (l, args), dst in self.rules.items(): + t_args = tuple([f(cls2states[state2cls[q]]) for q in args]) + new_rules[(l, t_args)] = f(cls2states[state2cls[dst]]) + return DFTA(new_rules, {f(cls2states[state2cls[q]]) for q in self.finals}) # type: ignore + + def map_states(self, mapping: Callable[[U], X]) -> "DFTA[X, V]": + return DFTA( + { + (l, tuple(map(mapping, args))): mapping(dst) + for (l, args), dst in self.rules.items() + }, + set(map(mapping, self.finals)), + ) + + def __repr__(self) -> str: + return str(self) + + def __str__(self) -> str: + s = "Print a DFTA\n" + for (P, args), dst in self.rules.items(): + add = "" + if len(args) > 0: + add = "(" + ", ".join(map(str, args)) + ")" + s += f"\t{P}{add} -> {dst}" + if dst in self.finals: + s += " (FINAL)" + s += "\n" + return s diff --git a/synth/syntax/dsl.py b/synth/syntax/dsl.py index 51fcd98a..b4076e07 100644 --- a/synth/syntax/dsl.py +++ b/synth/syntax/dsl.py @@ -1,19 +1,20 @@ import copy -from typing import Dict, Mapping, Optional, List as TList, Set, Tuple, Union +import itertools +from typing import Any, Callable, Dict, Mapping, Optional, List as TList, Set, Tuple +from synth.syntax.type_helper import FunctionType -from synth.syntax.type_system import Type, Arrow, List -from synth.syntax.program import Function, Primitive, Program, Variable +from synth.syntax.type_system import UNIT, Sum, Type, Arrow, List, UnknownType +from synth.syntax.program import Constant, Function, Primitive, Program, Variable class DSL: """ Object that represents a domain specific language - list_primitives: a list of primitives. - - Primitives can be considered equivalent with @: - + and +@3 are considered to be both '+' - This enables us to add specific constraints on some + versions. + Parameters: + ----------- + - syntax: maps primitive names to their types + - forbidden_patterns: forbidden local derivations """ @@ -26,7 +27,6 @@ def __init__( Primitive(primitive=p, type=t) for p, t in syntax.items() ] self.forbidden_patterns = forbidden_patterns or {} - self._forbidden_computed = False def __str__(self) -> str: s = "Print a DSL\n" @@ -35,12 +35,21 @@ def __str__(self) -> str: return s def instantiate_polymorphic_types(self, upper_bound_type_size: int = 10) -> None: + """ + Must be called before compilation into a grammar or parsing. + Instantiate all polymorphic types. + Parameters: + ----------- + - upper_bound_type_size: maximum type size of type instantiated for polymorphic types + """ # Generate all basic types set_basic_types: Set[Type] = set() for P in self.list_primitives: set_basic_types_P, set_polymorphic_types_P = P.type.decompose_type() set_basic_types = set_basic_types | set_basic_types_P + if UNIT in set_basic_types: + set_basic_types.remove(UNIT) set_types = set(set_basic_types) for type_ in set_basic_types: @@ -56,37 +65,173 @@ def instantiate_polymorphic_types(self, upper_bound_type_size: int = 10) -> None # Replace Primitive with Polymorphic types with their instanciated counterpart for P in self.list_primitives[:]: type_P = P.type - set_basic_types_P, set_polymorphic_types_P = type_P.decompose_type() + _, set_polymorphic_types_P = type_P.decompose_type() if set_polymorphic_types_P: set_instantiated_types: Set[Type] = set() set_instantiated_types.add(type_P) for poly_type in set_polymorphic_types_P: new_set_instantiated_types: Set[Type] = set() for type_ in set_types: + if ( + not poly_type.can_be(type_) + or type_.size() > upper_bound_type_size + ): + continue for instantiated_type in set_instantiated_types: unifier = {str(poly_type): type_} intermediate_type = copy.deepcopy(instantiated_type) new_type = intermediate_type.unify(unifier) - if new_type.size() <= upper_bound_type_size: - new_set_instantiated_types.add(new_type) + new_set_instantiated_types.add(new_type) set_instantiated_types = new_set_instantiated_types for type_ in set_instantiated_types: + instantiated_P = Primitive(P.primitive, type=type_) + if instantiated_P not in self.list_primitives: + self.list_primitives.append(instantiated_P) + self.list_primitives.remove(P) + + # Duplicate things for Sum types + for P in self.list_primitives[:]: + versions = P.type.all_versions() + if len(versions) > 1: + for type_ in versions: instantiated_P = Primitive(P.primitive, type=type_) self.list_primitives.append(instantiated_P) self.list_primitives.remove(P) + # Now remove all UNIT as parameters from signatures + for P in self.list_primitives[:]: + if any(arg == UNIT for arg in P.type.arguments()): + P.type = P.type.without_unit_arguments() + def __eq__(self, o: object) -> bool: return isinstance(o, DSL) and set(self.list_primitives) == set( o.list_primitives ) - def parse_program(self, program: str, type_request: Type) -> Program: + def fix_types( + self, + program: Program, + ) -> Program: + """ + Takes a program with possibly UnknownTypes anywhere and try to instantiate the types correctly. + This does not solves type equations, it is much weaker but should be enough for most use cases. + + Parameters: + ----------- + - program: the progam whose types needs fixing + + Returns: + ----------- + A parsed program that matches the given string + """ + return self.__fix_types__(program)[0] + + def __fix_types__( + self, + program: Program, + forced_type: Optional[Type] = None, + force_fix: bool = False, + ) -> Tuple[Program, bool]: + is_ambiguous = False + if isinstance(program, Function): + fixed_fun, ambiguous = self.__fix_types__( + program.function, force_fix=force_fix + ) + args = [ + self.__fix_types__(arg, arg_type)[0] + for arg, arg_type in zip(program.arguments, fixed_fun.type.arguments()) + ] + + if ambiguous and forced_type is not None: + print( + "before:", + fixed_fun, + "type:", + fixed_fun.type, + "args:", + args, + "target:", + FunctionType(*([arg.type for arg in args] + [forced_type])), + ) + fixed_fun = self.__fix_types__( + program.function, + FunctionType(*[arg.type for arg in args], forced_type), + force_fix=force_fix, + )[0] + print("after:", fixed_fun, "type:", fixed_fun.type) + args = [ + self.__fix_types__(arg, arg_type, force_fix=force_fix)[0] + for arg, arg_type in zip( + program.arguments, fixed_fun.type.arguments() + ) + ] + out: Program = Function(fixed_fun, args) + elif not force_fix and not program.type.is_under_specified(): + out = program + elif isinstance(program, Variable): + out = Variable(program.variable, forced_type or program.type) + elif isinstance(program, Constant): + out = Constant( + forced_type or program.type, program.value, program.has_value() + ) + elif isinstance(program, Primitive): + if forced_type is None: + matching = [ + p for p in self.list_primitives if p.primitive == program.primitive + ] + if len(matching) == 1: + forced_type = matching[0].type + elif len(matching) > 1: + is_ambiguous = True + forced_type = Sum(*list(map(lambda x: x.type, matching))) + out = Primitive(program.primitive, forced_type or program.type) + else: + assert False, "no implemented" + return out, is_ambiguous + + def auto_parse_program( + self, + program: str, + constants: Dict[str, Tuple[Type, Any]] = {}, + ) -> Program: """ Parse a program from its string representation given the type request. + It will try to automatically fix types to guess the type request. + + Parameters: + ----------- + - program: the string representation of the program, i.e. str(prog) + - constants: str representation of constants that map to their (type, value) + + Returns: + ----------- + A parsed program that matches the given string + """ + nvars = 0 + for s in program.split("var"): + i = 0 + while i < len(s) and s[i].isdigit(): + i += 1 + if i > 0: + nvars = max(int(s[:i]) + 1, nvars) + tr = FunctionType(*[UnknownType()] * (nvars + 1)) + return self.fix_types(self.parse_program(program, tr, constants, False)) + + def __parse_program__( + self, + program: str, + type_request: Type, + constants: Dict[str, Tuple[Type, Any]] = {}, + ) -> TList[Program]: + """ + Produce all possible interpretations of a parsed program. """ if " " in program: parts = list( - map(lambda p: self.parse_program(p, type_request), program.split(" ")) + map( + lambda p: self.__parse_program__(p, type_request, constants), + program.split(" "), + ) ) function_calls: TList[int] = [] level = 0 @@ -105,70 +250,117 @@ def parse_program(self, program: str, type_request: Type) -> Program: end += 1 levels.pop() - def parse_stack(l: TList[Program], function_calls: TList[int]) -> Program: - if len(l) == 1: - return l[0] - current = l.pop(0) - f_call = function_calls.pop(0) - if isinstance(current.type, Arrow) and f_call > 0: - args = [ - parse_stack(l, function_calls) - for _ in current.type.arguments()[:f_call] - ] - return Function(current, args) - return current - - sol = parse_stack(parts, function_calls) - assert ( - str(sol) == program - ), f"Failed parsing:{program} got:{sol} type request:{type_request} obtained:{sol.type}" - return sol + n = len(parts) + + def parse_stack(i: int) -> TList[Tuple[Program, int]]: + if i + 1 == n: + return [(p, n) for p in parts[-1]] + current = parts[i] + f_call = function_calls[i] + out: TList[Tuple[Program, int]] = [] + for some in current: + if some.type.is_instance(Arrow) and f_call > 0: + poss_args: TList[Tuple[TList[Program], int]] = [([], i + 1)] + for _ in some.type.arguments()[:f_call]: + next = [] + for poss, j in poss_args: + parsed = parse_stack(j) + for x, k in parsed: + next.append((poss + [x], k)) + poss_args = next + + for poss, j in poss_args: + out.append((Function(some, list(poss)), j)) + else: + out.append((some, i + 1)) + return out + + sols = parse_stack(0) + + return [p for p, _ in sols] else: program = program.strip("()") - for P in self.list_primitives: - if P.primitive == program: - return P - if program.startswith("var"): + matching: TList[Program] = [ + P for P in self.list_primitives if P.primitive == program + ] + if len(matching) > 0: + return matching + elif program.startswith("var"): varno = int(program[3:]) vart = type_request - if isinstance(type_request, Arrow): + if type_request.is_instance(Arrow): vart = type_request.arguments()[varno] - return Variable(varno, vart) - assert False, f"can't parse: {program}" - - def instantiate_forbidden(self) -> None: - if self._forbidden_computed: - return - self._forbidden_computed = True - forbidden_sets = self.forbidden_patterns - - # Complete sets - for source, forbid_set in forbidden_sets.items(): - for P1 in list(forbid_set): - for P2 in self.list_primitives: - if are_equivalent_primitives(P1, P2): - forbid_set.add(P2.primitive) - - # Now we have to complete keys - for source, forbid_set in list(forbidden_sets.items()): - for P in self.list_primitives: - if are_equivalent_primitives(P, source[0]): - forbidden_sets[(P.primitive, source[1])] = forbidden_sets[source] + return [Variable(varno, vart)] + elif program in constants: + t, val = constants[program] + return [Constant(t, val, True)] + assert False, f"can't parse: '{program}'" + + def parse_program( + self, + program: str, + type_request: Type, + constants: Dict[str, Tuple[Type, Any]] = {}, + check: bool = True, + ) -> Program: + """ + Parse a program from its string representation given the type request. + + Parameters: + ----------- + - program: the string representation of the program, i.e. str(prog) + - type_request: the type of the requested program in order to identify variable types + - constants: str representation of constants that map to their (type, value) + - check: ensure the program was correctly parsed with type checking + + Returns: + ----------- + A parsed program that matches the given string + """ + possibles = self.__parse_program__(program, type_request, constants) + if check: + coherents = [p for p in possibles if p.type_checks()] + assert ( + len(coherents) > 0 + ), f"failed to parse a program that type checks for: {program}" + return coherents[0] + return possibles[0] def get_primitive(self, name: str) -> Optional[Primitive]: + """ + Returns the Primitive object with the specified name if it exists and None otherwise + + Parameters: + ----------- + - name: the name of the primitive to get + """ for P in self.list_primitives: if P.primitive == name: return P return None + def instantiate_semantics( + self, semantics: Dict[str, Callable] + ) -> Dict[Primitive, Callable]: + """ + Transform the semantics dictionnary from strings to primitives. + """ + dico = {} + for key, f in semantics.items(): + for p in self.list_primitives: + if p.primitive == key: + dico[p] = f + return dico + + def __or__(self, other: "DSL") -> "DSL": + out = DSL({}) + out.list_primitives += self.list_primitives + for prim in other.list_primitives: + if prim not in self.list_primitives: + out.list_primitives.append(prim) + out.forbidden_patterns = {k: v for k, v in self.forbidden_patterns.items()} + for k, v in other.forbidden_patterns.items(): + if k not in out.forbidden_patterns: + out.forbidden_patterns[k] = v -def are_equivalent_primitives( - p1: Union[str, Primitive], p2: Union[str, Primitive] -) -> bool: - name1 = p1 if isinstance(p1, str) else p1.primitive - name2 = p2 if isinstance(p2, str) else p2.primitive - if "@" in name1: - name1 = name1[: name1.find("@")] - if "@" in name2: - name2 = name2[: name2.find("@")] - return name1 == name2 + return out diff --git a/synth/syntax/grammars/__init__.py b/synth/syntax/grammars/__init__.py index d74da782..9a5b73ef 100644 --- a/synth/syntax/grammars/__init__.py +++ b/synth/syntax/grammars/__init__.py @@ -1,12 +1,20 @@ from synth.syntax.grammars.cfg import CFG from synth.syntax.grammars.ttcfg import TTCFG -from synth.syntax.grammars.dfa import DFA from synth.syntax.grammars.grammar import Grammar from synth.syntax.grammars.det_grammar import DetGrammar from synth.syntax.grammars.tagged_det_grammar import ProbDetGrammar, TaggedDetGrammar -from synth.syntax.grammars.heap_search import ( - enumerate_prob_grammar, - enumerate_bucket_prob_grammar, +from synth.syntax.grammars.enumeration import ( + ProgramEnumerator, + bs_enumerate_prob_grammar, + bps_enumerate_prob_grammar, + hs_enumerate_prob_grammar, + hs_enumerate_prob_u_grammar, + hs_enumerate_bucket_prob_grammar, + hs_enumerate_bucket_prob_u_grammar, + cd_enumerate_prob_grammar, + as_enumerate_prob_grammar, + split, ) - -# from synth.syntax.grammars.pcfg_splitter import split +from synth.syntax.grammars.u_grammar import UGrammar +from synth.syntax.grammars.u_cfg import UCFG +from synth.syntax.grammars.tagged_u_grammar import ProbUGrammar, TaggedUGrammar diff --git a/synth/syntax/grammars/cfg.py b/synth/syntax/grammars/cfg.py index 191a56e4..8c0bedc8 100644 --- a/synth/syntax/grammars/cfg.py +++ b/synth/syntax/grammars/cfg.py @@ -1,12 +1,12 @@ from collections import deque -from math import prod +from functools import lru_cache from typing import Deque, Dict, Literal, Set, Tuple, List from synth.syntax.dsl import DSL from synth.syntax.grammars.det_grammar import DerivableProgram from synth.syntax.grammars.ttcfg import TTCFG, NGram from synth.syntax.program import Constant, Primitive, Variable -from synth.syntax.type_system import Arrow, Type +from synth.syntax.type_system import Type NoneType = Literal[None] @@ -20,28 +20,41 @@ class CFG(TTCFG[CFGState, NoneType]): """ def max_program_depth(self) -> int: + """ + Returns the maximum depth of a program contained in this grammar. + """ + if self.programs() < 0: + return -1 return max(S[1][0][1] for S in self.rules) + 1 def __hash__(self) -> int: return hash((self.start, str(self.rules))) - def size(self) -> int: - total_programs: Dict[Tuple[Type, Tuple[CFGState, NoneType]], int] = {} - for S in sorted(self.rules, key=lambda nt: nt[1][0][1], reverse=True): - total = 0 - for P in self.rules[S]: - args_P = self.rules[S][P][0] - if len(args_P) == 0: - total += 1 - else: - total += prod(total_programs[(C[0], (C[1], None))] for C in args_P) - total_programs[S] = total - return total_programs[self.start] - def clean(self) -> None: self._remove_non_productive_() self._remove_non_reachable_() + @lru_cache() + def programs(self) -> int: + count: Dict[Tuple[Type, CFGState], int] = {} + try: + for S in sorted(self.rules.keys(), key=lambda s: -s[1][0][1]): + total = 0 + for P in self.rules[S]: + local = 1 + for arg in self.rules[S][P][0]: + local *= count[arg] + total += local + count[(S[0], S[1][0])] = total + S = self.start + return count[(S[0], S[1][0])] + except KeyError: + # Recursive grammar + return -1 + + def is_recursive(self) -> bool: + return self.programs() == -1 + def _remove_non_reachable_(self) -> None: """ remove non-terminals which are not reachable from the initial non-terminal @@ -53,15 +66,16 @@ def _remove_non_reachable_(self) -> None: new_reach: Set[CFGNonTerminal] = set() reach.add(self.start) - for _ in range(self.max_program_depth()): + while reach: new_reach.clear() for S in reach: for P in self.rules[S]: args_P = self.rules[S][P][0] for arg in args_P: nctx = (arg[0], (arg[1], None)) - new_reach.add(nctx) - reachable.add(nctx) + if nctx not in reachable: + new_reach.add(nctx) + reachable.add(nctx) reach.clear() reach = new_reach.copy() @@ -77,14 +91,31 @@ def _remove_non_productive_(self) -> None: CFGNonTerminal, Dict[DerivableProgram, Tuple[List[Tuple[Type, CFGState]], NoneType]], ] = {} - for S in sorted(self.rules, key=lambda rule: rule[1][0][1], reverse=True): + # 1. determine the relevant non terminals + candidates = [S for S in self.rules] + next_candidates = [] + changed = True + while changed: + changed = False + for S in candidates: + for P in self.rules[S]: + args_P = self.rules[S][P][0] + if all((arg[0], (arg[1], None)) in new_rules for arg in args_P): + if S not in new_rules: + new_rules[S] = {} + if S not in new_rules: + next_candidates.append(S) + else: + changed = True + candidates = next_candidates + next_candidates = [] + # 2. get the relevant derivation rules + for S in new_rules: for P in self.rules[S]: args_P = self.rules[S][P][0] if all((arg[0], (arg[1], None)) in new_rules for arg in args_P): - if S not in new_rules: - new_rules[S] = {} new_rules[S][P] = self.rules[S][P] - + # 3. prune current grammar for S in set(self.rules): if S in new_rules: self.rules[S] = new_rules[S] @@ -106,7 +137,6 @@ def depth_constraint( dsl: DSL, type_request: Type, max_depth: int, - upper_bound_type_size: int = 10, min_variable_depth: int = 1, n_gram: int = 2, recursive: bool = False, @@ -116,24 +146,27 @@ def depth_constraint( Constructs a CFG from a DSL imposing bounds on size of the types and on the maximum program depth. - max_depth: int - is the maxium depth of programs allowed - uppder_bound_size_type: int - is the maximum size type allowed for polymorphic type instanciations - min_variable_depth: int - min depth at which variables and constants are allowed - n_gram: int - the context, a bigram depends only in the parent node - recursvie: bool - allows the generated programs to call themselves - constant_types: Set[Type] - the set of of types allowed for constant objects + Parameters: + ----------- + - max_depth: the maximum depth of programs allowed, if negative returns an infinite CFG + - min_variable_depth: min depth at which variables and constants are allowed + - n_gram: the context, a bigram depends only in the parent node + - recursive: enables the generated programs to call themselves + - constant_types: the set of of types allowed for constant objects """ - dsl.instantiate_polymorphic_types(upper_bound_type_size) + if max_depth < 0: + return CFG.infinite( + dsl, + type_request, + n_gram, + recursive, + constant_types, + ) - dsl.instantiate_forbidden() forbidden_sets = dsl.forbidden_patterns - if isinstance(type_request, Arrow): - return_type = type_request.returns() - args = type_request.arguments() - else: - return_type = type_request - args = [] + return_type = type_request.returns() + args = type_request.arguments() rules: Dict[ CFGNonTerminal, @@ -165,7 +198,7 @@ def depth_constraint( # Try to add constants from the DSL for P in dsl.list_primitives: type_P = P.type - if not isinstance(type_P, Arrow) and type_P == current_type: + if type_P == current_type: rules[non_terminal][P] = ([], None) # Function call if depth < max_depth - 1: @@ -198,12 +231,11 @@ def depth_constraint( list_to_be_treated.appendleft(new_context) rules[non_terminal][P] = (decorated_arguments_P, None) - # Try to use variable as if there were functions if depth >= min_variable_depth: for vi, varg in enumerate(args): arguments_V = varg.ends_with(current_type) - if arguments_V is not None: + if arguments_V is not None and len(varg.arguments()) > 0: V = Variable(vi, varg) decorated_arguments_V = [] for i, arg in enumerate(arguments_V): @@ -239,7 +271,133 @@ def depth_constraint( rules[non_terminal][P] = (decorated_arguments_self, None) - return CFG( + cfg = CFG( start=initital_ctx, rules=rules, ) + cfg.type_request = type_request + return cfg + + @classmethod + def infinite( + cls, + dsl: DSL, + type_request: Type, + n_gram: int = 2, + recursive: bool = False, + constant_types: Set[Type] = set(), + ) -> "CFG": + """ + Constructs a CFG from a DSL imposing bounds on size of the types. + Non terminals can be recursive. + + Parameters: + ----------- + - n_gram: the context, a bigram depends only in the parent node + - recursive: enables the generated programs to call themselves + - constant_types: the set of of types allowed for constant objects + """ + + forbidden_sets = dsl.forbidden_patterns + + return_type = type_request.returns() + args = type_request.arguments() + + rules: Dict[ + CFGNonTerminal, + Dict[DerivableProgram, Tuple[List[Tuple[Type, CFGState]], NoneType]], + ] = {} + + list_to_be_treated: Deque[CFGNonTerminal] = deque() + initital_ctx = (return_type, ((NGram(n_gram), 0), None)) + list_to_be_treated.append(initital_ctx) + done: Set[CFGNonTerminal] = set() + + while len(list_to_be_treated) > 0: + non_terminal = list_to_be_treated.pop() + current_type = non_terminal[0] + # Create rule if non existent + if non_terminal not in rules: + rules[non_terminal] = {} + + if non_terminal in done: + continue + done.add(non_terminal) + + # Add variables rules + for i in range(len(args)): + if current_type == args[i]: + var = Variable(i, current_type) + rules[non_terminal][var] = ([], None) + if current_type in constant_types: + cst = Constant(current_type) + rules[non_terminal][cst] = ([], None) + # Try to add constants from the DSL + for P in dsl.list_primitives: + type_P = P.type + if type_P == current_type: + rules[non_terminal][P] = ([], None) + # Function call + predecessors = non_terminal[1][0][0] + last_pred = predecessors.last() if len(predecessors) > 0 else None + forbidden = forbidden_sets.get( + (last_pred[0].primitive, last_pred[1]) + if last_pred and isinstance(last_pred[0], Primitive) + else ("", 0), + set(), + ) + # DSL Primitives + for P in dsl.list_primitives: + if P.primitive in forbidden: + continue + type_P = P.type + arguments_P = type_P.ends_with(current_type) + if arguments_P is not None: + decorated_arguments_P = [] + for i, arg in enumerate(arguments_P): + new_predecessors = predecessors.successor((P, i)) + new_context = ( + arg, + ((new_predecessors, 0), None), + ) + decorated_arguments_P.append((arg, (new_predecessors, 0))) + if ( + new_context not in list_to_be_treated + and new_context not in done + ): + list_to_be_treated.appendleft(new_context) + rules[non_terminal][P] = (decorated_arguments_P, None) + # Try to use variable as if there were functions + for vi, varg in enumerate(args): + arguments_V = varg.ends_with(current_type) + if arguments_V is not None: + V = Variable(vi, varg) + decorated_arguments_V = [] + for i, arg in enumerate(arguments_V): + new_predecessors = predecessors.successor((V, i)) + new_context = ( + arg, + ((new_predecessors, 0), None), + ) + decorated_arguments_V.append((arg, (new_predecessors, 0))) + if new_context not in list_to_be_treated: + list_to_be_treated.appendleft(new_context) + rules[non_terminal][V] = (decorated_arguments_V, None) + # Try to call self + if recursive: + arguments_self = type_request.ends_with(current_type) + if arguments_self is not None: + P = Primitive("@self", type_request) + decorated_arguments_self = [] + for i, arg in enumerate(arguments_self): + new_predecessors = predecessors.successor((P, i)) + new_context = ( + arg, + ((new_predecessors, 0), None), + ) + decorated_arguments_self.append((arg, (new_predecessors, 0))) + if new_context not in list_to_be_treated: + list_to_be_treated.appendleft(new_context) + rules[non_terminal][P] = (decorated_arguments_self, None) + + return CFG(start=initital_ctx, rules=rules) diff --git a/synth/syntax/grammars/det_grammar.py b/synth/syntax/grammars/det_grammar.py index 6d34e279..7bef2af8 100644 --- a/synth/syntax/grammars/det_grammar.py +++ b/synth/syntax/grammars/det_grammar.py @@ -1,18 +1,19 @@ from abc import ABC, abstractmethod from typing import ( + Any, Callable, Dict, List, Optional, + Set, Tuple, TypeVar, Generic, - Union, ) +from functools import lru_cache import copy -from synth.syntax.dsl import are_equivalent_primitives -from synth.syntax.grammars.grammar import Grammar +from synth.syntax.grammars.grammar import DerivableProgram, Grammar from synth.syntax.program import Constant, Function, Primitive, Program, Variable from synth.syntax.type_system import Arrow, Type @@ -21,10 +22,27 @@ W = TypeVar("W") T = TypeVar("T") -DerivableProgram = Union[Primitive, Variable, Constant] - class DetGrammar(Grammar, ABC, Generic[U, V, W]): + """ + Represents a deterministic grammar. + + (S) Non-terminals are Tuple[Type, U]. + (f) are Derivable programs + derivations are: + S -> f S1 ... Sk + there is no other derivation from S using f. + S1 ... Sk is of type V. + + When deriving an information of type W is maintained. + + Parameters: + ----------- + - start: the starting non-terminal of the grammar + - rules: the derivation rules + + """ + def __init__( self, start: Tuple[Type, U], @@ -37,6 +55,18 @@ def __init__( if clean: self.clean() + @lru_cache() + def primitives_used(self) -> Set[Primitive]: + """ + Returns the set of primitives used by this grammar. + """ + out: Set[Primitive] = set() + for S in self.rules: + for P in self.rules[S]: + if isinstance(P, Primitive): + out.add(P) + return out + def __hash__(self) -> int: return hash((self.start, str(self.rules))) @@ -62,20 +92,23 @@ def _guess_type_request_(self) -> Type: """ # Compute the type request type_req = self.start[0] - variables: List[Variable] = [] + self._variables: List[Variable] = [] for S in self.rules: for P in self.rules[S]: if isinstance(P, Variable): - if P not in variables: - variables.append(P) - n = len(variables) + if P not in self._variables: + self._variables.append(P) + n = len(self._variables) for i in range(n): j = n - i - 1 - for v in variables: + for v in self._variables: if v.variable == j: type_req = Arrow(v.type, type_req) return type_req + def variables(self) -> List[Variable]: + return self._variables[:] + def __contains__(self, program: Program) -> bool: return self.__contains_rec__(program, self.start, self.start_information())[0] @@ -148,10 +181,22 @@ def derive_all( @abstractmethod def arguments_length_for(self, S: Tuple[Type, U], P: DerivableProgram) -> int: + """ + Returns the number of arguments when deriving P from S. + """ + pass + + @abstractmethod + def instantiate_constants( + self, constants: Dict[Type, List[Any]] + ) -> "DetGrammar[U, V, W]": pass @abstractmethod def start_information(self) -> W: + """ + The starting information when deriving from a starting non-terminal. + """ pass def reduce_derivations( @@ -164,6 +209,8 @@ def reduce_derivations( """ Reduce the given program using the given reduce operator. + reduce: 'a, S, P, V -> 'a + reduce is called after derivation. """ @@ -195,52 +242,3 @@ def __reduce_derivations_rec__( value = reduce(value, start, program, self.rules[start][program]) return value, information, next return value, information, start - - def embed(self, program: Program) -> Optional[Program]: - """ - If the DSL has equivalent primitives, try to embed a program without equivalent primtives into this grammar. - """ - p = self.__embed__(program, self.start, self.start_information()) - return p - - def __embed__( - self, program: Program, start: Tuple[Type, U], information: W, level: int = 0 - ) -> Optional[Program]: - if isinstance(program, Function): - assert isinstance(program.function, Primitive) - possible_choices = [ - P - for P in self.rules[start] - if isinstance(P, Primitive) - and are_equivalent_primitives(P, program.function) - ] - for function in possible_choices: - args_P = program.arguments - new_information, next = self.derive( - copy.deepcopy(information), start, function - ) - nargs = [] - for arg in args_P: - narg = self.__embed__(arg, next, new_information, level + 1) - if narg is None: - break - nargs.append(narg) - new_information, lst = self.derive_all(new_information, next, narg) - next = lst[-1] - if len(nargs) != len(args_P): - continue - return Function(function, nargs) - return None - elif isinstance(program, Primitive): - possible_choices = [ - P - for P in self.rules[start] - if isinstance(P, Primitive) and are_equivalent_primitives(P, program) - ] - if len(possible_choices) == 0: - return None - elif len(possible_choices) == 1: - return possible_choices[0] - assert False, f"Ambigous possibilities: {possible_choices} from {start}" - else: - return program if program in self.rules[start] else None diff --git a/synth/syntax/grammars/dfa.py b/synth/syntax/grammars/dfa.py deleted file mode 100644 index 5baa0eff..00000000 --- a/synth/syntax/grammars/dfa.py +++ /dev/null @@ -1,36 +0,0 @@ -from typing import Dict, Generic, Tuple, TypeVar - - -U = TypeVar("U") -V = TypeVar("V") -W = TypeVar("W") -X = TypeVar("X") - - -class DFA(Generic[U, V]): - """ - Deterministic finite automaton. - Reads V elements from states U. - If there is no transition from U reading V it means it is non accepting. (there are no final states) - """ - - def __init__(self, initial: U, rules: Dict[U, Dict[V, U]]) -> None: - self.start = initial - self.rules = rules - - def __mul__(self, other: "DFA[W, X]") -> "DFA[Tuple[U, W], Tuple[V, X]]": - start = (self.start, other.start) - rules: Dict[Tuple[U, W], Dict[Tuple[V, X], Tuple[U, W]]] = {} - for S1 in self.rules: - for S2 in other.rules: - rules[(S1, S2)] = {} - for w1 in self.rules[S1]: - for w2 in other.rules[S2]: - rules[(S1, S2)][(w1, w2)] = ( - self.rules[S1][w1], - other.rules[S2][w2], - ) - return DFA(start, rules) - - def can_read(self, start: U, word: V) -> bool: - return start in self.rules and word in self.rules[start] diff --git a/synth/syntax/grammars/enumeration/__init__.py b/synth/syntax/grammars/enumeration/__init__.py new file mode 100644 index 00000000..61d84bbd --- /dev/null +++ b/synth/syntax/grammars/enumeration/__init__.py @@ -0,0 +1,25 @@ +from synth.syntax.grammars.enumeration.heap_search import ( + enumerate_prob_grammar as hs_enumerate_prob_grammar, + enumerate_bucket_prob_grammar as hs_enumerate_bucket_prob_grammar, +) +from synth.syntax.grammars.enumeration.u_heap_search import ( + enumerate_prob_u_grammar as hs_enumerate_prob_u_grammar, + enumerate_bucket_prob_u_grammar as hs_enumerate_bucket_prob_u_grammar, +) +from synth.syntax.grammars.enumeration.grammar_splitter import split +from synth.syntax.grammars.enumeration.program_enumerator import ProgramEnumerator + +from synth.syntax.grammars.enumeration.bee_search import ( + enumerate_prob_grammar as bs_enumerate_prob_grammar, +) +from synth.syntax.grammars.enumeration.beap_search import ( + enumerate_prob_grammar as bps_enumerate_prob_grammar, +) +from synth.syntax.grammars.enumeration.constant_delay import ( + enumerate_prob_grammar as cd_enumerate_prob_grammar, +) +from synth.syntax.grammars.enumeration.a_star import ( + enumerate_prob_grammar as as_enumerate_prob_grammar, +) + +enumerate_prob_grammar = cd_enumerate_prob_grammar diff --git a/synth/syntax/grammars/enumeration/a_star.py b/synth/syntax/grammars/enumeration/a_star.py new file mode 100644 index 00000000..1afc4ae9 --- /dev/null +++ b/synth/syntax/grammars/enumeration/a_star.py @@ -0,0 +1,133 @@ +from heapq import heappush, heappop +from typing import ( + Generator, + Generic, + List, + Optional, + Tuple, + TypeVar, + Union, +) +from dataclasses import dataclass, field + +import numpy as np + +from synth.filter.filter import Filter +from synth.syntax.grammars.enumeration.program_enumerator import ProgramEnumerator +from synth.syntax.grammars.tagged_u_grammar import ProbUGrammar +from synth.syntax.program import Function, Program +from synth.syntax.grammars.tagged_det_grammar import ProbDetGrammar, DerivableProgram +from synth.syntax.type_system import Type + +U = TypeVar("U") +V = TypeVar("V") +W = TypeVar("W") + + +def _build_( + elems: List[Tuple[DerivableProgram, Tuple[Type, U]]], G: ProbDetGrammar[U, V, W] +) -> Program: + P, S = elems.pop(0) + nargs = G.arguments_length_for(S, P) + if nargs == 0: + return P + else: + args = [] + while nargs > 0: + args.append(_build_(elems, G)) + nargs -= 1 + return Function(P, args) + + +@dataclass(order=True, frozen=True) +class HeapElement(Generic[U]): + priority: float + to_expand: List[Tuple[Type, U]] = field(compare=False) + parts: List[Tuple[DerivableProgram, Tuple[Type, U]]] = field(compare=False) + + def __repr__(self) -> str: + return f"({self.priority}, {self.parts})" + + def make_program(self, g: ProbDetGrammar[U, V, W]) -> Program: + return _build_(self.parts, g) + + +class AStar( + ProgramEnumerator[None], + Generic[U, V, W], +): + def __init__( + self, + G: ProbDetGrammar[U, V, W], + filter: Optional[Filter[Program]] = None, + ) -> None: + super().__init__(filter) + self.current: Optional[Program] = None + + self.G = G + self.start = G.start + self.rules = G.rules + + self.frontier: List[HeapElement[U]] = [] + + def probability(self, program: Program) -> float: + return self.G.probability(program) + + @classmethod + def name(cls) -> str: + return "a-star" + + def generator(self) -> Generator[Program, None, None]: + """ + A generator which outputs the next most probable program + """ + first = (self.G.start[0], self.G.start[1][0]) # type: ignore + heappush(self.frontier, HeapElement(0, [first], [])) + + while self.frontier: + elem = heappop(self.frontier) + if len(elem.to_expand) == 0: + p = elem.make_program(self.G) + if self._should_keep_subprogram(p): + yield p + else: + partS = elem.to_expand.pop() + S = (partS[0], (partS[1], None)) + for P in self.G.rules[S]: # type: ignore + args = self.G.rules[S][P][0] # type: ignore + p = self.G.probabilities[S][P] # type: ignore + new_el = HeapElement( + elem.priority + p, # type: ignore + elem.to_expand + list(args), + elem.parts + [(P, S)], + ) + heappush(self.frontier, new_el) + + def merge_program(self, representative: Program, other: Program) -> None: + """ + Merge other into representative. + In other words, other will no longer be generated through heap search + """ + pass + + def programs_in_banks(self) -> int: + return 0 + + def programs_in_queues(self) -> int: + return len(self.frontier) + + def clone(self, G: Union[ProbDetGrammar, ProbUGrammar]) -> "AStar[U, V, W]": + assert isinstance(G, ProbDetGrammar) + enum = self.__class__(G) + return enum + + +def enumerate_prob_grammar(G: ProbDetGrammar[U, V, W]) -> AStar[U, V, W]: + Gp: ProbDetGrammar = ProbDetGrammar( + G.grammar, + { + S: {P: -np.log(p) for P, p in val.items() if p > 0} + for S, val in G.probabilities.items() + }, + ) + return AStar(Gp) diff --git a/synth/syntax/grammars/enumeration/beap_search.py b/synth/syntax/grammars/enumeration/beap_search.py new file mode 100644 index 00000000..9e69ff0a --- /dev/null +++ b/synth/syntax/grammars/enumeration/beap_search.py @@ -0,0 +1,274 @@ +from itertools import product +from heapq import heappush, heappop, heapify +from typing import ( + Dict, + Generator, + Generic, + List, + Optional, + Set, + Tuple, + TypeVar, + Union, +) +from dataclasses import dataclass, field + +import numpy as np + +from synth.filter.filter import Filter +from synth.syntax.grammars.cfg import CFG +from synth.syntax.grammars.enumeration.program_enumerator import ProgramEnumerator +from synth.syntax.grammars.grammar import DerivableProgram +from synth.syntax.program import Program, Function +from synth.syntax.grammars.tagged_det_grammar import ProbDetGrammar +from synth.syntax.grammars.tagged_u_grammar import ProbUGrammar +from synth.syntax.type_system import Type + +U = TypeVar("U") +V = TypeVar("V") +W = TypeVar("W") + + +@dataclass(order=True, frozen=True) +class HeapElement: + cost: float + combination: List[int] + P: DerivableProgram = field(compare=False) + + def __repr__(self) -> str: + return f"({self.cost}, {self.combination}, {self.P})" + + +class BeapSearch( + ProgramEnumerator[None], + Generic[U, V, W], +): + def __init__( + self, G: ProbDetGrammar[U, V, W], filter: Optional[Filter[Program]] = None + ) -> None: + super().__init__(filter) + assert isinstance(G.grammar, CFG) + self.G = G + self.cfg: CFG = G.grammar + self._deleted: Set[Program] = set() + + # S -> cost list + # IDEA: Change from cost list to increase diffs + self._cost_lists: Dict[Tuple[Type, U], List[float]] = {} + # S -> cost_index -> program list + self._bank: Dict[Tuple[Type, U], Dict[int, List[Program]]] = {} + # S -> heap of HeapElement queued + self._queues: Dict[Tuple[Type, U], List[HeapElement]] = {} + # S -> cost index set + self._empties: Dict[Tuple[Type, U], Set[int]] = {} + + self._non_terminal_for: Dict[ + Tuple[Type, U], Dict[DerivableProgram, List[Tuple[Type, U]]] + ] = {} + + for S in self.G.grammar.rules: + self._cost_lists[S] = [] + self._bank[S] = {} + self._empties[S] = set() + self._queues[S] = [] + self._non_terminal_for[S] = { + P: [(Sp[0], (Sp[1], None)) for Sp in self.G.rules[S][P][0]] # type: ignore + for P in self.G.grammar.rules[S] + } + + def _init_non_terminal_(self, S: Tuple[Type, U]) -> None: + if len(self._cost_lists[S]) > 0: + return + self._cost_lists[S].append(1e99) + queue = self._queues[S] + for P in self.G.rules[S]: + # Init args + nargs = self.G.arguments_length_for(S, P) + cost = self.G.probabilities[S][P] + for Si in self._non_terminal_for[S][P]: + self._init_non_terminal_(Si) + cost += self._cost_lists[Si][0] + index_cost = [0] * nargs + heappush(queue, HeapElement(cost, index_cost, P)) + + self._cost_lists[S][0] = queue[0].cost + + def _reevaluate_(self) -> None: + if not self.cfg.is_recursive(): + return + changed = True + while changed: + changed = False + for S in list(self._queues.keys()): + new_queue = [ + HeapElement( + self.G.probabilities[S][el.P] + + sum( + self._cost_lists[Si][0] + for Si in self._non_terminal_for[S][el.P] + ), + el.combination, + el.P, + ) + for el in self._queues[S] + ] + if new_queue != self._queues[S]: + changed = True + heapify(new_queue) + self._queues[S] = new_queue + self._cost_lists[S][0] = self._queues[S][0].cost + + def generator(self) -> Generator[Program, None, None]: + self._init_non_terminal_(self.G.start) + self._reevaluate_() + n = 0 + failed = False + while not failed: + self._failed_by_empties = False + failed = True + for prog in self.query(self.G.start, n): + failed = False + yield prog + failed = failed and not self._failed_by_empties + n += 1 + + def programs_in_banks(self) -> int: + return sum(sum(len(x) for x in val.values()) for val in self._bank.values()) + + def programs_in_queues(self) -> int: + return sum(len(val) for val in self._queues.values()) + + def query( + self, S: Tuple[Type, U], cost_index: int + ) -> Generator[Program, None, None]: + # When we return this way, it actually mean that we have generated all programs that this non terminal could generate + if cost_index >= len(self._cost_lists[S]): + return + cost = self._cost_lists[S][cost_index] + has_generated_program = False + no_successor = True + bank = self._bank[S] + queue = self._queues[S] + while len(queue) > 0 and queue[0].cost == cost: + element = heappop(queue) + Sargs = self._non_terminal_for[S][element.P] + nargs = len(Sargs) + # necessary for finite grammars + arg_gen_failed = False + is_allowed_empty = False + # is_allowed_empty => arg_gen_failed + # Generate programs + args_possibles = [] + for i in range(nargs): + one_is_allowed_empty, possibles = self._query_list_( + Sargs[i], element.combination[i] + ) + is_allowed_empty |= one_is_allowed_empty + if len(possibles) == 0: + arg_gen_failed = True + if not one_is_allowed_empty: + break + args_possibles.append(possibles) + failed_for_other_reasons = arg_gen_failed and not is_allowed_empty + no_successor = no_successor and failed_for_other_reasons + # a Non terminal as arg is finite and we reached the end of enumeration + if failed_for_other_reasons: + continue + # Generate next combinations + for i in range(nargs): + cl = self._cost_lists[Sargs[i]] + # Finite grammar has reached the end of costs for Sarg[i] + if element.combination[i] + 1 >= len(cl): + # Either index_cost[i] > 1 so we break or + # index_cost[i] = 1 but then len(cl) = 1 so we need to check + if element.combination[i] + 1 > 1: + break + continue + index_cost = element.combination.copy() + index_cost[i] += 1 + new_cost = cost - cl[index_cost[i] - 1] + cl[index_cost[i]] + heappush(queue, HeapElement(new_cost, index_cost, element.P)) + # Avoid duplication with this condition + if index_cost[i] > 1: + break + # If empty cost index set then no need to generate programs + if is_allowed_empty: + continue + + if cost_index not in bank: + bank[cost_index] = [] + for new_args in product(*args_possibles): + if len(args_possibles) > 0: + new_program: Program = Function(element.P, list(new_args)) + else: + new_program = element.P + if new_program in self._deleted: + continue + elif not self._should_keep_subprogram(new_program): + self._deleted.add(new_program) + continue + has_generated_program = True + bank[cost_index].append(new_program) + yield new_program + if not has_generated_program: + # If we failed because of allowed empties we can tag this as allowed empty + if not no_successor: + self._empties[S].add(cost_index) + self._failed_by_empties = True + if len(queue) > 0: + next_cost = queue[0].cost + self._cost_lists[S].append(next_cost) + + def _query_list_( + self, S: Tuple[Type, U], cost_index: int + ) -> Tuple[bool, List[Program]]: + """ + returns is_allowed_empty, programs + """ + # It's an empty cost index but a valid one + if cost_index in self._empties[S]: + return True, [] + if cost_index >= len(self._cost_lists[S]): + return False, [] + bank = self._bank[S] + if cost_index in bank: + return False, bank[cost_index] + for x in self.query(S, cost_index): + pass + if cost_index in self._empties[S]: + return True, [] + return False, bank[cost_index] + + def merge_program(self, representative: Program, other: Program) -> None: + self._deleted.add(other) + for S in self.G.rules: + if S[0] != other.type: + continue + local_bank = self._bank[S] + for programs in local_bank.values(): + if other in programs: + programs.remove(other) + + def probability(self, program: Program) -> float: + return self.G.probability(program) + + @classmethod + def name(cls) -> str: + return "beap-search" + + def clone(self, G: Union[ProbDetGrammar, ProbUGrammar]) -> "BeapSearch[U, V, W]": + assert isinstance(G, ProbDetGrammar) + enum = self.__class__(G) + enum._deleted = self._deleted.copy() + return enum + + +def enumerate_prob_grammar(G: ProbDetGrammar[U, V, W]) -> BeapSearch[U, V, W]: + Gp: ProbDetGrammar = ProbDetGrammar( + G.grammar, + { + S: {P: -np.log(p) for P, p in val.items() if p > 0} + for S, val in G.probabilities.items() + }, + ) + return BeapSearch(Gp) diff --git a/synth/syntax/grammars/enumeration/bee_search.py b/synth/syntax/grammars/enumeration/bee_search.py new file mode 100644 index 00000000..198f428e --- /dev/null +++ b/synth/syntax/grammars/enumeration/bee_search.py @@ -0,0 +1,285 @@ +from collections import defaultdict +from itertools import product +from heapq import heappush, heappop +from typing import ( + Dict, + Generator, + Generic, + List, + Optional, + Set, + Tuple, + TypeVar, + Union, +) +from dataclasses import dataclass, field + +import numpy as np + +from synth.filter.filter import Filter +from synth.syntax.grammars.cfg import CFG +from synth.syntax.grammars.enumeration.program_enumerator import ProgramEnumerator +from synth.syntax.grammars.grammar import DerivableProgram +from synth.syntax.program import Program, Function +from synth.syntax.grammars.tagged_det_grammar import ProbDetGrammar +from synth.syntax.grammars.tagged_u_grammar import ProbUGrammar +from synth.syntax.type_system import Type + +U = TypeVar("U") +V = TypeVar("V") +W = TypeVar("W") + + +@dataclass(order=True, frozen=True) +class HeapElement: + cost: float + combination: List[int] + P: DerivableProgram = field(compare=False) + + def __repr__(self) -> str: + return f"({self.cost}, {self.combination}, {self.P})" + + +class BeeSearch( + ProgramEnumerator[None], + Generic[U, V, W], +): + def __init__( + self, G: ProbDetGrammar[U, V, W], filter: Optional[Filter[Program]] = None + ) -> None: + super().__init__(filter) + assert isinstance(G.grammar, CFG) + self.G = G + self._deleted: Set[Program] = set() + + self._cost_list: List[float] = [] + # S -> cost_index -> program list + self._bank: Dict[Tuple[Type, U], Dict[int, List[Program]]] = {} + # S -> heap of HeapElement queued + self._prog_queued: Dict[Tuple[Type, U], List[HeapElement]] = {} + # S -> max index currently queued + self._max_index: Dict[Tuple[Type, U], int] = defaultdict(int) + self._delayed: Dict[ + Tuple[Type, U], List[Tuple[List[int], DerivableProgram, Optional[int]]] + ] = defaultdict(list) + self._has_merged = False + # Fill terminals first + for S in self.G.rules: + self._bank[S] = {} + self._prog_queued[S] = [] + + for P in self.G.rules[S]: + nargs = self.G.arguments_length_for(S, P) + if nargs == 0: + self._add_combination_(S, P, []) + + # Init non terminals (otherwise add combination won't work correctly) + for S in self.G.rules: + for P in self.G.rules[S]: + nargs = self.G.arguments_length_for(S, P) + if nargs > 0: + index_cost = [0] * nargs + self._add_combination_(S, P, index_cost) + + def _add_combination_( + self, + S: Tuple[Type, U], + P: DerivableProgram, + index_cost: List[int], + changed_index: Optional[int] = None, + ) -> None: + # Check if it needs to be delayed or not + to_check = ( + [changed_index] + if changed_index is not None + else [i for i in range(len(index_cost))] + ) + for i in to_check: + if index_cost[i] >= len(self._cost_list): + self._delayed[S].append((index_cost, P, changed_index)) + return + # No need to delay add it + new_cost = self._index_cost2real_cost_(S, P, index_cost) + heappush( + self._prog_queued[S], + HeapElement(new_cost, index_cost, P), + ) + + def _trigger_delayed_(self) -> None: + copy = self._delayed.copy() + self._delayed.clear() + for S, elements in copy.items(): + for index_cost, P, to_check in elements: + self._add_combination_(S, P, index_cost, to_check) + + def _add_cost_(self, S: Tuple[Type, U], cost: float) -> Tuple[bool, int]: + cost_list = self._cost_list + if len(cost_list) > 0 and cost_list[-1] == cost: + return False, len(cost_list) - 1 + # assert len(cost_list) == 0 or cost > cost_list[-1], f"{cost} -> {cost_list}" + # print("adding:", cost, "to", cost_list) + cost_list.append(cost) + self._trigger_delayed_() + return True, len(cost_list) - 1 + + def _add_program_( + self, S: Tuple[Type, U], new_program: Program, cost_index: int + ) -> bool: + if new_program in self._deleted: + return False + if not self._should_keep_subprogram(new_program): + self._deleted.add(new_program) + return False + local_bank = self._bank[S] + if cost_index not in local_bank: + local_bank[cost_index] = [] + # assert max(local_bank.keys()) + 1 == len(self._cost_lists[S]), f"index:{cost_index} {local_bank.keys()} vs {len(self._cost_lists[S])}" + local_bank[cost_index].append(new_program) + return True + + def _index_cost2real_cost_( + self, S: Tuple[Type, U], P: DerivableProgram, indices: List[int] + ) -> float: + out = self.G.probabilities[S][P] + for i in range(self.G.arguments_length_for(S, P)): + out += self._cost_list[indices[i]] + return out + + def _non_terminal_for_( + self, S: Tuple[Type, U], P: DerivableProgram, index: int + ) -> Tuple[Type, U]: + Sp = self.G.rules[S][P][0][index] # type: ignore + return (Sp[0], (Sp[1], None)) # type: ignore + + def generator(self) -> Generator[Program, None, None]: + progs = self.G.programs() + infinite = progs < 0 + failed = 0 + while ( + infinite + or (self._has_merged and failed < 1000) + or (not self._has_merged and progs > 0) + ): + non_terminals, cost = self._next_cheapest_() + if cost is None: + break + if len(non_terminals) == 0: + break + failed += 1 + succ = False + for program in self._produce_programs_from_cost_(non_terminals, cost): + progs -= 1 + if not succ: + failed -= 1 + succ = True + yield program + + def _next_cheapest_(self) -> Tuple[List[Tuple[Type, U]], Optional[float]]: + """ + WORKS + """ + cheapest = None + non_terminals_container: List[Tuple[Type, U]] = [] + for S, heap in self._prog_queued.items(): + if len(heap) == 0: + continue + item = heap[0] + smallest_cost = item.cost + if cheapest is None or smallest_cost <= cheapest: + if cheapest is None or smallest_cost < cheapest: + non_terminals_container = [] + cheapest = smallest_cost + non_terminals_container.append(S) + return non_terminals_container, cheapest + + def _produce_programs_from_cost_( + self, non_terminals: List[Tuple[Type, U]], cost: float + ) -> Generator[Program, None, None]: + for S in non_terminals: + queue = self._prog_queued[S] + maxi = self._max_index[S] + # Add cost since we are pre-generative + cost_index = self._add_cost_(S, cost)[1] + + while queue and queue[0].cost == cost: + element = heappop(queue) + assert element.cost == cost + nargs = self.G.arguments_length_for(S, element.P) + Sargs = [self._non_terminal_for_(S, element.P, i) for i in range(nargs)] + # Generate next combinations + for i in range(nargs): + index_cost = element.combination.copy() + index_cost[i] += 1 + self._add_combination_(S, element.P, index_cost, i) + if index_cost[i] > maxi: + maxi = index_cost[i] + # Avoid duplication with this condition + if index_cost[i] > 1: + break + # Generate programs + args_possibles = [] + for i in range(nargs): + local_bank = self._bank[Sargs[i]] + ci = element.combination[i] + if ci not in local_bank or len(local_bank[ci]) == 0: + break + args_possibles.append(local_bank[ci]) + if len(args_possibles) != nargs: + # print("failed") + continue + for new_args in product(*args_possibles): + if len(args_possibles) == 0: + new_program: Program = element.P + else: + new_program = Function(element.P, list(new_args)) + if ( + self._add_program_(S, new_program, cost_index) + and S == self.G.start + ): + yield new_program + self._max_index[S] = maxi + + def merge_program(self, representative: Program, other: Program) -> None: + self._has_merged = True + self._deleted.add(other) + for S in self.G.rules: + if S[0] != other.type: + continue + local_bank = self._bank[S] + for programs in local_bank.values(): + if other in programs: + programs.remove(other) + + def probability(self, program: Program) -> float: + return self.G.probability(program) + + def programs_in_banks(self) -> int: + return sum(sum(len(x) for x in val.values()) for val in self._bank.values()) + + def programs_in_queues(self) -> int: + return sum(len(val) for val in self._delayed.values()) + sum( + len(val) for val in self._prog_queued.values() + ) + + @classmethod + def name(cls) -> str: + return "bee-search" + + def clone(self, G: Union[ProbDetGrammar, ProbUGrammar]) -> "BeeSearch[U, V, W]": + assert isinstance(G, ProbDetGrammar) + enum = self.__class__(G) + return enum + + +def enumerate_prob_grammar( + G: ProbDetGrammar[U, V, W], threshold: int = 2 +) -> BeeSearch[U, V, W]: + mult = 10**threshold + Gp: ProbDetGrammar = ProbDetGrammar( + G.grammar, + { + S: {P: int(-np.log(p) * mult) for P, p in val.items() if p > 0} + for S, val in G.probabilities.items() + }, + ) + return BeeSearch(Gp) diff --git a/synth/syntax/grammars/enumeration/constant_delay.py b/synth/syntax/grammars/enumeration/constant_delay.py new file mode 100644 index 00000000..473a132b --- /dev/null +++ b/synth/syntax/grammars/enumeration/constant_delay.py @@ -0,0 +1,452 @@ +from itertools import product +from heapq import heappush, heappop, heapify +from typing import ( + Dict, + Generator, + Generic, + List, + Optional, + Set, + Tuple, + TypeVar, + Union, +) +from dataclasses import dataclass, field + +import numpy as np + +from synth.filter.filter import Filter +from synth.syntax.grammars.cfg import CFG +from synth.syntax.grammars.enumeration.program_enumerator import ProgramEnumerator +from synth.syntax.grammars.enumeration.constant_delay_queue import CDQueue, CostTuple +from synth.syntax.grammars.grammar import DerivableProgram +from synth.syntax.program import Program, Function +from synth.syntax.grammars.tagged_det_grammar import ProbDetGrammar +from synth.syntax.grammars.tagged_u_grammar import ProbUGrammar +from synth.syntax.type_system import Type + +U = TypeVar("U") +V = TypeVar("V") +W = TypeVar("W") + + +@dataclass(order=True, frozen=True) +class Derivation: + cost: float + combination: int + P: DerivableProgram = field(compare=False) + + def __repr__(self) -> str: + return f"({self.cost}, {self.combination}, {self.P})" + + +class CDSearch( + ProgramEnumerator[None], + Generic[U, V, W], +): + def __init__( + self, + G: ProbDetGrammar[U, V, W], + filter: Optional[Filter[Program]] = None, + k: int = 5, + ) -> None: + super().__init__(filter) + assert isinstance(G.grammar, CFG) + self.G = G + self.cfg: CFG = G.grammar + self._deleted: Set[Program] = set() + + # compute larger M + all_costs = set( + self.G.probabilities[S][P] for S in self.G.rules for P in self.G.rules[S] + ) + self.basic_M = max(abs(x - y) for x in all_costs for y in all_costs) + self.M = ( + self.basic_M + * ( + max( + self.G.arguments_length_for(S, P) + for S in self.G.rules + for P in self.G.rules[S] + ) + + 1.0 + ) + * len(self.G.rules) + ) + + # Naming + # nt -> one non terminal + # derivation -> (S1, S2) + + self._queue_nt: Dict[Tuple[Type, U], List[Derivation]] = {} + self._queue_derivation: Dict[Tuple[Tuple[Type, U]], CDQueue] = {} + self._bank_nt: Dict[Tuple[Type, U], Dict[int, List[Program]]] = {} + self._bank_derivation: Dict[ + Tuple[Tuple[Type, U]], Dict[int, List[List[List[Program]]]] + ] = {} + self._cost_lists_nt: Dict[Tuple[Type, U], List[float]] = {} + self._cost_lists_derivation: Dict[Tuple[Tuple[Type, U]], List[float]] = {} + self._non_terminal_for: Dict[ + Tuple[Type, U], Dict[DerivableProgram, Tuple[Tuple[Type, U]]] + ] = {} + self._empties_nt: Dict[Tuple[Type, U], Set[int]] = {} + self._empties_derivation: Dict[Tuple[Tuple[Type, U]], Set[int]] = {} + + for S in self.G.grammar.rules: + self._queue_nt[S] = [] + self._cost_lists_nt[S] = [] + self._bank_nt[S] = {} + self._empties_nt[S] = set() + self._non_terminal_for[S] = { + P: tuple([(Sp[0], (Sp[1], None)) for Sp in self.G.rules[S][P][0]]) # type: ignore + for P in self.G.grammar.rules[S] + } + for P in self.G.grammar.rules[S]: + args = self._non_terminal_for[S][P] + if args and args not in self._queue_derivation: + self._queue_derivation[args] = CDQueue(int(self.M), k) + self._bank_derivation[args] = {} + self._cost_lists_derivation[args] = [] + self._empties_derivation[args] = set() + + def _peek_next_derivation_cost_( + self, S: Tuple[Type, U], P: DerivableProgram + ) -> float: + args = self._non_terminal_for[S][P] + if args: + return self._queue_derivation[args].peek().cost + return 0 + + def _queue_size_(self, S: Tuple[Type, U], P: DerivableProgram) -> int: + args = self._non_terminal_for[S][P] + return self._queue_derivation[args].size() + + def _init_derivation_(self, S: Tuple[Type, U], P: DerivableProgram) -> None: + args = self._non_terminal_for[S][P] + if len(self._cost_lists_derivation[args]) > 0: + return + cost = 0.0 + self._cost_lists_derivation[args].append(1e99) + queue = self._queue_derivation[args] + for Si in args: + self._init_non_terminal_(Si) + cost += self._cost_lists_nt[Si][0] + index_cost = [0] * len(args) + ct = CostTuple(cost, [index_cost]) + queue.push(ct) + queue.update() + self._cost_lists_derivation[args][0] = queue.peek().cost + + def _init_non_terminal_(self, S: Tuple[Type, U]) -> None: + if len(self._cost_lists_nt[S]) > 0: + return + self._cost_lists_nt[S].append(1e99) + queue = self._queue_nt[S] + for P in self.G.rules[S]: + args = self._non_terminal_for[S][P] + if args: + self._init_derivation_(S, P) + base_cost = self._cost_lists_derivation[args][0] + else: + base_cost = 0 + heappush(queue, Derivation(base_cost + self.G.probabilities[S][P], 0, P)) + + self._cost_lists_nt[S][0] = queue[0].cost + + def _reevaluate_derivation_(self, S: Tuple[Type, U], P: DerivableProgram) -> None: + args = self._non_terminal_for[S][P] + if args: + queue = self._queue_derivation[args] + elems = [] + while not queue.is_empty(): + elem = queue.pop() + elems.append( + CostTuple( + sum(self._queue_nt[Si][0].cost for Si in args), + elem.combinations, + ) + ) + elems = sorted(elems) + queue.clear() + while not (len(elems) == 0): + queue.push(elems.pop()) + self._cost_lists_derivation[args][0] = queue.peek().cost + + def _reevaluate_(self) -> None: + changed = True + while changed: + changed = False + + for S in list(self._queue_nt.keys()): + for P in self.G.rules[S]: + self._reevaluate_derivation_(S, P) + new_queue = [ + Derivation( + self.G.probabilities[S][el.P] + + self._peek_next_derivation_cost_(S, el.P), + el.combination, + el.P, + ) + for el in self._queue_nt[S] + ] + if new_queue != self._queue_nt[S]: + changed = True + heapify(new_queue) + self._queue_nt[S] = new_queue + self._cost_lists_nt[S][0] = self._queue_nt[S][0].cost + + def __compute_bounds__(self) -> None: + diff = {S: sorted(el.cost for el in self._queue_nt[S]) for S in self.G.rules} + + values = {S: int(diff[S][-1] - diff[S][0]) for S in self.G.rules} + changed = True + while changed: + changed = False + for S in self.G.rules: + for P in self.G.rules[S]: + maxi = max( + (values[arg] for arg in self._non_terminal_for[S][P]), default=0 + ) + if maxi > values[S]: + values[S] = maxi + changed = True + + # print("previous M:", self.M) + # print("new M:", values[self.G.start]) + # print(self.G) + # for S, val in values.items(): + # print("S=", S, "M=", val) + # Update queues + for arg in self._queue_derivation: + elems = [] + queue = self._queue_derivation[arg] + # print("before:", queue) + while not queue.is_empty(): + elems.append(queue.pop()) + # The max 1 is for a terminal rule where all derivations have same cost + M = max(1, max(values[X] for X in arg)) + self._queue_derivation[arg] = CDQueue(M, queue.k - 1 if M > 1000 else M) + elems = sorted(elems) + while not (len(elems) == 0): + self._queue_derivation[arg].push(elems.pop(0)) + assert len(self._queue_derivation[arg]) == 1 + + def generator(self) -> Generator[Program, None, None]: + self._init_non_terminal_(self.G.start) + self._reevaluate_() + # Update M + self.__compute_bounds__() + + n = 0 + failed = False + while not failed: + self._failed_by_empties = False + failed = True + for prog in self.query(self.G.start, n): + failed = False + yield prog + failed = failed and not self._failed_by_empties + n += 1 + + def programs_in_banks(self) -> int: + return sum(sum(len(x) for x in val.values()) for val in self._bank_nt.values()) + + def programs_in_queues(self) -> int: + return sum(len(val) for val in self._queue_nt.values()) + sum( + cd.size() for cd in self._queue_derivation.values() + ) + + def query_derivation( + self, S: Tuple[Type, U], P: DerivableProgram, cost_index: int + ) -> List[List[List[Program]]]: + args = self._non_terminal_for[S][P] + if cost_index >= len(self._cost_lists_derivation[args]): + return [] + # Memoization + bank = self._bank_derivation[args] + if cost_index in bank: + return bank[cost_index] + # Generation + bank[cost_index] = [] + queue = self._queue_derivation[args] + # print("before QUERY:", queue) + # print("\tS=", S) + no_successor = True + has_generated_program = False + + if not queue.is_empty(): + ct = queue.pop() + # print("\t\tAFTER POP:", queue) + # print("\t\tpopping:", ct) + for combination in ct.combinations: + arg_gen_failed = False + is_allowed_empty = False + # is_allowed_empty => arg_gen_failed + args_possibles = [] + for ci, Si in zip(combination, args): + one_is_allowed_empty, elems = self._query_list_(Si, ci) + is_allowed_empty |= one_is_allowed_empty + if len(elems) == 0: + arg_gen_failed = True + if not one_is_allowed_empty: + break + args_possibles.append(elems) + + failed_for_other_reasons = arg_gen_failed and not is_allowed_empty + no_successor = no_successor and failed_for_other_reasons + # a Non terminal as arg is finite and we reached the end of enumeration + if failed_for_other_reasons: + continue + + # add successors + for i in range(len(combination)): + cl = self._cost_lists_nt[args[i]] + if combination[i] + 1 >= len(cl): + if combination[i] + 1 > 1: + break + continue + index_cost = combination.copy() + index_cost[i] += 1 + new_cost = ct.cost - cl[index_cost[i] - 1] + cl[index_cost[i]] + # print("\t\tpushing:", new_cost, ">", ct.cost, "cost tuple:", index_cost) + queue.push(CostTuple(new_cost, [index_cost])) + # print("\t\tAFTER PUSH:", queue) + # Avoid duplication with this condition + if index_cost[i] > 1: + break + if is_allowed_empty: + continue + has_generated_program = True + bank[cost_index].append(args_possibles) + # print("after QUERY:", queue) + # print(f"[BANK {args}][{cost_index}] = {bank[cost_index]}") + if not has_generated_program: + # If we failed because of allowed empties we can tag this as allowed empty + if not no_successor: + self._empties_derivation[args].add(cost_index) + if not queue.is_empty(): + queue.update() + # print("BEFORE PEEK:", queue) + self._cost_lists_derivation[args].append(queue.peek().cost) + return bank[cost_index] + + def query( + self, S: Tuple[Type, U], cost_index: int + ) -> Generator[Program, None, None]: + # When we return this way, it actually mean that we have generated all programs that this non terminal could generate + if cost_index >= len(self._cost_lists_nt[S]): + return + cost = self._cost_lists_nt[S][cost_index] + bank = self._bank_nt[S] + queue = self._queue_nt[S] + has_generated_program = False + no_successor = True + while len(queue) > 0 and queue[0].cost == cost: + element = heappop(queue) + # print("[POP]:", element) + if cost_index not in bank: + bank[cost_index] = [] + args = self._non_terminal_for[S][element.P] + if args: + args_possibles = self.query_derivation( + S, element.P, element.combination + ) + is_empty = element.combination in self._empties_derivation[args] + # Finite nonterminal check + if element.combination + 1 < len(self._cost_lists_derivation[args]): + next_cost = ( + self.G.probabilities[S][element.P] + + self._cost_lists_derivation[args][element.combination + 1] + ) + heappush( + queue, Derivation(next_cost, element.combination + 1, element.P) + ) + no_successor = False + if is_empty: + continue + # Generate programs + for possibles in args_possibles: + # print("S", S, "P", element.P, "index:", element.combination, "args:", possibles) + for new_args in product(*possibles): + new_program: Program = Function(element.P, list(new_args)) + if new_program in self._deleted: + continue + elif not self._should_keep_subprogram(new_program): + self._deleted.add(new_program) + continue + has_generated_program = True + bank[cost_index].append(new_program) + yield new_program + else: + new_program = element.P + if new_program in self._deleted: + continue + elif not self._should_keep_subprogram(new_program): + self._deleted.add(new_program) + continue + bank[cost_index].append(new_program) + has_generated_program = True + yield new_program + if not has_generated_program: + if not no_successor: + self._failed_by_empties = True + self._empties_nt[S].add(cost_index) + if len(queue) > 0: + next_cost = queue[0].cost + self._cost_lists_nt[S].append(next_cost) + + def _query_list_( + self, S: Tuple[Type, U], cost_index: int + ) -> Tuple[bool, List[Program]]: + """ + returns is_allowed_empty, programs + """ + # It's an empty cost index but a valid one + if cost_index in self._empties_nt[S]: + return True, [] + if cost_index >= len(self._cost_lists_nt[S]): + return False, [] + bank = self._bank_nt[S] + if cost_index in bank: + return False, bank[cost_index] + for x in self.query(S, cost_index): + pass + if cost_index in self._empties_nt[S]: + return True, [] + return False, bank[cost_index] + + def merge_program(self, representative: Program, other: Program) -> None: + self._deleted.add(other) + for S in self.G.rules: + if S[0] != other.type: + continue + local_bank = self._bank_nt[S] + for programs in local_bank.values(): + if other in programs: + programs.remove(other) + + def probability(self, program: Program) -> float: + return self.G.probability(program) + + @classmethod + def name(cls) -> str: + return "cd-search" + + def clone(self, G: Union[ProbDetGrammar, ProbUGrammar]) -> "CDSearch[U, V, W]": + assert isinstance(G, ProbDetGrammar) + enum = self.__class__(G) + enum._deleted = self._deleted.copy() + return enum + + +def enumerate_prob_grammar( + G: ProbDetGrammar[U, V, W], k: int = 10, precision: float = 1e-5 +) -> CDSearch[U, V, W]: + Gp: ProbDetGrammar = ProbDetGrammar( + G.grammar, + { + S: {P: -int(np.log(p) * 1 / precision) for P, p in val.items() if p > 0} + for S, val in G.probabilities.items() + }, + ) + return CDSearch(Gp, k=k) diff --git a/synth/syntax/grammars/enumeration/constant_delay_queue.py b/synth/syntax/grammars/enumeration/constant_delay_queue.py new file mode 100644 index 00000000..8574deb9 --- /dev/null +++ b/synth/syntax/grammars/enumeration/constant_delay_queue.py @@ -0,0 +1,233 @@ +from dataclasses import dataclass +from typing import List, Optional, Tuple, Union, Any + + +@dataclass(order=True, frozen=True) +class CostTuple: + cost: float + combinations: List[List[int]] + + def __repr__(self) -> str: + return f"({self.cost}, {self.combinations})" + + +class CDQueue: + """ + Init: + push* + + Usage: + pop + push* + update + + """ + + def __init__(self, maxi: int, k: int) -> None: + # multiply otherwise when you get exactly maxi then it is equal to 0 + self.maxi = maxi * (k + 1) / k + self.k = k + 1 + self.mini: Optional[float] = None + self.cells: List[Tuple[int, Optional[Union[CostTuple, Any]]]] = [ + (0, None) for _ in range(self.k) + ] + self.translation = 0 + self.nelements = 0 + self.start: Optional[float] = None + self.n = 0 + self.clear() + + def update(self) -> None: + """ + Update its internal representation, should be done after all elements have been pushed. + """ + if self.nelements > 0: + while self.cells[self.translation][0] == 0: + self.translation = (self.translation + 1) % self.k + self.n += 1 + if self.nelements == 1: + self.mini = self.cells[self.translation][1].cost # type: ignore + self.start = self.mini + self.n = 0 + else: + self.mini = self.start + self.maxi * self.n / self.k # type: ignore + + def clear(self) -> None: + """ + Clear this queue of all of its elements. + """ + self.mini = None + self.cells = [(0, None) for _ in range(self.k)] + self.translation = 0 + self.nelements = 0 + self.start = None + self.n = 0 + + def push(self, element: CostTuple) -> None: + # print("PUSH") + if self.mini is None: + self.mini = element.cost + self.start = self.mini + self.n = 0 + assert element.cost - self.mini <= self.maxi + # a = f"[PUSH] {F.LIGHTYELLOW_EX}BEFORE{F.RESET}:{self}" + # print(f"[PUSH] {F.LIGHTYELLOW_EX}BEFORE{F.RESET}:", self) + # b = f"\t\t\t{element}" + # print("\t\t\t", element) + self.__push__( + element, element.cost - self.mini, self.maxi, self.cells, self.translation + ) + # if x or abs(element.cost - 11.040864126152853) <= 1e-3: + # print("n=", self.n, "unit:", self.maxi / self.k) + # print(a) + # print(b) + # print(f"[PUSH] {F.GREEN}AFTER{F.RESET}:", self) + + def __push__( + self, + element: CostTuple, + cost: float, + maxi: float, + cells: List, + translation: int, + add: bool = True, + ) -> None: + stack: List[Tuple[List[List[int]], int]] = [] + while True: + unit = maxi / self.k + lbi = int(cost / maxi * self.k) + index = (lbi + translation) % self.k + # assert cost >= 0, f"cost:{cost} element:{element} queue:{self}" + nelem, val = cells[index] + # print("\tfound:", nelem, "val=", val) + if nelem == 0: + # print("\t\tfilling...") + cells[index] = (1, element) + if add: + self.nelements += 1 + for c, i in stack: + c[i][0] += 1 + return + + elif nelem == 1: + if abs(val.cost - element.cost) > 1: + # print(f"\t\tsplitting with {val}... diff={abs(val.cost - element.cost)} ") + + cells[index] = [2, [(0, None) for _ in range(self.k)]] + self.__push__( + val, + cost + val.cost - element.cost - lbi * unit, + unit, + cells[index][1], + 0, + add=False, + ) + self.__push__( + element, + cost - lbi * unit, + unit, + cells[index][1], + 0, + add=False, + ) + if add: + self.nelements += 1 + for c, i in stack: + c[i][0] += 1 + return + else: + val.combinations.extend(element.combinations) + # print("\t\tmerging...") + return + else: + # print("\t\trecursive...") + stack.append((cells, index)) + cost = cost - lbi * unit + maxi = unit + cells = val + translation = 0 + return + + def __cleanup__(self, cells: List[Tuple[int, Any]], index: int) -> None: + # print(f"[CLEANUP] {F.LIGHTBLUE_EX}BEFORE{F.RESET}:", cells[index]) + n, _ = cells[index] + if n > 1: + if n == 2: + cells[index] = (1, self.__pop__(cells[index])) + # print("AFTER:", self) + else: + cells[index][0] -= 1 # type: ignore + else: + cells[index] = (0, None) + # print("[CLEANUP] AFTER:", cells[index]) + + def pop(self) -> CostTuple: + popped = self.__pop__(self.cells[self.translation]) + self.__cleanup__(self.cells, self.translation) + self.nelements -= 1 + self.last_pop = popped + return popped # type: ignore + + def __pop__(self, cells: Tuple[int, Any]) -> Optional[CostTuple]: + nelems, val = cells + if nelems <= 1: + return val # type: ignore + for i, elem in enumerate(val): + n, v = elem + if n > 0: + if n == 1: + val[i] = (0, None) + return v # type: ignore + else: + popped = self.__pop__(elem) + if popped is not None: + self.__cleanup__(val, i) + return popped + else: + val[i] = (0, None) + return None + + def peek(self) -> CostTuple: + # term, content = self.cells[self.translation] + # assert content is not None + popped = self.__peek__(self.cells[self.translation]) + # assert popped is not None + return popped # type: ignore + + def __peek__(self, cells: Tuple) -> Optional[CostTuple]: + nelems, val = cells + if nelems <= 1: + return val # type: ignore + for elem in val: + n, v = elem + if v is not None: + if n == 1: + return v # type: ignore + else: + popped = self.__peek__(elem) + if popped is not None: + return popped + return None + + def size(self) -> int: + return sum(self.__size__(el) for el in self.cells) + + def __size__(self, cell: Tuple) -> int: + terminal, val = cell + if val is None: + return 0 + elif terminal == 1: + return 1 + else: + return sum(max(1, self.__size__(elem)) for elem in val) + + def is_empty(self) -> bool: + return self.nelements == 0 + + def __repr__(self) -> str: + ordered = self.cells[self.translation :] + self.cells[: self.translation] + out = f"CDQueue[size={self.nelements}/{self.size()}, mini={self.mini}, maxi={self.mini + self.maxi * (self.k / (self.k + 1))}/{self.mini + self.maxi}, k={self.k}]\n\t{ordered}" # type: ignore + return out + + def __len__(self) -> int: + return self.nelements diff --git a/synth/syntax/grammars/enumeration/grammar_splitter.py b/synth/syntax/grammars/enumeration/grammar_splitter.py new file mode 100644 index 00000000..aa849e2d --- /dev/null +++ b/synth/syntax/grammars/enumeration/grammar_splitter.py @@ -0,0 +1,656 @@ +from collections import defaultdict +from typing import ( + Callable, + Dict, + Generic, + List, + Optional, + Tuple, + TypeVar, +) +import bisect +from dataclasses import dataclass, field +import copy + +import numpy as np + +from synth.syntax.grammars.tagged_det_grammar import DerivableProgram +from synth.syntax.grammars.tagged_u_grammar import ProbUGrammar +from synth.syntax.grammars.u_cfg import UCFG +from synth.syntax.program import Constant, Primitive, Program, Variable +from synth.syntax.type_system import Type, UnknownType + +U = TypeVar("U") + + +@dataclass(order=True, frozen=True) +class _Node(Generic[U]): + probability: float + for_next_derivation: Tuple[List[Tuple[Type, U]], Tuple[Type, U]] = field( + compare=False + ) + program: List[Program] = field(compare=False) + derivation_history: List[Tuple[Type, U]] = field(compare=False) + choices: List[List[Tuple[Type, U]]] = field(compare=False) + + +def __node_split__( + pcfg: ProbUGrammar[U, List[Tuple[Type, U]], List[Tuple[Type, U]]], node: _Node[U] +) -> Tuple[bool, List[_Node[U]]]: + """ + Split the specified node accordingly. + + Return: success, nodes: + - True, list of children nodes + - False, [node] + """ + output: List[_Node[U]] = [] + info, S = node.for_next_derivation + # If there is no next then it means this node can't be split + if S not in pcfg.tags: + return False, [node] + for P in pcfg.rules[S]: + p_prob = pcfg.probabilities[S][P] + # format is (info_state_stack, current_info_state, possibles) + next_derivations = pcfg.derive(info, S, P) + # Skip failed derivations + if len(next_derivations) == 0: + continue + for possible in next_derivations: + new_root = _Node( + node.probability * p_prob[tuple(possible[-1])], # type: ignore + (possible[0], possible[1]), + node.program + [P], + node.derivation_history + [S], + node.choices + [possible[-1]], + ) + output.append(new_root) + return True, output + + +def __split_nodes_until_quantity_reached__( + pcfg: ProbUGrammar[U, List[Tuple[Type, U]], List[Tuple[Type, U]]], quantity: int +) -> List[_Node[U]]: + """ + Start from the root node and split most probable node until the threshold number of nodes is reached. + """ + nodes: List[_Node[U]] = [] + for key, prob in pcfg.start_tags.items(): + nodes.append(_Node(prob, (pcfg.start_information(), key), [], [], [])) + while len(nodes) < quantity: + i = 1 + success, new_nodes = __node_split__(pcfg, nodes.pop()) + while not success: + i += 1 + nodes.append(new_nodes[0]) + success, new_nodes = __node_split__(pcfg, nodes.pop(-i)) + for new_node in new_nodes: + insertion_index: int = bisect.bisect(nodes, new_node) + nodes.insert(insertion_index, new_node) + + return nodes + + +def __all_compatible__( + pcfg: ProbUGrammar[U, List[Tuple[Type, U]], List[Tuple[Type, U]]], + node: _Node[U], + group: List[_Node[U]], +) -> bool: + return True # all(__are_compatible__(pcfg, node, node2) for node2 in group) + + +def __try_split_node_in_group__( + pcfg: ProbUGrammar[U, List[Tuple[Type, U]], List[Tuple[Type, U]]], + prob_groups: List[List], + group_index: int, +) -> bool: + group_a: List[_Node[U]] = prob_groups[group_index][1] + # Sort group by ascending probability + group_a_bis = sorted(group_a, key=lambda x: x.probability) + # Try splitting a node until success + i = 1 + success, new_nodes = __node_split__(pcfg, group_a_bis[-i]) + while not success and i < len(group_a): + i += 1 + success, new_nodes = __node_split__(pcfg, group_a_bis[-i]) + if i >= len(group_a): + return False + # Success, remove old node + group_a.pop(-i) + # Add new nodes + for new_node in new_nodes: + group_a.append(new_node) + return True + + +def __find_swap_for_group__( + pcfg: ProbUGrammar[U, List[Tuple[Type, U]], List[Tuple[Type, U]]], + prob_groups: List[List], + group_index: int, +) -> Optional[Tuple[int, Optional[int], int]]: + max_prob: float = prob_groups[-1][1] + min_prob: float = prob_groups[0][1] + group_a, prob = prob_groups[group_index] + best_swap: Optional[Tuple[int, Optional[int], int]] = None + current_score: float = max_prob / prob + + candidates = ( + list(range(len(prob_groups) - 1, group_index, -1)) + if group_index == 0 + else [len(prob_groups) - 1] + ) + + for i in candidates: + group_b, prob_b = prob_groups[i] + for j, node_a in enumerate(group_a): + pa: float = node_a.probability + reduced_prob: float = prob - pa + # Try all swaps + for k, node_b in enumerate(group_b): + pb: float = node_b.probability + if ( + pb < pa + or not __all_compatible__(pcfg, node_a, group_b) + or not __all_compatible__(pcfg, node_b, group_a) + ): + continue + new_mass_b: float = prob_b - pb + pa + mini = min_prob if group_index > 0 else reduced_prob + pb + maxi = ( + max(new_mass_b, prob_groups[-2][1]) + if j == len(prob_groups) - 1 + else max_prob + ) + new_score = maxi / mini + if new_score < current_score: + best_swap = (i, j, k) + current_score = new_score + # Consider taking something from b + for k, node_b in enumerate(group_b): + if not __all_compatible__(pcfg, node_b, group_a): + continue + pb = node_b.probability + if prob + pb > max_prob: + new_score = (prob + pb) / min_prob + else: + new_score = max_prob / (prob + pb) + if new_score < current_score: + best_swap = (i, None, k) + current_score = new_score + return best_swap + + +def __percolate_down__(prob_groups: List[List], group_index: int) -> None: + index = group_index + p = prob_groups[group_index][1] + while index > 0 and prob_groups[index - 1][1] > p: + prob_groups[index - 1], prob_groups[index] = ( + prob_groups[index], + prob_groups[index - 1], + ) + index -= 1 + + +def __percolate_up__(prob_groups: List[List], group_index: int) -> None: + index = group_index + p = prob_groups[group_index][1] + while index < len(prob_groups) - 2 and prob_groups[index + 1][1] < p: + prob_groups[index + 1], prob_groups[index] = ( + prob_groups[index], + prob_groups[index + 1], + ) + index += 1 + + +def __apply_swap__( + prob_groups: List[List], group_index: int, swap: Tuple[int, Optional[int], int] +) -> None: + j, k, l = swap + # App + if k: + node_a = prob_groups[group_index][0].pop(k) + prob_groups[group_index][1] -= node_a.probability + prob_groups[j][0].append(node_a) + prob_groups[j][1] += node_a.probability + + node_b = prob_groups[j][0].pop(l) + prob_groups[j][1] -= node_b.probability + prob_groups[group_index][0].append(node_b) + prob_groups[group_index][1] += node_b.probability + + __percolate_down__(prob_groups, -1) + __percolate_up__(prob_groups, group_index) + + +def __split_into_nodes__( + pcfg: ProbUGrammar[U, List[Tuple[Type, U]], List[Tuple[Type, U]]], + splits: int, + threshold: float, +) -> Tuple[List[List[_Node[U]]], float]: + nodes = __split_nodes_until_quantity_reached__(pcfg, splits) + # Create groups + groups: List[List[_Node[U]]] = [] + for node in nodes[:splits]: + groups.append([node]) + for node in nodes[splits:]: + # Add to first compatible group + added = False + for group in groups: + if __all_compatible__(pcfg, node, group): + group.append(node) + added = True + break + assert added + masses: List[float] = [np.sum([x.probability for x in group]) for group in groups] + prob_groups = sorted([[g, p] for g, p in zip(groups, masses)], key=lambda x: x[1]) # type: ignore + ratio: float = prob_groups[-1][1] / prob_groups[0][1] # type: ignore + made_progress = True + while ratio > threshold and made_progress: + made_progress = False + for i in range(splits - 1): + swap = __find_swap_for_group__(pcfg, prob_groups, i) + if swap: + made_progress = True + __apply_swap__(prob_groups, i, swap) + break + if not made_progress: + for i in range(splits - 1, 0, -1): + made_progress = __try_split_node_in_group__(pcfg, prob_groups, i) + if made_progress: + break + ratio = prob_groups[-1][1] / prob_groups[0][1] # type: ignore + return [g for g, _ in prob_groups], ratio # type: ignore + + +def __common_prefix__( + a: List[Tuple[Type, U]], b: List[Tuple[Type, U]] +) -> List[Tuple[Type, U]]: + if a == b: + return a + candidates = [] + if len(a) > 1: + candidates.append(__common_prefix__(a[1:], b)) + if len(b) >= 1 and a[0] == b[0]: + candidates.append([a[0]] + __common_prefix__(a[1:], b[1:])) + if len(b) > 1: + candidates.append(__common_prefix__(a, b[1:])) + # Take longest common prefix + lentghs = [len(x) for x in candidates] + if len(lentghs) == 0: + return [] + if max(lentghs) == lentghs[0]: + return candidates[0] + return candidates[1] + + +def __create_path__( + rules: Dict[ + Tuple[Type, Tuple[U, int]], + Dict[DerivableProgram, List[List[Tuple[Type, Tuple[U, int]]]]], + ], + probabilities: Dict[ + Tuple[Type, Tuple[U, int]], + Dict[DerivableProgram, Dict[List[Tuple[Type, Tuple[U, int]]], float]], + ], + original_pcfg: ProbUGrammar[U, List[Tuple[Type, U]], List[Tuple[Type, U]]], + Slist: List[Tuple[Type, U]], + Plist: List[Program], + Vlist: List[List[Tuple[Type, U]]], + map_state: Callable[[Tuple[Type, U]], Tuple[Type, Tuple[U, int]]], + original_start: Tuple[Type, U], + to_normalise: List[ + Tuple[List[Tuple[Type, U]], Tuple[Type, U], Program, List[Tuple[Type, U]]] + ], +) -> List[Tuple[Type, U]]: + # print("\tCREATING A PATH:", Plist) + info = original_pcfg.start_information() + for i, (S, P, v) in enumerate(zip(Slist, Plist, Vlist)): + if i == 0: + S = original_start + # print(f"\t\tpath:{S} -> {P} : {v}") + to_normalise.append((info, S, P, v)) + if i > 0: + info.pop(0) + derivation = original_pcfg.derive_specific(info, S, P, v) # type: ignore + assert derivation + next_derivation, current = derivation + # Update derivations + assert isinstance(P, (Primitive, Variable, Constant)) + # print(f"\t\tpath current:{current} next:{next_derivation}") + Sp = map_state(S) + mapped_v = [map_state(x) for x in v] + if Sp not in rules: + rules[Sp] = {P: []} + probabilities[Sp] = {P: {}} + if P not in rules[Sp]: + rules[Sp][P] = [] + probabilities[Sp][P] = {} + rules[Sp][P].append(mapped_v) + probabilities[Sp][P][tuple(mapped_v)] = original_pcfg.probabilities[S][P][ # type: ignore + tuple(v) # type: ignore + ] + info = [current] + next_derivation + return info + + +def __create_starts__( + min_prefix: List[Tuple[Type, U]], + map_state: Callable[[Tuple[Type, U]], Tuple[Type, Tuple[U, int]]], + rules: Dict[ + Tuple[Type, Tuple[U, int]], + Dict[DerivableProgram, List[List[Tuple[Type, Tuple[U, int]]]]], + ], + probabilities: Dict[ + Tuple[Type, Tuple[U, int]], + Dict[DerivableProgram, Dict[List[Tuple[Type, Tuple[U, int]]], float]], + ], + to_normalise: List[ + Tuple[List[Tuple[Type, U]], Tuple[Type, U], Program, List[Tuple[Type, U]]] + ], + group: List[_Node[U]], + original_pcfg: ProbUGrammar[U, List[Tuple[Type, U]], List[Tuple[Type, U]]], +) -> List[Tuple[Type, U]]: + """ + For each node in the group, create the starting path from the start non terminal to each node + """ + # Extract the start symbol + original_start = min_prefix.pop() + + # We also need to mark all contexts that should be filled + to_fill: List[Tuple[Type, U]] = [] + if len(min_prefix) > 0: + Slist = group[0].derivation_history[: len(min_prefix) + 1] + Plist = group[0].program[: len(min_prefix) + 1] + Vlist = group[0].choices[: len(min_prefix) + 1] + rem = __create_path__( + rules, + probabilities, + original_pcfg, + Slist, + Plist, + Vlist, + map_state, + Slist[0], + to_normalise, + ) + if rem and not isinstance(rem[0][0], UnknownType): + to_fill += rem + original_start = Slist[-1] + + # Now we need to make a path from the common prefix to each node's prefix + + for node in group: + program, prefix = ( + node.program, + node.derivation_history, + ) + # Create rules to follow the path + i = prefix.index(original_start) + if len(min_prefix) > 0: + i += 1 + ctx_path = prefix[i:] + program_path = program[i:] + v_path = node.choices[i:] + # print(prefix, "START:", original_start) + if len(ctx_path) > 0: + ctx_path[0] = original_start + rem = __create_path__( + rules, + probabilities, + original_pcfg, + ctx_path, + program_path, + v_path, + map_state, + original_start, + to_normalise, + ) + if rem and not isinstance(rem[0][0], UnknownType): + to_fill += rem + return to_fill + + +def __fill_grammar__( + map_state: Callable[[Tuple[Type, U]], Tuple[Type, Tuple[U, int]]], + rules: Dict[ + Tuple[Type, Tuple[U, int]], + Dict[DerivableProgram, List[List[Tuple[Type, Tuple[U, int]]]]], + ], + probabilities: Dict[ + Tuple[Type, Tuple[U, int]], + Dict[DerivableProgram, Dict[List[Tuple[Type, Tuple[U, int]]], float]], + ], + original_pcfg: ProbUGrammar[U, List[Tuple[Type, U]], List[Tuple[Type, U]]], + to_fill: List[Tuple[Type, U]], +) -> Dict[ + Tuple[Type, Tuple[U, int]], + Dict[DerivableProgram, Dict[Tuple[Tuple[Type, Tuple[U, int]], ...], float]], +]: + """ + Fill-in the grammar starting from the initial to_fill contexts. + """ + computed: Dict[ + Tuple[Type, Tuple[U, int]], + Dict[DerivableProgram, Dict[Tuple[Tuple[Type, Tuple[U, int]], ...], float]], + ] = defaultdict(lambda: defaultdict(dict)) + # Build rules from to_fill + while to_fill: + S = to_fill.pop() + Sp = map_state(S) + if Sp not in rules: + rules[Sp] = { + P: [[map_state(s) for s in v] for v in vList] + for P, vList in original_pcfg.rules[S].items() + } + for P in original_pcfg.rules[S]: + for state_list in original_pcfg.rules[S][P]: + for state in state_list: + to_fill.append(state) + probabilities[Sp] = { + P: {tuple(map_state(s) for s in k): p for k, p in dicoV.items()} # type: ignore + for P, dicoV in original_pcfg.probabilities[S].items() + } + computed[Sp] = probabilities[Sp] # type: ignore + return computed + + +def __fix_probabilities__( + map_state: Callable[[Tuple[Type, U]], Tuple[Type, Tuple[U, int]]], + probabilities: Dict[ + Tuple[Type, Tuple[U, int]], + Dict[DerivableProgram, Dict[List[Tuple[Type, Tuple[U, int]]], float]], + ], + original_pcfg: ProbUGrammar[U, List[Tuple[Type, U]], List[Tuple[Type, U]]], + computed: Dict[ + Tuple[Type, Tuple[U, int]], + Dict[DerivableProgram, Dict[Tuple[Tuple[Type, Tuple[U, int]], ...], float]], + ], + to_normalise: List[ + Tuple[List[Tuple[Type, U]], Tuple[Type, U], Program, List[Tuple[Type, U]]] + ], +) -> None: + while to_normalise: + for el in list(to_normalise): + info, S, cP, v = el + assert isinstance(cP, (Primitive, Variable, Constant)) + Sp = map_state(S) + if Sp not in probabilities: + probabilities[Sp] = {} + if cP not in probabilities[Sp]: + probabilities[Sp][cP] = {} + # print("\t", Sp, "->", P) + derivation = original_pcfg.derive_specific(info, S, cP, v) + assert derivation + _, current = derivation + # Compute the updated probabilities + new_prob = 0.0 + old_w = original_pcfg.probabilities[S][cP][tuple(v)] # type: ignore + if isinstance(current[0], UnknownType): + new_prob = 1 + else: + missed = False + currentP = map_state(current) + count = 0 + for Pp, v_dict in computed[currentP].items(): + if Pp not in computed[currentP]: + missed = True + break + for cv, p in v_dict.items(): + if cv not in v_dict: + missed = True + break + new_prob += p + count += 1 + if missed or count == 0: + continue + # print("for", S, "=>", P, "@", v) + # print("\tprob:", new_prob, "count:", count, "missed:", missed) + # Update according to Equation (1) + tmapped_v = tuple(map_state(x) for x in v) + # print("\t\t", Sp, "->", P) + probabilities[Sp][cP][tmapped_v] = old_w * new_prob # type: ignore + computed[Sp][cP][tmapped_v] = old_w * new_prob + to_normalise.remove(el) + + +def __pcfg_from__( + original_pcfg: ProbUGrammar[U, List[Tuple[Type, U]], List[Tuple[Type, U]]], + group: List[_Node[U]], +) -> ProbUGrammar[ + Tuple[U, int], List[Tuple[Type, Tuple[U, int]]], List[Tuple[Type, Tuple[U, int]]] +]: + # print() + # print("=" * 60) + # print("NODE") + # for node in group: + # print("\t", node.program) + # Find the common prefix to all + min_prefix = copy.deepcopy(group[0].derivation_history) + for node in group[1:]: + min_prefix = __common_prefix__(min_prefix, node.derivation_history) + # print("MIN PREFIX:", min_prefix) + + # Function to map states automatically + rule_nos = [0] + mapping: Dict[Tuple[Type, U], Tuple[Type, Tuple[U, int]]] = {} + + def map_state(s: Tuple[Type, U]) -> Tuple[Type, Tuple[U, int]]: + o = mapping.get(s, None) + if o is not None: + # print("\t", s, "=>", o) + return o + # print(s, "does not exist in mapping:", set(mapping.keys())) + mapping[s] = (s[0], (s[1], rule_nos[0])) + rule_nos[0] += 1 + return mapping[s] + + # New start states + starts = {map_state(s) for s in original_pcfg.grammar.starts} + + rules: Dict[ + Tuple[Type, Tuple[U, int]], + Dict[DerivableProgram, List[List[Tuple[Type, Tuple[U, int]]]]], + ] = {} + probabilities: Dict[ + Tuple[Type, Tuple[U, int]], + Dict[DerivableProgram, Dict[List[Tuple[Type, Tuple[U, int]]], float]], + ] = {} + start_probs: Dict[Tuple[Type, Tuple[U, int]], float] = {} + # List of non-terminals + info we need to normalise weirdly + to_normalise: List[ + Tuple[List[Tuple[Type, U]], Tuple[Type, U], Program, List[Tuple[Type, U]]] + ] = [] + # Our min_prefix may be something like (int, 1, (+, 1)) + # which means we already chose + + # But it is not in the PCFG + # Thus we need to add it + # In the general case we may as well have + -> + -> + as prefix this whole prefix needs to be added + to_fill = __create_starts__( + min_prefix, map_state, rules, probabilities, to_normalise, group, original_pcfg + ) + # print("BEFORE FILLING") + # print("to_fill:", to_fill) + # print(UCFG(starts, rules, clean=False)) + computed = __fill_grammar__(map_state, rules, probabilities, original_pcfg, to_fill) + # Now we can already have the new grammar + new_grammar = UCFG(starts, rules, clean=False) + # print("FINAL GRAMMAR BEFORE CLEANING") + # print(new_grammar) + new_grammar.clean() + # At this point we have all the needed rules + # However, the probabilites are incorrect + __fix_probabilities__( + map_state, probabilities, original_pcfg, computed, to_normalise + ) + for start in original_pcfg.start_tags: + Sp = map_state(start) + if Sp in new_grammar.starts: + start_probs[Sp] = original_pcfg.start_tags[start] + # The updated probabilities may not sum to 1 so we need to normalise them + # But let ProbDetGrammar do it with clean=True + + # Make probabilities coherent with rules + probabilities = { + S: { + P: {v: p for v, p in dicoV.items() if list(v) in new_grammar.rules[S][P]} + for P, dicoV in dicoP.items() + if P in new_grammar.rules[S] + } + for S, dicoP in probabilities.items() + if S in new_grammar.rules + } + # Now normalise as said earlier + grammar = ProbUGrammar(new_grammar, probabilities, start_probs) + grammar.normalise() + # print(grammar) + # print() + # print("=" * 80) + # print() + return grammar + + +# @overload +# def split( +# pcfg: ProbDetGrammar[U, List[Tuple[Type, U]], List[Tuple[Type, U]]], splits: int, desired_ratio: float = 1.1 +# ) -> Tuple[List[ProbDetGrammar[U, List[Tuple[Type, U]], List[Tuple[Type, U]]]], float]: +# pass + +# @overload +# def split( +# pcfg: ProbUGrammar[U, List[Tuple[Type, U]], List[Tuple[Type, U]]], splits: int, desired_ratio: float = 1.1 +# ) -> Tuple[List[ProbUGrammar[U, List[Tuple[Type, U]], List[Tuple[Type, U]]]], float]: +# pass + + +# def split( +# pcfg: Union[ProbUGrammar[U, List[Tuple[Type, U]], List[Tuple[Type, U]]], ProbDetGrammar[U, List[Tuple[Type, U]], List[Tuple[Type, U]]]], splits: int, desired_ratio: float = 1.1 +# ) -> Union[Tuple[List[ProbUGrammar[U, List[Tuple[Type, U]], List[Tuple[Type, U]]]], float], Tuple[List[ProbDetGrammar[U, List[Tuple[Type, U]], List[Tuple[Type, U]]]], float]]: +def split( + pcfg: ProbUGrammar[U, List[Tuple[Type, U]], List[Tuple[Type, U]]], + splits: int, + desired_ratio: float = 1.1, +) -> Tuple[ + List[ + ProbUGrammar[ + Tuple[U, int], + List[Tuple[Type, Tuple[U, int]]], + List[Tuple[Type, Tuple[U, int]]], + ] + ], + float, +]: + """ + Currently use exchange split. + Parameters: + splits: the number of splits (must be > 1, otherwise the pcfg returned does not match the type signature since the input one is returned) + desired_ratio: the max ratio authorized between the most probable group and the least probable pcfg + + Return: + a list of probabilistic grammars + the reached threshold + """ + if splits == 1: + return [pcfg], 1 # type: ignore + assert desired_ratio > 1, "The desired ratio must be > 1!" + groups, ratio = __split_into_nodes__(pcfg, splits, desired_ratio) + return [__pcfg_from__(pcfg, group) for group in groups if len(group) > 0], ratio diff --git a/synth/syntax/grammars/heap_search.py b/synth/syntax/grammars/enumeration/heap_search.py similarity index 60% rename from synth/syntax/grammars/heap_search.py rename to synth/syntax/grammars/enumeration/heap_search.py index d9a55952..2609dbeb 100644 --- a/synth/syntax/grammars/heap_search.py +++ b/synth/syntax/grammars/enumeration/heap_search.py @@ -15,7 +15,10 @@ from dataclasses import dataclass, field from abc import ABC, abstractmethod -from synth.syntax.program import Program, Function, Variable +from synth.filter.filter import Filter +from synth.syntax.grammars.enumeration.program_enumerator import ProgramEnumerator +from synth.syntax.grammars.tagged_u_grammar import ProbUGrammar +from synth.syntax.program import Program, Function from synth.syntax.grammars.tagged_det_grammar import ProbDetGrammar from synth.syntax.type_system import Type from synth.utils.ordered import Ordered @@ -34,9 +37,22 @@ def __repr__(self) -> str: return f"({self.priority}, {self.program})" -class HSEnumerator(ABC, Generic[U, V, W]): - def __init__(self, G: ProbDetGrammar[U, V, W]) -> None: +class HSEnumerator( + ProgramEnumerator[None], + ABC, + Generic[U, V, W], +): + def __init__( + self, + G: ProbDetGrammar[U, V, W], + threshold: Optional[Ordered] = None, + filter: Optional[Filter[Program]] = None, + ) -> None: + super().__init__(filter) self.current: Optional[Program] = None + self.threshold = threshold + + self.deleted: Set[Program] = set() self.G = G self.start = G.start @@ -51,6 +67,8 @@ def __init__(self, G: ProbDetGrammar[U, V, W]) -> None: # self.succ[S][P] is the successor of P from S self.succ: Dict[Tuple[Type, U], Dict[int, Program]] = {S: {} for S in symbols} + # self.pred[S][P] is the hash of the predecessor of P from S + self.pred: Dict[Tuple[Type, U], Dict[int, int]] = {S: {} for S in symbols} # self.hash_table_program[S] is the set of hashes of programs # ever added to the heap for S @@ -58,50 +76,48 @@ def __init__(self, G: ProbDetGrammar[U, V, W]) -> None: S: set() for S in symbols } - # self.hash_table_global[hash] = P maps - # hashes to programs for all programs ever added to some heap - self.hash_table_global: Dict[int, Program] = {} - self._init: Set[Tuple[Type, U]] = set() self.max_priority: Dict[ Union[Tuple[Type, U], Tuple[Tuple[Type, U], Program]], Program ] = {} - def __return_unique__(self, P: Program) -> Program: - """ - ensures that if a program appears in several heaps, - it is represented by the same object, - so we do not evaluate it several times - """ - hash_P = hash(P) - if hash_P in self.hash_table_global: - return self.hash_table_global[hash_P] - else: - self.hash_table_global[hash_P] = P - return P + def probability(self, program: Program) -> float: + return self.G.probability(program) + + @classmethod + def name(cls) -> str: + return "heap-search" def generator(self) -> Generator[Program, None, None]: """ A generator which outputs the next most probable program """ + self.__init_non_terminal__(self.G.start) + self._reevaluate_() + # Now we can init the heaps + for S in self.G.rules: + self.__init_heap__(S) + # And now that ALL heaps have been init + # Query(S, None) for all + for S in self.G.rules: + self.query(S, None) while True: program = self.query(self.start, self.current) if program is None: - break + return self.current = program + if not self._should_keep_subprogram(program): + self.deleted.add(program) + continue yield program - def __iter__(self) -> Generator[Program, None, None]: - return self.generator() - - def __init_non_terminal__(self, S: Tuple[Type, U]) -> None: - if S in self._init: - return - self._init.add(S) - # 1) Compute max probablities - best_program = None - best_priority: Optional[Ordered] = None + def __compute_max_prio__( + self, + S: Tuple[Type, U], + best_program: Optional[Program] = None, + best_priority: Optional[Ordered] = None, + ) -> Tuple[Optional[Program], Optional[Ordered]]: for P in self.rules[S]: nargs = self.G.arguments_length_for(S, P) P_unique: Program = P @@ -110,13 +126,16 @@ def __init_non_terminal__(self, S: Tuple[Type, U]) -> None: information, current = self.G.derive(self.G.start_information(), S, P) for _ in range(nargs): self.__init_non_terminal__(current) + if current not in self.max_priority: + break # Try to init sub Tuple[Type, U] in case they were not initialised arguments.append(self.max_priority[current]) information, lst = self.G.derive_all( information, current, arguments[-1] ) current = lst[-1] - + if len(arguments) != nargs: + continue new_program = Function( function=P_unique, arguments=arguments, @@ -124,37 +143,104 @@ def __init_non_terminal__(self, S: Tuple[Type, U]) -> None: P_unique = new_program priority = self.compute_priority(S, P_unique) self.max_priority[(S, P)] = P_unique - if not best_priority or priority < best_priority: + if best_priority is None or priority < best_priority: best_program = P_unique best_priority = priority - assert best_program - self.max_priority[S] = best_program + return best_program, best_priority + def __init_non_terminal__(self, S: Tuple[Type, U]) -> None: + if S in self._init: + return + if all((S, P) in self.max_priority for P in self.rules[S]): + return + self._init.add(S) + # 1) Compute max probablities + best_program, _ = self.__compute_max_prio__(S) + + self._init.remove(S) + if best_program is not None: + self.max_priority[S] = best_program + + def _reevaluate_(self) -> None: + changed = True + while changed: + changed = False + for S in list(self.G.rules.keys()): + old = self.max_priority.get(S, None) + self.__init_non_terminal__(S) + + if S not in self.max_priority or old != self.max_priority[S]: + changed = True + + def __init_heap__(self, S: Tuple[Type, U]) -> None: # 2) add P(max(S1),max(S2), ...) to self.heaps[S] for P in self.rules[S]: program = self.max_priority[(S, P)] hash_program = hash(program) # Remark: the program cannot already be in self.heaps[S] assert hash_program not in self.hash_table_program[S] + # Init heap all others so that query will work self.hash_table_program[S].add(hash_program) # we assume that the programs from max_probability # are represented by the same object - self.hash_table_global[hash_program] = program priority = self.compute_priority(S, program) - heappush( - self.heaps[S], - HeapElement(priority, program), - ) + if not self.threshold or priority < self.threshold: + heappush( + self.heaps[S], + HeapElement(priority, program), + ) - # 3) Do the 1st query - self.query(S, None) + def merge_program(self, representative: Program, other: Program) -> None: + """ + Merge other into representative. + In other words, other will no longer be generated through heap search + """ + our_hash = hash(other) + self.deleted.add(other) + for S in self.G.rules: + if our_hash in self.pred[S] and our_hash in self.succ[S]: + pred_hash = self.pred[S][our_hash] + nxt = self.succ[S][our_hash] + self.succ[S][pred_hash] = nxt + self.pred[S][hash(nxt)] = pred_hash + + def __add_successors__(self, succ: Program, S: Tuple[Type, U]) -> None: + if isinstance(succ, Function): + F = succ.function + information, lst = self.G.derive_all(self.G.start_information(), S, F) + S2 = lst[-1] + args_len = self.G.arguments_length_for(S, F) # type: ignore + for i in range(args_len): + # S2 is non-terminal symbol used to derive the i-th argument + succ_sub_program = self.query(S2, succ.arguments[i]) + if succ_sub_program: + new_arguments = succ.arguments[:] + new_arguments[i] = succ_sub_program + new_program = Function(F, new_arguments) + hash_new_program = hash(new_program) + if ( + hash_new_program not in self.hash_table_program[S] + and new_program not in self.deleted + ): + self.hash_table_program[S].add(hash_new_program) + try: + priority: Ordered = self.compute_priority(S, new_program) + if not self.threshold or priority < self.threshold: + heappush( + self.heaps[S], HeapElement(priority, new_program) + ) + except KeyError: + pass + if i + 1 < args_len: + information, lst = self.G.derive_all( + information, S2, succ.arguments[i] + ) + S2 = lst[-1] def query(self, S: Tuple[Type, U], program: Optional[Program]) -> Optional[Program]: """ computing the successor of program from S """ - if S not in self._init: - self.__init_non_terminal__(S) if program: hash_program = hash(program) else: @@ -168,52 +254,47 @@ def query(self, S: Tuple[Type, U], program: Optional[Program]) -> Optional[Progr try: element = heappop(self.heaps[S]) succ = element.program + while succ in self.deleted: + self.__add_successors__(succ, S) + element = heappop(self.heaps[S]) + succ = element.program except: return None # the heap is empty: there are no successors from S self.succ[S][hash_program] = succ # we store the successor + self.pred[S][hash(succ)] = hash_program # we store the predecessor # now we need to add all potential successors of succ in heaps[S] - if isinstance(succ, Function): - F = succ.function - information, lst = self.G.derive_all(self.G.start_information(), S, F) - S2 = lst[-1] - for i in range(self.G.arguments_length_for(S, F)): # type: ignore - # S2 is non-terminal symbol used to derive the i-th argument - succ_sub_program = self.query(S2, succ.arguments[i]) - if succ_sub_program: - new_arguments = succ.arguments[:] - new_arguments[i] = succ_sub_program - - new_program = self.__return_unique__(Function(F, new_arguments)) - hash_new_program = hash(new_program) - - if hash_new_program not in self.hash_table_program[S]: - self.hash_table_program[S].add(hash_new_program) - - priority: Ordered = self.compute_priority(S, new_program) - heappush(self.heaps[S], HeapElement(priority, new_program)) - information, lst = self.G.derive_all(information, S2, succ.arguments[i]) - S2 = lst[-1] - - if isinstance(succ, Variable): - return succ # if succ is a variable, there is no successor so we stop here - + self.__add_successors__(succ, S) return succ @abstractmethod def compute_priority(self, S: Tuple[Type, U], new_program: Program) -> Ordered: pass + def programs_in_banks(self) -> int: + return sum(len(val) for val in self.succ.values()) + + def programs_in_queues(self) -> int: + return sum(len(val) for val in self.heaps.values()) + + def clone(self, G: Union[ProbDetGrammar, ProbUGrammar]) -> "HSEnumerator[U, V, W]": + assert isinstance(G, ProbDetGrammar) + enum = self.__class__(G, self.threshold) + enum.deleted = self.deleted.copy() + return enum + class HeapSearch(HSEnumerator[U, V, W]): - def __init__(self, G: ProbDetGrammar[U, V, W]) -> None: - super().__init__(G) + def __init__(self, G: ProbDetGrammar[U, V, W], threshold: float = 0) -> None: + super().__init__(G, -threshold) self.probabilities: Dict[Program, Dict[Tuple[Type, U], float]] = defaultdict( lambda: {} ) def compute_priority(self, S: Tuple[Type, U], new_program: Program) -> float: + if new_program in self.probabilities and S in self.probabilities[new_program]: + return -self.probabilities[new_program][S] if isinstance(new_program, Function): F = new_program.function # We guarantee that F is a Primitive @@ -221,19 +302,23 @@ def compute_priority(self, S: Tuple[Type, U], new_program: Program) -> float: probability = self.G.probabilities[S][F] # type: ignore information, lst = self.G.derive_all(self.G.start_information(), S, F) S2 = lst[-1] - for i in range(self.G.arguments_length_for(S, F)): # type: ignore + args_len = self.G.arguments_length_for(S, F) # type: ignore + for i in range(args_len): arg = new_arguments[i] probability *= self.probabilities[arg][S2] - information, lst = self.G.derive_all(information, S2, arg) - S2 = lst[-1] + if i + 1 < args_len: + information, lst = self.G.derive_all(information, S2, arg) + S2 = lst[-1] else: probability = self.G.probabilities[S][new_program] # type: ignore self.probabilities[new_program][S] = probability return -probability -def enumerate_prob_grammar(G: ProbDetGrammar[U, V, W]) -> HeapSearch[U, V, W]: - return HeapSearch(G) +def enumerate_prob_grammar( + G: ProbDetGrammar[U, V, W], threshold: float = 0 +) -> HeapSearch[U, V, W]: + return HeapSearch(G, threshold) class Bucket(Ordered): @@ -287,6 +372,19 @@ def __iadd__(self, other: "Bucket") -> "Bucket": ) ) + def __add__(self, other: "Bucket") -> "Bucket": + if self.size == other.size: + dst = Bucket(self.size) + dst += self + dst += other + return dst + else: + raise RuntimeError( + "size mismatch, Bucket{}: {}, Bucket{}: {}".format( + self, self.size, other, other.size + ) + ) + def add_prob_uniform(self, probability: float) -> "Bucket": """ Given a probability add 1 in the relevant bucket assuming buckets are linearly distributed. diff --git a/synth/syntax/grammars/enumeration/program_enumerator.py b/synth/syntax/grammars/enumeration/program_enumerator.py new file mode 100644 index 00000000..0f85f986 --- /dev/null +++ b/synth/syntax/grammars/enumeration/program_enumerator.py @@ -0,0 +1,75 @@ +from typing import ( + Generator, + Generic, + Optional, + TypeVar, + Union, +) +from abc import ABC, abstractmethod + +from synth.syntax.grammars.tagged_det_grammar import ProbDetGrammar +from synth.syntax.grammars.tagged_u_grammar import ProbUGrammar +from synth.syntax.program import Program +from synth.filter import Filter + + +U = TypeVar("U") + + +class ProgramEnumerator(ABC, Generic[U]): + """ + Object that enumerates over programs. + When a program is generated a feedback of type U is expected. + If U is None then no feedback is expected. + """ + + def __init__(self, filter: Optional[Filter[Program]] = None) -> None: + super().__init__() + self.filter = filter + + @classmethod + @abstractmethod + def name(cls) -> str: + pass + + @abstractmethod + def generator(self) -> Generator[Program, U, None]: + pass + + def __iter__(self) -> Generator[Program, U, None]: + return self.generator() + + @abstractmethod + def programs_in_banks(self) -> int: + pass + + @abstractmethod + def programs_in_queues(self) -> int: + pass + + @abstractmethod + def probability(self, program: Program) -> float: + """ + Return the probability of generating the given program according to the grammar associated with this enumerator. + """ + pass + + def merge_program(self, representative: Program, other: Program) -> None: + """ + Merge other into representative. + Function used for observational equivalence, that means other and representative are semantically equivalent for the current task. + This is for a posteriori merging, it is rather inefficient compared to evaluating subprograms for most enumerative algorithms. + """ + pass + + def _should_keep_subprogram(self, program: Program) -> bool: + return self.filter is None or self.filter.accept(program) + + @abstractmethod + def clone( + self, grammar: Union[ProbDetGrammar, ProbUGrammar] + ) -> "ProgramEnumerator[U]": + """ + Clone this enumerator with the specified new grammar but remember every single program enumerated so that it does not enumerate them again. + """ + pass diff --git a/synth/syntax/grammars/enumeration/u_heap_search.py b/synth/syntax/grammars/enumeration/u_heap_search.py new file mode 100644 index 00000000..9dc1ac14 --- /dev/null +++ b/synth/syntax/grammars/enumeration/u_heap_search.py @@ -0,0 +1,445 @@ +from collections import defaultdict +from dataclasses import dataclass, field +from heapq import heappush, heappop +from typing import ( + Callable, + Dict, + Generator, + Generic, + List, + Optional, + Set, + Tuple, + TypeVar, + Union, +) +from abc import ABC, abstractmethod + +from synth.filter.filter import Filter +from synth.syntax.grammars.enumeration.program_enumerator import ProgramEnumerator +from synth.syntax.grammars.enumeration.heap_search import HeapElement, Bucket +from synth.syntax.program import Program, Function +from synth.syntax.grammars.tagged_u_grammar import ProbUGrammar +from synth.syntax.grammars.tagged_det_grammar import ProbDetGrammar +from synth.syntax.type_system import Type +from synth.utils.ordered import Ordered + +U = TypeVar("U") +V = TypeVar("V") +W = TypeVar("W") + + +@dataclass(order=True, frozen=True) +class StartHeapElement(Generic[U]): + priority: Ordered + program: Program = field(compare=False) + start: Tuple[Type, U] = field(compare=False) + + def __repr__(self) -> str: + return f"({self.priority}, {self.program}, {self.start})" + + +def __wrap__(el: Union[U, List[U]]) -> Union[U, Tuple[U, ...]]: + if isinstance(el, list): + return tuple(el) + return el + + +class UHSEnumerator(ProgramEnumerator[None], ABC, Generic[U, V, W]): + def __init__( + self, + G: ProbUGrammar[U, V, W], + threshold: Optional[Ordered] = None, + filter: Optional[Filter[Program]] = None, + ) -> None: + super().__init__(filter) + self.G = G + symbols = [S for S in self.G.rules] + self.threshold = threshold + self.deleted: Set[Program] = set() + + # self.heaps[S] is a heap containing programs generated from the non-terminal S + self.heaps: Dict[Tuple[Type, U], List[HeapElement]] = {S: [] for S in symbols} + + self._start_heap: List[StartHeapElement[U]] = [] + + # the same program can be pushed in different heaps, with different probabilities + # however, the same program cannot be pushed twice in the same heap + + # self.succ[S][P] is the successor of P from S + self.succ: Dict[Tuple[Type, U], Dict[int, Program]] = {S: {} for S in symbols} + # self.pred[S][P] is the hash of the predecessor of P from S + self.pred: Dict[Tuple[Type, U], Dict[int, int]] = {S: {} for S in symbols} + + # self.hash_table_program[S] is the set of hashes of programs + # ever added to the heap for S + self.hash_table_program: Dict[Tuple[Type, U], Set[int]] = { + S: set() for S in symbols + } + + self._keys: Dict[Tuple[Type, U], Dict[Program, V]] = defaultdict(dict) + + self._init: Set[Tuple[Type, U]] = set() + + self.max_priority: Dict[ + Union[Tuple[Type, U], Tuple[Tuple[Type, U], Program, V]], Program + ] = {} + + @classmethod + def name(cls) -> str: + return "u-heap-search" + + def programs_in_banks(self) -> int: + return sum(len(val) for val in self.succ.values()) + + def programs_in_queues(self) -> int: + return sum(len(val) for val in self.heaps.values()) + + def generator(self) -> Generator[Program, None, None]: + """ + A generator which outputs the next most probable program + """ + while True: + program = self.start_query() + if program is None: + break + if not self._should_keep_subprogram(program): + self.deleted.add(program) + continue + yield program + + def probability(self, program: Program) -> float: + return self.G.probability(program) + + def __init_helper__( + self, + Si: Tuple[Type, U], + information: W, + i: int, + args: List[Program], + nargs: int, + ) -> Generator[List[Program], None, None]: + # Try to init sub Tuple[Type, U] in case they were not initialised + self.__init_non_terminal__(Si) + args.append(self.max_priority[Si]) + if i + 1 >= nargs: + yield args + return + for information, lst in self.G.derive_all(information, Si, args[-1]): + Sip1 = lst[-1][0] + for sol in self.__init_helper__(Sip1, information, i + 1, args[:], nargs): + yield sol + + def __init_non_terminal__(self, S: Tuple[Type, U]) -> None: + if S in self._init: + return + self._init.add(S) + # 1) Compute max probablities + best_program: Optional[Program] = None + best_priority: Optional[Ordered] = None + for P in self.G.rules[S]: + nargs = self.G.arguments_length_for(S, P) + if nargs > 0: + for information, Si, v in self.G.derive( + self.G.start_information(), S, P + ): + for arguments in self.__init_helper__( + Si, information, 0, [], nargs + ): + new_program = Function( + function=P, + arguments=arguments, + ) + self._keys[S][new_program] = v + priority = self.compute_priority(S, new_program) + self.max_priority[(S, P, __wrap__(v))] = new_program # type: ignore + if not best_priority or priority < best_priority: + best_program = new_program + best_priority = priority + else: + some_v = list(self.G.tags[S][P].keys())[0] + self._keys[S][P] = some_v + priority = self.compute_priority(S, P) + self.max_priority[(S, P, some_v)] = P + if not best_priority or priority < best_priority: + best_program = P + best_priority = priority + assert best_program + self.max_priority[S] = best_program + + # 2) add P(max(S1),max(S2), ...) to self.heaps[S] + for P in self.G.rules[S]: + for v in self.G.tags[S][P]: + program = self.max_priority[(S, P, v)] + hash_program = hash(program) + # Remark: the program cannot already be in self.heaps[S] + assert hash_program not in self.hash_table_program[S] + self.hash_table_program[S].add(hash_program) + # we assume that the programs from max_probability + # are represented by the same object + priority = self.compute_priority(S, program) + assert program in self._keys[S] + if not self.threshold or priority < self.threshold: + heappush( + self.heaps[S], + HeapElement(priority, program), + ) + if S in self.G.starts: + heappush( + self._start_heap, + StartHeapElement( + self.adjust_priority_for_start(priority, S), program, S + ), + ) + + # 3) Do the 1st query + self.query(S, None) + + def start_query(self) -> Optional[Program]: + if len(self._init) == 0: + for start in self.G.starts: + self.query(start, None) + if len(self._start_heap) == 0: + return None + elem = heappop(self._start_heap) + self.query(elem.start, elem.program) + while elem.program in self.deleted: + elem = heappop(self._start_heap) + self.query(elem.start, elem.program) + return elem.program + + def __add_successors_to_heap__( + self, + succ: Function, + S: Tuple[Type, U], + Si: Tuple[Type, U], + info: W, + i: int, + v: V, + ) -> bool: + if i >= len(succ.arguments): + return True + # Si is non-terminal symbol used to derive the i-th argument + program = succ.arguments[i] + # Check if we are not using the correct derivation of S -> P + # If it is not the right one (from which we can dervie back its arguments) then we can't generate successors + for info, lst in self.G.derive_all( + info, Si, succ.arguments[i], hints=self._keys + ): + Sip1 = lst[-1][0] + if self.__add_successors_to_heap__(succ, S, Sip1, info, i + 1, v): + succ_sub_program = self.query(Si, succ.arguments[i]) + if succ_sub_program: + new_arguments = succ.arguments[:] + new_arguments[i] = succ_sub_program + new_program = Function(succ.function, new_arguments) + hash_new_program = hash(new_program) + if hash_new_program not in self.hash_table_program[S]: + self.hash_table_program[S].add(hash_new_program) + # try: + self._keys[S][new_program] = v + priority: Ordered = self.compute_priority(S, new_program) + if not self.threshold or priority < self.threshold: + heappush(self.heaps[S], HeapElement(priority, new_program)) + if S in self.G.starts: + heappush( + self._start_heap, + StartHeapElement( + self.adjust_priority_for_start(priority, S), + new_program, + S, + ), + ) + return True + return False + + def __add_successors__(self, succ: Program, S: Tuple[Type, U]) -> None: + if isinstance(succ, Function): + F = succ.function + tgt_v = self._keys[S][succ] + out = self.G.derive_specific(self.G.start_information(), S, F, tgt_v) # type: ignore + assert out is not None + info, Si = out + assert self.__add_successors_to_heap__(succ, S, Si, info, 0, tgt_v) + + def query(self, S: Tuple[Type, U], program: Optional[Program]) -> Optional[Program]: + """ + computing the successor of program from S + """ + if S not in self._init: + self.__init_non_terminal__(S) + if program: + hash_program = hash(program) + else: + hash_program = 123891 + + # if we have already computed the successor of program from S, we return its stored value + if hash_program in self.succ[S]: + return self.succ[S][hash_program] + + # otherwise the successor is the next element in the heap + try: + element = heappop(self.heaps[S]) + succ = element.program + while succ in self.deleted: + self.__add_successors__(succ, S) + element = heappop(self.heaps[S]) + succ = element.program + except: + return None # the heap is empty: there are no successors from S + + self.succ[S][hash_program] = succ # we store the successor + self.pred[S][hash(succ)] = hash_program # we store the predecessor + + # now we need to add all potential successors of succ in heaps[S] + self.__add_successors__(succ, S) + return succ + + def merge_program(self, representative: Program, other: Program) -> None: + """ + Merge other into representative. + In other words, other will no longer be generated through heap search + """ + our_hash = hash(other) + self.deleted.add(other) + for S in self.G.rules: + if our_hash in self.pred[S] and our_hash in self.succ[S]: + pred_hash = self.pred[S][our_hash] + nxt = self.succ[S][our_hash] + self.succ[S][pred_hash] = nxt + self.pred[S][hash(nxt)] = pred_hash + + @abstractmethod + def compute_priority(self, S: Tuple[Type, U], new_program: Program) -> Ordered: + pass + + @abstractmethod + def adjust_priority_for_start( + self, priority: Ordered, start: Tuple[Type, U] + ) -> Ordered: + pass + + def clone(self, G: Union[ProbDetGrammar, ProbUGrammar]) -> "UHSEnumerator[U, V, W]": + assert isinstance(G, ProbUGrammar) + enum = self.__class__(G, self.threshold) + enum.deleted = self.deleted.copy() + return enum + + +class UHeapSearch(UHSEnumerator[U, V, W]): + def __init__(self, G: ProbUGrammar[U, V, W], threshold: float = 0) -> None: + super().__init__(G, -threshold) + self.probabilities: Dict[Program, Dict[Tuple[Type, U], float]] = defaultdict( + lambda: {} + ) + + def adjust_priority_for_start( + self, priority: Ordered, start: Tuple[Type, U] + ) -> Ordered: + return priority * self.G.start_tags[start] # type: ignore + + def __prob__( + self, succ: Function, S: Tuple[Type, U], Si: Tuple[Type, U], info: W, i: int + ) -> float: + # Si is non-terminal symbol used to derive the i-th argument + arg = succ.arguments[i] + probability = self.probabilities[arg][Si] + if i + 1 >= len(succ.arguments): + return probability + for info, lst in self.G.derive_all(info, Si, arg, hints=self._keys): + Sip1 = lst[-1][0] + prob = self.__prob__(succ, S, Sip1, info, i + 1) + if prob >= 0: + return prob * probability + return -1 + + def compute_priority(self, S: Tuple[Type, U], new_program: Program) -> float: + # Try to hit cached probability + if new_program in self.probabilities and S in self.probabilities[new_program]: + return -self.probabilities[new_program][S] + if isinstance(new_program, Function): + F = new_program.function + v = self._keys[S][new_program] + out = self.G.derive_specific(self.G.start_information(), S, F, v) # type: ignore + assert out is not None + information, Si = out + # We guarantee that F is a Primitive + probability = self.G.probabilities[S][F][__wrap__(v)] # type: ignore + probability *= self.__prob__(new_program, S, Si, information, 0) + assert ( + probability >= 0 + ), f"Could not find {new_program} in {S} [{self.G.__contains_rec__(new_program, S, self.G.start_information())[0]}]" + else: + possibles = self.G.derive_all(self.G.start_information(), S, new_program) + assert len(possibles) == 1 + v = __wrap__(possibles[0][-1][-1][-1]) # type: ignore + probability = self.G.probabilities[S][new_program][v] # type: ignore + self.probabilities[new_program][S] = probability + return -probability + + +def enumerate_prob_u_grammar( + G: ProbUGrammar[U, V, W], threshold: float = 0 +) -> UHeapSearch[U, V, W]: + return UHeapSearch(G, threshold) + + +class BucketSearch(UHSEnumerator[U, V, W]): + def __init__(self, G: ProbUGrammar[U, V, W], bucket_size: int) -> None: + super().__init__(G) + self.bucket_tuples: Dict[Program, Dict[Tuple[Type, U], Bucket]] = defaultdict( + lambda: {} + ) + self.bucket_size = bucket_size + + def adjust_priority_for_start( + self, priority: Ordered, start: Tuple[Type, U] + ) -> Ordered: + return priority.add_prob_uniform(self.G.start_tags[start]) # type: ignore + + def __prob__( + self, succ: Function, S: Tuple[Type, U], Si: Tuple[Type, U], info: W, i: int + ) -> Optional[Bucket]: + # Si is non-terminal symbol used to derive the i-th argument + arg = succ.arguments[i] + if Si not in self.bucket_tuples[arg]: + return None + bucket = self.bucket_tuples[arg][Si] + if i + 1 >= len(succ.arguments): + return bucket + for info, lst in self.G.derive_all(info, Si, arg): + Sip1 = lst[-1][0] + prob = self.__prob__(succ, S, Sip1, info, i + 1) + if prob: + return prob + bucket + return None + + def compute_priority(self, S: Tuple[Type, U], new_program: Program) -> Bucket: + new_bucket = Bucket(self.bucket_size) + if isinstance(new_program, Function): + F = new_program.function + for information, lst in self.G.derive_all(self.G.start_information(), S, F): + Si = lst[-1][0] + v = __wrap__(lst[-1][-1]) + # We guarantee that F is a Primitive + probability = self.G.probabilities[S][F][v] # type: ignore + prob = self.__prob__(new_program, S, Si, information, 0) + if prob: + break + new_bucket.add_prob_uniform(probability) + assert prob is not None + new_bucket += prob + else: + possibles = self.G.derive_all(self.G.start_information(), S, new_program) + assert len(possibles) == 1 + v = __wrap__(possibles[0][-1][-1][-1]) + probability = self.G.probabilities[S][new_program][v] # type: ignore + new_bucket.add_prob_uniform(probability) + self.bucket_tuples[new_program][S] = new_bucket + return new_bucket + + +def enumerate_bucket_prob_u_grammar( + G: ProbUGrammar[U, V, W], bucket_size: int +) -> BucketSearch[U, V, W]: + return BucketSearch(G, bucket_size) diff --git a/synth/syntax/grammars/grammar.py b/synth/syntax/grammars/grammar.py index d13476aa..4d00adc2 100644 --- a/synth/syntax/grammars/grammar.py +++ b/synth/syntax/grammars/grammar.py @@ -1,6 +1,43 @@ from abc import ABC, abstractmethod +from dataclasses import dataclass, field +from typing import Any, Dict, List, Tuple, Union -from synth.syntax.program import Program +from synth.syntax.program import Constant, Primitive, Program, Variable +from synth.syntax.type_system import Type + +DerivableProgram = Union[Primitive, Variable, Constant] + + +@dataclass(frozen=True) +class NGram: + n: int + predecessors: List[Tuple[DerivableProgram, int]] = field(default_factory=lambda: []) + + _hash: "int | None" = field(init=False) + + def __post_init__(self): + object.__setattr__(self, "_hash", hash((self.n, tuple(self.predecessors)))) + + def __hash__(self) -> int: + return self._hash + + def __str__(self) -> str: + return str(self.predecessors) + + def __repr__(self) -> str: + return self.__str__() + + def __len__(self) -> int: + return len(self.predecessors) + + def successor(self, new_succ: Tuple[DerivableProgram, int]) -> "NGram": + new_pred = [new_succ] + self.predecessors + if len(new_pred) + 1 > self.n and self.n >= 0: + new_pred.pop() + return NGram(self.n, new_pred) + + def last(self) -> Tuple[DerivableProgram, int]: + return self.predecessors[0] class Grammar(ABC): @@ -14,3 +51,24 @@ def name(self) -> str: Returns the name of this class of grammar. """ pass + + @abstractmethod + def clean(self) -> None: + """ + Clean the grammar. + """ + pass + + @abstractmethod + def programs(self) -> int: + """ + Return the number of programs contained within this grammar. + """ + pass + + @abstractmethod + def instantiate_constants(self, constants: Dict[Type, List[Any]]) -> "Grammar": + """ + Replace all occurences of non instantiated constants with all possible values of instantiated ones. + """ + pass diff --git a/synth/syntax/grammars/pcfg_splitter.py b/synth/syntax/grammars/pcfg_splitter.py deleted file mode 100644 index 7127a0fb..00000000 --- a/synth/syntax/grammars/pcfg_splitter.py +++ /dev/null @@ -1,464 +0,0 @@ -# from typing import Dict, List, Optional, Tuple -# import bisect -# from dataclasses import dataclass, field -# import copy - -# import numpy as np - -# from synth.syntax.grammars.concrete_pcfg import ConcretePCFG, NonTerminal, PRules -# from synth.syntax.program import Program - - -# @dataclass(order=True, frozen=True) -# class Node: -# probability: float -# next_contexts: List[NonTerminal] = field(compare=False) -# program: List[Program] = field(compare=False) -# derivation_history: List[NonTerminal] = field(compare=False) - - -# def __common_prefix__(a: List[NonTerminal], b: List[NonTerminal]) -> List[NonTerminal]: -# if a == b: -# return a -# candidates = [] -# if len(a) > 1: -# candidates.append(__common_prefix__(a[1:], b)) -# if len(b) >= 1 and a[0] == b[0]: -# candidates.append([a[0]] + __common_prefix__(a[1:], b[1:])) -# if len(b) > 1: -# candidates.append(__common_prefix__(a, b[1:])) -# # Take longest common prefix -# lentghs = [len(x) for x in candidates] -# if len(lentghs) == 0: -# return [] -# if max(lentghs) == lentghs[0]: -# return candidates[0] -# return candidates[1] - - -# def __adapt_ctx__(S: NonTerminal, i: int) -> NonTerminal: -# pred = S.predecessors[0] -# return NonTerminal(S.type, [(pred[0], i)] + S.predecessors[1:], S.depth) - - -# def __create_path__( -# rules: PRules, -# original_pcfg: ConcretePCFG, -# rule_no: int, -# Slist: List[NonTerminal], -# Plist: List[Program], -# mapping: Dict[NonTerminal, NonTerminal], -# original_start: NonTerminal, -# ) -> int: -# for i, (S, P) in enumerate(zip(Slist, Plist)): -# if i == 0: -# S = original_start -# derivations = original_pcfg.rules[S][P][0] -# # Update derivations -# new_derivations = [] -# for nS in derivations: -# if nS not in Slist: -# new_derivations.append(nS) -# else: -# if nS in mapping: -# new_derivations.append(mapping[nS]) -# else: -# mS = __adapt_ctx__(nS, rule_no) -# mapping[nS] = mS -# new_derivations.append(mS) -# rule_no += 1 -# derivations = new_derivations -# # Update current S -# if i > 0: -# S = mapping[S] -# else: -# S = Slist[0] -# # Add rule -# rules[S] = {} -# rules[S][P] = derivations, 1 -# return rule_no - - -# def __pcfg_from__(original_pcfg: ConcretePCFG, group: List[Node]) -> ConcretePCFG: -# # Find the common prefix to all -# min_prefix = copy.deepcopy(group[0].derivation_history) -# for node in group[1:]: -# min_prefix = __common_prefix__(min_prefix, node.derivation_history) - -# # Extract the start symbol -# start = min_prefix.pop() - -# rules: Dict[NonTerminal, Dict[Program, Tuple[List[NonTerminal], float]]] = {} -# rule_no: int = ( -# max( -# max(x[1] for x in key.predecessors) if key.predecessors else 0 -# for key in original_pcfg.rules -# ) -# + 1 -# ) -# mapping: Dict[NonTerminal, NonTerminal] = {} -# # Our min_prefix may be something like (int, 1, (+, 1)) -# # which means we already chose + -# # But it is not in the PCFG -# # Thus we need to add it -# # In the general case we may as well have + -> + -> + as prefix this whole prefix needs to be added -# original_start = start -# if len(min_prefix) > 0: -# Slist = group[0].derivation_history[: len(min_prefix) + 1] -# Plist = group[0].program[: len(min_prefix) + 1] -# rule_no = __create_path__( -# rules, original_pcfg, rule_no, Slist, Plist, mapping, Slist[0] -# ) -# original_start = Slist[-1] -# start = mapping[original_start] - -# # Now we need to make a path from the common prefix to each node's prefix -# # We also need to mark all contexts that should be filled -# to_fill: List[NonTerminal] = [] -# for node in group: -# args, program, prefix = ( -# node.next_contexts, -# node.program, -# node.derivation_history, -# ) -# # Create rules to follow the path -# i = prefix.index(original_start) -# ctx_path = prefix[i:] -# program_path = program[i:] -# if len(ctx_path) > 0: -# ctx_path[0] = start -# rule_no = __create_path__( -# rules, -# original_pcfg, -# rule_no, -# ctx_path, -# program_path, -# mapping, -# original_start, -# ) -# # If there is no further derivation -# if not args: -# continue -# # Next derivation should be filled -# for arg in args: -# to_fill.append(arg) - -# # At this point rules can generate all partial programs -# # Get the S to normalize by descending depth order -# to_normalise = sorted(list(rules.keys()), key=lambda x: -x.depth) - -# # Build rules from to_fill -# while to_fill: -# S = to_fill.pop() -# rules[S] = {} -# for P in original_pcfg.rules[S]: -# args, w = original_pcfg.rules[S][P] -# rules[S][P] = (args[:], w) # copy list -# for arg in args: -# if arg not in rules: -# to_fill.append(arg) -# # At this point we have all the needed rules -# # However, the probabilites are incorrect -# while to_normalise: -# S = to_normalise.pop() -# if S not in original_pcfg.rules: -# continue -# # Compute the updated probabilities -# for P in list(rules[S]): -# args, _ = rules[S][P] -# # We have the following equation: -# # (1) w = old_w * remaining_fraction -# old_w = original_pcfg.rules[S][P][1] -# remaining_fraction: float = 1 -# # If there is a next derivation use it to compute the remaining_fraction -# if args: -# N = args[-1] -# remaining_fraction = sum(rules[N][K][1] for K in rules[N]) -# # Update according to Equation (1) -# rules[S][P] = args, old_w * remaining_fraction - -# # The updated probabilities may not sum to 1 so we need to normalise them -# # But let ConcretePCFG do it with clean=True - -# start = original_pcfg.start - -# # Ensure rules are depth ordered -# rules = { -# key: rules[key] for key in sorted(list(rules.keys()), key=lambda x: x.depth) -# } - -# return ConcretePCFG(start, rules, original_pcfg.max_program_depth, clean=True) - - -# def __node_split__(pcfg: ConcretePCFG, node: Node) -> Tuple[bool, List[Node]]: -# """ -# Split the specified node accordingly. - -# Return: success, nodes: -# - True, list of children nodes -# - False, [node] -# """ -# output: List[Node] = [] -# next_contexts = node.next_contexts -# # If there is no next then it means this node can't be split -# if len(next_contexts) == 0: -# return False, [node] -# new_context: NonTerminal = next_contexts.pop() -# for P in pcfg.rules[new_context]: -# args, p_prob = pcfg.rules[new_context][P] -# new_root = Node( -# node.probability * p_prob, -# next_contexts + args, -# node.program + [P], -# node.derivation_history + [new_context], -# ) -# output.append(new_root) -# return True, output - - -# def __split_nodes_until_quantity_reached__( -# pcfg: ConcretePCFG, quantity: int -# ) -> List[Node]: -# """ -# Start from the root node and split most probable node until the threshold number of nodes is reached. -# """ -# nodes: List[Node] = [Node(1.0, [pcfg.start], [], [])] -# while len(nodes) < quantity: -# i = 1 -# success, new_nodes = __node_split__(pcfg, nodes.pop()) -# while not success: -# i += 1 -# nodes.append(new_nodes[0]) -# success, new_nodes = __node_split__(pcfg, nodes.pop(-i)) -# for new_node in new_nodes: -# insertion_index: int = bisect.bisect(nodes, new_node) -# nodes.insert(insertion_index, new_node) - -# return nodes - - -# def __holes_of__(pcfg: ConcretePCFG, node: Node) -> List[NonTerminal]: -# stack = [pcfg.start] -# current = node.program[:] -# while current: -# S = stack.pop() -# P = current.pop(0) -# args = pcfg.rules[S][P][0] -# for arg in args: -# stack.append(arg) -# return stack - - -# def __is_fixing_any_hole__( -# pcfg: ConcretePCFG, node: Node, holes: List[NonTerminal] -# ) -> bool: -# current = node.program[:] -# stack = [pcfg.start] -# while current: -# S = stack.pop() -# if S in holes: -# return True -# P = current.pop(0) -# args = pcfg.rules[S][P][0] -# for arg in args: -# stack.append(arg) -# return False - - -# def __are_compatible__(pcfg: ConcretePCFG, node1: Node, node2: Node) -> bool: -# """ -# Two nodes prefix are compatible if one does not fix a context for the other. -# e.g. a -> b -> map -> * and c -> b -> map -> +1 -> * are incompatible. - -# In both cases map have the same context (bigram context) which is ((predecessor=b, argument=0), depth=2) thus are indistinguishables. -# However in the former all derivations are allowed in this context whereas in the latter +1 must be derived. -# Thus we cannot create a CFG that enables both. -# """ -# holes1 = __holes_of__(pcfg, node1) -# if __is_fixing_any_hole__(pcfg, node2, holes1): -# return False -# holes2 = __holes_of__(pcfg, node2) -# return not __is_fixing_any_hole__(pcfg, node1, holes2) - - -# def __all_compatible__(pcfg: ConcretePCFG, node: Node, group: List[Node]) -> bool: -# return all(__are_compatible__(pcfg, node, node2) for node2 in group) - - -# def __try_split_node_in_group__( -# pcfg: ConcretePCFG, prob_groups: List[List], group_index: int -# ) -> bool: -# group_a: List[Node] = prob_groups[group_index][1] -# # Sort group by ascending probability -# group_a_bis = sorted(group_a, key=lambda x: x.probability) -# # Try splitting a node until success -# i = 1 -# success, new_nodes = __node_split__(pcfg, group_a_bis[-i]) -# while not success and i < len(group_a): -# i += 1 -# success, new_nodes = __node_split__(pcfg, group_a_bis[-i]) -# if i >= len(group_a): -# return False -# # Success, remove old node -# group_a.pop(-i) -# # Add new nodes -# for new_node in new_nodes: -# group_a.append(new_node) -# return True - - -# def __find_swap_for_group__( -# pcfg: ConcretePCFG, prob_groups: List[List], group_index: int -# ) -> Optional[Tuple[int, Optional[int], int]]: -# max_prob: float = prob_groups[-1][1] -# min_prob: float = prob_groups[0][1] -# group_a, prob = prob_groups[group_index] -# best_swap: Optional[Tuple[int, Optional[int], int]] = None -# current_score: float = max_prob / prob - -# candidates = ( -# list(range(len(prob_groups) - 1, group_index, -1)) -# if group_index == 0 -# else [len(prob_groups) - 1] -# ) - -# for i in candidates: -# group_b, prob_b = prob_groups[i] -# for j, node_a in enumerate(group_a): -# pa: float = node_a.probability -# reduced_prob: float = prob - pa -# # Try all swaps -# for k, node_b in enumerate(group_b): -# pb: float = node_b.probability -# if ( -# pb < pa -# or not __all_compatible__(pcfg, node_a, group_b) -# or not __all_compatible__(pcfg, node_b, group_a) -# ): -# continue -# new_mass_b: float = prob_b - pb + pa -# mini = min_prob if group_index > 0 else reduced_prob + pb -# maxi = ( -# max(new_mass_b, prob_groups[-2][1]) -# if j == len(prob_groups) - 1 -# else max_prob -# ) -# new_score = maxi / mini -# if new_score < current_score: -# best_swap = (i, j, k) -# current_score = new_score -# # Consider taking something from b -# for k, node_b in enumerate(group_b): -# if not __all_compatible__(pcfg, node_b, group_a): -# continue -# pb = node_b.probability -# if prob + pb > max_prob: -# new_score = (prob + pb) / min_prob -# else: -# new_score = max_prob / (prob + pb) -# if new_score < current_score: -# best_swap = (i, None, k) -# current_score = new_score -# return best_swap - - -# def __percolate_down__(prob_groups: List[List], group_index: int) -> None: -# index = group_index -# p = prob_groups[group_index][1] -# while index > 0 and prob_groups[index - 1][1] > p: -# prob_groups[index - 1], prob_groups[index] = ( -# prob_groups[index], -# prob_groups[index - 1], -# ) -# index -= 1 - - -# def __percolate_up__(prob_groups: List[List], group_index: int) -> None: -# index = group_index -# p = prob_groups[group_index][1] -# while index < len(prob_groups) - 2 and prob_groups[index + 1][1] < p: -# prob_groups[index + 1], prob_groups[index] = ( -# prob_groups[index], -# prob_groups[index + 1], -# ) -# index += 1 - - -# def __apply_swap__( -# prob_groups: List[List], group_index: int, swap: Tuple[int, Optional[int], int] -# ) -> None: -# j, k, l = swap -# # App -# if k: -# node_a = prob_groups[group_index][0].pop(k) -# prob_groups[group_index][1] -= node_a.probability -# prob_groups[j][0].append(node_a) -# prob_groups[j][1] += node_a.probability - -# node_b = prob_groups[j][0].pop(l) -# prob_groups[j][1] -= node_b.probability -# prob_groups[group_index][0].append(node_b) -# prob_groups[group_index][1] += node_b.probability - -# __percolate_down__(prob_groups, -1) -# __percolate_up__(prob_groups, group_index) - - -# def __split_into_nodes__( -# pcfg: ConcretePCFG, splits: int, desired_ratio: float = 2 -# ) -> Tuple[List[List[Node]], float]: -# nodes = __split_nodes_until_quantity_reached__(pcfg, splits) - -# # Create groups -# groups: List[List[Node]] = [] -# for node in nodes[:splits]: -# groups.append([node]) -# for node in nodes[splits:]: -# # Add to first compatible group -# added = False -# for group in groups: -# if __all_compatible__(pcfg, node, group): -# group.append(node) -# added = True -# break -# assert added - -# # Improve -# improved = True -# masses: List[float] = [np.sum([x.probability for x in group]) for group in groups] -# prob_groups = sorted([[g, p] for g, p in zip(groups, masses)], key=lambda x: x[1]) # type: ignore -# ratio: float = prob_groups[-1][1] / prob_groups[0][1] # type: ignore -# while improved and ratio > desired_ratio: -# improved = False -# for i in range(splits - 1): -# swap = __find_swap_for_group__(pcfg, prob_groups, i) -# if swap: -# improved = True -# __apply_swap__(prob_groups, i, swap) -# break -# if not improved: -# for i in range(splits - 1, 0, -1): -# improved = __try_split_node_in_group__(pcfg, prob_groups, i) -# if improved: -# break -# ratio = prob_groups[-1][1] / prob_groups[0][1] # type: ignore -# return [g for g, _ in prob_groups], ratio # type: ignore - - -# def split( -# pcfg: ConcretePCFG, splits: int, desired_ratio: float = 1.1 -# ) -> Tuple[List[ConcretePCFG], float]: -# """ -# Currently use exchange split. -# Parameters: -# desired_ratio: the max ratio authorized between the most probable group and the least probable pcfg - -# Return: -# a list of ConcretePCFG -# the reached threshold -# """ -# if splits == 1: -# return [pcfg], 1 -# assert desired_ratio > 1, "The desired ratio must be > 1!" -# groups, ratio = __split_into_nodes__(pcfg, splits, desired_ratio) -# return [__pcfg_from__(pcfg, group) for group in groups if len(group) > 0], ratio diff --git a/synth/syntax/grammars/tagged_det_grammar.py b/synth/syntax/grammars/tagged_det_grammar.py index 9fe37757..c14e5b8a 100644 --- a/synth/syntax/grammars/tagged_det_grammar.py +++ b/synth/syntax/grammars/tagged_det_grammar.py @@ -1,20 +1,26 @@ from typing import ( + Any, + Callable, Dict, Generator, Generic, Iterable, List, + Literal, Optional, Tuple, TypeVar, + TYPE_CHECKING, ) import numpy as np -import vose -from synth.syntax.grammars.cfg import CFG, CFGNonTerminal, CFGState, NoneType +from synth.utils.vose_polyfill import Sampler as VoseSampler +if TYPE_CHECKING: + from synth.syntax.grammars.cfg import CFG +from synth.syntax.grammars.grammar import NGram from synth.syntax.grammars.det_grammar import DerivableProgram, DetGrammar -from synth.syntax.program import Function, Program +from synth.syntax.program import Constant, Function, Program from synth.syntax.type_system import Type T = TypeVar("T") @@ -22,6 +28,10 @@ V = TypeVar("V") W = TypeVar("W") +NoneType = Literal[None] +CFGState = Tuple[NGram, int] +CFGNonTerminal = Tuple[Type, Tuple[CFGState, NoneType]] + class TaggedDetGrammar(DetGrammar[U, V, W], Generic[T, U, V, W]): def __init__( @@ -32,6 +42,10 @@ def __init__( super().__init__(grammar.start, grammar.rules, clean=False) self.grammar = grammar self.tags = tags + self.type_request = grammar.type_request + + def programs(self) -> int: + return self.grammar.programs() def __hash__(self) -> int: return hash((self.start, self.grammar, str(self.tags))) @@ -81,7 +95,22 @@ def __add__( else: safe = {P: other.tags[S][P]} new_probs[S][P] = self.tags.get(S, safe)[P] + other.tags.get(S, safe)[P] # type: ignore - return TaggedDetGrammar(self.grammar, new_probs) + return self.__class__(self.grammar, new_probs) + + def instantiate_constants( + self, constants: Dict[Type, List[Any]] + ) -> "TaggedDetGrammar[T, U, V, W]": + tags: Dict[Tuple[Type, U], Dict[DerivableProgram, T]] = {} + + for S in self.tags: + tags[S] = {} + for P in self.tags[S]: + if isinstance(P, Constant) and P.type in constants: + for val in constants[P.type]: + tags[S][Constant(P.type, val, True)] = self.tags[S][P] + else: + tags[S][P] = self.tags[S][P] + return self.__class__(self.grammar.instantiate_constants(constants), tags) class ProbDetGrammar(TaggedDetGrammar[float, U, V, W]): @@ -93,6 +122,9 @@ def __init__( super().__init__(grammar, probabilities) self.ready_for_sampling = False + def __hash__(self) -> int: + return super().__hash__() + @property def probabilities(self) -> Dict[Tuple[Type, U], Dict[DerivableProgram, float]]: return self.tags @@ -134,7 +166,7 @@ def init_sampling(self, seed: Optional[int] = None) -> None: for i, S in enumerate(self.tags): P_list = list(self.tags[S].keys()) - self.vose_samplers[S] = vose.Sampler( + self.vose_samplers[S] = VoseSampler( np.array( [self.tags[S][P] for P in P_list], dtype=float, @@ -178,6 +210,23 @@ def sample_program( current = lst[-1] return Function(P, arguments) + def instantiate_constants( + self, constants: Dict[Type, List[Any]] + ) -> "ProbDetGrammar[U, V, W]": + tags: Dict[Tuple[Type, U], Dict[DerivableProgram, float]] = {} + + for S in self.tags: + tags[S] = {} + for P in self.tags[S]: + if isinstance(P, Constant) and P.type in constants: + for val in constants[P.type]: + tags[S][Constant(P.type, val, True)] = self.tags[S][P] / len( + constants[P.type] + ) + else: + tags[S][P] = self.tags[S][P] + return self.__class__(self.grammar.instantiate_constants(constants), tags) + @classmethod def uniform(cls, grammar: DetGrammar[U, V, W]) -> "ProbDetGrammar[U, V, W]": return ProbDetGrammar( @@ -188,10 +237,31 @@ def uniform(cls, grammar: DetGrammar[U, V, W]) -> "ProbDetGrammar[U, V, W]": }, ) + @classmethod + def random( + cls, + grammar: DetGrammar[U, V, W], + seed: Optional[int] = None, + gen: Callable[[np.random.Generator], float] = lambda prng: prng.uniform(), + ) -> "ProbDetGrammar[U, V, W]": + prng = np.random.default_rng(seed) + pg = ProbDetGrammar( + grammar, + {S: {_: gen(prng) for _ in grammar.rules[S]} for S in grammar.rules}, + ) + pg.normalise() + return pg + @classmethod def pcfg_from_samples( - cls, cfg: CFG, samples: Iterable[Program] + cls, cfg: "CFG", samples: Iterable[Program], remove_zero_rules: bool = False ) -> "ProbDetGrammar[Tuple[CFGState, NoneType], Tuple[List[Tuple[Type, CFGState]], NoneType], List[Tuple[Type, CFGState]]]": + """ + Produces the PCFG whose distribution is the mean distribution over the samples. + + Rules with probability zero are not removed unless remove_zero_rules is True. + + """ rules_cnt: Dict[CFGNonTerminal, Dict[DerivableProgram, int]] = {} for S in cfg.rules: rules_cnt[S] = {} @@ -224,9 +294,14 @@ def add_count(S: CFGNonTerminal, P: Program) -> bool: probabilities: Dict[CFGNonTerminal, Dict[DerivableProgram, float]] = {} for S in cfg.rules: total = sum(rules_cnt[S][P] for P in cfg.rules[S]) - if total > 0: - probabilities[S] = {} - for P in rules_cnt[S]: - probabilities[S][P] = rules_cnt[S][P] / total + if total <= 0: + if remove_zero_rules: + continue + total = 1 + probabilities[S] = {} + for P in rules_cnt[S]: + val = rules_cnt[S].get(P, 0) + if val > 0 or not remove_zero_rules: + probabilities[S][P] = val / total return ProbDetGrammar(cfg, probabilities) diff --git a/synth/syntax/grammars/tagged_u_grammar.py b/synth/syntax/grammars/tagged_u_grammar.py new file mode 100644 index 00000000..2cffc969 --- /dev/null +++ b/synth/syntax/grammars/tagged_u_grammar.py @@ -0,0 +1,312 @@ +from typing import ( + Any, + Callable, + Dict, + Generator, + Generic, + List, + Optional, + Tuple, + TypeVar, +) + +import numpy as np +from synth.utils.vose_polyfill import Sampler as VoseSampler + +from synth.syntax.grammars.det_grammar import DerivableProgram +from synth.syntax.grammars.u_grammar import UGrammar +from synth.syntax.program import Constant, Function, Program +from synth.syntax.type_system import Type + +T = TypeVar("T") +U = TypeVar("U") +V = TypeVar("V") +W = TypeVar("W") + + +class TaggedUGrammar(UGrammar[U, V, W], Generic[T, U, V, W]): + def __init__( + self, + grammar: UGrammar[U, V, W], + tags: Dict[Tuple[Type, U], Dict[DerivableProgram, Dict[V, T]]], + start_tags: Dict[Tuple[Type, U], T], + ): + super().__init__(grammar.starts, grammar.rules, clean=False) + self.grammar = grammar + self.tags = tags + self.start_tags = start_tags + + def programs(self) -> int: + return self.grammar.programs() + + def __hash__(self) -> int: + return hash((str(self.start_tags), self.grammar, str(self.tags))) + + def __eq__(self, o: object) -> bool: + return ( + isinstance(o, TaggedUGrammar) + and self.grammar == o.grammar + and self.tags == o.tags + and self.start_tags == o.start_tags + ) + + def name(self) -> str: + return "tagged" + self.grammar.name() + + def __str__(self) -> str: + s = f"Print a {self.name()}\n" + for S in reversed(self.grammar.rules): + add = f" [START p={self.start_tags[S]}]" if S in self.starts else "" + s += "#\n {}{}\n".format(S, add) + for P in self.grammar.rules[S]: + out = self.grammar.rules[S][P] + for possible in out: + s += " {} ~> {}\n".format( + self.tags[S][P][tuple(possible)], # type: ignore + self.grammar.__rule_to_str__(P, possible), + ) + return s + + def arguments_length_for(self, S: Tuple[Type, U], P: DerivableProgram) -> int: + return self.grammar.arguments_length_for(S, P) + + def derive( + self, information: W, S: Tuple[Type, U], P: DerivableProgram + ) -> List[Tuple[W, Tuple[Type, U], V]]: + return self.grammar.derive(information, S, P) + + def derive_specific( + self, information: W, S: Tuple[Type, U], P: DerivableProgram, v: V + ) -> Optional[Tuple[W, Tuple[Type, U]]]: + return self.grammar.derive_specific(information, S, P, v) + + def start_information(self) -> W: + return self.grammar.start_information() + + def __add__( + self, other: "TaggedUGrammar[T, U, V, W]" + ) -> "TaggedUGrammar[T, U, V, W]": + new_probs: Dict[Tuple[Type, U], Dict[DerivableProgram, Dict[V, T]]] = {} + for S in set(self.tags.keys()).union(other.tags.keys()): + new_probs[S] = {} + for P in set(self.tags.get(S, {})).union(other.tags.get(S, {})): + for key in self.tags[S][P]: + new_probs[S][P][key] = self.tags[S][P][key] + other.tags[S][P][key] # type: ignore + new_start_tags: Dict[Tuple[Type, U], T] = { + key: value + other.start_tags[key] # type: ignore + for key, value in self.start_tags.items() + } + return self.__class__(self.grammar, new_probs, new_start_tags) + + def instantiate_constants( + self, constants: Dict[Type, List[Any]] + ) -> "TaggedUGrammar[T, U, V, W]": + tags: Dict[Tuple[Type, U], Dict[DerivableProgram, Dict[V, T]]] = {} + + for S in self.tags: + tags[S] = {} + for P in self.tags[S]: + if isinstance(P, Constant) and P.type in constants: + for val in constants[P.type]: + tags[S][Constant(P.type, val, True)] = self.tags[S][P] + else: + tags[S][P] = self.tags[S][P] + return self.__class__( + self.grammar.instantiate_constants(constants), tags, self.start_tags + ) + + +class ProbUGrammar(TaggedUGrammar[float, U, V, W]): + def __init__( + self, + grammar: UGrammar[U, V, W], + probabilities: Dict[Tuple[Type, U], Dict[DerivableProgram, Dict[V, float]]], + start_probs: Dict[Tuple[Type, U], float], + ): + super().__init__(grammar, probabilities, start_probs) + self.ready_for_sampling = False + + def __hash__(self) -> int: + return super().__hash__() + + @property + def start_probabilities( + self, + ) -> Dict[Tuple[Type, U], float]: + return self.start_tags + + @property + def probabilities( + self, + ) -> Dict[Tuple[Type, U], Dict[DerivableProgram, Dict[V, float]]]: + return self.tags + + def name(self) -> str: + return "P" + self.grammar.name() + + def __mul__(self, other: float) -> "ProbUGrammar[U, V, W]": + return ProbUGrammar( + self.grammar, + { + S: {P: {v: p * other for v, p in lst.items()} for P, lst in v.items()} + for S, v in self.tags.items() + }, + {S: v * other for S, v in self.start_tags.items()}, + ) + + def __rmul__(self, other: float) -> "ProbUGrammar[U, V, W]": + return self.__mul__(other) + + def probability( + self, + program: Program, + start: Optional[Tuple[Type, U]] = None, + ) -> float: + try: + return self.reduce_derivations( + lambda current, S, P, V: current * self.tags[S][P][tuple(V)], # type: ignore + 1.0, + program, + start, + )[0] + except KeyError: + return 0 + + def init_sampling(self, seed: Optional[int] = None) -> None: + """ + seed = 0 <=> No Seeding + """ + self.ready_for_sampling = True + self.vose_samplers: Dict[Tuple[Type, U], Any] = {} + self.sampling_map: Dict[Tuple[Type, U], List[DerivableProgram]] = {} + self._vose_samplers_2: Dict[Tuple[Type, U], Dict[DerivableProgram, Any]] = {} + + for i, S in enumerate(self.tags): + P_list = list(self.tags[S].keys()) + self.vose_samplers[S] = VoseSampler( + np.array( + [sum(p for p in self.tags[S][P].values()) for P in P_list], + dtype=float, + ), + seed=seed + i if seed else None, + ) + self._vose_samplers_2[S] = {} + for P in P_list: + self._vose_samplers_2[S][P] = VoseSampler( + np.array( + [p for p in self.tags[S][P].values()], + dtype=float, + ) + / sum(p for p in self.tags[S][P].values()), + seed=seed + 7 * i if seed else None, + ) + self.sampling_map[S] = P_list + self._int2start = list(self.starts) + self._start_sampler = VoseSampler( + np.array( + [v for v in self.start_tags.values()], + dtype=float, + ), + seed=seed + len(self.tags) if seed else None, + ) + + def normalise(self) -> None: + for S in self.tags: + s = sum( + sum(self.tags[S][P][V] for V in self.tags[S][P]) for P in self.tags[S] + ) + for P in list(self.tags[S].keys()): + w = self.tags[S][P] + self.tags[S][P] = {v: p / s for v, p in w.items()} + + s = sum(v for v in self.start_tags.values()) + for S in self.start_tags: + self.start_tags[S] /= s + + def sampling(self) -> Generator[Program, None, None]: + """ + A generator that samples programs according to the PCFG G + """ + assert self.ready_for_sampling + while True: + yield self.sample_program() + + def sample_program( + self, S: Optional[Tuple[Type, U]] = None, information: Optional[W] = None + ) -> Program: + assert self.ready_for_sampling + if S is None: + S = self._int2start[self._start_sampler.sample()] + i: int = self.vose_samplers[S].sample() + P = self.sampling_map[S][i] + nargs = self.arguments_length_for(S, P) + if nargs == 0: + return P + arguments = [] + information = information or self.grammar.start_information() + i = self._vose_samplers_2[S][P].sample() + information, current, _ = self.grammar.derive(information, S, P)[i] + for _ in range(nargs): + arg = self.sample_program(current, information) + arguments.append(arg) + information, lst = self.grammar.derive_all(information, current, arg)[0] + current = lst[-1][0] + return Function(P, arguments) + + def instantiate_constants( + self, constants: Dict[Type, List[Any]] + ) -> "ProbUGrammar[U, V, W]": + tags: Dict[Tuple[Type, U], Dict[DerivableProgram, Dict[V, float]]] = {} + + for S in self.tags: + tags[S] = {} + for P in self.tags[S]: + if isinstance(P, Constant) and P.type in constants: + for val in constants[P.type]: + tags[S][Constant(P.type, val, True)] = { + k: v / len(constants[P.type]) + for k, v in self.tags[S][P].items() + } + else: + tags[S][P] = self.tags[S][P] + return self.__class__( + self.grammar.instantiate_constants(constants), tags, self.start_tags + ) + + @classmethod + def uniform(cls, grammar: UGrammar[U, V, W]) -> "ProbUGrammar[U, V, W]": + probs: Dict[Tuple[Type, U], Dict[DerivableProgram, Dict[V, float]]] = {} + for S in grammar.rules: + probs[S] = {} + n = sum(len(grammar.rules[S][P]) for P in grammar.rules[S]) + for P in grammar.rules[S]: + lst = grammar.rules[S][P] + if isinstance(lst[0], List): + probs[S][P] = {tuple(v): 1 / n for v in lst} # type: ignore + else: + probs[S][P] = {v: 1 / n for v in lst} + + start_probs = {start: 1 / len(grammar.starts) for start in grammar.starts} + return ProbUGrammar(grammar, probs, start_probs) + + @classmethod + def random( + cls, + grammar: UGrammar[U, V, W], + seed: Optional[int] = None, + gen: Callable[[np.random.Generator], float] = lambda prng: prng.uniform(), + ) -> "ProbUGrammar[U, V, W]": + prng = np.random.default_rng(seed) + pg = ProbUGrammar( + grammar, + { + S: { + P: {tuple(x) if isinstance(x, List) else x: gen(prng) for x in der} # type: ignore + for P, der in grammar.rules[S].items() + } + for S in grammar.rules + }, + {S: gen(prng) for S in grammar.starts}, + ) + pg.normalise() + return pg diff --git a/synth/syntax/grammars/ttcfg.py b/synth/syntax/grammars/ttcfg.py index 9a1a75fe..60145ea7 100644 --- a/synth/syntax/grammars/ttcfg.py +++ b/synth/syntax/grammars/ttcfg.py @@ -1,10 +1,12 @@ -from collections import deque -from dataclasses import dataclass, field +from collections import defaultdict, deque +from functools import lru_cache from typing import ( + Any, Callable, Deque, Dict, List, + Optional, Set, Tuple, TypeVar, @@ -14,10 +16,12 @@ ) from synth.syntax.dsl import DSL -from synth.syntax.grammars.dfa import DFA +from synth.syntax.automata.dfa import DFA +from synth.syntax.grammars.grammar import DerivableProgram, NGram +from synth.syntax.grammars.tagged_det_grammar import ProbDetGrammar from synth.syntax.program import Constant, Primitive, Variable from synth.syntax.type_system import Arrow, Type, UnknownType -from synth.syntax.grammars.det_grammar import DerivableProgram, DetGrammar +from synth.syntax.grammars.det_grammar import DetGrammar T = TypeVar("T") U = TypeVar("U") @@ -25,33 +29,6 @@ V = TypeVar("V") -@dataclass(frozen=True) -class NGram: - n: int - predecessors: List[Tuple[DerivableProgram, int]] = field(default_factory=lambda: []) - - def __hash__(self) -> int: - return hash((self.n, tuple(self.predecessors))) - - def __str__(self) -> str: - return str(self.predecessors) - - def __repr__(self) -> str: - return self.__str__() - - def __len__(self) -> int: - return len(self.predecessors) - - def successor(self, new_succ: Tuple[DerivableProgram, int]) -> "NGram": - new_pred = [new_succ] + self.predecessors - if len(new_pred) + 1 > self.n: - new_pred.pop() - return NGram(self.n, new_pred) - - def last(self) -> Tuple[DerivableProgram, int]: - return self.predecessors[0] - - class TTCFG( DetGrammar[Tuple[S, T], Tuple[List[Tuple[Type, S]], T], List[Tuple[Type, S]]], Generic[S, T], @@ -67,6 +44,9 @@ def __eq__(self, o: object) -> bool: and self.rules == o.rules ) + def __hash__(self) -> int: + return super().__hash__() + def derive( self, information: List[Tuple[Type, S]], @@ -89,15 +69,31 @@ def __mul__(self, other: "TTCFG[U, V]") -> "TTCFG[Tuple[S, U], Tuple[T, V]]": pass @overload - def __mul__(self, other: DFA[U, str]) -> "TTCFG[S, Tuple[T, U]]": + def __mul__(self, other: DFA[U, DerivableProgram]) -> "TTCFG[S, Tuple[T, U]]": + pass + + @overload + def __mul__( + self, other: DFA[U, Tuple[Tuple[Type, Tuple[S, T]], DerivableProgram]] + ) -> "TTCFG[S, Tuple[T, U]]": pass def __mul__( - self, other: Union["TTCFG[U, V]", DFA[U, str]] + self, + other: Union[ + "TTCFG[U, V]", + DFA[U, DerivableProgram], + DFA[U, Tuple[Tuple[Type, Tuple[S, T]], DerivableProgram]], + ], ) -> Union["TTCFG[S, Tuple[T, U]]", "TTCFG[Tuple[S, U], Tuple[T, V]]"]: if isinstance(other, TTCFG): return self.__mul_ttcfg__(other) - return self.__mul_dfa__(other) + elif isinstance(other, DFA): + if isinstance(list(other.rules[other.start].keys())[0], tuple): + return self.__mul_dfa__(other) # type: ignore + else: + return self.__mul_dfa_simple__(other) # type: ignore + assert False, f"Cannot multiply TTCFG with {other}" def __mul_ttcfg__(self, other: "TTCFG[U, V]") -> "TTCFG[Tuple[S, U], Tuple[T, V]]": assert ( @@ -141,7 +137,9 @@ def __mul_ttcfg__(self, other: "TTCFG[U, V]") -> "TTCFG[Tuple[S, U], Tuple[T, V] return TTCFG(start, rules, clean=True) - def __mul_dfa__(self, other: DFA[U, str]) -> "TTCFG[S, Tuple[T, U]]": + def __mul_dfa_simple__( + self, other: DFA[U, DerivableProgram] + ) -> "TTCFG[S, Tuple[T, U]]": rules: Dict[ Tuple[Type, Tuple[S, Tuple[T, U]]], Dict[ @@ -162,7 +160,7 @@ def __mul_dfa__(self, other: DFA[U, str]) -> "TTCFG[S, Tuple[T, U]]": rules[rule] = {} for P1 in self.rules[nT1]: for P2 in other.rules[nT2]: - if str(P1) != P2: + if P1 != P2: continue new_deriv = self.rules[nT1][P1][0][:] rules[rule][P1] = ( @@ -171,50 +169,92 @@ def __mul_dfa__(self, other: DFA[U, str]) -> "TTCFG[S, Tuple[T, U]]": ) return TTCFG(start, rules, clean=True) + def __mul_dfa__( + self, other: DFA[U, Tuple[Tuple[Type, Tuple[S, T]], DerivableProgram]] + ) -> "TTCFG[S, Tuple[T, U]]": + rules: Dict[ + Tuple[Type, Tuple[S, Tuple[T, U]]], + Dict[ + Union[Primitive, Variable, Constant], + Tuple[List[Tuple[Type, S]], Tuple[T, U]], + ], + ] = {} + start: Tuple[Type, Tuple[S, Tuple[T, U]]] = ( + self.start[0], + ( + self.start[1][0], + (self.start[1][1], other.start), + ), + ) + for nT1 in self.rules: + for nT2 in other.rules: + rule = (nT1[0], (nT1[1][0], (nT1[1][1], nT2))) + rules[rule] = {} + for S2, P2 in other.rules[nT2]: + if S2 != nT1: + continue + for P1 in self.rules[nT1]: + if P1 != P2: + continue + new_deriv = self.rules[nT1][P1][0][:] + rules[rule][P1] = ( + new_deriv, + (self.rules[nT1][P1][1], other.rules[nT2][(S2, P2)]), + ) + return TTCFG(start, rules, clean=True) + def clean(self) -> None: + # 1) Only keep reachable states new_rules: Dict[Tuple[Type, Tuple[S, T]], Set[DerivableProgram]] = {} list_to_be_treated: Deque[ - Tuple[Tuple[Type, S], T, List[Tuple[Type, S]]] + Tuple[Tuple[Type, Tuple[S, T]], List[Tuple[Type, S]]] ] = deque() - list_to_be_treated.append( - ((self.start[0], self.start[1][0]), self.start[1][1], []) - ) - - if isinstance(self.type_request, Arrow): - args = self.type_request.arguments() - else: - args = [] + list_to_be_treated.append((self.start, self.start_information())) while list_to_be_treated: - (current_type, non_terminal), current, stack = list_to_be_treated.pop() - rule = current_type, (non_terminal, current) - # Create rule if non existent + rule, info = list_to_be_treated.pop() if rule not in new_rules: new_rules[rule] = set() - else: - continue - # Try to add variables rules - for i in range(len(args)): - if current_type == args[i]: - var = Variable(i, current_type) - if var in self.rules[rule]: - new_rules[rule].add(var) - _, s = self.rules[rule][var] - if stack: - list_to_be_treated.append((stack[0], s, stack[1:])) - # DSL Primitives + # Create rule if non existent for P in self.rules[rule]: - type_P = P.type - arguments_P = type_P.ends_with(current_type) - if arguments_P is not None: - if P in self.rules[rule]: - decorated_arguments_P, new_el = self.rules[rule][P] - new_rules[rule].add(P) - tmp_stack = decorated_arguments_P + stack - if tmp_stack: - list_to_be_treated.append( - (tmp_stack[0], new_el, tmp_stack[1:]) - ) + new_rules[rule].add(P) + new_info, new_S = self.derive(info, rule, P) + if new_S in self.rules: + list_to_be_treated.append((new_S, new_info)) + + # 2) Remove empty non terminals + def clean() -> bool: + list_to_be_treated: Deque[ + Tuple[Tuple[Type, Tuple[S, T]], List[Tuple[Type, S]]] + ] = deque() + return_value = False + list_to_be_treated.append((self.start, self.start_information())) + while list_to_be_treated: + rule, info = list_to_be_treated.pop() + if rule not in new_rules: + continue + if len(new_rules[rule]) == 0: + del new_rules[rule] + return_value = True + continue + # Create rule if non existent + for P in list(new_rules[rule]): + new_info, new_S = self.derive(info, rule, P) + if ( + new_S not in new_rules + and new_S in self.rules + and len(new_info) >= len(info) + ): + new_rules[rule].remove(P) + if len(new_rules[rule]) == 0: + del new_rules[rule] + return_value = True + elif new_S in self.rules: + list_to_be_treated.append((new_S, new_info)) + return return_value + + while clean(): + pass self.rules = {S: {P: self.rules[S][P] for P in new_rules[S]} for S in new_rules} @@ -235,22 +275,118 @@ def __rule_to_str__( def name(self) -> str: return "TTCFG" + @lru_cache() + def programs(self) -> int: + """ + Return the total number of programs contained in this grammar. + """ + _counts: Dict[Tuple[Type, Tuple[S, T]], Dict[T, int]] = {} + + def __compute__(state: Tuple[Type, Tuple[S, T]]) -> Dict[T, int]: + if state in _counts: + return _counts[state] + if state not in self.rules: + return {state[1][1]: 1} + output: Dict[T, int] = defaultdict(int) + for P in self.rules[state]: + info, new_state = self.derive(self.start_information(), state, P) + local = __compute__(new_state) + while info: + base = info.pop() + next_local: Dict[T, int] = defaultdict(int) + for v, cnt in local.items(): + next_new_state = (base[0], (base[1], v)) + for nV, nC in __compute__(next_new_state).items(): + next_local[nV] += nC * cnt + local = next_local + for v, c in local.items(): + output[v] += c + _counts[state] = output + return output + + return sum(c for _, c in __compute__(self.start).items()) + + def programs_stochastic( + self, cfg: "TTCFG", samples: int = 10000, seed: Optional[int] = None + ) -> float: + """ + Provides an estimate of the number of programs in this grammar based on cfg. + cfg must contain this grammar. + Returns: the fraction of programs of cfg that this grammar contains + """ + pcfg = ProbDetGrammar.uniform(cfg) + pcfg.init_sampling(seed) + inside = 0 + for _ in range(samples): + if pcfg.sample_program() in self: + inside += 1 + return inside / samples + + def possible_outcomes_after( + self, + S: Tuple[Type, Tuple[S, T]], + P: Optional[DerivableProgram] = None, + info: Optional[List[Tuple[Type, S]]] = None, + ) -> Set[T]: + """ + Return the set of all possibles T that can be generated starting from S -> P or just from S. + """ + info = info or self.start_information() + if S not in self.rules: + return set() + if P is None: + # print(f" ask {S} ({info})") + out_plus = set() + for P in self.rules[S]: + out_plus |= self.possible_outcomes_after(S, P, info) + # print(f" end {S} ({info})") + + return out_plus + new_info, new_S = self.derive(info, S, P) + if new_S not in self.rules: + assert ( + len(new_info) == 0 + ), f"info:{new_info} from {S}->{P} ({info}) obtained: {new_S}" + return set([new_S[1][1]]) + # print(f" ask {S} -> {P} ({info}->{new_info})") + # Derive all possible children + out = set() + for PP in self.rules[new_S]: + candidates = self.possible_outcomes_after(new_S, PP, new_info) + out |= candidates + return out + + def instantiate_constants(self, constants: Dict[Type, List[Any]]) -> "TTCFG[S, T]": + rules: Dict[ + Tuple[Type, Tuple[S, T]], + Dict[DerivableProgram, Tuple[List[Tuple[Type, S]], T]], + ] = {} + for NT in self.rules: + rules[NT] = {} + for P in self.rules[NT]: + if isinstance(P, Constant) and P.type in constants: + for val in constants[P.type]: + rules[NT][Constant(P.type, val, True)] = self.rules[NT][P] + else: + rules[NT][P] = self.rules[NT][P] + # Cleaning produces infinite loop + return self.__class__(self.start, rules, clean=False) + @classmethod def size_constraint( cls, dsl: DSL, type_request: Type, max_size: int, n_gram: int = 2 - ) -> "TTCFG[NGram, int]": + ) -> "TTCFG[NGram, Tuple[int, int]]": """ Constructs a n-gram TT CFG from a DSL imposing the maximum program size. max_size: int - is the maxium depth of programs allowed """ - dsl.instantiate_forbidden() forbidden_sets = dsl.forbidden_patterns def __transition__( - state: Tuple[Type, Tuple[NGram, int]], + state: Tuple[Type, Tuple[NGram, Tuple[int, int]]], derivation: Union[Primitive, Variable, Constant], - ) -> Tuple[bool, int]: + ) -> Tuple[bool, Tuple[int, int]]: predecessors = state[1][0] last_pred = predecessors.last() if len(predecessors) > 0 else None if derivation in forbidden_sets.get( @@ -259,18 +395,23 @@ def __transition__( else ("", 0), set(), ): - return False, 0 - size = state[1][1] + return False, (0, 0) + size, future = state[1][1] if size > max_size: - return False, 0 - if not isinstance(derivation.type, Arrow): - return size + 1 <= max_size, size + 1 - return size + 1 + len(derivation.type.arguments()) <= max_size, size + 1 + return False, (0, 0) + if not derivation.type.is_instance(Arrow): + if future > 0: + return size + future <= max_size, (size + 1, future - 1) + return size + 1 + future <= max_size, (size + 1, future) + nargs = len(derivation.type.arguments()) + if future > 0: + return size + nargs + future <= max_size, (size + 1, future + nargs - 1) + return size + nargs + 1 + future <= max_size, (size + 1, future + nargs) return __saturation_build__( dsl, type_request, - (NGram(n_gram), 0), + (NGram(n_gram), (0, 0)), __transition__, lambda ctx, P, i, __: ctx[1][0].successor((P, i)), ) @@ -282,7 +423,6 @@ def at_most_k( """ Constructs a n-gram TT CFG from a DSL imposing at most k occurences of a certain primitive. """ - dsl.instantiate_forbidden() forbidden_sets = dsl.forbidden_patterns def __transition__( @@ -331,12 +471,8 @@ def __saturation_build__( Dict[DerivableProgram, Tuple[List[Tuple[Type, S]], T]], ] = {} - if isinstance(type_request, Arrow): - return_type = type_request.returns() - args = type_request.arguments() - else: - return_type = type_request - args = [] + return_type = type_request.returns() + args = type_request.arguments() list_to_be_treated: Deque[Tuple[Tuple[Type, S], T, List[Tuple[Type, S]]]] = deque() list_to_be_treated.append(((return_type, init[0]), init[1], [])) @@ -374,4 +510,4 @@ def __saturation_build__( if tmp_stack: list_to_be_treated.append((tmp_stack[0], new_el, tmp_stack[1:])) - return TTCFG((return_type, init), rules, clean=False) + return TTCFG((return_type, init), rules) diff --git a/synth/syntax/grammars/u_cfg.py b/synth/syntax/grammars/u_cfg.py new file mode 100644 index 00000000..732aed32 --- /dev/null +++ b/synth/syntax/grammars/u_cfg.py @@ -0,0 +1,365 @@ +from collections import defaultdict +from functools import lru_cache +from typing import ( + Any, + Dict, + List, + Optional, + Set, + Tuple, + TypeVar, + Generic, + Union, + overload, +) + +from synth.syntax.automata.tree_automaton import DFTA +from synth.syntax.dsl import DSL +from synth.syntax.grammars.cfg import CFG, CFGState +from synth.syntax.grammars.grammar import DerivableProgram, NGram +from synth.syntax.grammars.u_grammar import UGrammar +from synth.syntax.program import Constant +from synth.syntax.type_system import Type, UnknownType + +U = TypeVar("U") +V = TypeVar("V") +W = TypeVar("W") +T = TypeVar("T") + + +def __extract__(t: Tuple) -> Tuple: + while len(t) == 1 and isinstance(t, tuple): + t = t[0] + return t + + +def __d2state__(t: Union[Tuple[Type, U], Tuple[Tuple[Type, U], ...]]) -> Tuple[Type, U]: + t = __extract__(t) + if isinstance(t[0], tuple): + # Get the type + our_type = t + while isinstance(our_type, tuple): + our_type = our_type[0] # type: ignore + # Compute the rest + rest = [] + for tt in t: + rest.append(__extract__(tt)[1]) + return (our_type, tuple(rest)) + return t # type: ignore + + +class UCFG(UGrammar[U, List[Tuple[Type, U]], List[Tuple[Type, U]]], Generic[U]): + """ + Represents an unambigous context-free grammar. + + """ + + def __init__( + self, + starts: Set[Tuple[Type, U]], + rules: Dict[Tuple[Type, U], Dict[DerivableProgram, List[List[Tuple[Type, U]]]]], + clean: bool = True, + ): + self._some_start = list(starts)[0] + super().__init__(starts, rules, clean) + + def name(self) -> str: + return "UCFG" + + def __hash__(self) -> int: + return super().__hash__() + + def __rule_to_str__(self, P: DerivableProgram, out: List[Tuple[Type, U]]) -> str: + return "{}: {}".format(P, out) + + def clean(self) -> None: + """ + Clean this unambiguous grammar by removing non reachable, non productive rules. + """ + done: Set[Tuple[Tuple[Tuple[Type, U], ...], Tuple[Type, U]]] = set() + reached = {x for x in self.starts} + for x in self.starts: + done.add((tuple(self.start_information()), x)) + to_test = [(x, self.start_information()) for x in self.starts] + + while to_test: + S, info = to_test.pop() + for P in self.rules[S]: + for a in self.derive(info, S, P): + new_info, next_S, _ = a + i = len(done) + done.add((tuple(new_info), next_S)) + if len(done) == i: + continue + reached.add(next_S) + if isinstance(next_S[0], UnknownType): + continue + to_test.append((next_S, new_info)) + + self.rules = { + S: {P: possibles for P, possibles in dicP.items()} + for S, dicP in self.rules.items() + if S in reached + } + + # Clean starts + next_starts = set() + for S in self.starts: + has_one = False + for P in self.rules[S]: + for a in self.derive(self.start_information(), S, P): + new_info, next_S, _ = a + if (tuple(new_info), next_S) in done or isinstance( + next_S[0], UnknownType + ): + has_one = True + break + if has_one: + break + if has_one: + next_starts.add(S) + + self.starts = next_starts + + def arguments_length_for(self, S: Tuple[Type, U], P: DerivableProgram) -> int: + possibles = self.rules[S][P] + return len(possibles[0]) + + def start_information(self) -> List[Tuple[Type, U]]: + return [] + + def derive( + self, information: List[Tuple[Type, U]], S: Tuple[Type, U], P: DerivableProgram + ) -> List[Tuple[List[Tuple[Type, U]], Tuple[Type, U], List[Tuple[Type, U]]]]: + """ + Given the current information and the derivation S -> P, produces the list of possibles new information state and the next S after this derivation. + """ + if S not in self.rules or P not in self.rules[S]: + # This is important since we can fail to derive + return [] + candidates = self.rules[S][P] + out = [] + for args in candidates: + if args: + out.append((args[1:] + information, args[0], args)) + elif information: + out.append((information[1:], information[0], [])) + else: + # This indicates the end of a derivation + out.append(([], (UnknownType(), self._some_start[1]), [])) + return out + + def derive_specific( + self, + information: List[Tuple[Type, U]], + S: Tuple[Type, U], + P: DerivableProgram, + v: List[Tuple[Type, U]], + ) -> Optional[Tuple[List[Tuple[Type, U]], Tuple[Type, U]]]: + """ + Given the current information and the derivation S -> P, produces the new information state and the next S after this derivation. + """ + if len(v) == 0: + if information: + return (information[1:], information[0]) + else: + # This indicates the end of a derivation + return ([], (UnknownType(), self._some_start[1])) + for args in self.rules[S][P]: + if args == v: + return (args[1:] + information, args[0]) + return None + + @lru_cache() + def programs(self) -> int: + """ + Return the total number of programs contained in this grammar. + """ + _counts: Dict[Tuple[Type, U], int] = {} + + def __compute__(state: Tuple[Type, U]) -> int: + if state in _counts: + return _counts[state] + if state not in self.rules: + return 1 + total = 0 + for P in self.rules[state]: + possibles = self.derive(self.start_information(), state, P) + for _, _, args in possibles: + local = 1 + for arg in args: + local *= __compute__(arg) + total += local + _counts[state] = total + return total + + return sum(__compute__(start) for start in self.starts) + + def instantiate_constants(self, constants: Dict[Type, List[Any]]) -> "UCFG[U]": + rules: dict[ + Tuple[Type, U], dict[DerivableProgram, List[List[Tuple[Type, U]]]] + ] = {} + for NT in self.rules: + rules[NT] = {} + for P in self.rules[NT]: + if isinstance(P, Constant) and P.type in constants: + for val in constants[P.type]: + rules[NT][Constant(P.type, val, True)] = self.rules[NT][P] + else: + rules[NT][P] = self.rules[NT][P] + # Cleaning produces infinite loop + return self.__class__(self.starts, rules, clean=False) + + @classmethod + def depth_constraint( + cls, + dsl: DSL, + type_request: Type, + max_depth: int, + min_variable_depth: int = 1, + n_gram: int = 2, + recursive: bool = False, + constant_types: Set[Type] = set(), + ) -> "UCFG[CFGState]": + """ + Constructs a UCFG from a DSL imposing bounds on size of the types + and on the maximum program depth. + + Parameters: + ----------- + - max_depth: the maximum depth of programs allowed + - min_variable_depth: min depth at which variables and constants are allowed + - n_gram: the context, a bigram depends only in the parent node + - recursive: enables the generated programs to call themselves + - constant_types: the set of of types allowed for constant objects + """ + cfg = CFG.depth_constraint( + dsl, + type_request, + max_depth, + min_variable_depth, + n_gram, + recursive, + constant_types, + ) + return UCFG.from_CFG(cfg, True) + + @classmethod + def from_CFG(cls, cfg: CFG, clean: bool = False) -> "UCFG[CFGState]": + """ + Constructs a UCFG from the specified CFG + """ + rules: Dict[ + Tuple[Type, CFGState], + Dict[DerivableProgram, List[List[Tuple[Type, CFGState]]]], + ] = {} + for S in cfg.rules: + nS = (S[0], S[1][0]) + rules[nS] = {} + for P in cfg.rules[S]: + rules[nS][P] = [[SS for SS in cfg.rules[S][P][0]]] + return UCFG({(cfg.start[0], cfg.start[1][0])}, rules, clean) + + @overload + @classmethod + def from_DFTA( + cls, dfta: DFTA[Tuple[Type, U], DerivableProgram], clean: bool = True + ) -> "UCFG[U]": + pass + + @overload + @classmethod + def from_DFTA( + cls, + dfta: DFTA[Tuple[Tuple[Type, U], ...], DerivableProgram], + clean: bool = True, + ) -> "UCFG[Tuple[U, ...]]": + pass + + @classmethod + def from_DFTA( + cls, + dfta: Union[ + DFTA[Tuple[Tuple[Type, U], ...], DerivableProgram], + DFTA[Tuple[Type, U], DerivableProgram], + ], + clean: bool = True, + ) -> "Union[UCFG[U], UCFG[Tuple[U, ...]]]": + """ + Convert a DFTA into a UCFG representing the same language. + """ + + starts = {__d2state__(q) for q in dfta.finals} + + new_rules: Dict[ + Tuple[Type, U], + Dict[DerivableProgram, List[List[Tuple[Type, U]]]], + ] = {} + + stack: List[Tuple[Type, U]] = [el for el in starts] + while stack: + tgt = stack.pop() + if tgt in new_rules: + continue + new_rules[tgt] = defaultdict(list) + for (P, args), dst in dfta.rules.items(): + if __d2state__(dst) != tgt: + continue + new_rules[tgt][P].append([__d2state__(arg) for arg in args]) + for new_state in args: + if __d2state__(new_state) not in new_rules: + stack.append(__d2state__(new_state)) + + return UCFG(starts, new_rules, clean) + + @classmethod + def from_DFTA_with_ngrams( + cls, + dfta: Union[ + DFTA[Tuple[Tuple[Type, U], ...], DerivableProgram], + DFTA[Tuple[Type, U], DerivableProgram], + ], + ngram: int, + clean: bool = False, + ) -> "Union[UCFG[Tuple[NGram, U]], UCFG[Tuple[NGram, Tuple[U, ...]]]]": + """ + Convert a DFTA into a UCFG representing the same language and adds contextual information with ngrams. + If the DFTA is reduced then the UCFG is, therefore in that case clean should be set to False since cleaning can be expensive. + """ + + def local_d2state( + t: Union[Tuple[Type, U], Tuple[Tuple[Type, U], ...]], v: Optional[NGram] + ) -> Tuple[Type, Tuple[NGram, U]]: + a, b = __d2state__(t) + dst_v = v or NGram(ngram) + return (a, (dst_v, b)) + + match = lambda a, b: a[0] == b[0] and a[1][1] == b[1][1] + + starts = {local_d2state(q, None) for q in dfta.finals} + + new_rules: Dict[ + Tuple[Type, Tuple[NGram, U]], + Dict[DerivableProgram, List[List[Tuple[Type, Tuple[NGram, U]]]]], + ] = {} + + stack = [el for el in starts] + while stack: + tgt = stack.pop() + if tgt in new_rules: + continue + new_rules[tgt] = defaultdict(list) + last: NGram = tgt[1][0] + for (P, args), dst in dfta.rules.items(): + if not match(local_d2state(dst, None), tgt): + continue + new_args = [ + local_d2state(arg, last.successor((P, i))) # type: ignore + for i, arg in enumerate(args) + ] + new_rules[tgt][P].append(new_args) + for new_state in new_args: + if new_state not in new_rules: + stack.append(new_state) + + return UCFG(starts, new_rules, clean) diff --git a/synth/syntax/grammars/u_grammar.py b/synth/syntax/grammars/u_grammar.py new file mode 100644 index 00000000..c7839455 --- /dev/null +++ b/synth/syntax/grammars/u_grammar.py @@ -0,0 +1,327 @@ +from abc import ABC, abstractmethod +from typing import ( + Any, + Callable, + Dict, + List, + Optional, + Set, + Tuple, + TypeVar, + Generic, +) +from functools import lru_cache + +from synth.syntax.grammars.grammar import DerivableProgram, Grammar +from synth.syntax.program import Constant, Function, Primitive, Program, Variable +from synth.syntax.type_system import Arrow, Type + +U = TypeVar("U") +V = TypeVar("V") +W = TypeVar("W") +T = TypeVar("T") + + +class UGrammar(Grammar, ABC, Generic[U, V, W]): + """ + Represents an unambiguous grammar. + + (S) Non-terminals are Tuple[Type, U]. + (f) are Derivable programs + derivations are: + S -> f | S1 ... Sk + | S'1 ... S'k + where both cases are valid derivations. + S1 ... Sk, S'1 ... S'k are of type V. + + When deriving an information of type W is maintained. + + Parameters: + ----------- + - starts: the starts non terminal of the grammar + - rules: the derivation rules + + """ + + def __init__( + self, + starts: Set[Tuple[Type, U]], + rules: Dict[Tuple[Type, U], Dict[DerivableProgram, List[V]]], + clean: bool = True, + ): + self.starts = starts + self.rules = rules + self.type_request = self._guess_type_request_() + if clean: + self.clean() + + @lru_cache() + def primitives_used(self) -> Set[Primitive]: + """ + Returns the set of primitives used by this grammar. + """ + out: Set[Primitive] = set() + for S in self.rules: + for P in self.rules[S]: + if isinstance(P, Primitive): + out.add(P) + return out + + def __hash__(self) -> int: + return hash((tuple(self.starts), str(self.rules))) + + def __rule_to_str__(self, P: DerivableProgram, out: V) -> str: + return "{}: {}".format(P, out) + + def __str__(self) -> str: + s = f"Print a {self.name()}\n" + s += "starts: {}\n".format(self.starts) + for S in reversed(self.rules): + add = " [START]" if S in self.starts else "" + s += "#\n {}{}\n".format(S, add) + for P in self.rules[S]: + out = self.rules[S][P] + for possible in out: + s += " {}\n".format(self.__rule_to_str__(P, possible)) + return s + + def __repr__(self) -> str: + return self.__str__() + + def _guess_type_request_(self) -> Type: + """ + Guess the type request of this grammar. + """ + # Compute the type request + type_req = list(self.starts)[0][0] + variables: List[Variable] = [] + for S in self.rules: + for P in self.rules[S]: + if isinstance(P, Variable): + if P not in variables: + variables.append(P) + n = len(variables) + for i in range(n): + j = n - i - 1 + for v in variables: + if v.variable == j: + type_req = Arrow(v.type, type_req) + return type_req + + def __contains__(self, program: Program) -> bool: + return any( + self.__contains_rec__(program, start, self.start_information())[0] + for start in self.starts + ) + + def __contains_rec__( + self, program: Program, start: Tuple[Type, U], information: W + ) -> Tuple[bool, List[Tuple[W, Tuple[Type, U]]]]: + if start not in self.rules: + return False, [(information, start)] + if isinstance(program, Function): + function = program.function + args_P = program.arguments + if function not in self.rules[start]: + return False, [(information, start)] + possibles = [ + (a, b) + for a, b, _ in self.derive(information, start, function) # type: ignore + ] + for arg in args_P: + next_possibles = [] + for possible in possibles: + information, next = possible + contained, new_possibles = self.__contains_rec__( + arg, start=next, information=information + ) + if contained: + next_possibles += new_possibles + if len(next_possibles) == 0: + return False, [(information, next)] + possibles = next_possibles + return True, possibles + elif isinstance(program, (Primitive, Variable, Constant)): + if program not in self.rules[start]: + return False, [(information, start)] + possibles = [(a, b) for a, b, _ in self.derive(information, start, program)] + return True, possibles + return False, [(information, start)] + + def clean(self) -> None: + """ + Clean this deterministic grammar by removing non reachable, non productive rules. + """ + pass + + @abstractmethod + def derive( + self, information: W, S: Tuple[Type, U], P: DerivableProgram + ) -> List[Tuple[W, Tuple[Type, U], V]]: + """ + Given the current information and the derivation S -> P, produces the new information state and the next S after this derivation. + """ + pass + + def derive_specific( + self, information: W, S: Tuple[Type, U], P: DerivableProgram, v: V + ) -> Optional[Tuple[W, Tuple[Type, U]]]: + """ + Given the current information and the derivation S -> P, produces the new information state and the next S after this derivation. + """ + for a, b, c in self.derive(information, S, P): + if c == v: + return a, b + return None + + def derive_all( + self, + information: W, + S: Tuple[Type, U], + P: Program, + current: Optional[List[Tuple[Tuple[Type, U], V]]] = None, + hints: Optional[Dict[Tuple[Type, U], Dict[Program, V]]] = None, + ) -> List[Tuple[W, List[Tuple[Tuple[Type, U], V]]]]: + """ + Given current information and context S, produces the new information and all the contexts the grammar went through to derive program P. + """ + current = current or [] + if isinstance(P, (Primitive, Variable, Constant)): + if hints: + out_der = self.derive_specific(information, S, P, hints[S][P]) + assert out_der is not None + return [(out_der[0], current + [(out_der[1], hints[S][P])])] + else: + cur_possibles = self.derive(information, S, P) + out = [(info, current + [(ctx, pp)]) for info, ctx, pp in cur_possibles] + return out + + elif isinstance(P, Function): + F = P.function + # current.append(S) + # information, _ = self.derive_all(information, S, F, current) + if hints: + out_der = self.derive_specific(information, S, F, hints[S][P]) # type: ignore + assert out_der is not None + possibles = [(out_der[0], current + [(out_der[1], hints[S][P])])] + for arg in P.arguments: + next_possibles = [] + for possible in possibles: + information, crt = possible + S = crt[-1][0] + possibles_arg = self.derive_all( + information, S, arg, current, hints + ) + for information, next_crt in possibles_arg: + next_possibles.append((information, next_crt)) + possibles = next_possibles + return possibles + else: + possibles = self.derive_all(information, S, F, current) + for arg in P.arguments: + next_possibles = [] + for possible in possibles: + information, crt = possible + S = crt[-1][0] + possibles_arg = self.derive_all(information, S, arg, current) + for information, next_crt in possibles_arg: + next_possibles.append((information, next_crt)) + possibles = next_possibles + # information, _ = self.derive_all(information, S, arg, current) + # S = current[-1] + return possibles + assert False + + @abstractmethod + def arguments_length_for(self, S: Tuple[Type, U], P: DerivableProgram) -> int: + """ + Returns the number of arguments when deriving P from S. + """ + pass + + @abstractmethod + def instantiate_constants( + self, constants: Dict[Type, List[Any]] + ) -> "UGrammar[U, V, W]": + pass + + @abstractmethod + def start_information(self) -> W: + """ + The starting information when deriving from a starting non-terminal. + """ + pass + + def reduce_derivations( + self, + reduce: Callable[[T, Tuple[Type, U], DerivableProgram, V], T], + init: T, + program: Program, + start: Optional[Tuple[Type, U]] = None, + ) -> List[T]: + """ + Reduce the given program using the given reduce operator. + + reduce: 'a, S, P, V -> 'a + + reduce is called after derivation. + """ + + outputs = [] + if start is None: + alternatives: List[ + List[Tuple[Tuple[Type, U], Tuple[Type, U], DerivableProgram, V, W]] + ] = [] + for start in self.starts: + alternatives += self.__reduce_derivations_rec__( + reduce, program, start, self.start_information() + ) + else: + alternatives = self.__reduce_derivations_rec__( + reduce, program, start, self.start_information() + ) + for possibles in alternatives: + value = init + for __, S, P, v, _ in possibles: + value = reduce(value, S, P, v) + outputs.append(value) + return outputs + + def __reduce_derivations_rec__( + self, + reduce: Callable[[T, Tuple[Type, U], DerivableProgram, V], T], + program: Program, + start: Tuple[Type, U], + information: W, + ) -> List[List[Tuple[Tuple[Type, U], Tuple[Type, U], DerivableProgram, V, W]]]: + if isinstance(program, Function): + function = program.function + args_P = program.arguments + possibles: List[ + List[Tuple[Tuple[Type, U], Tuple[Type, U], DerivableProgram, V, W]] + ] = [ + [(b, start, function, c, a)] # type: ignore + for a, b, c in self.derive(information, start, function) # type: ignore + ] + for arg in args_P: + next_possibles = [] + for possible in possibles: + next, _, __, v, information = possible[-1] + alternatives = self.__reduce_derivations_rec__( + reduce, arg, start=next, information=information + ) + for alternative in alternatives: + next_possibles.append(possible + alternative) + possibles = next_possibles + # value, information, next = self.__reduce_derivations_rec__( + # reduce, value, arg, start=next, information=information + # ) + return next_possibles + elif isinstance(program, (Primitive, Variable, Constant)): + new_possibles = self.derive(information, start, program) + alternatives = [] + for new_possible in new_possibles: + information, next, v = new_possible + alternatives.append([(next, start, program, v, information)]) + return alternatives + return [] diff --git a/synth/syntax/program.py b/synth/syntax/program.py index 78f8a444..af64b2e4 100644 --- a/synth/syntax/program.py +++ b/synth/syntax/program.py @@ -1,13 +1,13 @@ -from abc import ABC, abstractstaticmethod -from typing import Generator, List as TList, Any, Optional, Set, Tuple +from abc import ABC, abstractmethod +from typing import Dict, Generator, List as TList, Any, Optional, Set, Tuple +import itertools from synth.syntax.type_system import ( - Arrow, - FunctionType, PrimitiveType, Type, UnknownType, ) +from synth.syntax.type_helper import FunctionType class Program(ABC): @@ -25,7 +25,16 @@ def __hash__(self) -> int: def __repr__(self) -> str: return self.__str__() + def uses_variables(self) -> bool: + """ + Returns true if a variable is used. + """ + return False + def used_variables(self) -> Set[int]: + """ + Returns the set of used variables numbers in this program. + """ s: Set[int] = set() self.__add_used_variables__(s) return s @@ -34,39 +43,117 @@ def __add_used_variables__(self, vars: Set[int]) -> None: pass def is_constant(self) -> bool: + """ + Returns true if this program is an instance of a Constant. + """ return False def is_invariant(self, constant_types: Set[PrimitiveType]) -> bool: + """SHOULD BE DELETED""" return True def count_constants(self) -> int: + """ + Returns the number of constants that are present in this program. + """ return int(self.is_constant()) - def length(self) -> int: + def constants(self) -> Generator[Optional["Constant"], None, None]: + """ + Iterates over all constants of this program, yields a None only if the program does NOT contain any constant. + """ + yield None + + @abstractmethod + def clone(self) -> "Program": + """ + Produces an exact clone (deep copy) of this program. + """ + pass + + def all_constants_instantiation( + self, constants: Dict[Type, TList[Any]] + ) -> Generator["Program", None, None]: + yield self + + def size(self) -> int: + """ + Returns the program's size. + """ return 1 def depth(self) -> int: + """ + Returns the program's depth seen as a tree. + """ return 1 def depth_first_iter(self) -> Generator["Program", None, None]: + """ + Depth first iteration over all objects that this program is built on. + + ``Function(P1, [P2, Function(P3, [P4])]).depth_first_iter()`` will yield + P1, P2, P3, P4, Function(P3, [P4]), Function(P1, [P2, Function(P3, [P4])]) + """ yield self - @abstractstaticmethod + def pretty_print(self) -> TList[str]: + """ + Represents this program as a list of operations. + """ + defined: Dict["Program", Tuple[int, str, str]] = {} + self.__pretty_print__(defined, 0) + data = sorted(defined.values()) + order = [x[2] for x in data if len(x[2]) > 0] + return order + + def __pretty_print__( + self, + defined: Dict["Program", Tuple[int, str, str]], + last: int, + ) -> int: + if self not in defined: + defined[self] = (0, str(self), "") + return last + + def __contains__(self, other: "Program") -> bool: + return self == other + + @staticmethod + @abstractmethod def __pickle__(o: "Program") -> Tuple: pass + def type_checks(self) -> bool: + return True + class Variable(Program): - __hash__ = Program.__hash__ + """ + Represents a variable (argument) in a program. - def is_invariant(self, constant_types: Set[PrimitiveType]) -> bool: - return False + Parameters: + ----------- + - variable: the argument index + - type: the variable's type + """ + + __hash__ = Program.__hash__ def __init__(self, variable: int, type: Type = UnknownType()): super().__init__(type) self.variable: int = variable self.hash = hash((self.variable, self.type)) + def is_invariant(self, constant_types: Set[PrimitiveType]) -> bool: + return False + + def uses_variables(self) -> bool: + return True + + def clone(self) -> "Program": + return Variable(self.variable) + def __add_used_variables__(self, vars: Set[int]) -> None: vars.add(self.variable) @@ -81,6 +168,16 @@ def __pickle__(o: Program) -> Tuple: # type: ignore[override] class Constant(Program): + """ + Represents a constant that may or may be assigned a value. + + Parameters: + ----------- + - type: the constant's type + - value: the constant's value + - has_value: explicitly indicate that this constant has been assigned a value + """ + __hash__ = Program.__hash__ def __init__(self, type: Type, value: Any = None, has_value: Optional[bool] = None): @@ -90,8 +187,23 @@ def __init__(self, type: Type, value: Any = None, has_value: Optional[bool] = No self.hash = hash((str(self.value), self._has_value, self.type)) def has_value(self) -> bool: + """ + Returns true if and only if this constant has been assigned a value. + """ return self._has_value + def constants(self) -> Generator[Optional["Constant"], None, None]: + yield self + + def clone(self) -> "Program": + return Constant(self.type, self.value, self._has_value) + + def all_constants_instantiation( + self, constants: Dict[Type, TList[Any]] + ) -> Generator["Program", None, None]: + for val in constants[self.type]: + yield Constant(self.type, val) + def __str__(self) -> str: if self.has_value(): return format(self.value) @@ -101,12 +213,20 @@ def is_constant(self) -> bool: return True def assign(self, value: Any) -> None: + """ + Assign a value to this constant. + """ self._has_value = True self.value = value + self.hash = hash((str(self.value), self._has_value, self.type)) def reset(self) -> None: + """ + Reset this constant as if no value was assigned to it. + """ self._has_value = False self.value = None + self.hash = hash((str(self.value), self._has_value, self.type)) def __eq__(self, other: Any) -> bool: return ( @@ -120,19 +240,27 @@ def __pickle__(o: Program) -> Tuple: # type: ignore[override] class Function(Program): + """ + Represents a function call, it supports partial application and the type is guessed automatically. + + Parameters: + ----------- + - function: the called function + - arguments: the arguments to the function + """ + __hash__ = Program.__hash__ def __init__(self, function: Program, arguments: TList[Program]): # Build automatically the type of the function type = function.type - assert isinstance(type, Arrow) args = type.arguments()[len(arguments) :] my_type = FunctionType(*args, type.returns()) super().__init__(my_type) self.function = function self.arguments = arguments - self.hash = hash(tuple([arg for arg in self.arguments] + [self.function])) + self.hash = hash((self.function, *self.arguments)) def __pickle__(o: Program) -> Tuple: # type: ignore[override] return Function, (o.function, o.arguments) # type: ignore @@ -146,6 +274,34 @@ def __str__(self) -> str: s += " " + format(arg) return s + ")" + def type_checks(self) -> bool: + return all( + arg.type.is_instance(f_arg_t) + for arg, f_arg_t in zip(self.arguments, self.function.type.arguments()) + ) + + def __pretty_print__( + self, + defined: Dict["Program", Tuple[int, str, str]], + last: int, + ) -> int: + if self in defined: + return last + out = [] + for arg in self.arguments: + last = arg.__pretty_print__(defined, last) + out.append(defined[arg][1]) + var_name = f"x{last}" + defined[self] = ( + last, + var_name, + f"{var_name}: {self.type} = {self.function}({', '.join(out)})", + ) + return last + 1 + + def clone(self) -> "Program": + return Function(self.function.clone(), [x.clone() for x in self.arguments]) + def __eq__(self, other: object) -> bool: return ( isinstance(other, Function) @@ -159,6 +315,30 @@ def is_constant(self) -> bool: arg.is_constant() for arg in self.arguments ) + def uses_variables(self) -> bool: + return self.function.uses_variables() or any( + arg.uses_variables() for arg in self.arguments + ) + + def constants(self) -> Generator[Optional["Constant"], None, None]: + g = [self.function.constants()] + [arg.constants() for arg in self.arguments] + for gen in g: + c = next(gen, None) + while c is not None: + yield c + c = next(gen, None) + + def all_constants_instantiation( + self, constants: Dict[Type, TList[Any]] + ) -> Generator["Program", None, None]: + for f in self.function.all_constants_instantiation(constants): + possibles = [ + list(arg.all_constants_instantiation(constants)) + for arg in self.arguments + ] + for args in itertools.product(*possibles): + yield Function(f, list(args)) + def is_invariant(self, constant_types: Set[PrimitiveType]) -> bool: return self.function.is_invariant(constant_types) and all( arg.is_invariant(constant_types) for arg in self.arguments @@ -169,8 +349,8 @@ def count_constants(self) -> int: [arg.count_constants() for arg in self.arguments] ) - def length(self) -> int: - return self.function.length() + sum([arg.length() for arg in self.arguments]) + def size(self) -> int: + return self.function.size() + sum([arg.size() for arg in self.arguments]) def depth(self) -> int: return 1 + max( @@ -189,6 +369,11 @@ def depth_first_iter(self) -> Generator["Program", None, None]: yield sub yield self + def __contains__(self, other: "Program") -> bool: + return self == other or any( + other in x for x in [self.function] + self.arguments + ) + class Lambda(Program): __hash__ = Program.__hash__ @@ -198,15 +383,31 @@ def __init__(self, body: Program, type: Type = UnknownType()): self.body = body self.hash = hash(94135 + hash(self.body)) + def clone(self) -> "Program": + return Lambda(self.body.clone()) + def __pickle__(o: Program) -> Tuple: # type: ignore[override] return Lambda, (o.body, o.type) # type: ignore def __str__(self) -> str: return "(lambda " + format(self.body) + ")" + def constants(self) -> Generator[Optional["Constant"], None, None]: + for x in self.body.constants(): + yield x + + def all_constants_instantiation( + self, constants: Dict[Type, TList[Any]] + ) -> Generator["Program", None, None]: + for val in self.body.all_constants_instantiation(constants): + yield Lambda(val) + def __eq__(self, other: Any) -> bool: return isinstance(other, Lambda) and self.body == other.body + def uses_variables(self) -> bool: + return self.body.uses_variables() + def __add_used_variables__(self, vars: Set[int]) -> None: return self.body.__add_used_variables__(vars) @@ -218,8 +419,20 @@ def depth_first_iter(self) -> Generator["Program", None, None]: yield sub yield self + def __contains__(self, other: "Program") -> bool: + return self == other or other in self.body + class Primitive(Program): + """ + Represents a DSL primitive. + + Parameters: + ----------- + - primitive: the name of the primitive + - type: the type of the primitive + """ + __hash__ = Program.__hash__ def __init__(self, primitive: str, type: Type = UnknownType()): @@ -227,6 +440,9 @@ def __init__(self, primitive: str, type: Type = UnknownType()): self.primitive = primitive self.hash = hash((self.primitive, self.type)) + def clone(self) -> "Program": + return Primitive(self.primitive, self.type) + def __pickle__(o: Program) -> Tuple: # type: ignore[override] return Primitive, (o.primitive, o.type) # type: ignore @@ -240,7 +456,11 @@ def __str__(self) -> str: return format(self.primitive) def __eq__(self, other: Any) -> bool: - return isinstance(other, Primitive) and self.primitive == other.primitive + return ( + isinstance(other, Primitive) + and self.primitive == other.primitive + and self.type == other.type + ) import copyreg diff --git a/synth/syntax/type_helper.py b/synth/syntax/type_helper.py new file mode 100644 index 00000000..6db25028 --- /dev/null +++ b/synth/syntax/type_helper.py @@ -0,0 +1,206 @@ +""" +An helper file that contains useful methods to make type creation and manipulation very easy. + +""" + +from synth.syntax.type_system import FixedPolymorphicType, Generic, PrimitiveType, Sum +from synth.syntax.type_system import ( + Type, + UnknownType, + Arrow, + EmptyList, + INT, + BOOL, + STRING, + UNIT, + List, + PolymorphicType, +) + +from typing import Any, Dict, Tuple, Union, overload, List as TList + + +def Optional(t: Type) -> Sum: + """ + Short-hand to create optional types. + """ + return Sum(UNIT, t) + + +def FunctionType(*args: Type) -> Type: + """ + Short-hand to create n-ary functions. + """ + n = len(args) - 1 + base = args[-1] + for i in range(n - 1, -1, -1): + base = Arrow(args[i], base) + return base + + +def guess_type(element: Any) -> Type: + """ + Guess the type of the given element. + Does not work for Arrow and Polymorphic Types. + """ + if isinstance(element, (TList, Tuple)): # type: ignore + if len(element) == 0: + return EmptyList + current: Type = UnknownType() + i = 0 + while i < len(element) and isinstance(current, UnknownType): + current = guess_type(element[i]) + i += 1 + return List(current) + if isinstance(element, bool): + return BOOL + elif isinstance(element, int): + return INT + elif isinstance(element, str): + return STRING + elif element is None: + return UNIT + return UnknownType() + + +_TOK_NONE = -1 +_TOK_PARENTHESIS = 0 +_TOK_BRACKETS = 1 +_TOK_INFIX = 2 +_TOK_POLYMORPHIC = 3 +_TOK_OR = 4 + + +def __matching__(text: str) -> int: + level = 0 + start = text[0] + for j, l in enumerate(text): + if l == start: + level += 1 + elif start == "(" and l == ")": + level -= 1 + elif start == "[" and l == "]": + level -= 1 + if level == 0: + return j + return -1 + + +__SPECIAL_TOKENS = " ()" + + +def __next_token__(text: str) -> Tuple[str, int, int]: + if text.startswith("(") or text.startswith("["): + i = __matching__(text) + return text[1:i], _TOK_BRACKETS if text[0] == "[" else _TOK_PARENTHESIS, i + 1 + elif text.startswith("|"): + return "", _TOK_OR, 1 + elif not text[0].isalpha() and not text[0] == "'": + i = 1 + while i < len(text) and not (text[i].isalpha() or text[i] in __SPECIAL_TOKENS): + i += 1 + return text[:i], _TOK_INFIX, i + i = 1 + while i < len(text) and (text[i].isalpha() or text[i].isdigit() or text[i] == "_"): + i += 1 + is_poly = len(text) > 0 and text[0] == "'" + return ( + text[is_poly:i], + _TOK_POLYMORPHIC if is_poly else _TOK_NONE, + i, + ) + + +@overload +def auto_type(el: str) -> Type: + """ + Automatically build the type from its string representation. + + Parameters: + ----------- + - el: the type's string representation + """ + pass + + +@overload +def auto_type(el: Dict[str, str]) -> Dict[str, Type]: + """ + Automatically build the type from its string representation for all values of the dictionnary. + + Parameters: + ----------- + - el: a dictionnary containing as values types string representations + """ + pass + + +def auto_type(el: Union[Dict[str, str], str]) -> Union[Dict[str, Type], Type]: + # Dictionnary part + if isinstance(el, dict): + return {k: auto_type(v) for k, v in el.items()} + # String part + stack = [] + text = el + last_infix = 0 + infix_stack = [] + or_flag = -1 + index = 1 + while len(text) > 0: + w, token, index = __next_token__(text) + # index also represents the number of chars consumed + if token == _TOK_PARENTHESIS: + stack.append(auto_type(w)) + elif token == _TOK_BRACKETS: + assert len(stack) > 0 + last = stack.pop() + assert isinstance( + last, PolymorphicType + ), f"Cannot restrain a non polymorphic type:{last}" + r = auto_type(w) + stack.append(FixedPolymorphicType(last.name, r)) + elif token == _TOK_POLYMORPHIC: + stack.append(PolymorphicType(w)) + elif token == _TOK_NONE: + if len(w) > 0: + if last_infix < len(stack) and or_flag < 0: + if w == "optional": + stack.append(Optional(stack.pop())) + else: + stack.append(Generic(w, stack.pop())) + else: + stack.append(PrimitiveType(w)) + elif token == _TOK_INFIX: + # old comment about arrows but the same for infix operators + # Okay a bit complicated since arrows should be built from right to left + # Notice than in no other case there might be a malformed arrow + # therefore it happens at the top level + # thus we can do it at the end (with the guarantee that the stack size will not be any lower than its current value from now on) + # even more interesting part: + # if the expression is well-formed then + # we just need to put arrows between all elements of the stacks + last_infix += 1 + infix_stack.append(w) + elif token == _TOK_OR: + or_flag = 0 + + # Manage or flags which consume things as they come + if or_flag == 0: + or_flag = 1 + elif or_flag == 1: + or_flag = -1 + assert len(stack) >= 2 + last = stack.pop() + stack.append(stack.pop() | last) + + # update text + text = text[index:].strip() + assert len(stack) >= 1 + while len(stack) > 1: + last = stack.pop() + w = infix_stack.pop() + if w == "->": + stack.append(Arrow(stack.pop(), last)) + else: + stack.append(Generic(w, stack.pop(), last, infix=True)) + return stack.pop() diff --git a/synth/syntax/type_system.py b/synth/syntax/type_system.py index a4fdaf1c..b9d129a4 100644 --- a/synth/syntax/type_system.py +++ b/synth/syntax/type_system.py @@ -1,10 +1,21 @@ """ Objective: define a type system. -A type can be either PolymorphicType, PrimitiveType, Arrow, or List +A type can be either PolymorphicType, FixedPolymorphicType, PrimitiveType, Generic, Arrow, Sum or List """ -from typing import Any, Dict, List as TList, Optional, Set, Tuple -from abc import ABC, abstractmethod, abstractstaticmethod +from itertools import product +from typing import Dict, List as TList, Optional, Set, Tuple, Union +from abc import ABC, abstractmethod + + +class TypeFunctor(ABC): + """ + Represents a type functor. + """ + + @abstractmethod + def __is_arg_an_instance__(self, arg: "Type") -> bool: + pass class Type(ABC): @@ -23,17 +34,62 @@ def __repr__(self) -> str: return self.__str__() def returns(self) -> "Type": + """ + The type returned by this arrow if this is an arrow or self otherwise. + """ return self def arguments(self) -> TList["Type"]: + """ + The arguments consumed by this arrow if this is an arrow or an empty list otherwise. + """ return [] + def without_unit_arguments(self) -> "Type": + return self + + def is_under_specified(self) -> bool: + """ + Returns True iff this type contains UnknownType + """ + return False + + def is_instance(self, other: Union["Type", TypeFunctor, type]) -> bool: + """ + Returns true if and only if this type is an instance of other. + This should be used instead of the default ``isinstance``. + """ + if isinstance(other, Type): + return other.__arg_is_a__(self) + elif isinstance(other, type): + return isinstance(self, other) + else: + return other.__is_arg_an_instance__(self) + + def __arg_is_a__(self, other: "Type") -> bool: + return other == self + def __contains__(self, t: "Type") -> bool: return self == t + def __or__(self, other: "Type") -> "Sum": + if isinstance(other, Sum): + return Sum(self, *other.types) + return Sum(self, other) + def is_polymorphic(self) -> bool: + """ + Returns true if and only if this type is polymorphic. + """ return False + def all_versions(self) -> TList["Type"]: + """ + Return all versions of this type. + Versions are created for each sum type. + """ + return [self] + def decompose_type(self) -> Tuple[Set["PrimitiveType"], Set["PolymorphicType"]]: """ Finds the set of basic types and polymorphic types @@ -53,6 +109,8 @@ def __decompose_type_rec__( def unify(self, unifier: Dict[str, "Type"]) -> "Type": """ + Instantiate this type with the specified named polymorphic types given in the dictionnary instantiated as the type values associated to their keys. + pre: `self.is_polymorphic() and all(not t.is_polymorphic() for t in dictionnary.values())` post: `not out.is_polymorphic() and match(self, out)` @@ -60,9 +118,15 @@ def unify(self, unifier: Dict[str, "Type"]) -> "Type": return self def depth(self) -> int: + """ + Returns the type's depth seen as a tree. + """ return 1 def size(self) -> int: + """ + Returns the type's size seen as a tree. + """ return 1 def ends_with(self, other: "Type") -> Optional[TList["Type"]]: @@ -86,17 +150,23 @@ def ends_with_rec( ) -> Optional[TList["Type"]]: if self == other: return arguments_list - if isinstance(self, Arrow): + if self.is_instance(Arrow): + assert isinstance(self, Arrow) arguments_list.append(self.type_in) return self.type_out.ends_with_rec(other, arguments_list) return None - @abstractstaticmethod - def __pickle__(o: "Type") -> Tuple: - pass - class PolymorphicType(Type): + """ + Represents a polymorphic type, like python's ``TypeVar``. + It is uniquely identified by its name. + + Parameters: + ----------- + - name: the name of this type + """ + __hash__ = Type.__hash__ def __init__(self, name: str): @@ -104,7 +174,10 @@ def __init__(self, name: str): self.name = name self.hash = hash(self.name) - def __pickle__(o: Type) -> Tuple: # type: ignore[override] + def __arg_is_a__(self, other: "Type") -> bool: + return True + + def __pickle__(o: Type) -> Tuple: return PolymorphicType, (o.name,) # type: ignore def __str__(self) -> str: @@ -126,15 +199,71 @@ def __decompose_type_rec__( def unify(self, unifier: Dict[str, "Type"]) -> "Type": return unifier.get(self.name, self) + def can_be(self, other: "Type") -> bool: + """ + Returns if this polymorphic type can be instanciated as the specified type. + """ + return True + + +class FixedPolymorphicType(PolymorphicType): + """ + Represents a polymorphic type, like python's ``TypeVar``. + It is uniquely identified by its name. + Although it can only be instantiated as one of the given types. + + Parameters: + ----------- + - name: the name of this type + - *types: the types to which this type can be instantiated to + """ + + __hash__ = Type.__hash__ + + def __init__(self, name: str, *types: Type): + super().__init__(name) + self.types = types + + def is_under_specified(self) -> bool: + return any(t.is_under_specified() for t in self.types) + + def __arg_is_a__(self, other: "Type") -> bool: + if isinstance(other, (Sum, FixedPolymorphicType)): + return all(any(x.is_instance(t) for t in self.types) for x in other.types) + return any(other.is_instance(t) for t in self.types) + + def __pickle__(o: Type) -> Tuple: + return FixedPolymorphicType, (o.name, *o.types) # type: ignore + + def can_be(self, other: "Type") -> bool: + if isinstance(other, (Sum, FixedPolymorphicType)): + return all(any(x.is_instance(t) for t in self.types) for x in other.types) + return any(other.is_instance(x) for x in self.types) + + def __eq__(self, o: object) -> bool: + return ( + isinstance(o, FixedPolymorphicType) + and len(set(o.types).symmetric_difference(self.types)) == 0 + ) + class PrimitiveType(Type): + """ + Represents a ground type like ``int``. + It is uniquely identified by its name. + + Parameters: + ----------- + - name: the name of this type + """ + __hash__ = Type.__hash__ def __init__(self, type_name: str): self.type_name = type_name self.hash = hash(self.type_name) - def __pickle__(o: Type) -> Tuple: # type: ignore[override] + def __pickle__(o: Type) -> Tuple: return PrimitiveType, (o.type_name,) # type: ignore def __str__(self) -> str: @@ -151,9 +280,86 @@ def __decompose_type_rec__( set_basic_types.add(self) +class Sum(Type): + """ + Represents a sum type, like python's ``Union``. + + Parameters: + ----------- + - *types: the types union + """ + + __hash__ = Type.__hash__ + + def __init__(self, *types: Type): + self.types = types + self.hash = hash(types) + + def all_versions(self) -> TList["Type"]: + v = [] + for t in self.types: + v += t.all_versions() + return v + + def is_under_specified(self) -> bool: + return any(t.is_under_specified() for t in self.types) + + def __arg_is_a__(self, other: "Type") -> bool: + if isinstance(other, (Sum, FixedPolymorphicType)): + return all(any(x.is_instance(t) for t in self.types) for x in other.types) + else: + return any(other.is_instance(t) for t in self.types) + + def __pickle__(o: Type) -> Tuple: + return Sum, tuple(x for x in o.types) # type: ignore + + def __str__(self) -> str: + return "[" + " | ".join(format(x) for x in self.types) + "]" + + def __contains__(self, t: Type) -> bool: + return super().__contains__(t) or any(t in tt for tt in self.types) + + def __or__(self, other: "Type") -> "Sum": + if isinstance(other, Sum): + x = list(other.types) + list(self.types) + return Sum(*x) + return Sum(other, *self.types) + + def __eq__(self, o: object) -> bool: + return ( + isinstance(o, Sum) + and len(set(o.types).symmetric_difference(self.types)) == 0 + ) + + def __decompose_type_rec__( + self, + set_basic_types: Set["PrimitiveType"], + set_polymorphic_types: Set["PolymorphicType"], + ) -> None: + for t in self.types: + t.__decompose_type_rec__(set_basic_types, set_polymorphic_types) + + def is_polymorphic(self) -> bool: + return any(t.is_polymorphic() for t in self.types) + + def unify(self, unifier: Dict[str, "Type"]) -> "Type": + return Sum(*[x.unify(unifier) for x in self.types]) + + def depth(self) -> int: + return max(t.depth() for t in self.types) + + def size(self) -> int: + return max(t.size() for t in self.types) + + class Arrow(Type): """ - Represents a function. + Represents a unary function. + + Parameters: + ----------- + - type_in: the argument's type + - type_out: the returned type """ __hash__ = Type.__hash__ @@ -161,11 +367,27 @@ class Arrow(Type): def __init__(self, type_in: Type, type_out: Type): self.type_in = type_in self.type_out = type_out + self.is_out_arrow = isinstance(self.type_out, Arrow) self.hash = hash((self.type_in, self.type_out)) - def __pickle__(o: Type) -> Tuple: # type: ignore[override] + def __pickle__(o: Type) -> Tuple: return Arrow, (o.type_in, o.type_out) # type: ignore + def all_versions(self) -> TList["Type"]: + a = self.type_in.all_versions() + b = self.type_out.all_versions() + return [Arrow(x, y) for x in a for y in b] + + def is_under_specified(self) -> bool: + return self.type_in.is_under_specified() or self.type_out.is_under_specified() + + def __arg_is_a__(self, other: "Type") -> bool: + return ( + isinstance(other, Arrow) + and other.type_in.is_instance(self.type_in) + and other.type_out.is_instance(self.type_out) + ) + def __str__(self) -> str: rep_in = format(self.type_in) rep_out = format(self.type_out) @@ -193,7 +415,7 @@ def returns(self) -> Type: """ Get the return type of this arrow. """ - if isinstance(self.type_out, Arrow): + if self.is_out_arrow: return self.type_out.returns() return self.type_out @@ -201,10 +423,17 @@ def arguments(self) -> TList[Type]: """ Get the list of arguments in the correct order of this arrow. """ - if isinstance(self.type_out, Arrow): + if self.is_out_arrow: return [self.type_in] + self.type_out.arguments() return [self.type_in] + def without_unit_arguments(self) -> "Type": + if self.type_in == UNIT: + return self.type_out + elif isinstance(self.type_in, Arrow) and self.type_in.type_out == UNIT: + return Arrow(self.type_in.type_in, self.type_out) + return self + def is_polymorphic(self) -> bool: return self.type_in.is_polymorphic() or self.type_out.is_polymorphic() @@ -218,48 +447,140 @@ def size(self) -> int: return 1 + self.type_in.size() + self.type_out.size() -class List(Type): +class Generic(Type): + """ + Represents a parametric type such as python's ``list`` or ``dict``. + It is uniquely identified by its name. + + Parameters: + ----------- + - name: the name of this type + - *types: the types this generic instance depends on + - infix: if this is an infix type like * for tuples, only used for __str__ + """ + __hash__ = Type.__hash__ - def __init__(self, element_type: Type): - self.element_type = element_type - self.hash = hash(18923 + hash(self.element_type)) + def __init__( + self, + name: str, + *types: Type, + infix: bool = False, + ): + self.types = types + self.infix = infix + self.name = name + self.hash = hash((self.name, self.types)) + + def all_versions(self) -> TList["Type"]: + v = [] + for t in self.types: + v.append(t.all_versions()) + out: TList[Type] = [] + for cand in product(*v): + out.append(Generic(self.name, *cand)) + return out - def __pickle__(o: Type) -> Tuple: # type: ignore[override] - return List, (o.element_type,) # type: ignore + def is_under_specified(self) -> bool: + return any(t.is_under_specified() for t in self.types) + + def __arg_is_a__(self, other: "Type") -> bool: + return ( + isinstance(other, Generic) + and other.name == self.name + and all(any(tt.is_instance(t) for t in self.types) for tt in other.types) + ) + + def __getstate__(self) -> Dict: + return {k: v for k, v in self.__dict__.items() if k != "hash"} + + def __setstate__(self, state: Dict) -> None: + self.__dict__ = state + self.hash = hash((self.name, self.types)) def __str__(self) -> str: - return "list({})".format(self.element_type) + base = " " if not self.infix else f" {self.name} " + base = base.join(format(x) for x in self.types) + if self.infix: + return f"({base})" + return base + " " + self.name def __contains__(self, t: Type) -> bool: - return super().__contains__(t) or t in self.element_type + return super().__contains__(t) or any(t in tt for tt in self.types) def __eq__(self, o: object) -> bool: - return isinstance(o, List) and o.element_type == self.element_type + return ( + isinstance(o, Generic) + and o.name == self.name + and all(x == y for x, y in zip(self.types, o.types)) + ) def __decompose_type_rec__( self, set_basic_types: Set["PrimitiveType"], set_polymorphic_types: Set["PolymorphicType"], ) -> None: - self.element_type.__decompose_type_rec__(set_basic_types, set_polymorphic_types) + for t in self.types: + t.__decompose_type_rec__(set_basic_types, set_polymorphic_types) def is_polymorphic(self) -> bool: - return self.element_type.is_polymorphic() + return any(t.is_polymorphic() for t in self.types) def unify(self, unifier: Dict[str, "Type"]) -> "Type": - return List(self.element_type.unify(unifier)) + return Generic(self.name, *[x.unify(unifier) for x in self.types]) def depth(self) -> int: - return 1 + self.element_type.depth() + return max(t.depth() for t in self.types) def size(self) -> int: - return 1 + self.element_type.size() + return 1 + sum(t.size() for t in self.types) + + +class GenericFunctor(TypeFunctor): + """ + Produces an instanciator for the specific generic type. + + Parameters: + ----------- + - name: the name of the generic type + - min_args: the minimum number of arguments, -1 for no min + - max_args: the maximum number of arguments, -1 for no max + - infix: whether the generic instanciated should be infix or not + """ + + def __init__( + self, + name: str, + min_args: int = -1, + max_args: int = -1, + infix: bool = False, + ) -> None: + super().__init__() + self.name = name + self.min_args = min_args + self.max_args = max_args + self.infix = infix + + def __call__(self, *types: Type) -> Type: + assert ( + self.max_args <= 0 or len(types) <= self.max_args + ), f"Too many arguments:{len(types)}>{self.max_args} to build a {self.name}" + assert ( + self.min_args <= 0 or len(types) >= self.min_args + ), f"Too few arguments:{len(types)}<{self.min_args} to build a {self.name}" + return Generic(self.name, *types, infix=self.infix) + + def __is_arg_an_instance__(self, arg: Type) -> bool: + return isinstance(arg, Generic) and arg.name == self.name + + +List = GenericFunctor("list", min_args=1, max_args=1) class UnknownType(Type): """ - In case we need to define an unknown type + Represents an unknown type. + Typically if you stumble upon this type, it is likely that something failed. """ __hash__ = Type.__hash__ @@ -268,14 +589,17 @@ def __init__(self) -> None: super().__init__() self.hash = hash(1984) - def __pickle__(o: Type) -> Tuple: # type: ignore[override] + def is_under_specified(self) -> bool: + return True + + def __pickle__(o: Type) -> Tuple: return UnknownType, () def __str__(self) -> str: return "UnknownType" def __eq__(self, __o: object) -> bool: - return False + return isinstance(__o, UnknownType) def __decompose_type_rec__( self, @@ -293,56 +617,30 @@ def __decompose_type_rec__( EmptyList = List(PolymorphicType("empty")) -def FunctionType(*args: Type) -> Type: - """ - Short-hand to create n-ary functions. - """ - types = list(args) - base = types.pop() - while types: - base = Arrow(types.pop(), base) - return base - - -def guess_type(element: Any) -> Type: - """ - Guess the type of the given element. - Does not work for Arrow and Polymorphic Types. - """ - if isinstance(element, (TList, Tuple)): # type: ignore - if len(element) == 0: - return EmptyList - current: Type = UnknownType() - i = 0 - while i < len(element) and isinstance(current, UnknownType): - current = guess_type(element[i]) - i += 1 - return List(current) - if isinstance(element, bool): - return BOOL - elif isinstance(element, int): - return INT - elif isinstance(element, str): - return STRING - elif element is None: - return UNIT - return UnknownType() - - def match(a: Type, b: Type) -> bool: """ Return true if a and b match, this considers polymorphic instanciations. """ if type(a) == type(b): - if isinstance(a, List): - return match(a.element_type, b.element_type) # type: ignore - elif isinstance(a, Arrow): + if isinstance(a, Generic): + return a.name == b.name and all( # type: ignore + match(x, y) + for x, y in zip(a.types, b.types) # type: ignore + ) + elif a.is_instance(Arrow): return match(a.type_in, b.type_in) and match(a.type_out, b.type_out) # type: ignore + elif isinstance(a, Sum): + return all(any(match(x, y) for y in b.types) for x in a.types) and all( # type: ignore + any(match(x, y) for y in a.types) + for x in b.types # type: ignore + ) elif isinstance(a, UnknownType): return False - return isinstance(a, PolymorphicType) or a == b + return ( + isinstance(a, PolymorphicType) and a.can_be(b) and b.can_be(a) # type: ignore + ) or a == b elif isinstance(a, PolymorphicType): - return True + return a.can_be(b) elif isinstance(b, PolymorphicType): return match(b, a) return False @@ -350,5 +648,13 @@ def match(a: Type, b: Type) -> bool: import copyreg -for cls in [PrimitiveType, PolymorphicType, List, Arrow, UnknownType]: +for cls in [ + PrimitiveType, + PolymorphicType, + FixedPolymorphicType, + # Generic, + Arrow, + Sum, + UnknownType, +]: copyreg.pickle(cls, cls.__pickle__) # type: ignore diff --git a/synth/task.py b/synth/task.py index de3fd269..cbf29c42 100644 --- a/synth/task.py +++ b/synth/task.py @@ -1,6 +1,7 @@ from dataclasses import dataclass, field from typing import ( Any, + Callable, Dict, Generic, Iterator, @@ -11,12 +12,13 @@ overload, Set, ) -import _pickle as cPickle # type: ignore +import pickle import bz2 from synth.specification import TaskSpecification from synth.syntax.program import Program from synth.syntax.type_system import Type +from synth.utils.data_storage import load_object, save_object T = TypeVar("T", bound=TaskSpecification) @@ -72,14 +74,16 @@ def save(self, path: str) -> None: Save this dataset in the specified file. The dataset file is compressed. """ - with bz2.BZ2File(path, "w") as fd: - cPickle.dump(self, fd) + save_object(path, self) @classmethod - def load(cls, path: str) -> "Dataset[T]": + def load( + cls, + path: str, + unpickler: Optional[Callable[[bz2.BZ2File], pickle.Unpickler]] = None, + ) -> "Dataset[T]": """ Load the dataset object stored in this file. """ - with bz2.BZ2File(path, "rb") as fd: - dataset: Dataset = cPickle.load(fd) - return dataset + d: Dataset = load_object(path, unpickler) + return d diff --git a/synth/utils/__init__.py b/synth/utils/__init__.py index 4edd820c..c5aa8372 100644 --- a/synth/utils/__init__.py +++ b/synth/utils/__init__.py @@ -1,5 +1,7 @@ """ Utility objects and functions that do not fit elsewhere. """ + import synth.utils.chrono as chrono from synth.utils.generator_utils import gen_take +from synth.utils.data_storage import load_object, save_object diff --git a/synth/utils/chrono.py b/synth/utils/chrono.py index e87ba263..3b587201 100644 --- a/synth/utils/chrono.py +++ b/synth/utils/chrono.py @@ -1,6 +1,7 @@ """ Module to measure time spent in parts of the code programmatically and easily. """ + from functools import wraps import time from dataclasses import dataclass, field diff --git a/synth/utils/data_storage.py b/synth/utils/data_storage.py new file mode 100644 index 00000000..d5e73328 --- /dev/null +++ b/synth/utils/data_storage.py @@ -0,0 +1,49 @@ +import bz2 +import pickle +from typing import Any, Callable, Optional +import pickletools + + +def load_object( + path: str, unpickler: Optional[Callable[[bz2.BZ2File], pickle.Unpickler]] = None +) -> Any: + """ + Load an arbitrary object from the specified file. + """ + with bz2.BZ2File(path, "rb") as fd: + if unpickler is None: + return pickle.load(fd) + else: + return unpickler(fd).load() + + +def save_object( + path: str, obj: Any, optimize: bool = True, compress_level: int = 9 +) -> None: + """ + Save an arbitrary object to the specified path. + Compression level must be in 1-9 where 9 is the highest level. + """ + with bz2.BZ2File(path, "w", compresslevel=compress_level) as fd: + content = pickle.dumps(obj) + if optimize: + content = pickletools.optimize(content) + fd.write(content) + + +def legacy_load_object(path: str, **kwargs: Any) -> Any: + """ + DEPRECATED + Load an arbitrary object from the specified file. + """ + with open(path, "rb") as fd: + return pickle.load(fd) + + +def legacy_save_object(path: str, obj: Any, **kwargs: Any) -> None: + """ + DEPRECATED + Save an arbitrary object to the specified path. + """ + with open(path, "wb") as fd: + pickle.dump(obj, fd) diff --git a/synth/utils/import_utils.py b/synth/utils/import_utils.py new file mode 100644 index 00000000..102bdb4d --- /dev/null +++ b/synth/utils/import_utils.py @@ -0,0 +1,40 @@ +import importlib +from types import SimpleNamespace +from typing import List, Tuple, TypeVar, Union, Iterable, Callable, Optional + +U = TypeVar("U") + + +def __try_names__(name: str, f: Callable[[str], U], prefixes: List[str]) -> U: + try: + return f(name) + except ModuleNotFoundError: + l = prefixes[0] + return __try_names__(l + "." + name, f, prefixes[1:]) + + +def import_file_function( + import_name: str, + keys: Iterable[Union[str, Tuple[str, str]]], + prefixes: List[str] = [], +) -> Callable[[bool], Optional[SimpleNamespace]]: + """ + Utility function that creates a simple loader for you if you only need + > from name.name import X, Y, ... + where "X, Y, ..." are elements of keys. + + prefixes is a list of prefixes to the import name to try in case of failure. + """ + + def loader(fully_load: bool = True) -> Optional[SimpleNamespace]: + if not fully_load: + return __try_names__(import_name, importlib.util.find_spec, prefixes) # type: ignore + + module = __try_names__(import_name, importlib.import_module, prefixes) + out = {} + for key in keys: + get, set = key if isinstance(key, tuple) else (key, key) + out[set] = module.__getattribute__(get) + return SimpleNamespace(**out) + + return loader diff --git a/synth/utils/ordered.py b/synth/utils/ordered.py index 690aec2f..273c8839 100644 --- a/synth/utils/ordered.py +++ b/synth/utils/ordered.py @@ -3,6 +3,10 @@ class Ordered(Protocol): + @abstractmethod + def __le__(self, other: Any) -> bool: + pass + @abstractmethod def __lt__(self, other: Any) -> bool: pass diff --git a/synth/utils/vose_polyfill.py b/synth/utils/vose_polyfill.py new file mode 100644 index 00000000..c8a03c01 --- /dev/null +++ b/synth/utils/vose_polyfill.py @@ -0,0 +1,94 @@ +from typing import Optional, Union +import numpy as np + + +class PythonSampler: + def __init__(self, weights: np.ndarray, seed: Optional[int] = None) -> None: + self.rng = np.random.default_rng(seed or 1) + n = len(weights) + alias = np.zeros(n, dtype=int) + proba = np.zeros(n, dtype=float) + # Compute the average probability and cache it for later use. + avg = 1.0 / n + # Create two stacks to act as worklists as we populate the tables. + small = [] + large = [] + # Populate the stacks with the input probabilities. + for i in range(n): + # If the probability is below the average probability, then we add it to the small + # list; otherwise we add it to the large list. + if weights[i] >= avg: + large.append(i) + else: + small.append(i) + # As a note: in the mathematical specification of the algorithm, we will always exhaust the + # small list before the big list. However, due to floating point inaccuracies, this is not + # necessarily true. Consequently, this inner loop (which tries to pair small and large + # elements) will have to check that both lists aren't empty. + while len(small) > 0 and len(large) > 0: + # Get the index of the small and the large probabilities. + less = small.pop(0) + more = large.pop(0) + # These probabilities have not yet been scaled up to be such that 1 / n is given weight + # 1.0. We do this here instead. + proba[less] = weights[less] * n + alias[less] = more + # Decrease the probability of the larger one by the appropriate amount. + weights[more] = weights[more] + weights[less] - avg + # If the new probability is less than the average, add it into the small list; + # otherwise add it to the large list. + if weights[more] >= avg: + large.append(more) + else: + small.append(more) + # At this point, everything is in one list, which means that the remaining probabilities + # should all be 1 / n. Based on this, set them appropriately. Due to numerical issues, we + # can't be sure which stack will hold the entries, so we empty both. + while len(small) > 0: + less = small.pop(0) + proba[less] = 1.0 + while len(large) > 0: + more = large.pop(0) + proba[more] = 1.0 + self.n = n + self.alias = alias + self.proba = proba + + def sample_1(self) -> int: + # Generate a fair die roll to determine which column to inspect. + col = int(self.rng.uniform(0, self.n)) + # Generate a biased coin toss to determine which option to pick. + heads = self.rng.uniform() < 0.5 + + # Based on the outcome, return either the column or its alias. + if heads: + return col + return self.alias[col] # type: ignore + + def sample( + self, k: int = 1, values: Optional[np.ndarray] = None + ) -> Union[int, np.ndarray]: + """Sample a random integer or a value from a given array. + + Parameters: + k: The number of integers to sample. If `k = 1`, then a single int (or float if values is not None) is returned. In any + other case, a numpy array is returned. + values: The numpy array of values from which to sample from. + + """ + if values is None: + if k == 1: + return self.sample_1() + return np.asarray([self.sample_1() for _ in range(k)]) + else: + if k == 1: + return values[self.sample_1()] # type: ignore + return np.asarray([values[self.sample_1()] for _ in range(k)]) + + +try: + import vose + + Sampler = vose.Sampler +except ImportError: + Sampler = PythonSampler diff --git a/tests/filtering/constraints/test_dfta_constraints.py b/tests/filtering/constraints/test_dfta_constraints.py new file mode 100644 index 00000000..e80408ab --- /dev/null +++ b/tests/filtering/constraints/test_dfta_constraints.py @@ -0,0 +1,201 @@ +from synth.syntax.grammars.cfg import CFG +from synth.syntax.grammars.u_cfg import UCFG +from synth.syntax.dsl import DSL +from synth.syntax.type_system import ( + INT, + STRING, + List, + PolymorphicType, + PrimitiveType, +) +from synth.syntax.type_helper import FunctionType +from synth.filter.constraints.dfta_constraints import add_dfta_constraints + + +syntax = { + "+": FunctionType(INT, INT, INT), + "-": FunctionType(INT, INT, INT), + "head": FunctionType(List(PolymorphicType("a")), PolymorphicType("a")), + "non_reachable": PrimitiveType("non_reachable"), + "1": INT, + "0": INT, + "non_productive": FunctionType(INT, STRING), +} +dsl = DSL(syntax) +cfg = CFG.depth_constraint(dsl, FunctionType(INT, INT), 4) + + +def test_restriction() -> None: + new_cfg = UCFG.from_DFTA(add_dfta_constraints(cfg, [], "(+ 1 _)", progress=False)) + print(new_cfg) + assert dsl.parse_program("(- 1 (+ 1 1))", cfg.type_request) not in new_cfg + assert dsl.parse_program("(- 1 (- 1 1))", cfg.type_request) not in new_cfg + assert dsl.parse_program("(+ 1 (+ 1 1))", cfg.type_request) in new_cfg + assert dsl.parse_program("(+ 1 (+ (+ 1 1) 1))", cfg.type_request) in new_cfg + assert dsl.parse_program("(+ 1 (+ 1 (+ 1 1)))", cfg.type_request) in new_cfg + assert dsl.parse_program("(+ 1 1)", cfg.type_request) in new_cfg + + new_cfg = UCFG.from_DFTA(add_dfta_constraints(cfg, ["(+ 1 _)"], progress=False)) + print(new_cfg) + assert dsl.parse_program("(- 1 (+ 1 1))", cfg.type_request) in new_cfg + assert dsl.parse_program("(- 1 (- 1 1))", cfg.type_request) in new_cfg + assert dsl.parse_program("(+ 1 (+ 1 1))", cfg.type_request) in new_cfg + assert dsl.parse_program("(+ 1 (+ (+ 1 1) 1))", cfg.type_request) not in new_cfg + assert dsl.parse_program("(+ 1 (+ 1 (+ 1 1)))", cfg.type_request) in new_cfg + assert dsl.parse_program("(+ 1 1)", cfg.type_request) in new_cfg + + +def test_multi_level_easy() -> None: + new_cfg = UCFG.from_DFTA( + add_dfta_constraints(cfg, [], "(+ 1 (- _ 1))", progress=False) + ) + # print(new_cfg) + assert dsl.parse_program("(- 1 (+ 1 1))", cfg.type_request) not in new_cfg + assert dsl.parse_program("(- 1 (- 1 1))", cfg.type_request) not in new_cfg + assert dsl.parse_program("(+ 1 (- 1 1))", cfg.type_request) in new_cfg + assert dsl.parse_program("(+ 1 1)", cfg.type_request) not in new_cfg + assert dsl.parse_program("(+ 1 (- 1 (+ 1 1)))", cfg.type_request) not in new_cfg + assert dsl.parse_program("(+ 1 (- (+ 1 1) 1))", cfg.type_request) in new_cfg + + new_cfg = UCFG.from_DFTA( + add_dfta_constraints(cfg, ["(+ 1 (- _ 1))"], progress=False) + ) + print(new_cfg) + assert dsl.parse_program("(+ 1 1)", cfg.type_request) not in new_cfg + assert dsl.parse_program("(- 1 (- 1 1))", cfg.type_request) in new_cfg + assert dsl.parse_program("(+ 1 (- 1 1))", cfg.type_request) in new_cfg + assert dsl.parse_program("(- 1 (+ 1 1))", cfg.type_request) not in new_cfg + assert dsl.parse_program("(+ 1 (+ 1 (+ 1 1)))", cfg.type_request) not in new_cfg + assert dsl.parse_program("(+ 1 (- (- 1 1) 1))", cfg.type_request) in new_cfg + + +def test_multi_level_hard() -> None: + new_cfg = UCFG.from_DFTA( + add_dfta_constraints(cfg, [], "(+ 1 (+ _ 1))", progress=False) + ) + # print(new_cfg) + assert dsl.parse_program("(- 1 (+ 1 1))", cfg.type_request) not in new_cfg + assert dsl.parse_program("(- 1 (- 1 1))", cfg.type_request) not in new_cfg + assert dsl.parse_program("(+ 1 (+ 1 1))", cfg.type_request) in new_cfg + assert dsl.parse_program("(+ 1 1)", cfg.type_request) not in new_cfg + assert dsl.parse_program("(+ 1 (+ 1 (+ 1 1)))", cfg.type_request) not in new_cfg + assert dsl.parse_program("(+ 1 (+ (+ 1 1) 1))", cfg.type_request) in new_cfg + + new_cfg = UCFG.from_DFTA( + add_dfta_constraints(cfg, ["(+ 1 (+ _ 1))"], progress=False) + ) + print(new_cfg) + assert dsl.parse_program("(+ 1 1)", cfg.type_request) not in new_cfg + assert dsl.parse_program("(- 1 (- 1 1))", cfg.type_request) in new_cfg + assert dsl.parse_program("(+ 1 (+ 1 1))", cfg.type_request) in new_cfg + assert dsl.parse_program("(- 1 (+ 1 1))", cfg.type_request) not in new_cfg + assert dsl.parse_program("(+ 1 (+ 1 (+ 1 1)))", cfg.type_request) not in new_cfg + assert dsl.parse_program("(+ 1 (+ (- 1 1) 1))", cfg.type_request) in new_cfg + + +def test_at_most() -> None: + new_cfg = UCFG.from_DFTA( + add_dfta_constraints(cfg, [], "(- #(1)<=1 _)", progress=False) + ) + print(new_cfg) + assert dsl.parse_program("(- 1 (+ 1 1))", cfg.type_request) in new_cfg + assert dsl.parse_program("(- 1 (- 1 1))", cfg.type_request) in new_cfg + assert dsl.parse_program("(+ 1 (+ 1 1))", cfg.type_request) not in new_cfg + assert dsl.parse_program("(- (- 1 1) 1)", cfg.type_request) not in new_cfg + assert dsl.parse_program("(+ 1 (+ 1 (- 1 1)))", cfg.type_request) not in new_cfg + assert dsl.parse_program("(+ 1 (+ (+ 1 1) 1))", cfg.type_request) not in new_cfg + + new_cfg = UCFG.from_DFTA( + add_dfta_constraints(cfg, ["(- #(1)<=1 _)"], progress=False) + ) + print(new_cfg) + assert dsl.parse_program("(- 1 (+ 1 1))", cfg.type_request) in new_cfg + assert dsl.parse_program("(- 1 (- 1 1))", cfg.type_request) in new_cfg + assert dsl.parse_program("(+ 1 (+ 1 1))", cfg.type_request) in new_cfg + assert dsl.parse_program("(- (- 1 1) 1)", cfg.type_request) not in new_cfg + assert dsl.parse_program("(+ 1 (+ 1 (- 1 1)))", cfg.type_request) in new_cfg + assert dsl.parse_program("(+ 1 (+ (+ 1 1) 1))", cfg.type_request) in new_cfg + + +def test_at_least() -> None: + new_cfg = UCFG.from_DFTA( + add_dfta_constraints(cfg, [], "(- _ #(1)>=2)", progress=False) + ) + print(new_cfg) + assert dsl.parse_program("(- 1 (+ 1 1))", cfg.type_request) in new_cfg + assert dsl.parse_program("(- 1 (- 1 1))", cfg.type_request) in new_cfg + assert dsl.parse_program("(+ 1 (+ 1 1))", cfg.type_request) not in new_cfg + assert dsl.parse_program("(- (- 1 1) 1)", cfg.type_request) not in new_cfg + assert dsl.parse_program("(+ 1 (+ 1 (- 1 1)))", cfg.type_request) not in new_cfg + assert dsl.parse_program("(- 1 (- (+ 1 1) 1))", cfg.type_request) in new_cfg + + new_cfg = UCFG.from_DFTA( + add_dfta_constraints(cfg, ["(- _ #(1)>=2)"], progress=False) + ) + print(new_cfg) + assert dsl.parse_program("(- 1 (+ 1 1))", cfg.type_request) in new_cfg + assert dsl.parse_program("(- 1 (- 1 1))", cfg.type_request) not in new_cfg + assert dsl.parse_program("(+ 1 (+ 1 1))", cfg.type_request) in new_cfg + assert dsl.parse_program("(- (- 1 1) 1)", cfg.type_request) not in new_cfg + assert dsl.parse_program("(+ 1 (+ 1 (- 1 1)))", cfg.type_request) not in new_cfg + assert dsl.parse_program("(- 1 (- 1 (+ 1 1)))", cfg.type_request) in new_cfg + + +def test_forbid_subtree() -> None: + new_cfg = UCFG.from_DFTA( + add_dfta_constraints(cfg, [], "(+ >^(var0) _)", progress=False) + ) + print(new_cfg) + assert dsl.parse_program("(- 1 (+ 1 1))", cfg.type_request) not in new_cfg + assert dsl.parse_program("(- 1 (- 1 1))", cfg.type_request) not in new_cfg + assert dsl.parse_program("(+ 1 (+ 1 1))", cfg.type_request) in new_cfg + assert dsl.parse_program("(+ var0 1)", cfg.type_request) not in new_cfg + assert dsl.parse_program("(+ 1 (+ 1 (+ var0 1)))", cfg.type_request) in new_cfg + + new_cfg = UCFG.from_DFTA( + add_dfta_constraints(cfg, ["(+ >^(var0) _)"], progress=False) + ) + print(new_cfg) + assert dsl.parse_program("(- 1 (+ 1 1))", cfg.type_request) in new_cfg + assert dsl.parse_program("(- 1 (- 1 1))", cfg.type_request) in new_cfg + assert dsl.parse_program("(+ 1 (+ 1 1))", cfg.type_request) in new_cfg + assert dsl.parse_program("(+ var0 1)", cfg.type_request) not in new_cfg + assert dsl.parse_program("(+ 1 (+ 1 (+ var0 1)))", cfg.type_request) not in new_cfg + + +def test_force_subtree() -> None: + new_cfg = UCFG.from_DFTA( + add_dfta_constraints(cfg, [], "(+ >(var0) _)", progress=False) + ) + print(new_cfg) + assert dsl.parse_program("(- 1 (+ 1 1))", cfg.type_request) not in new_cfg + assert dsl.parse_program("(- 1 (- 1 1))", cfg.type_request) not in new_cfg + assert dsl.parse_program("(+ 1 (+ 1 1))", cfg.type_request) not in new_cfg + assert dsl.parse_program("(+ var0 1)", cfg.type_request) in new_cfg + assert dsl.parse_program("(+ 1 (+ 1 (+ var0 1)))", cfg.type_request) not in new_cfg + assert dsl.parse_program("(+ (+ 1 (+ var0 1)) 1)", cfg.type_request) in new_cfg + + new_cfg = UCFG.from_DFTA( + add_dfta_constraints(cfg, ["(+ >(var0) _)"], progress=False) + ) + print(new_cfg) + assert dsl.parse_program("(- 1 (+ 1 1))", cfg.type_request) not in new_cfg + assert dsl.parse_program("(- 1 (- 1 1))", cfg.type_request) in new_cfg + assert dsl.parse_program("(+ 1 (+ 1 1))", cfg.type_request) not in new_cfg + assert dsl.parse_program("(+ var0 1)", cfg.type_request) in new_cfg + assert dsl.parse_program("(+ 1 (+ 1 (+ var0 1)))", cfg.type_request) not in new_cfg + assert dsl.parse_program("(+ (+ (+ var0 1) 1) 1)", cfg.type_request) in new_cfg + + +def test_multi_constraints() -> None: + new_cfg = UCFG.from_DFTA( + add_dfta_constraints(cfg, ["(+ 1 ^0)", "(- _ ^0)"], progress=False) + ) + print(new_cfg) + assert dsl.parse_program("(- 1 (+ 1 1))", cfg.type_request) in new_cfg + assert dsl.parse_program("(- 1 (- 1 1))", cfg.type_request) in new_cfg + assert dsl.parse_program("(+ 1 (+ 1 1))", cfg.type_request) in new_cfg + assert dsl.parse_program("(+ (+ 1 1) 1)", cfg.type_request) not in new_cfg + assert dsl.parse_program("(+ var0 1)", cfg.type_request) not in new_cfg + assert dsl.parse_program("(+ 1 (+ 1 (+ var0 1)))", cfg.type_request) not in new_cfg + assert dsl.parse_program("(+ 1 (+ 1 (+ 1 var0)))", cfg.type_request) in new_cfg diff --git a/tests/filtering/constraints/test_parsing.py b/tests/filtering/constraints/test_parsing.py new file mode 100644 index 00000000..9bfb6930 --- /dev/null +++ b/tests/filtering/constraints/test_parsing.py @@ -0,0 +1,52 @@ +from synth.syntax.grammars.ttcfg import TTCFG +from synth.syntax.dsl import DSL +from synth.syntax.program import Variable +from synth.syntax.type_system import ( + INT, + STRING, + List, + PolymorphicType, + PrimitiveType, +) +from synth.syntax.type_helper import FunctionType +from synth.filter.constraints.parsing import ( + parse_specification, + TokenAnything, + TokenAllow, + TokenAtLeast, + TokenAtMost, + TokenFunction, + TokenForceSubtree, + TokenForbidSubtree, +) + + +syntax = { + "+": FunctionType(INT, INT, INT), + "head": FunctionType(List(PolymorphicType("a")), PolymorphicType("a")), + "non_reachable": PrimitiveType("non_reachable"), + "1": INT, + "non_productive": FunctionType(INT, STRING), +} +dsl = DSL(syntax) +cfg = TTCFG.size_constraint(dsl, FunctionType(INT, INT), 20) +ONE = dsl.get_primitive("1") +PLUS = dsl.get_primitive("+") + + +def test_bases() -> None: + assert parse_specification("_", cfg) == TokenAnything() + assert parse_specification("#(1)<=3", cfg) == TokenAtMost([ONE], count=3) + assert parse_specification("(+ #(1)<=1 _)", cfg) == TokenFunction( + TokenAllow([PLUS]), args=[TokenAtMost([ONE], count=1), TokenAnything()] + ) + assert parse_specification("#(1,+)>=3", cfg) == TokenAtLeast([ONE, PLUS], count=3) + assert parse_specification("#(1,+,+)>=4", cfg) == TokenAtLeast([ONE, PLUS], count=4) + assert parse_specification("(+ 1 _)", cfg) == TokenFunction( + TokenAllow([PLUS]), + [TokenAllow([ONE]), TokenAnything()], + ) + assert parse_specification(">(var0)", cfg) == TokenForceSubtree([Variable(0, INT)]) + assert parse_specification(">^(1,var0)", cfg) == TokenForbidSubtree( + [ONE, Variable(0, INT)] + ) diff --git a/tests/filtering/constraints/test_ttcfg_constraints.py b/tests/filtering/constraints/test_ttcfg_constraints.py new file mode 100644 index 00000000..34d455eb --- /dev/null +++ b/tests/filtering/constraints/test_ttcfg_constraints.py @@ -0,0 +1,102 @@ +from synth.syntax.grammars.ttcfg import TTCFG +from synth.syntax.dsl import DSL +from synth.syntax.type_system import ( + INT, + STRING, + List, + PolymorphicType, + PrimitiveType, +) +from synth.syntax.type_helper import FunctionType +from synth.filter.constraints.ttcfg_constraints import add_constraints + + +syntax = { + "+": FunctionType(INT, INT, INT), + "-": FunctionType(INT, INT, INT), + "head": FunctionType(List(PolymorphicType("a")), PolymorphicType("a")), + "non_reachable": PrimitiveType("non_reachable"), + "1": INT, + "non_productive": FunctionType(INT, STRING), +} +dsl = DSL(syntax) +cfg = TTCFG.size_constraint(dsl, FunctionType(INT, INT), 9) +# cfg = CFG.depth_constraint(dsl, FunctionType(INT, INT), 4) + + +def test_restriction() -> None: + new_cfg = add_constraints(cfg, [], "(+ 1 _)", progress=False) + print(new_cfg) + assert dsl.parse_program("(- 1 (+ 1 1))", cfg.type_request) not in new_cfg + assert dsl.parse_program("(- 1 (- 1 1))", cfg.type_request) not in new_cfg + assert dsl.parse_program("(+ 1 (+ 1 1))", cfg.type_request) in new_cfg + assert dsl.parse_program("(+ 1 (+ (+ 1 1) 1))", cfg.type_request) in new_cfg + assert dsl.parse_program("(+ 1 (+ 1 (+ 1 1)))", cfg.type_request) in new_cfg + assert dsl.parse_program("(+ 1 1)", cfg.type_request) in new_cfg + + new_cfg = add_constraints(cfg, ["(+ 1 _)"], progress=False) + print(new_cfg) + assert dsl.parse_program("(- 1 (+ 1 1))", cfg.type_request) in new_cfg + assert dsl.parse_program("(- 1 (- 1 1))", cfg.type_request) in new_cfg + assert dsl.parse_program("(+ 1 (+ 1 1))", cfg.type_request) in new_cfg + assert dsl.parse_program("(+ 1 (+ (+ 1 1) 1))", cfg.type_request) not in new_cfg + assert dsl.parse_program("(+ 1 (+ 1 (+ 1 1)))", cfg.type_request) in new_cfg + assert dsl.parse_program("(+ 1 1)", cfg.type_request) in new_cfg + + +def test_multi_level() -> None: + new_cfg = add_constraints(cfg, [], "(+ 1 (+ _ 1))", progress=False) + print(new_cfg) + assert dsl.parse_program("(- 1 (+ 1 1))", cfg.type_request) not in new_cfg + assert dsl.parse_program("(- 1 (- 1 1))", cfg.type_request) not in new_cfg + assert dsl.parse_program("(+ 1 (+ 1 1))", cfg.type_request) in new_cfg + assert dsl.parse_program("(+ 1 1)", cfg.type_request) not in new_cfg + assert dsl.parse_program("(+ 1 (+ 1 (+ 1 1)))", cfg.type_request) not in new_cfg + assert dsl.parse_program("(+ 1 (+ (+ 1 1) 1))", cfg.type_request) in new_cfg + + new_cfg = add_constraints(cfg, ["(+ 1 (+ _ 1))"], progress=False) + print(new_cfg) + assert dsl.parse_program("(- 1 (+ 1 1))", cfg.type_request) not in new_cfg + assert dsl.parse_program("(- 1 (- 1 1))", cfg.type_request) in new_cfg + assert dsl.parse_program("(+ 1 (+ 1 1))", cfg.type_request) in new_cfg + assert dsl.parse_program("(+ 1 1)", cfg.type_request) not in new_cfg + assert dsl.parse_program("(+ 1 (+ 1 (+ 1 1)))", cfg.type_request) not in new_cfg + assert dsl.parse_program("(+ 1 (+ (+ 1 1) 1))", cfg.type_request) not in new_cfg + + +def test_at_most() -> None: + new_cfg = add_constraints(cfg, [], "(- #(1)<=1 _)", progress=False) + print(new_cfg) + assert dsl.parse_program("(- 1 (+ 1 1))", cfg.type_request) in new_cfg + assert dsl.parse_program("(- 1 (- 1 1))", cfg.type_request) in new_cfg + assert dsl.parse_program("(+ 1 (+ 1 1))", cfg.type_request) not in new_cfg + assert dsl.parse_program("(- (- 1 1) 1)", cfg.type_request) not in new_cfg + assert dsl.parse_program("(+ 1 (+ 1 (- 1 1)))", cfg.type_request) not in new_cfg + assert dsl.parse_program("(+ 1 (+ (+ 1 1) 1))", cfg.type_request) not in new_cfg + + new_cfg = add_constraints(cfg, ["(- #(1)<=1 _)"], progress=False) + print(new_cfg) + assert dsl.parse_program("(- 1 (+ 1 1))", cfg.type_request) in new_cfg + assert dsl.parse_program("(- 1 (- 1 1))", cfg.type_request) in new_cfg + assert dsl.parse_program("(+ 1 (+ 1 1))", cfg.type_request) in new_cfg + assert dsl.parse_program("(- (- 1 1) 1)", cfg.type_request) not in new_cfg + assert dsl.parse_program("(+ 1 (+ 1 (- 1 1)))", cfg.type_request) in new_cfg + assert dsl.parse_program("(+ 1 (+ (+ 1 1) 1))", cfg.type_request) in new_cfg + + +def test_var_dep() -> None: + new_cfg = add_constraints(cfg, [], "(+ >^(var0) _)", progress=False) + print(new_cfg) + assert dsl.parse_program("(- 1 (+ 1 1))", cfg.type_request) not in new_cfg + assert dsl.parse_program("(- 1 (- 1 1))", cfg.type_request) not in new_cfg + assert dsl.parse_program("(+ 1 (+ 1 1))", cfg.type_request) in new_cfg + assert dsl.parse_program("(+ var0 1)", cfg.type_request) not in new_cfg + assert dsl.parse_program("(+ 1 (+ 1 (+ var0 1)))", cfg.type_request) in new_cfg + + new_cfg = add_constraints(cfg, ["(+ >^(var0) _)"], progress=False) + print(new_cfg) + assert dsl.parse_program("(- 1 (+ 1 1))", cfg.type_request) in new_cfg + assert dsl.parse_program("(- 1 (- 1 1))", cfg.type_request) in new_cfg + assert dsl.parse_program("(+ 1 (+ 1 1))", cfg.type_request) in new_cfg + assert dsl.parse_program("(+ var0 1)", cfg.type_request) not in new_cfg + assert dsl.parse_program("(+ 1 (+ 1 (+ var0 1)))", cfg.type_request) not in new_cfg diff --git a/tests/nn/test_grammar_predictor.py b/tests/nn/test_det_grammar_predictor.py similarity index 89% rename from tests/nn/test_grammar_predictor.py rename to tests/nn/test_det_grammar_predictor.py index 1a491258..aedbc56a 100644 --- a/tests/nn/test_grammar_predictor.py +++ b/tests/nn/test_det_grammar_predictor.py @@ -2,15 +2,15 @@ import torch from torch.functional import Tensor -from synth.nn.grammar_predictor import GrammarPredictorLayer +from synth.nn.det_grammar_predictor import DetGrammarPredictorLayer from synth.nn.abstractions import cfg_bigram_without_depth from synth.syntax.grammars.cfg import CFG from synth.syntax.dsl import DSL from synth.syntax.program import Function, Primitive, Variable from synth.syntax.type_system import ( INT, - FunctionType, ) +from synth.syntax.type_helper import FunctionType syntax = { @@ -25,7 +25,7 @@ def test_forward() -> None: - layer = GrammarPredictorLayer(50, {cfg}, cfg_bigram_without_depth) + layer = DetGrammarPredictorLayer(50, {cfg}, cfg_bigram_without_depth) generator = torch.manual_seed(0) for _ in range(20): x = torch.randn((100, 50), generator=generator) @@ -34,7 +34,7 @@ def test_forward() -> None: def test_to_logpcfg() -> None: - layer = GrammarPredictorLayer(50, {cfg}, cfg_bigram_without_depth) + layer = DetGrammarPredictorLayer(50, {cfg}, cfg_bigram_without_depth) generator = torch.manual_seed(0) for _ in range(20): x = torch.randn((5, 50), generator=generator) @@ -51,7 +51,7 @@ def test_to_logpcfg() -> None: def test_logpcfg2pcfg() -> None: - layer = GrammarPredictorLayer(50, {cfg}, cfg_bigram_without_depth) + layer = DetGrammarPredictorLayer(50, {cfg}, cfg_bigram_without_depth) generator = torch.manual_seed(0) for _ in range(20): x = torch.randn((5, 50), generator=generator) @@ -74,7 +74,7 @@ def test_logpcfg2pcfg() -> None: def test_var_as_function() -> None: - layer = GrammarPredictorLayer(50, {cfg2, cfg}, cfg_bigram_without_depth) + layer = DetGrammarPredictorLayer(50, {cfg2, cfg}, cfg_bigram_without_depth) generator = torch.manual_seed(0) for _ in range(5): for c in [cfg, cfg2]: @@ -93,7 +93,7 @@ def test_var_as_function() -> None: def test_varprob() -> None: - layer = GrammarPredictorLayer(10, {cfg}, cfg_bigram_without_depth) + layer = DetGrammarPredictorLayer(10, {cfg}, cfg_bigram_without_depth) opti = torch.optim.AdamW(layer.parameters(), lr=1e-1) steps = 10 batch_size = 10 @@ -128,7 +128,7 @@ def test_varprob() -> None: def test_learning() -> None: - layer = GrammarPredictorLayer(10, {cfg}, cfg_bigram_without_depth) + layer = DetGrammarPredictorLayer(10, {cfg}, cfg_bigram_without_depth) opti = torch.optim.AdamW(layer.parameters(), lr=1e-1) steps = 10 mean_prob = [] @@ -163,8 +163,8 @@ def test_learning() -> None: assert mean_prob[-1] > 0.12 -def test_learning_cross_entropy() -> None: - layer = GrammarPredictorLayer(10, {cfg}, cfg_bigram_without_depth) +def test_learning_mse() -> None: + layer = DetGrammarPredictorLayer(10, {cfg}, cfg_bigram_without_depth) opti = torch.optim.AdamW(layer.parameters(), lr=1e-1) steps = 10 mean_prob = [] @@ -179,9 +179,7 @@ def test_learning_cross_entropy() -> None: inputs = torch.ones((batch_size, 10)) y = layer(inputs) opti.zero_grad() - loss = layer.loss_cross_entropy( - programs, [cfg.type_request for _ in programs], y - ) + loss = layer.loss_mse(programs, [cfg.type_request for _ in programs], y) loss.backward() opti.step() diff --git a/tests/nn/test_u_grammar_predictor.py b/tests/nn/test_u_grammar_predictor.py new file mode 100644 index 00000000..5439c83e --- /dev/null +++ b/tests/nn/test_u_grammar_predictor.py @@ -0,0 +1,203 @@ +import numpy as np +import torch +from torch.functional import Tensor + +from synth.nn.abstractions import ucfg_bigram +from synth.nn.u_grammar_predictor import UGrammarPredictorLayer +from synth.syntax.grammars.u_cfg import UCFG +from synth.syntax.dsl import DSL +from synth.syntax.program import Function, Primitive, Variable +from synth.syntax.type_system import ( + INT, +) +from synth.syntax.type_helper import FunctionType + + +syntax = { + "+": FunctionType(INT, INT, INT), + "-": FunctionType(INT, INT, INT), + "1": INT, +} + +dsl = DSL(syntax) +cfg = UCFG.depth_constraint(dsl, FunctionType(INT, INT), 4) +cfg2 = UCFG.depth_constraint(dsl, FunctionType(FunctionType(INT, INT), INT, INT), 5) + + +def test_forward() -> None: + layer = UGrammarPredictorLayer(50, {cfg}, ucfg_bigram) + generator = torch.manual_seed(0) + for _ in range(20): + x = torch.randn((100, 50), generator=generator) + y: Tensor = layer(x) + assert y.shape == torch.Size([x.shape[0], 16]) + + +def test_to_logpcfg() -> None: + layer = UGrammarPredictorLayer(50, {cfg}, ucfg_bigram) + generator = torch.manual_seed(0) + for _ in range(20): + x = torch.randn((5, 50), generator=generator) + y = layer(x) + for i in range(y.shape[0]): + log_pcfg = layer.tensor2log_prob_grammar( + y[i], cfg.type_request, total_variable_order=False + ) + for S in log_pcfg.rules: + total = sum( + sum(np.exp(t_prob.item()) for t_prob in dico.values()) + for P, dico in log_pcfg.tags[S].items() + ) + assert np.isclose(1, total) + + +def test_logpcfg2pcfg() -> None: + layer = UGrammarPredictorLayer(50, {cfg}, ucfg_bigram) + generator = torch.manual_seed(0) + for _ in range(20): + x = torch.randn((5, 50), generator=generator) + y = layer(x) + for i in range(y.shape[0]): + log_pcfg = layer.tensor2log_prob_grammar( + y[i], cfg.type_request, total_variable_order=False + ) + pcfg = log_pcfg.to_prob_u_grammar() + pcfg.init_sampling(0) + P = pcfg.sample_program() + # This line is important because it checks that a call to log_probability does not affect the probabilities + log_pcfg.log_probability(P).item() + for S in pcfg.rules: + for P in pcfg.rules[S]: + for v in pcfg.rules[S][P]: + target = np.exp(log_pcfg.tags[S][P][tuple(v)].item()) + assert np.isclose( + pcfg.probabilities[S][P][tuple(v)], target + ), f"S:{S}, P:{P} V:{v} pcfg_prob:{pcfg.probabilities[S][P][tuple(v)]} log_pcfg_prob:{target}" + + +def test_var_as_function() -> None: + layer = UGrammarPredictorLayer(50, {cfg2, cfg}, ucfg_bigram) + generator = torch.manual_seed(0) + for _ in range(5): + for c in [cfg, cfg2]: + x = torch.randn((5, 50), generator=generator) + y = layer(x) + for i in range(y.shape[0]): + log_pcfg = layer.tensor2log_prob_grammar( + y[i], c.type_request, total_variable_order=False + ) + pcfg = log_pcfg.to_prob_u_grammar() + pcfg.init_sampling(0) + P = pcfg.sample_program() + prob = pcfg.probability(P) + exp_logprob = np.exp(log_pcfg.log_probability(P).item()) + assert np.isclose( + prob, exp_logprob + ), f"P:{P} prob:{prob} explogprob:{exp_logprob}" + + +def test_varprob() -> None: + layer = UGrammarPredictorLayer(10, {cfg}, ucfg_bigram) + opti = torch.optim.AdamW(layer.parameters(), lr=1e-1) + steps = 10 + batch_size = 10 + programs = [ + Function( + Primitive("+", FunctionType(INT, INT, INT)), + [Variable(0, INT), Primitive("1", INT)], + ) + ] * batch_size + for _ in range(steps): + inputs = torch.ones((batch_size, 10)) + y = layer(inputs) + pcfgs = [ + layer.tensor2log_prob_grammar( + y[i], cfg.type_request, total_variable_order=False + ) + for i in range(batch_size) + ] + opti.zero_grad() + loss = layer.loss_negative_log_prob(programs, pcfgs) + loss.backward() + opti.step() + + for pcfg in pcfgs: + for S in pcfg.rules: + for P in pcfg.rules[S]: + if isinstance(P, Variable): + prob = np.exp(pcfg.tags[S][P][()].item()) + assert np.isclose( + prob, layer.variable_probability + ), f"S:{S}, P:{P} prob:{prob} target:{layer.variable_probability}" + + +def test_learning() -> None: + layer = UGrammarPredictorLayer(10, {cfg}, ucfg_bigram) + opti = torch.optim.AdamW(layer.parameters(), lr=1e-1) + steps = 10 + mean_prob = [] + batch_size = 10 + programs = [ + Function( + Primitive("+", FunctionType(INT, INT, INT)), + [Variable(0, INT), Primitive("1", INT)], + ) + ] * batch_size + for step in range(steps): + inputs = torch.ones((batch_size, 10)) + y = layer(inputs) + pcfgs = [ + layer.tensor2log_prob_grammar(y[i], cfg.type_request) + for i in range(batch_size) + ] + opti.zero_grad() + loss = layer.loss_negative_log_prob(programs, pcfgs) + loss.backward() + opti.step() + + with torch.no_grad(): + logprob = -layer.loss_negative_log_prob( + programs, pcfgs, length_normed=False + ).item() + mean_prob.append(np.exp(logprob)) + + for i in range(1, len(mean_prob)): + assert mean_prob[i - 1] < mean_prob[i], f"{mean_prob}" + + assert mean_prob[-1] > 0.12 + + +def test_learning_mse() -> None: + layer = UGrammarPredictorLayer(10, {cfg}, ucfg_bigram) + opti = torch.optim.AdamW(layer.parameters(), lr=1e-1) + steps = 10 + mean_prob = [] + batch_size = 10 + programs = [ + Function( + Primitive("+", FunctionType(INT, INT, INT)), + [Variable(0, INT), Primitive("1", INT)], + ) + ] * batch_size + for step in range(steps): + inputs = torch.ones((batch_size, 10)) + y = layer(inputs) + opti.zero_grad() + loss = layer.loss_mse(programs, [cfg.type_request for _ in programs], y) + loss.backward() + opti.step() + + with torch.no_grad(): + pcfgs = [ + layer.tensor2log_prob_grammar(y[i], cfg.type_request) + for i in range(batch_size) + ] + logprob = -layer.loss_negative_log_prob( + programs, pcfgs, length_normed=False + ).item() + mean_prob.append(np.exp(logprob)) + + for i in range(1, len(mean_prob)): + assert mean_prob[i - 1] < mean_prob[i], f"{mean_prob}" + + assert mean_prob[-1] > 0.12 diff --git a/tests/pbe/solvers/test_pbe_solver.py b/tests/pbe/solvers/test_pbe_solver.py new file mode 100644 index 00000000..04ec183d --- /dev/null +++ b/tests/pbe/solvers/test_pbe_solver.py @@ -0,0 +1,64 @@ +from synth.semantic.evaluator import DSLEvaluator +from synth.specification import PBE, Example +from synth.syntax.grammars.enumeration.heap_search import enumerate_prob_grammar +from synth.syntax.grammars.cfg import CFG +from synth.syntax.grammars.tagged_det_grammar import ProbDetGrammar +from synth.syntax.dsl import DSL +from synth.syntax.type_system import ( + INT, + STRING, + List, + PolymorphicType, + PrimitiveType, +) +from synth.syntax.type_helper import FunctionType +from synth.pbe.solvers import NaivePBESolver, CutoffPBESolver, PBESolver + +import pytest + +from synth.task import Task + + +syntax = { + "+": FunctionType(INT, INT, INT), + "-": FunctionType(INT, INT, INT), + "head": FunctionType(List(PolymorphicType("a")), PolymorphicType("a")), + "non_reachable": PrimitiveType("non_reachable"), + "1": INT, + "non_productive": FunctionType(INT, STRING), +} + +semantics = {"+": lambda x: lambda y: x + y, "-": lambda x: lambda y: x - y, "1": 1} + + +type_req = FunctionType(INT, INT) +int_lexicon = list(range(-100, 100)) +max_depth = 4 +dsl = DSL(syntax) +evaluator = DSLEvaluator(dsl.instantiate_semantics(semantics)) +testdata = [ + NaivePBESolver(evaluator), + CutoffPBESolver(evaluator), +] + +cfg = CFG.depth_constraint(dsl, FunctionType(INT, INT), 4) +pcfg = ProbDetGrammar.uniform(cfg) + + +tasks = [ + Task(cfg.type_request, PBE([Example([x], x + 2) for x in [3, 4, 9, 12]])), + Task(cfg.type_request, PBE([Example([x], x - 2) for x in [3, 4, 9, 12]])), +] + + +@pytest.mark.parametrize("solver", testdata) +def test_solving(solver: PBESolver) -> None: + for task in tasks: + failed = True + for program in solver.solve(task, enumerate_prob_grammar(pcfg), 10): + for example in task.specification.examples: + assert evaluator.eval(program, example.inputs) == example.output + failed = False + assert solver._score > 0 + break + assert not failed diff --git a/tests/pbe/solvers/test_restart_pbe_solver.py b/tests/pbe/solvers/test_restart_pbe_solver.py new file mode 100644 index 00000000..50858f7b --- /dev/null +++ b/tests/pbe/solvers/test_restart_pbe_solver.py @@ -0,0 +1,71 @@ +from synth.semantic.evaluator import DSLEvaluator +from synth.specification import PBE, Example +from synth.syntax.grammars.enumeration.heap_search import enumerate_prob_grammar +from synth.syntax.grammars.cfg import CFG +from synth.syntax.grammars.tagged_det_grammar import ProbDetGrammar +from synth.syntax.dsl import DSL +from synth.syntax.type_system import ( + INT, + STRING, + List, + PolymorphicType, + PrimitiveType, +) +from synth.syntax.type_helper import FunctionType +from synth.pbe.solvers import NaivePBESolver, CutoffPBESolver, PBESolver +from synth.pbe.solvers.restart_pbe_solver import RestartPBESolver + +import pytest + +from synth.task import Task + + +syntax = { + "+": FunctionType(INT, INT, INT), + "-": FunctionType(INT, INT, INT), + "head": FunctionType(List(PolymorphicType("a")), PolymorphicType("a")), + "non_reachable": PrimitiveType("non_reachable"), + "1": INT, + "non_productive": FunctionType(INT, STRING), +} + +semantics = {"+": lambda x: lambda y: x + y, "-": lambda x: lambda y: x - y, "1": 1} + + +type_req = FunctionType(INT, INT) +max_depth = 4 +dsl = DSL(syntax) +evaluator = DSLEvaluator(dsl.instantiate_semantics(semantics)) +testdata = [ + NaivePBESolver(evaluator), + CutoffPBESolver(evaluator), +] + +cfg = CFG.depth_constraint(dsl, type_req, max_depth) +pcfg = ProbDetGrammar.uniform(cfg) + + +tasks = [ + Task(cfg.type_request, PBE([Example([x], x + 2) for x in range(50)])), + Task(cfg.type_request, PBE([Example([x], x - 2) for x in range(50)])), +] + + +@pytest.mark.parametrize("solver", testdata) +def test_solving(solver: PBESolver) -> None: + real_solver = RestartPBESolver( + solver.evaluator, + lambda *args, **kwargs: solver, + restart_criterion=lambda self: len(self._data) - self._last_size > 3, + ) + for task in tasks: + failed = True + real_solver.reset_stats() + for program in real_solver.solve(task, enumerate_prob_grammar(pcfg), 5): + for example in task.specification.examples: + assert evaluator.eval(program, example.inputs) == example.output + failed = False + if real_solver._restarts <= 0: + continue + break + assert not failed diff --git a/tests/pbe/test_io_encoder.py b/tests/pbe/test_io_encoder.py index 848da1b5..4340e514 100644 --- a/tests/pbe/test_io_encoder.py +++ b/tests/pbe/test_io_encoder.py @@ -8,9 +8,9 @@ from synth.specification import PBE, Example from synth.syntax.type_system import ( INT, - FunctionType, List, ) +from synth.syntax.type_helper import FunctionType def test_encoding() -> None: diff --git a/tests/pbe/test_task_generator.py b/tests/pbe/test_task_generator.py index 1dd64798..ce1f39a8 100644 --- a/tests/pbe/test_task_generator.py +++ b/tests/pbe/test_task_generator.py @@ -7,11 +7,11 @@ from synth.syntax.type_system import ( INT, STRING, - FunctionType, List, PolymorphicType, PrimitiveType, ) +from synth.syntax.type_helper import FunctionType syntax = { @@ -38,7 +38,7 @@ def test_gen() -> None: pcfg.init_sampling(20) g = TaskGenerator( LexiconSampler(int_lexicon, seed=10), - DSLEvaluator(semantics), + DSLEvaluator(dsl.instantiate_semantics(semantics)), LexiconSampler([type_req], seed=10), LexiconSampler(samples_lexicon, [0.25, 0.5, 0.25], seed=10), {pcfg}, @@ -61,7 +61,7 @@ def test_seed() -> None: pcfg.init_sampling(10) g1 = TaskGenerator( LexiconSampler(int_lexicon, seed=10), - DSLEvaluator(semantics), + DSLEvaluator(dsl.instantiate_semantics(semantics)), LexiconSampler([type_req], seed=10), LexiconSampler([2, 3, 4], [0.25, 0.5, 0.25], seed=10), {pcfg}, @@ -71,7 +71,7 @@ def test_seed() -> None: pcfg.init_sampling(10) g2 = TaskGenerator( LexiconSampler(int_lexicon, seed=10), - DSLEvaluator(semantics), + DSLEvaluator(dsl.instantiate_semantics(semantics)), LexiconSampler([type_req], seed=10), LexiconSampler([2, 3, 4], [0.25, 0.5, 0.25], seed=10), {pcfg}, diff --git a/tests/pruning/type_constraints/test_pattern_constraints.py b/tests/pruning/type_constraints/test_pattern_constraints.py deleted file mode 100644 index f6ee73b0..00000000 --- a/tests/pruning/type_constraints/test_pattern_constraints.py +++ /dev/null @@ -1,133 +0,0 @@ -from collections import defaultdict -from typing import Set, Tuple -from synth.pruning.type_constraints.utils import get_prefix -from synth.syntax.grammars.cfg import CFG, CFGState -from synth.syntax.dsl import DSL -from synth.syntax.program import Primitive -from synth.syntax.type_system import INT, FunctionType, Type -from synth.pruning.type_constraints.pattern_constraints import ( - produce_new_syntax_for_constraints, -) - - -syntax = { - "+": FunctionType(INT, INT, INT), - "-": FunctionType(INT, INT, INT), - "*": FunctionType(INT, INT, INT), - "0": INT, - "1": INT, - "2": INT, -} -constraints = ["+ $(var0) _", "- (+ (+ _ _) _) _", "* ^+,-,* _"] -depth = 4 -type_request = FunctionType(INT, INT, INT) - - -def test_produce() -> None: - old_size = -1 - - for _ in range(2): - new_syntax, new_tr = produce_new_syntax_for_constraints( - syntax, constraints, type_request, progress=True - ) - size = CFG.depth_constraint(DSL(new_syntax), new_tr, depth).size() - if old_size == -1: - old_size = size - assert old_size == size - - -def test_var_constraints() -> None: - dsl = DSL(syntax) - p1 = dsl.parse_program("(+ var1 1)", type_request) - p2 = dsl.parse_program("(- var0 0)", type_request) - p3 = dsl.parse_program("(+ var0 0)", type_request) - new_syntax, new_tr = produce_new_syntax_for_constraints( - syntax, constraints[:1], type_request, progress=False - ) - cfg = CFG.depth_constraint(DSL(new_syntax), new_tr, depth) - assert p1 not in cfg - assert p2 not in cfg - assert p3 not in cfg - - -def test_forbid_constraints() -> None: - dsl = DSL(syntax) - p1 = dsl.parse_program("(* (+ 0 1) 1)", type_request) - p2 = dsl.parse_program("(* 0 (+ 1 1))", type_request) - p3 = dsl.parse_program("(* 2 (* (* 1 2) 2))", type_request) - new_syntax, new_tr = produce_new_syntax_for_constraints( - syntax, constraints[2:], type_request, progress=False - ) - cfg = CFG.depth_constraint(DSL(new_syntax), new_tr, depth) - p2 = cfg.embed(p2) - assert p1 not in cfg - assert p2 in cfg - assert p3 not in cfg - - -def test_nested_constraints() -> None: - dsl = DSL(syntax) - p1 = dsl.parse_program("(- (+ 0 1) 1)", type_request) - p2 = dsl.parse_program("(- 0 (+ (+ 1 1) 1))", type_request) - p3 = dsl.parse_program("(- (+ (+ 1 2) 2) 2)", type_request) - new_syntax, new_tr = produce_new_syntax_for_constraints( - syntax, constraints[1:2], type_request, progress=False - ) - cfg = CFG.depth_constraint(DSL(new_syntax), new_tr, depth) - assert p1 not in cfg - assert p2 not in cfg - assert p3 not in cfg - - -def __exist_equivalent_path_from_start__(cfg: CFG, pset: Set[Primitive]) -> bool: - plist = list(pset) - r = cfg.rules[cfg.start] - for i, p1 in enumerate(plist): - for p2 in plist[i + 1 :]: - if all( - __exist_equivalent_path__(cfg, arg1, arg2) - for arg1, arg2 in zip(r[p1][0], r[p2][0]) - ): - print(f"\tstart -> {p1} and start -> {p2}") - return True - return False - - -def __exist_equivalent_path__( - cfg: CFG, s1: Tuple[Type, CFGState], s2: Tuple[Type, CFGState] -) -> bool: - if s1 == s2: - return True - r1 = cfg.rules[(s1[0], (s1[1], None))] - r2 = cfg.rules[(s2[0], (s2[1], None))] - for P1 in r1: - for P2 in r2: - if P1 == P2: - print(f"\t{s1} -> {P1} and {s2} -> {P2}") - return True - if ( - isinstance(P1, Primitive) - and isinstance(P2, Primitive) - and get_prefix(P1.primitive) == get_prefix(P2.primitive) - ): - if all( - __exist_equivalent_path__(cfg, arg1, arg2) - for arg1, arg2 in zip(r1[P1][0], r2[P2][0]) - ): - print(f"\t{s1} -> {P1} and {s2} -> {P2}") - return True - return False - - -def test_unambiguous() -> None: - new_syntax, new_tr = produce_new_syntax_for_constraints( - syntax, constraints[1:2], type_request, progress=False - ) - cfg = CFG.depth_constraint(DSL(new_syntax), new_tr, depth) - equivalents = defaultdict(set) - for P in cfg.rules[cfg.start]: - if isinstance(P, Primitive): - equivalents[get_prefix(P.primitive)].add(P) - for plist in equivalents.values(): - if len(plist) > 1: - assert not __exist_equivalent_path_from_start__(cfg, plist) diff --git a/tests/pruning/type_constraints/test_sketch.py b/tests/pruning/type_constraints/test_sketch.py deleted file mode 100644 index 5b0325fe..00000000 --- a/tests/pruning/type_constraints/test_sketch.py +++ /dev/null @@ -1,132 +0,0 @@ -from collections import defaultdict -from typing import Set, Tuple -from synth.pruning.type_constraints.utils import get_prefix -from synth.syntax.grammars.cfg import CFG, CFGState -from synth.syntax.dsl import DSL -from synth.syntax.program import Primitive -from synth.syntax.type_system import INT, FunctionType, Type -from synth.pruning.type_constraints import ( - produce_new_syntax_for_sketch, -) - - -syntax = { - "+": FunctionType(INT, INT, INT), - "-": FunctionType(INT, INT, INT), - "*": FunctionType(INT, INT, INT), - "0": INT, - "1": INT, - "2": INT, -} -depth = 4 -type_request = FunctionType(INT, INT, INT) - - -def test_produce() -> None: - old_size = -1 - - for _ in range(2): - new_syntax, new_tr = produce_new_syntax_for_sketch( - syntax, "+ _ _", type_request - ) - size = CFG.depth_constraint(DSL(new_syntax), new_tr, depth).size() - if old_size == -1: - old_size = size - assert old_size == size - - -def test_var_constraints() -> None: - dsl = DSL(syntax) - p1 = dsl.parse_program("(+ var0 1)", type_request) - p2 = dsl.parse_program("(- 1 var0)", type_request) - p3 = dsl.parse_program("(+ (+ 1 var0) 0)", type_request) - new_syntax, new_tr = produce_new_syntax_for_sketch( - syntax, "(+ 1 $(var0))", type_request - ) - cfg = CFG.depth_constraint(DSL(new_syntax), new_tr, depth) - assert p1 not in cfg - assert p2 not in cfg - assert p3 not in cfg - - -def test_forbid_constraints() -> None: - dsl = DSL(syntax) - p1 = dsl.parse_program("(* (+ 0 1) 1)", type_request) - p2 = dsl.parse_program("(* 0 (+ 1 1))", type_request) - p3 = dsl.parse_program("(* 2 (* (* 1 2) 2))", type_request) - new_syntax, new_tr = produce_new_syntax_for_sketch( - syntax, "* ^+,-,* _", type_request - ) - cfg = CFG.depth_constraint(DSL(new_syntax), new_tr, depth) - p2 = cfg.embed(p2) - assert p1 not in cfg - assert p2 in cfg - assert p3 not in cfg - - -def test_nested_constraints() -> None: - dsl = DSL(syntax) - p1 = dsl.parse_program("(- (+ 0 1) 1)", type_request) - p2 = dsl.parse_program("(- 0 (+ (+ 1 1) 1))", type_request) - p3 = dsl.parse_program("(- (+ (+ 1 2) 2) 2)", type_request) - new_syntax, new_tr = produce_new_syntax_for_sketch( - syntax, "- (+ (+ _ _) _) _", type_request - ) - cfg = CFG.depth_constraint(DSL(new_syntax), new_tr, depth) - assert p1 not in cfg - assert p2 not in cfg - assert p3 not in cfg - - -def __exist_equivalent_path_from_start__(cfg: CFG, pset: Set[Primitive]) -> bool: - plist = list(pset) - r = cfg.rules[cfg.start] - for i, p1 in enumerate(plist): - for p2 in plist[i + 1 :]: - if all( - __exist_equivalent_path__(cfg, arg1, arg2) - for arg1, arg2 in zip(r[p1][0], r[p2][0]) - ): - print(f"\tstart -> {p1} and start -> {p2}") - return True - return False - - -def __exist_equivalent_path__( - cfg: CFG, s1: Tuple[Type, CFGState], s2: Tuple[Type, CFGState] -) -> bool: - if s1 == s2: - return True - r1 = cfg.rules[(s1[0], (s1[1], None))] - r2 = cfg.rules[(s2[0], (s2[1], None))] - for P1 in r1: - for P2 in r2: - if P1 == P2: - print(f"\t{s1} -> {P1} and {s2} -> {P2}") - return True - if ( - isinstance(P1, Primitive) - and isinstance(P2, Primitive) - and get_prefix(P1.primitive) == get_prefix(P2.primitive) - ): - if all( - __exist_equivalent_path__(cfg, arg1, arg2) - for arg1, arg2 in zip(r1[P1][0], r2[P2][0]) - ): - print(f"\t{s1} -> {P1} and {s2} -> {P2}") - return True - return False - - -def test_unambiguous() -> None: - new_syntax, new_tr = produce_new_syntax_for_sketch( - syntax, "- (+ (+ _ _) _) _", type_request - ) - cfg = CFG.depth_constraint(DSL(new_syntax), new_tr, depth) - equivalents = defaultdict(set) - for P in cfg.rules[cfg.start]: - if isinstance(P, Primitive): - equivalents[get_prefix(P.primitive)].add(P) - for plist in equivalents.values(): - if len(plist) > 1: - assert not __exist_equivalent_path_from_start__(cfg, plist) diff --git a/tests/pruning/type_constraints/test_utils.py b/tests/pruning/type_constraints/test_utils.py deleted file mode 100644 index 0284973b..00000000 --- a/tests/pruning/type_constraints/test_utils.py +++ /dev/null @@ -1,42 +0,0 @@ -from synth.syntax.type_system import BOOL, INT, FunctionType, PrimitiveType -from synth.pruning.type_constraints.utils import clean, export_syntax_to_python - -BOOL_0 = PrimitiveType("bool@0") -syntax = { - "+": FunctionType(INT, INT, INT), - "-": FunctionType(INT, INT, INT), - "*": FunctionType(INT, INT, INT), - "and@0": FunctionType(BOOL, BOOL, BOOL_0), - "or@0": FunctionType(BOOL, BOOL, BOOL_0), - "not@0": FunctionType(BOOL, BOOL_0), - "<=@0": FunctionType(INT, INT, BOOL_0), - "==@0": FunctionType(INT, INT, BOOL_0), - "<=": FunctionType(INT, INT, BOOL), - "==": FunctionType(INT, INT, BOOL), - "and": FunctionType(BOOL, BOOL, BOOL), - "or": FunctionType(BOOL, BOOL, BOOL), - "not": FunctionType(BOOL, BOOL), - "ite": FunctionType(BOOL, INT, INT, INT), - "0": INT, - "1": INT, - "2": INT, -} - - -def test_export() -> None: - out = export_syntax_to_python(syntax, varname="koala") - out = out.replace("int", "INT") - out = out.replace("bool", "BOOL") - out = out.replace("BOOL@0", "bool@0") - imports = "from synth.syntax.type_system import BOOL, INT, FunctionType, PrimitiveType, Arrow\n" - exec(imports + out) - koala = eval(out[out.rfind("= {") + 2 :]) - assert koala == syntax - - -def test_clean() -> None: - new_syntax = {k: v for k, v in syntax.items()} - clean(new_syntax, None) - for P in syntax.keys(): - if "@" in P: - assert P not in new_syntax diff --git a/tests/semantic/test_evaluator.py b/tests/semantic/test_evaluator.py index 50bc2574..914dfc3e 100644 --- a/tests/semantic/test_evaluator.py +++ b/tests/semantic/test_evaluator.py @@ -5,15 +5,16 @@ from synth.syntax.type_system import ( INT, STRING, - FunctionType, List, PolymorphicType, PrimitiveType, ) +from synth.syntax.type_helper import FunctionType syntax = { "+1": FunctionType(INT, INT), + "0": INT, "head": FunctionType(List(PolymorphicType("a")), PolymorphicType("a")), "non_reachable": PrimitiveType("non_reachable"), "non_productive": FunctionType(INT, STRING), @@ -21,49 +22,84 @@ semantics = { "+1": lambda x: x + 1, + "0": 0, } max_depth = 4 dsl = DSL(syntax) cfg = CFG.depth_constraint(dsl, FunctionType(INT, INT), max_depth) +other_syntax = {"+1": FunctionType(INT, INT), "0": INT, "2": INT, "True": STRING} + +other_semantics = { + "+1": lambda x: x + 1, + "0": 0, + "True": True, + "2": 2, +} +other_dsl = DSL(other_syntax) +other_cfg = CFG.depth_constraint(other_dsl, FunctionType(INT, INT), max_depth) + def test_eval() -> None: - eval = DSLEvaluator(semantics) + eval = DSLEvaluator(dsl.instantiate_semantics(semantics)) pcfg = ProbDetGrammar.uniform(cfg) pcfg.init_sampling(0) for _ in range(100): program = pcfg.sample_program() try: for i in range(-25, 25): - assert eval.eval(program, [i]) == program.length() + i - 1 + if len(program.used_variables()) == 0: + assert eval.eval(program, [i]) == program.size() - 1 + else: + assert eval.eval(program, [i]) == program.size() + i - 1 except Exception as e: assert False, e def test_supports_list() -> None: - eval = DSLEvaluator(semantics) + eval = DSLEvaluator(dsl.instantiate_semantics(semantics)) pcfg = ProbDetGrammar.uniform(cfg) pcfg.init_sampling(0) for _ in range(100): program = pcfg.sample_program() try: for i in range(-25, 25): - assert eval.eval(program, [i, [i]]) == program.length() + i - 1 + if len(program.used_variables()) == 0: + assert eval.eval(program, [i]) == program.size() - 1 + else: + assert eval.eval(program, [i]) == program.size() + i - 1 except Exception as e: assert False, e def test_use_cache() -> None: - eval = DSLEvaluator(semantics) + eval = DSLEvaluator(dsl.instantiate_semantics(semantics)) pcfg = ProbDetGrammar.uniform(cfg) pcfg.init_sampling(0) for _ in range(100): program = pcfg.sample_program() try: for i in range(-25, 25): - assert eval.eval(program, [i]) == program.length() + i - 1 - assert ( - eval._cache[__tuplify__([i])][program] == program.length() + i - 1 - ) + if len(program.used_variables()) == 0: + assert eval.eval(program, [i]) == program.size() - 1 + assert eval._cache[__tuplify__([i])][program] == program.size() - 1 + else: + assert eval.eval(program, [i]) == program.size() + i - 1 + assert ( + eval._cache[__tuplify__([i])][program] == program.size() + i - 1 + ) except Exception as e: assert False, e + + +def test_compress() -> None: + eval = DSLEvaluator(other_dsl.instantiate_semantics(other_semantics)) + p = other_dsl.auto_parse_program("(+1 0)") + pp = other_dsl.auto_parse_program("1", constants={"1": (INT, 1)}) + c = eval.compress(p) + assert c != p + assert c == pp + p = other_dsl.auto_parse_program("(+1 (+1 0))") + pp = other_dsl.auto_parse_program("2") + c = eval.compress(p) + assert c == pp diff --git a/tests/syntax/automata/test_tree_automaton.py b/tests/syntax/automata/test_tree_automaton.py new file mode 100644 index 00000000..83d23e25 --- /dev/null +++ b/tests/syntax/automata/test_tree_automaton.py @@ -0,0 +1,102 @@ +from itertools import product +from typing import Dict, Set, Tuple + +from synth.syntax.dsl import DSL +from synth.syntax.type_system import ( + INT, + STRING, + List, + PolymorphicType, + PrimitiveType, + Type, +) +from synth.syntax.type_helper import FunctionType +from synth.syntax.grammars.grammar import DerivableProgram, NGram +from synth.syntax.automata.tree_automaton import DFTA +from synth.syntax.grammars.cfg import CFG + + +def cfg2dfta( + grammar: CFG, +) -> DFTA[Tuple[Type, int], DerivableProgram]: + StateT = Tuple[Type, int] + dfta_rules: Dict[Tuple[DerivableProgram, Tuple[StateT, ...]], StateT] = {} + max_depth = grammar.max_program_depth() + all_cases: Dict[ + Tuple[int, Tuple[Type, ...]], Set[Tuple[Tuple[Type, int], ...]] + ] = {} + for S in grammar.rules: + for P in grammar.rules[S]: + args = grammar.rules[S][P][0] + if len(args) == 0: + dfta_rules[(P, ())] = (P.type, 0) + else: + key = (len(args), tuple([arg[0] for arg in args])) + if key not in all_cases: + all_cases[key] = set( + [ + tuple(x) + for x in product( + *[ + [(arg[0], j) for j in range(max_depth)] + for arg in args + ] + ) + ] + ) + for nargs in all_cases[key]: + dfta_rules[(P, nargs)] = ( + S[0], + max(i for _, i in nargs) + 1, + ) + r = grammar.type_request.returns() + return DFTA(dfta_rules, {(r, x) for x in range(max_depth)}) + + +import pytest + +syntax = { + "+": FunctionType(INT, INT, INT), + "head": FunctionType(List(PolymorphicType("a")), PolymorphicType("a")), + "non_reachable": PrimitiveType("non_reachable"), + "1": INT, + "non_productive": FunctionType(INT, STRING), +} +dsl = DSL(syntax) +max_depths = [3, 7, 11] + + +@pytest.mark.parametrize("max_depth", max_depths) +def test_reduce(max_depth: int) -> None: + cfg = CFG.depth_constraint(dsl, FunctionType(INT, INT), max_depth, n_gram=1) + dfta = cfg2dfta(cfg) + dfta.reduce() + for (P, args), dst in dfta.rules.items(): + assert not ( + all(x == 0 for x in args) and len(args) > 0 + ), f"Unreachable rule: {P} {args}" + assert dst != max_depth, f"Unproductive rule: {P} {args} -> {dst}" + + +@pytest.mark.parametrize("max_depth", max_depths) +def test_states(max_depth: int) -> None: + cfg = CFG.depth_constraint(dsl, FunctionType(INT, INT), max_depth, n_gram=1) + dfta = cfg2dfta(cfg) + dfta.reduce() + for (P, args), dst in dfta.rules.items(): + if dst[1] < 1: + continue + state = (dst[0], ((NGram(1), dst[1] - 1), None)) + assert state in cfg.rules + assert P in cfg.rules[state] + assert all(a[0] == b[0] for a, b in zip(args, cfg.rules[state][P][0])) + + +@pytest.mark.parametrize("max_depth", max_depths) +def test_minimise(max_depth: int) -> None: + cfg = CFG.depth_constraint(dsl, FunctionType(INT, INT), max_depth) + dfta = cfg2dfta(cfg) + dfta.reduce() + ndfta = dfta.minimise() + for P, args in ndfta.rules: + assert not (all(x == (0,) for x in args) and len(args) > 0) diff --git a/tests/syntax/grammars/enumeration/test_a_star.py b/tests/syntax/grammars/enumeration/test_a_star.py new file mode 100644 index 00000000..e61b7908 --- /dev/null +++ b/tests/syntax/grammars/enumeration/test_a_star.py @@ -0,0 +1,71 @@ +from synth.syntax.grammars.enumeration.a_star import ( + enumerate_prob_grammar, +) +from synth.syntax.grammars.cfg import CFG +from synth.syntax.grammars.ttcfg import TTCFG +from synth.syntax.grammars.tagged_det_grammar import ProbDetGrammar +from synth.syntax.dsl import DSL +from synth.syntax.type_system import ( + INT, + STRING, + List, + PolymorphicType, + PrimitiveType, +) +from synth.syntax.type_helper import FunctionType, auto_type + +import pytest + + +syntax = { + "+": FunctionType(INT, INT, INT), + "head": FunctionType(List(PolymorphicType("a")), PolymorphicType("a")), + "non_reachable": PrimitiveType("non_reachable"), + "1": INT, + "2": INT, + "non_productive": FunctionType(INT, STRING), +} +dsl = DSL(syntax) +dsl.instantiate_polymorphic_types() +testdata = [ + CFG.depth_constraint(dsl, FunctionType(INT, INT), 3), + CFG.depth_constraint(dsl, FunctionType(INT, INT), 4), +] + + +@pytest.mark.parametrize("cfg", testdata) +def test_unicity_a_star(cfg: TTCFG) -> None: + pcfg = ProbDetGrammar.uniform(cfg) + seen = set() + print(cfg) + for program in enumerate_prob_grammar(pcfg): + assert program not in seen + seen.add(program) + # print(pcfg.grammar) + assert len(seen) == cfg.programs() + + +@pytest.mark.parametrize("cfg", testdata) +def test_order_a_star(cfg: TTCFG) -> None: + pcfg = ProbDetGrammar.uniform(cfg) + last = 1.0 + for program in enumerate_prob_grammar(pcfg): + p = pcfg.probability(program) + assert p <= last + last = p + + +def test_infinite() -> None: + pcfg = ProbDetGrammar.random( + CFG.infinite(dsl, testdata[0].type_request, n_gram=1), 1 + ) + count = 10000 + last = 1.0 + for program in enumerate_prob_grammar(pcfg): + count -= 1 + p = pcfg.probability(program) + assert -1e-12 <= last - p, f"failed at program n°{count}:{program}" + last = p + if count < 0: + break + assert count == -1 diff --git a/tests/syntax/grammars/enumeration/test_beap_search.py b/tests/syntax/grammars/enumeration/test_beap_search.py new file mode 100644 index 00000000..85b3292d --- /dev/null +++ b/tests/syntax/grammars/enumeration/test_beap_search.py @@ -0,0 +1,89 @@ +from synth.syntax.grammars.enumeration.beap_search import ( + enumerate_prob_grammar, +) +from synth.syntax.grammars.cfg import CFG +from synth.syntax.grammars.ttcfg import TTCFG +from synth.syntax.grammars.tagged_det_grammar import ProbDetGrammar +from synth.syntax.dsl import DSL +from synth.syntax.type_system import ( + INT, + STRING, + List, + PolymorphicType, + PrimitiveType, +) +from synth.syntax.type_helper import FunctionType, auto_type + +import pytest + + +syntax = { + "+": FunctionType(INT, INT, INT), + "head": FunctionType(List(PolymorphicType("a")), PolymorphicType("a")), + "non_reachable": PrimitiveType("non_reachable"), + "1": INT, + "2": INT, + "non_productive": FunctionType(INT, STRING), +} +dsl = DSL(syntax) +dsl.instantiate_polymorphic_types() +testdata = [ + CFG.depth_constraint(dsl, FunctionType(INT, INT), 3), + CFG.depth_constraint(dsl, FunctionType(INT, INT), 4), +] + + +@pytest.mark.parametrize("cfg", testdata) +def test_unicity_beep_search(cfg: TTCFG) -> None: + pcfg = ProbDetGrammar.uniform(cfg) + seen = set() + for program in enumerate_prob_grammar(pcfg): + assert program not in seen + seen.add(program) + # print(pcfg.grammar) + assert len(seen) == cfg.programs() + + +@pytest.mark.parametrize("cfg", testdata) +def test_order_beep_search(cfg: TTCFG) -> None: + pcfg = ProbDetGrammar.uniform(cfg) + last = 1.0 + for program in enumerate_prob_grammar(pcfg): + p = pcfg.probability(program) + assert p <= last + last = p + + +def test_infinite() -> None: + pcfg = ProbDetGrammar.random( + CFG.infinite(dsl, testdata[0].type_request, n_gram=1), 1 + ) + count = 10000 + last = 1.0 + for program in enumerate_prob_grammar(pcfg): + count -= 1 + p = pcfg.probability(program) + assert -1e-12 <= last - p, f"failed at program n°{count}:{program}" + last = p + if count < 0: + break + assert count == -1 + + +@pytest.mark.parametrize("cfg", testdata) +def test_merge(cfg: TTCFG) -> None: + pcfg = ProbDetGrammar.uniform(cfg) + seen = set() + for program in enumerate_prob_grammar(pcfg): + assert program not in seen + seen.add(program) + en = enumerate_prob_grammar(pcfg) + removed = dsl.parse_program("(+ 1 1)", auto_type("int")) + en.merge_program(dsl.parse_program("2", auto_type("int")), removed) + new_seen = set() + for program in en: + assert removed not in program + new_seen.add(program) + diff = seen.difference(new_seen) + for x in diff: + assert removed in x diff --git a/tests/syntax/grammars/enumeration/test_bee_search.py b/tests/syntax/grammars/enumeration/test_bee_search.py new file mode 100644 index 00000000..58f679f3 --- /dev/null +++ b/tests/syntax/grammars/enumeration/test_bee_search.py @@ -0,0 +1,89 @@ +from synth.syntax.grammars.enumeration.bee_search import ( + enumerate_prob_grammar, +) +from synth.syntax.grammars.cfg import CFG +from synth.syntax.grammars.ttcfg import TTCFG +from synth.syntax.grammars.tagged_det_grammar import ProbDetGrammar +from synth.syntax.dsl import DSL +from synth.syntax.type_system import ( + INT, + STRING, + List, + PolymorphicType, + PrimitiveType, +) +from synth.syntax.type_helper import FunctionType, auto_type + +import pytest + + +syntax = { + "+": FunctionType(INT, INT, INT), + "head": FunctionType(List(PolymorphicType("a")), PolymorphicType("a")), + "non_reachable": PrimitiveType("non_reachable"), + "1": INT, + "2": INT, + "non_productive": FunctionType(INT, STRING), +} +dsl = DSL(syntax) +dsl.instantiate_polymorphic_types() +testdata = [ + CFG.depth_constraint(dsl, FunctionType(INT, INT), 3), + CFG.depth_constraint(dsl, FunctionType(INT, INT), 4), +] + + +@pytest.mark.parametrize("cfg", testdata) +def test_unicity_beeSearch(cfg: TTCFG) -> None: + pcfg = ProbDetGrammar.uniform(cfg) + seen = set() + for program in enumerate_prob_grammar(pcfg): + assert program not in seen + seen.add(program) + # print(pcfg.grammar) + assert len(seen) == cfg.programs() + + +@pytest.mark.parametrize("cfg", testdata) +def test_order_beeSearch(cfg: TTCFG) -> None: + pcfg = ProbDetGrammar.uniform(cfg) + last = 1.0 + for program in enumerate_prob_grammar(pcfg): + p = pcfg.probability(program) + assert p <= last + last = p + + +@pytest.mark.parametrize("cfg", testdata) +def test_merge(cfg: TTCFG) -> None: + pcfg = ProbDetGrammar.uniform(cfg) + seen = set() + for program in enumerate_prob_grammar(pcfg): + assert program not in seen + seen.add(program) + en = enumerate_prob_grammar(pcfg) + removed = dsl.parse_program("(+ 1 1)", auto_type("int")) + en.merge_program(dsl.parse_program("2", auto_type("int")), removed) + new_seen = set() + for program in en: + assert removed not in program + new_seen.add(program) + diff = seen.difference(new_seen) + for x in diff: + assert removed in x + + +def test_infinite() -> None: + pcfg = ProbDetGrammar.random( + CFG.infinite(dsl, testdata[0].type_request, n_gram=1), 1 + ) + count = 10000 + last = 1.0 + for program in enumerate_prob_grammar(pcfg): + count -= 1 + p = pcfg.probability(program) + assert -1e-12 <= last - p, f"failed at program n°{count}:{program}" + last = p + if count < 0: + break + assert count == -1 diff --git a/tests/syntax/grammars/enumeration/test_constant_delay.py b/tests/syntax/grammars/enumeration/test_constant_delay.py new file mode 100644 index 00000000..a569dcab --- /dev/null +++ b/tests/syntax/grammars/enumeration/test_constant_delay.py @@ -0,0 +1,87 @@ +from synth.syntax.grammars.enumeration.constant_delay import ( + enumerate_prob_grammar, +) +from synth.syntax.grammars.enumeration.beap_search import ( + enumerate_prob_grammar as enumerate, +) + +from synth.syntax.grammars.cfg import CFG +from synth.syntax.grammars.ttcfg import TTCFG +from synth.syntax.grammars.tagged_det_grammar import ProbDetGrammar +from synth.syntax.dsl import DSL +from synth.syntax.type_system import ( + INT, + STRING, + List, + PolymorphicType, + PrimitiveType, +) +from synth.syntax.type_helper import FunctionType, auto_type + +import numpy as np + +import pytest + + +syntax = { + "+": FunctionType(INT, INT, INT), + "head": FunctionType(List(PolymorphicType("a")), PolymorphicType("a")), + "non_reachable": PrimitiveType("non_reachable"), + "1": INT, + "2": INT, + "non_productive": FunctionType(INT, STRING), +} +dsl = DSL(syntax) +dsl.instantiate_polymorphic_types() +testdata = [ + CFG.depth_constraint(dsl, FunctionType(INT, INT), 3), + CFG.depth_constraint(dsl, FunctionType(INT, INT), 4), +] +kvals = [4, 16, 64] +precision = [1e-2, 1e-4, 1e-8] + + +@pytest.mark.parametrize("cfg", testdata) +@pytest.mark.parametrize("k", kvals) +@pytest.mark.parametrize("precis", precision) +def test_unicity_beep_search(cfg: TTCFG, k: int, precis: float) -> None: + pcfg = ProbDetGrammar.uniform(cfg) + seen = set() + for program in enumerate_prob_grammar(pcfg, k, precis): + assert program not in seen + seen.add(program) + # print(pcfg.grammar) + assert len(seen) == cfg.programs() + + +@pytest.mark.parametrize("cfg", testdata) +@pytest.mark.parametrize("k", kvals) +@pytest.mark.parametrize("precis", precision) +def test_order_beep_search(cfg: TTCFG, k: int, precis: float) -> None: + # pcfg = ProbDetGrammar.uniform(cfg) + pcfg = ProbDetGrammar.random(cfg, seed=4) + last = 1.0 + for program in enumerate_prob_grammar(pcfg, k, precis): + p = pcfg.probability(program) + assert p <= last or abs(p / last) <= 1 + precis * (2**4) + last = p + + +@pytest.mark.parametrize("k", kvals) +@pytest.mark.parametrize("precis", precision) +def test_infinite(k: int, precis: float) -> None: + pcfg = ProbDetGrammar.random( + CFG.infinite(dsl, testdata[0].type_request, n_gram=1), 1 + ) + count = 10000 + last = 1.0 + for program in enumerate_prob_grammar(pcfg, k, precis): + count -= 1 + p = pcfg.probability(program) + assert ( + -1e-12 <= last - p or abs(p / last) <= 1 + precis + ), f"failed at program n°{count}:{program}, p={p} last={last}, p={np.log(p)} last={np.log(last)}" + last = p + if count < 0: + break + assert count == -1 diff --git a/tests/syntax/grammars/enumeration/test_grammar_splitter.py b/tests/syntax/grammars/enumeration/test_grammar_splitter.py new file mode 100644 index 00000000..1a7de8f4 --- /dev/null +++ b/tests/syntax/grammars/enumeration/test_grammar_splitter.py @@ -0,0 +1,54 @@ +import pytest +from synth.syntax.grammars.enumeration.u_heap_search import enumerate_prob_u_grammar +from synth.syntax.grammars.enumeration.grammar_splitter import split +from synth.syntax.grammars.tagged_u_grammar import ProbUGrammar +from synth.syntax.grammars.u_cfg import UCFG +from synth.syntax.dsl import DSL +from synth.syntax.type_system import ( + INT, + STRING, + List, + PolymorphicType, + PrimitiveType, +) +from synth.syntax.type_helper import FunctionType + + +syntax = { + "+": FunctionType(INT, INT, INT), + "-": FunctionType(INT, INT, INT), + "head": FunctionType(List(PolymorphicType("a")), PolymorphicType("a")), + "non_reachable": PrimitiveType("non_reachable"), + "1": INT, + "non_productive": FunctionType(INT, STRING), +} +dsl = DSL(syntax) +pucfg = ProbUGrammar.uniform(UCFG.depth_constraint(dsl, FunctionType(INT, INT), 3)) +testdata = list(range(2, 5)) +seen = set() +for program in enumerate_prob_u_grammar(pucfg): + seen.add(program) + + +@pytest.mark.parametrize("splits", testdata) +def test_unicity(splits: int) -> None: + fragments, _ = split(pucfg, splits, desired_ratio=1.05) + seen = set() + for sub_pcfg in fragments: + print(sub_pcfg) + for program in enumerate_prob_u_grammar(sub_pcfg): + assert program not in seen + seen.add(program) + + +@pytest.mark.parametrize("splits", testdata) +def test_none_missing(splits: int) -> None: + fragments, _ = split(pucfg, splits, desired_ratio=1.05) + new_seen = set() + for sub_pcfg in fragments: + a = set() + for program in enumerate_prob_u_grammar(sub_pcfg): + a.add(program) + new_seen |= a + assert len(new_seen.difference(seen)) == 0, new_seen.difference(seen) + assert len(seen.difference(new_seen)) == 0, seen.difference(new_seen) diff --git a/tests/syntax/grammars/enumeration/test_heap_search.py b/tests/syntax/grammars/enumeration/test_heap_search.py new file mode 100644 index 00000000..54f68ee2 --- /dev/null +++ b/tests/syntax/grammars/enumeration/test_heap_search.py @@ -0,0 +1,136 @@ +from synth.syntax.grammars.enumeration.heap_search import ( + Bucket, + enumerate_prob_grammar, + enumerate_bucket_prob_grammar, +) +from synth.syntax.grammars.cfg import CFG +from synth.syntax.grammars.ttcfg import TTCFG +from synth.syntax.grammars.tagged_det_grammar import ProbDetGrammar +from synth.syntax.dsl import DSL +from synth.syntax.type_system import ( + INT, + STRING, + List, + PolymorphicType, + PrimitiveType, +) +from synth.syntax.type_helper import FunctionType, auto_type + +import pytest + + +syntax = { + "+": FunctionType(INT, INT, INT), + "head": FunctionType(List(PolymorphicType("a")), PolymorphicType("a")), + "non_reachable": PrimitiveType("non_reachable"), + "1": INT, + "2": INT, + "non_productive": FunctionType(INT, STRING), +} +dsl = DSL(syntax) +dsl.instantiate_polymorphic_types() +testdata = [ + CFG.depth_constraint(dsl, FunctionType(INT, INT), 3), + TTCFG.size_constraint(dsl, FunctionType(INT, INT), 5), +] + + +@pytest.mark.parametrize("cfg", testdata) +def test_unicity_heapSearch(cfg: TTCFG) -> None: + pcfg = ProbDetGrammar.uniform(cfg) + seen = set() + for program in enumerate_prob_grammar(pcfg): + assert program not in seen + seen.add(program) + assert len(seen) == cfg.programs() + + +@pytest.mark.parametrize("cfg", testdata) +def test_order_heapSearch(cfg: TTCFG) -> None: + pcfg = ProbDetGrammar.uniform(cfg) + last = 1.0 + for program in enumerate_prob_grammar(pcfg): + p = pcfg.probability(program) + assert p <= last + last = p + + +@pytest.mark.parametrize("cfg", testdata) +def test_threshold(cfg: TTCFG) -> None: + pcfg = ProbDetGrammar.uniform(cfg) + threshold = 0.15 + seen = set() + for program in enumerate_prob_grammar(pcfg): + p = pcfg.probability(program) + if p <= threshold: + break + seen.add(p) + seent = set() + for program in enumerate_prob_grammar(pcfg, threshold): + p = pcfg.probability(program) + assert p > threshold + seent.add(p) + + assert len(seent.symmetric_difference(seen)) == 0 + + +@pytest.mark.parametrize("cfg", testdata) +def test_unicity_bucketSearch(cfg: TTCFG) -> None: + pcfg = ProbDetGrammar.uniform(cfg) + for bucketSize in range(3, 10): + seen = set() + for program in enumerate_bucket_prob_grammar(pcfg, bucket_size=bucketSize): + assert program not in seen + seen.add(program) + assert len(seen) == cfg.programs() + + +@pytest.mark.parametrize("cfg", testdata) +def test_order_bucketSearch(cfg: TTCFG) -> None: + pcfg = ProbDetGrammar.uniform(cfg) + for bucketSize in range(3, 10): + last = Bucket(bucketSize) + for program in enumerate_bucket_prob_grammar(pcfg, bucket_size=bucketSize): + p = pcfg.reduce_derivations( + lambda b, S, P, _: b.add_prob_uniform(pcfg.probabilities[S][P]), + Bucket(bucketSize), + program, + ) + assert p.size == bucketSize + assert p >= last or last == Bucket(bucketSize) + last = p + + +@pytest.mark.parametrize("cfg", testdata) +def test_merge(cfg: TTCFG) -> None: + pcfg = ProbDetGrammar.uniform(cfg) + seen = set() + for program in enumerate_prob_grammar(pcfg): + assert program not in seen + seen.add(program) + en = enumerate_prob_grammar(pcfg) + removed = dsl.parse_program("(+ 1 1)", auto_type("int")) + en.merge_program(dsl.parse_program("2", auto_type("int")), removed) + new_seen = set() + for program in en: + assert removed not in program + new_seen.add(program) + diff = seen.difference(new_seen) + for x in diff: + assert removed in x + + +def test_infinite() -> None: + pcfg = ProbDetGrammar.random( + CFG.infinite(dsl, testdata[0].type_request, n_gram=1), 1 + ) + count = 10000 + last = 1.0 + for program in enumerate_prob_grammar(pcfg): + count -= 1 + p = pcfg.probability(program) + assert -1e-12 <= last - p, f"failed at program n°{count}:{program}" + last = p + if count < 0: + break + assert count == -1 diff --git a/tests/syntax/grammars/enumeration/test_u_heap_search.py b/tests/syntax/grammars/enumeration/test_u_heap_search.py new file mode 100644 index 00000000..7eba0910 --- /dev/null +++ b/tests/syntax/grammars/enumeration/test_u_heap_search.py @@ -0,0 +1,157 @@ +from typing import TypeVar +from synth.filter.constraints import add_dfta_constraints +from synth.syntax.grammars.cfg import CFG +from synth.syntax.grammars.enumeration.heap_search import enumerate_prob_grammar +from synth.syntax.grammars.enumeration.u_heap_search import ( + Bucket, + enumerate_prob_u_grammar, + enumerate_bucket_prob_u_grammar, +) +from synth.syntax.grammars.tagged_det_grammar import ProbDetGrammar +from synth.syntax.grammars.u_cfg import UCFG +from synth.syntax.grammars.u_cfg import UCFG +from synth.syntax.grammars.tagged_u_grammar import ProbUGrammar +from synth.syntax.dsl import DSL +from synth.syntax.type_system import ( + INT, + STRING, + List, + PolymorphicType, + PrimitiveType, +) +from synth.syntax.type_helper import FunctionType, auto_type + +import pytest + + +syntax = { + "+": FunctionType(INT, INT, INT), + "-": FunctionType(INT, INT, INT), + "head": FunctionType(List(PolymorphicType("a")), PolymorphicType("a")), + "non_reachable": PrimitiveType("non_reachable"), + "1": INT, + "2": INT, + "0": INT, + "non_productive": FunctionType(INT, STRING), +} +dsl = DSL(syntax) +dsl.instantiate_polymorphic_types() +testdata = [ + UCFG.depth_constraint(dsl, FunctionType(INT, INT), 3), + UCFG.from_DFTA_with_ngrams( + add_dfta_constraints( + CFG.depth_constraint(dsl, FunctionType(INT, INT), 3), + ["(+ 1 ^0)", "(- _ ^0)"], + ), + 2, + ), +] + + +def test_equality() -> None: + base = CFG.depth_constraint(dsl, FunctionType(INT, INT), 3, min_variable_depth=0) + ucfg = UCFG.from_DFTA_with_ngrams( + add_dfta_constraints(base, [], progress=False), + 2, + ) + + seen = set() + for program in enumerate_prob_u_grammar(ProbUGrammar.uniform(ucfg)): + seen.add(program) + seen2 = set() + for program in enumerate_prob_grammar(ProbDetGrammar.uniform(base)): + seen2.add(program) + assert len(seen.difference(seen2)) == 0 + + +@pytest.mark.parametrize("cfg", testdata) +def test_unicity_heapSearch(cfg: UCFG) -> None: + pcfg = ProbUGrammar.uniform(cfg) + seen = set() + for program in enumerate_prob_u_grammar(pcfg): + assert program not in seen + seen.add(program) + assert len(seen) == cfg.programs() + + +@pytest.mark.parametrize("cfg", testdata) +def test_order_heapSearch(cfg: UCFG) -> None: + pcfg = ProbUGrammar.uniform(cfg) + last = 1.0 + for program in enumerate_prob_u_grammar(pcfg): + p = pcfg.probability(program) + assert (p - last) <= 1e-7 + last = p + + +@pytest.mark.parametrize("cfg", testdata) +def test_threshold(cfg: UCFG) -> None: + pcfg = ProbUGrammar.uniform(cfg) + threshold = 0.15 + seen = set() + for program in enumerate_prob_u_grammar(pcfg): + p = pcfg.probability(program) + if p <= threshold: + break + seen.add(p) + seent = set() + for program in enumerate_prob_u_grammar(pcfg, threshold): + p = pcfg.probability(program) + assert p > threshold + seent.add(p) + + assert len(seent.symmetric_difference(seen)) == 0 + + +@pytest.mark.parametrize("cfg", testdata) +def test_unicity_bucketSearch(cfg: UCFG) -> None: + pcfg = ProbUGrammar.uniform(cfg) + for bucketSize in range(3, 10): + seen = set() + for program in enumerate_bucket_prob_u_grammar(pcfg, bucket_size=bucketSize): + assert program not in seen + seen.add(program) + assert len(seen) == cfg.programs() + + +U = TypeVar("U") + + +@pytest.mark.parametrize("cfg", testdata) +def test_order_bucketSearch(cfg: UCFG[U]) -> None: + pcfg = ProbUGrammar.uniform(cfg) + for bucketSize in range(3, 10): + last = Bucket(bucketSize) + for program in enumerate_bucket_prob_u_grammar(pcfg, bucket_size=bucketSize): + outs = pcfg.reduce_derivations( + lambda b, S, P, v: b.add_prob_uniform( + pcfg.probabilities[S][P][tuple(v)] + ), + Bucket(bucketSize), + program, + ) + assert len(outs) == 1 + p = outs[0] + assert p.size == bucketSize + assert p >= last or last == Bucket(bucketSize) + last = p + + +@pytest.mark.parametrize("cfg", testdata) +def test_merge(cfg: UCFG[U]) -> None: + pcfg = ProbUGrammar.uniform(cfg) + seen = set() + for program in enumerate_prob_u_grammar(pcfg): + assert program not in seen + seen.add(program) + en = enumerate_prob_u_grammar(pcfg) + removed = dsl.parse_program("(- 1 1)", auto_type("int")) + en.merge_program(dsl.parse_program("0", auto_type("int")), removed) + new_seen = set() + print(cfg) + for program in en: + assert removed not in program + new_seen.add(program) + diff = seen.difference(new_seen) + for x in diff: + assert removed in x diff --git a/tests/syntax/grammars/test_cfg.py b/tests/syntax/grammars/test_cfg.py index 2c404f33..00d98d08 100644 --- a/tests/syntax/grammars/test_cfg.py +++ b/tests/syntax/grammars/test_cfg.py @@ -5,12 +5,13 @@ INT, STRING, Arrow, - FunctionType, List, PolymorphicType, PrimitiveType, ) +from synth.syntax.type_helper import FunctionType +import pytest syntax = { "+": FunctionType(INT, INT, INT), @@ -19,39 +20,50 @@ "1": INT, "non_productive": FunctionType(INT, STRING), } +dsl = DSL(syntax) +max_depths = [3, 7, 11] def test_function_as_variable() -> None: dsl = DSL(syntax) max_depth = 5 cfg = CFG.depth_constraint(dsl, FunctionType(Arrow(INT, INT), INT), max_depth) - assert cfg.size() > 0 + assert cfg.programs() > 0 -def test_clean() -> None: - dsl = DSL(syntax) - for max_depth in [3, 7, 11]: - cfg = CFG.depth_constraint(dsl, FunctionType(INT, INT), max_depth) - for rule in cfg.rules: - assert rule[1][0][1] <= max_depth - for P in cfg.rules[rule]: - if isinstance(P, Primitive): - assert P.primitive != "non_reachable" - assert P.primitive != "non_productive" - assert P.primitive != "head" - - -def test_depth_constraint() -> None: - dsl = DSL(syntax) - for max_depth in [3, 7, 11]: - cfg = CFG.depth_constraint(dsl, FunctionType(INT, INT), max_depth) - res = dsl.parse_program("(+ 1 var0)", FunctionType(INT, INT)) - print(cfg) - while res.depth() <= max_depth: - assert ( - res in cfg - ), f"Program depth:{res.depth()} should be in the TTCFG max_depth:{max_depth}" - res = dsl.parse_program(f"(+ {res} var0)", FunctionType(INT, INT)) +@pytest.mark.parametrize("max_depth", max_depths) +def test_clean(max_depth: int) -> None: + cfg = CFG.depth_constraint(dsl, FunctionType(INT, INT), max_depth) + for rule in cfg.rules: + assert rule[1][0][1] <= max_depth + for P in cfg.rules[rule]: + if isinstance(P, Primitive): + assert P.primitive != "non_reachable" + assert P.primitive != "non_productive" + assert P.primitive != "head" + + +@pytest.mark.parametrize("max_depth", max_depths) +def test_depth_constraint(max_depth: int) -> None: + cfg = CFG.depth_constraint(dsl, FunctionType(INT, INT), max_depth) + res = dsl.parse_program("(+ 1 var0)", FunctionType(INT, INT)) + print(cfg) + while res.depth() <= max_depth: + assert ( + res in cfg + ), f"Program depth:{res.depth()} should be in the TTCFG max_depth:{max_depth}" + res = dsl.parse_program(f"(+ {res} var0)", FunctionType(INT, INT)) + assert ( + res not in cfg + ), f"Program depth:{res.depth()} should NOT be in the TTCFG max_depth:{max_depth}" + + +def test_infinite() -> None: + cfg = CFG.infinite(dsl, FunctionType(INT, INT), n_gram=2) + res = dsl.parse_program("(+ 1 var0)", FunctionType(INT, INT)) + print(cfg) + while res.depth() <= 30: assert ( - res not in cfg - ), f"Program depth:{res.depth()} should NOT be in the TTCFG max_depth:{max_depth}" + res in cfg + ), f"Program depth:{res.depth()} should be in the infinite TTCFG" + res = dsl.parse_program(f"(+ {res} var0)", FunctionType(INT, INT)) diff --git a/tests/syntax/grammars/test_heap_search.py b/tests/syntax/grammars/test_heap_search.py deleted file mode 100644 index 913f592e..00000000 --- a/tests/syntax/grammars/test_heap_search.py +++ /dev/null @@ -1,80 +0,0 @@ -from synth.syntax.grammars.heap_search import ( - Bucket, - enumerate_prob_grammar, - enumerate_bucket_prob_grammar, -) -from synth.syntax.grammars.cfg import CFG -from synth.syntax.grammars.tagged_det_grammar import ProbDetGrammar -from synth.syntax.dsl import DSL -from synth.syntax.type_system import ( - INT, - STRING, - FunctionType, - List, - PolymorphicType, - PrimitiveType, -) - - -syntax = { - "+": FunctionType(INT, INT, INT), - "head": FunctionType(List(PolymorphicType("a")), PolymorphicType("a")), - "non_reachable": PrimitiveType("non_reachable"), - "1": INT, - "non_productive": FunctionType(INT, STRING), -} - - -def test_unicity_heapSearch() -> None: - dsl = DSL(syntax) - max_depth = 3 - cfg = CFG.depth_constraint(dsl, FunctionType(INT, INT), max_depth) - pcfg = ProbDetGrammar.uniform(cfg) - seen = set() - for program in enumerate_prob_grammar(pcfg): - assert program not in seen - seen.add(program) - assert len(seen) == cfg.size() - - -def test_order_heapSearch() -> None: - dsl = DSL(syntax) - max_depth = 3 - cfg = CFG.depth_constraint(dsl, FunctionType(INT, INT), max_depth) - pcfg = ProbDetGrammar.uniform(cfg) - last = 1.0 - for program in enumerate_prob_grammar(pcfg): - p = pcfg.probability(program) - assert p <= last - last = p - - -def test_unicity_bucketSearch() -> None: - dsl = DSL(syntax) - max_depth = 3 - cfg = CFG.depth_constraint(dsl, FunctionType(INT, INT), max_depth) - pcfg = ProbDetGrammar.uniform(cfg) - for bucketSize in range(3, 10): - seen = set() - for program in enumerate_bucket_prob_grammar(pcfg, bucket_size=bucketSize): - assert program not in seen - seen.add(program) - assert len(seen) == cfg.size() - - -def test_order_bucketSearch() -> None: - dsl = DSL(syntax) - max_depth = 3 - cfg = CFG.depth_constraint(dsl, FunctionType(INT, INT), max_depth) - pcfg = ProbDetGrammar.uniform(cfg) - for bucketSize in range(3, 10): - last = Bucket(bucketSize) - for program in enumerate_bucket_prob_grammar(pcfg, bucket_size=bucketSize): - p = pcfg.reduce_derivations( - lambda b, S, P, _: b.add_prob_uniform(pcfg.probabilities[S][P]), - Bucket(bucketSize), - program, - ) - assert p.size == bucketSize - assert p >= last or last == Bucket(bucketSize) - last = p diff --git a/tests/syntax/grammars/test_pcfg_splitter.py b/tests/syntax/grammars/test_pcfg_splitter.py deleted file mode 100644 index e406df1d..00000000 --- a/tests/syntax/grammars/test_pcfg_splitter.py +++ /dev/null @@ -1,57 +0,0 @@ -# from synth.syntax.grammars.heap_search import enumerate_pcfg -# from synth.syntax.grammars.pcfg_splitter import split -# from synth.syntax.grammars.cfg import CFG -# from synth.syntax.grammars.concrete_pcfg import ConcretePCFG -# from synth.syntax.dsl import DSL -# from synth.syntax.type_system import ( -# INT, -# STRING, -# FunctionType, -# List, -# PolymorphicType, -# PrimitiveType, -# ) - - -# syntax = { -# "+": FunctionType(INT, INT, INT), -# "-": FunctionType(INT, INT, INT), -# "head": FunctionType(List(PolymorphicType("a")), PolymorphicType("a")), -# "non_reachable": PrimitiveType("non_reachable"), -# "1": INT, -# "non_productive": FunctionType(INT, STRING), -# } - - -# def test_unicity() -> None: -# dsl = DSL(syntax) -# max_depth = 4 -# cfg = CFG.from_dsl(dsl, FunctionType(INT, INT), max_depth) -# pcfg = ConcretePCFG.uniform(cfg) -# for splits in [2, 4, 5]: -# fragments, _ = split(pcfg, splits, desired_ratio=1.05) -# seen = set() -# for sub_pcfg in fragments: -# for program in enumerate_pcfg(sub_pcfg): -# assert program not in seen -# seen.add(program) - - -# def prout_test_none_missing() -> None: -# dsl = DSL(syntax) -# max_depth = 3 -# cfg = CFG.from_dsl(dsl, FunctionType(INT, INT), max_depth) -# pcfg = ConcretePCFG.uniform(cfg) -# seen = set() -# for program in enumerate_pcfg(pcfg): -# seen.add(program) -# for splits in [2, 4, 5]: -# fragments, _ = split(pcfg, splits, desired_ratio=1.05) -# new_seen = set() -# for sub_pcfg in fragments: -# a = set() -# for program in enumerate_pcfg(sub_pcfg): -# a.add(program) -# new_seen |= a -# assert len(seen.difference(new_seen)) == 0, seen.difference(new_seen) -# assert len(new_seen.difference(seen)) == 0, new_seen.difference(seen) diff --git a/tests/syntax/grammars/test_tagged_det_grammar.py b/tests/syntax/grammars/test_tagged_det_grammar.py index 12e16e3e..63e1afcd 100644 --- a/tests/syntax/grammars/test_tagged_det_grammar.py +++ b/tests/syntax/grammars/test_tagged_det_grammar.py @@ -7,12 +7,13 @@ from synth.syntax.type_system import ( INT, STRING, - FunctionType, List, PolymorphicType, PrimitiveType, ) +from synth.syntax.type_helper import FunctionType +import pytest syntax = { "+": FunctionType(INT, INT, INT), @@ -23,65 +24,62 @@ "2": INT, "non_productive": FunctionType(INT, STRING), } +dsl = DSL(syntax) +max_depths = [3, 7, 11] -def test_from_cfg() -> None: - dsl = DSL(syntax) - for max_depth in [3, 7, 11]: - cfg = CFG.depth_constraint(dsl, FunctionType(INT, INT), max_depth) - pcfg = ProbDetGrammar.uniform(cfg) - for rule in pcfg.rules: - n = len(pcfg.rules[rule]) - for P in pcfg.rules[rule]: - prob = pcfg.probabilities[rule][P] - assert np.isclose(prob, 1 / n) +@pytest.mark.parametrize("max_depth", max_depths) +def test_from_cfg(max_depth: int) -> None: + cfg = CFG.depth_constraint(dsl, FunctionType(INT, INT), max_depth) + pcfg = ProbDetGrammar.uniform(cfg) + for rule in pcfg.rules: + n = len(pcfg.rules[rule]) + for P in pcfg.rules[rule]: + prob = pcfg.probabilities[rule][P] + assert np.isclose(prob, 1 / n) -def test_from_ttcfg() -> None: - dsl = DSL(syntax) - for max_depth in [3, 7, 11]: - cfg = TTCFG.size_constraint(dsl, FunctionType(INT, INT), max_depth) - pcfg = ProbDetGrammar.uniform(cfg) - for rule in pcfg.rules: - n = len(pcfg.rules[rule]) - for P in pcfg.rules[rule]: - prob = pcfg.probabilities[rule][P] - assert np.isclose(prob, 1 / n) +@pytest.mark.parametrize("max_depth", max_depths) +def test_from_ttcfg(max_depth: int) -> None: + cfg = TTCFG.size_constraint(dsl, FunctionType(INT, INT), max_depth) + pcfg = ProbDetGrammar.uniform(cfg) + for rule in pcfg.rules: + n = len(pcfg.rules[rule]) + for P in pcfg.rules[rule]: + prob = pcfg.probabilities[rule][P] + assert np.isclose(prob, 1 / n) -def test_ready_for_sampling() -> None: - dsl = DSL(syntax) - for max_depth in [3, 7, 11]: - cfg = CFG.depth_constraint(dsl, FunctionType(INT, INT), max_depth) - pcfg = ProbDetGrammar.uniform(cfg) - assert not pcfg.ready_for_sampling - pcfg.init_sampling() - assert pcfg.ready_for_sampling +@pytest.mark.parametrize("max_depth", max_depths) +def test_ready_for_sampling(max_depth: int) -> None: + cfg = CFG.depth_constraint(dsl, FunctionType(INT, INT), max_depth) + pcfg = ProbDetGrammar.uniform(cfg) + assert not pcfg.ready_for_sampling + pcfg.init_sampling() + assert pcfg.ready_for_sampling -def test_seeding() -> None: - dsl = DSL(syntax) - for max_depth in [3, 7, 11]: - seed = 100 - cfg = CFG.depth_constraint(dsl, FunctionType(INT, INT), max_depth) - pcfg = ProbDetGrammar.uniform(cfg) - pcfg.init_sampling(seed) - g1 = pcfg.sampling() - cpy = ProbDetGrammar.uniform(cfg) - cpy.init_sampling(seed) - assert pcfg == cpy - g2 = cpy.sampling() - for _ in range(200): - p1, p2 = next(g1), next(g2) - assert p1 == p2, f"[n°{_}]: {p1} != {p2}" +@pytest.mark.parametrize("max_depth", max_depths) +def test_seeding(max_depth: int) -> None: + seed = 100 + cfg = CFG.depth_constraint(dsl, FunctionType(INT, INT), max_depth) + pcfg = ProbDetGrammar.uniform(cfg) + pcfg.init_sampling(seed) + g1 = pcfg.sampling() + cpy = ProbDetGrammar.uniform(cfg) + cpy.init_sampling(seed) + assert pcfg == cpy + g2 = cpy.sampling() + for _ in range(200): + p1, p2 = next(g1), next(g2) + assert p1 == p2, f"[n°{_}]: {p1} != {p2}" -def test_depth() -> None: - dsl = DSL(syntax) - for max_depth in [3, 7, 11]: - cfg = CFG.depth_constraint(dsl, FunctionType(INT, INT), max_depth) - pcfg = ProbDetGrammar.uniform(cfg) - pcfg.init_sampling(0) - g = pcfg.sampling() - for _ in range(200): - assert next(g).depth() <= max_depth +@pytest.mark.parametrize("max_depth", max_depths) +def test_depth(max_depth: int) -> None: + cfg = CFG.depth_constraint(dsl, FunctionType(INT, INT), max_depth) + pcfg = ProbDetGrammar.uniform(cfg) + pcfg.init_sampling(0) + g = pcfg.sampling() + for _ in range(200): + assert next(g).depth() <= max_depth diff --git a/tests/syntax/grammars/test_tagged_u_grammar.py b/tests/syntax/grammars/test_tagged_u_grammar.py new file mode 100644 index 00000000..eeeb531d --- /dev/null +++ b/tests/syntax/grammars/test_tagged_u_grammar.py @@ -0,0 +1,87 @@ +import numpy as np + +from synth.syntax.grammars.tagged_u_grammar import ProbUGrammar +from synth.syntax.grammars.u_cfg import UCFG +from synth.syntax.dsl import DSL +from synth.syntax.type_system import ( + INT, + STRING, + List, + PolymorphicType, + PrimitiveType, +) +from synth.syntax.type_helper import FunctionType + +import pytest + +syntax = { + "+": FunctionType(INT, INT, INT), + "-": FunctionType(INT, INT, INT), + "head": FunctionType(List(PolymorphicType("a")), PolymorphicType("a")), + "non_reachable": PrimitiveType("non_reachable"), + "1": INT, + "2": INT, + "non_productive": FunctionType(INT, STRING), +} + +dsl = DSL(syntax) +max_depths = [3, 7, 11] + + +@pytest.mark.parametrize("max_depth", max_depths) +def test_from_cfg(max_depth: int) -> None: + cfg = UCFG.depth_constraint(dsl, FunctionType(INT, INT), max_depth) + pcfg = ProbUGrammar.uniform(cfg) + for rule in pcfg.rules: + n = sum(len(pcfg.rules[rule][P]) for P in pcfg.rules[rule]) + for P in pcfg.rules[rule]: + dico = pcfg.probabilities[rule][P] + for _, prob in dico.items(): + assert np.isclose(prob, 1 / n) + + +# def test_from_ttcfg() -> None: +# dsl = DSL(syntax) +# for max_depth in [3, 7, 11]: +# cfg = TTCFG.size_constraint(dsl, FunctionType(INT, INT), max_depth) +# pcfg = ProbUGrammar.uniform(cfg) +# for rule in pcfg.rules: +# n = len(pcfg.rules[rule]) +# for P in pcfg.rules[rule]: +# prob = pcfg.probabilities[rule][P] +# assert np.isclose(prob, 1 / n) + + +@pytest.mark.parametrize("max_depth", max_depths) +def test_ready_for_sampling(max_depth: int) -> None: + cfg = UCFG.depth_constraint(dsl, FunctionType(INT, INT), max_depth) + pcfg = ProbUGrammar.uniform(cfg) + assert not pcfg.ready_for_sampling + pcfg.init_sampling() + assert pcfg.ready_for_sampling + + +@pytest.mark.parametrize("max_depth", max_depths) +def test_seeding(max_depth: int) -> None: + seed = 100 + cfg = UCFG.depth_constraint(dsl, FunctionType(INT, INT), max_depth) + pcfg = ProbUGrammar.uniform(cfg) + pcfg.init_sampling(seed) + g1 = pcfg.sampling() + cpy = ProbUGrammar.uniform(cfg) + cpy.init_sampling(seed) + assert pcfg == cpy + g2 = cpy.sampling() + for _ in range(200): + p1, p2 = next(g1), next(g2) + assert p1 == p2, f"[n°{_}]: {p1} != {p2}" + + +@pytest.mark.parametrize("max_depth", max_depths) +def test_depth(max_depth: int) -> None: + cfg = UCFG.depth_constraint(dsl, FunctionType(INT, INT), max_depth) + pcfg = ProbUGrammar.uniform(cfg) + pcfg.init_sampling(0) + g = pcfg.sampling() + for _ in range(200): + assert next(g).depth() <= max_depth diff --git a/tests/syntax/grammars/test_ttcfg.py b/tests/syntax/grammars/test_ttcfg.py index ecab631e..113cb40a 100644 --- a/tests/syntax/grammars/test_ttcfg.py +++ b/tests/syntax/grammars/test_ttcfg.py @@ -4,12 +4,13 @@ from synth.syntax.type_system import ( INT, STRING, - FunctionType, List, PolymorphicType, PrimitiveType, ) +from synth.syntax.type_helper import FunctionType +import pytest syntax = { "+": FunctionType(INT, INT, INT), @@ -18,71 +19,77 @@ "1": INT, "non_productive": FunctionType(INT, STRING), } +dsl = DSL(syntax) +dsl.instantiate_polymorphic_types() +max_sizes = [3, 7, 11] -def test_clean() -> None: - dsl = DSL(syntax) - for max_size in [3, 7, 11]: - cfg = TTCFG.size_constraint(dsl, FunctionType(INT, INT), max_size) - for rule in cfg.rules: - for P in cfg.rules[rule]: - if isinstance(P, Primitive): - assert P.primitive != "non_reachable" - assert P.primitive != "non_productive" - assert P.primitive != "head" - else: - assert P.type == INT +@pytest.mark.parametrize("max_size", max_sizes) +def test_clean(max_size: int) -> None: + cfg = TTCFG.size_constraint(dsl, FunctionType(INT, INT), max_size) + for rule in cfg.rules: + for P in cfg.rules[rule]: + if isinstance(P, Primitive): + assert P.primitive != "non_reachable" + assert P.primitive != "non_productive" + assert P.primitive != "head" + else: + assert P.type == INT -def test_size_constraint() -> None: - dsl = DSL(syntax) - for max_size in [3, 7, 11]: - cfg = TTCFG.size_constraint(dsl, FunctionType(INT, INT), max_size) - size1 = dsl.parse_program("(+ 1 var0)", FunctionType(INT, INT)) - res = size1 - while res.length() <= max_size: - assert ( - res in cfg - ), f"Program size:{res.length()} should be in the TTCFG max_size:{max_size}" - res = dsl.parse_program(f"(+ {res} var0)", FunctionType(INT, INT)) +@pytest.mark.parametrize("max_size,progs", [(1, 2), (3, 2 + 4), (5, 22)]) +def test_size(max_size: int, progs: int) -> None: + cfg = TTCFG.size_constraint(dsl, FunctionType(INT, INT), max_size) + size = cfg.programs() + print(cfg) + assert size == progs + + +@pytest.mark.parametrize("max_size", max_sizes) +def test_size_constraint(max_size: int) -> None: + cfg = TTCFG.size_constraint(dsl, FunctionType(INT, INT), max_size) + size1 = dsl.parse_program("(+ 1 var0)", FunctionType(INT, INT)) + res = size1 + while res.size() <= max_size: assert ( - res not in cfg - ), f"Program size:{res.length()} should NOT be in the TTCFG max_size:{max_size}" + res in cfg + ), f"Program size:{res.size()} should be in the TTCFG max_size:{max_size}" + res = dsl.parse_program(f"(+ {res} var0)", FunctionType(INT, INT)) + assert ( + res not in cfg + ), f"Program size:{res.size()} should NOT be in the TTCFG max_size:{max_size}" -def test_at_most() -> None: - dsl = DSL(syntax) - for max_occ in [3, 7, 11]: - cfg = TTCFG.at_most_k(dsl, FunctionType(INT, INT), "+", max_occ) - res = dsl.parse_program("(+ 1 var0)", FunctionType(INT, INT)) - while res.depth() - 1 <= max_occ: - assert ( - res in cfg - ), f"Occurences:{res.depth() - 1} should be in the TTCFG max occurences:{max_occ}" - res = dsl.parse_program(f"(+ {res} var0)", FunctionType(INT, INT)) +@pytest.mark.parametrize("max_occ", [3, 4, 5]) +def test_at_most(max_occ: int) -> None: + cfg = TTCFG.at_most_k(dsl, FunctionType(INT, INT), "+", max_occ) + res = dsl.parse_program("(+ 1 var0)", FunctionType(INT, INT)) + while res.depth() - 1 <= max_occ: assert ( - res not in cfg - ), f"Occurences:{res.depth() - 1} should NOT be in the TTCFG max occurences:{max_occ}" + res in cfg + ), f"Occurences:{res.depth() - 1} should be in the TTCFG max occurences:{max_occ}" + res = dsl.parse_program(f"(+ {res} var0)", FunctionType(INT, INT)) + assert ( + res not in cfg + ), f"Occurences:{res.depth() - 1} should NOT be in the TTCFG max occurences:{max_occ}" -def test_clean() -> None: - dsl = DSL(syntax) - for max_size in [3, 7, 11]: - cfg = TTCFG.size_constraint(dsl, FunctionType(INT, INT), max_size) - for rule in cfg.rules: - for P in cfg.rules[rule]: - if isinstance(P, Primitive): - assert P.primitive != "non_reachable" - assert P.primitive != "non_productive" - assert P.primitive != "head" +@pytest.mark.parametrize("max_size", max_sizes) +def test_clean(max_size: int) -> None: + cfg = TTCFG.size_constraint(dsl, FunctionType(INT, INT), max_size) + for rule in cfg.rules: + for P in cfg.rules[rule]: + if isinstance(P, Primitive): + assert P.primitive != "non_reachable" + assert P.primitive != "non_productive" + assert P.primitive != "head" - cpy = TTCFG.size_constraint(dsl, FunctionType(INT, INT), max_size) - cpy.clean() - assert cfg == cpy + cpy = TTCFG.size_constraint(dsl, FunctionType(INT, INT), max_size) + cpy.clean() + assert cfg == cpy def test_product() -> None: - dsl = DSL(syntax) max_size = 3 cfg1 = TTCFG.size_constraint(dsl, FunctionType(INT, INT), max_size * 2) cfg2 = TTCFG.size_constraint(dsl, FunctionType(INT, INT), max_size) @@ -90,11 +97,11 @@ def test_product() -> None: assert cfg size1 = dsl.parse_program("(+ 1 var0)", FunctionType(INT, INT)) res = size1 - while res.length() <= max_size: + while res.size() <= max_size: assert ( res in cfg - ), f"Program size:{res.length()} should be in the TTCFG max_size:{max_size}" + ), f"Program size:{res.size()} should be in the TTCFG max_size:{max_size}" res = dsl.parse_program(f"(+ {res} var0)", FunctionType(INT, INT)) assert ( res not in cfg - ), f"Program size:{res.length()} should NOT be in the TTCFG max_size:{max_size}" + ), f"Program size:{res.size()} should NOT be in the TTCFG max_size:{max_size}" diff --git a/tests/syntax/grammars/test_ucfg.py b/tests/syntax/grammars/test_ucfg.py new file mode 100644 index 00000000..2bdff3ed --- /dev/null +++ b/tests/syntax/grammars/test_ucfg.py @@ -0,0 +1,68 @@ +from synth.syntax.grammars.cfg import CFG +from synth.syntax.grammars.u_cfg import UCFG +from synth.syntax.dsl import DSL +from synth.syntax.program import Primitive +from synth.syntax.type_system import ( + INT, + STRING, + List, + PolymorphicType, + PrimitiveType, +) +from synth.syntax.type_helper import FunctionType + +import pytest + + +syntax = { + "+": FunctionType(INT, INT, INT), + "head": FunctionType(List(PolymorphicType("a")), PolymorphicType("a")), + "non_reachable": PrimitiveType("non_reachable"), + "1": INT, + "non_productive": FunctionType(INT, STRING), +} +dsl = DSL(syntax) +dsl.instantiate_polymorphic_types() +max_depths = [3, 7, 11] + + +@pytest.mark.parametrize("max_depth", max_depths) +def test_size(max_depth: int) -> None: + ucfg = UCFG.depth_constraint(dsl, FunctionType(INT, INT), max_depth) + cfg = CFG.depth_constraint(dsl, FunctionType(INT, INT), max_depth) + assert cfg.programs() == ucfg.programs() + + +@pytest.mark.parametrize("max_depth", max_depths) +def test_clean(max_depth: int) -> None: + dirty_ucfg = UCFG.from_CFG( + CFG.depth_constraint(dsl, FunctionType(INT, INT), max_depth), clean=False + ) + clean_ucfg = UCFG.from_CFG( + CFG.depth_constraint(dsl, FunctionType(INT, INT), max_depth), clean=True + ) + + assert clean_ucfg.programs() == dirty_ucfg.programs() + + for rule in clean_ucfg.rules: + assert rule[1][1] <= max_depth + for P in clean_ucfg.rules[rule]: + if isinstance(P, Primitive): + assert P.primitive != "non_reachable" + assert P.primitive != "non_productive" + assert P.primitive != "head" + + +@pytest.mark.parametrize("max_depth", max_depths) +def test_depth_constraint(max_depth: int) -> None: + cfg = UCFG.depth_constraint(dsl, FunctionType(INT, INT), max_depth) + res = dsl.parse_program("(+ 1 var0)", FunctionType(INT, INT)) + print(cfg) + while res.depth() <= max_depth: + assert ( + res in cfg + ), f"Program depth:{res.depth()} should be in the TTCFG max_depth:{max_depth}" + res = dsl.parse_program(f"(+ {res} var0)", FunctionType(INT, INT)) + assert ( + res not in cfg + ), f"Program depth:{res.depth()} should NOT be in the TTCFG max_depth:{max_depth}" diff --git a/tests/syntax/test_dsl.py b/tests/syntax/test_dsl.py index 44021b50..6754e3b6 100644 --- a/tests/syntax/test_dsl.py +++ b/tests/syntax/test_dsl.py @@ -1,5 +1,6 @@ from synth.syntax.dsl import DSL -from synth.syntax.type_system import INT, FunctionType, PolymorphicType +from synth.syntax.type_system import INT, PolymorphicType +from synth.syntax.type_helper import FunctionType syntax = { diff --git a/tests/syntax/test_program.py b/tests/syntax/test_program.py index 2d124afb..97cc5945 100644 --- a/tests/syntax/test_program.py +++ b/tests/syntax/test_program.py @@ -2,7 +2,8 @@ import random from synth.syntax.program import Primitive, Function, Program, Variable -from synth.syntax.type_system import BOOL, INT, FunctionType +from synth.syntax.type_system import BOOL, INT +from synth.syntax.type_helper import FunctionType def __gen2list__(g: Generator) -> List: diff --git a/tests/syntax/test_type_helper.py b/tests/syntax/test_type_helper.py new file mode 100644 index 00000000..cf14436f --- /dev/null +++ b/tests/syntax/test_type_helper.py @@ -0,0 +1,82 @@ +from synth.syntax.type_system import ( + STRING, + INT, + BOOL, + FixedPolymorphicType, + Generic, + GenericFunctor, + PolymorphicType, + List, + Arrow, + PrimitiveType, + UnknownType, + match, +) +from synth.syntax.type_helper import auto_type, FunctionType, guess_type, Optional +import random + + +def test_guess_type() -> None: + # Bool + assert guess_type(True) == BOOL + assert guess_type(False) == BOOL + # Int + random.seed(0) + for _ in range(100): + assert guess_type(random.randint(-100, 100)) == INT + # String + assert guess_type("") == STRING + # List + assert match(guess_type([]), List(PolymorphicType(""))) + assert guess_type([True]) == List(BOOL) + assert guess_type([""]) == List(STRING) + assert guess_type([1]) == List(INT) + # Unknown + assert isinstance(guess_type(int), UnknownType) + + +def test_FunctionType() -> None: + assert FunctionType(INT, BOOL, STRING, List(INT)) == Arrow( + INT, Arrow(BOOL, Arrow(STRING, List(INT))) + ) + + +def test_auto_type_base() -> None: + assert PrimitiveType("int") == auto_type("int") + assert PrimitiveType("bb") == auto_type("bb") + assert PolymorphicType("bb") == auto_type("'bb") + assert PolymorphicType("aa") == auto_type("'aa") + assert PrimitiveType("a_a") == auto_type("a_a") + + +def test_auto_type_advanced() -> None: + assert List(PrimitiveType("int")) == auto_type("int list") + assert List(PolymorphicType("a")) == auto_type("'a list") + + some = GenericFunctor("some", min_args=1, max_args=1) + + assert some(PolymorphicType("a")) == auto_type("'a some") + assert Optional(PolymorphicType("a")) == auto_type("'a optional") + assert Optional(some(PolymorphicType("a"))) == auto_type("'a some optional") + + x = PrimitiveType("bb") | PolymorphicType("aa") + assert x == auto_type("bb | 'aa") + assert x == auto_type("bb|'aa") + assert x == auto_type("'aa | bb") + + +def test_auto_type_arrows() -> None: + a = PrimitiveType("a") + b = PrimitiveType("b") + assert FunctionType(a, b) == auto_type("a->b") + assert FunctionType(a, b, b) == auto_type("a->b->b") + assert FunctionType(a, FunctionType(a, b), b) == auto_type("a->(a->b)->b") + + assert Generic("*", a, b, infix=True) == auto_type("a*b") + + +def test_auto_type_fixed_poly() -> None: + x = FixedPolymorphicType("z", PrimitiveType("b") | PrimitiveType("c")) + assert x == auto_type("'z[b|c]") + assert x == auto_type("'z [b|c]") + assert x == auto_type("'z[ b|c ]") diff --git a/tests/syntax/test_type_system.py b/tests/syntax/test_type_system.py index 2b9d8fba..a8762781 100644 --- a/tests/syntax/test_type_system.py +++ b/tests/syntax/test_type_system.py @@ -1,20 +1,18 @@ +import copy from synth.syntax.type_system import ( STRING, INT, BOOL, - FunctionType, + FixedPolymorphicType, PolymorphicType, List, Arrow, PrimitiveType, Type, - UnknownType, match, - guess_type, EmptyList, ) from typing import List as TList, Set, Tuple -import random def test_hash() -> None: @@ -60,33 +58,14 @@ def test_match() -> None: assert match(PolymorphicType("a"), t1) assert match(t1, PolymorphicType("a")) - if isinstance(t1, List): + if t1.is_instance(List): assert match(t1, List(PolymorphicType("a"))) assert match(List(PolymorphicType("a")), t1) - elif isinstance(t1, Arrow): - assert match(t1, Arrow(PolymorphicType("a"), t1.type_out)) - assert match(t1, Arrow(t1.type_in, PolymorphicType("a"))) - assert match(Arrow(PolymorphicType("a"), t1.type_out), t1) - assert match(Arrow(t1.type_in, PolymorphicType("a")), t1) - - -def test_guess_type() -> None: - # Bool - assert guess_type(True) == BOOL - assert guess_type(False) == BOOL - # Int - random.seed(0) - for _ in range(100): - assert guess_type(random.randint(-100, 100)) == INT - # String - assert guess_type("") == STRING - # List - assert match(guess_type([]), List(PolymorphicType(""))) - assert guess_type([True]) == List(BOOL) - assert guess_type([""]) == List(STRING) - assert guess_type([1]) == List(INT) - # Unknown - assert isinstance(guess_type(int), UnknownType) + elif t1.is_instance(Arrow): + assert match(t1, Arrow(PolymorphicType("a"), t1.type_out)) # type: ignore + assert match(t1, Arrow(t1.type_in, PolymorphicType("a"))) # type: ignore + assert match(Arrow(PolymorphicType("a"), t1.type_out), t1) # type: ignore + assert match(Arrow(t1.type_in, PolymorphicType("a")), t1) # type: ignore def test_decompose_type() -> None: @@ -123,12 +102,6 @@ def test_unify() -> None: ) -def test_FunctionType() -> None: - assert FunctionType(INT, BOOL, STRING, List(INT)) == Arrow( - INT, Arrow(BOOL, Arrow(STRING, List(INT))) - ) - - def test_contains() -> None: t = Arrow( Arrow(INT, EmptyList), @@ -138,3 +111,42 @@ def test_contains() -> None: assert BOOL not in t assert PolymorphicType("b") in t assert Arrow(INT, EmptyList) in t + + +def test_is_a() -> None: + types = [BOOL, INT, List(INT), Arrow(INT, INT)] + for i, x in enumerate(types): + for y in types[i + 1 :]: + print("x:", x) + print("y:", y) + assert not x.is_instance(y) + assert not y.is_instance(x) + assert x.is_instance(x | y) + assert y.is_instance(x | y) + assert not (x | y).is_instance(x) + assert (x | y).is_instance(x | y) + + assert List(INT).is_instance(List(PolymorphicType("a"))) + assert List(INT).is_instance(List(INT | BOOL)) + assert not List(STRING).is_instance(List(INT | BOOL)) + + +def test_can_be() -> None: + z = FixedPolymorphicType("z", INT, BOOL) + types = [BOOL, INT, List(INT), Arrow(INT, INT)] + for i, x in enumerate(types): + assert z.can_be(x) == (i <= 1) + + +def test_pickle() -> None: + types = [ + BOOL, + INT, + List(INT), + Arrow(INT, INT), + PolymorphicType("a"), + FixedPolymorphicType("b", INT), + ] + for t in types: + tprime = copy.deepcopy(t) + assert tprime == t diff --git a/tests/test_task.py b/tests/test_task.py index e5921d66..5df73a17 100644 --- a/tests/test_task.py +++ b/tests/test_task.py @@ -1,7 +1,8 @@ import random import pathlib -from synth.syntax.type_system import INT, FunctionType +from synth.syntax.type_system import INT +from synth.syntax.type_helper import FunctionType from synth.syntax.program import Variable from synth.task import Task, Dataset from synth.specification import PBE, Example diff --git a/uv.lock b/uv.lock new file mode 100644 index 00000000..09da1d5c --- /dev/null +++ b/uv.lock @@ -0,0 +1,1461 @@ +version = 1 +revision = 1 +requires-python = ">=3.10" +resolution-markers = [ + "python_full_version >= '3.12'", + "python_full_version == '3.11.*'", + "python_full_version < '3.11'", +] + +[[package]] +name = "absl-py" +version = "2.3.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/10/2a/c93173ffa1b39c1d0395b7e842bbdc62e556ca9d8d3b5572926f3e4ca752/absl_py-2.3.1.tar.gz", hash = "sha256:a97820526f7fbfd2ec1bce83f3f25e3a14840dac0d8e02a0b71cd75db3f77fc9", size = 116588 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/8f/aa/ba0014cc4659328dc818a28827be78e6d97312ab0cb98105a770924dc11e/absl_py-2.3.1-py3-none-any.whl", hash = "sha256:eeecf07f0c2a93ace0772c92e596ace6d3d3996c042b2128459aaae2a76de11d", size = 135811 }, +] + +[[package]] +name = "colorama" +version = "0.4.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d8/53/6f443c9a4a8358a93a6792e2acffb9d9d5cb0a5cfd8802644b7b1c9a02e4/colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44", size = 27697 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335 }, +] + +[[package]] +name = "contourpy" +version = "1.3.2" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version < '3.11'", +] +dependencies = [ + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/66/54/eb9bfc647b19f2009dd5c7f5ec51c4e6ca831725f1aea7a993034f483147/contourpy-1.3.2.tar.gz", hash = "sha256:b6945942715a034c671b7fc54f9588126b0b8bf23db2696e3ca8328f3ff0ab54", size = 13466130 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/12/a3/da4153ec8fe25d263aa48c1a4cbde7f49b59af86f0b6f7862788c60da737/contourpy-1.3.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ba38e3f9f330af820c4b27ceb4b9c7feee5fe0493ea53a8720f4792667465934", size = 268551 }, + { url = "https://files.pythonhosted.org/packages/2f/6c/330de89ae1087eb622bfca0177d32a7ece50c3ef07b28002de4757d9d875/contourpy-1.3.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:dc41ba0714aa2968d1f8674ec97504a8f7e334f48eeacebcaa6256213acb0989", size = 253399 }, + { url = "https://files.pythonhosted.org/packages/c1/bd/20c6726b1b7f81a8bee5271bed5c165f0a8e1f572578a9d27e2ccb763cb2/contourpy-1.3.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9be002b31c558d1ddf1b9b415b162c603405414bacd6932d031c5b5a8b757f0d", size = 312061 }, + { url = "https://files.pythonhosted.org/packages/22/fc/a9665c88f8a2473f823cf1ec601de9e5375050f1958cbb356cdf06ef1ab6/contourpy-1.3.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8d2e74acbcba3bfdb6d9d8384cdc4f9260cae86ed9beee8bd5f54fee49a430b9", size = 351956 }, + { url = "https://files.pythonhosted.org/packages/25/eb/9f0a0238f305ad8fb7ef42481020d6e20cf15e46be99a1fcf939546a177e/contourpy-1.3.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e259bced5549ac64410162adc973c5e2fb77f04df4a439d00b478e57a0e65512", size = 320872 }, + { url = "https://files.pythonhosted.org/packages/32/5c/1ee32d1c7956923202f00cf8d2a14a62ed7517bdc0ee1e55301227fc273c/contourpy-1.3.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ad687a04bc802cbe8b9c399c07162a3c35e227e2daccf1668eb1f278cb698631", size = 325027 }, + { url = "https://files.pythonhosted.org/packages/83/bf/9baed89785ba743ef329c2b07fd0611d12bfecbedbdd3eeecf929d8d3b52/contourpy-1.3.2-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:cdd22595308f53ef2f891040ab2b93d79192513ffccbd7fe19be7aa773a5e09f", size = 1306641 }, + { url = "https://files.pythonhosted.org/packages/d4/cc/74e5e83d1e35de2d28bd97033426b450bc4fd96e092a1f7a63dc7369b55d/contourpy-1.3.2-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:b4f54d6a2defe9f257327b0f243612dd051cc43825587520b1bf74a31e2f6ef2", size = 1374075 }, + { url = "https://files.pythonhosted.org/packages/0c/42/17f3b798fd5e033b46a16f8d9fcb39f1aba051307f5ebf441bad1ecf78f8/contourpy-1.3.2-cp310-cp310-win32.whl", hash = "sha256:f939a054192ddc596e031e50bb13b657ce318cf13d264f095ce9db7dc6ae81c0", size = 177534 }, + { url = "https://files.pythonhosted.org/packages/54/ec/5162b8582f2c994721018d0c9ece9dc6ff769d298a8ac6b6a652c307e7df/contourpy-1.3.2-cp310-cp310-win_amd64.whl", hash = "sha256:c440093bbc8fc21c637c03bafcbef95ccd963bc6e0514ad887932c18ca2a759a", size = 221188 }, + { url = "https://files.pythonhosted.org/packages/b3/b9/ede788a0b56fc5b071639d06c33cb893f68b1178938f3425debebe2dab78/contourpy-1.3.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6a37a2fb93d4df3fc4c0e363ea4d16f83195fc09c891bc8ce072b9d084853445", size = 269636 }, + { url = "https://files.pythonhosted.org/packages/e6/75/3469f011d64b8bbfa04f709bfc23e1dd71be54d05b1b083be9f5b22750d1/contourpy-1.3.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:b7cd50c38f500bbcc9b6a46643a40e0913673f869315d8e70de0438817cb7773", size = 254636 }, + { url = "https://files.pythonhosted.org/packages/8d/2f/95adb8dae08ce0ebca4fd8e7ad653159565d9739128b2d5977806656fcd2/contourpy-1.3.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d6658ccc7251a4433eebd89ed2672c2ed96fba367fd25ca9512aa92a4b46c4f1", size = 313053 }, + { url = "https://files.pythonhosted.org/packages/c3/a6/8ccf97a50f31adfa36917707fe39c9a0cbc24b3bbb58185577f119736cc9/contourpy-1.3.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:70771a461aaeb335df14deb6c97439973d253ae70660ca085eec25241137ef43", size = 352985 }, + { url = "https://files.pythonhosted.org/packages/1d/b6/7925ab9b77386143f39d9c3243fdd101621b4532eb126743201160ffa7e6/contourpy-1.3.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:65a887a6e8c4cd0897507d814b14c54a8c2e2aa4ac9f7686292f9769fcf9a6ab", size = 323750 }, + { url = "https://files.pythonhosted.org/packages/c2/f3/20c5d1ef4f4748e52d60771b8560cf00b69d5c6368b5c2e9311bcfa2a08b/contourpy-1.3.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3859783aefa2b8355697f16642695a5b9792e7a46ab86da1118a4a23a51a33d7", size = 326246 }, + { url = "https://files.pythonhosted.org/packages/8c/e5/9dae809e7e0b2d9d70c52b3d24cba134dd3dad979eb3e5e71f5df22ed1f5/contourpy-1.3.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:eab0f6db315fa4d70f1d8ab514e527f0366ec021ff853d7ed6a2d33605cf4b83", size = 1308728 }, + { url = "https://files.pythonhosted.org/packages/e2/4a/0058ba34aeea35c0b442ae61a4f4d4ca84d6df8f91309bc2d43bb8dd248f/contourpy-1.3.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:d91a3ccc7fea94ca0acab82ceb77f396d50a1f67412efe4c526f5d20264e6ecd", size = 1375762 }, + { url = "https://files.pythonhosted.org/packages/09/33/7174bdfc8b7767ef2c08ed81244762d93d5c579336fc0b51ca57b33d1b80/contourpy-1.3.2-cp311-cp311-win32.whl", hash = "sha256:1c48188778d4d2f3d48e4643fb15d8608b1d01e4b4d6b0548d9b336c28fc9b6f", size = 178196 }, + { url = "https://files.pythonhosted.org/packages/5e/fe/4029038b4e1c4485cef18e480b0e2cd2d755448bb071eb9977caac80b77b/contourpy-1.3.2-cp311-cp311-win_amd64.whl", hash = "sha256:5ebac872ba09cb8f2131c46b8739a7ff71de28a24c869bcad554477eb089a878", size = 222017 }, + { url = "https://files.pythonhosted.org/packages/34/f7/44785876384eff370c251d58fd65f6ad7f39adce4a093c934d4a67a7c6b6/contourpy-1.3.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:4caf2bcd2969402bf77edc4cb6034c7dd7c0803213b3523f111eb7460a51b8d2", size = 271580 }, + { url = "https://files.pythonhosted.org/packages/93/3b/0004767622a9826ea3d95f0e9d98cd8729015768075d61f9fea8eeca42a8/contourpy-1.3.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:82199cb78276249796419fe36b7386bd8d2cc3f28b3bc19fe2454fe2e26c4c15", size = 255530 }, + { url = "https://files.pythonhosted.org/packages/e7/bb/7bd49e1f4fa805772d9fd130e0d375554ebc771ed7172f48dfcd4ca61549/contourpy-1.3.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:106fab697af11456fcba3e352ad50effe493a90f893fca6c2ca5c033820cea92", size = 307688 }, + { url = "https://files.pythonhosted.org/packages/fc/97/e1d5dbbfa170725ef78357a9a0edc996b09ae4af170927ba8ce977e60a5f/contourpy-1.3.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d14f12932a8d620e307f715857107b1d1845cc44fdb5da2bc8e850f5ceba9f87", size = 347331 }, + { url = "https://files.pythonhosted.org/packages/6f/66/e69e6e904f5ecf6901be3dd16e7e54d41b6ec6ae3405a535286d4418ffb4/contourpy-1.3.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:532fd26e715560721bb0d5fc7610fce279b3699b018600ab999d1be895b09415", size = 318963 }, + { url = "https://files.pythonhosted.org/packages/a8/32/b8a1c8965e4f72482ff2d1ac2cd670ce0b542f203c8e1d34e7c3e6925da7/contourpy-1.3.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f26b383144cf2d2c29f01a1e8170f50dacf0eac02d64139dcd709a8ac4eb3cfe", size = 323681 }, + { url = "https://files.pythonhosted.org/packages/30/c6/12a7e6811d08757c7162a541ca4c5c6a34c0f4e98ef2b338791093518e40/contourpy-1.3.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:c49f73e61f1f774650a55d221803b101d966ca0c5a2d6d5e4320ec3997489441", size = 1308674 }, + { url = "https://files.pythonhosted.org/packages/2a/8a/bebe5a3f68b484d3a2b8ffaf84704b3e343ef1addea528132ef148e22b3b/contourpy-1.3.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:3d80b2c0300583228ac98d0a927a1ba6a2ba6b8a742463c564f1d419ee5b211e", size = 1380480 }, + { url = "https://files.pythonhosted.org/packages/34/db/fcd325f19b5978fb509a7d55e06d99f5f856294c1991097534360b307cf1/contourpy-1.3.2-cp312-cp312-win32.whl", hash = "sha256:90df94c89a91b7362e1142cbee7568f86514412ab8a2c0d0fca72d7e91b62912", size = 178489 }, + { url = "https://files.pythonhosted.org/packages/01/c8/fadd0b92ffa7b5eb5949bf340a63a4a496a6930a6c37a7ba0f12acb076d6/contourpy-1.3.2-cp312-cp312-win_amd64.whl", hash = "sha256:8c942a01d9163e2e5cfb05cb66110121b8d07ad438a17f9e766317bcb62abf73", size = 223042 }, + { url = "https://files.pythonhosted.org/packages/2e/61/5673f7e364b31e4e7ef6f61a4b5121c5f170f941895912f773d95270f3a2/contourpy-1.3.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:de39db2604ae755316cb5967728f4bea92685884b1e767b7c24e983ef5f771cb", size = 271630 }, + { url = "https://files.pythonhosted.org/packages/ff/66/a40badddd1223822c95798c55292844b7e871e50f6bfd9f158cb25e0bd39/contourpy-1.3.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:3f9e896f447c5c8618f1edb2bafa9a4030f22a575ec418ad70611450720b5b08", size = 255670 }, + { url = "https://files.pythonhosted.org/packages/1e/c7/cf9fdee8200805c9bc3b148f49cb9482a4e3ea2719e772602a425c9b09f8/contourpy-1.3.2-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:71e2bd4a1c4188f5c2b8d274da78faab884b59df20df63c34f74aa1813c4427c", size = 306694 }, + { url = "https://files.pythonhosted.org/packages/dd/e7/ccb9bec80e1ba121efbffad7f38021021cda5be87532ec16fd96533bb2e0/contourpy-1.3.2-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:de425af81b6cea33101ae95ece1f696af39446db9682a0b56daaa48cfc29f38f", size = 345986 }, + { url = "https://files.pythonhosted.org/packages/dc/49/ca13bb2da90391fa4219fdb23b078d6065ada886658ac7818e5441448b78/contourpy-1.3.2-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:977e98a0e0480d3fe292246417239d2d45435904afd6d7332d8455981c408b85", size = 318060 }, + { url = "https://files.pythonhosted.org/packages/c8/65/5245ce8c548a8422236c13ffcdcdada6a2a812c361e9e0c70548bb40b661/contourpy-1.3.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:434f0adf84911c924519d2b08fc10491dd282b20bdd3fa8f60fd816ea0b48841", size = 322747 }, + { url = "https://files.pythonhosted.org/packages/72/30/669b8eb48e0a01c660ead3752a25b44fdb2e5ebc13a55782f639170772f9/contourpy-1.3.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:c66c4906cdbc50e9cba65978823e6e00b45682eb09adbb78c9775b74eb222422", size = 1308895 }, + { url = "https://files.pythonhosted.org/packages/05/5a/b569f4250decee6e8d54498be7bdf29021a4c256e77fe8138c8319ef8eb3/contourpy-1.3.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:8b7fc0cd78ba2f4695fd0a6ad81a19e7e3ab825c31b577f384aa9d7817dc3bef", size = 1379098 }, + { url = "https://files.pythonhosted.org/packages/19/ba/b227c3886d120e60e41b28740ac3617b2f2b971b9f601c835661194579f1/contourpy-1.3.2-cp313-cp313-win32.whl", hash = "sha256:15ce6ab60957ca74cff444fe66d9045c1fd3e92c8936894ebd1f3eef2fff075f", size = 178535 }, + { url = "https://files.pythonhosted.org/packages/12/6e/2fed56cd47ca739b43e892707ae9a13790a486a3173be063681ca67d2262/contourpy-1.3.2-cp313-cp313-win_amd64.whl", hash = "sha256:e1578f7eafce927b168752ed7e22646dad6cd9bca673c60bff55889fa236ebf9", size = 223096 }, + { url = "https://files.pythonhosted.org/packages/54/4c/e76fe2a03014a7c767d79ea35c86a747e9325537a8b7627e0e5b3ba266b4/contourpy-1.3.2-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:0475b1f6604896bc7c53bb070e355e9321e1bc0d381735421a2d2068ec56531f", size = 285090 }, + { url = "https://files.pythonhosted.org/packages/7b/e2/5aba47debd55d668e00baf9651b721e7733975dc9fc27264a62b0dd26eb8/contourpy-1.3.2-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:c85bb486e9be652314bb5b9e2e3b0d1b2e643d5eec4992c0fbe8ac71775da739", size = 268643 }, + { url = "https://files.pythonhosted.org/packages/a1/37/cd45f1f051fe6230f751cc5cdd2728bb3a203f5619510ef11e732109593c/contourpy-1.3.2-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:745b57db7758f3ffc05a10254edd3182a2a83402a89c00957a8e8a22f5582823", size = 310443 }, + { url = "https://files.pythonhosted.org/packages/8b/a2/36ea6140c306c9ff6dd38e3bcec80b3b018474ef4d17eb68ceecd26675f4/contourpy-1.3.2-cp313-cp313t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:970e9173dbd7eba9b4e01aab19215a48ee5dd3f43cef736eebde064a171f89a5", size = 349865 }, + { url = "https://files.pythonhosted.org/packages/95/b7/2fc76bc539693180488f7b6cc518da7acbbb9e3b931fd9280504128bf956/contourpy-1.3.2-cp313-cp313t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c6c4639a9c22230276b7bffb6a850dfc8258a2521305e1faefe804d006b2e532", size = 321162 }, + { url = "https://files.pythonhosted.org/packages/f4/10/76d4f778458b0aa83f96e59d65ece72a060bacb20cfbee46cf6cd5ceba41/contourpy-1.3.2-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cc829960f34ba36aad4302e78eabf3ef16a3a100863f0d4eeddf30e8a485a03b", size = 327355 }, + { url = "https://files.pythonhosted.org/packages/43/a3/10cf483ea683f9f8ab096c24bad3cce20e0d1dd9a4baa0e2093c1c962d9d/contourpy-1.3.2-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:d32530b534e986374fc19eaa77fcb87e8a99e5431499949b828312bdcd20ac52", size = 1307935 }, + { url = "https://files.pythonhosted.org/packages/78/73/69dd9a024444489e22d86108e7b913f3528f56cfc312b5c5727a44188471/contourpy-1.3.2-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:e298e7e70cf4eb179cc1077be1c725b5fd131ebc81181bf0c03525c8abc297fd", size = 1372168 }, + { url = "https://files.pythonhosted.org/packages/0f/1b/96d586ccf1b1a9d2004dd519b25fbf104a11589abfd05484ff12199cca21/contourpy-1.3.2-cp313-cp313t-win32.whl", hash = "sha256:d0e589ae0d55204991450bb5c23f571c64fe43adaa53f93fc902a84c96f52fe1", size = 189550 }, + { url = "https://files.pythonhosted.org/packages/b0/e6/6000d0094e8a5e32ad62591c8609e269febb6e4db83a1c75ff8868b42731/contourpy-1.3.2-cp313-cp313t-win_amd64.whl", hash = "sha256:78e9253c3de756b3f6a5174d024c4835acd59eb3f8e2ca13e775dbffe1558f69", size = 238214 }, + { url = "https://files.pythonhosted.org/packages/33/05/b26e3c6ecc05f349ee0013f0bb850a761016d89cec528a98193a48c34033/contourpy-1.3.2-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:fd93cc7f3139b6dd7aab2f26a90dde0aa9fc264dbf70f6740d498a70b860b82c", size = 265681 }, + { url = "https://files.pythonhosted.org/packages/2b/25/ac07d6ad12affa7d1ffed11b77417d0a6308170f44ff20fa1d5aa6333f03/contourpy-1.3.2-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:107ba8a6a7eec58bb475329e6d3b95deba9440667c4d62b9b6063942b61d7f16", size = 315101 }, + { url = "https://files.pythonhosted.org/packages/8f/4d/5bb3192bbe9d3f27e3061a6a8e7733c9120e203cb8515767d30973f71030/contourpy-1.3.2-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:ded1706ed0c1049224531b81128efbd5084598f18d8a2d9efae833edbd2b40ad", size = 220599 }, + { url = "https://files.pythonhosted.org/packages/ff/c0/91f1215d0d9f9f343e4773ba6c9b89e8c0cc7a64a6263f21139da639d848/contourpy-1.3.2-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:5f5964cdad279256c084b69c3f412b7801e15356b16efa9d78aa974041903da0", size = 266807 }, + { url = "https://files.pythonhosted.org/packages/d4/79/6be7e90c955c0487e7712660d6cead01fa17bff98e0ea275737cc2bc8e71/contourpy-1.3.2-pp311-pypy311_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:49b65a95d642d4efa8f64ba12558fcb83407e58a2dfba9d796d77b63ccfcaff5", size = 318729 }, + { url = "https://files.pythonhosted.org/packages/87/68/7f46fb537958e87427d98a4074bcde4b67a70b04900cfc5ce29bc2f556c1/contourpy-1.3.2-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:8c5acb8dddb0752bf252e01a3035b21443158910ac16a3b0d20e7fed7d534ce5", size = 221791 }, +] + +[[package]] +name = "contourpy" +version = "1.3.3" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.12'", + "python_full_version == '3.11.*'", +] +dependencies = [ + { name = "numpy", version = "2.3.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/58/01/1253e6698a07380cd31a736d248a3f2a50a7c88779a1813da27503cadc2a/contourpy-1.3.3.tar.gz", hash = "sha256:083e12155b210502d0bca491432bb04d56dc3432f95a979b429f2848c3dbe880", size = 13466174 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/91/2e/c4390a31919d8a78b90e8ecf87cd4b4c4f05a5b48d05ec17db8e5404c6f4/contourpy-1.3.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:709a48ef9a690e1343202916450bc48b9e51c049b089c7f79a267b46cffcdaa1", size = 288773 }, + { url = "https://files.pythonhosted.org/packages/0d/44/c4b0b6095fef4dc9c420e041799591e3b63e9619e3044f7f4f6c21c0ab24/contourpy-1.3.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:23416f38bfd74d5d28ab8429cc4d63fa67d5068bd711a85edb1c3fb0c3e2f381", size = 270149 }, + { url = "https://files.pythonhosted.org/packages/30/2e/dd4ced42fefac8470661d7cb7e264808425e6c5d56d175291e93890cce09/contourpy-1.3.3-cp311-cp311-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:929ddf8c4c7f348e4c0a5a3a714b5c8542ffaa8c22954862a46ca1813b667ee7", size = 329222 }, + { url = "https://files.pythonhosted.org/packages/f2/74/cc6ec2548e3d276c71389ea4802a774b7aa3558223b7bade3f25787fafc2/contourpy-1.3.3-cp311-cp311-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:9e999574eddae35f1312c2b4b717b7885d4edd6cb46700e04f7f02db454e67c1", size = 377234 }, + { url = "https://files.pythonhosted.org/packages/03/b3/64ef723029f917410f75c09da54254c5f9ea90ef89b143ccadb09df14c15/contourpy-1.3.3-cp311-cp311-manylinux_2_26_s390x.manylinux_2_28_s390x.whl", hash = "sha256:0bf67e0e3f482cb69779dd3061b534eb35ac9b17f163d851e2a547d56dba0a3a", size = 380555 }, + { url = "https://files.pythonhosted.org/packages/5f/4b/6157f24ca425b89fe2eb7e7be642375711ab671135be21e6faa100f7448c/contourpy-1.3.3-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:51e79c1f7470158e838808d4a996fa9bac72c498e93d8ebe5119bc1e6becb0db", size = 355238 }, + { url = "https://files.pythonhosted.org/packages/98/56/f914f0dd678480708a04cfd2206e7c382533249bc5001eb9f58aa693e200/contourpy-1.3.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:598c3aaece21c503615fd59c92a3598b428b2f01bfb4b8ca9c4edeecc2438620", size = 1326218 }, + { url = "https://files.pythonhosted.org/packages/fb/d7/4a972334a0c971acd5172389671113ae82aa7527073980c38d5868ff1161/contourpy-1.3.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:322ab1c99b008dad206d406bb61d014cf0174df491ae9d9d0fac6a6fda4f977f", size = 1392867 }, + { url = "https://files.pythonhosted.org/packages/75/3e/f2cc6cd56dc8cff46b1a56232eabc6feea52720083ea71ab15523daab796/contourpy-1.3.3-cp311-cp311-win32.whl", hash = "sha256:fd907ae12cd483cd83e414b12941c632a969171bf90fc937d0c9f268a31cafff", size = 183677 }, + { url = "https://files.pythonhosted.org/packages/98/4b/9bd370b004b5c9d8045c6c33cf65bae018b27aca550a3f657cdc99acdbd8/contourpy-1.3.3-cp311-cp311-win_amd64.whl", hash = "sha256:3519428f6be58431c56581f1694ba8e50626f2dd550af225f82fb5f5814d2a42", size = 225234 }, + { url = "https://files.pythonhosted.org/packages/d9/b6/71771e02c2e004450c12b1120a5f488cad2e4d5b590b1af8bad060360fe4/contourpy-1.3.3-cp311-cp311-win_arm64.whl", hash = "sha256:15ff10bfada4bf92ec8b31c62bf7c1834c244019b4a33095a68000d7075df470", size = 193123 }, + { url = "https://files.pythonhosted.org/packages/be/45/adfee365d9ea3d853550b2e735f9d66366701c65db7855cd07621732ccfc/contourpy-1.3.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:b08a32ea2f8e42cf1d4be3169a98dd4be32bafe4f22b6c4cb4ba810fa9e5d2cb", size = 293419 }, + { url = "https://files.pythonhosted.org/packages/53/3e/405b59cfa13021a56bba395a6b3aca8cec012b45bf177b0eaf7a202cde2c/contourpy-1.3.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:556dba8fb6f5d8742f2923fe9457dbdd51e1049c4a43fd3986a0b14a1d815fc6", size = 273979 }, + { url = "https://files.pythonhosted.org/packages/d4/1c/a12359b9b2ca3a845e8f7f9ac08bdf776114eb931392fcad91743e2ea17b/contourpy-1.3.3-cp312-cp312-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:92d9abc807cf7d0e047b95ca5d957cf4792fcd04e920ca70d48add15c1a90ea7", size = 332653 }, + { url = "https://files.pythonhosted.org/packages/63/12/897aeebfb475b7748ea67b61e045accdfcf0d971f8a588b67108ed7f5512/contourpy-1.3.3-cp312-cp312-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:b2e8faa0ed68cb29af51edd8e24798bb661eac3bd9f65420c1887b6ca89987c8", size = 379536 }, + { url = "https://files.pythonhosted.org/packages/43/8a/a8c584b82deb248930ce069e71576fc09bd7174bbd35183b7943fb1064fd/contourpy-1.3.3-cp312-cp312-manylinux_2_26_s390x.manylinux_2_28_s390x.whl", hash = "sha256:626d60935cf668e70a5ce6ff184fd713e9683fb458898e4249b63be9e28286ea", size = 384397 }, + { url = "https://files.pythonhosted.org/packages/cc/8f/ec6289987824b29529d0dfda0d74a07cec60e54b9c92f3c9da4c0ac732de/contourpy-1.3.3-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4d00e655fcef08aba35ec9610536bfe90267d7ab5ba944f7032549c55a146da1", size = 362601 }, + { url = "https://files.pythonhosted.org/packages/05/0a/a3fe3be3ee2dceb3e615ebb4df97ae6f3828aa915d3e10549ce016302bd1/contourpy-1.3.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:451e71b5a7d597379ef572de31eeb909a87246974d960049a9848c3bc6c41bf7", size = 1331288 }, + { url = "https://files.pythonhosted.org/packages/33/1d/acad9bd4e97f13f3e2b18a3977fe1b4a37ecf3d38d815333980c6c72e963/contourpy-1.3.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:459c1f020cd59fcfe6650180678a9993932d80d44ccde1fa1868977438f0b411", size = 1403386 }, + { url = "https://files.pythonhosted.org/packages/cf/8f/5847f44a7fddf859704217a99a23a4f6417b10e5ab1256a179264561540e/contourpy-1.3.3-cp312-cp312-win32.whl", hash = "sha256:023b44101dfe49d7d53932be418477dba359649246075c996866106da069af69", size = 185018 }, + { url = "https://files.pythonhosted.org/packages/19/e8/6026ed58a64563186a9ee3f29f41261fd1828f527dd93d33b60feca63352/contourpy-1.3.3-cp312-cp312-win_amd64.whl", hash = "sha256:8153b8bfc11e1e4d75bcb0bff1db232f9e10b274e0929de9d608027e0d34ff8b", size = 226567 }, + { url = "https://files.pythonhosted.org/packages/d1/e2/f05240d2c39a1ed228d8328a78b6f44cd695f7ef47beb3e684cf93604f86/contourpy-1.3.3-cp312-cp312-win_arm64.whl", hash = "sha256:07ce5ed73ecdc4a03ffe3e1b3e3c1166db35ae7584be76f65dbbe28a7791b0cc", size = 193655 }, + { url = "https://files.pythonhosted.org/packages/68/35/0167aad910bbdb9599272bd96d01a9ec6852f36b9455cf2ca67bd4cc2d23/contourpy-1.3.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:177fb367556747a686509d6fef71d221a4b198a3905fe824430e5ea0fda54eb5", size = 293257 }, + { url = "https://files.pythonhosted.org/packages/96/e4/7adcd9c8362745b2210728f209bfbcf7d91ba868a2c5f40d8b58f54c509b/contourpy-1.3.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:d002b6f00d73d69333dac9d0b8d5e84d9724ff9ef044fd63c5986e62b7c9e1b1", size = 274034 }, + { url = "https://files.pythonhosted.org/packages/73/23/90e31ceeed1de63058a02cb04b12f2de4b40e3bef5e082a7c18d9c8ae281/contourpy-1.3.3-cp313-cp313-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:348ac1f5d4f1d66d3322420f01d42e43122f43616e0f194fc1c9f5d830c5b286", size = 334672 }, + { url = "https://files.pythonhosted.org/packages/ed/93/b43d8acbe67392e659e1d984700e79eb67e2acb2bd7f62012b583a7f1b55/contourpy-1.3.3-cp313-cp313-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:655456777ff65c2c548b7c454af9c6f33f16c8884f11083244b5819cc214f1b5", size = 381234 }, + { url = "https://files.pythonhosted.org/packages/46/3b/bec82a3ea06f66711520f75a40c8fc0b113b2a75edb36aa633eb11c4f50f/contourpy-1.3.3-cp313-cp313-manylinux_2_26_s390x.manylinux_2_28_s390x.whl", hash = "sha256:644a6853d15b2512d67881586bd03f462c7ab755db95f16f14d7e238f2852c67", size = 385169 }, + { url = "https://files.pythonhosted.org/packages/4b/32/e0f13a1c5b0f8572d0ec6ae2f6c677b7991fafd95da523159c19eff0696a/contourpy-1.3.3-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4debd64f124ca62069f313a9cb86656ff087786016d76927ae2cf37846b006c9", size = 362859 }, + { url = "https://files.pythonhosted.org/packages/33/71/e2a7945b7de4e58af42d708a219f3b2f4cff7386e6b6ab0a0fa0033c49a9/contourpy-1.3.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:a15459b0f4615b00bbd1e91f1b9e19b7e63aea7483d03d804186f278c0af2659", size = 1332062 }, + { url = "https://files.pythonhosted.org/packages/12/fc/4e87ac754220ccc0e807284f88e943d6d43b43843614f0a8afa469801db0/contourpy-1.3.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:ca0fdcd73925568ca027e0b17ab07aad764be4706d0a925b89227e447d9737b7", size = 1403932 }, + { url = "https://files.pythonhosted.org/packages/a6/2e/adc197a37443f934594112222ac1aa7dc9a98faf9c3842884df9a9d8751d/contourpy-1.3.3-cp313-cp313-win32.whl", hash = "sha256:b20c7c9a3bf701366556e1b1984ed2d0cedf999903c51311417cf5f591d8c78d", size = 185024 }, + { url = "https://files.pythonhosted.org/packages/18/0b/0098c214843213759692cc638fce7de5c289200a830e5035d1791d7a2338/contourpy-1.3.3-cp313-cp313-win_amd64.whl", hash = "sha256:1cadd8b8969f060ba45ed7c1b714fe69185812ab43bd6b86a9123fe8f99c3263", size = 226578 }, + { url = "https://files.pythonhosted.org/packages/8a/9a/2f6024a0c5995243cd63afdeb3651c984f0d2bc727fd98066d40e141ad73/contourpy-1.3.3-cp313-cp313-win_arm64.whl", hash = "sha256:fd914713266421b7536de2bfa8181aa8c699432b6763a0ea64195ebe28bff6a9", size = 193524 }, + { url = "https://files.pythonhosted.org/packages/c0/b3/f8a1a86bd3298513f500e5b1f5fd92b69896449f6cab6a146a5d52715479/contourpy-1.3.3-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:88df9880d507169449d434c293467418b9f6cbe82edd19284aa0409e7fdb933d", size = 306730 }, + { url = "https://files.pythonhosted.org/packages/3f/11/4780db94ae62fc0c2053909b65dc3246bd7cecfc4f8a20d957ad43aa4ad8/contourpy-1.3.3-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:d06bb1f751ba5d417047db62bca3c8fde202b8c11fb50742ab3ab962c81e8216", size = 287897 }, + { url = "https://files.pythonhosted.org/packages/ae/15/e59f5f3ffdd6f3d4daa3e47114c53daabcb18574a26c21f03dc9e4e42ff0/contourpy-1.3.3-cp313-cp313t-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e4e6b05a45525357e382909a4c1600444e2a45b4795163d3b22669285591c1ae", size = 326751 }, + { url = "https://files.pythonhosted.org/packages/0f/81/03b45cfad088e4770b1dcf72ea78d3802d04200009fb364d18a493857210/contourpy-1.3.3-cp313-cp313t-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:ab3074b48c4e2cf1a960e6bbeb7f04566bf36b1861d5c9d4d8ac04b82e38ba20", size = 375486 }, + { url = "https://files.pythonhosted.org/packages/0c/ba/49923366492ffbdd4486e970d421b289a670ae8cf539c1ea9a09822b371a/contourpy-1.3.3-cp313-cp313t-manylinux_2_26_s390x.manylinux_2_28_s390x.whl", hash = "sha256:6c3d53c796f8647d6deb1abe867daeb66dcc8a97e8455efa729516b997b8ed99", size = 388106 }, + { url = "https://files.pythonhosted.org/packages/9f/52/5b00ea89525f8f143651f9f03a0df371d3cbd2fccd21ca9b768c7a6500c2/contourpy-1.3.3-cp313-cp313t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:50ed930df7289ff2a8d7afeb9603f8289e5704755c7e5c3bbd929c90c817164b", size = 352548 }, + { url = "https://files.pythonhosted.org/packages/32/1d/a209ec1a3a3452d490f6b14dd92e72280c99ae3d1e73da74f8277d4ee08f/contourpy-1.3.3-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:4feffb6537d64b84877da813a5c30f1422ea5739566abf0bd18065ac040e120a", size = 1322297 }, + { url = "https://files.pythonhosted.org/packages/bc/9e/46f0e8ebdd884ca0e8877e46a3f4e633f6c9c8c4f3f6e72be3fe075994aa/contourpy-1.3.3-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:2b7e9480ffe2b0cd2e787e4df64270e3a0440d9db8dc823312e2c940c167df7e", size = 1391023 }, + { url = "https://files.pythonhosted.org/packages/b9/70/f308384a3ae9cd2209e0849f33c913f658d3326900d0ff5d378d6a1422d2/contourpy-1.3.3-cp313-cp313t-win32.whl", hash = "sha256:283edd842a01e3dcd435b1c5116798d661378d83d36d337b8dde1d16a5fc9ba3", size = 196157 }, + { url = "https://files.pythonhosted.org/packages/b2/dd/880f890a6663b84d9e34a6f88cded89d78f0091e0045a284427cb6b18521/contourpy-1.3.3-cp313-cp313t-win_amd64.whl", hash = "sha256:87acf5963fc2b34825e5b6b048f40e3635dd547f590b04d2ab317c2619ef7ae8", size = 240570 }, + { url = "https://files.pythonhosted.org/packages/80/99/2adc7d8ffead633234817ef8e9a87115c8a11927a94478f6bb3d3f4d4f7d/contourpy-1.3.3-cp313-cp313t-win_arm64.whl", hash = "sha256:3c30273eb2a55024ff31ba7d052dde990d7d8e5450f4bbb6e913558b3d6c2301", size = 199713 }, + { url = "https://files.pythonhosted.org/packages/72/8b/4546f3ab60f78c514ffb7d01a0bd743f90de36f0019d1be84d0a708a580a/contourpy-1.3.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:fde6c716d51c04b1c25d0b90364d0be954624a0ee9d60e23e850e8d48353d07a", size = 292189 }, + { url = "https://files.pythonhosted.org/packages/fd/e1/3542a9cb596cadd76fcef413f19c79216e002623158befe6daa03dbfa88c/contourpy-1.3.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:cbedb772ed74ff5be440fa8eee9bd49f64f6e3fc09436d9c7d8f1c287b121d77", size = 273251 }, + { url = "https://files.pythonhosted.org/packages/b1/71/f93e1e9471d189f79d0ce2497007731c1e6bf9ef6d1d61b911430c3db4e5/contourpy-1.3.3-cp314-cp314-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:22e9b1bd7a9b1d652cd77388465dc358dafcd2e217d35552424aa4f996f524f5", size = 335810 }, + { url = "https://files.pythonhosted.org/packages/91/f9/e35f4c1c93f9275d4e38681a80506b5510e9327350c51f8d4a5a724d178c/contourpy-1.3.3-cp314-cp314-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:a22738912262aa3e254e4f3cb079a95a67132fc5a063890e224393596902f5a4", size = 382871 }, + { url = "https://files.pythonhosted.org/packages/b5/71/47b512f936f66a0a900d81c396a7e60d73419868fba959c61efed7a8ab46/contourpy-1.3.3-cp314-cp314-manylinux_2_26_s390x.manylinux_2_28_s390x.whl", hash = "sha256:afe5a512f31ee6bd7d0dda52ec9864c984ca3d66664444f2d72e0dc4eb832e36", size = 386264 }, + { url = "https://files.pythonhosted.org/packages/04/5f/9ff93450ba96b09c7c2b3f81c94de31c89f92292f1380261bd7195bea4ea/contourpy-1.3.3-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f64836de09927cba6f79dcd00fdd7d5329f3fccc633468507079c829ca4db4e3", size = 363819 }, + { url = "https://files.pythonhosted.org/packages/3e/a6/0b185d4cc480ee494945cde102cb0149ae830b5fa17bf855b95f2e70ad13/contourpy-1.3.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:1fd43c3be4c8e5fd6e4f2baeae35ae18176cf2e5cced681cca908addf1cdd53b", size = 1333650 }, + { url = "https://files.pythonhosted.org/packages/43/d7/afdc95580ca56f30fbcd3060250f66cedbde69b4547028863abd8aa3b47e/contourpy-1.3.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:6afc576f7b33cf00996e5c1102dc2a8f7cc89e39c0b55df93a0b78c1bd992b36", size = 1404833 }, + { url = "https://files.pythonhosted.org/packages/e2/e2/366af18a6d386f41132a48f033cbd2102e9b0cf6345d35ff0826cd984566/contourpy-1.3.3-cp314-cp314-win32.whl", hash = "sha256:66c8a43a4f7b8df8b71ee1840e4211a3c8d93b214b213f590e18a1beca458f7d", size = 189692 }, + { url = "https://files.pythonhosted.org/packages/7d/c2/57f54b03d0f22d4044b8afb9ca0e184f8b1afd57b4f735c2fa70883dc601/contourpy-1.3.3-cp314-cp314-win_amd64.whl", hash = "sha256:cf9022ef053f2694e31d630feaacb21ea24224be1c3ad0520b13d844274614fd", size = 232424 }, + { url = "https://files.pythonhosted.org/packages/18/79/a9416650df9b525737ab521aa181ccc42d56016d2123ddcb7b58e926a42c/contourpy-1.3.3-cp314-cp314-win_arm64.whl", hash = "sha256:95b181891b4c71de4bb404c6621e7e2390745f887f2a026b2d99e92c17892339", size = 198300 }, + { url = "https://files.pythonhosted.org/packages/1f/42/38c159a7d0f2b7b9c04c64ab317042bb6952b713ba875c1681529a2932fe/contourpy-1.3.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:33c82d0138c0a062380332c861387650c82e4cf1747aaa6938b9b6516762e772", size = 306769 }, + { url = "https://files.pythonhosted.org/packages/c3/6c/26a8205f24bca10974e77460de68d3d7c63e282e23782f1239f226fcae6f/contourpy-1.3.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:ea37e7b45949df430fe649e5de8351c423430046a2af20b1c1961cae3afcda77", size = 287892 }, + { url = "https://files.pythonhosted.org/packages/66/06/8a475c8ab718ebfd7925661747dbb3c3ee9c82ac834ccb3570be49d129f4/contourpy-1.3.3-cp314-cp314t-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d304906ecc71672e9c89e87c4675dc5c2645e1f4269a5063b99b0bb29f232d13", size = 326748 }, + { url = "https://files.pythonhosted.org/packages/b4/a3/c5ca9f010a44c223f098fccd8b158bb1cb287378a31ac141f04730dc49be/contourpy-1.3.3-cp314-cp314t-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:ca658cd1a680a5c9ea96dc61cdbae1e85c8f25849843aa799dfd3cb370ad4fbe", size = 375554 }, + { url = "https://files.pythonhosted.org/packages/80/5b/68bd33ae63fac658a4145088c1e894405e07584a316738710b636c6d0333/contourpy-1.3.3-cp314-cp314t-manylinux_2_26_s390x.manylinux_2_28_s390x.whl", hash = "sha256:ab2fd90904c503739a75b7c8c5c01160130ba67944a7b77bbf36ef8054576e7f", size = 388118 }, + { url = "https://files.pythonhosted.org/packages/40/52/4c285a6435940ae25d7410a6c36bda5145839bc3f0beb20c707cda18b9d2/contourpy-1.3.3-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b7301b89040075c30e5768810bc96a8e8d78085b47d8be6e4c3f5a0b4ed478a0", size = 352555 }, + { url = "https://files.pythonhosted.org/packages/24/ee/3e81e1dd174f5c7fefe50e85d0892de05ca4e26ef1c9a59c2a57e43b865a/contourpy-1.3.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:2a2a8b627d5cc6b7c41a4beff6c5ad5eb848c88255fda4a8745f7e901b32d8e4", size = 1322295 }, + { url = "https://files.pythonhosted.org/packages/3c/b2/6d913d4d04e14379de429057cd169e5e00f6c2af3bb13e1710bcbdb5da12/contourpy-1.3.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:fd6ec6be509c787f1caf6b247f0b1ca598bef13f4ddeaa126b7658215529ba0f", size = 1391027 }, + { url = "https://files.pythonhosted.org/packages/93/8a/68a4ec5c55a2971213d29a9374913f7e9f18581945a7a31d1a39b5d2dfe5/contourpy-1.3.3-cp314-cp314t-win32.whl", hash = "sha256:e74a9a0f5e3fff48fb5a7f2fd2b9b70a3fe014a67522f79b7cca4c0c7e43c9ae", size = 202428 }, + { url = "https://files.pythonhosted.org/packages/fa/96/fd9f641ffedc4fa3ace923af73b9d07e869496c9cc7a459103e6e978992f/contourpy-1.3.3-cp314-cp314t-win_amd64.whl", hash = "sha256:13b68d6a62db8eafaebb8039218921399baf6e47bf85006fd8529f2a08ef33fc", size = 250331 }, + { url = "https://files.pythonhosted.org/packages/ae/8c/469afb6465b853afff216f9528ffda78a915ff880ed58813ba4faf4ba0b6/contourpy-1.3.3-cp314-cp314t-win_arm64.whl", hash = "sha256:b7448cb5a725bb1e35ce88771b86fba35ef418952474492cf7c764059933ff8b", size = 203831 }, + { url = "https://files.pythonhosted.org/packages/a5/29/8dcfe16f0107943fa92388c23f6e05cff0ba58058c4c95b00280d4c75a14/contourpy-1.3.3-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:cd5dfcaeb10f7b7f9dc8941717c6c2ade08f587be2226222c12b25f0483ed497", size = 278809 }, + { url = "https://files.pythonhosted.org/packages/85/a9/8b37ef4f7dafeb335daee3c8254645ef5725be4d9c6aa70b50ec46ef2f7e/contourpy-1.3.3-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:0c1fc238306b35f246d61a1d416a627348b5cf0648648a031e14bb8705fcdfe8", size = 261593 }, + { url = "https://files.pythonhosted.org/packages/0a/59/ebfb8c677c75605cc27f7122c90313fd2f375ff3c8d19a1694bda74aaa63/contourpy-1.3.3-pp311-pypy311_pp73-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:70f9aad7de812d6541d29d2bbf8feb22ff7e1c299523db288004e3157ff4674e", size = 302202 }, + { url = "https://files.pythonhosted.org/packages/3c/37/21972a15834d90bfbfb009b9d004779bd5a07a0ec0234e5ba8f64d5736f4/contourpy-1.3.3-pp311-pypy311_pp73-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5ed3657edf08512fc3fe81b510e35c2012fbd3081d2e26160f27ca28affec989", size = 329207 }, + { url = "https://files.pythonhosted.org/packages/0c/58/bd257695f39d05594ca4ad60df5bcb7e32247f9951fd09a9b8edb82d1daa/contourpy-1.3.3-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:3d1a3799d62d45c18bafd41c5fa05120b96a28079f2393af559b843d1a966a77", size = 225315 }, +] + +[[package]] +name = "cycler" +version = "0.12.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a9/95/a3dbbb5028f35eafb79008e7522a75244477d2838f38cbb722248dabc2a8/cycler-0.12.1.tar.gz", hash = "sha256:88bb128f02ba341da8ef447245a9e138fae777f6a23943da4540077d3601eb1c", size = 7615 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e7/05/c19819d5e3d95294a6f5947fb9b9629efb316b96de511b418c53d245aae6/cycler-0.12.1-py3-none-any.whl", hash = "sha256:85cef7cff222d8644161529808465972e51340599459b8ac3ccbac5a854e0d30", size = 8321 }, +] + +[[package]] +name = "cython" +version = "3.1.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/18/40/7b17cd866158238db704965da1b5849af261dbad393ea3ac966f934b2d39/cython-3.1.2.tar.gz", hash = "sha256:6bbf7a953fa6762dfecdec015e3b054ba51c0121a45ad851fa130f63f5331381", size = 3184825 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/4c/5e/c89172b252697acd6a440a2efead37685f8f2c42ea0d906098cbfb9aed69/cython-3.1.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:0f2add8b23cb19da3f546a688cd8f9e0bfc2776715ebf5e283bc3113b03ff008", size = 2973977 }, + { url = "https://files.pythonhosted.org/packages/9c/e5/d7fb67187193c5763d59a4b70d86a92be18b05b01737af8bfca7bafea0d3/cython-3.1.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:0d6248a2ae155ca4c42d7fa6a9a05154d62e695d7736bc17e1b85da6dcc361df", size = 2836988 }, + { url = "https://files.pythonhosted.org/packages/23/3a/5b92bfff9c1cc1179a493684d0e6a893ee7cd69c4f1977813000ea76c5d7/cython-3.1.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:262bf49d9da64e2a34c86cbf8de4aa37daffb0f602396f116cca1ed47dc4b9f2", size = 3212933 }, + { url = "https://files.pythonhosted.org/packages/b4/eb/8c47ba21177929f9122e7aceca9fe1f9f5a037e705226f8a5a9113fb53ba/cython-3.1.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ae53ae93c699d5f113953a9869df2fc269d8e173f9aa0616c6d8d6e12b4e9827", size = 3332955 }, + { url = "https://files.pythonhosted.org/packages/2a/a7/e29079146154c4c0403dfb5b9b51c183e0887fc19727aacc3946246c5898/cython-3.1.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b417c5d046ce676ee595ec7955ed47a68ad6f419cbf8c2a8708e55a3b38dfa35", size = 3394613 }, + { url = "https://files.pythonhosted.org/packages/94/18/dd10c4531c0e918b20300ee23b32a4bffa5cbacaa8e8dd19fa6b02b260fe/cython-3.1.2-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:af127da4b956e0e906e552fad838dc3fb6b6384164070ceebb0d90982a8ae25a", size = 3257573 }, + { url = "https://files.pythonhosted.org/packages/19/09/0998fa0c42c6cc56fdcba6bb757abe13fc4456a5a063dacb5331e30d7560/cython-3.1.2-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:9be3d4954b46fd0f2dceac011d470f658eaf819132db52fbd1cf226ee60348db", size = 3479007 }, + { url = "https://files.pythonhosted.org/packages/9d/1c/e107d8bc45ab1f3c2205c7f4a17b3c594126b72f7fc2d78b304f5ae72434/cython-3.1.2-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:63da49672c4bb022b4de9d37bab6c29953dbf5a31a2f40dffd0cf0915dcd7a17", size = 3414055 }, + { url = "https://files.pythonhosted.org/packages/13/25/5c1177bbc23263ba82b60a754383a001c57798d3f7982ea9b5fd3916c1fa/cython-3.1.2-cp310-cp310-win32.whl", hash = "sha256:2d8291dbbc1cb86b8d60c86fe9cbf99ec72de28cb157cbe869c95df4d32efa96", size = 2484860 }, + { url = "https://files.pythonhosted.org/packages/f5/19/119287fa7e3c8268d33ac6213fc7e7d6e9b74b239d459073d285362ebf2a/cython-3.1.2-cp310-cp310-win_amd64.whl", hash = "sha256:e1f30a1339e03c80968a371ef76bf27a6648c5646cccd14a97e731b6957db97a", size = 2679771 }, + { url = "https://files.pythonhosted.org/packages/1f/de/502ddebaf5fe78f13cd6361acdd74710d3a5b15c22a9edc0ea4c873a59a5/cython-3.1.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:5548573e0912d7dc80579827493315384c462e2f15797b91a8ed177686d31eb9", size = 3007792 }, + { url = "https://files.pythonhosted.org/packages/bb/c8/91b00bc68effba9ba1ff5b33988052ac4d98fc1ac3021ade7261661299c6/cython-3.1.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:4bf3ea5bc50d80762c490f42846820a868a6406fdb5878ae9e4cc2f11b50228a", size = 2870798 }, + { url = "https://files.pythonhosted.org/packages/f4/4b/29d290f14607785112c00a5e1685d766f433531bbd6a11ad229ab61b7a70/cython-3.1.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:20ce53951d06ab2bca39f153d9c5add1d631c2a44d58bf67288c9d631be9724e", size = 3131280 }, + { url = "https://files.pythonhosted.org/packages/38/3c/7c61e9ce25377ec7c4aa0b7ceeed34559ebca7b5cfd384672ba64eeaa4ba/cython-3.1.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e05a36224e3002d48c7c1c695b3771343bd16bc57eab60d6c5d5e08f3cbbafd8", size = 3223898 }, + { url = "https://files.pythonhosted.org/packages/10/96/2d3fbe7e50e98b53ac86fefb48b64262b2e1304b3495e8e25b3cd1c3473e/cython-3.1.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:dbc0fc0777c7ab82297c01c61a1161093a22a41714f62e8c35188a309bd5db8e", size = 3291527 }, + { url = "https://files.pythonhosted.org/packages/bd/e4/4cd3624e250d86f05bdb121a567865b9cca75cdc6dce4eedd68e626ea4f8/cython-3.1.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:18161ef3dd0e90a944daa2be468dd27696712a5f792d6289e97d2a31298ad688", size = 3184034 }, + { url = "https://files.pythonhosted.org/packages/24/de/f8c1243c3e50ec95cb81f3a7936c8cf162f28050db8683e291c3861b46a0/cython-3.1.2-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:ca45020950cd52d82189d6dfb6225737586be6fe7b0b9d3fadd7daca62eff531", size = 3386084 }, + { url = "https://files.pythonhosted.org/packages/c8/95/2365937da44741ef0781bb9ecc1f8f52b38b65acb7293b5fc7c3eaee5346/cython-3.1.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:aaae97d6d07610224be2b73a93e9e3dd85c09aedfd8e47054e3ef5a863387dae", size = 3309974 }, + { url = "https://files.pythonhosted.org/packages/9b/b8/280eed114110a1a3aa9e2e76bcd06cdd5ef0df7ab77c0be9d5378ca28c57/cython-3.1.2-cp311-cp311-win32.whl", hash = "sha256:3d439d9b19e7e70f6ff745602906d282a853dd5219d8e7abbf355de680c9d120", size = 2482942 }, + { url = "https://files.pythonhosted.org/packages/a2/50/0aa65be5a4ab65bde3224b8fd23ed795f699d1e724ac109bb0a32036b82d/cython-3.1.2-cp311-cp311-win_amd64.whl", hash = "sha256:8efa44ee2f1876e40eb5e45f6513a19758077c56bf140623ccab43d31f873b61", size = 2686535 }, + { url = "https://files.pythonhosted.org/packages/22/86/9393ab7204d5bb65f415dd271b658c18f57b9345d06002cae069376a5a7a/cython-3.1.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:9c2c4b6f9a941c857b40168b3f3c81d514e509d985c2dcd12e1a4fea9734192e", size = 3015898 }, + { url = "https://files.pythonhosted.org/packages/f9/b8/3d10ac37ab7b7ee60bc6bfb48f6682ebee7fddaccf56e1e135f0d46ca79f/cython-3.1.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:bdbc115bbe1b8c1dcbcd1b03748ea87fa967eb8dfc3a1a9bb243d4a382efcff4", size = 2846204 }, + { url = "https://files.pythonhosted.org/packages/f8/34/637771d8e10ebabc34a34cdd0d63fe797b66c334e150189955bf6442d710/cython-3.1.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c05111f89db1ca98edc0675cfaa62be47b3ff519a29876eb095532a9f9e052b8", size = 3080671 }, + { url = "https://files.pythonhosted.org/packages/6b/c8/383ad1851fb272920a152c5a30bb6f08c3471b5438079d9488fc3074a170/cython-3.1.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f6e7188df8709be32cfdfadc7c3782e361c929df9132f95e1bbc90a340dca3c7", size = 3199022 }, + { url = "https://files.pythonhosted.org/packages/e6/11/20adc8f2db37a29f245e8fd4b8b8a8245fce4bbbd128185cc9a7b1065e4c/cython-3.1.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1c0ecc71e60a051732c2607b8eb8f2a03a5dac09b28e52b8af323c329db9987b", size = 3241337 }, + { url = "https://files.pythonhosted.org/packages/6f/0b/491f1fd3e177cccb6bb6d52f9609f78d395edde83ac47ebb06d21717ca29/cython-3.1.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:f27143cf88835c8bcc9bf3304953f23f377d1d991e8942982fe7be344c7cfce3", size = 3131808 }, + { url = "https://files.pythonhosted.org/packages/db/d2/5e7053a3214c9baa7ad72940555eb87cf4750e597f10b2bb43db62c3f39f/cython-3.1.2-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:d8c43566701133f53bf13485839d8f3f309095fe0d3b9d0cd5873073394d2edc", size = 3340319 }, + { url = "https://files.pythonhosted.org/packages/95/42/4842f8ddac9b36c94ae08b23c7fcde3f930c1dd49ac8992bb5320a4d96b5/cython-3.1.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:a3bb893e85f027a929c1764bb14db4c31cbdf8a96f59a78f608f2ba7cfbbce95", size = 3287370 }, + { url = "https://files.pythonhosted.org/packages/03/0d/417745ed75d414176e50310087b43299a3e611e75c379ff998f60f2ca1a8/cython-3.1.2-cp312-cp312-win32.whl", hash = "sha256:12c5902f105e43ca9af7874cdf87a23627f98c15d5a4f6d38bc9d334845145c0", size = 2487734 }, + { url = "https://files.pythonhosted.org/packages/8e/82/df61d09ab81979ba171a8252af8fb8a3b26a0f19d1330c2679c11fe41667/cython-3.1.2-cp312-cp312-win_amd64.whl", hash = "sha256:06789eb7bd2e55b38b9dd349e9309f794aee0fed99c26ea5c9562d463877763f", size = 2695542 }, + { url = "https://files.pythonhosted.org/packages/7b/e2/355354a00a4ee7029b89767a280272f91c7e68b6edb686690992aaa6c32c/cython-3.1.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:cc22e5f18af436c894b90c257130346930fdc860d7f42b924548c591672beeef", size = 2999991 }, + { url = "https://files.pythonhosted.org/packages/7c/d6/fb1033396585fd900adda9a410624b96d2a37b5f7f3685f0bdc5fa2bafe0/cython-3.1.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:42c7bffb0fe9898996c7eef9eb74ce3654553c7a3a3f3da66e5a49f801904ce0", size = 2831764 }, + { url = "https://files.pythonhosted.org/packages/28/46/2bbcd5a8a67e4ec0dbdf73b0b85add085e401d782cdc9291673aeaf05fc2/cython-3.1.2-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:88dc7fd54bfae78c366c6106a759f389000ea4dfe8ed9568af9d2f612825a164", size = 3068467 }, + { url = "https://files.pythonhosted.org/packages/b3/9b/20a8a12d1454416141479380f7722f2ad298d2b41d0d7833fc409894715d/cython-3.1.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:80d0ce057672ca50728153757d022842d5dcec536b50c79615a22dda2a874ea0", size = 3186690 }, + { url = "https://files.pythonhosted.org/packages/3a/9f/20cdecae7966dfbaff198952bcee745e402072a3b6565dfebb41202b55f8/cython-3.1.2-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:eda6a43f1b78eae0d841698916eef661d15f8bc8439c266a964ea4c504f05612", size = 3212888 }, + { url = "https://files.pythonhosted.org/packages/fe/6a/ae723af7a2c9fe9e737468c046d953b34d427093c4974d34c15174cf7efe/cython-3.1.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:b4c516d103e87c2e9c1ab85227e4d91c7484c1ba29e25f8afbf67bae93fee164", size = 3117859 }, + { url = "https://files.pythonhosted.org/packages/f5/e9/25a5f5c962f2f331dc2ff74a62046e35ec0ffd08f0da6fa51261101a5e2e/cython-3.1.2-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:7542f1d18ab2cd22debc72974ec9e53437a20623d47d6001466e430538d7df54", size = 3315382 }, + { url = "https://files.pythonhosted.org/packages/4f/d2/2ee59f5e31b1d7e397ca0f3899559681a44dd3502fa8b68d2bb285f54aa7/cython-3.1.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:63335513c06dcec4ecdaa8598f36c969032149ffd92a461f641ee363dc83c7ad", size = 3273216 }, + { url = "https://files.pythonhosted.org/packages/a7/88/e792eb40d8a17010793da2f6c0f72624ec2b7964fccba8d5c544aed16400/cython-3.1.2-cp313-cp313-win32.whl", hash = "sha256:b377d542299332bfeb61ec09c57821b10f1597304394ba76544f4d07780a16df", size = 2482057 }, + { url = "https://files.pythonhosted.org/packages/c2/94/65ba40faeafe74845ba22b61aff7d73475671c3bd24bffc6cba53f3b0063/cython-3.1.2-cp313-cp313-win_amd64.whl", hash = "sha256:8ab1319c77f15b0ae04b3fb03588df3afdec4cf79e90eeea5c961e0ebd8fdf72", size = 2693103 }, + { url = "https://files.pythonhosted.org/packages/25/d6/ef8557d5e75cc57d55df579af4976935ee111a85bbee4a5b72354e257066/cython-3.1.2-py3-none-any.whl", hash = "sha256:d23fd7ffd7457205f08571a42b108a3cf993e83a59fe4d72b42e6fc592cf2639", size = 1224753 }, +] + +[[package]] +name = "exceptiongroup" +version = "1.3.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions", marker = "python_full_version < '3.11'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/0b/9f/a65090624ecf468cdca03533906e7c69ed7588582240cfe7cc9e770b50eb/exceptiongroup-1.3.0.tar.gz", hash = "sha256:b241f5885f560bc56a59ee63ca4c6a8bfa46ae4ad651af316d4e81817bb9fd88", size = 29749 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/36/f4/c6e662dade71f56cd2f3735141b265c3c79293c109549c1e6933b0651ffc/exceptiongroup-1.3.0-py3-none-any.whl", hash = "sha256:4d111e6e0c13d0644cad6ddaa7ed0261a0b36971f6d23e7ec9b4b9097da78a10", size = 16674 }, +] + +[[package]] +name = "filelock" +version = "3.18.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/0a/10/c23352565a6544bdc5353e0b15fc1c563352101f30e24bf500207a54df9a/filelock-3.18.0.tar.gz", hash = "sha256:adbc88eabb99d2fec8c9c1b229b171f18afa655400173ddc653d5d01501fb9f2", size = 18075 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/4d/36/2a115987e2d8c300a974597416d9de88f2444426de9571f4b59b2cca3acc/filelock-3.18.0-py3-none-any.whl", hash = "sha256:c401f4f8377c4464e6db25fff06205fd89bdd83b65eb0488ed1b160f780e21de", size = 16215 }, +] + +[[package]] +name = "fonttools" +version = "4.59.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/8a/27/ec3c723bfdf86f34c5c82bf6305df3e0f0d8ea798d2d3a7cb0c0a866d286/fonttools-4.59.0.tar.gz", hash = "sha256:be392ec3529e2f57faa28709d60723a763904f71a2b63aabe14fee6648fe3b14", size = 3532521 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1c/1f/3dcae710b7c4b56e79442b03db64f6c9f10c3348f7af40339dffcefb581e/fonttools-4.59.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:524133c1be38445c5c0575eacea42dbd44374b310b1ffc4b60ff01d881fabb96", size = 2761846 }, + { url = "https://files.pythonhosted.org/packages/eb/0e/ae3a1884fa1549acac1191cc9ec039142f6ac0e9cbc139c2e6a3dab967da/fonttools-4.59.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:21e606b2d38fed938dde871c5736822dd6bda7a4631b92e509a1f5cd1b90c5df", size = 2332060 }, + { url = "https://files.pythonhosted.org/packages/75/46/58bff92a7216829159ac7bdb1d05a48ad1b8ab8c539555f12d29fdecfdd4/fonttools-4.59.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e93df708c69a193fc7987192f94df250f83f3851fda49413f02ba5dded639482", size = 4852354 }, + { url = "https://files.pythonhosted.org/packages/05/57/767e31e48861045d89691128bd81fd4c62b62150f9a17a666f731ce4f197/fonttools-4.59.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:62224a9bb85b4b66d1b46d45cbe43d71cbf8f527d332b177e3b96191ffbc1e64", size = 4781132 }, + { url = "https://files.pythonhosted.org/packages/d7/78/adb5e9b0af5c6ce469e8b0e112f144eaa84b30dd72a486e9c778a9b03b31/fonttools-4.59.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:b8974b2a266b54c96709bd5e239979cddfd2dbceed331aa567ea1d7c4a2202db", size = 4832901 }, + { url = "https://files.pythonhosted.org/packages/ac/92/bc3881097fbf3d56d112bec308c863c058e5d4c9c65f534e8ae58450ab8a/fonttools-4.59.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:209b75943d158f610b78320eacb5539aa9e920bee2c775445b2846c65d20e19d", size = 4940140 }, + { url = "https://files.pythonhosted.org/packages/4a/54/39cdb23f0eeda2e07ae9cb189f2b6f41da89aabc682d3a387b3ff4a4ed29/fonttools-4.59.0-cp310-cp310-win32.whl", hash = "sha256:4c908a7036f0f3677f8afa577bcd973e3e20ddd2f7c42a33208d18bee95cdb6f", size = 2215890 }, + { url = "https://files.pythonhosted.org/packages/d8/eb/f8388d9e19f95d8df2449febe9b1a38ddd758cfdb7d6de3a05198d785d61/fonttools-4.59.0-cp310-cp310-win_amd64.whl", hash = "sha256:8b4309a2775e4feee7356e63b163969a215d663399cce1b3d3b65e7ec2d9680e", size = 2260191 }, + { url = "https://files.pythonhosted.org/packages/06/96/520733d9602fa1bf6592e5354c6721ac6fc9ea72bc98d112d0c38b967199/fonttools-4.59.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:841b2186adce48903c0fef235421ae21549020eca942c1da773ac380b056ab3c", size = 2782387 }, + { url = "https://files.pythonhosted.org/packages/87/6a/170fce30b9bce69077d8eec9bea2cfd9f7995e8911c71be905e2eba6368b/fonttools-4.59.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:9bcc1e77fbd1609198966ded6b2a9897bd6c6bcbd2287a2fc7d75f1a254179c5", size = 2342194 }, + { url = "https://files.pythonhosted.org/packages/b0/b6/7c8166c0066856f1408092f7968ac744060cf72ca53aec9036106f57eeca/fonttools-4.59.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:37c377f7cb2ab2eca8a0b319c68146d34a339792f9420fca6cd49cf28d370705", size = 5032333 }, + { url = "https://files.pythonhosted.org/packages/eb/0c/707c5a19598eafcafd489b73c4cb1c142102d6197e872f531512d084aa76/fonttools-4.59.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:fa39475eaccb98f9199eccfda4298abaf35ae0caec676ffc25b3a5e224044464", size = 4974422 }, + { url = "https://files.pythonhosted.org/packages/f6/e7/6d33737d9fe632a0f59289b6f9743a86d2a9d0673de2a0c38c0f54729822/fonttools-4.59.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:d3972b13148c1d1fbc092b27678a33b3080d1ac0ca305742b0119b75f9e87e38", size = 5010631 }, + { url = "https://files.pythonhosted.org/packages/63/e1/a4c3d089ab034a578820c8f2dff21ef60daf9668034a1e4fb38bb1cc3398/fonttools-4.59.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:a408c3c51358c89b29cfa5317cf11518b7ce5de1717abb55c5ae2d2921027de6", size = 5122198 }, + { url = "https://files.pythonhosted.org/packages/09/77/ca82b9c12fa4de3c520b7760ee61787640cf3fde55ef1b0bfe1de38c8153/fonttools-4.59.0-cp311-cp311-win32.whl", hash = "sha256:6770d7da00f358183d8fd5c4615436189e4f683bdb6affb02cad3d221d7bb757", size = 2214216 }, + { url = "https://files.pythonhosted.org/packages/ab/25/5aa7ca24b560b2f00f260acf32c4cf29d7aaf8656e159a336111c18bc345/fonttools-4.59.0-cp311-cp311-win_amd64.whl", hash = "sha256:84fc186980231a287b28560d3123bd255d3c6b6659828c642b4cf961e2b923d0", size = 2261879 }, + { url = "https://files.pythonhosted.org/packages/e2/77/b1c8af22f4265e951cd2e5535dbef8859efcef4fb8dee742d368c967cddb/fonttools-4.59.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:f9b3a78f69dcbd803cf2fb3f972779875b244c1115481dfbdd567b2c22b31f6b", size = 2767562 }, + { url = "https://files.pythonhosted.org/packages/ff/5a/aeb975699588176bb357e8b398dfd27e5d3a2230d92b81ab8cbb6187358d/fonttools-4.59.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:57bb7e26928573ee7c6504f54c05860d867fd35e675769f3ce01b52af38d48e2", size = 2335168 }, + { url = "https://files.pythonhosted.org/packages/54/97/c6101a7e60ae138c4ef75b22434373a0da50a707dad523dd19a4889315bf/fonttools-4.59.0-cp312-cp312-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:4536f2695fe5c1ffb528d84a35a7d3967e5558d2af58b4775e7ab1449d65767b", size = 4909850 }, + { url = "https://files.pythonhosted.org/packages/bd/6c/fa4d18d641054f7bff878cbea14aa9433f292b9057cb1700d8e91a4d5f4f/fonttools-4.59.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:885bde7d26e5b40e15c47bd5def48b38cbd50830a65f98122a8fb90962af7cd1", size = 4955131 }, + { url = "https://files.pythonhosted.org/packages/20/5c/331947fc1377deb928a69bde49f9003364f5115e5cbe351eea99e39412a2/fonttools-4.59.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:6801aeddb6acb2c42eafa45bc1cb98ba236871ae6f33f31e984670b749a8e58e", size = 4899667 }, + { url = "https://files.pythonhosted.org/packages/8a/46/b66469dfa26b8ff0baa7654b2cc7851206c6d57fe3abdabbaab22079a119/fonttools-4.59.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:31003b6a10f70742a63126b80863ab48175fb8272a18ca0846c0482968f0588e", size = 5051349 }, + { url = "https://files.pythonhosted.org/packages/2e/05/ebfb6b1f3a4328ab69787d106a7d92ccde77ce66e98659df0f9e3f28d93d/fonttools-4.59.0-cp312-cp312-win32.whl", hash = "sha256:fbce6dae41b692a5973d0f2158f782b9ad05babc2c2019a970a1094a23909b1b", size = 2201315 }, + { url = "https://files.pythonhosted.org/packages/09/45/d2bdc9ea20bbadec1016fd0db45696d573d7a26d95ab5174ffcb6d74340b/fonttools-4.59.0-cp312-cp312-win_amd64.whl", hash = "sha256:332bfe685d1ac58ca8d62b8d6c71c2e52a6c64bc218dc8f7825c9ea51385aa01", size = 2249408 }, + { url = "https://files.pythonhosted.org/packages/f3/bb/390990e7c457d377b00890d9f96a3ca13ae2517efafb6609c1756e213ba4/fonttools-4.59.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:78813b49d749e1bb4db1c57f2d4d7e6db22c253cb0a86ad819f5dc197710d4b2", size = 2758704 }, + { url = "https://files.pythonhosted.org/packages/df/6f/d730d9fcc9b410a11597092bd2eb9ca53e5438c6cb90e4b3047ce1b723e9/fonttools-4.59.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:401b1941ce37e78b8fd119b419b617277c65ae9417742a63282257434fd68ea2", size = 2330764 }, + { url = "https://files.pythonhosted.org/packages/75/b4/b96bb66f6f8cc4669de44a158099b249c8159231d254ab6b092909388be5/fonttools-4.59.0-cp313-cp313-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:efd7e6660674e234e29937bc1481dceb7e0336bfae75b856b4fb272b5093c5d4", size = 4890699 }, + { url = "https://files.pythonhosted.org/packages/b5/57/7969af50b26408be12baa317c6147588db5b38af2759e6df94554dbc5fdb/fonttools-4.59.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:51ab1ff33c19e336c02dee1e9fd1abd974a4ca3d8f7eef2a104d0816a241ce97", size = 4952934 }, + { url = "https://files.pythonhosted.org/packages/d6/e2/dd968053b6cf1f46c904f5bd409b22341477c017d8201619a265e50762d3/fonttools-4.59.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:a9bf8adc9e1f3012edc8f09b08336272aec0c55bc677422273e21280db748f7c", size = 4892319 }, + { url = "https://files.pythonhosted.org/packages/6b/95/a59810d8eda09129f83467a4e58f84205dc6994ebaeb9815406363e07250/fonttools-4.59.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:37e01c6ec0c98599778c2e688350d624fa4770fbd6144551bd5e032f1199171c", size = 5034753 }, + { url = "https://files.pythonhosted.org/packages/a5/84/51a69ee89ff8d1fea0c6997e946657e25a3f08513de8435fe124929f3eef/fonttools-4.59.0-cp313-cp313-win32.whl", hash = "sha256:70d6b3ceaa9cc5a6ac52884f3b3d9544e8e231e95b23f138bdb78e6d4dc0eae3", size = 2199688 }, + { url = "https://files.pythonhosted.org/packages/a0/ee/f626cd372932d828508137a79b85167fdcf3adab2e3bed433f295c596c6a/fonttools-4.59.0-cp313-cp313-win_amd64.whl", hash = "sha256:26731739daa23b872643f0e4072d5939960237d540c35c14e6a06d47d71ca8fe", size = 2248560 }, + { url = "https://files.pythonhosted.org/packages/d0/9c/df0ef2c51845a13043e5088f7bb988ca6cd5bb82d5d4203d6a158aa58cf2/fonttools-4.59.0-py3-none-any.whl", hash = "sha256:241313683afd3baacb32a6bd124d0bce7404bc5280e12e291bae1b9bba28711d", size = 1128050 }, +] + +[[package]] +name = "fsspec" +version = "2025.7.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/8b/02/0835e6ab9cfc03916fe3f78c0956cfcdb6ff2669ffa6651065d5ebf7fc98/fsspec-2025.7.0.tar.gz", hash = "sha256:786120687ffa54b8283d942929540d8bc5ccfa820deb555a2b5d0ed2b737bf58", size = 304432 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2f/e0/014d5d9d7a4564cf1c40b5039bc882db69fd881111e03ab3657ac0b218e2/fsspec-2025.7.0-py3-none-any.whl", hash = "sha256:8b012e39f63c7d5f10474de957f3ab793b47b45ae7d39f2fb735f8bbe25c0e21", size = 199597 }, +] + +[[package]] +name = "grpcio" +version = "1.74.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/38/b4/35feb8f7cab7239c5b94bd2db71abb3d6adb5f335ad8f131abb6060840b6/grpcio-1.74.0.tar.gz", hash = "sha256:80d1f4fbb35b0742d3e3d3bb654b7381cd5f015f8497279a1e9c21ba623e01b1", size = 12756048 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/66/54/68e51a90797ad7afc5b0a7881426c337f6a9168ebab73c3210b76aa7c90d/grpcio-1.74.0-cp310-cp310-linux_armv7l.whl", hash = "sha256:85bd5cdf4ed7b2d6438871adf6afff9af7096486fcf51818a81b77ef4dd30907", size = 5481935 }, + { url = "https://files.pythonhosted.org/packages/32/2a/af817c7e9843929e93e54d09c9aee2555c2e8d81b93102a9426b36e91833/grpcio-1.74.0-cp310-cp310-macosx_11_0_universal2.whl", hash = "sha256:68c8ebcca945efff9d86d8d6d7bfb0841cf0071024417e2d7f45c5e46b5b08eb", size = 10986796 }, + { url = "https://files.pythonhosted.org/packages/d5/94/d67756638d7bb07750b07d0826c68e414124574b53840ba1ff777abcd388/grpcio-1.74.0-cp310-cp310-manylinux_2_17_aarch64.whl", hash = "sha256:e154d230dc1bbbd78ad2fdc3039fa50ad7ffcf438e4eb2fa30bce223a70c7486", size = 5983663 }, + { url = "https://files.pythonhosted.org/packages/35/f5/c5e4853bf42148fea8532d49e919426585b73eafcf379a712934652a8de9/grpcio-1.74.0-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e8978003816c7b9eabe217f88c78bc26adc8f9304bf6a594b02e5a49b2ef9c11", size = 6653765 }, + { url = "https://files.pythonhosted.org/packages/fd/75/a1991dd64b331d199935e096cc9daa3415ee5ccbe9f909aa48eded7bba34/grpcio-1.74.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c3d7bd6e3929fd2ea7fbc3f562e4987229ead70c9ae5f01501a46701e08f1ad9", size = 6215172 }, + { url = "https://files.pythonhosted.org/packages/01/a4/7cef3dbb3b073d0ce34fd507efc44ac4c9442a0ef9fba4fb3f5c551efef5/grpcio-1.74.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:136b53c91ac1d02c8c24201bfdeb56f8b3ac3278668cbb8e0ba49c88069e1bdc", size = 6329142 }, + { url = "https://files.pythonhosted.org/packages/bf/d3/587920f882b46e835ad96014087054655312400e2f1f1446419e5179a383/grpcio-1.74.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:fe0f540750a13fd8e5da4b3eaba91a785eea8dca5ccd2bc2ffe978caa403090e", size = 7018632 }, + { url = "https://files.pythonhosted.org/packages/1f/95/c70a3b15a0bc83334b507e3d2ae20ee8fa38d419b8758a4d838f5c2a7d32/grpcio-1.74.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:4e4181bfc24413d1e3a37a0b7889bea68d973d4b45dd2bc68bb766c140718f82", size = 6509641 }, + { url = "https://files.pythonhosted.org/packages/4b/06/2e7042d06247d668ae69ea6998eca33f475fd4e2855f94dcb2aa5daef334/grpcio-1.74.0-cp310-cp310-win32.whl", hash = "sha256:1733969040989f7acc3d94c22f55b4a9501a30f6aaacdbccfaba0a3ffb255ab7", size = 3817478 }, + { url = "https://files.pythonhosted.org/packages/93/20/e02b9dcca3ee91124060b65bbf5b8e1af80b3b76a30f694b44b964ab4d71/grpcio-1.74.0-cp310-cp310-win_amd64.whl", hash = "sha256:9e912d3c993a29df6c627459af58975b2e5c897d93287939b9d5065f000249b5", size = 4493971 }, + { url = "https://files.pythonhosted.org/packages/e7/77/b2f06db9f240a5abeddd23a0e49eae2b6ac54d85f0e5267784ce02269c3b/grpcio-1.74.0-cp311-cp311-linux_armv7l.whl", hash = "sha256:69e1a8180868a2576f02356565f16635b99088da7df3d45aaa7e24e73a054e31", size = 5487368 }, + { url = "https://files.pythonhosted.org/packages/48/99/0ac8678a819c28d9a370a663007581744a9f2a844e32f0fa95e1ddda5b9e/grpcio-1.74.0-cp311-cp311-macosx_11_0_universal2.whl", hash = "sha256:8efe72fde5500f47aca1ef59495cb59c885afe04ac89dd11d810f2de87d935d4", size = 10999804 }, + { url = "https://files.pythonhosted.org/packages/45/c6/a2d586300d9e14ad72e8dc211c7aecb45fe9846a51e558c5bca0c9102c7f/grpcio-1.74.0-cp311-cp311-manylinux_2_17_aarch64.whl", hash = "sha256:a8f0302f9ac4e9923f98d8e243939a6fb627cd048f5cd38595c97e38020dffce", size = 5987667 }, + { url = "https://files.pythonhosted.org/packages/c9/57/5f338bf56a7f22584e68d669632e521f0de460bb3749d54533fc3d0fca4f/grpcio-1.74.0-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2f609a39f62a6f6f05c7512746798282546358a37ea93c1fcbadf8b2fed162e3", size = 6655612 }, + { url = "https://files.pythonhosted.org/packages/82/ea/a4820c4c44c8b35b1903a6c72a5bdccec92d0840cf5c858c498c66786ba5/grpcio-1.74.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c98e0b7434a7fa4e3e63f250456eaef52499fba5ae661c58cc5b5477d11e7182", size = 6219544 }, + { url = "https://files.pythonhosted.org/packages/a4/17/0537630a921365928f5abb6d14c79ba4dcb3e662e0dbeede8af4138d9dcf/grpcio-1.74.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:662456c4513e298db6d7bd9c3b8df6f75f8752f0ba01fb653e252ed4a59b5a5d", size = 6334863 }, + { url = "https://files.pythonhosted.org/packages/e2/a6/85ca6cb9af3f13e1320d0a806658dca432ff88149d5972df1f7b51e87127/grpcio-1.74.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:3d14e3c4d65e19d8430a4e28ceb71ace4728776fd6c3ce34016947474479683f", size = 7019320 }, + { url = "https://files.pythonhosted.org/packages/4f/a7/fe2beab970a1e25d2eff108b3cf4f7d9a53c185106377a3d1989216eba45/grpcio-1.74.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:1bf949792cee20d2078323a9b02bacbbae002b9e3b9e2433f2741c15bdeba1c4", size = 6514228 }, + { url = "https://files.pythonhosted.org/packages/6a/c2/2f9c945c8a248cebc3ccda1b7a1bf1775b9d7d59e444dbb18c0014e23da6/grpcio-1.74.0-cp311-cp311-win32.whl", hash = "sha256:55b453812fa7c7ce2f5c88be3018fb4a490519b6ce80788d5913f3f9d7da8c7b", size = 3817216 }, + { url = "https://files.pythonhosted.org/packages/ff/d1/a9cf9c94b55becda2199299a12b9feef0c79946b0d9d34c989de6d12d05d/grpcio-1.74.0-cp311-cp311-win_amd64.whl", hash = "sha256:86ad489db097141a907c559988c29718719aa3e13370d40e20506f11b4de0d11", size = 4495380 }, + { url = "https://files.pythonhosted.org/packages/4c/5d/e504d5d5c4469823504f65687d6c8fb97b7f7bf0b34873b7598f1df24630/grpcio-1.74.0-cp312-cp312-linux_armv7l.whl", hash = "sha256:8533e6e9c5bd630ca98062e3a1326249e6ada07d05acf191a77bc33f8948f3d8", size = 5445551 }, + { url = "https://files.pythonhosted.org/packages/43/01/730e37056f96f2f6ce9f17999af1556df62ee8dab7fa48bceeaab5fd3008/grpcio-1.74.0-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:2918948864fec2a11721d91568effffbe0a02b23ecd57f281391d986847982f6", size = 10979810 }, + { url = "https://files.pythonhosted.org/packages/79/3d/09fd100473ea5c47083889ca47ffd356576173ec134312f6aa0e13111dee/grpcio-1.74.0-cp312-cp312-manylinux_2_17_aarch64.whl", hash = "sha256:60d2d48b0580e70d2e1954d0d19fa3c2e60dd7cbed826aca104fff518310d1c5", size = 5941946 }, + { url = "https://files.pythonhosted.org/packages/8a/99/12d2cca0a63c874c6d3d195629dcd85cdf5d6f98a30d8db44271f8a97b93/grpcio-1.74.0-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3601274bc0523f6dc07666c0e01682c94472402ac2fd1226fd96e079863bfa49", size = 6621763 }, + { url = "https://files.pythonhosted.org/packages/9d/2c/930b0e7a2f1029bbc193443c7bc4dc2a46fedb0203c8793dcd97081f1520/grpcio-1.74.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:176d60a5168d7948539def20b2a3adcce67d72454d9ae05969a2e73f3a0feee7", size = 6180664 }, + { url = "https://files.pythonhosted.org/packages/db/d5/ff8a2442180ad0867717e670f5ec42bfd8d38b92158ad6bcd864e6d4b1ed/grpcio-1.74.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:e759f9e8bc908aaae0412642afe5416c9f983a80499448fcc7fab8692ae044c3", size = 6301083 }, + { url = "https://files.pythonhosted.org/packages/b0/ba/b361d390451a37ca118e4ec7dccec690422e05bc85fba2ec72b06cefec9f/grpcio-1.74.0-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:9e7c4389771855a92934b2846bd807fc25a3dfa820fd912fe6bd8136026b2707", size = 6994132 }, + { url = "https://files.pythonhosted.org/packages/3b/0c/3a5fa47d2437a44ced74141795ac0251bbddeae74bf81df3447edd767d27/grpcio-1.74.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:cce634b10aeab37010449124814b05a62fb5f18928ca878f1bf4750d1f0c815b", size = 6489616 }, + { url = "https://files.pythonhosted.org/packages/ae/95/ab64703b436d99dc5217228babc76047d60e9ad14df129e307b5fec81fd0/grpcio-1.74.0-cp312-cp312-win32.whl", hash = "sha256:885912559974df35d92219e2dc98f51a16a48395f37b92865ad45186f294096c", size = 3807083 }, + { url = "https://files.pythonhosted.org/packages/84/59/900aa2445891fc47a33f7d2f76e00ca5d6ae6584b20d19af9c06fa09bf9a/grpcio-1.74.0-cp312-cp312-win_amd64.whl", hash = "sha256:42f8fee287427b94be63d916c90399ed310ed10aadbf9e2e5538b3e497d269bc", size = 4490123 }, + { url = "https://files.pythonhosted.org/packages/d4/d8/1004a5f468715221450e66b051c839c2ce9a985aa3ee427422061fcbb6aa/grpcio-1.74.0-cp313-cp313-linux_armv7l.whl", hash = "sha256:2bc2d7d8d184e2362b53905cb1708c84cb16354771c04b490485fa07ce3a1d89", size = 5449488 }, + { url = "https://files.pythonhosted.org/packages/94/0e/33731a03f63740d7743dced423846c831d8e6da808fcd02821a4416df7fa/grpcio-1.74.0-cp313-cp313-macosx_11_0_universal2.whl", hash = "sha256:c14e803037e572c177ba54a3e090d6eb12efd795d49327c5ee2b3bddb836bf01", size = 10974059 }, + { url = "https://files.pythonhosted.org/packages/0d/c6/3d2c14d87771a421205bdca991467cfe473ee4c6a1231c1ede5248c62ab8/grpcio-1.74.0-cp313-cp313-manylinux_2_17_aarch64.whl", hash = "sha256:f6ec94f0e50eb8fa1744a731088b966427575e40c2944a980049798b127a687e", size = 5945647 }, + { url = "https://files.pythonhosted.org/packages/c5/83/5a354c8aaff58594eef7fffebae41a0f8995a6258bbc6809b800c33d4c13/grpcio-1.74.0-cp313-cp313-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:566b9395b90cc3d0d0c6404bc8572c7c18786ede549cdb540ae27b58afe0fb91", size = 6626101 }, + { url = "https://files.pythonhosted.org/packages/3f/ca/4fdc7bf59bf6994aa45cbd4ef1055cd65e2884de6113dbd49f75498ddb08/grpcio-1.74.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e1ea6176d7dfd5b941ea01c2ec34de9531ba494d541fe2057c904e601879f249", size = 6182562 }, + { url = "https://files.pythonhosted.org/packages/fd/48/2869e5b2c1922583686f7ae674937986807c2f676d08be70d0a541316270/grpcio-1.74.0-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:64229c1e9cea079420527fa8ac45d80fc1e8d3f94deaa35643c381fa8d98f362", size = 6303425 }, + { url = "https://files.pythonhosted.org/packages/a6/0e/bac93147b9a164f759497bc6913e74af1cb632c733c7af62c0336782bd38/grpcio-1.74.0-cp313-cp313-musllinux_1_1_i686.whl", hash = "sha256:0f87bddd6e27fc776aacf7ebfec367b6d49cad0455123951e4488ea99d9b9b8f", size = 6996533 }, + { url = "https://files.pythonhosted.org/packages/84/35/9f6b2503c1fd86d068b46818bbd7329db26a87cdd8c01e0d1a9abea1104c/grpcio-1.74.0-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:3b03d8f2a07f0fea8c8f74deb59f8352b770e3900d143b3d1475effcb08eec20", size = 6491489 }, + { url = "https://files.pythonhosted.org/packages/75/33/a04e99be2a82c4cbc4039eb3a76f6c3632932b9d5d295221389d10ac9ca7/grpcio-1.74.0-cp313-cp313-win32.whl", hash = "sha256:b6a73b2ba83e663b2480a90b82fdae6a7aa6427f62bf43b29912c0cfd1aa2bfa", size = 3805811 }, + { url = "https://files.pythonhosted.org/packages/34/80/de3eb55eb581815342d097214bed4c59e806b05f1b3110df03b2280d6dfd/grpcio-1.74.0-cp313-cp313-win_amd64.whl", hash = "sha256:fd3c71aeee838299c5887230b8a1822795325ddfea635edd82954c1eaa831e24", size = 4489214 }, +] + +[[package]] +name = "iniconfig" +version = "2.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f2/97/ebf4da567aa6827c909642694d71c9fcf53e5b504f2d96afea02718862f3/iniconfig-2.1.0.tar.gz", hash = "sha256:3abbd2e30b36733fee78f9c7f7308f2d0050e88f0087fd25c2645f63c773e1c7", size = 4793 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2c/e1/e6716421ea10d38022b952c159d5161ca1193197fb744506875fbb87ea7b/iniconfig-2.1.0-py3-none-any.whl", hash = "sha256:9deba5723312380e77435581c6bf4935c94cbfab9b1ed33ef8d238ea168eb760", size = 6050 }, +] + +[[package]] +name = "jinja2" +version = "3.1.6" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "markupsafe" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/df/bf/f7da0350254c0ed7c72f3e33cef02e048281fec7ecec5f032d4aac52226b/jinja2-3.1.6.tar.gz", hash = "sha256:0137fb05990d35f1275a587e9aee6d56da821fc83491a0fb838183be43f66d6d", size = 245115 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/62/a1/3d680cbfd5f4b8f15abc1d571870c5fc3e594bb582bc3b64ea099db13e56/jinja2-3.1.6-py3-none-any.whl", hash = "sha256:85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67", size = 134899 }, +] + +[[package]] +name = "kiwisolver" +version = "1.4.8" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/82/59/7c91426a8ac292e1cdd53a63b6d9439abd573c875c3f92c146767dd33faf/kiwisolver-1.4.8.tar.gz", hash = "sha256:23d5f023bdc8c7e54eb65f03ca5d5bb25b601eac4d7f1a042888a1f45237987e", size = 97538 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/47/5f/4d8e9e852d98ecd26cdf8eaf7ed8bc33174033bba5e07001b289f07308fd/kiwisolver-1.4.8-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:88c6f252f6816a73b1f8c904f7bbe02fd67c09a69f7cb8a0eecdbf5ce78e63db", size = 124623 }, + { url = "https://files.pythonhosted.org/packages/1d/70/7f5af2a18a76fe92ea14675f8bd88ce53ee79e37900fa5f1a1d8e0b42998/kiwisolver-1.4.8-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:c72941acb7b67138f35b879bbe85be0f6c6a70cab78fe3ef6db9c024d9223e5b", size = 66720 }, + { url = "https://files.pythonhosted.org/packages/c6/13/e15f804a142353aefd089fadc8f1d985561a15358c97aca27b0979cb0785/kiwisolver-1.4.8-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:ce2cf1e5688edcb727fdf7cd1bbd0b6416758996826a8be1d958f91880d0809d", size = 65413 }, + { url = "https://files.pythonhosted.org/packages/ce/6d/67d36c4d2054e83fb875c6b59d0809d5c530de8148846b1370475eeeece9/kiwisolver-1.4.8-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:c8bf637892dc6e6aad2bc6d4d69d08764166e5e3f69d469e55427b6ac001b19d", size = 1650826 }, + { url = "https://files.pythonhosted.org/packages/de/c6/7b9bb8044e150d4d1558423a1568e4f227193662a02231064e3824f37e0a/kiwisolver-1.4.8-cp310-cp310-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:034d2c891f76bd3edbdb3ea11140d8510dca675443da7304205a2eaa45d8334c", size = 1628231 }, + { url = "https://files.pythonhosted.org/packages/b6/38/ad10d437563063eaaedbe2c3540a71101fc7fb07a7e71f855e93ea4de605/kiwisolver-1.4.8-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d47b28d1dfe0793d5e96bce90835e17edf9a499b53969b03c6c47ea5985844c3", size = 1408938 }, + { url = "https://files.pythonhosted.org/packages/52/ce/c0106b3bd7f9e665c5f5bc1e07cc95b5dabd4e08e3dad42dbe2faad467e7/kiwisolver-1.4.8-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:eb158fe28ca0c29f2260cca8c43005329ad58452c36f0edf298204de32a9a3ed", size = 1422799 }, + { url = "https://files.pythonhosted.org/packages/d0/87/efb704b1d75dc9758087ba374c0f23d3254505edaedd09cf9d247f7878b9/kiwisolver-1.4.8-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d5536185fce131780ebd809f8e623bf4030ce1b161353166c49a3c74c287897f", size = 1354362 }, + { url = "https://files.pythonhosted.org/packages/eb/b3/fd760dc214ec9a8f208b99e42e8f0130ff4b384eca8b29dd0efc62052176/kiwisolver-1.4.8-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:369b75d40abedc1da2c1f4de13f3482cb99e3237b38726710f4a793432b1c5ff", size = 2222695 }, + { url = "https://files.pythonhosted.org/packages/a2/09/a27fb36cca3fc01700687cc45dae7a6a5f8eeb5f657b9f710f788748e10d/kiwisolver-1.4.8-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:641f2ddf9358c80faa22e22eb4c9f54bd3f0e442e038728f500e3b978d00aa7d", size = 2370802 }, + { url = "https://files.pythonhosted.org/packages/3d/c3/ba0a0346db35fe4dc1f2f2cf8b99362fbb922d7562e5f911f7ce7a7b60fa/kiwisolver-1.4.8-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:d561d2d8883e0819445cfe58d7ddd673e4015c3c57261d7bdcd3710d0d14005c", size = 2334646 }, + { url = "https://files.pythonhosted.org/packages/41/52/942cf69e562f5ed253ac67d5c92a693745f0bed3c81f49fc0cbebe4d6b00/kiwisolver-1.4.8-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:1732e065704b47c9afca7ffa272f845300a4eb959276bf6970dc07265e73b605", size = 2467260 }, + { url = "https://files.pythonhosted.org/packages/32/26/2d9668f30d8a494b0411d4d7d4ea1345ba12deb6a75274d58dd6ea01e951/kiwisolver-1.4.8-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:bcb1ebc3547619c3b58a39e2448af089ea2ef44b37988caf432447374941574e", size = 2288633 }, + { url = "https://files.pythonhosted.org/packages/98/99/0dd05071654aa44fe5d5e350729961e7bb535372935a45ac89a8924316e6/kiwisolver-1.4.8-cp310-cp310-win_amd64.whl", hash = "sha256:89c107041f7b27844179ea9c85d6da275aa55ecf28413e87624d033cf1f6b751", size = 71885 }, + { url = "https://files.pythonhosted.org/packages/6c/fc/822e532262a97442989335394d441cd1d0448c2e46d26d3e04efca84df22/kiwisolver-1.4.8-cp310-cp310-win_arm64.whl", hash = "sha256:b5773efa2be9eb9fcf5415ea3ab70fc785d598729fd6057bea38d539ead28271", size = 65175 }, + { url = "https://files.pythonhosted.org/packages/da/ed/c913ee28936c371418cb167b128066ffb20bbf37771eecc2c97edf8a6e4c/kiwisolver-1.4.8-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:a4d3601908c560bdf880f07d94f31d734afd1bb71e96585cace0e38ef44c6d84", size = 124635 }, + { url = "https://files.pythonhosted.org/packages/4c/45/4a7f896f7467aaf5f56ef093d1f329346f3b594e77c6a3c327b2d415f521/kiwisolver-1.4.8-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:856b269c4d28a5c0d5e6c1955ec36ebfd1651ac00e1ce0afa3e28da95293b561", size = 66717 }, + { url = "https://files.pythonhosted.org/packages/5f/b4/c12b3ac0852a3a68f94598d4c8d569f55361beef6159dce4e7b624160da2/kiwisolver-1.4.8-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:c2b9a96e0f326205af81a15718a9073328df1173a2619a68553decb7097fd5d7", size = 65413 }, + { url = "https://files.pythonhosted.org/packages/a9/98/1df4089b1ed23d83d410adfdc5947245c753bddfbe06541c4aae330e9e70/kiwisolver-1.4.8-cp311-cp311-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c5020c83e8553f770cb3b5fc13faac40f17e0b205bd237aebd21d53d733adb03", size = 1343994 }, + { url = "https://files.pythonhosted.org/packages/8d/bf/b4b169b050c8421a7c53ea1ea74e4ef9c335ee9013216c558a047f162d20/kiwisolver-1.4.8-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:dace81d28c787956bfbfbbfd72fdcef014f37d9b48830829e488fdb32b49d954", size = 1434804 }, + { url = "https://files.pythonhosted.org/packages/66/5a/e13bd341fbcf73325ea60fdc8af752addf75c5079867af2e04cc41f34434/kiwisolver-1.4.8-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:11e1022b524bd48ae56c9b4f9296bce77e15a2e42a502cceba602f804b32bb79", size = 1450690 }, + { url = "https://files.pythonhosted.org/packages/9b/4f/5955dcb376ba4a830384cc6fab7d7547bd6759fe75a09564910e9e3bb8ea/kiwisolver-1.4.8-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3b9b4d2892fefc886f30301cdd80debd8bb01ecdf165a449eb6e78f79f0fabd6", size = 1376839 }, + { url = "https://files.pythonhosted.org/packages/3a/97/5edbed69a9d0caa2e4aa616ae7df8127e10f6586940aa683a496c2c280b9/kiwisolver-1.4.8-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3a96c0e790ee875d65e340ab383700e2b4891677b7fcd30a699146f9384a2bb0", size = 1435109 }, + { url = "https://files.pythonhosted.org/packages/13/fc/e756382cb64e556af6c1809a1bbb22c141bbc2445049f2da06b420fe52bf/kiwisolver-1.4.8-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:23454ff084b07ac54ca8be535f4174170c1094a4cff78fbae4f73a4bcc0d4dab", size = 2245269 }, + { url = "https://files.pythonhosted.org/packages/76/15/e59e45829d7f41c776d138245cabae6515cb4eb44b418f6d4109c478b481/kiwisolver-1.4.8-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:87b287251ad6488e95b4f0b4a79a6d04d3ea35fde6340eb38fbd1ca9cd35bbbc", size = 2393468 }, + { url = "https://files.pythonhosted.org/packages/e9/39/483558c2a913ab8384d6e4b66a932406f87c95a6080112433da5ed668559/kiwisolver-1.4.8-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:b21dbe165081142b1232a240fc6383fd32cdd877ca6cc89eab93e5f5883e1c25", size = 2355394 }, + { url = "https://files.pythonhosted.org/packages/01/aa/efad1fbca6570a161d29224f14b082960c7e08268a133fe5dc0f6906820e/kiwisolver-1.4.8-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:768cade2c2df13db52475bd28d3a3fac8c9eff04b0e9e2fda0f3760f20b3f7fc", size = 2490901 }, + { url = "https://files.pythonhosted.org/packages/c9/4f/15988966ba46bcd5ab9d0c8296914436720dd67fca689ae1a75b4ec1c72f/kiwisolver-1.4.8-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:d47cfb2650f0e103d4bf68b0b5804c68da97272c84bb12850d877a95c056bd67", size = 2312306 }, + { url = "https://files.pythonhosted.org/packages/2d/27/bdf1c769c83f74d98cbc34483a972f221440703054894a37d174fba8aa68/kiwisolver-1.4.8-cp311-cp311-win_amd64.whl", hash = "sha256:ed33ca2002a779a2e20eeb06aea7721b6e47f2d4b8a8ece979d8ba9e2a167e34", size = 71966 }, + { url = "https://files.pythonhosted.org/packages/4a/c9/9642ea855604aeb2968a8e145fc662edf61db7632ad2e4fb92424be6b6c0/kiwisolver-1.4.8-cp311-cp311-win_arm64.whl", hash = "sha256:16523b40aab60426ffdebe33ac374457cf62863e330a90a0383639ce14bf44b2", size = 65311 }, + { url = "https://files.pythonhosted.org/packages/fc/aa/cea685c4ab647f349c3bc92d2daf7ae34c8e8cf405a6dcd3a497f58a2ac3/kiwisolver-1.4.8-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:d6af5e8815fd02997cb6ad9bbed0ee1e60014438ee1a5c2444c96f87b8843502", size = 124152 }, + { url = "https://files.pythonhosted.org/packages/c5/0b/8db6d2e2452d60d5ebc4ce4b204feeb16176a851fd42462f66ade6808084/kiwisolver-1.4.8-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:bade438f86e21d91e0cf5dd7c0ed00cda0f77c8c1616bd83f9fc157fa6760d31", size = 66555 }, + { url = "https://files.pythonhosted.org/packages/60/26/d6a0db6785dd35d3ba5bf2b2df0aedc5af089962c6eb2cbf67a15b81369e/kiwisolver-1.4.8-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:b83dc6769ddbc57613280118fb4ce3cd08899cc3369f7d0e0fab518a7cf37fdb", size = 65067 }, + { url = "https://files.pythonhosted.org/packages/c9/ed/1d97f7e3561e09757a196231edccc1bcf59d55ddccefa2afc9c615abd8e0/kiwisolver-1.4.8-cp312-cp312-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:111793b232842991be367ed828076b03d96202c19221b5ebab421ce8bcad016f", size = 1378443 }, + { url = "https://files.pythonhosted.org/packages/29/61/39d30b99954e6b46f760e6289c12fede2ab96a254c443639052d1b573fbc/kiwisolver-1.4.8-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:257af1622860e51b1a9d0ce387bf5c2c4f36a90594cb9514f55b074bcc787cfc", size = 1472728 }, + { url = "https://files.pythonhosted.org/packages/0c/3e/804163b932f7603ef256e4a715e5843a9600802bb23a68b4e08c8c0ff61d/kiwisolver-1.4.8-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:69b5637c3f316cab1ec1c9a12b8c5f4750a4c4b71af9157645bf32830e39c03a", size = 1478388 }, + { url = "https://files.pythonhosted.org/packages/8a/9e/60eaa75169a154700be74f875a4d9961b11ba048bef315fbe89cb6999056/kiwisolver-1.4.8-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:782bb86f245ec18009890e7cb8d13a5ef54dcf2ebe18ed65f795e635a96a1c6a", size = 1413849 }, + { url = "https://files.pythonhosted.org/packages/bc/b3/9458adb9472e61a998c8c4d95cfdfec91c73c53a375b30b1428310f923e4/kiwisolver-1.4.8-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cc978a80a0db3a66d25767b03688f1147a69e6237175c0f4ffffaaedf744055a", size = 1475533 }, + { url = "https://files.pythonhosted.org/packages/e4/7a/0a42d9571e35798de80aef4bb43a9b672aa7f8e58643d7bd1950398ffb0a/kiwisolver-1.4.8-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:36dbbfd34838500a31f52c9786990d00150860e46cd5041386f217101350f0d3", size = 2268898 }, + { url = "https://files.pythonhosted.org/packages/d9/07/1255dc8d80271400126ed8db35a1795b1a2c098ac3a72645075d06fe5c5d/kiwisolver-1.4.8-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:eaa973f1e05131de5ff3569bbba7f5fd07ea0595d3870ed4a526d486fe57fa1b", size = 2425605 }, + { url = "https://files.pythonhosted.org/packages/84/df/5a3b4cf13780ef6f6942df67b138b03b7e79e9f1f08f57c49957d5867f6e/kiwisolver-1.4.8-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:a66f60f8d0c87ab7f59b6fb80e642ebb29fec354a4dfad687ca4092ae69d04f4", size = 2375801 }, + { url = "https://files.pythonhosted.org/packages/8f/10/2348d068e8b0f635c8c86892788dac7a6b5c0cb12356620ab575775aad89/kiwisolver-1.4.8-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:858416b7fb777a53f0c59ca08190ce24e9abbd3cffa18886a5781b8e3e26f65d", size = 2520077 }, + { url = "https://files.pythonhosted.org/packages/32/d8/014b89fee5d4dce157d814303b0fce4d31385a2af4c41fed194b173b81ac/kiwisolver-1.4.8-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:085940635c62697391baafaaeabdf3dd7a6c3643577dde337f4d66eba021b2b8", size = 2338410 }, + { url = "https://files.pythonhosted.org/packages/bd/72/dfff0cc97f2a0776e1c9eb5bef1ddfd45f46246c6533b0191887a427bca5/kiwisolver-1.4.8-cp312-cp312-win_amd64.whl", hash = "sha256:01c3d31902c7db5fb6182832713d3b4122ad9317c2c5877d0539227d96bb2e50", size = 71853 }, + { url = "https://files.pythonhosted.org/packages/dc/85/220d13d914485c0948a00f0b9eb419efaf6da81b7d72e88ce2391f7aed8d/kiwisolver-1.4.8-cp312-cp312-win_arm64.whl", hash = "sha256:a3c44cb68861de93f0c4a8175fbaa691f0aa22550c331fefef02b618a9dcb476", size = 65424 }, + { url = "https://files.pythonhosted.org/packages/79/b3/e62464a652f4f8cd9006e13d07abad844a47df1e6537f73ddfbf1bc997ec/kiwisolver-1.4.8-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:1c8ceb754339793c24aee1c9fb2485b5b1f5bb1c2c214ff13368431e51fc9a09", size = 124156 }, + { url = "https://files.pythonhosted.org/packages/8d/2d/f13d06998b546a2ad4f48607a146e045bbe48030774de29f90bdc573df15/kiwisolver-1.4.8-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:54a62808ac74b5e55a04a408cda6156f986cefbcf0ada13572696b507cc92fa1", size = 66555 }, + { url = "https://files.pythonhosted.org/packages/59/e3/b8bd14b0a54998a9fd1e8da591c60998dc003618cb19a3f94cb233ec1511/kiwisolver-1.4.8-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:68269e60ee4929893aad82666821aaacbd455284124817af45c11e50a4b42e3c", size = 65071 }, + { url = "https://files.pythonhosted.org/packages/f0/1c/6c86f6d85ffe4d0ce04228d976f00674f1df5dc893bf2dd4f1928748f187/kiwisolver-1.4.8-cp313-cp313-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:34d142fba9c464bc3bbfeff15c96eab0e7310343d6aefb62a79d51421fcc5f1b", size = 1378053 }, + { url = "https://files.pythonhosted.org/packages/4e/b9/1c6e9f6dcb103ac5cf87cb695845f5fa71379021500153566d8a8a9fc291/kiwisolver-1.4.8-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3ddc373e0eef45b59197de815b1b28ef89ae3955e7722cc9710fb91cd77b7f47", size = 1472278 }, + { url = "https://files.pythonhosted.org/packages/ee/81/aca1eb176de671f8bda479b11acdc42c132b61a2ac861c883907dde6debb/kiwisolver-1.4.8-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:77e6f57a20b9bd4e1e2cedda4d0b986ebd0216236f0106e55c28aea3d3d69b16", size = 1478139 }, + { url = "https://files.pythonhosted.org/packages/49/f4/e081522473671c97b2687d380e9e4c26f748a86363ce5af48b4a28e48d06/kiwisolver-1.4.8-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:08e77738ed7538f036cd1170cbed942ef749137b1311fa2bbe2a7fda2f6bf3cc", size = 1413517 }, + { url = "https://files.pythonhosted.org/packages/8f/e9/6a7d025d8da8c4931522922cd706105aa32b3291d1add8c5427cdcd66e63/kiwisolver-1.4.8-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a5ce1e481a74b44dd5e92ff03ea0cb371ae7a0268318e202be06c8f04f4f1246", size = 1474952 }, + { url = "https://files.pythonhosted.org/packages/82/13/13fa685ae167bee5d94b415991c4fc7bb0a1b6ebea6e753a87044b209678/kiwisolver-1.4.8-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:fc2ace710ba7c1dfd1a3b42530b62b9ceed115f19a1656adefce7b1782a37794", size = 2269132 }, + { url = "https://files.pythonhosted.org/packages/ef/92/bb7c9395489b99a6cb41d502d3686bac692586db2045adc19e45ee64ed23/kiwisolver-1.4.8-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:3452046c37c7692bd52b0e752b87954ef86ee2224e624ef7ce6cb21e8c41cc1b", size = 2425997 }, + { url = "https://files.pythonhosted.org/packages/ed/12/87f0e9271e2b63d35d0d8524954145837dd1a6c15b62a2d8c1ebe0f182b4/kiwisolver-1.4.8-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:7e9a60b50fe8b2ec6f448fe8d81b07e40141bfced7f896309df271a0b92f80f3", size = 2376060 }, + { url = "https://files.pythonhosted.org/packages/02/6e/c8af39288edbce8bf0fa35dee427b082758a4b71e9c91ef18fa667782138/kiwisolver-1.4.8-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:918139571133f366e8362fa4a297aeba86c7816b7ecf0bc79168080e2bd79957", size = 2520471 }, + { url = "https://files.pythonhosted.org/packages/13/78/df381bc7b26e535c91469f77f16adcd073beb3e2dd25042efd064af82323/kiwisolver-1.4.8-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:e063ef9f89885a1d68dd8b2e18f5ead48653176d10a0e324e3b0030e3a69adeb", size = 2338793 }, + { url = "https://files.pythonhosted.org/packages/d0/dc/c1abe38c37c071d0fc71c9a474fd0b9ede05d42f5a458d584619cfd2371a/kiwisolver-1.4.8-cp313-cp313-win_amd64.whl", hash = "sha256:a17b7c4f5b2c51bb68ed379defd608a03954a1845dfed7cc0117f1cc8a9b7fd2", size = 71855 }, + { url = "https://files.pythonhosted.org/packages/a0/b6/21529d595b126ac298fdd90b705d87d4c5693de60023e0efcb4f387ed99e/kiwisolver-1.4.8-cp313-cp313-win_arm64.whl", hash = "sha256:3cd3bc628b25f74aedc6d374d5babf0166a92ff1317f46267f12d2ed54bc1d30", size = 65430 }, + { url = "https://files.pythonhosted.org/packages/34/bd/b89380b7298e3af9b39f49334e3e2a4af0e04819789f04b43d560516c0c8/kiwisolver-1.4.8-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:370fd2df41660ed4e26b8c9d6bbcad668fbe2560462cba151a721d49e5b6628c", size = 126294 }, + { url = "https://files.pythonhosted.org/packages/83/41/5857dc72e5e4148eaac5aa76e0703e594e4465f8ab7ec0fc60e3a9bb8fea/kiwisolver-1.4.8-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:84a2f830d42707de1d191b9490ac186bf7997a9495d4e9072210a1296345f7dc", size = 67736 }, + { url = "https://files.pythonhosted.org/packages/e1/d1/be059b8db56ac270489fb0b3297fd1e53d195ba76e9bbb30e5401fa6b759/kiwisolver-1.4.8-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:7a3ad337add5148cf51ce0b55642dc551c0b9d6248458a757f98796ca7348712", size = 66194 }, + { url = "https://files.pythonhosted.org/packages/e1/83/4b73975f149819eb7dcf9299ed467eba068ecb16439a98990dcb12e63fdd/kiwisolver-1.4.8-cp313-cp313t-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7506488470f41169b86d8c9aeff587293f530a23a23a49d6bc64dab66bedc71e", size = 1465942 }, + { url = "https://files.pythonhosted.org/packages/c7/2c/30a5cdde5102958e602c07466bce058b9d7cb48734aa7a4327261ac8e002/kiwisolver-1.4.8-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2f0121b07b356a22fb0414cec4666bbe36fd6d0d759db3d37228f496ed67c880", size = 1595341 }, + { url = "https://files.pythonhosted.org/packages/ff/9b/1e71db1c000385aa069704f5990574b8244cce854ecd83119c19e83c9586/kiwisolver-1.4.8-cp313-cp313t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d6d6bd87df62c27d4185de7c511c6248040afae67028a8a22012b010bc7ad062", size = 1598455 }, + { url = "https://files.pythonhosted.org/packages/85/92/c8fec52ddf06231b31cbb779af77e99b8253cd96bd135250b9498144c78b/kiwisolver-1.4.8-cp313-cp313t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:291331973c64bb9cce50bbe871fb2e675c4331dab4f31abe89f175ad7679a4d7", size = 1522138 }, + { url = "https://files.pythonhosted.org/packages/0b/51/9eb7e2cd07a15d8bdd976f6190c0164f92ce1904e5c0c79198c4972926b7/kiwisolver-1.4.8-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:893f5525bb92d3d735878ec00f781b2de998333659507d29ea4466208df37bed", size = 1582857 }, + { url = "https://files.pythonhosted.org/packages/0f/95/c5a00387a5405e68ba32cc64af65ce881a39b98d73cc394b24143bebc5b8/kiwisolver-1.4.8-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:b47a465040146981dc9db8647981b8cb96366fbc8d452b031e4f8fdffec3f26d", size = 2293129 }, + { url = "https://files.pythonhosted.org/packages/44/83/eeb7af7d706b8347548313fa3a3a15931f404533cc54fe01f39e830dd231/kiwisolver-1.4.8-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:99cea8b9dd34ff80c521aef46a1dddb0dcc0283cf18bde6d756f1e6f31772165", size = 2421538 }, + { url = "https://files.pythonhosted.org/packages/05/f9/27e94c1b3eb29e6933b6986ffc5fa1177d2cd1f0c8efc5f02c91c9ac61de/kiwisolver-1.4.8-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:151dffc4865e5fe6dafce5480fab84f950d14566c480c08a53c663a0020504b6", size = 2390661 }, + { url = "https://files.pythonhosted.org/packages/d9/d4/3c9735faa36ac591a4afcc2980d2691000506050b7a7e80bcfe44048daa7/kiwisolver-1.4.8-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:577facaa411c10421314598b50413aa1ebcf5126f704f1e5d72d7e4e9f020d90", size = 2546710 }, + { url = "https://files.pythonhosted.org/packages/4c/fa/be89a49c640930180657482a74970cdcf6f7072c8d2471e1babe17a222dc/kiwisolver-1.4.8-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:be4816dc51c8a471749d664161b434912eee82f2ea66bd7628bd14583a833e85", size = 2349213 }, + { url = "https://files.pythonhosted.org/packages/1f/f9/ae81c47a43e33b93b0a9819cac6723257f5da2a5a60daf46aa5c7226ea85/kiwisolver-1.4.8-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:e7a019419b7b510f0f7c9dceff8c5eae2392037eae483a7f9162625233802b0a", size = 60403 }, + { url = "https://files.pythonhosted.org/packages/58/ca/f92b5cb6f4ce0c1ebfcfe3e2e42b96917e16f7090e45b21102941924f18f/kiwisolver-1.4.8-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:286b18e86682fd2217a48fc6be6b0f20c1d0ed10958d8dc53453ad58d7be0bf8", size = 58657 }, + { url = "https://files.pythonhosted.org/packages/80/28/ae0240f732f0484d3a4dc885d055653c47144bdf59b670aae0ec3c65a7c8/kiwisolver-1.4.8-pp310-pypy310_pp73-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4191ee8dfd0be1c3666ccbac178c5a05d5f8d689bbe3fc92f3c4abec817f8fe0", size = 84948 }, + { url = "https://files.pythonhosted.org/packages/5d/eb/78d50346c51db22c7203c1611f9b513075f35c4e0e4877c5dde378d66043/kiwisolver-1.4.8-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7cd2785b9391f2873ad46088ed7599a6a71e762e1ea33e87514b1a441ed1da1c", size = 81186 }, + { url = "https://files.pythonhosted.org/packages/43/f8/7259f18c77adca88d5f64f9a522792e178b2691f3748817a8750c2d216ef/kiwisolver-1.4.8-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c07b29089b7ba090b6f1a669f1411f27221c3662b3a1b7010e67b59bb5a6f10b", size = 80279 }, + { url = "https://files.pythonhosted.org/packages/3a/1d/50ad811d1c5dae091e4cf046beba925bcae0a610e79ae4c538f996f63ed5/kiwisolver-1.4.8-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:65ea09a5a3faadd59c2ce96dc7bf0f364986a315949dc6374f04396b0d60e09b", size = 71762 }, +] + +[[package]] +name = "markdown" +version = "3.8.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d7/c2/4ab49206c17f75cb08d6311171f2d65798988db4360c4d1485bd0eedd67c/markdown-3.8.2.tar.gz", hash = "sha256:247b9a70dd12e27f67431ce62523e675b866d254f900c4fe75ce3dda62237c45", size = 362071 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/96/2b/34cc11786bc00d0f04d0f5fdc3a2b1ae0b6239eef72d3d345805f9ad92a1/markdown-3.8.2-py3-none-any.whl", hash = "sha256:5c83764dbd4e00bdd94d85a19b8d55ccca20fe35b2e678a1422b380324dd5f24", size = 106827 }, +] + +[[package]] +name = "markupsafe" +version = "3.0.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/b2/97/5d42485e71dfc078108a86d6de8fa46db44a1a9295e89c5d6d4a06e23a62/markupsafe-3.0.2.tar.gz", hash = "sha256:ee55d3edf80167e48ea11a923c7386f4669df67d7994554387f84e7d8b0a2bf0", size = 20537 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/04/90/d08277ce111dd22f77149fd1a5d4653eeb3b3eaacbdfcbae5afb2600eebd/MarkupSafe-3.0.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:7e94c425039cde14257288fd61dcfb01963e658efbc0ff54f5306b06054700f8", size = 14357 }, + { url = "https://files.pythonhosted.org/packages/04/e1/6e2194baeae0bca1fae6629dc0cbbb968d4d941469cbab11a3872edff374/MarkupSafe-3.0.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9e2d922824181480953426608b81967de705c3cef4d1af983af849d7bd619158", size = 12393 }, + { url = "https://files.pythonhosted.org/packages/1d/69/35fa85a8ece0a437493dc61ce0bb6d459dcba482c34197e3efc829aa357f/MarkupSafe-3.0.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:38a9ef736c01fccdd6600705b09dc574584b89bea478200c5fbf112a6b0d5579", size = 21732 }, + { url = "https://files.pythonhosted.org/packages/22/35/137da042dfb4720b638d2937c38a9c2df83fe32d20e8c8f3185dbfef05f7/MarkupSafe-3.0.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bbcb445fa71794da8f178f0f6d66789a28d7319071af7a496d4d507ed566270d", size = 20866 }, + { url = "https://files.pythonhosted.org/packages/29/28/6d029a903727a1b62edb51863232152fd335d602def598dade38996887f0/MarkupSafe-3.0.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:57cb5a3cf367aeb1d316576250f65edec5bb3be939e9247ae594b4bcbc317dfb", size = 20964 }, + { url = "https://files.pythonhosted.org/packages/cc/cd/07438f95f83e8bc028279909d9c9bd39e24149b0d60053a97b2bc4f8aa51/MarkupSafe-3.0.2-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:3809ede931876f5b2ec92eef964286840ed3540dadf803dd570c3b7e13141a3b", size = 21977 }, + { url = "https://files.pythonhosted.org/packages/29/01/84b57395b4cc062f9c4c55ce0df7d3108ca32397299d9df00fedd9117d3d/MarkupSafe-3.0.2-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:e07c3764494e3776c602c1e78e298937c3315ccc9043ead7e685b7f2b8d47b3c", size = 21366 }, + { url = "https://files.pythonhosted.org/packages/bd/6e/61ebf08d8940553afff20d1fb1ba7294b6f8d279df9fd0c0db911b4bbcfd/MarkupSafe-3.0.2-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:b424c77b206d63d500bcb69fa55ed8d0e6a3774056bdc4839fc9298a7edca171", size = 21091 }, + { url = "https://files.pythonhosted.org/packages/11/23/ffbf53694e8c94ebd1e7e491de185124277964344733c45481f32ede2499/MarkupSafe-3.0.2-cp310-cp310-win32.whl", hash = "sha256:fcabf5ff6eea076f859677f5f0b6b5c1a51e70a376b0579e0eadef8db48c6b50", size = 15065 }, + { url = "https://files.pythonhosted.org/packages/44/06/e7175d06dd6e9172d4a69a72592cb3f7a996a9c396eee29082826449bbc3/MarkupSafe-3.0.2-cp310-cp310-win_amd64.whl", hash = "sha256:6af100e168aa82a50e186c82875a5893c5597a0c1ccdb0d8b40240b1f28b969a", size = 15514 }, + { url = "https://files.pythonhosted.org/packages/6b/28/bbf83e3f76936960b850435576dd5e67034e200469571be53f69174a2dfd/MarkupSafe-3.0.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:9025b4018f3a1314059769c7bf15441064b2207cb3f065e6ea1e7359cb46db9d", size = 14353 }, + { url = "https://files.pythonhosted.org/packages/6c/30/316d194b093cde57d448a4c3209f22e3046c5bb2fb0820b118292b334be7/MarkupSafe-3.0.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:93335ca3812df2f366e80509ae119189886b0f3c2b81325d39efdb84a1e2ae93", size = 12392 }, + { url = "https://files.pythonhosted.org/packages/f2/96/9cdafba8445d3a53cae530aaf83c38ec64c4d5427d975c974084af5bc5d2/MarkupSafe-3.0.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2cb8438c3cbb25e220c2ab33bb226559e7afb3baec11c4f218ffa7308603c832", size = 23984 }, + { url = "https://files.pythonhosted.org/packages/f1/a4/aefb044a2cd8d7334c8a47d3fb2c9f328ac48cb349468cc31c20b539305f/MarkupSafe-3.0.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a123e330ef0853c6e822384873bef7507557d8e4a082961e1defa947aa59ba84", size = 23120 }, + { url = "https://files.pythonhosted.org/packages/8d/21/5e4851379f88f3fad1de30361db501300d4f07bcad047d3cb0449fc51f8c/MarkupSafe-3.0.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1e084f686b92e5b83186b07e8a17fc09e38fff551f3602b249881fec658d3eca", size = 23032 }, + { url = "https://files.pythonhosted.org/packages/00/7b/e92c64e079b2d0d7ddf69899c98842f3f9a60a1ae72657c89ce2655c999d/MarkupSafe-3.0.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:d8213e09c917a951de9d09ecee036d5c7d36cb6cb7dbaece4c71a60d79fb9798", size = 24057 }, + { url = "https://files.pythonhosted.org/packages/f9/ac/46f960ca323037caa0a10662ef97d0a4728e890334fc156b9f9e52bcc4ca/MarkupSafe-3.0.2-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:5b02fb34468b6aaa40dfc198d813a641e3a63b98c2b05a16b9f80b7ec314185e", size = 23359 }, + { url = "https://files.pythonhosted.org/packages/69/84/83439e16197337b8b14b6a5b9c2105fff81d42c2a7c5b58ac7b62ee2c3b1/MarkupSafe-3.0.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:0bff5e0ae4ef2e1ae4fdf2dfd5b76c75e5c2fa4132d05fc1b0dabcd20c7e28c4", size = 23306 }, + { url = "https://files.pythonhosted.org/packages/9a/34/a15aa69f01e2181ed8d2b685c0d2f6655d5cca2c4db0ddea775e631918cd/MarkupSafe-3.0.2-cp311-cp311-win32.whl", hash = "sha256:6c89876f41da747c8d3677a2b540fb32ef5715f97b66eeb0c6b66f5e3ef6f59d", size = 15094 }, + { url = "https://files.pythonhosted.org/packages/da/b8/3a3bd761922d416f3dc5d00bfbed11f66b1ab89a0c2b6e887240a30b0f6b/MarkupSafe-3.0.2-cp311-cp311-win_amd64.whl", hash = "sha256:70a87b411535ccad5ef2f1df5136506a10775d267e197e4cf531ced10537bd6b", size = 15521 }, + { url = "https://files.pythonhosted.org/packages/22/09/d1f21434c97fc42f09d290cbb6350d44eb12f09cc62c9476effdb33a18aa/MarkupSafe-3.0.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:9778bd8ab0a994ebf6f84c2b949e65736d5575320a17ae8984a77fab08db94cf", size = 14274 }, + { url = "https://files.pythonhosted.org/packages/6b/b0/18f76bba336fa5aecf79d45dcd6c806c280ec44538b3c13671d49099fdd0/MarkupSafe-3.0.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:846ade7b71e3536c4e56b386c2a47adf5741d2d8b94ec9dc3e92e5e1ee1e2225", size = 12348 }, + { url = "https://files.pythonhosted.org/packages/e0/25/dd5c0f6ac1311e9b40f4af06c78efde0f3b5cbf02502f8ef9501294c425b/MarkupSafe-3.0.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1c99d261bd2d5f6b59325c92c73df481e05e57f19837bdca8413b9eac4bd8028", size = 24149 }, + { url = "https://files.pythonhosted.org/packages/f3/f0/89e7aadfb3749d0f52234a0c8c7867877876e0a20b60e2188e9850794c17/MarkupSafe-3.0.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e17c96c14e19278594aa4841ec148115f9c7615a47382ecb6b82bd8fea3ab0c8", size = 23118 }, + { url = "https://files.pythonhosted.org/packages/d5/da/f2eeb64c723f5e3777bc081da884b414671982008c47dcc1873d81f625b6/MarkupSafe-3.0.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:88416bd1e65dcea10bc7569faacb2c20ce071dd1f87539ca2ab364bf6231393c", size = 22993 }, + { url = "https://files.pythonhosted.org/packages/da/0e/1f32af846df486dce7c227fe0f2398dc7e2e51d4a370508281f3c1c5cddc/MarkupSafe-3.0.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:2181e67807fc2fa785d0592dc2d6206c019b9502410671cc905d132a92866557", size = 24178 }, + { url = "https://files.pythonhosted.org/packages/c4/f6/bb3ca0532de8086cbff5f06d137064c8410d10779c4c127e0e47d17c0b71/MarkupSafe-3.0.2-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:52305740fe773d09cffb16f8ed0427942901f00adedac82ec8b67752f58a1b22", size = 23319 }, + { url = "https://files.pythonhosted.org/packages/a2/82/8be4c96ffee03c5b4a034e60a31294daf481e12c7c43ab8e34a1453ee48b/MarkupSafe-3.0.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:ad10d3ded218f1039f11a75f8091880239651b52e9bb592ca27de44eed242a48", size = 23352 }, + { url = "https://files.pythonhosted.org/packages/51/ae/97827349d3fcffee7e184bdf7f41cd6b88d9919c80f0263ba7acd1bbcb18/MarkupSafe-3.0.2-cp312-cp312-win32.whl", hash = "sha256:0f4ca02bea9a23221c0182836703cbf8930c5e9454bacce27e767509fa286a30", size = 15097 }, + { url = "https://files.pythonhosted.org/packages/c1/80/a61f99dc3a936413c3ee4e1eecac96c0da5ed07ad56fd975f1a9da5bc630/MarkupSafe-3.0.2-cp312-cp312-win_amd64.whl", hash = "sha256:8e06879fc22a25ca47312fbe7c8264eb0b662f6db27cb2d3bbbc74b1df4b9b87", size = 15601 }, + { url = "https://files.pythonhosted.org/packages/83/0e/67eb10a7ecc77a0c2bbe2b0235765b98d164d81600746914bebada795e97/MarkupSafe-3.0.2-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:ba9527cdd4c926ed0760bc301f6728ef34d841f405abf9d4f959c478421e4efd", size = 14274 }, + { url = "https://files.pythonhosted.org/packages/2b/6d/9409f3684d3335375d04e5f05744dfe7e9f120062c9857df4ab490a1031a/MarkupSafe-3.0.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:f8b3d067f2e40fe93e1ccdd6b2e1d16c43140e76f02fb1319a05cf2b79d99430", size = 12352 }, + { url = "https://files.pythonhosted.org/packages/d2/f5/6eadfcd3885ea85fe2a7c128315cc1bb7241e1987443d78c8fe712d03091/MarkupSafe-3.0.2-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:569511d3b58c8791ab4c2e1285575265991e6d8f8700c7be0e88f86cb0672094", size = 24122 }, + { url = "https://files.pythonhosted.org/packages/0c/91/96cf928db8236f1bfab6ce15ad070dfdd02ed88261c2afafd4b43575e9e9/MarkupSafe-3.0.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:15ab75ef81add55874e7ab7055e9c397312385bd9ced94920f2802310c930396", size = 23085 }, + { url = "https://files.pythonhosted.org/packages/c2/cf/c9d56af24d56ea04daae7ac0940232d31d5a8354f2b457c6d856b2057d69/MarkupSafe-3.0.2-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f3818cb119498c0678015754eba762e0d61e5b52d34c8b13d770f0719f7b1d79", size = 22978 }, + { url = "https://files.pythonhosted.org/packages/2a/9f/8619835cd6a711d6272d62abb78c033bda638fdc54c4e7f4272cf1c0962b/MarkupSafe-3.0.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:cdb82a876c47801bb54a690c5ae105a46b392ac6099881cdfb9f6e95e4014c6a", size = 24208 }, + { url = "https://files.pythonhosted.org/packages/f9/bf/176950a1792b2cd2102b8ffeb5133e1ed984547b75db47c25a67d3359f77/MarkupSafe-3.0.2-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:cabc348d87e913db6ab4aa100f01b08f481097838bdddf7c7a84b7575b7309ca", size = 23357 }, + { url = "https://files.pythonhosted.org/packages/ce/4f/9a02c1d335caabe5c4efb90e1b6e8ee944aa245c1aaaab8e8a618987d816/MarkupSafe-3.0.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:444dcda765c8a838eaae23112db52f1efaf750daddb2d9ca300bcae1039adc5c", size = 23344 }, + { url = "https://files.pythonhosted.org/packages/ee/55/c271b57db36f748f0e04a759ace9f8f759ccf22b4960c270c78a394f58be/MarkupSafe-3.0.2-cp313-cp313-win32.whl", hash = "sha256:bcf3e58998965654fdaff38e58584d8937aa3096ab5354d493c77d1fdd66d7a1", size = 15101 }, + { url = "https://files.pythonhosted.org/packages/29/88/07df22d2dd4df40aba9f3e402e6dc1b8ee86297dddbad4872bd5e7b0094f/MarkupSafe-3.0.2-cp313-cp313-win_amd64.whl", hash = "sha256:e6a2a455bd412959b57a172ce6328d2dd1f01cb2135efda2e4576e8a23fa3b0f", size = 15603 }, + { url = "https://files.pythonhosted.org/packages/62/6a/8b89d24db2d32d433dffcd6a8779159da109842434f1dd2f6e71f32f738c/MarkupSafe-3.0.2-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:b5a6b3ada725cea8a5e634536b1b01c30bcdcd7f9c6fff4151548d5bf6b3a36c", size = 14510 }, + { url = "https://files.pythonhosted.org/packages/7a/06/a10f955f70a2e5a9bf78d11a161029d278eeacbd35ef806c3fd17b13060d/MarkupSafe-3.0.2-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:a904af0a6162c73e3edcb969eeeb53a63ceeb5d8cf642fade7d39e7963a22ddb", size = 12486 }, + { url = "https://files.pythonhosted.org/packages/34/cf/65d4a571869a1a9078198ca28f39fba5fbb910f952f9dbc5220afff9f5e6/MarkupSafe-3.0.2-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4aa4e5faecf353ed117801a068ebab7b7e09ffb6e1d5e412dc852e0da018126c", size = 25480 }, + { url = "https://files.pythonhosted.org/packages/0c/e3/90e9651924c430b885468b56b3d597cabf6d72be4b24a0acd1fa0e12af67/MarkupSafe-3.0.2-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c0ef13eaeee5b615fb07c9a7dadb38eac06a0608b41570d8ade51c56539e509d", size = 23914 }, + { url = "https://files.pythonhosted.org/packages/66/8c/6c7cf61f95d63bb866db39085150df1f2a5bd3335298f14a66b48e92659c/MarkupSafe-3.0.2-cp313-cp313t-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d16a81a06776313e817c951135cf7340a3e91e8c1ff2fac444cfd75fffa04afe", size = 23796 }, + { url = "https://files.pythonhosted.org/packages/bb/35/cbe9238ec3f47ac9a7c8b3df7a808e7cb50fe149dc7039f5f454b3fba218/MarkupSafe-3.0.2-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:6381026f158fdb7c72a168278597a5e3a5222e83ea18f543112b2662a9b699c5", size = 25473 }, + { url = "https://files.pythonhosted.org/packages/e6/32/7621a4382488aa283cc05e8984a9c219abad3bca087be9ec77e89939ded9/MarkupSafe-3.0.2-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:3d79d162e7be8f996986c064d1c7c817f6df3a77fe3d6859f6f9e7be4b8c213a", size = 24114 }, + { url = "https://files.pythonhosted.org/packages/0d/80/0985960e4b89922cb5a0bac0ed39c5b96cbc1a536a99f30e8c220a996ed9/MarkupSafe-3.0.2-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:131a3c7689c85f5ad20f9f6fb1b866f402c445b220c19fe4308c0b147ccd2ad9", size = 24098 }, + { url = "https://files.pythonhosted.org/packages/82/78/fedb03c7d5380df2427038ec8d973587e90561b2d90cd472ce9254cf348b/MarkupSafe-3.0.2-cp313-cp313t-win32.whl", hash = "sha256:ba8062ed2cf21c07a9e295d5b8a2a5ce678b913b45fdf68c32d95d6c1291e0b6", size = 15208 }, + { url = "https://files.pythonhosted.org/packages/4f/65/6079a46068dfceaeabb5dcad6d674f5f5c61a6fa5673746f42a9f4c233b3/MarkupSafe-3.0.2-cp313-cp313t-win_amd64.whl", hash = "sha256:e444a31f8db13eb18ada366ab3cf45fd4b31e4db1236a4448f68778c1d1a5a2f", size = 15739 }, +] + +[[package]] +name = "matplotlib" +version = "3.10.5" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "contourpy", version = "1.3.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, + { name = "contourpy", version = "1.3.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, + { name = "cycler" }, + { name = "fonttools" }, + { name = "kiwisolver" }, + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, + { name = "numpy", version = "2.3.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, + { name = "packaging" }, + { name = "pillow" }, + { name = "pyparsing" }, + { name = "python-dateutil" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/43/91/f2939bb60b7ebf12478b030e0d7f340247390f402b3b189616aad790c366/matplotlib-3.10.5.tar.gz", hash = "sha256:352ed6ccfb7998a00881692f38b4ca083c691d3e275b4145423704c34c909076", size = 34804044 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/89/5355cdfe43242cb4d1a64a67cb6831398b665ad90e9702c16247cbd8d5ab/matplotlib-3.10.5-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:5d4773a6d1c106ca05cb5a5515d277a6bb96ed09e5c8fab6b7741b8fcaa62c8f", size = 8229094 }, + { url = "https://files.pythonhosted.org/packages/34/bc/ba802650e1c69650faed261a9df004af4c6f21759d7a1ec67fe972f093b3/matplotlib-3.10.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:dc88af74e7ba27de6cbe6faee916024ea35d895ed3d61ef6f58c4ce97da7185a", size = 8091464 }, + { url = "https://files.pythonhosted.org/packages/ac/64/8d0c8937dee86c286625bddb1902efacc3e22f2b619f5b5a8df29fe5217b/matplotlib-3.10.5-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:64c4535419d5617f7363dad171a5a59963308e0f3f813c4bed6c9e6e2c131512", size = 8653163 }, + { url = "https://files.pythonhosted.org/packages/11/dc/8dfc0acfbdc2fc2336c72561b7935cfa73db9ca70b875d8d3e1b3a6f371a/matplotlib-3.10.5-cp310-cp310-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a277033048ab22d34f88a3c5243938cef776493f6201a8742ed5f8b553201343", size = 9490635 }, + { url = "https://files.pythonhosted.org/packages/54/02/e3fdfe0f2e9fb05f3a691d63876639dbf684170fdcf93231e973104153b4/matplotlib-3.10.5-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:e4a6470a118a2e93022ecc7d3bd16b3114b2004ea2bf014fff875b3bc99b70c6", size = 9539036 }, + { url = "https://files.pythonhosted.org/packages/c1/29/82bf486ff7f4dbedfb11ccc207d0575cbe3be6ea26f75be514252bde3d70/matplotlib-3.10.5-cp310-cp310-win_amd64.whl", hash = "sha256:7e44cada61bec8833c106547786814dd4a266c1b2964fd25daa3804f1b8d4467", size = 8093529 }, + { url = "https://files.pythonhosted.org/packages/aa/c7/1f2db90a1d43710478bb1e9b57b162852f79234d28e4f48a28cc415aa583/matplotlib-3.10.5-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:dcfc39c452c6a9f9028d3e44d2d721484f665304857188124b505b2c95e1eecf", size = 8239216 }, + { url = "https://files.pythonhosted.org/packages/82/6d/ca6844c77a4f89b1c9e4d481c412e1d1dbabf2aae2cbc5aa2da4a1d6683e/matplotlib-3.10.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:903352681b59f3efbf4546985142a9686ea1d616bb054b09a537a06e4b892ccf", size = 8102130 }, + { url = "https://files.pythonhosted.org/packages/1d/1e/5e187a30cc673a3e384f3723e5f3c416033c1d8d5da414f82e4e731128ea/matplotlib-3.10.5-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:080c3676a56b8ee1c762bcf8fca3fe709daa1ee23e6ef06ad9f3fc17332f2d2a", size = 8666471 }, + { url = "https://files.pythonhosted.org/packages/03/c0/95540d584d7d645324db99a845ac194e915ef75011a0d5e19e1b5cee7e69/matplotlib-3.10.5-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4b4984d5064a35b6f66d2c11d668565f4389b1119cc64db7a4c1725bc11adffc", size = 9500518 }, + { url = "https://files.pythonhosted.org/packages/ba/2e/e019352099ea58b4169adb9c6e1a2ad0c568c6377c2b677ee1f06de2adc7/matplotlib-3.10.5-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:3967424121d3a46705c9fa9bdb0931de3228f13f73d7bb03c999c88343a89d89", size = 9552372 }, + { url = "https://files.pythonhosted.org/packages/b7/81/3200b792a5e8b354f31f4101ad7834743ad07b6d620259f2059317b25e4d/matplotlib-3.10.5-cp311-cp311-win_amd64.whl", hash = "sha256:33775bbeb75528555a15ac29396940128ef5613cf9a2d31fb1bfd18b3c0c0903", size = 8100634 }, + { url = "https://files.pythonhosted.org/packages/52/46/a944f6f0c1f5476a0adfa501969d229ce5ae60cf9a663be0e70361381f89/matplotlib-3.10.5-cp311-cp311-win_arm64.whl", hash = "sha256:c61333a8e5e6240e73769d5826b9a31d8b22df76c0778f8480baf1b4b01c9420", size = 7978880 }, + { url = "https://files.pythonhosted.org/packages/66/1e/c6f6bcd882d589410b475ca1fc22e34e34c82adff519caf18f3e6dd9d682/matplotlib-3.10.5-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:00b6feadc28a08bd3c65b2894f56cf3c94fc8f7adcbc6ab4516ae1e8ed8f62e2", size = 8253056 }, + { url = "https://files.pythonhosted.org/packages/53/e6/d6f7d1b59413f233793dda14419776f5f443bcccb2dfc84b09f09fe05dbe/matplotlib-3.10.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ee98a5c5344dc7f48dc261b6ba5d9900c008fc12beb3fa6ebda81273602cc389", size = 8110131 }, + { url = "https://files.pythonhosted.org/packages/66/2b/bed8a45e74957549197a2ac2e1259671cd80b55ed9e1fe2b5c94d88a9202/matplotlib-3.10.5-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:a17e57e33de901d221a07af32c08870ed4528db0b6059dce7d7e65c1122d4bea", size = 8669603 }, + { url = "https://files.pythonhosted.org/packages/7e/a7/315e9435b10d057f5e52dfc603cd353167ae28bb1a4e033d41540c0067a4/matplotlib-3.10.5-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:97b9d6443419085950ee4a5b1ee08c363e5c43d7176e55513479e53669e88468", size = 9508127 }, + { url = "https://files.pythonhosted.org/packages/7f/d9/edcbb1f02ca99165365d2768d517898c22c6040187e2ae2ce7294437c413/matplotlib-3.10.5-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:ceefe5d40807d29a66ae916c6a3915d60ef9f028ce1927b84e727be91d884369", size = 9566926 }, + { url = "https://files.pythonhosted.org/packages/3b/d9/6dd924ad5616c97b7308e6320cf392c466237a82a2040381163b7500510a/matplotlib-3.10.5-cp312-cp312-win_amd64.whl", hash = "sha256:c04cba0f93d40e45b3c187c6c52c17f24535b27d545f757a2fffebc06c12b98b", size = 8107599 }, + { url = "https://files.pythonhosted.org/packages/0e/f3/522dc319a50f7b0279fbe74f86f7a3506ce414bc23172098e8d2bdf21894/matplotlib-3.10.5-cp312-cp312-win_arm64.whl", hash = "sha256:a41bcb6e2c8e79dc99c5511ae6f7787d2fb52efd3d805fff06d5d4f667db16b2", size = 7978173 }, + { url = "https://files.pythonhosted.org/packages/8d/05/4f3c1f396075f108515e45cb8d334aff011a922350e502a7472e24c52d77/matplotlib-3.10.5-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:354204db3f7d5caaa10e5de74549ef6a05a4550fdd1c8f831ab9bca81efd39ed", size = 8253586 }, + { url = "https://files.pythonhosted.org/packages/2f/2c/e084415775aac7016c3719fe7006cdb462582c6c99ac142f27303c56e243/matplotlib-3.10.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:b072aac0c3ad563a2b3318124756cb6112157017f7431626600ecbe890df57a1", size = 8110715 }, + { url = "https://files.pythonhosted.org/packages/52/1b/233e3094b749df16e3e6cd5a44849fd33852e692ad009cf7de00cf58ddf6/matplotlib-3.10.5-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:d52fd5b684d541b5a51fb276b2b97b010c75bee9aa392f96b4a07aeb491e33c7", size = 8669397 }, + { url = "https://files.pythonhosted.org/packages/e8/ec/03f9e003a798f907d9f772eed9b7c6a9775d5bd00648b643ebfb88e25414/matplotlib-3.10.5-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ee7a09ae2f4676276f5a65bd9f2bd91b4f9fbaedf49f40267ce3f9b448de501f", size = 9508646 }, + { url = "https://files.pythonhosted.org/packages/91/e7/c051a7a386680c28487bca27d23b02d84f63e3d2a9b4d2fc478e6a42e37e/matplotlib-3.10.5-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:ba6c3c9c067b83481d647af88b4e441d532acdb5ef22178a14935b0b881188f4", size = 9567424 }, + { url = "https://files.pythonhosted.org/packages/36/c2/24302e93ff431b8f4173ee1dd88976c8d80483cadbc5d3d777cef47b3a1c/matplotlib-3.10.5-cp313-cp313-win_amd64.whl", hash = "sha256:07442d2692c9bd1cceaa4afb4bbe5b57b98a7599de4dabfcca92d3eea70f9ebe", size = 8107809 }, + { url = "https://files.pythonhosted.org/packages/0b/33/423ec6a668d375dad825197557ed8fbdb74d62b432c1ed8235465945475f/matplotlib-3.10.5-cp313-cp313-win_arm64.whl", hash = "sha256:48fe6d47380b68a37ccfcc94f009530e84d41f71f5dae7eda7c4a5a84aa0a674", size = 7978078 }, + { url = "https://files.pythonhosted.org/packages/51/17/521fc16ec766455c7bb52cc046550cf7652f6765ca8650ff120aa2d197b6/matplotlib-3.10.5-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:3b80eb8621331449fc519541a7461987f10afa4f9cfd91afcd2276ebe19bd56c", size = 8295590 }, + { url = "https://files.pythonhosted.org/packages/f8/12/23c28b2c21114c63999bae129fce7fd34515641c517ae48ce7b7dcd33458/matplotlib-3.10.5-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:47a388908e469d6ca2a6015858fa924e0e8a2345a37125948d8e93a91c47933e", size = 8158518 }, + { url = "https://files.pythonhosted.org/packages/81/f8/aae4eb25e8e7190759f3cb91cbeaa344128159ac92bb6b409e24f8711f78/matplotlib-3.10.5-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:8b6b49167d208358983ce26e43aa4196073b4702858670f2eb111f9a10652b4b", size = 8691815 }, + { url = "https://files.pythonhosted.org/packages/d0/ba/450c39ebdd486bd33a359fc17365ade46c6a96bf637bbb0df7824de2886c/matplotlib-3.10.5-cp313-cp313t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8a8da0453a7fd8e3da114234ba70c5ba9ef0e98f190309ddfde0f089accd46ea", size = 9522814 }, + { url = "https://files.pythonhosted.org/packages/89/11/9c66f6a990e27bb9aa023f7988d2d5809cb98aa39c09cbf20fba75a542ef/matplotlib-3.10.5-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:52c6573dfcb7726a9907b482cd5b92e6b5499b284ffacb04ffbfe06b3e568124", size = 9573917 }, + { url = "https://files.pythonhosted.org/packages/b3/69/8b49394de92569419e5e05e82e83df9b749a0ff550d07631ea96ed2eb35a/matplotlib-3.10.5-cp313-cp313t-win_amd64.whl", hash = "sha256:a23193db2e9d64ece69cac0c8231849db7dd77ce59c7b89948cf9d0ce655a3ce", size = 8181034 }, + { url = "https://files.pythonhosted.org/packages/47/23/82dc435bb98a2fc5c20dffcac8f0b083935ac28286413ed8835df40d0baa/matplotlib-3.10.5-cp313-cp313t-win_arm64.whl", hash = "sha256:56da3b102cf6da2776fef3e71cd96fcf22103a13594a18ac9a9b31314e0be154", size = 8023337 }, + { url = "https://files.pythonhosted.org/packages/ac/e0/26b6cfde31f5383503ee45dcb7e691d45dadf0b3f54639332b59316a97f8/matplotlib-3.10.5-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:96ef8f5a3696f20f55597ffa91c28e2e73088df25c555f8d4754931515512715", size = 8253591 }, + { url = "https://files.pythonhosted.org/packages/c1/89/98488c7ef7ea20ea659af7499628c240a608b337af4be2066d644cfd0a0f/matplotlib-3.10.5-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:77fab633e94b9da60512d4fa0213daeb76d5a7b05156840c4fd0399b4b818837", size = 8112566 }, + { url = "https://files.pythonhosted.org/packages/52/67/42294dfedc82aea55e1a767daf3263aacfb5a125f44ba189e685bab41b6f/matplotlib-3.10.5-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:27f52634315e96b1debbfdc5c416592edcd9c4221bc2f520fd39c33db5d9f202", size = 9513281 }, + { url = "https://files.pythonhosted.org/packages/e7/68/f258239e0cf34c2cbc816781c7ab6fca768452e6bf1119aedd2bd4a882a3/matplotlib-3.10.5-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:525f6e28c485c769d1f07935b660c864de41c37fd716bfa64158ea646f7084bb", size = 9780873 }, + { url = "https://files.pythonhosted.org/packages/89/64/f4881554006bd12e4558bd66778bdd15d47b00a1f6c6e8b50f6208eda4b3/matplotlib-3.10.5-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:1f5f3ec4c191253c5f2b7c07096a142c6a1c024d9f738247bfc8e3f9643fc975", size = 9568954 }, + { url = "https://files.pythonhosted.org/packages/06/f8/42779d39c3f757e1f012f2dda3319a89fb602bd2ef98ce8faf0281f4febd/matplotlib-3.10.5-cp314-cp314-win_amd64.whl", hash = "sha256:707f9c292c4cd4716f19ab8a1f93f26598222cd931e0cd98fbbb1c5994bf7667", size = 8237465 }, + { url = "https://files.pythonhosted.org/packages/cf/f8/153fd06b5160f0cd27c8b9dd797fcc9fb56ac6a0ebf3c1f765b6b68d3c8a/matplotlib-3.10.5-cp314-cp314-win_arm64.whl", hash = "sha256:21a95b9bf408178d372814de7baacd61c712a62cae560b5e6f35d791776f6516", size = 8108898 }, + { url = "https://files.pythonhosted.org/packages/9a/ee/c4b082a382a225fe0d2a73f1f57cf6f6f132308805b493a54c8641006238/matplotlib-3.10.5-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:a6b310f95e1102a8c7c817ef17b60ee5d1851b8c71b63d9286b66b177963039e", size = 8295636 }, + { url = "https://files.pythonhosted.org/packages/30/73/2195fa2099718b21a20da82dfc753bf2af58d596b51aefe93e359dd5915a/matplotlib-3.10.5-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:94986a242747a0605cb3ff1cb98691c736f28a59f8ffe5175acaeb7397c49a5a", size = 8158575 }, + { url = "https://files.pythonhosted.org/packages/f6/e9/a08cdb34618a91fa08f75e6738541da5cacde7c307cea18ff10f0d03fcff/matplotlib-3.10.5-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1ff10ea43288f0c8bab608a305dc6c918cc729d429c31dcbbecde3b9f4d5b569", size = 9522815 }, + { url = "https://files.pythonhosted.org/packages/4e/bb/34d8b7e0d1bb6d06ef45db01dfa560d5a67b1c40c0b998ce9ccde934bb09/matplotlib-3.10.5-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f6adb644c9d040ffb0d3434e440490a66cf73dbfa118a6f79cd7568431f7a012", size = 9783514 }, + { url = "https://files.pythonhosted.org/packages/12/09/d330d1e55dcca2e11b4d304cc5227f52e2512e46828d6249b88e0694176e/matplotlib-3.10.5-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:4fa40a8f98428f789a9dcacd625f59b7bc4e3ef6c8c7c80187a7a709475cf592", size = 9573932 }, + { url = "https://files.pythonhosted.org/packages/eb/3b/f70258ac729aa004aca673800a53a2b0a26d49ca1df2eaa03289a1c40f81/matplotlib-3.10.5-cp314-cp314t-win_amd64.whl", hash = "sha256:95672a5d628b44207aab91ec20bf59c26da99de12b88f7e0b1fb0a84a86ff959", size = 8322003 }, + { url = "https://files.pythonhosted.org/packages/5b/60/3601f8ce6d76a7c81c7f25a0e15fde0d6b66226dd187aa6d2838e6374161/matplotlib-3.10.5-cp314-cp314t-win_arm64.whl", hash = "sha256:2efaf97d72629e74252e0b5e3c46813e9eeaa94e011ecf8084a971a31a97f40b", size = 8153849 }, + { url = "https://files.pythonhosted.org/packages/e4/eb/7d4c5de49eb78294e1a8e2be8a6ecff8b433e921b731412a56cd1abd3567/matplotlib-3.10.5-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:b5fa2e941f77eb579005fb804026f9d0a1082276118d01cc6051d0d9626eaa7f", size = 8222360 }, + { url = "https://files.pythonhosted.org/packages/16/8a/e435db90927b66b16d69f8f009498775f4469f8de4d14b87856965e58eba/matplotlib-3.10.5-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:1fc0d2a3241cdcb9daaca279204a3351ce9df3c0e7e621c7e04ec28aaacaca30", size = 8087462 }, + { url = "https://files.pythonhosted.org/packages/0b/dd/06c0e00064362f5647f318e00b435be2ff76a1bdced97c5eaf8347311fbe/matplotlib-3.10.5-pp310-pypy310_pp73-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:8dee65cb1424b7dc982fe87895b5613d4e691cc57117e8af840da0148ca6c1d7", size = 8659802 }, + { url = "https://files.pythonhosted.org/packages/dc/d6/e921be4e1a5f7aca5194e1f016cb67ec294548e530013251f630713e456d/matplotlib-3.10.5-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:160e125da27a749481eaddc0627962990f6029811dbeae23881833a011a0907f", size = 8233224 }, + { url = "https://files.pythonhosted.org/packages/ec/74/a2b9b04824b9c349c8f1b2d21d5af43fa7010039427f2b133a034cb09e59/matplotlib-3.10.5-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:ac3d50760394d78a3c9be6b28318fe22b494c4fcf6407e8fd4794b538251899b", size = 8098539 }, + { url = "https://files.pythonhosted.org/packages/fc/66/cd29ebc7f6c0d2a15d216fb572573e8fc38bd5d6dec3bd9d7d904c0949f7/matplotlib-3.10.5-pp311-pypy311_pp73-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:6c49465bf689c4d59d174d0c7795fb42a21d4244d11d70e52b8011987367ac61", size = 8672192 }, +] + +[[package]] +name = "mpmath" +version = "1.3.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e0/47/dd32fa426cc72114383ac549964eecb20ecfd886d1e5ccf5340b55b02f57/mpmath-1.3.0.tar.gz", hash = "sha256:7a28eb2a9774d00c7bc92411c19a89209d5da7c4c9a9e227be8330a23a25b91f", size = 508106 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/43/e3/7d92a15f894aa0c9c4b49b8ee9ac9850d6e63b03c9c32c0367a13ae62209/mpmath-1.3.0-py3-none-any.whl", hash = "sha256:a0b2b9fe80bbcd81a6647ff13108738cfb482d481d826cc0e02f5b35e5c88d2c", size = 536198 }, +] + +[[package]] +name = "mypy" +version = "1.17.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "mypy-extensions" }, + { name = "pathspec" }, + { name = "tomli", marker = "python_full_version < '3.11'" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/8e/22/ea637422dedf0bf36f3ef238eab4e455e2a0dcc3082b5cc067615347ab8e/mypy-1.17.1.tar.gz", hash = "sha256:25e01ec741ab5bb3eec8ba9cdb0f769230368a22c959c4937360efb89b7e9f01", size = 3352570 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/77/a9/3d7aa83955617cdf02f94e50aab5c830d205cfa4320cf124ff64acce3a8e/mypy-1.17.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:3fbe6d5555bf608c47203baa3e72dbc6ec9965b3d7c318aa9a4ca76f465bd972", size = 11003299 }, + { url = "https://files.pythonhosted.org/packages/83/e8/72e62ff837dd5caaac2b4a5c07ce769c8e808a00a65e5d8f94ea9c6f20ab/mypy-1.17.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:80ef5c058b7bce08c83cac668158cb7edea692e458d21098c7d3bce35a5d43e7", size = 10125451 }, + { url = "https://files.pythonhosted.org/packages/7d/10/f3f3543f6448db11881776f26a0ed079865926b0c841818ee22de2c6bbab/mypy-1.17.1-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c4a580f8a70c69e4a75587bd925d298434057fe2a428faaf927ffe6e4b9a98df", size = 11916211 }, + { url = "https://files.pythonhosted.org/packages/06/bf/63e83ed551282d67bb3f7fea2cd5561b08d2bb6eb287c096539feb5ddbc5/mypy-1.17.1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:dd86bb649299f09d987a2eebb4d52d10603224500792e1bee18303bbcc1ce390", size = 12652687 }, + { url = "https://files.pythonhosted.org/packages/69/66/68f2eeef11facf597143e85b694a161868b3b006a5fbad50e09ea117ef24/mypy-1.17.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:a76906f26bd8d51ea9504966a9c25419f2e668f012e0bdf3da4ea1526c534d94", size = 12896322 }, + { url = "https://files.pythonhosted.org/packages/a3/87/8e3e9c2c8bd0d7e071a89c71be28ad088aaecbadf0454f46a540bda7bca6/mypy-1.17.1-cp310-cp310-win_amd64.whl", hash = "sha256:e79311f2d904ccb59787477b7bd5d26f3347789c06fcd7656fa500875290264b", size = 9507962 }, + { url = "https://files.pythonhosted.org/packages/46/cf/eadc80c4e0a70db1c08921dcc220357ba8ab2faecb4392e3cebeb10edbfa/mypy-1.17.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:ad37544be07c5d7fba814eb370e006df58fed8ad1ef33ed1649cb1889ba6ff58", size = 10921009 }, + { url = "https://files.pythonhosted.org/packages/5d/c1/c869d8c067829ad30d9bdae051046561552516cfb3a14f7f0347b7d973ee/mypy-1.17.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:064e2ff508e5464b4bd807a7c1625bc5047c5022b85c70f030680e18f37273a5", size = 10047482 }, + { url = "https://files.pythonhosted.org/packages/98/b9/803672bab3fe03cee2e14786ca056efda4bb511ea02dadcedde6176d06d0/mypy-1.17.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:70401bbabd2fa1aa7c43bb358f54037baf0586f41e83b0ae67dd0534fc64edfd", size = 11832883 }, + { url = "https://files.pythonhosted.org/packages/88/fb/fcdac695beca66800918c18697b48833a9a6701de288452b6715a98cfee1/mypy-1.17.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e92bdc656b7757c438660f775f872a669b8ff374edc4d18277d86b63edba6b8b", size = 12566215 }, + { url = "https://files.pythonhosted.org/packages/7f/37/a932da3d3dace99ee8eb2043b6ab03b6768c36eb29a02f98f46c18c0da0e/mypy-1.17.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:c1fdf4abb29ed1cb091cf432979e162c208a5ac676ce35010373ff29247bcad5", size = 12751956 }, + { url = "https://files.pythonhosted.org/packages/8c/cf/6438a429e0f2f5cab8bc83e53dbebfa666476f40ee322e13cac5e64b79e7/mypy-1.17.1-cp311-cp311-win_amd64.whl", hash = "sha256:ff2933428516ab63f961644bc49bc4cbe42bbffb2cd3b71cc7277c07d16b1a8b", size = 9507307 }, + { url = "https://files.pythonhosted.org/packages/17/a2/7034d0d61af8098ec47902108553122baa0f438df8a713be860f7407c9e6/mypy-1.17.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:69e83ea6553a3ba79c08c6e15dbd9bfa912ec1e493bf75489ef93beb65209aeb", size = 11086295 }, + { url = "https://files.pythonhosted.org/packages/14/1f/19e7e44b594d4b12f6ba8064dbe136505cec813549ca3e5191e40b1d3cc2/mypy-1.17.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:1b16708a66d38abb1e6b5702f5c2c87e133289da36f6a1d15f6a5221085c6403", size = 10112355 }, + { url = "https://files.pythonhosted.org/packages/5b/69/baa33927e29e6b4c55d798a9d44db5d394072eef2bdc18c3e2048c9ed1e9/mypy-1.17.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:89e972c0035e9e05823907ad5398c5a73b9f47a002b22359b177d40bdaee7056", size = 11875285 }, + { url = "https://files.pythonhosted.org/packages/90/13/f3a89c76b0a41e19490b01e7069713a30949d9a6c147289ee1521bcea245/mypy-1.17.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:03b6d0ed2b188e35ee6d5c36b5580cffd6da23319991c49ab5556c023ccf1341", size = 12737895 }, + { url = "https://files.pythonhosted.org/packages/23/a1/c4ee79ac484241301564072e6476c5a5be2590bc2e7bfd28220033d2ef8f/mypy-1.17.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:c837b896b37cd103570d776bda106eabb8737aa6dd4f248451aecf53030cdbeb", size = 12931025 }, + { url = "https://files.pythonhosted.org/packages/89/b8/7409477be7919a0608900e6320b155c72caab4fef46427c5cc75f85edadd/mypy-1.17.1-cp312-cp312-win_amd64.whl", hash = "sha256:665afab0963a4b39dff7c1fa563cc8b11ecff7910206db4b2e64dd1ba25aed19", size = 9584664 }, + { url = "https://files.pythonhosted.org/packages/5b/82/aec2fc9b9b149f372850291827537a508d6c4d3664b1750a324b91f71355/mypy-1.17.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:93378d3203a5c0800c6b6d850ad2f19f7a3cdf1a3701d3416dbf128805c6a6a7", size = 11075338 }, + { url = "https://files.pythonhosted.org/packages/07/ac/ee93fbde9d2242657128af8c86f5d917cd2887584cf948a8e3663d0cd737/mypy-1.17.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:15d54056f7fe7a826d897789f53dd6377ec2ea8ba6f776dc83c2902b899fee81", size = 10113066 }, + { url = "https://files.pythonhosted.org/packages/5a/68/946a1e0be93f17f7caa56c45844ec691ca153ee8b62f21eddda336a2d203/mypy-1.17.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:209a58fed9987eccc20f2ca94afe7257a8f46eb5df1fb69958650973230f91e6", size = 11875473 }, + { url = "https://files.pythonhosted.org/packages/9f/0f/478b4dce1cb4f43cf0f0d00fba3030b21ca04a01b74d1cd272a528cf446f/mypy-1.17.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:099b9a5da47de9e2cb5165e581f158e854d9e19d2e96b6698c0d64de911dd849", size = 12744296 }, + { url = "https://files.pythonhosted.org/packages/ca/70/afa5850176379d1b303f992a828de95fc14487429a7139a4e0bdd17a8279/mypy-1.17.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:fa6ffadfbe6994d724c5a1bb6123a7d27dd68fc9c059561cd33b664a79578e14", size = 12914657 }, + { url = "https://files.pythonhosted.org/packages/53/f9/4a83e1c856a3d9c8f6edaa4749a4864ee98486e9b9dbfbc93842891029c2/mypy-1.17.1-cp313-cp313-win_amd64.whl", hash = "sha256:9a2b7d9180aed171f033c9f2fc6c204c1245cf60b0cb61cf2e7acc24eea78e0a", size = 9593320 }, + { url = "https://files.pythonhosted.org/packages/38/56/79c2fac86da57c7d8c48622a05873eaab40b905096c33597462713f5af90/mypy-1.17.1-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:15a83369400454c41ed3a118e0cc58bd8123921a602f385cb6d6ea5df050c733", size = 11040037 }, + { url = "https://files.pythonhosted.org/packages/4d/c3/adabe6ff53638e3cad19e3547268482408323b1e68bf082c9119000cd049/mypy-1.17.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:55b918670f692fc9fba55c3298d8a3beae295c5cded0a55dccdc5bbead814acd", size = 10131550 }, + { url = "https://files.pythonhosted.org/packages/b8/c5/2e234c22c3bdeb23a7817af57a58865a39753bde52c74e2c661ee0cfc640/mypy-1.17.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:62761474061feef6f720149d7ba876122007ddc64adff5ba6f374fda35a018a0", size = 11872963 }, + { url = "https://files.pythonhosted.org/packages/ab/26/c13c130f35ca8caa5f2ceab68a247775648fdcd6c9a18f158825f2bc2410/mypy-1.17.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c49562d3d908fd49ed0938e5423daed8d407774a479b595b143a3d7f87cdae6a", size = 12710189 }, + { url = "https://files.pythonhosted.org/packages/82/df/c7d79d09f6de8383fe800521d066d877e54d30b4fb94281c262be2df84ef/mypy-1.17.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:397fba5d7616a5bc60b45c7ed204717eaddc38f826e3645402c426057ead9a91", size = 12900322 }, + { url = "https://files.pythonhosted.org/packages/b8/98/3d5a48978b4f708c55ae832619addc66d677f6dc59f3ebad71bae8285ca6/mypy-1.17.1-cp314-cp314-win_amd64.whl", hash = "sha256:9d6b20b97d373f41617bd0708fd46aa656059af57f2ef72aa8c7d6a2b73b74ed", size = 9751879 }, + { url = "https://files.pythonhosted.org/packages/1d/f3/8fcd2af0f5b806f6cf463efaffd3c9548a28f84220493ecd38d127b6b66d/mypy-1.17.1-py3-none-any.whl", hash = "sha256:a9f52c0351c21fe24c21d8c0eb1f62967b262d6729393397b6f443c3b773c3b9", size = 2283411 }, +] + +[[package]] +name = "mypy-extensions" +version = "1.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a2/6e/371856a3fb9d31ca8dac321cda606860fa4548858c0cc45d9d1d4ca2628b/mypy_extensions-1.1.0.tar.gz", hash = "sha256:52e68efc3284861e772bbcd66823fde5ae21fd2fdb51c62a211403730b916558", size = 6343 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/79/7b/2c79738432f5c924bef5071f933bcc9efd0473bac3b4aa584a6f7c1c8df8/mypy_extensions-1.1.0-py3-none-any.whl", hash = "sha256:1be4cccdb0f2482337c4743e60421de3a356cd97508abadd57d47403e94f5505", size = 4963 }, +] + +[[package]] +name = "networkx" +version = "3.4.2" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version < '3.11'", +] +sdist = { url = "https://files.pythonhosted.org/packages/fd/1d/06475e1cd5264c0b870ea2cc6fdb3e37177c1e565c43f56ff17a10e3937f/networkx-3.4.2.tar.gz", hash = "sha256:307c3669428c5362aab27c8a1260aa8f47c4e91d3891f48be0141738d8d053e1", size = 2151368 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b9/54/dd730b32ea14ea797530a4479b2ed46a6fb250f682a9cfb997e968bf0261/networkx-3.4.2-py3-none-any.whl", hash = "sha256:df5d4365b724cf81b8c6a7312509d0c22386097011ad1abe274afd5e9d3bbc5f", size = 1723263 }, +] + +[[package]] +name = "networkx" +version = "3.5" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.12'", + "python_full_version == '3.11.*'", +] +sdist = { url = "https://files.pythonhosted.org/packages/6c/4f/ccdb8ad3a38e583f214547fd2f7ff1fc160c43a75af88e6aec213404b96a/networkx-3.5.tar.gz", hash = "sha256:d4c6f9cf81f52d69230866796b82afbccdec3db7ae4fbd1b65ea750feed50037", size = 2471065 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/eb/8d/776adee7bbf76365fdd7f2552710282c79a4ead5d2a46408c9043a2b70ba/networkx-3.5-py3-none-any.whl", hash = "sha256:0030d386a9a06dee3565298b4a734b68589749a544acbb6c412dc9e2489ec6ec", size = 2034406 }, +] + +[[package]] +name = "numpy" +version = "2.2.6" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version < '3.11'", +] +sdist = { url = "https://files.pythonhosted.org/packages/76/21/7d2a95e4bba9dc13d043ee156a356c0a8f0c6309dff6b21b4d71a073b8a8/numpy-2.2.6.tar.gz", hash = "sha256:e29554e2bef54a90aa5cc07da6ce955accb83f21ab5de01a62c8478897b264fd", size = 20276440 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9a/3e/ed6db5be21ce87955c0cbd3009f2803f59fa08df21b5df06862e2d8e2bdd/numpy-2.2.6-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:b412caa66f72040e6d268491a59f2c43bf03eb6c96dd8f0307829feb7fa2b6fb", size = 21165245 }, + { url = "https://files.pythonhosted.org/packages/22/c2/4b9221495b2a132cc9d2eb862e21d42a009f5a60e45fc44b00118c174bff/numpy-2.2.6-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8e41fd67c52b86603a91c1a505ebaef50b3314de0213461c7a6e99c9a3beff90", size = 14360048 }, + { url = "https://files.pythonhosted.org/packages/fd/77/dc2fcfc66943c6410e2bf598062f5959372735ffda175b39906d54f02349/numpy-2.2.6-cp310-cp310-macosx_14_0_arm64.whl", hash = "sha256:37e990a01ae6ec7fe7fa1c26c55ecb672dd98b19c3d0e1d1f326fa13cb38d163", size = 5340542 }, + { url = "https://files.pythonhosted.org/packages/7a/4f/1cb5fdc353a5f5cc7feb692db9b8ec2c3d6405453f982435efc52561df58/numpy-2.2.6-cp310-cp310-macosx_14_0_x86_64.whl", hash = "sha256:5a6429d4be8ca66d889b7cf70f536a397dc45ba6faeb5f8c5427935d9592e9cf", size = 6878301 }, + { url = "https://files.pythonhosted.org/packages/eb/17/96a3acd228cec142fcb8723bd3cc39c2a474f7dcf0a5d16731980bcafa95/numpy-2.2.6-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:efd28d4e9cd7d7a8d39074a4d44c63eda73401580c5c76acda2ce969e0a38e83", size = 14297320 }, + { url = "https://files.pythonhosted.org/packages/b4/63/3de6a34ad7ad6646ac7d2f55ebc6ad439dbbf9c4370017c50cf403fb19b5/numpy-2.2.6-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fc7b73d02efb0e18c000e9ad8b83480dfcd5dfd11065997ed4c6747470ae8915", size = 16801050 }, + { url = "https://files.pythonhosted.org/packages/07/b6/89d837eddef52b3d0cec5c6ba0456c1bf1b9ef6a6672fc2b7873c3ec4e2e/numpy-2.2.6-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:74d4531beb257d2c3f4b261bfb0fc09e0f9ebb8842d82a7b4209415896adc680", size = 15807034 }, + { url = "https://files.pythonhosted.org/packages/01/c8/dc6ae86e3c61cfec1f178e5c9f7858584049b6093f843bca541f94120920/numpy-2.2.6-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:8fc377d995680230e83241d8a96def29f204b5782f371c532579b4f20607a289", size = 18614185 }, + { url = "https://files.pythonhosted.org/packages/5b/c5/0064b1b7e7c89137b471ccec1fd2282fceaae0ab3a9550f2568782d80357/numpy-2.2.6-cp310-cp310-win32.whl", hash = "sha256:b093dd74e50a8cba3e873868d9e93a85b78e0daf2e98c6797566ad8044e8363d", size = 6527149 }, + { url = "https://files.pythonhosted.org/packages/a3/dd/4b822569d6b96c39d1215dbae0582fd99954dcbcf0c1a13c61783feaca3f/numpy-2.2.6-cp310-cp310-win_amd64.whl", hash = "sha256:f0fd6321b839904e15c46e0d257fdd101dd7f530fe03fd6359c1ea63738703f3", size = 12904620 }, + { url = "https://files.pythonhosted.org/packages/da/a8/4f83e2aa666a9fbf56d6118faaaf5f1974d456b1823fda0a176eff722839/numpy-2.2.6-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:f9f1adb22318e121c5c69a09142811a201ef17ab257a1e66ca3025065b7f53ae", size = 21176963 }, + { url = "https://files.pythonhosted.org/packages/b3/2b/64e1affc7972decb74c9e29e5649fac940514910960ba25cd9af4488b66c/numpy-2.2.6-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:c820a93b0255bc360f53eca31a0e676fd1101f673dda8da93454a12e23fc5f7a", size = 14406743 }, + { url = "https://files.pythonhosted.org/packages/4a/9f/0121e375000b5e50ffdd8b25bf78d8e1a5aa4cca3f185d41265198c7b834/numpy-2.2.6-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:3d70692235e759f260c3d837193090014aebdf026dfd167834bcba43e30c2a42", size = 5352616 }, + { url = "https://files.pythonhosted.org/packages/31/0d/b48c405c91693635fbe2dcd7bc84a33a602add5f63286e024d3b6741411c/numpy-2.2.6-cp311-cp311-macosx_14_0_x86_64.whl", hash = "sha256:481b49095335f8eed42e39e8041327c05b0f6f4780488f61286ed3c01368d491", size = 6889579 }, + { url = "https://files.pythonhosted.org/packages/52/b8/7f0554d49b565d0171eab6e99001846882000883998e7b7d9f0d98b1f934/numpy-2.2.6-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b64d8d4d17135e00c8e346e0a738deb17e754230d7e0810ac5012750bbd85a5a", size = 14312005 }, + { url = "https://files.pythonhosted.org/packages/b3/dd/2238b898e51bd6d389b7389ffb20d7f4c10066d80351187ec8e303a5a475/numpy-2.2.6-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba10f8411898fc418a521833e014a77d3ca01c15b0c6cdcce6a0d2897e6dbbdf", size = 16821570 }, + { url = "https://files.pythonhosted.org/packages/83/6c/44d0325722cf644f191042bf47eedad61c1e6df2432ed65cbe28509d404e/numpy-2.2.6-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:bd48227a919f1bafbdda0583705e547892342c26fb127219d60a5c36882609d1", size = 15818548 }, + { url = "https://files.pythonhosted.org/packages/ae/9d/81e8216030ce66be25279098789b665d49ff19eef08bfa8cb96d4957f422/numpy-2.2.6-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:9551a499bf125c1d4f9e250377c1ee2eddd02e01eac6644c080162c0c51778ab", size = 18620521 }, + { url = "https://files.pythonhosted.org/packages/6a/fd/e19617b9530b031db51b0926eed5345ce8ddc669bb3bc0044b23e275ebe8/numpy-2.2.6-cp311-cp311-win32.whl", hash = "sha256:0678000bb9ac1475cd454c6b8c799206af8107e310843532b04d49649c717a47", size = 6525866 }, + { url = "https://files.pythonhosted.org/packages/31/0a/f354fb7176b81747d870f7991dc763e157a934c717b67b58456bc63da3df/numpy-2.2.6-cp311-cp311-win_amd64.whl", hash = "sha256:e8213002e427c69c45a52bbd94163084025f533a55a59d6f9c5b820774ef3303", size = 12907455 }, + { url = "https://files.pythonhosted.org/packages/82/5d/c00588b6cf18e1da539b45d3598d3557084990dcc4331960c15ee776ee41/numpy-2.2.6-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:41c5a21f4a04fa86436124d388f6ed60a9343a6f767fced1a8a71c3fbca038ff", size = 20875348 }, + { url = "https://files.pythonhosted.org/packages/66/ee/560deadcdde6c2f90200450d5938f63a34b37e27ebff162810f716f6a230/numpy-2.2.6-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:de749064336d37e340f640b05f24e9e3dd678c57318c7289d222a8a2f543e90c", size = 14119362 }, + { url = "https://files.pythonhosted.org/packages/3c/65/4baa99f1c53b30adf0acd9a5519078871ddde8d2339dc5a7fde80d9d87da/numpy-2.2.6-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:894b3a42502226a1cac872f840030665f33326fc3dac8e57c607905773cdcde3", size = 5084103 }, + { url = "https://files.pythonhosted.org/packages/cc/89/e5a34c071a0570cc40c9a54eb472d113eea6d002e9ae12bb3a8407fb912e/numpy-2.2.6-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:71594f7c51a18e728451bb50cc60a3ce4e6538822731b2933209a1f3614e9282", size = 6625382 }, + { url = "https://files.pythonhosted.org/packages/f8/35/8c80729f1ff76b3921d5c9487c7ac3de9b2a103b1cd05e905b3090513510/numpy-2.2.6-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f2618db89be1b4e05f7a1a847a9c1c0abd63e63a1607d892dd54668dd92faf87", size = 14018462 }, + { url = "https://files.pythonhosted.org/packages/8c/3d/1e1db36cfd41f895d266b103df00ca5b3cbe965184df824dec5c08c6b803/numpy-2.2.6-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fd83c01228a688733f1ded5201c678f0c53ecc1006ffbc404db9f7a899ac6249", size = 16527618 }, + { url = "https://files.pythonhosted.org/packages/61/c6/03ed30992602c85aa3cd95b9070a514f8b3c33e31124694438d88809ae36/numpy-2.2.6-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:37c0ca431f82cd5fa716eca9506aefcabc247fb27ba69c5062a6d3ade8cf8f49", size = 15505511 }, + { url = "https://files.pythonhosted.org/packages/b7/25/5761d832a81df431e260719ec45de696414266613c9ee268394dd5ad8236/numpy-2.2.6-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:fe27749d33bb772c80dcd84ae7e8df2adc920ae8297400dabec45f0dedb3f6de", size = 18313783 }, + { url = "https://files.pythonhosted.org/packages/57/0a/72d5a3527c5ebffcd47bde9162c39fae1f90138c961e5296491ce778e682/numpy-2.2.6-cp312-cp312-win32.whl", hash = "sha256:4eeaae00d789f66c7a25ac5f34b71a7035bb474e679f410e5e1a94deb24cf2d4", size = 6246506 }, + { url = "https://files.pythonhosted.org/packages/36/fa/8c9210162ca1b88529ab76b41ba02d433fd54fecaf6feb70ef9f124683f1/numpy-2.2.6-cp312-cp312-win_amd64.whl", hash = "sha256:c1f9540be57940698ed329904db803cf7a402f3fc200bfe599334c9bd84a40b2", size = 12614190 }, + { url = "https://files.pythonhosted.org/packages/f9/5c/6657823f4f594f72b5471f1db1ab12e26e890bb2e41897522d134d2a3e81/numpy-2.2.6-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:0811bb762109d9708cca4d0b13c4f67146e3c3b7cf8d34018c722adb2d957c84", size = 20867828 }, + { url = "https://files.pythonhosted.org/packages/dc/9e/14520dc3dadf3c803473bd07e9b2bd1b69bc583cb2497b47000fed2fa92f/numpy-2.2.6-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:287cc3162b6f01463ccd86be154f284d0893d2b3ed7292439ea97eafa8170e0b", size = 14143006 }, + { url = "https://files.pythonhosted.org/packages/4f/06/7e96c57d90bebdce9918412087fc22ca9851cceaf5567a45c1f404480e9e/numpy-2.2.6-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:f1372f041402e37e5e633e586f62aa53de2eac8d98cbfb822806ce4bbefcb74d", size = 5076765 }, + { url = "https://files.pythonhosted.org/packages/73/ed/63d920c23b4289fdac96ddbdd6132e9427790977d5457cd132f18e76eae0/numpy-2.2.6-cp313-cp313-macosx_14_0_x86_64.whl", hash = "sha256:55a4d33fa519660d69614a9fad433be87e5252f4b03850642f88993f7b2ca566", size = 6617736 }, + { url = "https://files.pythonhosted.org/packages/85/c5/e19c8f99d83fd377ec8c7e0cf627a8049746da54afc24ef0a0cb73d5dfb5/numpy-2.2.6-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f92729c95468a2f4f15e9bb94c432a9229d0d50de67304399627a943201baa2f", size = 14010719 }, + { url = "https://files.pythonhosted.org/packages/19/49/4df9123aafa7b539317bf6d342cb6d227e49f7a35b99c287a6109b13dd93/numpy-2.2.6-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1bc23a79bfabc5d056d106f9befb8d50c31ced2fbc70eedb8155aec74a45798f", size = 16526072 }, + { url = "https://files.pythonhosted.org/packages/b2/6c/04b5f47f4f32f7c2b0e7260442a8cbcf8168b0e1a41ff1495da42f42a14f/numpy-2.2.6-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:e3143e4451880bed956e706a3220b4e5cf6172ef05fcc397f6f36a550b1dd868", size = 15503213 }, + { url = "https://files.pythonhosted.org/packages/17/0a/5cd92e352c1307640d5b6fec1b2ffb06cd0dabe7d7b8227f97933d378422/numpy-2.2.6-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:b4f13750ce79751586ae2eb824ba7e1e8dba64784086c98cdbbcc6a42112ce0d", size = 18316632 }, + { url = "https://files.pythonhosted.org/packages/f0/3b/5cba2b1d88760ef86596ad0f3d484b1cbff7c115ae2429678465057c5155/numpy-2.2.6-cp313-cp313-win32.whl", hash = "sha256:5beb72339d9d4fa36522fc63802f469b13cdbe4fdab4a288f0c441b74272ebfd", size = 6244532 }, + { url = "https://files.pythonhosted.org/packages/cb/3b/d58c12eafcb298d4e6d0d40216866ab15f59e55d148a5658bb3132311fcf/numpy-2.2.6-cp313-cp313-win_amd64.whl", hash = "sha256:b0544343a702fa80c95ad5d3d608ea3599dd54d4632df855e4c8d24eb6ecfa1c", size = 12610885 }, + { url = "https://files.pythonhosted.org/packages/6b/9e/4bf918b818e516322db999ac25d00c75788ddfd2d2ade4fa66f1f38097e1/numpy-2.2.6-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:0bca768cd85ae743b2affdc762d617eddf3bcf8724435498a1e80132d04879e6", size = 20963467 }, + { url = "https://files.pythonhosted.org/packages/61/66/d2de6b291507517ff2e438e13ff7b1e2cdbdb7cb40b3ed475377aece69f9/numpy-2.2.6-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:fc0c5673685c508a142ca65209b4e79ed6740a4ed6b2267dbba90f34b0b3cfda", size = 14225144 }, + { url = "https://files.pythonhosted.org/packages/e4/25/480387655407ead912e28ba3a820bc69af9adf13bcbe40b299d454ec011f/numpy-2.2.6-cp313-cp313t-macosx_14_0_arm64.whl", hash = "sha256:5bd4fc3ac8926b3819797a7c0e2631eb889b4118a9898c84f585a54d475b7e40", size = 5200217 }, + { url = "https://files.pythonhosted.org/packages/aa/4a/6e313b5108f53dcbf3aca0c0f3e9c92f4c10ce57a0a721851f9785872895/numpy-2.2.6-cp313-cp313t-macosx_14_0_x86_64.whl", hash = "sha256:fee4236c876c4e8369388054d02d0e9bb84821feb1a64dd59e137e6511a551f8", size = 6712014 }, + { url = "https://files.pythonhosted.org/packages/b7/30/172c2d5c4be71fdf476e9de553443cf8e25feddbe185e0bd88b096915bcc/numpy-2.2.6-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e1dda9c7e08dc141e0247a5b8f49cf05984955246a327d4c48bda16821947b2f", size = 14077935 }, + { url = "https://files.pythonhosted.org/packages/12/fb/9e743f8d4e4d3c710902cf87af3512082ae3d43b945d5d16563f26ec251d/numpy-2.2.6-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f447e6acb680fd307f40d3da4852208af94afdfab89cf850986c3ca00562f4fa", size = 16600122 }, + { url = "https://files.pythonhosted.org/packages/12/75/ee20da0e58d3a66f204f38916757e01e33a9737d0b22373b3eb5a27358f9/numpy-2.2.6-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:389d771b1623ec92636b0786bc4ae56abafad4a4c513d36a55dce14bd9ce8571", size = 15586143 }, + { url = "https://files.pythonhosted.org/packages/76/95/bef5b37f29fc5e739947e9ce5179ad402875633308504a52d188302319c8/numpy-2.2.6-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:8e9ace4a37db23421249ed236fdcdd457d671e25146786dfc96835cd951aa7c1", size = 18385260 }, + { url = "https://files.pythonhosted.org/packages/09/04/f2f83279d287407cf36a7a8053a5abe7be3622a4363337338f2585e4afda/numpy-2.2.6-cp313-cp313t-win32.whl", hash = "sha256:038613e9fb8c72b0a41f025a7e4c3f0b7a1b5d768ece4796b674c8f3fe13efff", size = 6377225 }, + { url = "https://files.pythonhosted.org/packages/67/0e/35082d13c09c02c011cf21570543d202ad929d961c02a147493cb0c2bdf5/numpy-2.2.6-cp313-cp313t-win_amd64.whl", hash = "sha256:6031dd6dfecc0cf9f668681a37648373bddd6421fff6c66ec1624eed0180ee06", size = 12771374 }, + { url = "https://files.pythonhosted.org/packages/9e/3b/d94a75f4dbf1ef5d321523ecac21ef23a3cd2ac8b78ae2aac40873590229/numpy-2.2.6-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:0b605b275d7bd0c640cad4e5d30fa701a8d59302e127e5f79138ad62762c3e3d", size = 21040391 }, + { url = "https://files.pythonhosted.org/packages/17/f4/09b2fa1b58f0fb4f7c7963a1649c64c4d315752240377ed74d9cd878f7b5/numpy-2.2.6-pp310-pypy310_pp73-macosx_14_0_x86_64.whl", hash = "sha256:7befc596a7dc9da8a337f79802ee8adb30a552a94f792b9c9d18c840055907db", size = 6786754 }, + { url = "https://files.pythonhosted.org/packages/af/30/feba75f143bdc868a1cc3f44ccfa6c4b9ec522b36458e738cd00f67b573f/numpy-2.2.6-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ce47521a4754c8f4593837384bd3424880629f718d87c5d44f8ed763edd63543", size = 16643476 }, + { url = "https://files.pythonhosted.org/packages/37/48/ac2a9584402fb6c0cd5b5d1a91dcf176b15760130dd386bbafdbfe3640bf/numpy-2.2.6-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:d042d24c90c41b54fd506da306759e06e568864df8ec17ccc17e9e884634fd00", size = 12812666 }, +] + +[[package]] +name = "numpy" +version = "2.3.2" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.12'", + "python_full_version == '3.11.*'", +] +sdist = { url = "https://files.pythonhosted.org/packages/37/7d/3fec4199c5ffb892bed55cff901e4f39a58c81df9c44c280499e92cad264/numpy-2.3.2.tar.gz", hash = "sha256:e0486a11ec30cdecb53f184d496d1c6a20786c81e55e41640270130056f8ee48", size = 20489306 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/96/26/1320083986108998bd487e2931eed2aeedf914b6e8905431487543ec911d/numpy-2.3.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:852ae5bed3478b92f093e30f785c98e0cb62fa0a939ed057c31716e18a7a22b9", size = 21259016 }, + { url = "https://files.pythonhosted.org/packages/c4/2b/792b341463fa93fc7e55abbdbe87dac316c5b8cb5e94fb7a59fb6fa0cda5/numpy-2.3.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:7a0e27186e781a69959d0230dd9909b5e26024f8da10683bd6344baea1885168", size = 14451158 }, + { url = "https://files.pythonhosted.org/packages/b7/13/e792d7209261afb0c9f4759ffef6135b35c77c6349a151f488f531d13595/numpy-2.3.2-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:f0a1a8476ad77a228e41619af2fa9505cf69df928e9aaa165746584ea17fed2b", size = 5379817 }, + { url = "https://files.pythonhosted.org/packages/49/ce/055274fcba4107c022b2113a213c7287346563f48d62e8d2a5176ad93217/numpy-2.3.2-cp311-cp311-macosx_14_0_x86_64.whl", hash = "sha256:cbc95b3813920145032412f7e33d12080f11dc776262df1712e1638207dde9e8", size = 6913606 }, + { url = "https://files.pythonhosted.org/packages/17/f2/e4d72e6bc5ff01e2ab613dc198d560714971900c03674b41947e38606502/numpy-2.3.2-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f75018be4980a7324edc5930fe39aa391d5734531b1926968605416ff58c332d", size = 14589652 }, + { url = "https://files.pythonhosted.org/packages/c8/b0/fbeee3000a51ebf7222016e2939b5c5ecf8000a19555d04a18f1e02521b8/numpy-2.3.2-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:20b8200721840f5621b7bd03f8dcd78de33ec522fc40dc2641aa09537df010c3", size = 16938816 }, + { url = "https://files.pythonhosted.org/packages/a9/ec/2f6c45c3484cc159621ea8fc000ac5a86f1575f090cac78ac27193ce82cd/numpy-2.3.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1f91e5c028504660d606340a084db4b216567ded1056ea2b4be4f9d10b67197f", size = 16370512 }, + { url = "https://files.pythonhosted.org/packages/b5/01/dd67cf511850bd7aefd6347aaae0956ed415abea741ae107834aae7d6d4e/numpy-2.3.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:fb1752a3bb9a3ad2d6b090b88a9a0ae1cd6f004ef95f75825e2f382c183b2097", size = 18884947 }, + { url = "https://files.pythonhosted.org/packages/a7/17/2cf60fd3e6a61d006778735edf67a222787a8c1a7842aed43ef96d777446/numpy-2.3.2-cp311-cp311-win32.whl", hash = "sha256:4ae6863868aaee2f57503c7a5052b3a2807cf7a3914475e637a0ecd366ced220", size = 6599494 }, + { url = "https://files.pythonhosted.org/packages/d5/03/0eade211c504bda872a594f045f98ddcc6caef2b7c63610946845e304d3f/numpy-2.3.2-cp311-cp311-win_amd64.whl", hash = "sha256:240259d6564f1c65424bcd10f435145a7644a65a6811cfc3201c4a429ba79170", size = 13087889 }, + { url = "https://files.pythonhosted.org/packages/13/32/2c7979d39dafb2a25087e12310fc7f3b9d3c7d960df4f4bc97955ae0ce1d/numpy-2.3.2-cp311-cp311-win_arm64.whl", hash = "sha256:4209f874d45f921bde2cff1ffcd8a3695f545ad2ffbef6d3d3c6768162efab89", size = 10459560 }, + { url = "https://files.pythonhosted.org/packages/00/6d/745dd1c1c5c284d17725e5c802ca4d45cfc6803519d777f087b71c9f4069/numpy-2.3.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:bc3186bea41fae9d8e90c2b4fb5f0a1f5a690682da79b92574d63f56b529080b", size = 20956420 }, + { url = "https://files.pythonhosted.org/packages/bc/96/e7b533ea5740641dd62b07a790af5d9d8fec36000b8e2d0472bd7574105f/numpy-2.3.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:2f4f0215edb189048a3c03bd5b19345bdfa7b45a7a6f72ae5945d2a28272727f", size = 14184660 }, + { url = "https://files.pythonhosted.org/packages/2b/53/102c6122db45a62aa20d1b18c9986f67e6b97e0d6fbc1ae13e3e4c84430c/numpy-2.3.2-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:8b1224a734cd509f70816455c3cffe13a4f599b1bf7130f913ba0e2c0b2006c0", size = 5113382 }, + { url = "https://files.pythonhosted.org/packages/2b/21/376257efcbf63e624250717e82b4fae93d60178f09eb03ed766dbb48ec9c/numpy-2.3.2-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:3dcf02866b977a38ba3ec10215220609ab9667378a9e2150615673f3ffd6c73b", size = 6647258 }, + { url = "https://files.pythonhosted.org/packages/91/ba/f4ebf257f08affa464fe6036e13f2bf9d4642a40228781dc1235da81be9f/numpy-2.3.2-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:572d5512df5470f50ada8d1972c5f1082d9a0b7aa5944db8084077570cf98370", size = 14281409 }, + { url = "https://files.pythonhosted.org/packages/59/ef/f96536f1df42c668cbacb727a8c6da7afc9c05ece6d558927fb1722693e1/numpy-2.3.2-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8145dd6d10df13c559d1e4314df29695613575183fa2e2d11fac4c208c8a1f73", size = 16641317 }, + { url = "https://files.pythonhosted.org/packages/f6/a7/af813a7b4f9a42f498dde8a4c6fcbff8100eed00182cc91dbaf095645f38/numpy-2.3.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:103ea7063fa624af04a791c39f97070bf93b96d7af7eb23530cd087dc8dbe9dc", size = 16056262 }, + { url = "https://files.pythonhosted.org/packages/8b/5d/41c4ef8404caaa7f05ed1cfb06afe16a25895260eacbd29b4d84dff2920b/numpy-2.3.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:fc927d7f289d14f5e037be917539620603294454130b6de200091e23d27dc9be", size = 18579342 }, + { url = "https://files.pythonhosted.org/packages/a1/4f/9950e44c5a11636f4a3af6e825ec23003475cc9a466edb7a759ed3ea63bd/numpy-2.3.2-cp312-cp312-win32.whl", hash = "sha256:d95f59afe7f808c103be692175008bab926b59309ade3e6d25009e9a171f7036", size = 6320610 }, + { url = "https://files.pythonhosted.org/packages/7c/2f/244643a5ce54a94f0a9a2ab578189c061e4a87c002e037b0829dd77293b6/numpy-2.3.2-cp312-cp312-win_amd64.whl", hash = "sha256:9e196ade2400c0c737d93465327d1ae7c06c7cb8a1756121ebf54b06ca183c7f", size = 12786292 }, + { url = "https://files.pythonhosted.org/packages/54/cd/7b5f49d5d78db7badab22d8323c1b6ae458fbf86c4fdfa194ab3cd4eb39b/numpy-2.3.2-cp312-cp312-win_arm64.whl", hash = "sha256:ee807923782faaf60d0d7331f5e86da7d5e3079e28b291973c545476c2b00d07", size = 10194071 }, + { url = "https://files.pythonhosted.org/packages/1c/c0/c6bb172c916b00700ed3bf71cb56175fd1f7dbecebf8353545d0b5519f6c/numpy-2.3.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:c8d9727f5316a256425892b043736d63e89ed15bbfe6556c5ff4d9d4448ff3b3", size = 20949074 }, + { url = "https://files.pythonhosted.org/packages/20/4e/c116466d22acaf4573e58421c956c6076dc526e24a6be0903219775d862e/numpy-2.3.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:efc81393f25f14d11c9d161e46e6ee348637c0a1e8a54bf9dedc472a3fae993b", size = 14177311 }, + { url = "https://files.pythonhosted.org/packages/78/45/d4698c182895af189c463fc91d70805d455a227261d950e4e0f1310c2550/numpy-2.3.2-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:dd937f088a2df683cbb79dda9a772b62a3e5a8a7e76690612c2737f38c6ef1b6", size = 5106022 }, + { url = "https://files.pythonhosted.org/packages/9f/76/3e6880fef4420179309dba72a8c11f6166c431cf6dee54c577af8906f914/numpy-2.3.2-cp313-cp313-macosx_14_0_x86_64.whl", hash = "sha256:11e58218c0c46c80509186e460d79fbdc9ca1eb8d8aee39d8f2dc768eb781089", size = 6640135 }, + { url = "https://files.pythonhosted.org/packages/34/fa/87ff7f25b3c4ce9085a62554460b7db686fef1e0207e8977795c7b7d7ba1/numpy-2.3.2-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5ad4ebcb683a1f99f4f392cc522ee20a18b2bb12a2c1c42c3d48d5a1adc9d3d2", size = 14278147 }, + { url = "https://files.pythonhosted.org/packages/1d/0f/571b2c7a3833ae419fe69ff7b479a78d313581785203cc70a8db90121b9a/numpy-2.3.2-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:938065908d1d869c7d75d8ec45f735a034771c6ea07088867f713d1cd3bbbe4f", size = 16635989 }, + { url = "https://files.pythonhosted.org/packages/24/5a/84ae8dca9c9a4c592fe11340b36a86ffa9fd3e40513198daf8a97839345c/numpy-2.3.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:66459dccc65d8ec98cc7df61307b64bf9e08101f9598755d42d8ae65d9a7a6ee", size = 16053052 }, + { url = "https://files.pythonhosted.org/packages/57/7c/e5725d99a9133b9813fcf148d3f858df98511686e853169dbaf63aec6097/numpy-2.3.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:a7af9ed2aa9ec5950daf05bb11abc4076a108bd3c7db9aa7251d5f107079b6a6", size = 18577955 }, + { url = "https://files.pythonhosted.org/packages/ae/11/7c546fcf42145f29b71e4d6f429e96d8d68e5a7ba1830b2e68d7418f0bbd/numpy-2.3.2-cp313-cp313-win32.whl", hash = "sha256:906a30249315f9c8e17b085cc5f87d3f369b35fedd0051d4a84686967bdbbd0b", size = 6311843 }, + { url = "https://files.pythonhosted.org/packages/aa/6f/a428fd1cb7ed39b4280d057720fed5121b0d7754fd2a9768640160f5517b/numpy-2.3.2-cp313-cp313-win_amd64.whl", hash = "sha256:c63d95dc9d67b676e9108fe0d2182987ccb0f11933c1e8959f42fa0da8d4fa56", size = 12782876 }, + { url = "https://files.pythonhosted.org/packages/65/85/4ea455c9040a12595fb6c43f2c217257c7b52dd0ba332c6a6c1d28b289fe/numpy-2.3.2-cp313-cp313-win_arm64.whl", hash = "sha256:b05a89f2fb84d21235f93de47129dd4f11c16f64c87c33f5e284e6a3a54e43f2", size = 10192786 }, + { url = "https://files.pythonhosted.org/packages/80/23/8278f40282d10c3f258ec3ff1b103d4994bcad78b0cba9208317f6bb73da/numpy-2.3.2-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:4e6ecfeddfa83b02318f4d84acf15fbdbf9ded18e46989a15a8b6995dfbf85ab", size = 21047395 }, + { url = "https://files.pythonhosted.org/packages/1f/2d/624f2ce4a5df52628b4ccd16a4f9437b37c35f4f8a50d00e962aae6efd7a/numpy-2.3.2-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:508b0eada3eded10a3b55725b40806a4b855961040180028f52580c4729916a2", size = 14300374 }, + { url = "https://files.pythonhosted.org/packages/f6/62/ff1e512cdbb829b80a6bd08318a58698867bca0ca2499d101b4af063ee97/numpy-2.3.2-cp313-cp313t-macosx_14_0_arm64.whl", hash = "sha256:754d6755d9a7588bdc6ac47dc4ee97867271b17cee39cb87aef079574366db0a", size = 5228864 }, + { url = "https://files.pythonhosted.org/packages/7d/8e/74bc18078fff03192d4032cfa99d5a5ca937807136d6f5790ce07ca53515/numpy-2.3.2-cp313-cp313t-macosx_14_0_x86_64.whl", hash = "sha256:a9f66e7d2b2d7712410d3bc5684149040ef5f19856f20277cd17ea83e5006286", size = 6737533 }, + { url = "https://files.pythonhosted.org/packages/19/ea/0731efe2c9073ccca5698ef6a8c3667c4cf4eea53fcdcd0b50140aba03bc/numpy-2.3.2-cp313-cp313t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:de6ea4e5a65d5a90c7d286ddff2b87f3f4ad61faa3db8dabe936b34c2275b6f8", size = 14352007 }, + { url = "https://files.pythonhosted.org/packages/cf/90/36be0865f16dfed20f4bc7f75235b963d5939707d4b591f086777412ff7b/numpy-2.3.2-cp313-cp313t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a3ef07ec8cbc8fc9e369c8dcd52019510c12da4de81367d8b20bc692aa07573a", size = 16701914 }, + { url = "https://files.pythonhosted.org/packages/94/30/06cd055e24cb6c38e5989a9e747042b4e723535758e6153f11afea88c01b/numpy-2.3.2-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:27c9f90e7481275c7800dc9c24b7cc40ace3fdb970ae4d21eaff983a32f70c91", size = 16132708 }, + { url = "https://files.pythonhosted.org/packages/9a/14/ecede608ea73e58267fd7cb78f42341b3b37ba576e778a1a06baffbe585c/numpy-2.3.2-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:07b62978075b67eee4065b166d000d457c82a1efe726cce608b9db9dd66a73a5", size = 18651678 }, + { url = "https://files.pythonhosted.org/packages/40/f3/2fe6066b8d07c3685509bc24d56386534c008b462a488b7f503ba82b8923/numpy-2.3.2-cp313-cp313t-win32.whl", hash = "sha256:c771cfac34a4f2c0de8e8c97312d07d64fd8f8ed45bc9f5726a7e947270152b5", size = 6441832 }, + { url = "https://files.pythonhosted.org/packages/0b/ba/0937d66d05204d8f28630c9c60bc3eda68824abde4cf756c4d6aad03b0c6/numpy-2.3.2-cp313-cp313t-win_amd64.whl", hash = "sha256:72dbebb2dcc8305c431b2836bcc66af967df91be793d63a24e3d9b741374c450", size = 12927049 }, + { url = "https://files.pythonhosted.org/packages/e9/ed/13542dd59c104d5e654dfa2ac282c199ba64846a74c2c4bcdbc3a0f75df1/numpy-2.3.2-cp313-cp313t-win_arm64.whl", hash = "sha256:72c6df2267e926a6d5286b0a6d556ebe49eae261062059317837fda12ddf0c1a", size = 10262935 }, + { url = "https://files.pythonhosted.org/packages/c9/7c/7659048aaf498f7611b783e000c7268fcc4dcf0ce21cd10aad7b2e8f9591/numpy-2.3.2-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:448a66d052d0cf14ce9865d159bfc403282c9bc7bb2a31b03cc18b651eca8b1a", size = 20950906 }, + { url = "https://files.pythonhosted.org/packages/80/db/984bea9d4ddf7112a04cfdfb22b1050af5757864cfffe8e09e44b7f11a10/numpy-2.3.2-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:546aaf78e81b4081b2eba1d105c3b34064783027a06b3ab20b6eba21fb64132b", size = 14185607 }, + { url = "https://files.pythonhosted.org/packages/e4/76/b3d6f414f4eca568f469ac112a3b510938d892bc5a6c190cb883af080b77/numpy-2.3.2-cp314-cp314-macosx_14_0_arm64.whl", hash = "sha256:87c930d52f45df092f7578889711a0768094debf73cfcde105e2d66954358125", size = 5114110 }, + { url = "https://files.pythonhosted.org/packages/9e/d2/6f5e6826abd6bca52392ed88fe44a4b52aacb60567ac3bc86c67834c3a56/numpy-2.3.2-cp314-cp314-macosx_14_0_x86_64.whl", hash = "sha256:8dc082ea901a62edb8f59713c6a7e28a85daddcb67454c839de57656478f5b19", size = 6642050 }, + { url = "https://files.pythonhosted.org/packages/c4/43/f12b2ade99199e39c73ad182f103f9d9791f48d885c600c8e05927865baf/numpy-2.3.2-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:af58de8745f7fa9ca1c0c7c943616c6fe28e75d0c81f5c295810e3c83b5be92f", size = 14296292 }, + { url = "https://files.pythonhosted.org/packages/5d/f9/77c07d94bf110a916b17210fac38680ed8734c236bfed9982fd8524a7b47/numpy-2.3.2-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:fed5527c4cf10f16c6d0b6bee1f89958bccb0ad2522c8cadc2efd318bcd545f5", size = 16638913 }, + { url = "https://files.pythonhosted.org/packages/9b/d1/9d9f2c8ea399cc05cfff8a7437453bd4e7d894373a93cdc46361bbb49a7d/numpy-2.3.2-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:095737ed986e00393ec18ec0b21b47c22889ae4b0cd2d5e88342e08b01141f58", size = 16071180 }, + { url = "https://files.pythonhosted.org/packages/4c/41/82e2c68aff2a0c9bf315e47d61951099fed65d8cb2c8d9dc388cb87e947e/numpy-2.3.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:b5e40e80299607f597e1a8a247ff8d71d79c5b52baa11cc1cce30aa92d2da6e0", size = 18576809 }, + { url = "https://files.pythonhosted.org/packages/14/14/4b4fd3efb0837ed252d0f583c5c35a75121038a8c4e065f2c259be06d2d8/numpy-2.3.2-cp314-cp314-win32.whl", hash = "sha256:7d6e390423cc1f76e1b8108c9b6889d20a7a1f59d9a60cac4a050fa734d6c1e2", size = 6366410 }, + { url = "https://files.pythonhosted.org/packages/11/9e/b4c24a6b8467b61aced5c8dc7dcfce23621baa2e17f661edb2444a418040/numpy-2.3.2-cp314-cp314-win_amd64.whl", hash = "sha256:b9d0878b21e3918d76d2209c924ebb272340da1fb51abc00f986c258cd5e957b", size = 12918821 }, + { url = "https://files.pythonhosted.org/packages/0e/0f/0dc44007c70b1007c1cef86b06986a3812dd7106d8f946c09cfa75782556/numpy-2.3.2-cp314-cp314-win_arm64.whl", hash = "sha256:2738534837c6a1d0c39340a190177d7d66fdf432894f469728da901f8f6dc910", size = 10477303 }, + { url = "https://files.pythonhosted.org/packages/8b/3e/075752b79140b78ddfc9c0a1634d234cfdbc6f9bbbfa6b7504e445ad7d19/numpy-2.3.2-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:4d002ecf7c9b53240be3bb69d80f86ddbd34078bae04d87be81c1f58466f264e", size = 21047524 }, + { url = "https://files.pythonhosted.org/packages/fe/6d/60e8247564a72426570d0e0ea1151b95ce5bd2f1597bb878a18d32aec855/numpy-2.3.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:293b2192c6bcce487dbc6326de5853787f870aeb6c43f8f9c6496db5b1781e45", size = 14300519 }, + { url = "https://files.pythonhosted.org/packages/4d/73/d8326c442cd428d47a067070c3ac6cc3b651a6e53613a1668342a12d4479/numpy-2.3.2-cp314-cp314t-macosx_14_0_arm64.whl", hash = "sha256:0a4f2021a6da53a0d580d6ef5db29947025ae8b35b3250141805ea9a32bbe86b", size = 5228972 }, + { url = "https://files.pythonhosted.org/packages/34/2e/e71b2d6dad075271e7079db776196829019b90ce3ece5c69639e4f6fdc44/numpy-2.3.2-cp314-cp314t-macosx_14_0_x86_64.whl", hash = "sha256:9c144440db4bf3bb6372d2c3e49834cc0ff7bb4c24975ab33e01199e645416f2", size = 6737439 }, + { url = "https://files.pythonhosted.org/packages/15/b0/d004bcd56c2c5e0500ffc65385eb6d569ffd3363cb5e593ae742749b2daa/numpy-2.3.2-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f92d6c2a8535dc4fe4419562294ff957f83a16ebdec66df0805e473ffaad8bd0", size = 14352479 }, + { url = "https://files.pythonhosted.org/packages/11/e3/285142fcff8721e0c99b51686426165059874c150ea9ab898e12a492e291/numpy-2.3.2-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:cefc2219baa48e468e3db7e706305fcd0c095534a192a08f31e98d83a7d45fb0", size = 16702805 }, + { url = "https://files.pythonhosted.org/packages/33/c3/33b56b0e47e604af2c7cd065edca892d180f5899599b76830652875249a3/numpy-2.3.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:76c3e9501ceb50b2ff3824c3589d5d1ab4ac857b0ee3f8f49629d0de55ecf7c2", size = 16133830 }, + { url = "https://files.pythonhosted.org/packages/6e/ae/7b1476a1f4d6a48bc669b8deb09939c56dd2a439db1ab03017844374fb67/numpy-2.3.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:122bf5ed9a0221b3419672493878ba4967121514b1d7d4656a7580cd11dddcbf", size = 18652665 }, + { url = "https://files.pythonhosted.org/packages/14/ba/5b5c9978c4bb161034148ade2de9db44ec316fab89ce8c400db0e0c81f86/numpy-2.3.2-cp314-cp314t-win32.whl", hash = "sha256:6f1ae3dcb840edccc45af496f312528c15b1f79ac318169d094e85e4bb35fdf1", size = 6514777 }, + { url = "https://files.pythonhosted.org/packages/eb/46/3dbaf0ae7c17cdc46b9f662c56da2054887b8d9e737c1476f335c83d33db/numpy-2.3.2-cp314-cp314t-win_amd64.whl", hash = "sha256:087ffc25890d89a43536f75c5fe8770922008758e8eeeef61733957041ed2f9b", size = 13111856 }, + { url = "https://files.pythonhosted.org/packages/c1/9e/1652778bce745a67b5fe05adde60ed362d38eb17d919a540e813d30f6874/numpy-2.3.2-cp314-cp314t-win_arm64.whl", hash = "sha256:092aeb3449833ea9c0bf0089d70c29ae480685dd2377ec9cdbbb620257f84631", size = 10544226 }, + { url = "https://files.pythonhosted.org/packages/cf/ea/50ebc91d28b275b23b7128ef25c3d08152bc4068f42742867e07a870a42a/numpy-2.3.2-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:14a91ebac98813a49bc6aa1a0dfc09513dcec1d97eaf31ca21a87221a1cdcb15", size = 21130338 }, + { url = "https://files.pythonhosted.org/packages/9f/57/cdd5eac00dd5f137277355c318a955c0d8fb8aa486020c22afd305f8b88f/numpy-2.3.2-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:71669b5daae692189540cffc4c439468d35a3f84f0c88b078ecd94337f6cb0ec", size = 14375776 }, + { url = "https://files.pythonhosted.org/packages/83/85/27280c7f34fcd305c2209c0cdca4d70775e4859a9eaa92f850087f8dea50/numpy-2.3.2-pp311-pypy311_pp73-macosx_14_0_arm64.whl", hash = "sha256:69779198d9caee6e547adb933941ed7520f896fd9656834c300bdf4dd8642712", size = 5304882 }, + { url = "https://files.pythonhosted.org/packages/48/b4/6500b24d278e15dd796f43824e69939d00981d37d9779e32499e823aa0aa/numpy-2.3.2-pp311-pypy311_pp73-macosx_14_0_x86_64.whl", hash = "sha256:2c3271cc4097beb5a60f010bcc1cc204b300bb3eafb4399376418a83a1c6373c", size = 6818405 }, + { url = "https://files.pythonhosted.org/packages/9b/c9/142c1e03f199d202da8e980c2496213509291b6024fd2735ad28ae7065c7/numpy-2.3.2-pp311-pypy311_pp73-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8446acd11fe3dc1830568c941d44449fd5cb83068e5c70bd5a470d323d448296", size = 14419651 }, + { url = "https://files.pythonhosted.org/packages/8b/95/8023e87cbea31a750a6c00ff9427d65ebc5fef104a136bfa69f76266d614/numpy-2.3.2-pp311-pypy311_pp73-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:aa098a5ab53fa407fded5870865c6275a5cd4101cfdef8d6fafc48286a96e981", size = 16760166 }, + { url = "https://files.pythonhosted.org/packages/78/e3/6690b3f85a05506733c7e90b577e4762517404ea78bab2ca3a5cb1aeb78d/numpy-2.3.2-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:6936aff90dda378c09bea075af0d9c675fe3a977a9d2402f95a87f440f59f619", size = 12977811 }, +] + +[[package]] +name = "nvidia-cublas-cu12" +version = "12.6.4.1" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/af/eb/ff4b8c503fa1f1796679dce648854d58751982426e4e4b37d6fce49d259c/nvidia_cublas_cu12-12.6.4.1-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:08ed2686e9875d01b58e3cb379c6896df8e76c75e0d4a7f7dace3d7b6d9ef8eb", size = 393138322 }, +] + +[[package]] +name = "nvidia-cuda-cupti-cu12" +version = "12.6.80" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/49/60/7b6497946d74bcf1de852a21824d63baad12cd417db4195fc1bfe59db953/nvidia_cuda_cupti_cu12-12.6.80-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:6768bad6cab4f19e8292125e5f1ac8aa7d1718704012a0e3272a6f61c4bce132", size = 8917980 }, + { url = "https://files.pythonhosted.org/packages/a5/24/120ee57b218d9952c379d1e026c4479c9ece9997a4fb46303611ee48f038/nvidia_cuda_cupti_cu12-12.6.80-py3-none-manylinux2014_x86_64.whl", hash = "sha256:a3eff6cdfcc6a4c35db968a06fcadb061cbc7d6dde548609a941ff8701b98b73", size = 8917972 }, +] + +[[package]] +name = "nvidia-cuda-nvrtc-cu12" +version = "12.6.77" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/75/2e/46030320b5a80661e88039f59060d1790298b4718944a65a7f2aeda3d9e9/nvidia_cuda_nvrtc_cu12-12.6.77-py3-none-manylinux2014_x86_64.whl", hash = "sha256:35b0cc6ee3a9636d5409133e79273ce1f3fd087abb0532d2d2e8fff1fe9efc53", size = 23650380 }, +] + +[[package]] +name = "nvidia-cuda-runtime-cu12" +version = "12.6.77" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e1/23/e717c5ac26d26cf39a27fbc076240fad2e3b817e5889d671b67f4f9f49c5/nvidia_cuda_runtime_cu12-12.6.77-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:ba3b56a4f896141e25e19ab287cd71e52a6a0f4b29d0d31609f60e3b4d5219b7", size = 897690 }, + { url = "https://files.pythonhosted.org/packages/f0/62/65c05e161eeddbafeca24dc461f47de550d9fa8a7e04eb213e32b55cfd99/nvidia_cuda_runtime_cu12-12.6.77-py3-none-manylinux2014_x86_64.whl", hash = "sha256:a84d15d5e1da416dd4774cb42edf5e954a3e60cc945698dc1d5be02321c44dc8", size = 897678 }, +] + +[[package]] +name = "nvidia-cudnn-cu12" +version = "9.5.1.17" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "nvidia-cublas-cu12" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/2a/78/4535c9c7f859a64781e43c969a3a7e84c54634e319a996d43ef32ce46f83/nvidia_cudnn_cu12-9.5.1.17-py3-none-manylinux_2_28_x86_64.whl", hash = "sha256:30ac3869f6db17d170e0e556dd6cc5eee02647abc31ca856634d5a40f82c15b2", size = 570988386 }, +] + +[[package]] +name = "nvidia-cufft-cu12" +version = "11.3.0.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "nvidia-nvjitlink-cu12" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/8f/16/73727675941ab8e6ffd86ca3a4b7b47065edcca7a997920b831f8147c99d/nvidia_cufft_cu12-11.3.0.4-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:ccba62eb9cef5559abd5e0d54ceed2d9934030f51163df018532142a8ec533e5", size = 200221632 }, + { url = "https://files.pythonhosted.org/packages/60/de/99ec247a07ea40c969d904fc14f3a356b3e2a704121675b75c366b694ee1/nvidia_cufft_cu12-11.3.0.4-py3-none-manylinux2014_x86_64.whl", hash = "sha256:768160ac89f6f7b459bee747e8d175dbf53619cfe74b2a5636264163138013ca", size = 200221622 }, +] + +[[package]] +name = "nvidia-cufile-cu12" +version = "1.11.1.6" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b2/66/cc9876340ac68ae71b15c743ddb13f8b30d5244af344ec8322b449e35426/nvidia_cufile_cu12-1.11.1.6-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:cc23469d1c7e52ce6c1d55253273d32c565dd22068647f3aa59b3c6b005bf159", size = 1142103 }, +] + +[[package]] +name = "nvidia-curand-cu12" +version = "10.3.7.77" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/73/1b/44a01c4e70933637c93e6e1a8063d1e998b50213a6b65ac5a9169c47e98e/nvidia_curand_cu12-10.3.7.77-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:a42cd1344297f70b9e39a1e4f467a4e1c10f1da54ff7a85c12197f6c652c8bdf", size = 56279010 }, + { url = "https://files.pythonhosted.org/packages/4a/aa/2c7ff0b5ee02eaef890c0ce7d4f74bc30901871c5e45dee1ae6d0083cd80/nvidia_curand_cu12-10.3.7.77-py3-none-manylinux2014_x86_64.whl", hash = "sha256:99f1a32f1ac2bd134897fc7a203f779303261268a65762a623bf30cc9fe79117", size = 56279000 }, +] + +[[package]] +name = "nvidia-cusolver-cu12" +version = "11.7.1.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "nvidia-cublas-cu12" }, + { name = "nvidia-cusparse-cu12" }, + { name = "nvidia-nvjitlink-cu12" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/f0/6e/c2cf12c9ff8b872e92b4a5740701e51ff17689c4d726fca91875b07f655d/nvidia_cusolver_cu12-11.7.1.2-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:e9e49843a7707e42022babb9bcfa33c29857a93b88020c4e4434656a655b698c", size = 158229790 }, + { url = "https://files.pythonhosted.org/packages/9f/81/baba53585da791d043c10084cf9553e074548408e04ae884cfe9193bd484/nvidia_cusolver_cu12-11.7.1.2-py3-none-manylinux2014_x86_64.whl", hash = "sha256:6cf28f17f64107a0c4d7802be5ff5537b2130bfc112f25d5a30df227058ca0e6", size = 158229780 }, +] + +[[package]] +name = "nvidia-cusparse-cu12" +version = "12.5.4.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "nvidia-nvjitlink-cu12" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/06/1e/b8b7c2f4099a37b96af5c9bb158632ea9e5d9d27d7391d7eb8fc45236674/nvidia_cusparse_cu12-12.5.4.2-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:7556d9eca156e18184b94947ade0fba5bb47d69cec46bf8660fd2c71a4b48b73", size = 216561367 }, + { url = "https://files.pythonhosted.org/packages/43/ac/64c4316ba163e8217a99680c7605f779accffc6a4bcd0c778c12948d3707/nvidia_cusparse_cu12-12.5.4.2-py3-none-manylinux2014_x86_64.whl", hash = "sha256:23749a6571191a215cb74d1cdbff4a86e7b19f1200c071b3fcf844a5bea23a2f", size = 216561357 }, +] + +[[package]] +name = "nvidia-cusparselt-cu12" +version = "0.6.3" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3b/9a/72ef35b399b0e183bc2e8f6f558036922d453c4d8237dab26c666a04244b/nvidia_cusparselt_cu12-0.6.3-py3-none-manylinux2014_x86_64.whl", hash = "sha256:e5c8a26c36445dd2e6812f1177978a24e2d37cacce7e090f297a688d1ec44f46", size = 156785796 }, +] + +[[package]] +name = "nvidia-nccl-cu12" +version = "2.26.2" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/67/ca/f42388aed0fddd64ade7493dbba36e1f534d4e6fdbdd355c6a90030ae028/nvidia_nccl_cu12-2.26.2-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:694cf3879a206553cc9d7dbda76b13efaf610fdb70a50cba303de1b0d1530ac6", size = 201319755 }, +] + +[[package]] +name = "nvidia-nvjitlink-cu12" +version = "12.6.85" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9d/d7/c5383e47c7e9bf1c99d5bd2a8c935af2b6d705ad831a7ec5c97db4d82f4f/nvidia_nvjitlink_cu12-12.6.85-py3-none-manylinux2010_x86_64.manylinux_2_12_x86_64.whl", hash = "sha256:eedc36df9e88b682efe4309aa16b5b4e78c2407eac59e8c10a6a47535164369a", size = 19744971 }, +] + +[[package]] +name = "nvidia-nvtx-cu12" +version = "12.6.77" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/56/9a/fff8376f8e3d084cd1530e1ef7b879bb7d6d265620c95c1b322725c694f4/nvidia_nvtx_cu12-12.6.77-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:b90bed3df379fa79afbd21be8e04a0314336b8ae16768b58f2d34cb1d04cd7d2", size = 89276 }, + { url = "https://files.pythonhosted.org/packages/9e/4e/0d0c945463719429b7bd21dece907ad0bde437a2ff12b9b12fee94722ab0/nvidia_nvtx_cu12-12.6.77-py3-none-manylinux2014_x86_64.whl", hash = "sha256:6574241a3ec5fdc9334353ab8c479fe75841dbe8f4532a8fc97ce63503330ba1", size = 89265 }, +] + +[[package]] +name = "packaging" +version = "25.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a1/d4/1fc4078c65507b51b96ca8f8c3ba19e6a61c8253c72794544580a7b6c24d/packaging-25.0.tar.gz", hash = "sha256:d443872c98d677bf60f6a1f2f8c1cb748e8fe762d2bf9d3148b5599295b0fc4f", size = 165727 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/20/12/38679034af332785aac8774540895e234f4d07f7545804097de4b666afd8/packaging-25.0-py3-none-any.whl", hash = "sha256:29572ef2b1f17581046b3a2227d5c611fb25ec70ca1ba8554b24b0e69331a484", size = 66469 }, +] + +[[package]] +name = "pathspec" +version = "0.12.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ca/bc/f35b8446f4531a7cb215605d100cd88b7ac6f44ab3fc94870c120ab3adbf/pathspec-0.12.1.tar.gz", hash = "sha256:a482d51503a1ab33b1c67a6c3813a26953dbdc71c31dacaef9a838c4e29f5712", size = 51043 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cc/20/ff623b09d963f88bfde16306a54e12ee5ea43e9b597108672ff3a408aad6/pathspec-0.12.1-py3-none-any.whl", hash = "sha256:a0d503e138a4c123b27490a4f7beda6a01c6f288df0e4a8b79c7eb0dc7b4cc08", size = 31191 }, +] + +[[package]] +name = "pillow" +version = "11.3.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f3/0d/d0d6dea55cd152ce3d6767bb38a8fc10e33796ba4ba210cbab9354b6d238/pillow-11.3.0.tar.gz", hash = "sha256:3828ee7586cd0b2091b6209e5ad53e20d0649bbe87164a459d0676e035e8f523", size = 47113069 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/4c/5d/45a3553a253ac8763f3561371432a90bdbe6000fbdcf1397ffe502aa206c/pillow-11.3.0-cp310-cp310-macosx_10_10_x86_64.whl", hash = "sha256:1b9c17fd4ace828b3003dfd1e30bff24863e0eb59b535e8f80194d9cc7ecf860", size = 5316554 }, + { url = "https://files.pythonhosted.org/packages/7c/c8/67c12ab069ef586a25a4a79ced553586748fad100c77c0ce59bb4983ac98/pillow-11.3.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:65dc69160114cdd0ca0f35cb434633c75e8e7fad4cf855177a05bf38678f73ad", size = 4686548 }, + { url = "https://files.pythonhosted.org/packages/2f/bd/6741ebd56263390b382ae4c5de02979af7f8bd9807346d068700dd6d5cf9/pillow-11.3.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:7107195ddc914f656c7fc8e4a5e1c25f32e9236ea3ea860f257b0436011fddd0", size = 5859742 }, + { url = "https://files.pythonhosted.org/packages/ca/0b/c412a9e27e1e6a829e6ab6c2dca52dd563efbedf4c9c6aa453d9a9b77359/pillow-11.3.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:cc3e831b563b3114baac7ec2ee86819eb03caa1a2cef0b481a5675b59c4fe23b", size = 7633087 }, + { url = "https://files.pythonhosted.org/packages/59/9d/9b7076aaf30f5dd17e5e5589b2d2f5a5d7e30ff67a171eb686e4eecc2adf/pillow-11.3.0-cp310-cp310-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f1f182ebd2303acf8c380a54f615ec883322593320a9b00438eb842c1f37ae50", size = 5963350 }, + { url = "https://files.pythonhosted.org/packages/f0/16/1a6bf01fb622fb9cf5c91683823f073f053005c849b1f52ed613afcf8dae/pillow-11.3.0-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4445fa62e15936a028672fd48c4c11a66d641d2c05726c7ec1f8ba6a572036ae", size = 6631840 }, + { url = "https://files.pythonhosted.org/packages/7b/e6/6ff7077077eb47fde78739e7d570bdcd7c10495666b6afcd23ab56b19a43/pillow-11.3.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:71f511f6b3b91dd543282477be45a033e4845a40278fa8dcdbfdb07109bf18f9", size = 6074005 }, + { url = "https://files.pythonhosted.org/packages/c3/3a/b13f36832ea6d279a697231658199e0a03cd87ef12048016bdcc84131601/pillow-11.3.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:040a5b691b0713e1f6cbe222e0f4f74cd233421e105850ae3b3c0ceda520f42e", size = 6708372 }, + { url = "https://files.pythonhosted.org/packages/6c/e4/61b2e1a7528740efbc70b3d581f33937e38e98ef3d50b05007267a55bcb2/pillow-11.3.0-cp310-cp310-win32.whl", hash = "sha256:89bd777bc6624fe4115e9fac3352c79ed60f3bb18651420635f26e643e3dd1f6", size = 6277090 }, + { url = "https://files.pythonhosted.org/packages/a9/d3/60c781c83a785d6afbd6a326ed4d759d141de43aa7365725cbcd65ce5e54/pillow-11.3.0-cp310-cp310-win_amd64.whl", hash = "sha256:19d2ff547c75b8e3ff46f4d9ef969a06c30ab2d4263a9e287733aa8b2429ce8f", size = 6985988 }, + { url = "https://files.pythonhosted.org/packages/9f/28/4f4a0203165eefb3763939c6789ba31013a2e90adffb456610f30f613850/pillow-11.3.0-cp310-cp310-win_arm64.whl", hash = "sha256:819931d25e57b513242859ce1876c58c59dc31587847bf74cfe06b2e0cb22d2f", size = 2422899 }, + { url = "https://files.pythonhosted.org/packages/db/26/77f8ed17ca4ffd60e1dcd220a6ec6d71210ba398cfa33a13a1cd614c5613/pillow-11.3.0-cp311-cp311-macosx_10_10_x86_64.whl", hash = "sha256:1cd110edf822773368b396281a2293aeb91c90a2db00d78ea43e7e861631b722", size = 5316531 }, + { url = "https://files.pythonhosted.org/packages/cb/39/ee475903197ce709322a17a866892efb560f57900d9af2e55f86db51b0a5/pillow-11.3.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:9c412fddd1b77a75aa904615ebaa6001f169b26fd467b4be93aded278266b288", size = 4686560 }, + { url = "https://files.pythonhosted.org/packages/d5/90/442068a160fd179938ba55ec8c97050a612426fae5ec0a764e345839f76d/pillow-11.3.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:7d1aa4de119a0ecac0a34a9c8bde33f34022e2e8f99104e47a3ca392fd60e37d", size = 5870978 }, + { url = "https://files.pythonhosted.org/packages/13/92/dcdd147ab02daf405387f0218dcf792dc6dd5b14d2573d40b4caeef01059/pillow-11.3.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:91da1d88226663594e3f6b4b8c3c8d85bd504117d043740a8e0ec449087cc494", size = 7641168 }, + { url = "https://files.pythonhosted.org/packages/6e/db/839d6ba7fd38b51af641aa904e2960e7a5644d60ec754c046b7d2aee00e5/pillow-11.3.0-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:643f189248837533073c405ec2f0bb250ba54598cf80e8c1e043381a60632f58", size = 5973053 }, + { url = "https://files.pythonhosted.org/packages/f2/2f/d7675ecae6c43e9f12aa8d58b6012683b20b6edfbdac7abcb4e6af7a3784/pillow-11.3.0-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:106064daa23a745510dabce1d84f29137a37224831d88eb4ce94bb187b1d7e5f", size = 6640273 }, + { url = "https://files.pythonhosted.org/packages/45/ad/931694675ede172e15b2ff03c8144a0ddaea1d87adb72bb07655eaffb654/pillow-11.3.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:cd8ff254faf15591e724dc7c4ddb6bf4793efcbe13802a4ae3e863cd300b493e", size = 6082043 }, + { url = "https://files.pythonhosted.org/packages/3a/04/ba8f2b11fc80d2dd462d7abec16351b45ec99cbbaea4387648a44190351a/pillow-11.3.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:932c754c2d51ad2b2271fd01c3d121daaa35e27efae2a616f77bf164bc0b3e94", size = 6715516 }, + { url = "https://files.pythonhosted.org/packages/48/59/8cd06d7f3944cc7d892e8533c56b0acb68399f640786313275faec1e3b6f/pillow-11.3.0-cp311-cp311-win32.whl", hash = "sha256:b4b8f3efc8d530a1544e5962bd6b403d5f7fe8b9e08227c6b255f98ad82b4ba0", size = 6274768 }, + { url = "https://files.pythonhosted.org/packages/f1/cc/29c0f5d64ab8eae20f3232da8f8571660aa0ab4b8f1331da5c2f5f9a938e/pillow-11.3.0-cp311-cp311-win_amd64.whl", hash = "sha256:1a992e86b0dd7aeb1f053cd506508c0999d710a8f07b4c791c63843fc6a807ac", size = 6986055 }, + { url = "https://files.pythonhosted.org/packages/c6/df/90bd886fabd544c25addd63e5ca6932c86f2b701d5da6c7839387a076b4a/pillow-11.3.0-cp311-cp311-win_arm64.whl", hash = "sha256:30807c931ff7c095620fe04448e2c2fc673fcbb1ffe2a7da3fb39613489b1ddd", size = 2423079 }, + { url = "https://files.pythonhosted.org/packages/40/fe/1bc9b3ee13f68487a99ac9529968035cca2f0a51ec36892060edcc51d06a/pillow-11.3.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:fdae223722da47b024b867c1ea0be64e0df702c5e0a60e27daad39bf960dd1e4", size = 5278800 }, + { url = "https://files.pythonhosted.org/packages/2c/32/7e2ac19b5713657384cec55f89065fb306b06af008cfd87e572035b27119/pillow-11.3.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:921bd305b10e82b4d1f5e802b6850677f965d8394203d182f078873851dada69", size = 4686296 }, + { url = "https://files.pythonhosted.org/packages/8e/1e/b9e12bbe6e4c2220effebc09ea0923a07a6da1e1f1bfbc8d7d29a01ce32b/pillow-11.3.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:eb76541cba2f958032d79d143b98a3a6b3ea87f0959bbe256c0b5e416599fd5d", size = 5871726 }, + { url = "https://files.pythonhosted.org/packages/8d/33/e9200d2bd7ba00dc3ddb78df1198a6e80d7669cce6c2bdbeb2530a74ec58/pillow-11.3.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:67172f2944ebba3d4a7b54f2e95c786a3a50c21b88456329314caaa28cda70f6", size = 7644652 }, + { url = "https://files.pythonhosted.org/packages/41/f1/6f2427a26fc683e00d985bc391bdd76d8dd4e92fac33d841127eb8fb2313/pillow-11.3.0-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:97f07ed9f56a3b9b5f49d3661dc9607484e85c67e27f3e8be2c7d28ca032fec7", size = 5977787 }, + { url = "https://files.pythonhosted.org/packages/e4/c9/06dd4a38974e24f932ff5f98ea3c546ce3f8c995d3f0985f8e5ba48bba19/pillow-11.3.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:676b2815362456b5b3216b4fd5bd89d362100dc6f4945154ff172e206a22c024", size = 6645236 }, + { url = "https://files.pythonhosted.org/packages/40/e7/848f69fb79843b3d91241bad658e9c14f39a32f71a301bcd1d139416d1be/pillow-11.3.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3e184b2f26ff146363dd07bde8b711833d7b0202e27d13540bfe2e35a323a809", size = 6086950 }, + { url = "https://files.pythonhosted.org/packages/0b/1a/7cff92e695a2a29ac1958c2a0fe4c0b2393b60aac13b04a4fe2735cad52d/pillow-11.3.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:6be31e3fc9a621e071bc17bb7de63b85cbe0bfae91bb0363c893cbe67247780d", size = 6723358 }, + { url = "https://files.pythonhosted.org/packages/26/7d/73699ad77895f69edff76b0f332acc3d497f22f5d75e5360f78cbcaff248/pillow-11.3.0-cp312-cp312-win32.whl", hash = "sha256:7b161756381f0918e05e7cb8a371fff367e807770f8fe92ecb20d905d0e1c149", size = 6275079 }, + { url = "https://files.pythonhosted.org/packages/8c/ce/e7dfc873bdd9828f3b6e5c2bbb74e47a98ec23cc5c74fc4e54462f0d9204/pillow-11.3.0-cp312-cp312-win_amd64.whl", hash = "sha256:a6444696fce635783440b7f7a9fc24b3ad10a9ea3f0ab66c5905be1c19ccf17d", size = 6986324 }, + { url = "https://files.pythonhosted.org/packages/16/8f/b13447d1bf0b1f7467ce7d86f6e6edf66c0ad7cf44cf5c87a37f9bed9936/pillow-11.3.0-cp312-cp312-win_arm64.whl", hash = "sha256:2aceea54f957dd4448264f9bf40875da0415c83eb85f55069d89c0ed436e3542", size = 2423067 }, + { url = "https://files.pythonhosted.org/packages/1e/93/0952f2ed8db3a5a4c7a11f91965d6184ebc8cd7cbb7941a260d5f018cd2d/pillow-11.3.0-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:1c627742b539bba4309df89171356fcb3cc5a9178355b2727d1b74a6cf155fbd", size = 2128328 }, + { url = "https://files.pythonhosted.org/packages/4b/e8/100c3d114b1a0bf4042f27e0f87d2f25e857e838034e98ca98fe7b8c0a9c/pillow-11.3.0-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:30b7c02f3899d10f13d7a48163c8969e4e653f8b43416d23d13d1bbfdc93b9f8", size = 2170652 }, + { url = "https://files.pythonhosted.org/packages/aa/86/3f758a28a6e381758545f7cdb4942e1cb79abd271bea932998fc0db93cb6/pillow-11.3.0-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:7859a4cc7c9295f5838015d8cc0a9c215b77e43d07a25e460f35cf516df8626f", size = 2227443 }, + { url = "https://files.pythonhosted.org/packages/01/f4/91d5b3ffa718df2f53b0dc109877993e511f4fd055d7e9508682e8aba092/pillow-11.3.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:ec1ee50470b0d050984394423d96325b744d55c701a439d2bd66089bff963d3c", size = 5278474 }, + { url = "https://files.pythonhosted.org/packages/f9/0e/37d7d3eca6c879fbd9dba21268427dffda1ab00d4eb05b32923d4fbe3b12/pillow-11.3.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:7db51d222548ccfd274e4572fdbf3e810a5e66b00608862f947b163e613b67dd", size = 4686038 }, + { url = "https://files.pythonhosted.org/packages/ff/b0/3426e5c7f6565e752d81221af9d3676fdbb4f352317ceafd42899aaf5d8a/pillow-11.3.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:2d6fcc902a24ac74495df63faad1884282239265c6839a0a6416d33faedfae7e", size = 5864407 }, + { url = "https://files.pythonhosted.org/packages/fc/c1/c6c423134229f2a221ee53f838d4be9d82bab86f7e2f8e75e47b6bf6cd77/pillow-11.3.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:f0f5d8f4a08090c6d6d578351a2b91acf519a54986c055af27e7a93feae6d3f1", size = 7639094 }, + { url = "https://files.pythonhosted.org/packages/ba/c9/09e6746630fe6372c67c648ff9deae52a2bc20897d51fa293571977ceb5d/pillow-11.3.0-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c37d8ba9411d6003bba9e518db0db0c58a680ab9fe5179f040b0463644bc9805", size = 5973503 }, + { url = "https://files.pythonhosted.org/packages/d5/1c/a2a29649c0b1983d3ef57ee87a66487fdeb45132df66ab30dd37f7dbe162/pillow-11.3.0-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:13f87d581e71d9189ab21fe0efb5a23e9f28552d5be6979e84001d3b8505abe8", size = 6642574 }, + { url = "https://files.pythonhosted.org/packages/36/de/d5cc31cc4b055b6c6fd990e3e7f0f8aaf36229a2698501bcb0cdf67c7146/pillow-11.3.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:023f6d2d11784a465f09fd09a34b150ea4672e85fb3d05931d89f373ab14abb2", size = 6084060 }, + { url = "https://files.pythonhosted.org/packages/d5/ea/502d938cbaeec836ac28a9b730193716f0114c41325db428e6b280513f09/pillow-11.3.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:45dfc51ac5975b938e9809451c51734124e73b04d0f0ac621649821a63852e7b", size = 6721407 }, + { url = "https://files.pythonhosted.org/packages/45/9c/9c5e2a73f125f6cbc59cc7087c8f2d649a7ae453f83bd0362ff7c9e2aee2/pillow-11.3.0-cp313-cp313-win32.whl", hash = "sha256:a4d336baed65d50d37b88ca5b60c0fa9d81e3a87d4a7930d3880d1624d5b31f3", size = 6273841 }, + { url = "https://files.pythonhosted.org/packages/23/85/397c73524e0cd212067e0c969aa245b01d50183439550d24d9f55781b776/pillow-11.3.0-cp313-cp313-win_amd64.whl", hash = "sha256:0bce5c4fd0921f99d2e858dc4d4d64193407e1b99478bc5cacecba2311abde51", size = 6978450 }, + { url = "https://files.pythonhosted.org/packages/17/d2/622f4547f69cd173955194b78e4d19ca4935a1b0f03a302d655c9f6aae65/pillow-11.3.0-cp313-cp313-win_arm64.whl", hash = "sha256:1904e1264881f682f02b7f8167935cce37bc97db457f8e7849dc3a6a52b99580", size = 2423055 }, + { url = "https://files.pythonhosted.org/packages/dd/80/a8a2ac21dda2e82480852978416cfacd439a4b490a501a288ecf4fe2532d/pillow-11.3.0-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:4c834a3921375c48ee6b9624061076bc0a32a60b5532b322cc0ea64e639dd50e", size = 5281110 }, + { url = "https://files.pythonhosted.org/packages/44/d6/b79754ca790f315918732e18f82a8146d33bcd7f4494380457ea89eb883d/pillow-11.3.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:5e05688ccef30ea69b9317a9ead994b93975104a677a36a8ed8106be9260aa6d", size = 4689547 }, + { url = "https://files.pythonhosted.org/packages/49/20/716b8717d331150cb00f7fdd78169c01e8e0c219732a78b0e59b6bdb2fd6/pillow-11.3.0-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:1019b04af07fc0163e2810167918cb5add8d74674b6267616021ab558dc98ced", size = 5901554 }, + { url = "https://files.pythonhosted.org/packages/74/cf/a9f3a2514a65bb071075063a96f0a5cf949c2f2fce683c15ccc83b1c1cab/pillow-11.3.0-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:f944255db153ebb2b19c51fe85dd99ef0ce494123f21b9db4877ffdfc5590c7c", size = 7669132 }, + { url = "https://files.pythonhosted.org/packages/98/3c/da78805cbdbee9cb43efe8261dd7cc0b4b93f2ac79b676c03159e9db2187/pillow-11.3.0-cp313-cp313t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1f85acb69adf2aaee8b7da124efebbdb959a104db34d3a2cb0f3793dbae422a8", size = 6005001 }, + { url = "https://files.pythonhosted.org/packages/6c/fa/ce044b91faecf30e635321351bba32bab5a7e034c60187fe9698191aef4f/pillow-11.3.0-cp313-cp313t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:05f6ecbeff5005399bb48d198f098a9b4b6bdf27b8487c7f38ca16eeb070cd59", size = 6668814 }, + { url = "https://files.pythonhosted.org/packages/7b/51/90f9291406d09bf93686434f9183aba27b831c10c87746ff49f127ee80cb/pillow-11.3.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:a7bc6e6fd0395bc052f16b1a8670859964dbd7003bd0af2ff08342eb6e442cfe", size = 6113124 }, + { url = "https://files.pythonhosted.org/packages/cd/5a/6fec59b1dfb619234f7636d4157d11fb4e196caeee220232a8d2ec48488d/pillow-11.3.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:83e1b0161c9d148125083a35c1c5a89db5b7054834fd4387499e06552035236c", size = 6747186 }, + { url = "https://files.pythonhosted.org/packages/49/6b/00187a044f98255225f172de653941e61da37104a9ea60e4f6887717e2b5/pillow-11.3.0-cp313-cp313t-win32.whl", hash = "sha256:2a3117c06b8fb646639dce83694f2f9eac405472713fcb1ae887469c0d4f6788", size = 6277546 }, + { url = "https://files.pythonhosted.org/packages/e8/5c/6caaba7e261c0d75bab23be79f1d06b5ad2a2ae49f028ccec801b0e853d6/pillow-11.3.0-cp313-cp313t-win_amd64.whl", hash = "sha256:857844335c95bea93fb39e0fa2726b4d9d758850b34075a7e3ff4f4fa3aa3b31", size = 6985102 }, + { url = "https://files.pythonhosted.org/packages/f3/7e/b623008460c09a0cb38263c93b828c666493caee2eb34ff67f778b87e58c/pillow-11.3.0-cp313-cp313t-win_arm64.whl", hash = "sha256:8797edc41f3e8536ae4b10897ee2f637235c94f27404cac7297f7b607dd0716e", size = 2424803 }, + { url = "https://files.pythonhosted.org/packages/73/f4/04905af42837292ed86cb1b1dabe03dce1edc008ef14c473c5c7e1443c5d/pillow-11.3.0-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:d9da3df5f9ea2a89b81bb6087177fb1f4d1c7146d583a3fe5c672c0d94e55e12", size = 5278520 }, + { url = "https://files.pythonhosted.org/packages/41/b0/33d79e377a336247df6348a54e6d2a2b85d644ca202555e3faa0cf811ecc/pillow-11.3.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:0b275ff9b04df7b640c59ec5a3cb113eefd3795a8df80bac69646ef699c6981a", size = 4686116 }, + { url = "https://files.pythonhosted.org/packages/49/2d/ed8bc0ab219ae8768f529597d9509d184fe8a6c4741a6864fea334d25f3f/pillow-11.3.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:0743841cabd3dba6a83f38a92672cccbd69af56e3e91777b0ee7f4dba4385632", size = 5864597 }, + { url = "https://files.pythonhosted.org/packages/b5/3d/b932bb4225c80b58dfadaca9d42d08d0b7064d2d1791b6a237f87f661834/pillow-11.3.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:2465a69cf967b8b49ee1b96d76718cd98c4e925414ead59fdf75cf0fd07df673", size = 7638246 }, + { url = "https://files.pythonhosted.org/packages/09/b5/0487044b7c096f1b48f0d7ad416472c02e0e4bf6919541b111efd3cae690/pillow-11.3.0-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:41742638139424703b4d01665b807c6468e23e699e8e90cffefe291c5832b027", size = 5973336 }, + { url = "https://files.pythonhosted.org/packages/a8/2d/524f9318f6cbfcc79fbc004801ea6b607ec3f843977652fdee4857a7568b/pillow-11.3.0-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:93efb0b4de7e340d99057415c749175e24c8864302369e05914682ba642e5d77", size = 6642699 }, + { url = "https://files.pythonhosted.org/packages/6f/d2/a9a4f280c6aefedce1e8f615baaa5474e0701d86dd6f1dede66726462bbd/pillow-11.3.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7966e38dcd0fa11ca390aed7c6f20454443581d758242023cf36fcb319b1a874", size = 6083789 }, + { url = "https://files.pythonhosted.org/packages/fe/54/86b0cd9dbb683a9d5e960b66c7379e821a19be4ac5810e2e5a715c09a0c0/pillow-11.3.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:98a9afa7b9007c67ed84c57c9e0ad86a6000da96eaa638e4f8abe5b65ff83f0a", size = 6720386 }, + { url = "https://files.pythonhosted.org/packages/e7/95/88efcaf384c3588e24259c4203b909cbe3e3c2d887af9e938c2022c9dd48/pillow-11.3.0-cp314-cp314-win32.whl", hash = "sha256:02a723e6bf909e7cea0dac1b0e0310be9d7650cd66222a5f1c571455c0a45214", size = 6370911 }, + { url = "https://files.pythonhosted.org/packages/2e/cc/934e5820850ec5eb107e7b1a72dd278140731c669f396110ebc326f2a503/pillow-11.3.0-cp314-cp314-win_amd64.whl", hash = "sha256:a418486160228f64dd9e9efcd132679b7a02a5f22c982c78b6fc7dab3fefb635", size = 7117383 }, + { url = "https://files.pythonhosted.org/packages/d6/e9/9c0a616a71da2a5d163aa37405e8aced9a906d574b4a214bede134e731bc/pillow-11.3.0-cp314-cp314-win_arm64.whl", hash = "sha256:155658efb5e044669c08896c0c44231c5e9abcaadbc5cd3648df2f7c0b96b9a6", size = 2511385 }, + { url = "https://files.pythonhosted.org/packages/1a/33/c88376898aff369658b225262cd4f2659b13e8178e7534df9e6e1fa289f6/pillow-11.3.0-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:59a03cdf019efbfeeed910bf79c7c93255c3d54bc45898ac2a4140071b02b4ae", size = 5281129 }, + { url = "https://files.pythonhosted.org/packages/1f/70/d376247fb36f1844b42910911c83a02d5544ebd2a8bad9efcc0f707ea774/pillow-11.3.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:f8a5827f84d973d8636e9dc5764af4f0cf2318d26744b3d902931701b0d46653", size = 4689580 }, + { url = "https://files.pythonhosted.org/packages/eb/1c/537e930496149fbac69efd2fc4329035bbe2e5475b4165439e3be9cb183b/pillow-11.3.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:ee92f2fd10f4adc4b43d07ec5e779932b4eb3dbfbc34790ada5a6669bc095aa6", size = 5902860 }, + { url = "https://files.pythonhosted.org/packages/bd/57/80f53264954dcefeebcf9dae6e3eb1daea1b488f0be8b8fef12f79a3eb10/pillow-11.3.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:c96d333dcf42d01f47b37e0979b6bd73ec91eae18614864622d9b87bbd5bbf36", size = 7670694 }, + { url = "https://files.pythonhosted.org/packages/70/ff/4727d3b71a8578b4587d9c276e90efad2d6fe0335fd76742a6da08132e8c/pillow-11.3.0-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4c96f993ab8c98460cd0c001447bff6194403e8b1d7e149ade5f00594918128b", size = 6005888 }, + { url = "https://files.pythonhosted.org/packages/05/ae/716592277934f85d3be51d7256f3636672d7b1abfafdc42cf3f8cbd4b4c8/pillow-11.3.0-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:41342b64afeba938edb034d122b2dda5db2139b9a4af999729ba8818e0056477", size = 6670330 }, + { url = "https://files.pythonhosted.org/packages/e7/bb/7fe6cddcc8827b01b1a9766f5fdeb7418680744f9082035bdbabecf1d57f/pillow-11.3.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:068d9c39a2d1b358eb9f245ce7ab1b5c3246c7c8c7d9ba58cfa5b43146c06e50", size = 6114089 }, + { url = "https://files.pythonhosted.org/packages/8b/f5/06bfaa444c8e80f1a8e4bff98da9c83b37b5be3b1deaa43d27a0db37ef84/pillow-11.3.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:a1bc6ba083b145187f648b667e05a2534ecc4b9f2784c2cbe3089e44868f2b9b", size = 6748206 }, + { url = "https://files.pythonhosted.org/packages/f0/77/bc6f92a3e8e6e46c0ca78abfffec0037845800ea38c73483760362804c41/pillow-11.3.0-cp314-cp314t-win32.whl", hash = "sha256:118ca10c0d60b06d006be10a501fd6bbdfef559251ed31b794668ed569c87e12", size = 6377370 }, + { url = "https://files.pythonhosted.org/packages/4a/82/3a721f7d69dca802befb8af08b7c79ebcab461007ce1c18bd91a5d5896f9/pillow-11.3.0-cp314-cp314t-win_amd64.whl", hash = "sha256:8924748b688aa210d79883357d102cd64690e56b923a186f35a82cbc10f997db", size = 7121500 }, + { url = "https://files.pythonhosted.org/packages/89/c7/5572fa4a3f45740eaab6ae86fcdf7195b55beac1371ac8c619d880cfe948/pillow-11.3.0-cp314-cp314t-win_arm64.whl", hash = "sha256:79ea0d14d3ebad43ec77ad5272e6ff9bba5b679ef73375ea760261207fa8e0aa", size = 2512835 }, + { url = "https://files.pythonhosted.org/packages/6f/8b/209bd6b62ce8367f47e68a218bffac88888fdf2c9fcf1ecadc6c3ec1ebc7/pillow-11.3.0-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:3cee80663f29e3843b68199b9d6f4f54bd1d4a6b59bdd91bceefc51238bcb967", size = 5270556 }, + { url = "https://files.pythonhosted.org/packages/2e/e6/231a0b76070c2cfd9e260a7a5b504fb72da0a95279410fa7afd99d9751d6/pillow-11.3.0-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:b5f56c3f344f2ccaf0dd875d3e180f631dc60a51b314295a3e681fe8cf851fbe", size = 4654625 }, + { url = "https://files.pythonhosted.org/packages/13/f4/10cf94fda33cb12765f2397fc285fa6d8eb9c29de7f3185165b702fc7386/pillow-11.3.0-pp310-pypy310_pp73-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:e67d793d180c9df62f1f40aee3accca4829d3794c95098887edc18af4b8b780c", size = 4874207 }, + { url = "https://files.pythonhosted.org/packages/72/c9/583821097dc691880c92892e8e2d41fe0a5a3d6021f4963371d2f6d57250/pillow-11.3.0-pp310-pypy310_pp73-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:d000f46e2917c705e9fb93a3606ee4a819d1e3aa7a9b442f6444f07e77cf5e25", size = 6583939 }, + { url = "https://files.pythonhosted.org/packages/3b/8e/5c9d410f9217b12320efc7c413e72693f48468979a013ad17fd690397b9a/pillow-11.3.0-pp310-pypy310_pp73-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:527b37216b6ac3a12d7838dc3bd75208ec57c1c6d11ef01902266a5a0c14fc27", size = 4957166 }, + { url = "https://files.pythonhosted.org/packages/62/bb/78347dbe13219991877ffb3a91bf09da8317fbfcd4b5f9140aeae020ad71/pillow-11.3.0-pp310-pypy310_pp73-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:be5463ac478b623b9dd3937afd7fb7ab3d79dd290a28e2b6df292dc75063eb8a", size = 5581482 }, + { url = "https://files.pythonhosted.org/packages/d9/28/1000353d5e61498aaeaaf7f1e4b49ddb05f2c6575f9d4f9f914a3538b6e1/pillow-11.3.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:8dc70ca24c110503e16918a658b869019126ecfe03109b754c402daff12b3d9f", size = 6984596 }, + { url = "https://files.pythonhosted.org/packages/9e/e3/6fa84033758276fb31da12e5fb66ad747ae83b93c67af17f8c6ff4cc8f34/pillow-11.3.0-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:7c8ec7a017ad1bd562f93dbd8505763e688d388cde6e4a010ae1486916e713e6", size = 5270566 }, + { url = "https://files.pythonhosted.org/packages/5b/ee/e8d2e1ab4892970b561e1ba96cbd59c0d28cf66737fc44abb2aec3795a4e/pillow-11.3.0-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:9ab6ae226de48019caa8074894544af5b53a117ccb9d3b3dcb2871464c829438", size = 4654618 }, + { url = "https://files.pythonhosted.org/packages/f2/6d/17f80f4e1f0761f02160fc433abd4109fa1548dcfdca46cfdadaf9efa565/pillow-11.3.0-pp311-pypy311_pp73-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:fe27fb049cdcca11f11a7bfda64043c37b30e6b91f10cb5bab275806c32f6ab3", size = 4874248 }, + { url = "https://files.pythonhosted.org/packages/de/5f/c22340acd61cef960130585bbe2120e2fd8434c214802f07e8c03596b17e/pillow-11.3.0-pp311-pypy311_pp73-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:465b9e8844e3c3519a983d58b80be3f668e2a7a5db97f2784e7079fbc9f9822c", size = 6583963 }, + { url = "https://files.pythonhosted.org/packages/31/5e/03966aedfbfcbb4d5f8aa042452d3361f325b963ebbadddac05b122e47dd/pillow-11.3.0-pp311-pypy311_pp73-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5418b53c0d59b3824d05e029669efa023bbef0f3e92e75ec8428f3799487f361", size = 4957170 }, + { url = "https://files.pythonhosted.org/packages/cc/2d/e082982aacc927fc2cab48e1e731bdb1643a1406acace8bed0900a61464e/pillow-11.3.0-pp311-pypy311_pp73-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:504b6f59505f08ae014f724b6207ff6222662aab5cc9542577fb084ed0676ac7", size = 5581505 }, + { url = "https://files.pythonhosted.org/packages/34/e7/ae39f538fd6844e982063c3a5e4598b8ced43b9633baa3a85ef33af8c05c/pillow-11.3.0-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:c84d689db21a1c397d001aa08241044aa2069e7587b398c8cc63020390b1c1b8", size = 6984598 }, +] + +[[package]] +name = "pltpublish" +version = "1.0.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "matplotlib" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b9/c9/d720aa05436f1fed003a1966a2d7d772c313952a9efa84a604d79dc604d9/pltpublish-1.0.3.tar.gz", hash = "sha256:3eb7dc3f8f87307dbf72b64aa45034de12e28cd6bf29fb8cc67b05d49f9cad5c", size = 4195 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/85/b3/ab60e88b9c42c9fa2d59d22b2a77eaf7fc083a3148fb71ffad87cd314819/pltpublish-1.0.3-py3-none-any.whl", hash = "sha256:d06977f57679643c59b1d1a2cec923f823642fab53f5589e02c666cfa6e4a09b", size = 5053 }, +] + +[[package]] +name = "pluggy" +version = "1.6.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f9/e2/3e91f31a7d2b083fe6ef3fa267035b518369d9511ffab804f839851d2779/pluggy-1.6.0.tar.gz", hash = "sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3", size = 69412 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538 }, +] + +[[package]] +name = "protobuf" +version = "6.31.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/52/f3/b9655a711b32c19720253f6f06326faf90580834e2e83f840472d752bc8b/protobuf-6.31.1.tar.gz", hash = "sha256:d8cac4c982f0b957a4dc73a80e2ea24fab08e679c0de9deb835f4a12d69aca9a", size = 441797 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f3/6f/6ab8e4bf962fd5570d3deaa2d5c38f0a363f57b4501047b5ebeb83ab1125/protobuf-6.31.1-cp310-abi3-win32.whl", hash = "sha256:7fa17d5a29c2e04b7d90e5e32388b8bfd0e7107cd8e616feef7ed3fa6bdab5c9", size = 423603 }, + { url = "https://files.pythonhosted.org/packages/44/3a/b15c4347dd4bf3a1b0ee882f384623e2063bb5cf9fa9d57990a4f7df2fb6/protobuf-6.31.1-cp310-abi3-win_amd64.whl", hash = "sha256:426f59d2964864a1a366254fa703b8632dcec0790d8862d30034d8245e1cd447", size = 435283 }, + { url = "https://files.pythonhosted.org/packages/6a/c9/b9689a2a250264a84e66c46d8862ba788ee7a641cdca39bccf64f59284b7/protobuf-6.31.1-cp39-abi3-macosx_10_9_universal2.whl", hash = "sha256:6f1227473dc43d44ed644425268eb7c2e488ae245d51c6866d19fe158e207402", size = 425604 }, + { url = "https://files.pythonhosted.org/packages/76/a1/7a5a94032c83375e4fe7e7f56e3976ea6ac90c5e85fac8576409e25c39c3/protobuf-6.31.1-cp39-abi3-manylinux2014_aarch64.whl", hash = "sha256:a40fc12b84c154884d7d4c4ebd675d5b3b5283e155f324049ae396b95ddebc39", size = 322115 }, + { url = "https://files.pythonhosted.org/packages/fa/b1/b59d405d64d31999244643d88c45c8241c58f17cc887e73bcb90602327f8/protobuf-6.31.1-cp39-abi3-manylinux2014_x86_64.whl", hash = "sha256:4ee898bf66f7a8b0bd21bce523814e6fbd8c6add948045ce958b73af7e8878c6", size = 321070 }, + { url = "https://files.pythonhosted.org/packages/f7/af/ab3c51ab7507a7325e98ffe691d9495ee3d3aa5f589afad65ec920d39821/protobuf-6.31.1-py3-none-any.whl", hash = "sha256:720a6c7e6b77288b85063569baae8536671b39f15cc22037ec7045658d80489e", size = 168724 }, +] + +[[package]] +name = "pygments" +version = "2.19.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/b0/77/a5b8c569bf593b0140bde72ea885a803b82086995367bf2037de0159d924/pygments-2.19.2.tar.gz", hash = "sha256:636cb2477cec7f8952536970bc533bc43743542f70392ae026374600add5b887", size = 4968631 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c7/21/705964c7812476f378728bdf590ca4b771ec72385c533964653c68e86bdc/pygments-2.19.2-py3-none-any.whl", hash = "sha256:86540386c03d588bb81d44bc3928634ff26449851e99741617ecb9037ee5ec0b", size = 1225217 }, +] + +[[package]] +name = "pyparsing" +version = "3.2.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/bb/22/f1129e69d94ffff626bdb5c835506b3a5b4f3d070f17ea295e12c2c6f60f/pyparsing-3.2.3.tar.gz", hash = "sha256:b9c13f1ab8b3b542f72e28f634bad4de758ab3ce4546e4301970ad6fa77c38be", size = 1088608 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/05/e7/df2285f3d08fee213f2d041540fa4fc9ca6c2d44cf36d3a035bf2a8d2bcc/pyparsing-3.2.3-py3-none-any.whl", hash = "sha256:a749938e02d6fd0b59b356ca504a24982314bb090c383e3cf201c95ef7e2bfcf", size = 111120 }, +] + +[[package]] +name = "pytest" +version = "8.4.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "exceptiongroup", marker = "python_full_version < '3.11'" }, + { name = "iniconfig" }, + { name = "packaging" }, + { name = "pluggy" }, + { name = "pygments" }, + { name = "tomli", marker = "python_full_version < '3.11'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/08/ba/45911d754e8eba3d5a841a5ce61a65a685ff1798421ac054f85aa8747dfb/pytest-8.4.1.tar.gz", hash = "sha256:7c67fd69174877359ed9371ec3af8a3d2b04741818c51e5e99cc1742251fa93c", size = 1517714 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/29/16/c8a903f4c4dffe7a12843191437d7cd8e32751d5de349d45d3fe69544e87/pytest-8.4.1-py3-none-any.whl", hash = "sha256:539c70ba6fcead8e78eebbf1115e8b589e7565830d7d006a8723f19ac8a0afb7", size = 365474 }, +] + +[[package]] +name = "python-dateutil" +version = "2.9.0.post0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "six" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/66/c0/0c8b6ad9f17a802ee498c46e004a0eb49bc148f2fd230864601a86dcf6db/python-dateutil-2.9.0.post0.tar.gz", hash = "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3", size = 342432 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ec/57/56b9bcc3c9c6a792fcbaf139543cee77261f3651ca9da0c93f5c1221264b/python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427", size = 229892 }, +] + +[[package]] +name = "ruff" +version = "0.12.7" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a1/81/0bd3594fa0f690466e41bd033bdcdf86cba8288345ac77ad4afbe5ec743a/ruff-0.12.7.tar.gz", hash = "sha256:1fc3193f238bc2d7968772c82831a4ff69252f673be371fb49663f0068b7ec71", size = 5197814 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e1/d2/6cb35e9c85e7a91e8d22ab32ae07ac39cc34a71f1009a6f9e4a2a019e602/ruff-0.12.7-py3-none-linux_armv6l.whl", hash = "sha256:76e4f31529899b8c434c3c1dede98c4483b89590e15fb49f2d46183801565303", size = 11852189 }, + { url = "https://files.pythonhosted.org/packages/63/5b/a4136b9921aa84638f1a6be7fb086f8cad0fde538ba76bda3682f2599a2f/ruff-0.12.7-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:789b7a03e72507c54fb3ba6209e4bb36517b90f1a3569ea17084e3fd295500fb", size = 12519389 }, + { url = "https://files.pythonhosted.org/packages/a8/c9/3e24a8472484269b6b1821794141f879c54645a111ded4b6f58f9ab0705f/ruff-0.12.7-py3-none-macosx_11_0_arm64.whl", hash = "sha256:2e1c2a3b8626339bb6369116e7030a4cf194ea48f49b64bb505732a7fce4f4e3", size = 11743384 }, + { url = "https://files.pythonhosted.org/packages/26/7c/458dd25deeb3452c43eaee853c0b17a1e84169f8021a26d500ead77964fd/ruff-0.12.7-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:32dec41817623d388e645612ec70d5757a6d9c035f3744a52c7b195a57e03860", size = 11943759 }, + { url = "https://files.pythonhosted.org/packages/7f/8b/658798472ef260ca050e400ab96ef7e85c366c39cf3dfbef4d0a46a528b6/ruff-0.12.7-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:47ef751f722053a5df5fa48d412dbb54d41ab9b17875c6840a58ec63ff0c247c", size = 11654028 }, + { url = "https://files.pythonhosted.org/packages/a8/86/9c2336f13b2a3326d06d39178fd3448dcc7025f82514d1b15816fe42bfe8/ruff-0.12.7-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a828a5fc25a3efd3e1ff7b241fd392686c9386f20e5ac90aa9234a5faa12c423", size = 13225209 }, + { url = "https://files.pythonhosted.org/packages/76/69/df73f65f53d6c463b19b6b312fd2391dc36425d926ec237a7ed028a90fc1/ruff-0.12.7-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:5726f59b171111fa6a69d82aef48f00b56598b03a22f0f4170664ff4d8298efb", size = 14182353 }, + { url = "https://files.pythonhosted.org/packages/58/1e/de6cda406d99fea84b66811c189b5ea139814b98125b052424b55d28a41c/ruff-0.12.7-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:74e6f5c04c4dd4aba223f4fe6e7104f79e0eebf7d307e4f9b18c18362124bccd", size = 13631555 }, + { url = "https://files.pythonhosted.org/packages/6f/ae/625d46d5164a6cc9261945a5e89df24457dc8262539ace3ac36c40f0b51e/ruff-0.12.7-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5d0bfe4e77fba61bf2ccadf8cf005d6133e3ce08793bbe870dd1c734f2699a3e", size = 12667556 }, + { url = "https://files.pythonhosted.org/packages/55/bf/9cb1ea5e3066779e42ade8d0cd3d3b0582a5720a814ae1586f85014656b6/ruff-0.12.7-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:06bfb01e1623bf7f59ea749a841da56f8f653d641bfd046edee32ede7ff6c606", size = 12939784 }, + { url = "https://files.pythonhosted.org/packages/55/7f/7ead2663be5627c04be83754c4f3096603bf5e99ed856c7cd29618c691bd/ruff-0.12.7-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:e41df94a957d50083fd09b916d6e89e497246698c3f3d5c681c8b3e7b9bb4ac8", size = 11771356 }, + { url = "https://files.pythonhosted.org/packages/17/40/a95352ea16edf78cd3a938085dccc55df692a4d8ba1b3af7accbe2c806b0/ruff-0.12.7-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:4000623300563c709458d0ce170c3d0d788c23a058912f28bbadc6f905d67afa", size = 11612124 }, + { url = "https://files.pythonhosted.org/packages/4d/74/633b04871c669e23b8917877e812376827c06df866e1677f15abfadc95cb/ruff-0.12.7-py3-none-musllinux_1_2_i686.whl", hash = "sha256:69ffe0e5f9b2cf2b8e289a3f8945b402a1b19eff24ec389f45f23c42a3dd6fb5", size = 12479945 }, + { url = "https://files.pythonhosted.org/packages/be/34/c3ef2d7799c9778b835a76189c6f53c179d3bdebc8c65288c29032e03613/ruff-0.12.7-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:a07a5c8ffa2611a52732bdc67bf88e243abd84fe2d7f6daef3826b59abbfeda4", size = 12998677 }, + { url = "https://files.pythonhosted.org/packages/77/ab/aca2e756ad7b09b3d662a41773f3edcbd262872a4fc81f920dc1ffa44541/ruff-0.12.7-py3-none-win32.whl", hash = "sha256:c928f1b2ec59fb77dfdf70e0419408898b63998789cc98197e15f560b9e77f77", size = 11756687 }, + { url = "https://files.pythonhosted.org/packages/b4/71/26d45a5042bc71db22ddd8252ca9d01e9ca454f230e2996bb04f16d72799/ruff-0.12.7-py3-none-win_amd64.whl", hash = "sha256:9c18f3d707ee9edf89da76131956aba1270c6348bfee8f6c647de841eac7194f", size = 12912365 }, + { url = "https://files.pythonhosted.org/packages/4c/9b/0b8aa09817b63e78d94b4977f18b1fcaead3165a5ee49251c5d5c245bb2d/ruff-0.12.7-py3-none-win_arm64.whl", hash = "sha256:dfce05101dbd11833a0776716d5d1578641b7fddb537fe7fa956ab85d1769b69", size = 11982083 }, +] + +[[package]] +name = "setuptools" +version = "80.9.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/18/5d/3bf57dcd21979b887f014ea83c24ae194cfcd12b9e0fda66b957c69d1fca/setuptools-80.9.0.tar.gz", hash = "sha256:f36b47402ecde768dbfafc46e8e4207b4360c654f1f3bb84475f0a28628fb19c", size = 1319958 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a3/dc/17031897dae0efacfea57dfd3a82fdd2a2aeb58e0ff71b77b87e44edc772/setuptools-80.9.0-py3-none-any.whl", hash = "sha256:062d34222ad13e0cc312a4c02d73f059e86a4acbfbdea8f8f76b28c99f306922", size = 1201486 }, +] + +[[package]] +name = "six" +version = "1.17.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/94/e7/b2c673351809dca68a0e064b6af791aa332cf192da575fd474ed7d6f16a2/six-1.17.0.tar.gz", hash = "sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81", size = 34031 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b7/ce/149a00dd41f10bc29e5921b496af8b574d8413afcd5e30dfa0ed46c2cc5e/six-1.17.0-py2.py3-none-any.whl", hash = "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274", size = 11050 }, +] + +[[package]] +name = "sympy" +version = "1.14.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "mpmath" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/83/d3/803453b36afefb7c2bb238361cd4ae6125a569b4db67cd9e79846ba2d68c/sympy-1.14.0.tar.gz", hash = "sha256:d3d3fe8df1e5a0b42f0e7bdf50541697dbe7d23746e894990c030e2b05e72517", size = 7793921 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a2/09/77d55d46fd61b4a135c444fc97158ef34a095e5681d0a6c10b75bf356191/sympy-1.14.0-py3-none-any.whl", hash = "sha256:e091cc3e99d2141a0ba2847328f5479b05d94a6635cb96148ccb3f34671bd8f5", size = 6299353 }, +] + +[[package]] +name = "synth" +version = "0.1.0" +source = { editable = "." } +dependencies = [ + { name = "colorama" }, + { name = "cython" }, + { name = "matplotlib" }, + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, + { name = "numpy", version = "2.3.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, + { name = "pltpublish" }, + { name = "tensorboard" }, + { name = "torch" }, + { name = "tqdm" }, + { name = "vose" }, +] + +[package.dev-dependencies] +dev = [ + { name = "mypy" }, + { name = "pytest" }, + { name = "ruff" }, +] + +[package.metadata] +requires-dist = [ + { name = "colorama", specifier = ">=0.4.4" }, + { name = "cython", specifier = ">=0.29" }, + { name = "matplotlib", specifier = ">=3.5.1" }, + { name = "numpy", specifier = ">=1.22" }, + { name = "pltpublish", specifier = ">=0.1.0" }, + { name = "tensorboard", specifier = ">=0" }, + { name = "torch", specifier = ">=1.13.1" }, + { name = "tqdm", specifier = ">=4.63.0" }, + { name = "vose", git = "https://github.com/Theomat/vose.git" }, +] + +[package.metadata.requires-dev] +dev = [ + { name = "mypy", specifier = ">=0.910" }, + { name = "pytest", specifier = ">=7.2.0" }, + { name = "ruff", specifier = ">=0.4.3" }, +] + +[[package]] +name = "tensorboard" +version = "2.20.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "absl-py" }, + { name = "grpcio" }, + { name = "markdown" }, + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, + { name = "numpy", version = "2.3.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, + { name = "packaging" }, + { name = "pillow" }, + { name = "protobuf" }, + { name = "setuptools" }, + { name = "tensorboard-data-server" }, + { name = "werkzeug" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/9c/d9/a5db55f88f258ac669a92858b70a714bbbd5acd993820b41ec4a96a4d77f/tensorboard-2.20.0-py3-none-any.whl", hash = "sha256:9dc9f978cb84c0723acf9a345d96c184f0293d18f166bb8d59ee098e6cfaaba6", size = 5525680 }, +] + +[[package]] +name = "tensorboard-data-server" +version = "0.7.2" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7a/13/e503968fefabd4c6b2650af21e110aa8466fe21432cd7c43a84577a89438/tensorboard_data_server-0.7.2-py3-none-any.whl", hash = "sha256:7e0610d205889588983836ec05dc098e80f97b7e7bbff7e994ebb78f578d0ddb", size = 2356 }, + { url = "https://files.pythonhosted.org/packages/b7/85/dabeaf902892922777492e1d253bb7e1264cadce3cea932f7ff599e53fea/tensorboard_data_server-0.7.2-py3-none-macosx_10_9_x86_64.whl", hash = "sha256:9fe5d24221b29625dbc7328b0436ca7fc1c23de4acf4d272f1180856e32f9f60", size = 4823598 }, + { url = "https://files.pythonhosted.org/packages/73/c6/825dab04195756cf8ff2e12698f22513b3db2f64925bdd41671bfb33aaa5/tensorboard_data_server-0.7.2-py3-none-manylinux_2_31_x86_64.whl", hash = "sha256:ef687163c24185ae9754ed5650eb5bc4d84ff257aabdc33f0cc6f74d8ba54530", size = 6590363 }, +] + +[[package]] +name = "tomli" +version = "2.2.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/18/87/302344fed471e44a87289cf4967697d07e532f2421fdaf868a303cbae4ff/tomli-2.2.1.tar.gz", hash = "sha256:cd45e1dc79c835ce60f7404ec8119f2eb06d38b1deba146f07ced3bbc44505ff", size = 17175 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/43/ca/75707e6efa2b37c77dadb324ae7d9571cb424e61ea73fad7c56c2d14527f/tomli-2.2.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:678e4fa69e4575eb77d103de3df8a895e1591b48e740211bd1067378c69e8249", size = 131077 }, + { url = "https://files.pythonhosted.org/packages/c7/16/51ae563a8615d472fdbffc43a3f3d46588c264ac4f024f63f01283becfbb/tomli-2.2.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:023aa114dd824ade0100497eb2318602af309e5a55595f76b626d6d9f3b7b0a6", size = 123429 }, + { url = "https://files.pythonhosted.org/packages/f1/dd/4f6cd1e7b160041db83c694abc78e100473c15d54620083dbd5aae7b990e/tomli-2.2.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ece47d672db52ac607a3d9599a9d48dcb2f2f735c6c2d1f34130085bb12b112a", size = 226067 }, + { url = "https://files.pythonhosted.org/packages/a9/6b/c54ede5dc70d648cc6361eaf429304b02f2871a345bbdd51e993d6cdf550/tomli-2.2.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6972ca9c9cc9f0acaa56a8ca1ff51e7af152a9f87fb64623e31d5c83700080ee", size = 236030 }, + { url = "https://files.pythonhosted.org/packages/1f/47/999514fa49cfaf7a92c805a86c3c43f4215621855d151b61c602abb38091/tomli-2.2.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c954d2250168d28797dd4e3ac5cf812a406cd5a92674ee4c8f123c889786aa8e", size = 240898 }, + { url = "https://files.pythonhosted.org/packages/73/41/0a01279a7ae09ee1573b423318e7934674ce06eb33f50936655071d81a24/tomli-2.2.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:8dd28b3e155b80f4d54beb40a441d366adcfe740969820caf156c019fb5c7ec4", size = 229894 }, + { url = "https://files.pythonhosted.org/packages/55/18/5d8bc5b0a0362311ce4d18830a5d28943667599a60d20118074ea1b01bb7/tomli-2.2.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:e59e304978767a54663af13c07b3d1af22ddee3bb2fb0618ca1593e4f593a106", size = 245319 }, + { url = "https://files.pythonhosted.org/packages/92/a3/7ade0576d17f3cdf5ff44d61390d4b3febb8a9fc2b480c75c47ea048c646/tomli-2.2.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:33580bccab0338d00994d7f16f4c4ec25b776af3ffaac1ed74e0b3fc95e885a8", size = 238273 }, + { url = "https://files.pythonhosted.org/packages/72/6f/fa64ef058ac1446a1e51110c375339b3ec6be245af9d14c87c4a6412dd32/tomli-2.2.1-cp311-cp311-win32.whl", hash = "sha256:465af0e0875402f1d226519c9904f37254b3045fc5084697cefb9bdde1ff99ff", size = 98310 }, + { url = "https://files.pythonhosted.org/packages/6a/1c/4a2dcde4a51b81be3530565e92eda625d94dafb46dbeb15069df4caffc34/tomli-2.2.1-cp311-cp311-win_amd64.whl", hash = "sha256:2d0f2fdd22b02c6d81637a3c95f8cd77f995846af7414c5c4b8d0545afa1bc4b", size = 108309 }, + { url = "https://files.pythonhosted.org/packages/52/e1/f8af4c2fcde17500422858155aeb0d7e93477a0d59a98e56cbfe75070fd0/tomli-2.2.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:4a8f6e44de52d5e6c657c9fe83b562f5f4256d8ebbfe4ff922c495620a7f6cea", size = 132762 }, + { url = "https://files.pythonhosted.org/packages/03/b8/152c68bb84fc00396b83e7bbddd5ec0bd3dd409db4195e2a9b3e398ad2e3/tomli-2.2.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:8d57ca8095a641b8237d5b079147646153d22552f1c637fd3ba7f4b0b29167a8", size = 123453 }, + { url = "https://files.pythonhosted.org/packages/c8/d6/fc9267af9166f79ac528ff7e8c55c8181ded34eb4b0e93daa767b8841573/tomli-2.2.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4e340144ad7ae1533cb897d406382b4b6fede8890a03738ff1683af800d54192", size = 233486 }, + { url = "https://files.pythonhosted.org/packages/5c/51/51c3f2884d7bab89af25f678447ea7d297b53b5a3b5730a7cb2ef6069f07/tomli-2.2.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:db2b95f9de79181805df90bedc5a5ab4c165e6ec3fe99f970d0e302f384ad222", size = 242349 }, + { url = "https://files.pythonhosted.org/packages/ab/df/bfa89627d13a5cc22402e441e8a931ef2108403db390ff3345c05253935e/tomli-2.2.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:40741994320b232529c802f8bc86da4e1aa9f413db394617b9a256ae0f9a7f77", size = 252159 }, + { url = "https://files.pythonhosted.org/packages/9e/6e/fa2b916dced65763a5168c6ccb91066f7639bdc88b48adda990db10c8c0b/tomli-2.2.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:400e720fe168c0f8521520190686ef8ef033fb19fc493da09779e592861b78c6", size = 237243 }, + { url = "https://files.pythonhosted.org/packages/b4/04/885d3b1f650e1153cbb93a6a9782c58a972b94ea4483ae4ac5cedd5e4a09/tomli-2.2.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:02abe224de6ae62c19f090f68da4e27b10af2b93213d36cf44e6e1c5abd19fdd", size = 259645 }, + { url = "https://files.pythonhosted.org/packages/9c/de/6b432d66e986e501586da298e28ebeefd3edc2c780f3ad73d22566034239/tomli-2.2.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:b82ebccc8c8a36f2094e969560a1b836758481f3dc360ce9a3277c65f374285e", size = 244584 }, + { url = "https://files.pythonhosted.org/packages/1c/9a/47c0449b98e6e7d1be6cbac02f93dd79003234ddc4aaab6ba07a9a7482e2/tomli-2.2.1-cp312-cp312-win32.whl", hash = "sha256:889f80ef92701b9dbb224e49ec87c645ce5df3fa2cc548664eb8a25e03127a98", size = 98875 }, + { url = "https://files.pythonhosted.org/packages/ef/60/9b9638f081c6f1261e2688bd487625cd1e660d0a85bd469e91d8db969734/tomli-2.2.1-cp312-cp312-win_amd64.whl", hash = "sha256:7fc04e92e1d624a4a63c76474610238576942d6b8950a2d7f908a340494e67e4", size = 109418 }, + { url = "https://files.pythonhosted.org/packages/04/90/2ee5f2e0362cb8a0b6499dc44f4d7d48f8fff06d28ba46e6f1eaa61a1388/tomli-2.2.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:f4039b9cbc3048b2416cc57ab3bda989a6fcf9b36cf8937f01a6e731b64f80d7", size = 132708 }, + { url = "https://files.pythonhosted.org/packages/c0/ec/46b4108816de6b385141f082ba99e315501ccd0a2ea23db4a100dd3990ea/tomli-2.2.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:286f0ca2ffeeb5b9bd4fcc8d6c330534323ec51b2f52da063b11c502da16f30c", size = 123582 }, + { url = "https://files.pythonhosted.org/packages/a0/bd/b470466d0137b37b68d24556c38a0cc819e8febe392d5b199dcd7f578365/tomli-2.2.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a92ef1a44547e894e2a17d24e7557a5e85a9e1d0048b0b5e7541f76c5032cb13", size = 232543 }, + { url = "https://files.pythonhosted.org/packages/d9/e5/82e80ff3b751373f7cead2815bcbe2d51c895b3c990686741a8e56ec42ab/tomli-2.2.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9316dc65bed1684c9a98ee68759ceaed29d229e985297003e494aa825ebb0281", size = 241691 }, + { url = "https://files.pythonhosted.org/packages/05/7e/2a110bc2713557d6a1bfb06af23dd01e7dde52b6ee7dadc589868f9abfac/tomli-2.2.1-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e85e99945e688e32d5a35c1ff38ed0b3f41f43fad8df0bdf79f72b2ba7bc5272", size = 251170 }, + { url = "https://files.pythonhosted.org/packages/64/7b/22d713946efe00e0adbcdfd6d1aa119ae03fd0b60ebed51ebb3fa9f5a2e5/tomli-2.2.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:ac065718db92ca818f8d6141b5f66369833d4a80a9d74435a268c52bdfa73140", size = 236530 }, + { url = "https://files.pythonhosted.org/packages/38/31/3a76f67da4b0cf37b742ca76beaf819dca0ebef26d78fc794a576e08accf/tomli-2.2.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:d920f33822747519673ee656a4b6ac33e382eca9d331c87770faa3eef562aeb2", size = 258666 }, + { url = "https://files.pythonhosted.org/packages/07/10/5af1293da642aded87e8a988753945d0cf7e00a9452d3911dd3bb354c9e2/tomli-2.2.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:a198f10c4d1b1375d7687bc25294306e551bf1abfa4eace6650070a5c1ae2744", size = 243954 }, + { url = "https://files.pythonhosted.org/packages/5b/b9/1ed31d167be802da0fc95020d04cd27b7d7065cc6fbefdd2f9186f60d7bd/tomli-2.2.1-cp313-cp313-win32.whl", hash = "sha256:d3f5614314d758649ab2ab3a62d4f2004c825922f9e370b29416484086b264ec", size = 98724 }, + { url = "https://files.pythonhosted.org/packages/c7/32/b0963458706accd9afcfeb867c0f9175a741bf7b19cd424230714d722198/tomli-2.2.1-cp313-cp313-win_amd64.whl", hash = "sha256:a38aa0308e754b0e3c67e344754dff64999ff9b513e691d0e786265c93583c69", size = 109383 }, + { url = "https://files.pythonhosted.org/packages/6e/c2/61d3e0f47e2b74ef40a68b9e6ad5984f6241a942f7cd3bbfbdbd03861ea9/tomli-2.2.1-py3-none-any.whl", hash = "sha256:cb55c73c5f4408779d0cf3eef9f762b9c9f147a77de7b258bef0a5628adc85cc", size = 14257 }, +] + +[[package]] +name = "torch" +version = "2.7.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "filelock" }, + { name = "fsspec" }, + { name = "jinja2" }, + { name = "networkx", version = "3.4.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, + { name = "networkx", version = "3.5", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, + { name = "nvidia-cublas-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "nvidia-cuda-cupti-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "nvidia-cuda-nvrtc-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "nvidia-cuda-runtime-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "nvidia-cudnn-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "nvidia-cufft-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "nvidia-cufile-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "nvidia-curand-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "nvidia-cusolver-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "nvidia-cusparse-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "nvidia-cusparselt-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "nvidia-nccl-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "nvidia-nvjitlink-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "nvidia-nvtx-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "setuptools", marker = "python_full_version >= '3.12'" }, + { name = "sympy" }, + { name = "triton", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "typing-extensions" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/6a/27/2e06cb52adf89fe6e020963529d17ed51532fc73c1e6d1b18420ef03338c/torch-2.7.1-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:a103b5d782af5bd119b81dbcc7ffc6fa09904c423ff8db397a1e6ea8fd71508f", size = 99089441 }, + { url = "https://files.pythonhosted.org/packages/0a/7c/0a5b3aee977596459ec45be2220370fde8e017f651fecc40522fd478cb1e/torch-2.7.1-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:fe955951bdf32d182ee8ead6c3186ad54781492bf03d547d31771a01b3d6fb7d", size = 821154516 }, + { url = "https://files.pythonhosted.org/packages/f9/91/3d709cfc5e15995fb3fe7a6b564ce42280d3a55676dad672205e94f34ac9/torch-2.7.1-cp310-cp310-win_amd64.whl", hash = "sha256:885453d6fba67d9991132143bf7fa06b79b24352f4506fd4d10b309f53454162", size = 216093147 }, + { url = "https://files.pythonhosted.org/packages/92/f6/5da3918414e07da9866ecb9330fe6ffdebe15cb9a4c5ada7d4b6e0a6654d/torch-2.7.1-cp310-none-macosx_11_0_arm64.whl", hash = "sha256:d72acfdb86cee2a32c0ce0101606f3758f0d8bb5f8f31e7920dc2809e963aa7c", size = 68630914 }, + { url = "https://files.pythonhosted.org/packages/11/56/2eae3494e3d375533034a8e8cf0ba163363e996d85f0629441fa9d9843fe/torch-2.7.1-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:236f501f2e383f1cb861337bdf057712182f910f10aeaf509065d54d339e49b2", size = 99093039 }, + { url = "https://files.pythonhosted.org/packages/e5/94/34b80bd172d0072c9979708ccd279c2da2f55c3ef318eceec276ab9544a4/torch-2.7.1-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:06eea61f859436622e78dd0cdd51dbc8f8c6d76917a9cf0555a333f9eac31ec1", size = 821174704 }, + { url = "https://files.pythonhosted.org/packages/50/9e/acf04ff375b0b49a45511c55d188bcea5c942da2aaf293096676110086d1/torch-2.7.1-cp311-cp311-win_amd64.whl", hash = "sha256:8273145a2e0a3c6f9fd2ac36762d6ee89c26d430e612b95a99885df083b04e52", size = 216095937 }, + { url = "https://files.pythonhosted.org/packages/5b/2b/d36d57c66ff031f93b4fa432e86802f84991477e522adcdffd314454326b/torch-2.7.1-cp311-none-macosx_11_0_arm64.whl", hash = "sha256:aea4fc1bf433d12843eb2c6b2204861f43d8364597697074c8d38ae2507f8730", size = 68640034 }, + { url = "https://files.pythonhosted.org/packages/87/93/fb505a5022a2e908d81fe9a5e0aa84c86c0d5f408173be71c6018836f34e/torch-2.7.1-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:27ea1e518df4c9de73af7e8a720770f3628e7f667280bce2be7a16292697e3fa", size = 98948276 }, + { url = "https://files.pythonhosted.org/packages/56/7e/67c3fe2b8c33f40af06326a3d6ae7776b3e3a01daa8f71d125d78594d874/torch-2.7.1-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:c33360cfc2edd976c2633b3b66c769bdcbbf0e0b6550606d188431c81e7dd1fc", size = 821025792 }, + { url = "https://files.pythonhosted.org/packages/a1/37/a37495502bc7a23bf34f89584fa5a78e25bae7b8da513bc1b8f97afb7009/torch-2.7.1-cp312-cp312-win_amd64.whl", hash = "sha256:d8bf6e1856ddd1807e79dc57e54d3335f2b62e6f316ed13ed3ecfe1fc1df3d8b", size = 216050349 }, + { url = "https://files.pythonhosted.org/packages/3a/60/04b77281c730bb13460628e518c52721257814ac6c298acd25757f6a175c/torch-2.7.1-cp312-none-macosx_11_0_arm64.whl", hash = "sha256:787687087412c4bd68d315e39bc1223f08aae1d16a9e9771d95eabbb04ae98fb", size = 68645146 }, + { url = "https://files.pythonhosted.org/packages/66/81/e48c9edb655ee8eb8c2a6026abdb6f8d2146abd1f150979ede807bb75dcb/torch-2.7.1-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:03563603d931e70722dce0e11999d53aa80a375a3d78e6b39b9f6805ea0a8d28", size = 98946649 }, + { url = "https://files.pythonhosted.org/packages/3a/24/efe2f520d75274fc06b695c616415a1e8a1021d87a13c68ff9dce733d088/torch-2.7.1-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:d632f5417b6980f61404a125b999ca6ebd0b8b4bbdbb5fbbba44374ab619a412", size = 821033192 }, + { url = "https://files.pythonhosted.org/packages/dd/d9/9c24d230333ff4e9b6807274f6f8d52a864210b52ec794c5def7925f4495/torch-2.7.1-cp313-cp313-win_amd64.whl", hash = "sha256:23660443e13995ee93e3d844786701ea4ca69f337027b05182f5ba053ce43b38", size = 216055668 }, + { url = "https://files.pythonhosted.org/packages/95/bf/e086ee36ddcef9299f6e708d3b6c8487c1651787bb9ee2939eb2a7f74911/torch-2.7.1-cp313-cp313t-macosx_14_0_arm64.whl", hash = "sha256:0da4f4dba9f65d0d203794e619fe7ca3247a55ffdcbd17ae8fb83c8b2dc9b585", size = 68925988 }, + { url = "https://files.pythonhosted.org/packages/69/6a/67090dcfe1cf9048448b31555af6efb149f7afa0a310a366adbdada32105/torch-2.7.1-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:e08d7e6f21a617fe38eeb46dd2213ded43f27c072e9165dc27300c9ef9570934", size = 99028857 }, + { url = "https://files.pythonhosted.org/packages/90/1c/48b988870823d1cc381f15ec4e70ed3d65e043f43f919329b0045ae83529/torch-2.7.1-cp313-cp313t-manylinux_2_28_x86_64.whl", hash = "sha256:30207f672328a42df4f2174b8f426f354b2baa0b7cca3a0adb3d6ab5daf00dc8", size = 821098066 }, + { url = "https://files.pythonhosted.org/packages/7b/eb/10050d61c9d5140c5dc04a89ed3257ef1a6b93e49dd91b95363d757071e0/torch-2.7.1-cp313-cp313t-win_amd64.whl", hash = "sha256:79042feca1c634aaf6603fe6feea8c6b30dfa140a6bbc0b973e2260c7e79a22e", size = 216336310 }, + { url = "https://files.pythonhosted.org/packages/b1/29/beb45cdf5c4fc3ebe282bf5eafc8dfd925ead7299b3c97491900fe5ed844/torch-2.7.1-cp313-none-macosx_11_0_arm64.whl", hash = "sha256:988b0cbc4333618a1056d2ebad9eb10089637b659eb645434d0809d8d937b946", size = 68645708 }, +] + +[[package]] +name = "tqdm" +version = "4.67.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a8/4b/29b4ef32e036bb34e4ab51796dd745cdba7ed47ad142a9f4a1eb8e0c744d/tqdm-4.67.1.tar.gz", hash = "sha256:f8aef9c52c08c13a65f30ea34f4e5aac3fd1a34959879d7e59e63027286627f2", size = 169737 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d0/30/dc54f88dd4a2b5dc8a0279bdd7270e735851848b762aeb1c1184ed1f6b14/tqdm-4.67.1-py3-none-any.whl", hash = "sha256:26445eca388f82e72884e0d580d5464cd801a3ea01e63e5601bdff9ba6a48de2", size = 78540 }, +] + +[[package]] +name = "triton" +version = "3.3.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "setuptools" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/8d/a9/549e51e9b1b2c9b854fd761a1d23df0ba2fbc60bd0c13b489ffa518cfcb7/triton-3.3.1-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b74db445b1c562844d3cfad6e9679c72e93fdfb1a90a24052b03bb5c49d1242e", size = 155600257 }, + { url = "https://files.pythonhosted.org/packages/21/2f/3e56ea7b58f80ff68899b1dbe810ff257c9d177d288c6b0f55bf2fe4eb50/triton-3.3.1-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b31e3aa26f8cb3cc5bf4e187bf737cbacf17311e1112b781d4a059353dfd731b", size = 155689937 }, + { url = "https://files.pythonhosted.org/packages/24/5f/950fb373bf9c01ad4eb5a8cd5eaf32cdf9e238c02f9293557a2129b9c4ac/triton-3.3.1-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9999e83aba21e1a78c1f36f21bce621b77bcaa530277a50484a7cb4a822f6e43", size = 155669138 }, + { url = "https://files.pythonhosted.org/packages/74/1f/dfb531f90a2d367d914adfee771babbd3f1a5b26c3f5fbc458dee21daa78/triton-3.3.1-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b89d846b5a4198317fec27a5d3a609ea96b6d557ff44b56c23176546023c4240", size = 155673035 }, + { url = "https://files.pythonhosted.org/packages/28/71/bd20ffcb7a64c753dc2463489a61bf69d531f308e390ad06390268c4ea04/triton-3.3.1-cp313-cp313t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a3198adb9d78b77818a5388bff89fa72ff36f9da0bc689db2f0a651a67ce6a42", size = 155735832 }, +] + +[[package]] +name = "typing-extensions" +version = "4.14.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/98/5a/da40306b885cc8c09109dc2e1abd358d5684b1425678151cdaed4731c822/typing_extensions-4.14.1.tar.gz", hash = "sha256:38b39f4aeeab64884ce9f74c94263ef78f3c22467c8724005483154c26648d36", size = 107673 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b5/00/d631e67a838026495268c2f6884f3711a15a9a2a96cd244fdaea53b823fb/typing_extensions-4.14.1-py3-none-any.whl", hash = "sha256:d1e1e3b58374dc93031d6eda2420a48ea44a36c2b4766a4fdeb3710755731d76", size = 43906 }, +] + +[[package]] +name = "vose" +version = "0.0.4" +source = { git = "https://github.com/Theomat/vose.git#6de8d98ab188aa26be9e22c88a9ad22ae966efa5" } +dependencies = [ + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, + { name = "numpy", version = "2.3.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, +] + +[[package]] +name = "werkzeug" +version = "3.1.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "markupsafe" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/9f/69/83029f1f6300c5fb2471d621ab06f6ec6b3324685a2ce0f9777fd4a8b71e/werkzeug-3.1.3.tar.gz", hash = "sha256:60723ce945c19328679790e3282cc758aa4a6040e4bb330f53d30fa546d44746", size = 806925 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/52/24/ab44c871b0f07f491e5d2ad12c9bd7358e527510618cb1b803a88e986db1/werkzeug-3.1.3-py3-none-any.whl", hash = "sha256:54b78bf3716d19a65be4fceccc0d1d7b89e608834989dfae50ea87564639213e", size = 224498 }, +]