improve constraints file output

[dhellmann]

Work harder to produce a unique list of constraints that meet all of
our requirements.

[dhellmann]

Some sample output for the command using a build graph for instructlab:

$ fromager graph explain-duplicates ./bootstrap-output.cuda-ubi9/work-dir/graph.json

numpy
  1.26.4
    numpy<2.0.0 matches ['1.26.4']
      vllm==0.5.2.2
      outlines==0.0.46
    numpy<2,>=1.23.2; python_version == "3.11" matches ['1.26.4']
      pandas==2.2.1
    numpy<2.0 matches ['1.26.4']
      xformers==0.0.27
      deepspeed==0.14.4
    numpy<2.0.0,>=1.26.4; python_version >= "3.11" matches ['1.26.4']
      instructlab==0.18.0b6
    numpy<2.0.0,>=1.17 matches ['1.26.4']
      accelerate==0.33.0
    numpy<2.0.0,>=1.23.5; python_version >= "3.9" and python_version < "3.12" matches ['1.26.4']
      pandas-stubs==2.2.2.240805
    numpy<2.0.0,>=1.26.4; python_version != "3.10" matches ['1.26.4']
      instructlab-training==0.3.1
      instructlab-training==0.3.1
    numpy<2.0.0,>=1.18.2 matches ['1.26.4']
      trl==0.9.6
  2.0.0
    numpy>=1.17 matches ['2.0.0', '1.26.4']
      datasets==2.20.0
      transformers==4.43.4
      gguf==0.9.1
      evaluate==0.4.2
      peft==0.12.0
      sacrebleu==2.4.1
    numpy>=1.16.6 matches ['2.0.0', '1.26.4']
      pyarrow==17.0.0
    numpy<2.1,>=1.22 matches ['2.0.0', '1.26.4']
      numba==0.60.0
    numpy>=1.23.0 matches ['2.0.0', '1.26.4']
      numexpr==2.10.1
    numpy matches ['2.0.0', '1.26.4']
      rouge-score==0.1.2
      pythran==0.16.1
      bitsandbytes==0.43.3
    numpy>=1.19.5 matches ['2.0.0', '1.26.4']
      scikit-learn==1.5.1
    numpy<2.3,>=1.23.5 matches ['2.0.0', '1.26.4']
      scipy==1.14.0
    numpy>=1.20.0 matches ['2.0.0', '1.26.4']
      llama-cpp-python==0.2.79
  * numpy==1.26.4 usable by all consumers

starlette
  0.37.2
    starlette<0.38.0,>=0.37.2 matches ['0.37.2']
      fastapi==0.112.0
  0.38.2
    starlette<1.0.0,>=0.30.0 matches ['0.37.2', '0.38.2']
      prometheus-fastapi-instrumentator==7.0.0
    starlette matches ['0.37.2', '0.38.2']
      sse-starlette==2.1.3
      starlette-context==0.3.6
  * starlette==0.37.2 usable by all consumers

tenacity
  8.5.0
    tenacity!=8.4.0,<9.0.0,>=8.1.0 matches ['8.5.0']
      langchain-core==0.2.28
  9.0.0
    tenacity!=8.4.0,>=8.3.0 matches ['9.0.0', '8.5.0']
      instructlab-sdg==0.2.4
  * tenacity==8.5.0 usable by all consumers

And the constraints

$ fromager graph to-constraints ./bootstrap-output.cuda-ubi9/work-dir/graph.json
loading settings from /home/dhellmann/devel/rhel-ai-build-pipelines/builder/overrides/settings.yaml
absl-py==2.1.0
accelerate==0.33.0
aiofiles==24.1.0
aiohappyeyeballs==2.3.4
aiohttp==3.10.1
aiosignal==1.3.1
annotated-types==0.7.0
anyio==4.4.0
appdirs==1.4.4
attrs==24.2.0
bitsandbytes==0.43.3
certifi==2024.7.4
chardet==5.2.0
charset-normalizer==3.3.2
click==8.1.7
click-didyoumean==0.3.1
cloudpickle==3.0.0
colorama==0.4.6
dataproperty==1.0.1
datasets==2.20.0
deepspeed==0.14.4
dill==0.3.8
diskcache==5.6.3
distro==1.9.0
docker-pycreds==0.4.0
docstring-parser==0.16
einops==0.8.0
evaluate==0.4.2
fastapi==0.112.0
fastchat==0.1.0
filelock==3.15.4
flash-attn==2.5.7
frozenlist==1.4.1
fsspec==2024.5.0
gguf==0.9.1
gitdb==4.0.11
gitpython==3.1.43
h11==0.14.0
hjson==3.1.0
httpcore==1.0.5
httptools==0.6.1
httpx==0.27.0
huggingface-hub==0.24.5
idna==3.7
importlib-metadata==8.2.0
instructlab==0.18.0b6
instructlab-dolomite==0.1.1
instructlab-eval==0.1.1
instructlab-quantize==0.1.0
instructlab-schema==0.4.0
instructlab-sdg==0.2.4
instructlab-training==0.3.1
interegular==0.3.3
jinja2==3.1.4
jiter==0.5.0
joblib==1.4.2
jsonlines==4.0.0
jsonpatch==1.33
jsonpointer==3.0.0
jsonschema==4.23.0
jsonschema-specifications==2023.12.1
langchain-core==0.2.28
langchain-text-splitters==0.2.2
langsmith==0.1.98
lark==1.1.9
llama-cpp-python==0.2.79
llvmlite==0.43.0
lm-eval==0.4.3
lm-format-enforcer==0.10.3
lxml==5.2.2
markdown-it-py==3.0.0
markupsafe==2.1.5
mbstrdecoder==1.1.3
mdurl==0.1.2
more-itertools==10.3.0
mpmath==1.3.0
multidict==6.0.5
multiprocess==0.70.16
nest-asyncio==1.6.0
networkx==3.3
ninja==1.11.1.1
nltk==3.8.1
numba==0.60.0
numexpr==2.10.1
# WARNING: fromager selected numpy==1.26.4 from: ['1.26.4', '2.0.0']
numpy==1.26.4
nvidia-ml-py==12.555.43
openai==1.40.0
orjson==3.10.6
outlines==0.0.46
packaging==24.1
pandas==2.2.1
pandas-stubs==2.2.2.240805
pathspec==0.12.1
pathvalidate==3.2.0
peft==0.12.0
pillow==10.4.0
platformdirs==4.2.2
portalocker==2.10.1
prometheus-client==0.20.0
prometheus-fastapi-instrumentator==7.0.0
prompt-toolkit==3.0.47
protobuf==4.25.4
psutil==6.0.0
py-cpuinfo==9.0.0
pyairports==2.1.1
pyarrow==17.0.0
pyarrow-hotfix==0.6
pybind11==2.13.1
pycountry==24.6.1
pydantic==2.8.2
pydantic-core==2.20.1
pydantic-settings==2.4.0
pydantic-yaml==1.3.0
pygments==2.18.0
pytablewriter==1.2.0
python-dateutil==2.9.0.post0
python-dotenv==1.0.1
pytz==2024.1
pyyaml==6.0.1
pyzmq==26.1.0
referencing==0.35.1
regex==2024.7.24
requests==2.32.3
rich==13.7.1
rouge-score==0.1.2
rpds-py==0.18.1
ruamel-yaml==0.18.6
ruamel-yaml-clib==0.2.8
sacrebleu==2.4.1
safetensors==0.4.4
scikit-learn==1.5.1
scipy==1.14.0
sentencepiece==0.2.0
sentry-sdk==2.12.0
setproctitle==1.3.3
setuptools==72.1.0
shortuuid==1.0.13
shtab==1.7.1
six==1.16.0
smmap==5.0.1
sniffio==1.3.1
sqlitedict==2.1.0
sse-starlette==2.1.3
# WARNING: fromager selected starlette==0.37.2 from: ['0.37.2', '0.38.2']
starlette==0.37.2
starlette-context==0.3.6
sympy==1.13.1
tabledata==1.3.3
tabulate==0.9.0
tcolorpy==0.1.6
# WARNING: fromager selected tenacity==8.5.0 from: ['8.5.0', '9.0.0']
tenacity==8.5.0
threadpoolctl==3.5.0
tiktoken==0.7.0
tokenizers==0.19.1
toml==0.10.2
torch==2.3.1
tqdm==4.66.5
tqdm-multiprocess==0.0.11
transformers==4.43.4
triton==2.3.1
trl==0.9.6
typepy==1.3.2
types-pytz==2024.1.0.20240417
typing-extensions==4.12.2
tyro==0.8.5
tzdata==2024.1
urllib3==2.2.2
uvicorn==0.30.5
uvloop==0.19.0
vllm==0.5.2.2
vllm-flash-attn==2.5.9.post1
wandb==0.16.6
watchfiles==0.22.0
wcwidth==0.2.13
websockets==12.0
wheel==0.44.0
word2number==1.1
xformers==0.0.27
xxhash==3.4.1
yamllint==1.35.1
yarl==1.9.4
zipp==3.19.2
zstandard==0.23.0

[shubhbapna]

Looks pretty good! I think later we can refactor and create a separate graph module that handles some of the graph operations and the related data types

[tiran]

Good work! I have added some minor suggestions around reporting and logging.

[shubhbapna]

just a nit pick, otherwise LGTM! Will wait for final approval from @tiran

[rd4398]

This looks good!
I will wait for other reviewers to approve and merge.