Skip to content

Commit

Permalink
[devscripts] install_deps: Add script and migrate to it
Browse files Browse the repository at this point in the history
Authored by: bashonly
  • Loading branch information
bashonly authored and Grub4K committed Feb 11, 2024
1 parent fd64777 commit b8a433a
Show file tree
Hide file tree
Showing 8 changed files with 95 additions and 35 deletions.
36 changes: 19 additions & 17 deletions .github/workflows/build.yml
Expand Up @@ -121,16 +121,14 @@ jobs:
- name: Install Requirements
run: |
sudo apt -y install zip pandoc man sed
reqs=$(mktemp)
cat > "$reqs" << EOF
cat > ./requirements.txt << EOF
python=3.10.*
pyinstaller
cffi
brotli-python
secretstorage
EOF
sed -E '/^(brotli|secretstorage).*/d' requirements.txt >> "$reqs"
mamba create -n build --file "$reqs"
python devscripts/install_deps.py --print \
--exclude brotli --exclude brotlicffi \
--include secretstorage --include pyinstaller >> ./requirements.txt
mamba create -n build --file ./requirements.txt
- name: Prepare
run: |
Expand Down Expand Up @@ -203,12 +201,13 @@ jobs:
apt update
apt -y install zlib1g-dev python3.8 python3.8-dev python3.8-distutils python3-pip
python3.8 -m pip install -U pip setuptools wheel
# Cannot access requirements.txt from the repo directory at this stage
# Cannot access any files from the repo directory at this stage
python3.8 -m pip install -U Pyinstaller mutagen pycryptodomex websockets brotli certifi secretstorage
run: |
cd repo
python3.8 -m pip install -U Pyinstaller secretstorage -r requirements.txt # Cached version may be out of date
python3.8 devscripts/install_deps.py -o --include build
python3.8 devscripts/install_deps.py --include pyinstaller --include secretstorage # Cached version may be out of date
python3.8 devscripts/update-version.py -c "${{ inputs.channel }}" -r "${{ needs.process.outputs.origin }}" "${{ inputs.version }}"
python3.8 devscripts/make_lazy_extractors.py
python3.8 -m bundle.pyinstaller
Expand Down Expand Up @@ -240,9 +239,10 @@ jobs:
- name: Install Requirements
run: |
brew install coreutils
python3 -m pip install -U --user pip setuptools wheel
python3 devscripts/install_deps.py --user -o --include build
python3 devscripts/install_deps.py --print --include pyinstaller > requirements.txt
# We need to ignore wheels otherwise we break universal2 builds
python3 -m pip install -U --user --no-binary :all: Pyinstaller -r requirements.txt
python3 -m pip install -U --user --no-binary :all: -r requirements.txt
- name: Prepare
run: |
Expand Down Expand Up @@ -293,8 +293,8 @@ jobs:
- name: Install Requirements
run: |
brew install coreutils
python3 -m pip install -U --user pip setuptools wheel
python3 -m pip install -U --user Pyinstaller -r requirements.txt
python3 devscripts/install_deps.py --user -o --include build
python3 devscripts/install_deps.py --user --include pyinstaller
- name: Prepare
run: |
Expand Down Expand Up @@ -333,8 +333,9 @@ jobs:
python-version: "3.8"
- name: Install Requirements
run: | # Custom pyinstaller built with https://github.com/yt-dlp/pyinstaller-builds
python -m pip install -U pip setuptools wheel py2exe
pip install -U "https://yt-dlp.github.io/Pyinstaller-Builds/x86_64/pyinstaller-5.8.0-py3-none-any.whl" -r requirements.txt
python devscripts/install_deps.py -o --include build
python devscripts/install_deps.py --include py2exe
python -m pip install -U "https://yt-dlp.github.io/Pyinstaller-Builds/x86_64/pyinstaller-5.8.0-py3-none-any.whl"
- name: Prepare
run: |
Expand Down Expand Up @@ -382,8 +383,9 @@ jobs:
architecture: "x86"
- name: Install Requirements
run: |
python -m pip install -U pip setuptools wheel
pip install -U "https://yt-dlp.github.io/Pyinstaller-Builds/i686/pyinstaller-5.8.0-py3-none-any.whl" -r requirements.txt
python devscripts/install_deps.py -o --include build
python devscripts/install_deps.py
python -m pip install -U "https://yt-dlp.github.io/Pyinstaller-Builds/i686/pyinstaller-5.8.0-py3-none-any.whl"
- name: Prepare
run: |
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/core.yml
Expand Up @@ -53,7 +53,7 @@ jobs:
with:
python-version: ${{ matrix.python-version }}
- name: Install test requirements
run: pip install pytest -r requirements.txt
run: python3 ./devscripts/install_deps.py --include dev
- name: Run tests
continue-on-error: False
run: |
Expand Down
4 changes: 2 additions & 2 deletions .github/workflows/download.yml
Expand Up @@ -15,7 +15,7 @@ jobs:
with:
python-version: 3.9
- name: Install test requirements
run: pip install pytest -r requirements.txt
run: python3 ./devscripts/install_deps.py --include dev
- name: Run tests
continue-on-error: true
run: python3 ./devscripts/run_tests.py download
Expand All @@ -42,7 +42,7 @@ jobs:
with:
python-version: ${{ matrix.python-version }}
- name: Install test requirements
run: pip install pytest -r requirements.txt
run: python3 ./devscripts/install_deps.py --include dev
- name: Run tests
continue-on-error: true
run: python3 ./devscripts/run_tests.py download
6 changes: 3 additions & 3 deletions .github/workflows/quick-test.yml
Expand Up @@ -15,7 +15,7 @@ jobs:
with:
python-version: '3.8'
- name: Install test requirements
run: pip install pytest -r requirements.txt
run: python3 ./devscripts/install_deps.py --include dev
- name: Run tests
run: |
python3 -m yt_dlp -v || true
Expand All @@ -28,8 +28,8 @@ jobs:
- uses: actions/checkout@v4
- uses: actions/setup-python@v4
- name: Install flake8
run: pip install flake8
run: python3 ./devscripts/install_deps.py -o --include dev
- name: Make lazy extractors
run: python devscripts/make_lazy_extractors.py
run: python3 ./devscripts/make_lazy_extractors.py
- name: Run flake8
run: flake8 .
3 changes: 1 addition & 2 deletions .github/workflows/release.yml
Expand Up @@ -253,8 +253,7 @@ jobs:
- name: Install Requirements
run: |
sudo apt -y install pandoc man
python -m pip install -U pip setuptools wheel twine
python -m pip install -U -r requirements.txt
python devscripts/install_deps.py -o --include build
- name: Prepare
env:
Expand Down
5 changes: 3 additions & 2 deletions README.md
Expand Up @@ -324,7 +324,7 @@ If you do not have the necessary dependencies for a task you are attempting, yt-
To build the standalone executable, you must have Python and `pyinstaller` (plus any of yt-dlp's [optional dependencies](#dependencies) if needed). The executable will be built for the same architecture (x86/ARM, 32/64 bit) as the Python used. You can run the following commands:

```
python3 -m pip install -U pyinstaller -r requirements.txt
python3 devscripts/install_deps.py --include pyinstaller
python3 devscripts/make_lazy_extractors.py
python3 -m bundle.pyinstaller
```
Expand All @@ -351,13 +351,14 @@ While we provide the option to build with [py2exe](https://www.py2exe.org), it i
If you wish to build it anyway, install Python (if it is not already installed) and you can run the following commands:

```
py -m pip install -U py2exe -r requirements.txt
py devscripts/install_deps.py --include py2exe
py devscripts/make_lazy_extractors.py
py -m bundle.py2exe
```

### Related scripts

* **`devscripts/install_deps.py`** - Install dependencies for yt-dlp.
* **`devscripts/update-version.py`** - Update the version number based on current date.
* **`devscripts/set-variant.py`** - Set the build variant of the executable.
* **`devscripts/make_changelog.py`** - Create a markdown changelog using short commit messages and update `CONTRIBUTORS` file.
Expand Down
66 changes: 66 additions & 0 deletions devscripts/install_deps.py
@@ -0,0 +1,66 @@
#!/usr/bin/env python3

# Allow execution from anywhere
import os
import sys

sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))

import argparse
import re
import subprocess

from devscripts.tomlparse import parse_toml
from devscripts.utils import read_file


def parse_args():
parser = argparse.ArgumentParser(description='Install dependencies for yt-dlp')
parser.add_argument(
'input', nargs='?', metavar='TOMLFILE', default='pyproject.toml', help='Input file (default: %(default)s)')
parser.add_argument(
'-e', '--exclude', metavar='REQUIREMENT', action='append', help='Exclude a required dependency')
parser.add_argument(
'-i', '--include', metavar='GROUP', action='append', help='Include an optional dependency group')
parser.add_argument(
'-o', '--only-optional', action='store_true', help='Only install optional dependencies')
parser.add_argument(
'-p', '--print', action='store_true', help='Only print a requirements.txt to stdout')
parser.add_argument(
'-u', '--user', action='store_true', help='Install with pip as --user')
return parser.parse_args()


def main():
args = parse_args()
toml_data = parse_toml(read_file(args.input))
deps = toml_data['project']['dependencies']
targets = deps.copy() if not args.only_optional else []

for exclude in args.exclude or []:
for dep in deps:
simplified_dep = re.match(r'[\w-]+', dep)[0]
if dep in targets and (exclude.lower() == simplified_dep.lower() or exclude == dep):
targets.remove(dep)

optional_deps = toml_data['project']['optional-dependencies']
for include in args.include or []:
group = optional_deps.get(include)
if group:
targets.extend(group)

if args.print:
for target in targets:
print(target)
return

pip_args = [sys.executable, '-m', 'pip', 'install', '-U']
if args.user:
pip_args.append('--user')
pip_args.extend(targets)

return subprocess.call(pip_args)


if __name__ == '__main__':
sys.exit(main())
8 changes: 0 additions & 8 deletions requirements.txt

This file was deleted.

0 comments on commit b8a433a

Please sign in to comment.