diff --git a/.github/actionlint.yml b/.github/actionlint.yml index bdd3901a37..ee319fef54 100644 --- a/.github/actionlint.yml +++ b/.github/actionlint.yml @@ -1,5 +1,4 @@ config-variables: - - KEEP_CACHE_WARM - PUSH_VERSION_COMMIT - UPDATE_TO_VERIFICATION - PYPI_PROJECT diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index b5033341ad..ea19bdfb13 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -78,6 +78,7 @@ permissions: {} jobs: process: + name: Process runs-on: ubuntu-latest outputs: origin: ${{ steps.process_inputs.outputs.origin }} @@ -145,7 +146,6 @@ jobs: 'runner': 'ubuntu-24.04-arm', 'qemu_platform': 'linux/arm/v7', 'onefile': False, - 'cache_requirements': True, 'update_to': 'yt-dlp/yt-dlp@2023.03.04', }], 'musllinux': [{ @@ -174,7 +174,6 @@ jobs: exe.setdefault('qemu_platform', None) exe.setdefault('onefile', True) exe.setdefault('onedir', True) - exe.setdefault('cache_requirements', False) exe.setdefault('python_version', os.environ['PYTHON_VERSION']) exe.setdefault('update_to', os.environ['UPDATE_TO']) if not any(INPUTS.get(key) for key in EXE_MAP): @@ -185,6 +184,7 @@ jobs: f.write(f'matrix={json.dumps(matrix)}') unix: + name: unix needs: [process] if: inputs.unix permissions: @@ -197,12 +197,12 @@ jobs: UPDATE_TO: yt-dlp/yt-dlp@2025.09.05 steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 with: fetch-depth: 0 # Needed for changelog persist-credentials: false - - uses: actions/setup-python@v6 + - uses: actions/setup-python@83679a892e2d95755f2dac6acb0bfd1e9ac5d548 # v6.1.0 with: python-version: "3.10" @@ -231,7 +231,7 @@ jobs: [[ "${version}" != "${downgraded_version}" ]] - name: Upload artifacts - uses: actions/upload-artifact@v6 + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 with: name: build-bin-${{ github.job }} path: | @@ -261,28 +261,16 @@ jobs: SKIP_ONEFILE_BUILD: ${{ (!matrix.onefile && '1') || '' }} steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 with: persist-credentials: false - - name: Cache requirements - if: matrix.cache_requirements - id: cache-venv - uses: actions/cache@v5 - env: - SEGMENT_DOWNLOAD_TIMEOUT_MINS: 1 - with: - path: | - venv - key: cache-reqs-${{ matrix.os }}_${{ matrix.arch }}-${{ github.ref }}-${{ needs.process.outputs.timestamp }} - restore-keys: | - cache-reqs-${{ matrix.os }}_${{ matrix.arch }}-${{ github.ref }}- - cache-reqs-${{ matrix.os }}_${{ matrix.arch }}- - - name: Set up QEMU if: matrix.qemu_platform - uses: docker/setup-qemu-action@v3 + uses: docker/setup-qemu-action@c7c53464625b32c7a7e944ae62b3e17d2b600130 # v3.7.0 with: + image: tonistiigi/binfmt:qemu-v10.0.4-56@sha256:30cc9a4d03765acac9be2ed0afc23af1ad018aed2c28ea4be8c2eb9afe03fbd1 + cache-image: false platforms: ${{ matrix.qemu_platform }} - name: Build executable @@ -306,7 +294,7 @@ jobs: docker compose up --build --exit-code-from "${SERVICE}" "${SERVICE}" - name: Upload artifacts - uses: actions/upload-artifact@v6 + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 with: name: build-bin-${{ matrix.os }}_${{ matrix.arch }} path: | @@ -314,6 +302,7 @@ jobs: compression-level: 0 macos: + name: macos needs: [process] if: inputs.macos permissions: @@ -326,25 +315,12 @@ jobs: UPDATE_TO: yt-dlp/yt-dlp@2025.09.05 steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 with: persist-credentials: false # NB: Building universal2 does not work with python from actions/setup-python - - name: Cache requirements - id: cache-venv - uses: actions/cache@v5 - env: - SEGMENT_DOWNLOAD_TIMEOUT_MINS: 1 - with: - path: | - ~/yt-dlp-build-venv - key: cache-reqs-${{ github.job }}-${{ github.ref }}-${{ needs.process.outputs.timestamp }} - restore-keys: | - cache-reqs-${{ github.job }}-${{ github.ref }}- - cache-reqs-${{ github.job }}- - - name: Install Requirements run: | brew install coreutils @@ -408,7 +384,7 @@ jobs: [[ "$version" != "$downgraded_version" ]] - name: Upload artifacts - uses: actions/upload-artifact@v6 + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 with: name: build-bin-${{ github.job }} path: | @@ -459,29 +435,15 @@ jobs: PYI_WHEEL: pyinstaller-${{ matrix.pyi_version }}-py3-none-${{ matrix.platform_tag }}.whl steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 with: persist-credentials: false - - uses: actions/setup-python@v6 + - uses: actions/setup-python@83679a892e2d95755f2dac6acb0bfd1e9ac5d548 # v6.1.0 with: python-version: ${{ matrix.python_version }} architecture: ${{ matrix.arch }} - - name: Cache requirements - id: cache-venv - if: matrix.arch == 'arm64' - uses: actions/cache@v5 - env: - SEGMENT_DOWNLOAD_TIMEOUT_MINS: 1 - with: - path: | - /yt-dlp-build-venv - key: ${{ env.BASE_CACHE_KEY }}-${{ github.ref }}-${{ needs.process.outputs.timestamp }} - restore-keys: | - ${{ env.BASE_CACHE_KEY }}-${{ github.ref }}- - ${{ env.BASE_CACHE_KEY }}- - - name: Install Requirements env: ARCH: ${{ matrix.arch }} @@ -489,6 +451,8 @@ jobs: PYI_HASH: ${{ matrix.pyi_hash }} shell: pwsh run: | + $ErrorActionPreference = "Stop" + $PSNativeCommandUseErrorActionPreference = $true python -m venv /yt-dlp-build-venv /yt-dlp-build-venv/Scripts/Activate.ps1 python -m pip install -U pip @@ -506,12 +470,16 @@ jobs: - name: Prepare shell: pwsh run: | + $ErrorActionPreference = "Stop" + $PSNativeCommandUseErrorActionPreference = $true python devscripts/update-version.py -c "${Env:CHANNEL}" -r "${Env:ORIGIN}" "${Env:VERSION}" python devscripts/make_lazy_extractors.py - name: Build shell: pwsh run: | + $ErrorActionPreference = "Stop" + $PSNativeCommandUseErrorActionPreference = $true /yt-dlp-build-venv/Scripts/Activate.ps1 python -m bundle.pyinstaller python -m bundle.pyinstaller --onedir @@ -521,6 +489,8 @@ jobs: if: vars.UPDATE_TO_VERIFICATION shell: pwsh run: | + $ErrorActionPreference = "Stop" + $PSNativeCommandUseErrorActionPreference = $true $name = "yt-dlp${Env:SUFFIX}" Copy-Item "./dist/${name}.exe" "./dist/${name}_downgraded.exe" $version = & "./dist/${name}.exe" --version @@ -531,7 +501,7 @@ jobs: } - name: Upload artifacts - uses: actions/upload-artifact@v6 + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 with: name: build-bin-${{ github.job }}-${{ matrix.arch }} path: | @@ -540,6 +510,7 @@ jobs: compression-level: 0 meta_files: + name: Metadata files needs: - process - unix @@ -550,13 +521,14 @@ jobs: runs-on: ubuntu-latest steps: - name: Download artifacts - uses: actions/download-artifact@v7 + uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 with: path: artifact pattern: build-bin-* merge-multiple: true - name: Make SHA2-SUMS files + shell: bash run: | cd ./artifact/ # make sure SHA sums are also printed to stdout @@ -618,7 +590,7 @@ jobs: done - name: Upload artifacts - uses: actions/upload-artifact@v6 + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 with: name: build-${{ github.job }} path: | diff --git a/.github/workflows/cache-warmer.yml b/.github/workflows/cache-warmer.yml deleted file mode 100644 index 4a5589c7a2..0000000000 --- a/.github/workflows/cache-warmer.yml +++ /dev/null @@ -1,25 +0,0 @@ -name: Keep cache warm -on: - workflow_dispatch: - schedule: - - cron: '0 22 1,6,11,16,21,27 * *' - -permissions: {} - -jobs: - build: - if: | - vars.KEEP_CACHE_WARM || github.event_name == 'workflow_dispatch' - permissions: - contents: read - uses: ./.github/workflows/build.yml - with: - version: '999999' - channel: stable - origin: ${{ github.repository }} - unix: false - linux: false - linux_armv7l: true - musllinux: false - macos: true - windows: true diff --git a/.github/workflows/challenge-tests.yml b/.github/workflows/challenge-tests.yml index fa684e6644..8a98545750 100644 --- a/.github/workflows/challenge-tests.yml +++ b/.github/workflows/challenge-tests.yml @@ -37,28 +37,30 @@ jobs: env: QJS_VERSION: '2025-04-26' # Earliest version with rope strings steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 with: persist-credentials: false - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v6 + uses: actions/setup-python@83679a892e2d95755f2dac6acb0bfd1e9ac5d548 # v6.1.0 with: python-version: ${{ matrix.python-version }} - name: Install Deno - uses: denoland/setup-deno@v2 + uses: denoland/setup-deno@e95548e56dfa95d4e1a28d6f422fafe75c4c26fb # v2.0.3 with: deno-version: '2.0.0' # minimum supported version - name: Install Bun - uses: oven-sh/setup-bun@v2 + uses: oven-sh/setup-bun@735343b667d3e6f658f44d0eca948eb6282f2b76 # v2.0.2 with: # minimum supported version is 1.0.31 but earliest available Windows version is 1.1.0 bun-version: ${{ (matrix.os == 'windows-latest' && '1.1.0') || '1.0.31' }} + no-cache: true - name: Install Node - uses: actions/setup-node@v6 + uses: actions/setup-node@395ad3262231945c25e8478fd5baf05154b1d79f # v6.1.0 with: node-version: '20.0' # minimum supported version - name: Install QuickJS (Linux) if: matrix.os == 'ubuntu-latest' + shell: bash run: | wget "https://bellard.org/quickjs/binary_releases/quickjs-linux-x86_64-${QJS_VERSION}.zip" -O quickjs.zip unzip quickjs.zip qjs @@ -67,15 +69,19 @@ jobs: if: matrix.os == 'windows-latest' shell: pwsh run: | + $ErrorActionPreference = "Stop" + $PSNativeCommandUseErrorActionPreference = $true Invoke-WebRequest "https://bellard.org/quickjs/binary_releases/quickjs-win-x86_64-${Env:QJS_VERSION}.zip" -OutFile quickjs.zip unzip quickjs.zip - name: Install test requirements + shell: bash run: | python ./devscripts/install_deps.py --print --omit-default --include-extra test > requirements.txt python ./devscripts/install_deps.py --print -c certifi -c requests -c urllib3 -c yt-dlp-ejs >> requirements.txt python -m pip install -U -r requirements.txt - name: Run tests timeout-minutes: 15 + shell: bash run: | python -m yt_dlp -v --js-runtimes node --js-runtimes bun --js-runtimes quickjs || true python ./devscripts/run_tests.py test/test_jsc -k download diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 5d0d06e095..c9eb40df41 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -11,14 +11,18 @@ on: permissions: {} +concurrency: + group: codeql-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + jobs: analyze: name: Analyze (${{ matrix.language }}) runs-on: ubuntu-latest permissions: - actions: read + actions: read # Needed by github/codeql-action if repository is private contents: read - security-events: write + security-events: write # Needed to use github/codeql-action with Github Advanced Security strategy: fail-fast: false @@ -27,17 +31,17 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v6 + uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 with: persist-credentials: false - name: Initialize CodeQL - uses: github/codeql-action/init@v4 + uses: github/codeql-action/init@5d4e8d1aca955e8d8589aabd499c5cae939e33c7 # v4.31.9 with: languages: ${{ matrix.language }} build-mode: none - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v4 + uses: github/codeql-action/analyze@5d4e8d1aca955e8d8589aabd499c5cae939e33c7 # v4.31.9 with: category: "/language:${{matrix.language}}" diff --git a/.github/workflows/core.yml b/.github/workflows/core.yml index 3ac9495cea..2d0dfae8a0 100644 --- a/.github/workflows/core.yml +++ b/.github/workflows/core.yml @@ -57,12 +57,12 @@ jobs: - os: windows-latest python-version: pypy-3.11 steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 with: fetch-depth: 0 persist-credentials: false - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v6 + uses: actions/setup-python@83679a892e2d95755f2dac6acb0bfd1e9ac5d548 # v6.1.0 with: python-version: ${{ matrix.python-version }} - name: Install test requirements diff --git a/.github/workflows/download.yml b/.github/workflows/download.yml deleted file mode 100644 index e9c4d75a71..0000000000 --- a/.github/workflows/download.yml +++ /dev/null @@ -1,56 +0,0 @@ -name: Download Tests -on: [push, pull_request] - -permissions: {} - -jobs: - quick: - name: Quick Download Tests - if: "contains(github.event.head_commit.message, 'ci run dl')" - permissions: - contents: read - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v6 - with: - persist-credentials: false - - name: Set up Python - uses: actions/setup-python@v6 - with: - python-version: '3.10' - - name: Install test requirements - run: python ./devscripts/install_deps.py --include-extra dev - - name: Run tests - continue-on-error: true - run: python ./devscripts/run_tests.py download - - full: - name: Full Download Tests - if: "contains(github.event.head_commit.message, 'ci run dl all')" - permissions: - contents: read - runs-on: ${{ matrix.os }} - strategy: - fail-fast: true - matrix: - os: [ubuntu-latest] - python-version: ['3.11', '3.12', '3.13', '3.14', pypy-3.11] - include: - # atleast one of each CPython/PyPy tests must be in windows - - os: windows-latest - python-version: '3.10' - - os: windows-latest - python-version: pypy-3.11 - steps: - - uses: actions/checkout@v6 - with: - persist-credentials: false - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v6 - with: - python-version: ${{ matrix.python-version }} - - name: Install test requirements - run: python ./devscripts/install_deps.py --include-extra dev - - name: Run tests - continue-on-error: true - run: python ./devscripts/run_tests.py download diff --git a/.github/workflows/issue-lockdown.yml b/.github/workflows/issue-lockdown.yml index 100b9f93f2..09f47ee622 100644 --- a/.github/workflows/issue-lockdown.yml +++ b/.github/workflows/issue-lockdown.yml @@ -10,7 +10,7 @@ jobs: name: Issue Lockdown if: vars.ISSUE_LOCKDOWN permissions: - issues: write + issues: write # Needed to lock issues runs-on: ubuntu-latest steps: - name: "Lock new issue" diff --git a/.github/workflows/quick-test.yml b/.github/workflows/quick-test.yml index 410bf3520f..7584790d7f 100644 --- a/.github/workflows/quick-test.yml +++ b/.github/workflows/quick-test.yml @@ -3,6 +3,10 @@ on: [push, pull_request] permissions: {} +concurrency: + group: quick-test-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + jobs: tests: name: Core Test @@ -11,17 +15,19 @@ jobs: contents: read runs-on: ubuntu-latest steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 with: persist-credentials: false - name: Set up Python 3.10 - uses: actions/setup-python@v6 + uses: actions/setup-python@83679a892e2d95755f2dac6acb0bfd1e9ac5d548 # v6.1.0 with: python-version: '3.10' - name: Install test requirements + shell: bash run: python ./devscripts/install_deps.py --omit-default --include-extra test - name: Run tests timeout-minutes: 15 + shell: bash run: | python3 -m yt_dlp -v || true python3 ./devscripts/run_tests.py --pytest-args '--reruns 2 --reruns-delay 3.0' core @@ -32,10 +38,10 @@ jobs: contents: read runs-on: ubuntu-latest steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 with: persist-credentials: false - - uses: actions/setup-python@v6 + - uses: actions/setup-python@83679a892e2d95755f2dac6acb0bfd1e9ac5d548 # v6.1.0 with: python-version: '3.10' - name: Install dev dependencies @@ -47,4 +53,5 @@ jobs: - name: Run autopep8 run: autopep8 --diff . - name: Check file mode + shell: bash run: git ls-files --format="%(objectmode) %(path)" yt_dlp/ | ( ! grep -v "^100644" ) diff --git a/.github/workflows/release-master.yml b/.github/workflows/release-master.yml index 0e7478e452..da8e75d696 100644 --- a/.github/workflows/release-master.yml +++ b/.github/workflows/release-master.yml @@ -19,30 +19,34 @@ permissions: {} jobs: release: + name: Publish Github release if: vars.BUILD_MASTER permissions: - contents: write - id-token: write # mandatory for trusted publishing + contents: write # May be needed to publish release + id-token: write # Needed for trusted publishing uses: ./.github/workflows/release.yml with: prerelease: true source: ${{ (github.repository != 'yt-dlp/yt-dlp' && vars.MASTER_ARCHIVE_REPO) || 'master' }} target: 'master' - secrets: inherit + secrets: + ARCHIVE_REPO_TOKEN: ${{ secrets.ARCHIVE_REPO_TOKEN }} + GPG_SIGNING_KEY: ${{ secrets.GPG_SIGNING_KEY }} publish_pypi: + name: Publish to PyPI needs: [release] if: vars.MASTER_PYPI_PROJECT permissions: - id-token: write # mandatory for trusted publishing + id-token: write # Needed for trusted publishing runs-on: ubuntu-latest steps: - name: Download artifacts - uses: actions/download-artifact@v7 + uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 with: path: dist name: build-pypi - name: Publish to PyPI - uses: pypa/gh-action-pypi-publish@release/v1 + uses: pypa/gh-action-pypi-publish@ed0c53931b1dc9bd32cbe73a98c7f6766f8a527e # v1.13.0 with: verbose: true diff --git a/.github/workflows/release-nightly.yml b/.github/workflows/release-nightly.yml index e0b7a3729c..a6026a199a 100644 --- a/.github/workflows/release-nightly.yml +++ b/.github/workflows/release-nightly.yml @@ -7,6 +7,7 @@ permissions: {} jobs: check_nightly: + name: Check for new commits if: vars.BUILD_NIGHTLY permissions: contents: read @@ -14,12 +15,13 @@ jobs: outputs: commit: ${{ steps.check_for_new_commits.outputs.commit }} steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 with: fetch-depth: 0 persist-credentials: false - name: Check for new commits id: check_for_new_commits + shell: bash run: | relevant_files=( "yt_dlp/*.py" @@ -36,31 +38,35 @@ jobs: echo "commit=$(git log --format=%H -1 --since="24 hours ago" -- "${relevant_files[@]}")" | tee "$GITHUB_OUTPUT" release: + name: Publish Github release needs: [check_nightly] if: ${{ needs.check_nightly.outputs.commit }} permissions: - contents: write - id-token: write # mandatory for trusted publishing + contents: write # May be needed to publish release + id-token: write # Needed for trusted publishing uses: ./.github/workflows/release.yml with: prerelease: true source: ${{ (github.repository != 'yt-dlp/yt-dlp' && vars.NIGHTLY_ARCHIVE_REPO) || 'nightly' }} target: 'nightly' - secrets: inherit + secrets: + ARCHIVE_REPO_TOKEN: ${{ secrets.ARCHIVE_REPO_TOKEN }} + GPG_SIGNING_KEY: ${{ secrets.GPG_SIGNING_KEY }} publish_pypi: + name: Publish to PyPI needs: [release] if: vars.NIGHTLY_PYPI_PROJECT permissions: - id-token: write # mandatory for trusted publishing + id-token: write # Needed for trusted publishing runs-on: ubuntu-latest steps: - name: Download artifacts - uses: actions/download-artifact@v7 + uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 with: path: dist name: build-pypi - name: Publish to PyPI - uses: pypa/gh-action-pypi-publish@release/v1 + uses: pypa/gh-action-pypi-publish@ed0c53931b1dc9bd32cbe73a98c7f6766f8a527e # v1.13.0 with: verbose: true diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index e06bd09f0d..584ac13be9 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -22,6 +22,11 @@ on: required: false default: true type: boolean + secrets: + ARCHIVE_REPO_TOKEN: + required: false + GPG_SIGNING_KEY: + required: false workflow_dispatch: inputs: source: @@ -60,25 +65,26 @@ permissions: {} jobs: prepare: + name: Prepare permissions: - contents: write + contents: write # Needed to git-push the release commit runs-on: ubuntu-latest outputs: channel: ${{ steps.setup_variables.outputs.channel }} version: ${{ steps.setup_variables.outputs.version }} target_repo: ${{ steps.setup_variables.outputs.target_repo }} - target_repo_token: ${{ steps.setup_variables.outputs.target_repo_token }} target_tag: ${{ steps.setup_variables.outputs.target_tag }} pypi_project: ${{ steps.setup_variables.outputs.pypi_project }} pypi_suffix: ${{ steps.setup_variables.outputs.pypi_suffix }} head_sha: ${{ steps.get_target.outputs.head_sha }} steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 with: fetch-depth: 0 + persist-credentials: true # Needed to git-push the release commit - - uses: actions/setup-python@v6 + - uses: actions/setup-python@83679a892e2d95755f2dac6acb0bfd1e9ac5d548 # v6.1.0 with: python-version: "3.10" # Keep this in sync with test-workflows.yml @@ -103,8 +109,6 @@ jobs: TARGET_PYPI_SUFFIX: ${{ vars[format('{0}_pypi_suffix', steps.process_inputs.outputs.target_repo)] }} SOURCE_ARCHIVE_REPO: ${{ vars[format('{0}_archive_repo', steps.process_inputs.outputs.source_repo)] }} TARGET_ARCHIVE_REPO: ${{ vars[format('{0}_archive_repo', steps.process_inputs.outputs.target_repo)] }} - HAS_SOURCE_ARCHIVE_REPO_TOKEN: ${{ !!secrets[format('{0}_archive_repo_token', steps.process_inputs.outputs.source_repo)] }} - HAS_TARGET_ARCHIVE_REPO_TOKEN: ${{ !!secrets[format('{0}_archive_repo_token', steps.process_inputs.outputs.target_repo)] }} HAS_ARCHIVE_REPO_TOKEN: ${{ !!secrets.ARCHIVE_REPO_TOKEN }} run: | python -m devscripts.setup_variables @@ -149,6 +153,7 @@ jobs: run: git push origin "${GITHUB_EVENT_REF}" build: + name: Build needs: [prepare] permissions: contents: read @@ -162,19 +167,20 @@ jobs: GPG_SIGNING_KEY: ${{ secrets.GPG_SIGNING_KEY }} publish_pypi: + name: Publish to PyPI needs: [prepare, build] if: ${{ needs.prepare.outputs.pypi_project }} permissions: contents: read - id-token: write # mandatory for trusted publishing + id-token: write # Needed for trusted publishing runs-on: ubuntu-latest steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 with: fetch-depth: 0 # Needed for changelog persist-credentials: false - - uses: actions/setup-python@v6 + - uses: actions/setup-python@83679a892e2d95755f2dac6acb0bfd1e9ac5d548 # v6.1.0 with: python-version: "3.10" @@ -210,7 +216,7 @@ jobs: - name: Upload artifacts if: github.event_name != 'workflow_dispatch' - uses: actions/upload-artifact@v6 + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 with: name: build-pypi path: | @@ -219,14 +225,15 @@ jobs: - name: Publish to PyPI if: github.event_name == 'workflow_dispatch' - uses: pypa/gh-action-pypi-publish@release/v1 + uses: pypa/gh-action-pypi-publish@ed0c53931b1dc9bd32cbe73a98c7f6766f8a527e # v1.13.0 with: verbose: true publish: + name: Publish Github release needs: [prepare, build] permissions: - contents: write + contents: write # Needed by gh to publish release to Github runs-on: ubuntu-latest env: TARGET_REPO: ${{ needs.prepare.outputs.target_repo }} @@ -234,16 +241,16 @@ jobs: VERSION: ${{ needs.prepare.outputs.version }} HEAD_SHA: ${{ needs.prepare.outputs.head_sha }} steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 with: fetch-depth: 0 persist-credentials: false - - uses: actions/download-artifact@v7 + - uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 with: path: artifact pattern: build-* merge-multiple: true - - uses: actions/setup-python@v6 + - uses: actions/setup-python@83679a892e2d95755f2dac6acb0bfd1e9ac5d548 # v6.1.0 with: python-version: "3.10" @@ -284,7 +291,7 @@ jobs: - name: Publish to archive repo env: - GH_TOKEN: ${{ secrets[needs.prepare.outputs.target_repo_token] }} + GH_TOKEN: ${{ secrets.ARCHIVE_REPO_TOKEN }} GH_REPO: ${{ needs.prepare.outputs.target_repo }} TITLE_PREFIX: ${{ startswith(env.TARGET_REPO, 'yt-dlp/') && 'yt-dlp ' || '' }} TITLE: ${{ inputs.target != env.TARGET_REPO && inputs.target || needs.prepare.outputs.channel }} diff --git a/.github/workflows/sanitize-comment.yml b/.github/workflows/sanitize-comment.yml index a14c5a8eb0..5faf4cb3f7 100644 --- a/.github/workflows/sanitize-comment.yml +++ b/.github/workflows/sanitize-comment.yml @@ -11,8 +11,8 @@ jobs: name: Sanitize comment if: vars.SANITIZE_COMMENT && !github.event.issue.pull_request permissions: - issues: write + issues: write # Needed by yt-dlp/sanitize-comment to edit comments runs-on: ubuntu-latest steps: - name: Sanitize comment - uses: yt-dlp/sanitize-comment@v1 + uses: yt-dlp/sanitize-comment@4536c691101b89f5373d50fe8a7980cae146346b # v1.0.0 diff --git a/.github/workflows/test-workflows.yml b/.github/workflows/test-workflows.yml index f82ce5b91a..8e0eba5ddd 100644 --- a/.github/workflows/test-workflows.yml +++ b/.github/workflows/test-workflows.yml @@ -1,14 +1,18 @@ name: Test and lint workflows on: push: + branches: [master] paths: + - .github/*.yml - .github/workflows/* - bundle/docker/linux/*.sh - devscripts/setup_variables.py - devscripts/setup_variables_tests.py - devscripts/utils.py pull_request: + branches: [master] paths: + - .github/*.yml - .github/workflows/* - bundle/docker/linux/*.sh - devscripts/setup_variables.py @@ -17,6 +21,10 @@ on: permissions: {} +concurrency: + group: test-workflows-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + env: ACTIONLINT_VERSION: "1.7.9" ACTIONLINT_SHA256SUM: 233b280d05e100837f4af1433c7b40a5dcb306e3aa68fb4f17f8a7f45a7df7b4 @@ -29,15 +37,16 @@ jobs: contents: read runs-on: ubuntu-latest steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 with: persist-credentials: false - - uses: actions/setup-python@v6 + - uses: actions/setup-python@83679a892e2d95755f2dac6acb0bfd1e9ac5d548 # v6.1.0 with: python-version: "3.10" # Keep this in sync with release.yml's prepare job - name: Install requirements env: ACTIONLINT_TARBALL: ${{ format('actionlint_{0}_linux_amd64.tar.gz', env.ACTIONLINT_VERSION) }} + shell: bash run: | python -m devscripts.install_deps --omit-default --include-extra test sudo apt -y install shellcheck @@ -55,3 +64,20 @@ jobs: - name: Test GHA devscripts run: | pytest -Werror --tb=short --color=yes devscripts/setup_variables_tests.py + + zizmor: + name: Run zizmor + permissions: + contents: read + actions: read # Needed by zizmorcore/zizmor-action if repository is private + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 + with: + persist-credentials: false + - name: Run zizmor + uses: zizmorcore/zizmor-action@e639db99335bc9038abc0e066dfcd72e23d26fb4 # v0.3.0 + with: + advanced-security: false + persona: pedantic + version: v1.19.0 diff --git a/.github/zizmor.yml b/.github/zizmor.yml new file mode 100644 index 0000000000..01645c87e8 --- /dev/null +++ b/.github/zizmor.yml @@ -0,0 +1,15 @@ +rules: + concurrency-limits: + ignore: + - build.yml # Can only be triggered by maintainers or cronjob + - issue-lockdown.yml # It *should* run for *every* new issue + - release-nightly.yml # Can only be triggered by once-daily cronjob + - release.yml # Can only be triggered by maintainers or cronjob + - sanitize-comment.yml # It *should* run for *every* new comment/edit + obfuscation: + ignore: + - release.yml # Not actual obfuscation + unpinned-uses: + config: + policies: + "*": hash-pin diff --git a/bundle/docker/compose.yml b/bundle/docker/compose.yml index 19a011d7a2..ee78eb4fcc 100644 --- a/bundle/docker/compose.yml +++ b/bundle/docker/compose.yml @@ -26,7 +26,7 @@ services: platforms: - "linux/amd64" args: - VERIFYIMAGE: quay.io/pypa/manylinux2014_x86_64:latest + VERIFYIMAGE: quay.io/pypa/manylinux2014_x86_64:2025.12.19-1@sha256:b716645f9aecd0c1418283af930804bbdbd68a73d855a60101c5aab8548d737d environment: EXE_NAME: ${EXE_NAME:?} UPDATE_TO: @@ -61,7 +61,7 @@ services: platforms: - "linux/arm64" args: - VERIFYIMAGE: quay.io/pypa/manylinux2014_aarch64:latest + VERIFYIMAGE: quay.io/pypa/manylinux2014_aarch64:2025.12.19-1@sha256:36cbe6638c7c605c2b44a92e35751baa537ec8902112f790139d89c7e1ccd2a4 environment: EXE_NAME: ${EXE_NAME:?} UPDATE_TO: @@ -97,7 +97,7 @@ services: platforms: - "linux/arm/v7" args: - VERIFYIMAGE: arm32v7/debian:bullseye + VERIFYIMAGE: arm32v7/debian:bullseye@sha256:9d544bf6ff73e36b8df1b7e415f6c8ee40ed84a0f3a26970cac8ea88b0ccf2ac environment: EXE_NAME: ${EXE_NAME:?} UPDATE_TO: @@ -132,7 +132,7 @@ services: platforms: - "linux/amd64" args: - VERIFYIMAGE: alpine:3.22 + VERIFYIMAGE: alpine:3.23.2@sha256:865b95f46d98cf867a156fe4a135ad3fe50d2056aa3f25ed31662dff6da4eb62 environment: EXE_NAME: ${EXE_NAME:?} UPDATE_TO: @@ -168,7 +168,7 @@ services: platforms: - "linux/arm64" args: - VERIFYIMAGE: alpine:3.22 + VERIFYIMAGE: alpine:3.23.2@sha256:865b95f46d98cf867a156fe4a135ad3fe50d2056aa3f25ed31662dff6da4eb62 environment: EXE_NAME: ${EXE_NAME:?} UPDATE_TO: diff --git a/devscripts/setup_variables.py b/devscripts/setup_variables.py index a45a36835c..a5bde4701c 100644 --- a/devscripts/setup_variables.py +++ b/devscripts/setup_variables.py @@ -21,8 +21,6 @@ def setup_variables(environment): SOURCE_PYPI_PROJECT, SOURCE_PYPI_SUFFIX, TARGET_PYPI_PROJECT, TARGET_PYPI_SUFFIX, SOURCE_ARCHIVE_REPO, TARGET_ARCHIVE_REPO, - HAS_SOURCE_ARCHIVE_REPO_TOKEN, - HAS_TARGET_ARCHIVE_REPO_TOKEN, HAS_ARCHIVE_REPO_TOKEN `INPUTS` must contain these keys: @@ -37,8 +35,6 @@ def setup_variables(environment): PROCESSED = json.loads(environment['PROCESSED']) source_channel = None - does_not_have_needed_token = False - target_repo_token = None pypi_project = None pypi_suffix = None @@ -81,28 +77,19 @@ def setup_variables(environment): target_repo = REPOSITORY if target_repo != REPOSITORY: target_repo = environment['TARGET_ARCHIVE_REPO'] - target_repo_token = f'{PROCESSED["target_repo"].upper()}_ARCHIVE_REPO_TOKEN' - if not json.loads(environment['HAS_TARGET_ARCHIVE_REPO_TOKEN']): - does_not_have_needed_token = True pypi_project = environment['TARGET_PYPI_PROJECT'] or None pypi_suffix = environment['TARGET_PYPI_SUFFIX'] or None else: target_tag = source_tag or version if source_channel: target_repo = source_channel - target_repo_token = f'{PROCESSED["source_repo"].upper()}_ARCHIVE_REPO_TOKEN' - if not json.loads(environment['HAS_SOURCE_ARCHIVE_REPO_TOKEN']): - does_not_have_needed_token = True pypi_project = environment['SOURCE_PYPI_PROJECT'] or None pypi_suffix = environment['SOURCE_PYPI_SUFFIX'] or None else: target_repo = REPOSITORY - if does_not_have_needed_token: - if not json.loads(environment['HAS_ARCHIVE_REPO_TOKEN']): - print(f'::error::Repository access secret {target_repo_token} not found') - return None - target_repo_token = 'ARCHIVE_REPO_TOKEN' + if target_repo != REPOSITORY and not json.loads(environment['HAS_ARCHIVE_REPO_TOKEN']): + return None if target_repo == REPOSITORY and not INPUTS['prerelease']: pypi_project = environment['PYPI_PROJECT'] or None @@ -111,7 +98,6 @@ def setup_variables(environment): 'channel': resolved_source, 'version': version, 'target_repo': target_repo, - 'target_repo_token': target_repo_token, 'target_tag': target_tag, 'pypi_project': pypi_project, 'pypi_suffix': pypi_suffix, @@ -147,6 +133,7 @@ if __name__ == '__main__': outputs = setup_variables(dict(os.environ)) if not outputs: + print('::error::Repository access secret ARCHIVE_REPO_TOKEN not found') sys.exit(1) print('::group::Output variables') diff --git a/devscripts/setup_variables_tests.py b/devscripts/setup_variables_tests.py index 42abba9d1f..22efe0a804 100644 --- a/devscripts/setup_variables_tests.py +++ b/devscripts/setup_variables_tests.py @@ -9,8 +9,10 @@ import json from devscripts.setup_variables import STABLE_REPOSITORY, process_inputs, setup_variables from devscripts.utils import calculate_version +GENERATE_TEST_DATA = object() -def _test(github_repository, note, repo_vars, repo_secrets, inputs, expected=None, ignore_revision=False): + +def _test(github_repository, note, repo_vars, repo_secrets, inputs, expected, ignore_revision=False): inp = inputs.copy() inp.setdefault('linux_armv7l', True) inp.setdefault('prerelease', False) @@ -33,16 +35,19 @@ def _test(github_repository, note, repo_vars, repo_secrets, inputs, expected=Non 'TARGET_PYPI_SUFFIX': variables.get(f'{target_repo}_PYPI_SUFFIX') or '', 'SOURCE_ARCHIVE_REPO': variables.get(f'{source_repo}_ARCHIVE_REPO') or '', 'TARGET_ARCHIVE_REPO': variables.get(f'{target_repo}_ARCHIVE_REPO') or '', - 'HAS_SOURCE_ARCHIVE_REPO_TOKEN': json.dumps(bool(secrets.get(f'{source_repo}_ARCHIVE_REPO_TOKEN'))), - 'HAS_TARGET_ARCHIVE_REPO_TOKEN': json.dumps(bool(secrets.get(f'{target_repo}_ARCHIVE_REPO_TOKEN'))), 'HAS_ARCHIVE_REPO_TOKEN': json.dumps(bool(secrets.get('ARCHIVE_REPO_TOKEN'))), } result = setup_variables(env) - if not expected: + + if expected is GENERATE_TEST_DATA: print(' {\n' + '\n'.join(f' {k!r}: {v!r},' for k, v in result.items()) + '\n }') return + if expected is None: + assert result is None, f'expected error/None but got dict: {github_repository} {note}' + return + exp = expected.copy() if ignore_revision: assert len(result['version']) == len(exp['version']), f'revision missing: {github_repository} {note}' @@ -77,7 +82,6 @@ def test_setup_variables(): 'channel': 'stable', 'version': DEFAULT_VERSION, 'target_repo': STABLE_REPOSITORY, - 'target_repo_token': None, 'target_tag': DEFAULT_VERSION, 'pypi_project': 'yt-dlp', 'pypi_suffix': None, @@ -91,7 +95,6 @@ def test_setup_variables(): 'channel': 'nightly', 'version': DEFAULT_VERSION_WITH_REVISION, 'target_repo': 'yt-dlp/yt-dlp-nightly-builds', - 'target_repo_token': 'ARCHIVE_REPO_TOKEN', 'target_tag': DEFAULT_VERSION_WITH_REVISION, 'pypi_project': 'yt-dlp', 'pypi_suffix': 'dev', @@ -106,7 +109,6 @@ def test_setup_variables(): 'channel': 'nightly', 'version': DEFAULT_VERSION_WITH_REVISION, 'target_repo': 'yt-dlp/yt-dlp-nightly-builds', - 'target_repo_token': 'ARCHIVE_REPO_TOKEN', 'target_tag': DEFAULT_VERSION_WITH_REVISION, 'pypi_project': 'yt-dlp', 'pypi_suffix': 'dev', @@ -120,7 +122,6 @@ def test_setup_variables(): 'channel': 'master', 'version': DEFAULT_VERSION_WITH_REVISION, 'target_repo': 'yt-dlp/yt-dlp-master-builds', - 'target_repo_token': 'ARCHIVE_REPO_TOKEN', 'target_tag': DEFAULT_VERSION_WITH_REVISION, 'pypi_project': None, 'pypi_suffix': None, @@ -135,7 +136,6 @@ def test_setup_variables(): 'channel': 'master', 'version': DEFAULT_VERSION_WITH_REVISION, 'target_repo': 'yt-dlp/yt-dlp-master-builds', - 'target_repo_token': 'ARCHIVE_REPO_TOKEN', 'target_tag': DEFAULT_VERSION_WITH_REVISION, 'pypi_project': None, 'pypi_suffix': None, @@ -149,7 +149,6 @@ def test_setup_variables(): 'channel': 'stable', 'version': DEFAULT_VERSION_WITH_REVISION, 'target_repo': STABLE_REPOSITORY, - 'target_repo_token': None, 'target_tag': 'experimental', 'pypi_project': None, 'pypi_suffix': None, @@ -163,7 +162,6 @@ def test_setup_variables(): 'channel': 'stable', 'version': DEFAULT_VERSION_WITH_REVISION, 'target_repo': STABLE_REPOSITORY, - 'target_repo_token': None, 'target_tag': 'experimental', 'pypi_project': None, 'pypi_suffix': None, @@ -175,7 +173,6 @@ def test_setup_variables(): 'channel': FORK_REPOSITORY, 'version': DEFAULT_VERSION_WITH_REVISION, 'target_repo': FORK_REPOSITORY, - 'target_repo_token': None, 'target_tag': DEFAULT_VERSION_WITH_REVISION, 'pypi_project': None, 'pypi_suffix': None, @@ -186,7 +183,6 @@ def test_setup_variables(): 'channel': FORK_REPOSITORY, 'version': DEFAULT_VERSION_WITH_REVISION, 'target_repo': FORK_REPOSITORY, - 'target_repo_token': None, 'target_tag': DEFAULT_VERSION_WITH_REVISION, 'pypi_project': None, 'pypi_suffix': None, @@ -201,7 +197,6 @@ def test_setup_variables(): 'channel': f'{FORK_REPOSITORY}@nightly', 'version': DEFAULT_VERSION_WITH_REVISION, 'target_repo': FORK_REPOSITORY, - 'target_repo_token': None, 'target_tag': 'nightly', 'pypi_project': None, 'pypi_suffix': None, @@ -216,7 +211,6 @@ def test_setup_variables(): 'channel': f'{FORK_REPOSITORY}@master', 'version': DEFAULT_VERSION_WITH_REVISION, 'target_repo': FORK_REPOSITORY, - 'target_repo_token': None, 'target_tag': 'master', 'pypi_project': None, 'pypi_suffix': None, @@ -227,7 +221,6 @@ def test_setup_variables(): 'channel': FORK_REPOSITORY, 'version': f'{DEFAULT_VERSION[:10]}.123', 'target_repo': FORK_REPOSITORY, - 'target_repo_token': None, 'target_tag': f'{DEFAULT_VERSION[:10]}.123', 'pypi_project': None, 'pypi_suffix': None, @@ -239,7 +232,6 @@ def test_setup_variables(): 'channel': FORK_REPOSITORY, 'version': DEFAULT_VERSION, 'target_repo': FORK_REPOSITORY, - 'target_repo_token': None, 'target_tag': DEFAULT_VERSION, 'pypi_project': None, 'pypi_suffix': None, @@ -250,19 +242,16 @@ def test_setup_variables(): 'channel': FORK_REPOSITORY, 'version': DEFAULT_VERSION_WITH_REVISION, 'target_repo': FORK_REPOSITORY, - 'target_repo_token': None, 'target_tag': DEFAULT_VERSION_WITH_REVISION, 'pypi_project': None, 'pypi_suffix': None, }, ignore_revision=True) _test( - FORK_REPOSITORY, 'fork w/NIGHTLY_ARCHIVE_REPO_TOKEN, nightly', { + FORK_REPOSITORY, 'fork, nightly', { 'NIGHTLY_ARCHIVE_REPO': f'{FORK_ORG}/yt-dlp-nightly-builds', 'PYPI_PROJECT': 'yt-dlp-test', - }, { - 'NIGHTLY_ARCHIVE_REPO_TOKEN': '1', - }, { + }, BASE_REPO_SECRETS, { 'source': f'{FORK_ORG}/yt-dlp-nightly-builds', 'target': 'nightly', 'prerelease': True, @@ -270,19 +259,16 @@ def test_setup_variables(): 'channel': f'{FORK_ORG}/yt-dlp-nightly-builds', 'version': DEFAULT_VERSION_WITH_REVISION, 'target_repo': f'{FORK_ORG}/yt-dlp-nightly-builds', - 'target_repo_token': 'NIGHTLY_ARCHIVE_REPO_TOKEN', 'target_tag': DEFAULT_VERSION_WITH_REVISION, 'pypi_project': None, 'pypi_suffix': None, }, ignore_revision=True) _test( - FORK_REPOSITORY, 'fork w/MASTER_ARCHIVE_REPO_TOKEN, master', { + FORK_REPOSITORY, 'fork, master', { 'MASTER_ARCHIVE_REPO': f'{FORK_ORG}/yt-dlp-master-builds', 'MASTER_PYPI_PROJECT': 'yt-dlp-test', 'MASTER_PYPI_SUFFIX': 'dev', - }, { - 'MASTER_ARCHIVE_REPO_TOKEN': '1', - }, { + }, BASE_REPO_SECRETS, { 'source': f'{FORK_ORG}/yt-dlp-master-builds', 'target': 'master', 'prerelease': True, @@ -290,7 +276,6 @@ def test_setup_variables(): 'channel': f'{FORK_ORG}/yt-dlp-master-builds', 'version': DEFAULT_VERSION_WITH_REVISION, 'target_repo': f'{FORK_ORG}/yt-dlp-master-builds', - 'target_repo_token': 'MASTER_ARCHIVE_REPO_TOKEN', 'target_tag': DEFAULT_VERSION_WITH_REVISION, 'pypi_project': 'yt-dlp-test', 'pypi_suffix': 'dev', @@ -302,7 +287,6 @@ def test_setup_variables(): 'channel': f'{FORK_REPOSITORY}@experimental', 'version': DEFAULT_VERSION_WITH_REVISION, 'target_repo': FORK_REPOSITORY, - 'target_repo_token': None, 'target_tag': 'experimental', 'pypi_project': None, 'pypi_suffix': None, @@ -317,8 +301,15 @@ def test_setup_variables(): 'channel': 'stable', 'version': DEFAULT_VERSION_WITH_REVISION, 'target_repo': FORK_REPOSITORY, - 'target_repo_token': None, 'target_tag': 'experimental', 'pypi_project': None, 'pypi_suffix': None, }, ignore_revision=True) + + _test( + STABLE_REPOSITORY, 'official vars but no ARCHIVE_REPO_TOKEN, nightly', + BASE_REPO_VARS, {}, { + 'source': 'nightly', + 'target': 'nightly', + 'prerelease': True, + }, None)