From a2eb11b42832b460c0fe40a1fe36d7d9863bb1d7 Mon Sep 17 00:00:00 2001
From: Simon Sawicki <contact@grub4k.xyz>
Date: Tue, 23 Jul 2024 17:03:56 +0200
Subject: [PATCH] [ci] Implement release workflow

---
 .github/workflows/build.yml   | 170 ++++++++++++++++++++++++++++++++++
 .github/workflows/release.yml | 147 +++++++++++++++++++++++++++++
 devscripts/changelog.py       | 100 ++++++++++++++++++++
 devscripts/update_version.py  |  46 +++++++++
 4 files changed, 463 insertions(+)
 create mode 100644 .github/workflows/build.yml
 create mode 100644 .github/workflows/release.yml
 create mode 100755 devscripts/changelog.py
 create mode 100755 devscripts/update_version.py

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
new file mode 100644
index 000000000..f394fba73
--- /dev/null
+++ b/.github/workflows/build.yml
@@ -0,0 +1,170 @@
+name: Build Artifacts
+
+on:
+  workflow_call:
+    inputs:
+      version:
+        required: true
+        type: string
+      unix:
+        default: true
+        type: boolean
+      windows32:
+        default: true
+        type: boolean
+
+  workflow_dispatch:
+    inputs:
+      version:
+        description: |
+          VERSION: yyyy.mm.dd[.rev] or rev
+        required: true
+        type: string
+      unix:
+        description: youtube-dl, youtube-dl.tar.gz
+        default: true
+        type: boolean
+      windows32:
+        description: youtube-dl.exe
+        default: true
+        type: boolean
+
+permissions:
+  contents: read
+
+jobs:
+  unix:
+    if: inputs.unix
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0  # Needed for changelog
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.9"
+
+      - name: Install Requirements
+        run: |
+          sudo apt -y install zip pandoc man sed
+
+      - name: Prepare
+        run: |
+          python devscripts/update_version.py "${{ inputs.version }}"
+          python devscripts/changelog.py --update
+          python devscripts/make_lazy_extractors.py youtube_dl/extractor/lazy_extractors.py
+
+      - name: Build Unix platform-independent binary
+        run: |
+          make all tar
+
+      - name: Upload artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: build-bin-${{ github.job }}
+          path: |
+            youtube-dl
+            youtube-dl.tar.gz
+          compression-level: 0
+
+  windows32:
+    if: inputs.windows32
+    runs-on: windows-2022
+    env:
+      PYCRYPTO: pycrypto-2.6.1-cp34-none-win32
+      # Temporary workaround for Python 3.4/5 failures - May 2024
+      PIP_TRUSTED_HOST: "pypi.python.org pypi.org files.pythonhosted.org"
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python 3.4
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.4"
+          architecture: x86
+
+      - name: Install packages
+        # https://pip.pypa.io/en/stable/news/#v19-2
+        # https://setuptools.pypa.io/en/latest/history.html#v44-0-0
+        # https://wheel.readthedocs.io/en/stable/news.html
+        # https://pypi.org/project/py2exe/0.9.2.2
+        shell: bash
+        run: |
+          python -m pip install --upgrade \
+            "pip<19.2" \
+            "setuptools<44" \
+            "wheel<0.34.0" \
+            "py2exe==0.9.2.2" \
+          ;
+
+      - name: PyCrypto cache
+        id: cache_pycrypto
+        uses: actions/cache@v4
+        with:
+          key: ${{ env.PYCRYPTO }}
+          path: ./${{ env.PYCRYPTO }}
+
+      - name: PyCrypto download
+        if: |
+          steps.cache_pycrypto.outputs.cache-hit != 'true'
+        shell: bash
+        run: |
+          mkdir -p "${PYCRYPTO}"
+          cd "${PYCRYPTO}"
+          curl -L -O "https://web.archive.org/web/20200627032153/http://www.voidspace.org.uk/python/pycrypto-2.6.1/${PYCRYPTO}.whl"
+
+      - name: PyCrypto install
+        shell: bash
+        run: |
+          python -m pip install "./${PYCRYPTO}/${PYCRYPTO}.whl"
+
+      - name: Prepare
+        run: |
+          python devscripts/update_version.py "${{ inputs.version }}"
+          python devscripts/make_lazy_extractors.py youtube_dl/extractor/lazy_extractors.py
+
+      - name: Build binary
+        run: python setup.py py2exe
+
+      - name: Upload artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: build-bin-${{ github.job }}
+          path: |
+            youtube-dl.exe
+          compression-level: 0
+
+  meta_files:
+    if: always() && !cancelled()
+    needs:
+      - unix
+      - windows32
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/download-artifact@v4
+        with:
+          path: artifact
+          pattern: build-bin-*
+          merge-multiple: true
+
+      - name: Make SHA2-SUMS files
+        run: |
+          cd ./artifact/
+          # make sure SHA sums are also printed to stdout
+          sha256sum -- * | tee ../SHA2-256SUMS
+          sha512sum -- * | tee ../SHA2-512SUMS
+          # also print as permanent annotations to the summary page
+          while read -r shasum; do
+            echo "::notice title=${shasum##* }::sha256: ${shasum% *}"
+          done < ../SHA2-256SUMS
+
+      - name: Upload artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: build-${{ github.job }}
+          path: |
+            SHA*SUMS*
+          compression-level: 0
+          overwrite: true
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
new file mode 100644
index 000000000..2a441a3a8
--- /dev/null
+++ b/.github/workflows/release.yml
@@ -0,0 +1,147 @@
+name: Release
+
+on:
+  workflow_dispatch:
+    inputs:
+      version:
+        description: |
+          VERSION: yyyy.mm.dd[.rev] or rev
+          (default: auto-generated)
+        required: false
+        default: ""
+        type: string
+      prerelease:
+        description: Pre-release
+        default: false
+        type: boolean
+
+jobs:
+  prepare:
+    permissions:
+      contents: write
+    runs-on: ubuntu-latest
+    outputs:
+      version: ${{ steps.setup_variables.outputs.version }}
+      head_sha: ${{ steps.get_target.outputs.head_sha }}
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - uses: actions/setup-python@v5
+        with:
+            python-version: "3.10"
+
+      - name: Setup variables
+        id: setup_variables
+        run: |
+          revision="${{ (inputs.prerelease || !vars.PUSH_VERSION_COMMIT) && '$(date -u +"%H%M%S")' || '' }}"
+          version="$(
+            python devscripts/update_version.py \
+            ${{ inputs.version || '"${revision}"' }} )"
+          echo "::group::Output variables"
+          cat << EOF | tee -a "$GITHUB_OUTPUT"
+          version=${version}
+          EOF
+          echo "::endgroup::"
+
+      - name: Update documentation
+        env:
+          version: ${{ steps.setup_variables.outputs.version }}
+          target_repo: ${{ steps.setup_variables.outputs.target_repo }}
+        if: |
+          !inputs.prerelease
+        run: |
+          python devscripts/changelog.py --update
+          make README.md
+          make issuetemplates
+          make supportedsites
+
+      - name: Push to release
+        id: push_release
+        env:
+          version: ${{ steps.setup_variables.outputs.version }}
+          creator: ${{ github.event.sender.login }}
+        if: |
+          !inputs.prerelease
+        run: |
+          git config --global user.name "github-actions[bot]"
+          git config --global user.email "41898282+github-actions[bot]@users.noreply.github.com"
+          git add -u
+          git commit -m "Release ${version}" \
+            -m "Created by: ${creator}" \
+            -m ":ci skip all"
+          git push origin --force master:release
+
+      - name: Get target commitish
+        id: get_target
+        run: |
+          echo "head_sha=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT"
+
+      - name: Update master
+        env:
+          target_repo: ${{ steps.setup_variables.outputs.target_repo }}
+        if: |
+          vars.PUSH_VERSION_COMMIT != '' && !inputs.prerelease
+        run: |
+          git push origin ${{ github.event.ref }}
+
+  build:
+    needs: prepare
+    uses: ./.github/workflows/build.yml
+    with:
+      version: ${{ needs.prepare.outputs.version }}
+    permissions:
+      contents: read
+
+  publish:
+    needs: [prepare, build]
+    permissions:
+      contents: write
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - uses: actions/download-artifact@v4
+        with:
+          path: artifact
+          pattern: build-*
+          merge-multiple: true
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.10"
+
+      - name: Generate release notes
+        run: |
+          cat >> ./RELEASE_NOTES << EOF
+          <details><summary><h3>Changelog</h3></summary>
+
+          $(python devscripts/changelog.py)
+
+          </details>
+          EOF
+          cat > ./PRERELEASE_NOTES << EOF
+          **This is a pre-release build**
+          ---
+
+          $(cat ./RELEASE_NOTES)
+          EOF
+
+      - name: Publish release
+        env:
+          GH_TOKEN: ${{ github.token }}
+          version: ${{ needs.prepare.outputs.version }}
+          head_sha: ${{ needs.prepare.outputs.head_sha }}
+        run: |
+          gh release create \
+            --notes-file ${{ inputs.prerelease && 'PRERELEASE_NOTES' || 'RELEASE_NOTES' }} \
+            --target ${{ env.head_sha }} \
+            --title "youtube-dl ${version}" \
+            ${{ inputs.prerelease && '--prerelease' || '' }} \
+            "${version}" \
+            artifact/*
diff --git a/devscripts/changelog.py b/devscripts/changelog.py
new file mode 100755
index 000000000..097328ede
--- /dev/null
+++ b/devscripts/changelog.py
@@ -0,0 +1,100 @@
+#!/usr/bin/env python
+from __future__ import unicode_literals
+
+import os
+import subprocess
+import sys
+
+
+def run(args):
+    process = subprocess.Popen(args, stdout=subprocess.PIPE, universal_newlines=True)
+    return process.communicate()[0].strip()
+
+
+def is_core(short):
+    prefix = None
+    if ']' in short:
+        prefix = short.partition(']')[0][1:]
+    elif ': ' in short:
+        prefix = short.partition(': ')[0]
+
+    if not prefix or ' ' in prefix:
+        return True
+
+    prefix = prefix.partition(':')[0].lower()
+    if prefix.startswith('extractor/'):
+        prefix = prefix[len('extractor/'):]
+    if prefix.endswith('ie'):
+        prefix = prefix[:-len('ie')]
+    return not os.path.exists('youtube_dl/extractor/%s.py' % prefix)
+
+
+def format_line(markdown, short, sha):
+    if not markdown:
+        return '* ' + short
+
+    return '* [%s](https://github.com/ytdl-org/youtube-dl/commit/%s)' % (short, sha)
+
+
+def generate_changelog(markdown):
+    most_recent_tag = run([
+        'git', 'tag', '--list', '--sort=-v:refname',
+        '????.??.??', '????.??.??.?',
+    ]).split('\n')[0]
+    lines = run([
+        'git', 'log',
+        '--format=format:%H%n%s', '--no-merges', '-z',
+        most_recent_tag + '..HEAD',
+    ]).split('\x00')
+
+    core = []
+    extractor = []
+    for line in lines:
+        if not line:
+            continue
+        sha, short = line.split('\n')
+
+        if ' * ' in short:
+            short = short.partition(' * ')[0]
+
+        target = core if is_core(short) else extractor
+        target.append((sha, short))
+
+    result = []
+    if core:
+        result.append('#### Core' if markdown else 'Core')
+        for sha, short in core:
+            result.append(format_line(markdown, short, sha))
+        result.append('')
+
+    if extractor:
+        result.append('#### Extractor' if markdown else 'Extractor')
+        for sha, short in extractor:
+            result.append(format_line(markdown, short, sha))
+        result.append('')
+
+    return '\n'.join(result)
+
+
+def read_version():
+    with open('youtube_dl/version.py', 'r') as f:
+        exec(compile(f.read(), 'youtube_dl/version.py', 'exec'))
+
+    return locals()['__version__']
+
+
+update_in_place = len(sys.argv) > 1 and sys.argv[1] == '--update'
+changelog = generate_changelog(not update_in_place)
+
+if not update_in_place:
+    print(changelog)
+    sys.exit()
+
+with open('ChangeLog', 'rb') as file:
+    data = file.read()
+
+with open('ChangeLog', 'wb') as file:
+    file.write(('version %s\n\n' % read_version()).encode('utf-8'))
+    file.write(changelog.encode('utf-8'))
+    file.write('\n\n'.encode('utf-8'))
+    file.write(data)
diff --git a/devscripts/update_version.py b/devscripts/update_version.py
new file mode 100755
index 000000000..c39c98966
--- /dev/null
+++ b/devscripts/update_version.py
@@ -0,0 +1,46 @@
+#!/usr/bin/env python
+from __future__ import unicode_literals
+
+import datetime as dt
+import sys
+
+
+VERSION_FILE_FORMAT = '''\
+# Autogenerated by devscripts/update_version.py
+from __future__ import unicode_literals
+
+__version__ = {!r}
+'''
+
+
+def split_version(version):
+    if '.' not in version:
+        return None, version
+
+    version_list = version.split('.')
+    version = '.'.join(version_list[:3])
+    revision = version_list[3] if len(version_list) > 3 else None
+
+    return version, revision
+
+
+with open('youtube_dl/version.py', 'r') as f:
+    exec(compile(f.read(), 'youtube_dl/version.py', 'exec'))
+
+old_ver, old_rev = split_version(locals()['__version__'])
+ver, rev = split_version(sys.argv[1]) if len(sys.argv) > 1 else (None, None)
+
+if not ver:
+    ver = (
+        dt.datetime.now(dt.timezone.utc) if sys.version_info >= (3,)
+        else dt.datetime.utcnow()).strftime('%Y.%m.%d')
+    if not rev and old_ver == ver:
+        rev = str(int(old_rev or 0) + 1)
+
+if rev:
+    ver = ver + '.' + rev
+
+with open('youtube_dl/version.py', 'w') as f:
+    f.write(VERSION_FILE_FORMAT.format(ver))
+
+print(ver)