diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 000000000..f394fba73 --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,170 @@ +name: Build Artifacts + +on: + workflow_call: + inputs: + version: + required: true + type: string + unix: + default: true + type: boolean + windows32: + default: true + type: boolean + + workflow_dispatch: + inputs: + version: + description: | + VERSION: yyyy.mm.dd[.rev] or rev + required: true + type: string + unix: + description: youtube-dl, youtube-dl.tar.gz + default: true + type: boolean + windows32: + description: youtube-dl.exe + default: true + type: boolean + +permissions: + contents: read + +jobs: + unix: + if: inputs.unix + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 # Needed for changelog + + - uses: actions/setup-python@v5 + with: + python-version: "3.9" + + - name: Install Requirements + run: | + sudo apt -y install zip pandoc man sed + + - name: Prepare + run: | + python devscripts/update_version.py "${{ inputs.version }}" + python devscripts/changelog.py --update + python devscripts/make_lazy_extractors.py youtube_dl/extractor/lazy_extractors.py + + - name: Build Unix platform-independent binary + run: | + make all tar + + - name: Upload artifacts + uses: actions/upload-artifact@v4 + with: + name: build-bin-${{ github.job }} + path: | + youtube-dl + youtube-dl.tar.gz + compression-level: 0 + + windows32: + if: inputs.windows32 + runs-on: windows-2022 + env: + PYCRYPTO: pycrypto-2.6.1-cp34-none-win32 + # Temporary workaround for Python 3.4/5 failures - May 2024 + PIP_TRUSTED_HOST: "pypi.python.org pypi.org files.pythonhosted.org" + + steps: + - uses: actions/checkout@v4 + + - name: Set up Python 3.4 + uses: actions/setup-python@v5 + with: + python-version: "3.4" + architecture: x86 + + - name: Install packages + # https://pip.pypa.io/en/stable/news/#v19-2 + # https://setuptools.pypa.io/en/latest/history.html#v44-0-0 + # https://wheel.readthedocs.io/en/stable/news.html + # https://pypi.org/project/py2exe/0.9.2.2 + shell: bash + run: | + python -m pip install --upgrade \ + "pip<19.2" \ + "setuptools<44" \ + "wheel<0.34.0" \ + "py2exe==0.9.2.2" \ + ; + + - name: PyCrypto cache + id: cache_pycrypto + uses: actions/cache@v4 + with: + key: ${{ env.PYCRYPTO }} + path: ./${{ env.PYCRYPTO }} + + - name: PyCrypto download + if: | + steps.cache_pycrypto.outputs.cache-hit != 'true' + shell: bash + run: | + mkdir -p "${PYCRYPTO}" + cd "${PYCRYPTO}" + curl -L -O "https://web.archive.org/web/20200627032153/http://www.voidspace.org.uk/python/pycrypto-2.6.1/${PYCRYPTO}.whl" + + - name: PyCrypto install + shell: bash + run: | + python -m pip install "./${PYCRYPTO}/${PYCRYPTO}.whl" + + - name: Prepare + run: | + python devscripts/update_version.py "${{ inputs.version }}" + python devscripts/make_lazy_extractors.py youtube_dl/extractor/lazy_extractors.py + + - name: Build binary + run: python setup.py py2exe + + - name: Upload artifacts + uses: actions/upload-artifact@v4 + with: + name: build-bin-${{ github.job }} + path: | + youtube-dl.exe + compression-level: 0 + + meta_files: + if: always() && !cancelled() + needs: + - unix + - windows32 + runs-on: ubuntu-latest + steps: + - uses: actions/download-artifact@v4 + with: + path: artifact + pattern: build-bin-* + merge-multiple: true + + - name: Make SHA2-SUMS files + run: | + cd ./artifact/ + # make sure SHA sums are also printed to stdout + sha256sum -- * | tee ../SHA2-256SUMS + sha512sum -- * | tee ../SHA2-512SUMS + # also print as permanent annotations to the summary page + while read -r shasum; do + echo "::notice title=${shasum##* }::sha256: ${shasum% *}" + done < ../SHA2-256SUMS + + - name: Upload artifacts + uses: actions/upload-artifact@v4 + with: + name: build-${{ github.job }} + path: | + SHA*SUMS* + compression-level: 0 + overwrite: true diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 000000000..2a441a3a8 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,147 @@ +name: Release + +on: + workflow_dispatch: + inputs: + version: + description: | + VERSION: yyyy.mm.dd[.rev] or rev + (default: auto-generated) + required: false + default: "" + type: string + prerelease: + description: Pre-release + default: false + type: boolean + +jobs: + prepare: + permissions: + contents: write + runs-on: ubuntu-latest + outputs: + version: ${{ steps.setup_variables.outputs.version }} + head_sha: ${{ steps.get_target.outputs.head_sha }} + + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - uses: actions/setup-python@v5 + with: + python-version: "3.10" + + - name: Setup variables + id: setup_variables + run: | + revision="${{ (inputs.prerelease || !vars.PUSH_VERSION_COMMIT) && '$(date -u +"%H%M%S")' || '' }}" + version="$( + python devscripts/update_version.py \ + ${{ inputs.version || '"${revision}"' }} )" + echo "::group::Output variables" + cat << EOF | tee -a "$GITHUB_OUTPUT" + version=${version} + EOF + echo "::endgroup::" + + - name: Update documentation + env: + version: ${{ steps.setup_variables.outputs.version }} + target_repo: ${{ steps.setup_variables.outputs.target_repo }} + if: | + !inputs.prerelease + run: | + python devscripts/changelog.py --update + make README.md + make issuetemplates + make supportedsites + + - name: Push to release + id: push_release + env: + version: ${{ steps.setup_variables.outputs.version }} + creator: ${{ github.event.sender.login }} + if: | + !inputs.prerelease + run: | + git config --global user.name "github-actions[bot]" + git config --global user.email "41898282+github-actions[bot]@users.noreply.github.com" + git add -u + git commit -m "Release ${version}" \ + -m "Created by: ${creator}" \ + -m ":ci skip all" + git push origin --force master:release + + - name: Get target commitish + id: get_target + run: | + echo "head_sha=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT" + + - name: Update master + env: + target_repo: ${{ steps.setup_variables.outputs.target_repo }} + if: | + vars.PUSH_VERSION_COMMIT != '' && !inputs.prerelease + run: | + git push origin ${{ github.event.ref }} + + build: + needs: prepare + uses: ./.github/workflows/build.yml + with: + version: ${{ needs.prepare.outputs.version }} + permissions: + contents: read + + publish: + needs: [prepare, build] + permissions: + contents: write + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - uses: actions/download-artifact@v4 + with: + path: artifact + pattern: build-* + merge-multiple: true + + - uses: actions/setup-python@v5 + with: + python-version: "3.10" + + - name: Generate release notes + run: | + cat >> ./RELEASE_NOTES << EOF +

Changelog

+ + $(python devscripts/changelog.py) + +
+ EOF + cat > ./PRERELEASE_NOTES << EOF + **This is a pre-release build** + --- + + $(cat ./RELEASE_NOTES) + EOF + + - name: Publish release + env: + GH_TOKEN: ${{ github.token }} + version: ${{ needs.prepare.outputs.version }} + head_sha: ${{ needs.prepare.outputs.head_sha }} + run: | + gh release create \ + --notes-file ${{ inputs.prerelease && 'PRERELEASE_NOTES' || 'RELEASE_NOTES' }} \ + --target ${{ env.head_sha }} \ + --title "youtube-dl ${version}" \ + ${{ inputs.prerelease && '--prerelease' || '' }} \ + "${version}" \ + artifact/* diff --git a/devscripts/changelog.py b/devscripts/changelog.py new file mode 100755 index 000000000..097328ede --- /dev/null +++ b/devscripts/changelog.py @@ -0,0 +1,100 @@ +#!/usr/bin/env python +from __future__ import unicode_literals + +import os +import subprocess +import sys + + +def run(args): + process = subprocess.Popen(args, stdout=subprocess.PIPE, universal_newlines=True) + return process.communicate()[0].strip() + + +def is_core(short): + prefix = None + if ']' in short: + prefix = short.partition(']')[0][1:] + elif ': ' in short: + prefix = short.partition(': ')[0] + + if not prefix or ' ' in prefix: + return True + + prefix = prefix.partition(':')[0].lower() + if prefix.startswith('extractor/'): + prefix = prefix[len('extractor/'):] + if prefix.endswith('ie'): + prefix = prefix[:-len('ie')] + return not os.path.exists('youtube_dl/extractor/%s.py' % prefix) + + +def format_line(markdown, short, sha): + if not markdown: + return '* ' + short + + return '* [%s](https://github.com/ytdl-org/youtube-dl/commit/%s)' % (short, sha) + + +def generate_changelog(markdown): + most_recent_tag = run([ + 'git', 'tag', '--list', '--sort=-v:refname', + '????.??.??', '????.??.??.?', + ]).split('\n')[0] + lines = run([ + 'git', 'log', + '--format=format:%H%n%s', '--no-merges', '-z', + most_recent_tag + '..HEAD', + ]).split('\x00') + + core = [] + extractor = [] + for line in lines: + if not line: + continue + sha, short = line.split('\n') + + if ' * ' in short: + short = short.partition(' * ')[0] + + target = core if is_core(short) else extractor + target.append((sha, short)) + + result = [] + if core: + result.append('#### Core' if markdown else 'Core') + for sha, short in core: + result.append(format_line(markdown, short, sha)) + result.append('') + + if extractor: + result.append('#### Extractor' if markdown else 'Extractor') + for sha, short in extractor: + result.append(format_line(markdown, short, sha)) + result.append('') + + return '\n'.join(result) + + +def read_version(): + with open('youtube_dl/version.py', 'r') as f: + exec(compile(f.read(), 'youtube_dl/version.py', 'exec')) + + return locals()['__version__'] + + +update_in_place = len(sys.argv) > 1 and sys.argv[1] == '--update' +changelog = generate_changelog(not update_in_place) + +if not update_in_place: + print(changelog) + sys.exit() + +with open('ChangeLog', 'rb') as file: + data = file.read() + +with open('ChangeLog', 'wb') as file: + file.write(('version %s\n\n' % read_version()).encode('utf-8')) + file.write(changelog.encode('utf-8')) + file.write('\n\n'.encode('utf-8')) + file.write(data) diff --git a/devscripts/update_version.py b/devscripts/update_version.py new file mode 100755 index 000000000..c39c98966 --- /dev/null +++ b/devscripts/update_version.py @@ -0,0 +1,46 @@ +#!/usr/bin/env python +from __future__ import unicode_literals + +import datetime as dt +import sys + + +VERSION_FILE_FORMAT = '''\ +# Autogenerated by devscripts/update_version.py +from __future__ import unicode_literals + +__version__ = {!r} +''' + + +def split_version(version): + if '.' not in version: + return None, version + + version_list = version.split('.') + version = '.'.join(version_list[:3]) + revision = version_list[3] if len(version_list) > 3 else None + + return version, revision + + +with open('youtube_dl/version.py', 'r') as f: + exec(compile(f.read(), 'youtube_dl/version.py', 'exec')) + +old_ver, old_rev = split_version(locals()['__version__']) +ver, rev = split_version(sys.argv[1]) if len(sys.argv) > 1 else (None, None) + +if not ver: + ver = ( + dt.datetime.now(dt.timezone.utc) if sys.version_info >= (3,) + else dt.datetime.utcnow()).strftime('%Y.%m.%d') + if not rev and old_ver == ver: + rev = str(int(old_rev or 0) + 1) + +if rev: + ver = ver + '.' + rev + +with open('youtube_dl/version.py', 'w') as f: + f.write(VERSION_FILE_FORMAT.format(ver)) + +print(ver)