From d925c303939e6b6d7683cc1a6b26a42f4c7775a0 Mon Sep 17 00:00:00 2001 From: arkon Date: Sat, 4 Jun 2022 20:55:39 -0400 Subject: [PATCH] Build modules in chunks (#12067) --- .github/workflows/build_pull_request.yml | 84 +++++++++++++++------ .github/workflows/build_push.yml | 94 +++++++++++++++++------- settings.gradle.kts | 49 ++++++------ 3 files changed, 159 insertions(+), 68 deletions(-) diff --git a/.github/workflows/build_pull_request.yml b/.github/workflows/build_pull_request.yml index da5a1cb32..ea160a11f 100644 --- a/.github/workflows/build_pull_request.yml +++ b/.github/workflows/build_pull_request.yml @@ -5,10 +5,18 @@ on: paths-ignore: - '**.md' +env: + CI_CHUNK_SIZE: 75 + jobs: - check_wrapper: - name: Validate Gradle Wrapper + prepare: + name: Prepare job runs-on: ubuntu-latest + outputs: + individualMatrix: ${{ steps.generate-matrices.outputs.individualMatrix }} + multisrcMatrix: ${{ steps.generate-matrices.outputs.multisrcMatrix }} + env: + CI_MODULE_GEN: true steps: - name: Clone repo uses: actions/checkout@v3 @@ -16,16 +24,54 @@ jobs: - name: Validate Gradle Wrapper uses: gradle/wrapper-validation-action@v1 + - name: Set up JDK + uses: actions/setup-java@v3 + with: + java-version: 8 + distribution: adopt + + - name: Copy CI files + run: | + mkdir -p ~/.gradle + cp .github/runner-files/ci-gradle.properties ~/.gradle/gradle.properties + + - name: Generate multisrc sources + uses: gradle/gradle-command-action@v2 + with: + arguments: :multisrc:generateExtensions + + - name: Get number of modules + run: | + set -x + ./gradlew -q projects | grep '.*extensions\:\(individual\|multisrc\)\:.*\:.*' > projects.txt + + echo "NUM_INDIVIDUAL_MODULES=$(cat projects.txt | grep '.*\:individual\:.*' | wc -l)" >> $GITHUB_ENV + echo "NUM_MULTISRC_MODULES=$(cat projects.txt | grep '.*\:multisrc\:.*' | wc -l)" >> $GITHUB_ENV + + - id: generate-matrices + name: Create output matrices + uses: actions/github-script@v6 + with: + script: | + const numIndividualModules = process.env.NUM_INDIVIDUAL_MODULES; + const numMultisrcModules = process.env.NUM_MULTISRC_MODULES; + const chunkSize = process.env.CI_CHUNK_SIZE; + + const numIndividualChunks = Math.ceil(numIndividualModules / chunkSize); + const numMultisrcChunks = Math.ceil(numMultisrcModules / chunkSize); + + console.log(`Individual modules: ${numIndividualModules} (${numIndividualChunks} chunks of ${chunkSize})`); + console.log(`Multi-source modules: ${numMultisrcModules} (${numMultisrcChunks} chunks of ${chunkSize})`); + + core.setOutput('individualMatrix', { 'chunk': [...Array(numIndividualChunks).keys()] }); + core.setOutput('multisrcMatrix', { 'chunk': [...Array(numMultisrcChunks).keys()] }); + build_multisrc: name: Build multisrc modules - needs: check_wrapper + needs: prepare runs-on: ubuntu-latest strategy: - fail-fast: false - matrix: - lang: [all, en, ar, de, es, fr, id, it, ja, ko, pt, ru, th, tr, vi, zh, bg, hi, pl] - # Full list of locales in project, but no APKs are produced for some of them here: - # lang: [all, en, ar, ca, de, es, fr, id, it, ja, ko, pt, ru, th, tr, vi, zh, bg, hi, pl] + matrix: ${{ fromJSON(needs.prepare.outputs.multisrcMatrix) }} steps: - name: Checkout PR uses: actions/checkout@v3 @@ -36,7 +82,7 @@ jobs: java-version: 8 distribution: adopt - - name: Copy CI gradle.properties + - name: Copy CI files run: | mkdir -p ~/.gradle cp .github/runner-files/ci-gradle.properties ~/.gradle/gradle.properties @@ -44,30 +90,26 @@ jobs: - name: Generate sources from the multi-source library uses: gradle/gradle-command-action@v2 env: - CI_MULTISRC: "true" + CI_MODULE_GEN: "true" with: arguments: :multisrc:generateExtensions cache-read-only: true - - name: Build "${{ matrix.lang }}" extensions + - name: Build extensions (chunk ${{ matrix.chunk }}) uses: gradle/gradle-command-action@v2 env: CI_MULTISRC: "true" - CI_MATRIX_LANG: ${{ matrix.lang }} + CI_CHUNK_NUM: ${{ matrix.chunk }} with: arguments: assembleDebug cache-read-only: true build_individual: name: Build individual modules - needs: check_wrapper + needs: prepare runs-on: ubuntu-latest strategy: - fail-fast: false - matrix: - lang: [all, en, ar, ca, de, es, fr, id, it, ja, ko, pt, ru, th, tr, vi, zh] - # Full list of locales in project, but no APKs are produced for some of them here: - # lang: [all, en, ar, ca, de, es, fr, id, it, ja, ko, pt, ru, th, tr, vi, zh, bg, hi, pl] + matrix: ${{ fromJSON(needs.prepare.outputs.individualMatrix) }} steps: - name: Checkout PR uses: actions/checkout@v3 @@ -78,16 +120,16 @@ jobs: java-version: 8 distribution: adopt - - name: Copy CI gradle.properties + - name: Copy CI files run: | mkdir -p ~/.gradle cp .github/runner-files/ci-gradle.properties ~/.gradle/gradle.properties - - name: Build "${{ matrix.lang }}" extensions + - name: Build extensions (chunk ${{ matrix.chunk }}) uses: gradle/gradle-command-action@v2 env: CI_MULTISRC: "false" - CI_MATRIX_LANG: ${{ matrix.lang }} + CI_CHUNK_NUM: ${{ matrix.chunk }} with: arguments: assembleDebug cache-read-only: true diff --git a/.github/workflows/build_push.yml b/.github/workflows/build_push.yml index 3dc214fd8..c5c211e24 100644 --- a/.github/workflows/build_push.yml +++ b/.github/workflows/build_push.yml @@ -7,10 +7,18 @@ on: paths-ignore: - '**.md' +env: + CI_CHUNK_SIZE: 75 + jobs: - check_wrapper: - name: Validate Gradle Wrapper + prepare: + name: Prepare job runs-on: ubuntu-latest + outputs: + individualMatrix: ${{ steps.generate-matrices.outputs.individualMatrix }} + multisrcMatrix: ${{ steps.generate-matrices.outputs.multisrcMatrix }} + env: + CI_MODULE_GEN: true steps: - name: Cancel previous runs uses: styfle/cancel-workflow-action@0.9.1 @@ -24,16 +32,54 @@ jobs: - name: Validate Gradle Wrapper uses: gradle/wrapper-validation-action@v1 + - name: Set up JDK + uses: actions/setup-java@v3 + with: + java-version: 8 + distribution: adopt + + - name: Copy CI files + run: | + mkdir -p ~/.gradle + cp .github/runner-files/ci-gradle.properties ~/.gradle/gradle.properties + + - name: Generate multisrc sources + uses: gradle/gradle-command-action@v2 + with: + arguments: :multisrc:generateExtensions + + - name: Get number of modules + run: | + set -x + ./gradlew -q projects | grep '.*extensions\:\(individual\|multisrc\)\:.*\:.*' > projects.txt + + echo "NUM_INDIVIDUAL_MODULES=$(cat projects.txt | grep '.*\:individual\:.*' | wc -l)" >> $GITHUB_ENV + echo "NUM_MULTISRC_MODULES=$(cat projects.txt | grep '.*\:multisrc\:.*' | wc -l)" >> $GITHUB_ENV + + - id: generate-matrices + name: Create output matrices + uses: actions/github-script@v6 + with: + script: | + const numIndividualModules = process.env.NUM_INDIVIDUAL_MODULES; + const numMultisrcModules = process.env.NUM_MULTISRC_MODULES; + const chunkSize = process.env.CI_CHUNK_SIZE; + + const numIndividualChunks = Math.ceil(numIndividualModules / chunkSize); + const numMultisrcChunks = Math.ceil(numMultisrcModules / chunkSize); + + console.log(`Individual modules: ${numIndividualModules} (${numIndividualChunks} chunks of ${chunkSize})`); + console.log(`Multi-source modules: ${numMultisrcModules} (${numMultisrcChunks} chunks of ${chunkSize})`); + + core.setOutput('individualMatrix', { 'chunk': [...Array(numIndividualChunks).keys()] }); + core.setOutput('multisrcMatrix', { 'chunk': [...Array(numMultisrcChunks).keys()] }); + build_multisrc: name: Build multisrc modules - needs: check_wrapper + needs: prepare runs-on: ubuntu-latest strategy: - fail-fast: false - matrix: - lang: [all, en, ar, de, es, fr, id, it, ja, ko, pt, ru, th, tr, vi, zh, bg, hi, pl] - # Full list of locales in project, but no APKs are produced for some of them here: - # lang: [all, en, ar, ca, de, es, fr, id, it, ja, ko, pt, ru, th, tr, vi, zh, bg, hi, pl] + matrix: ${{ fromJSON(needs.prepare.outputs.multisrcMatrix) }} steps: - name: Checkout master branch uses: actions/checkout@v3 @@ -53,26 +99,26 @@ jobs: - name: Generate sources from the multi-source library uses: gradle/gradle-command-action@v2 env: - CI_MULTISRC: "true" + CI_MODULE_GEN: "true" with: - arguments: :multisrc:generateExtensions -x ktFormat -x ktLint + arguments: :multisrc:generateExtensions - - name: Build "${{ matrix.lang }}" extensions + - name: Build extensions (chunk ${{ matrix.chunk }}) uses: gradle/gradle-command-action@v2 env: CI_MULTISRC: "true" - CI_MATRIX_LANG: ${{ matrix.lang }} + CI_CHUNK_NUM: ${{ matrix.chunk }} ALIAS: ${{ secrets.ALIAS }} KEY_STORE_PASSWORD: ${{ secrets.KEY_STORE_PASSWORD }} KEY_PASSWORD: ${{ secrets.KEY_PASSWORD }} with: - arguments: assembleRelease -x lintKotlin + arguments: assembleRelease - - name: Upload "${{ matrix.lang }}" APKs + - name: Upload APKs (chunk ${{ matrix.chunk }}) uses: actions/upload-artifact@v2 if: "github.repository == 'tachiyomiorg/tachiyomi-extensions'" with: - name: "multisrc-${{ matrix.lang }}-apks" + name: "multisrc-apks-${{ matrix.chunk }}" path: "**/*.apk" retention-days: 1 @@ -81,14 +127,10 @@ jobs: build_individual: name: Build individual modules - needs: check_wrapper + needs: prepare runs-on: ubuntu-latest strategy: - fail-fast: false - matrix: - lang: [all, en, ar, ca, de, es, fr, id, it, ja, ko, pt, ru, th, tr, vi, zh] - # Full list of locales in project, but no APKs are produced for some of them here: - # lang: [all, en, ar, ca, de, es, fr, id, it, ja, ko, pt, ru, th, tr, vi, zh, bg, hi, pl] + matrix: ${{ fromJSON(needs.prepare.outputs.individualMatrix) }} steps: - name: Checkout master branch uses: actions/checkout@v3 @@ -105,22 +147,22 @@ jobs: cp .github/runner-files/ci-gradle.properties ~/.gradle/gradle.properties echo ${{ secrets.SIGNING_KEY }} | base64 -d > signingkey.jks - - name: Build "${{ matrix.lang }}" extensions + - name: Build extensions (chunk ${{ matrix.chunk }}) uses: gradle/gradle-command-action@v2 env: CI_MULTISRC: "false" - CI_MATRIX_LANG: ${{ matrix.lang }} + CI_CHUNK_NUM: ${{ matrix.chunk }} ALIAS: ${{ secrets.ALIAS }} KEY_STORE_PASSWORD: ${{ secrets.KEY_STORE_PASSWORD }} KEY_PASSWORD: ${{ secrets.KEY_PASSWORD }} with: - arguments: assembleRelease -x lintKotlin + arguments: assembleRelease - - name: Upload "${{ matrix.lang }}" APKs + - name: Upload APKs (chunk ${{ matrix.chunk }}) uses: actions/upload-artifact@v2 if: "github.repository == 'tachiyomiorg/tachiyomi-extensions'" with: - name: "individual-${{ matrix.lang }}-apks" + name: "individual-apks-${{ matrix.chunk }}" path: "**/*.apk" retention-days: 1 diff --git a/settings.gradle.kts b/settings.gradle.kts index 459698180..b09f644d5 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -6,7 +6,7 @@ project(":lib-ratelimit").projectDir = File("lib/ratelimit") include(":lib-dataimage") project(":lib-dataimage").projectDir = File("lib/dataimage") -if (System.getenv("CI") == null) { +if (System.getenv("CI") == null || System.getenv("CI_MODULE_GEN") == "true") { // Local development (full project build) include(":multisrc") @@ -20,7 +20,7 @@ if (System.getenv("CI") == null) { project(name).projectDir = File("src/${dir.name}/${subdir.name}") } } - // Loads generated extensions from multisrc + // Loads all generated extensions from multisrc File(rootDir, "generated-src").eachDir { dir -> dir.eachDir { subdir -> val name = ":extensions:multisrc:${dir.name}:${subdir.name}" @@ -44,36 +44,43 @@ if (System.getenv("CI") == null) { // Running in CI (GitHub Actions) val isMultisrc = System.getenv("CI_MULTISRC") == "true" - val lang = System.getenv("CI_MATRIX_LANG") + val chunkSize = System.getenv("CI_CHUNK_SIZE").toInt() + val chunk = System.getenv("CI_CHUNK_NUM").toInt() if (isMultisrc) { include(":multisrc") project(":multisrc").projectDir = File("multisrc") // Loads generated extensions from multisrc - File(rootDir, "generated-src").eachDir { dir -> - if (dir.name == lang) { - dir.eachDir { subdir -> - val name = ":extensions:multisrc:${dir.name}:${subdir.name}" - include(name) - project(name).projectDir = File("generated-src/${dir.name}/${subdir.name}") - } - } + File(rootDir, "generated-src").getChunk(chunk, chunkSize)?.forEach { + val name = ":extensions:multisrc:${it.parentFile.name}:${it.name}" + println(name) + include(name) + project(name).projectDir = File("generated-src/${it.parentFile.name}/${it.name}") } } else { - // Loads all extensions - File(rootDir, "src").eachDir { dir -> - if (dir.name == lang) { - dir.eachDir { subdir -> - val name = ":extensions:individual:${dir.name}:${subdir.name}" - include(name) - project(name).projectDir = File("src/${dir.name}/${subdir.name}") - } - } + // Loads individual extensions + File(rootDir, "src").getChunk(chunk, chunkSize)?.forEach { + val name = ":extensions:individual:${it.parentFile.name}:${it.name}" + println(name) + include(name) + project(name).projectDir = File("src/${it.parentFile.name}/${it.name}") } } } -inline fun File.eachDir(block: (File) -> Unit) { +fun File.getChunk(chunk: Int, chunkSize: Int): List? { + return listFiles() + // Lang folder + ?.filter { it.isDirectory } + // Extension subfolders + ?.mapNotNull { dir -> dir.listFiles()?.filter { it.isDirectory } } + ?.flatten() + ?.sortedBy { it.name } + ?.chunked(chunkSize) + ?.get(chunk) +} + +fun File.eachDir(block: (File) -> Unit) { listFiles()?.filter { it.isDirectory }?.forEach { block(it) } }