Build modules in chunks (#12067)

.github/workflows/build_pull_request.yml

@@ -5,10 +5,18 @@ on:
     paths-ignore:
       - '**.md'
 
+env:
+  CI_CHUNK_SIZE: 75
+
 jobs:
-  check_wrapper:
+  prepare:
-    name: Validate Gradle Wrapper
+    name: Prepare job
     runs-on: ubuntu-latest
+    outputs:
+      individualMatrix: ${{ steps.generate-matrices.outputs.individualMatrix }}
+      multisrcMatrix: ${{ steps.generate-matrices.outputs.multisrcMatrix }}
+    env:
+      CI_MODULE_GEN: true
     steps:
       - name: Clone repo
         uses: actions/checkout@v3
@@ -16,16 +24,54 @@ jobs:
       - name: Validate Gradle Wrapper
         uses: gradle/wrapper-validation-action@v1
 
+      - name: Set up JDK
+        uses: actions/setup-java@v3
+        with:
+          java-version: 8
+          distribution: adopt
+
+      - name: Copy CI files
+        run: |
+          mkdir -p ~/.gradle
+          cp .github/runner-files/ci-gradle.properties ~/.gradle/gradle.properties
+
+      - name: Generate multisrc sources
+        uses: gradle/gradle-command-action@v2
+        with:
+          arguments: :multisrc:generateExtensions
+
+      - name: Get number of modules
+        run: |
+          set -x
+          ./gradlew -q projects | grep '.*extensions\:\(individual\|multisrc\)\:.*\:.*' > projects.txt
+
+          echo "NUM_INDIVIDUAL_MODULES=$(cat projects.txt | grep '.*\:individual\:.*' | wc -l)" >> $GITHUB_ENV
+          echo "NUM_MULTISRC_MODULES=$(cat projects.txt | grep '.*\:multisrc\:.*' | wc -l)" >> $GITHUB_ENV
+
+      - id: generate-matrices
+        name: Create output matrices
+        uses: actions/github-script@v6
+        with:
+          script: |
+            const numIndividualModules = process.env.NUM_INDIVIDUAL_MODULES;
+            const numMultisrcModules = process.env.NUM_MULTISRC_MODULES;
+            const chunkSize = process.env.CI_CHUNK_SIZE;
+
+            const numIndividualChunks = Math.ceil(numIndividualModules / chunkSize);
+            const numMultisrcChunks = Math.ceil(numMultisrcModules / chunkSize);
+
+            console.log(`Individual modules: ${numIndividualModules} (${numIndividualChunks} chunks of ${chunkSize})`);
+            console.log(`Multi-source modules: ${numMultisrcModules} (${numMultisrcChunks} chunks of ${chunkSize})`);
+
+            core.setOutput('individualMatrix', { 'chunk': [...Array(numIndividualChunks).keys()] });
+            core.setOutput('multisrcMatrix', { 'chunk': [...Array(numMultisrcChunks).keys()] });
+
   build_multisrc:
     name: Build multisrc modules
-    needs: check_wrapper
+    needs: prepare
     runs-on: ubuntu-latest
     strategy:
-      fail-fast: false
+      matrix: ${{ fromJSON(needs.prepare.outputs.multisrcMatrix) }}
-      matrix:
-        lang: [all, en, ar, de, es, fr, id, it, ja, ko, pt, ru, th, tr, vi, zh, bg, hi, pl]
-        # Full list of locales in project, but no APKs are produced for some of them here:
-        # lang: [all, en, ar, ca, de, es, fr, id, it, ja, ko, pt, ru, th, tr, vi, zh, bg, hi, pl]
     steps:
       - name: Checkout PR
         uses: actions/checkout@v3
@@ -36,7 +82,7 @@ jobs:
           java-version: 8
           distribution: adopt
 
-      - name: Copy CI gradle.properties
+      - name: Copy CI files
         run: |
           mkdir -p ~/.gradle
           cp .github/runner-files/ci-gradle.properties ~/.gradle/gradle.properties
@@ -44,30 +90,26 @@ jobs:
       - name: Generate sources from the multi-source library
         uses: gradle/gradle-command-action@v2
         env:
-          CI_MULTISRC: "true"
+          CI_MODULE_GEN: "true"
         with:
           arguments: :multisrc:generateExtensions
           cache-read-only: true
 
-      - name: Build "${{ matrix.lang }}" extensions
+      - name: Build extensions (chunk ${{ matrix.chunk }})
         uses: gradle/gradle-command-action@v2
         env:
           CI_MULTISRC: "true"
-          CI_MATRIX_LANG: ${{ matrix.lang }}
+          CI_CHUNK_NUM: ${{ matrix.chunk }}
         with:
           arguments: assembleDebug
           cache-read-only: true
 
   build_individual:
     name: Build individual modules
-    needs: check_wrapper
+    needs: prepare
     runs-on: ubuntu-latest
     strategy:
-      fail-fast: false
+      matrix: ${{ fromJSON(needs.prepare.outputs.individualMatrix) }}
-      matrix:
-        lang: [all, en, ar, ca, de, es, fr, id, it, ja, ko, pt, ru, th, tr, vi, zh]
-        # Full list of locales in project, but no APKs are produced for some of them here:
-        # lang: [all, en, ar, ca, de, es, fr, id, it, ja, ko, pt, ru, th, tr, vi, zh, bg, hi, pl]
     steps:
       - name: Checkout PR
         uses: actions/checkout@v3
@@ -78,16 +120,16 @@ jobs:
           java-version: 8
           distribution: adopt
 
-      - name: Copy CI gradle.properties
+      - name: Copy CI files
         run: |
           mkdir -p ~/.gradle
           cp .github/runner-files/ci-gradle.properties ~/.gradle/gradle.properties
 
-      - name: Build "${{ matrix.lang }}" extensions
+      - name: Build extensions (chunk ${{ matrix.chunk }})
         uses: gradle/gradle-command-action@v2
         env:
           CI_MULTISRC: "false"
-          CI_MATRIX_LANG: ${{ matrix.lang }}
+          CI_CHUNK_NUM: ${{ matrix.chunk }}
         with:
           arguments: assembleDebug
           cache-read-only: true

.github/workflows/build_push.yml

@@ -7,10 +7,18 @@ on:
     paths-ignore:
       - '**.md'
 
+env:
+  CI_CHUNK_SIZE: 75
+
 jobs:
-  check_wrapper:
+  prepare:
-    name: Validate Gradle Wrapper
+    name: Prepare job
     runs-on: ubuntu-latest
+    outputs:
+      individualMatrix: ${{ steps.generate-matrices.outputs.individualMatrix }}
+      multisrcMatrix: ${{ steps.generate-matrices.outputs.multisrcMatrix }}
+    env:
+      CI_MODULE_GEN: true
     steps:
       - name: Cancel previous runs
         uses: styfle/cancel-workflow-action@0.9.1
@@ -24,16 +32,54 @@ jobs:
       - name: Validate Gradle Wrapper
         uses: gradle/wrapper-validation-action@v1
 
+      - name: Set up JDK
+        uses: actions/setup-java@v3
+        with:
+          java-version: 8
+          distribution: adopt
+
+      - name: Copy CI files
+        run: |
+          mkdir -p ~/.gradle
+          cp .github/runner-files/ci-gradle.properties ~/.gradle/gradle.properties
+
+      - name: Generate multisrc sources
+        uses: gradle/gradle-command-action@v2
+        with:
+          arguments: :multisrc:generateExtensions
+
+      - name: Get number of modules
+        run: |
+          set -x
+          ./gradlew -q projects | grep '.*extensions\:\(individual\|multisrc\)\:.*\:.*' > projects.txt
+
+          echo "NUM_INDIVIDUAL_MODULES=$(cat projects.txt | grep '.*\:individual\:.*' | wc -l)" >> $GITHUB_ENV
+          echo "NUM_MULTISRC_MODULES=$(cat projects.txt | grep '.*\:multisrc\:.*' | wc -l)" >> $GITHUB_ENV
+
+      - id: generate-matrices
+        name: Create output matrices
+        uses: actions/github-script@v6
+        with:
+          script: |
+            const numIndividualModules = process.env.NUM_INDIVIDUAL_MODULES;
+            const numMultisrcModules = process.env.NUM_MULTISRC_MODULES;
+            const chunkSize = process.env.CI_CHUNK_SIZE;
+
+            const numIndividualChunks = Math.ceil(numIndividualModules / chunkSize);
+            const numMultisrcChunks = Math.ceil(numMultisrcModules / chunkSize);
+
+            console.log(`Individual modules: ${numIndividualModules} (${numIndividualChunks} chunks of ${chunkSize})`);
+            console.log(`Multi-source modules: ${numMultisrcModules} (${numMultisrcChunks} chunks of ${chunkSize})`);
+
+            core.setOutput('individualMatrix', { 'chunk': [...Array(numIndividualChunks).keys()] });
+            core.setOutput('multisrcMatrix', { 'chunk': [...Array(numMultisrcChunks).keys()] });
+
   build_multisrc:
     name: Build multisrc modules
-    needs: check_wrapper
+    needs: prepare
     runs-on: ubuntu-latest
     strategy:
-      fail-fast: false
+      matrix: ${{ fromJSON(needs.prepare.outputs.multisrcMatrix) }}
-      matrix:
-        lang: [all, en, ar, de, es, fr, id, it, ja, ko, pt, ru, th, tr, vi, zh, bg, hi, pl]
-        # Full list of locales in project, but no APKs are produced for some of them here:
-        # lang: [all, en, ar, ca, de, es, fr, id, it, ja, ko, pt, ru, th, tr, vi, zh, bg, hi, pl]
     steps:
       - name: Checkout master branch
         uses: actions/checkout@v3
@@ -53,26 +99,26 @@ jobs:
       - name: Generate sources from the multi-source library
         uses: gradle/gradle-command-action@v2
         env:
-          CI_MULTISRC: "true"
+          CI_MODULE_GEN: "true"
         with:
-          arguments: :multisrc:generateExtensions -x ktFormat -x ktLint
+          arguments: :multisrc:generateExtensions
 
-      - name: Build "${{ matrix.lang }}" extensions
+      - name: Build extensions (chunk ${{ matrix.chunk }})
         uses: gradle/gradle-command-action@v2
         env:
           CI_MULTISRC: "true"
-          CI_MATRIX_LANG: ${{ matrix.lang }}
+          CI_CHUNK_NUM: ${{ matrix.chunk }}
           ALIAS: ${{ secrets.ALIAS }}
           KEY_STORE_PASSWORD: ${{ secrets.KEY_STORE_PASSWORD }}
           KEY_PASSWORD: ${{ secrets.KEY_PASSWORD }}
         with:
-          arguments: assembleRelease -x lintKotlin
+          arguments: assembleRelease
 
-      - name: Upload "${{ matrix.lang }}" APKs
+      - name: Upload APKs (chunk ${{ matrix.chunk }})
         uses: actions/upload-artifact@v2
         if: "github.repository == 'tachiyomiorg/tachiyomi-extensions'"
         with:
-          name: "multisrc-${{ matrix.lang }}-apks"
+          name: "multisrc-apks-${{ matrix.chunk }}"
           path: "**/*.apk"
           retention-days: 1
 
@@ -81,14 +127,10 @@ jobs:
 
   build_individual:
     name: Build individual modules
-    needs: check_wrapper
+    needs: prepare
     runs-on: ubuntu-latest
     strategy:
-      fail-fast: false
+      matrix: ${{ fromJSON(needs.prepare.outputs.individualMatrix) }}
-      matrix:
-        lang: [all, en, ar, ca, de, es, fr, id, it, ja, ko, pt, ru, th, tr, vi, zh]
-        # Full list of locales in project, but no APKs are produced for some of them here:
-        # lang: [all, en, ar, ca, de, es, fr, id, it, ja, ko, pt, ru, th, tr, vi, zh, bg, hi, pl]
     steps:
       - name: Checkout master branch
         uses: actions/checkout@v3
@@ -105,22 +147,22 @@ jobs:
           cp .github/runner-files/ci-gradle.properties ~/.gradle/gradle.properties
           echo ${{ secrets.SIGNING_KEY }} | base64 -d > signingkey.jks
 
-      - name: Build "${{ matrix.lang }}" extensions
+      - name: Build extensions (chunk ${{ matrix.chunk }})
         uses: gradle/gradle-command-action@v2
         env:
           CI_MULTISRC: "false"
-          CI_MATRIX_LANG: ${{ matrix.lang }}
+          CI_CHUNK_NUM: ${{ matrix.chunk }}
           ALIAS: ${{ secrets.ALIAS }}
           KEY_STORE_PASSWORD: ${{ secrets.KEY_STORE_PASSWORD }}
           KEY_PASSWORD: ${{ secrets.KEY_PASSWORD }}
         with:
-          arguments: assembleRelease -x lintKotlin
+          arguments: assembleRelease
 
-      - name: Upload "${{ matrix.lang }}" APKs
+      - name: Upload APKs (chunk ${{ matrix.chunk }})
         uses: actions/upload-artifact@v2
         if: "github.repository == 'tachiyomiorg/tachiyomi-extensions'"
         with:
-          name: "individual-${{ matrix.lang }}-apks"
+          name: "individual-apks-${{ matrix.chunk }}"
           path: "**/*.apk"
           retention-days: 1
 

settings.gradle.kts

@@ -6,7 +6,7 @@ project(":lib-ratelimit").projectDir = File("lib/ratelimit")
 include(":lib-dataimage")
 project(":lib-dataimage").projectDir = File("lib/dataimage")
 
-if (System.getenv("CI") == null) {
+if (System.getenv("CI") == null || System.getenv("CI_MODULE_GEN") == "true") {
     // Local development (full project build)
 
     include(":multisrc")
@@ -20,7 +20,7 @@ if (System.getenv("CI") == null) {
             project(name).projectDir = File("src/${dir.name}/${subdir.name}")
         }
     }
-    // Loads generated extensions from multisrc
+    // Loads all generated extensions from multisrc
     File(rootDir, "generated-src").eachDir { dir ->
         dir.eachDir { subdir ->
             val name = ":extensions:multisrc:${dir.name}:${subdir.name}"
@@ -44,36 +44,43 @@ if (System.getenv("CI") == null) {
     // Running in CI (GitHub Actions)
 
     val isMultisrc = System.getenv("CI_MULTISRC") == "true"
-    val lang = System.getenv("CI_MATRIX_LANG")
+    val chunkSize = System.getenv("CI_CHUNK_SIZE").toInt()
+    val chunk = System.getenv("CI_CHUNK_NUM").toInt()
 
     if (isMultisrc) {
         include(":multisrc")
         project(":multisrc").projectDir = File("multisrc")
 
         // Loads generated extensions from multisrc
-        File(rootDir, "generated-src").eachDir { dir ->
+        File(rootDir, "generated-src").getChunk(chunk, chunkSize)?.forEach {
-            if (dir.name == lang) {
+            val name = ":extensions:multisrc:${it.parentFile.name}:${it.name}"
-                dir.eachDir { subdir ->
+            println(name)
-                    val name = ":extensions:multisrc:${dir.name}:${subdir.name}"
+            include(name)
-                    include(name)
+            project(name).projectDir = File("generated-src/${it.parentFile.name}/${it.name}")
-                    project(name).projectDir = File("generated-src/${dir.name}/${subdir.name}")
-                }
-            }
         }
     } else {
-        // Loads all extensions
+        // Loads individual extensions
-        File(rootDir, "src").eachDir { dir ->
+        File(rootDir, "src").getChunk(chunk, chunkSize)?.forEach {
-            if (dir.name == lang) {
+            val name = ":extensions:individual:${it.parentFile.name}:${it.name}"
-                dir.eachDir { subdir ->
+            println(name)
-                    val name = ":extensions:individual:${dir.name}:${subdir.name}"
+            include(name)
-                    include(name)
+            project(name).projectDir = File("src/${it.parentFile.name}/${it.name}")
-                    project(name).projectDir = File("src/${dir.name}/${subdir.name}")
-                }
-            }
         }
     }
 }
 
-inline fun File.eachDir(block: (File) -> Unit) {
+fun File.getChunk(chunk: Int, chunkSize: Int): List<File>? {
+    return listFiles()
+        // Lang folder
+        ?.filter { it.isDirectory }
+        // Extension subfolders
+        ?.mapNotNull { dir -> dir.listFiles()?.filter { it.isDirectory } }
+        ?.flatten()
+        ?.sortedBy { it.name }
+        ?.chunked(chunkSize)
+        ?.get(chunk)
+}
+
+fun File.eachDir(block: (File) -> Unit) {
     listFiles()?.filter { it.isDirectory }?.forEach { block(it) }
 }