Build modules in chunks (#12067)
This commit is contained in:
arkon
GitHub
parent
42e6a5064f
commit
d925c30393
|
|
@ -5,10 +5,18 @@ on:
|
|||
paths-ignore:
|
||||
- '**.md'
|
||||
|
||||
env:
|
||||
CI_CHUNK_SIZE: 75
|
||||
|
||||
jobs:
|
||||
check_wrapper:
|
||||
name: Validate Gradle Wrapper
|
||||
prepare:
|
||||
name: Prepare job
|
||||
runs-on: ubuntu-latest
|
||||
outputs:
|
||||
individualMatrix: ${{ steps.generate-matrices.outputs.individualMatrix }}
|
||||
multisrcMatrix: ${{ steps.generate-matrices.outputs.multisrcMatrix }}
|
||||
env:
|
||||
CI_MODULE_GEN: true
|
||||
steps:
|
||||
- name: Clone repo
|
||||
uses: actions/checkout@v3
|
||||
|
|
@ -16,16 +24,54 @@ jobs:
|
|||
- name: Validate Gradle Wrapper
|
||||
uses: gradle/wrapper-validation-action@v1
|
||||
|
||||
- name: Set up JDK
|
||||
uses: actions/setup-java@v3
|
||||
with:
|
||||
java-version: 8
|
||||
distribution: adopt
|
||||
|
||||
- name: Copy CI files
|
||||
run: |
|
||||
mkdir -p ~/.gradle
|
||||
cp .github/runner-files/ci-gradle.properties ~/.gradle/gradle.properties
|
||||
|
||||
- name: Generate multisrc sources
|
||||
uses: gradle/gradle-command-action@v2
|
||||
with:
|
||||
arguments: :multisrc:generateExtensions
|
||||
|
||||
- name: Get number of modules
|
||||
run: |
|
||||
set -x
|
||||
./gradlew -q projects | grep '.*extensions\:\(individual\|multisrc\)\:.*\:.*' > projects.txt
|
||||
|
||||
echo "NUM_INDIVIDUAL_MODULES=$(cat projects.txt | grep '.*\:individual\:.*' | wc -l)" >> $GITHUB_ENV
|
||||
echo "NUM_MULTISRC_MODULES=$(cat projects.txt | grep '.*\:multisrc\:.*' | wc -l)" >> $GITHUB_ENV
|
||||
|
||||
- id: generate-matrices
|
||||
name: Create output matrices
|
||||
uses: actions/github-script@v6
|
||||
with:
|
||||
script: |
|
||||
const numIndividualModules = process.env.NUM_INDIVIDUAL_MODULES;
|
||||
const numMultisrcModules = process.env.NUM_MULTISRC_MODULES;
|
||||
const chunkSize = process.env.CI_CHUNK_SIZE;
|
||||
|
||||
const numIndividualChunks = Math.ceil(numIndividualModules / chunkSize);
|
||||
const numMultisrcChunks = Math.ceil(numMultisrcModules / chunkSize);
|
||||
|
||||
console.log(`Individual modules: ${numIndividualModules} (${numIndividualChunks} chunks of ${chunkSize})`);
|
||||
console.log(`Multi-source modules: ${numMultisrcModules} (${numMultisrcChunks} chunks of ${chunkSize})`);
|
||||
|
||||
core.setOutput('individualMatrix', { 'chunk': [...Array(numIndividualChunks).keys()] });
|
||||
core.setOutput('multisrcMatrix', { 'chunk': [...Array(numMultisrcChunks).keys()] });
|
||||
|
||||
build_multisrc:
|
||||
name: Build multisrc modules
|
||||
needs: check_wrapper
|
||||
needs: prepare
|
||||
runs-on: ubuntu-latest
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
lang: [all, en, ar, de, es, fr, id, it, ja, ko, pt, ru, th, tr, vi, zh, bg, hi, pl]
|
||||
# Full list of locales in project, but no APKs are produced for some of them here:
|
||||
# lang: [all, en, ar, ca, de, es, fr, id, it, ja, ko, pt, ru, th, tr, vi, zh, bg, hi, pl]
|
||||
matrix: ${{ fromJSON(needs.prepare.outputs.multisrcMatrix) }}
|
||||
steps:
|
||||
- name: Checkout PR
|
||||
uses: actions/checkout@v3
|
||||
|
|
@ -36,7 +82,7 @@ jobs:
|
|||
java-version: 8
|
||||
distribution: adopt
|
||||
|
||||
- name: Copy CI gradle.properties
|
||||
- name: Copy CI files
|
||||
run: |
|
||||
mkdir -p ~/.gradle
|
||||
cp .github/runner-files/ci-gradle.properties ~/.gradle/gradle.properties
|
||||
|
|
@ -44,30 +90,26 @@ jobs:
|
|||
- name: Generate sources from the multi-source library
|
||||
uses: gradle/gradle-command-action@v2
|
||||
env:
|
||||
CI_MULTISRC: "true"
|
||||
CI_MODULE_GEN: "true"
|
||||
with:
|
||||
arguments: :multisrc:generateExtensions
|
||||
cache-read-only: true
|
||||
|
||||
- name: Build "${{ matrix.lang }}" extensions
|
||||
- name: Build extensions (chunk ${{ matrix.chunk }})
|
||||
uses: gradle/gradle-command-action@v2
|
||||
env:
|
||||
CI_MULTISRC: "true"
|
||||
CI_MATRIX_LANG: ${{ matrix.lang }}
|
||||
CI_CHUNK_NUM: ${{ matrix.chunk }}
|
||||
with:
|
||||
arguments: assembleDebug
|
||||
cache-read-only: true
|
||||
|
||||
build_individual:
|
||||
name: Build individual modules
|
||||
needs: check_wrapper
|
||||
needs: prepare
|
||||
runs-on: ubuntu-latest
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
lang: [all, en, ar, ca, de, es, fr, id, it, ja, ko, pt, ru, th, tr, vi, zh]
|
||||
# Full list of locales in project, but no APKs are produced for some of them here:
|
||||
# lang: [all, en, ar, ca, de, es, fr, id, it, ja, ko, pt, ru, th, tr, vi, zh, bg, hi, pl]
|
||||
matrix: ${{ fromJSON(needs.prepare.outputs.individualMatrix) }}
|
||||
steps:
|
||||
- name: Checkout PR
|
||||
uses: actions/checkout@v3
|
||||
|
|
@ -78,16 +120,16 @@ jobs:
|
|||
java-version: 8
|
||||
distribution: adopt
|
||||
|
||||
- name: Copy CI gradle.properties
|
||||
- name: Copy CI files
|
||||
run: |
|
||||
mkdir -p ~/.gradle
|
||||
cp .github/runner-files/ci-gradle.properties ~/.gradle/gradle.properties
|
||||
|
||||
- name: Build "${{ matrix.lang }}" extensions
|
||||
- name: Build extensions (chunk ${{ matrix.chunk }})
|
||||
uses: gradle/gradle-command-action@v2
|
||||
env:
|
||||
CI_MULTISRC: "false"
|
||||
CI_MATRIX_LANG: ${{ matrix.lang }}
|
||||
CI_CHUNK_NUM: ${{ matrix.chunk }}
|
||||
with:
|
||||
arguments: assembleDebug
|
||||
cache-read-only: true
|
||||
|
|
|
|||
|
|
@ -7,10 +7,18 @@ on:
|
|||
paths-ignore:
|
||||
- '**.md'
|
||||
|
||||
env:
|
||||
CI_CHUNK_SIZE: 75
|
||||
|
||||
jobs:
|
||||
check_wrapper:
|
||||
name: Validate Gradle Wrapper
|
||||
prepare:
|
||||
name: Prepare job
|
||||
runs-on: ubuntu-latest
|
||||
outputs:
|
||||
individualMatrix: ${{ steps.generate-matrices.outputs.individualMatrix }}
|
||||
multisrcMatrix: ${{ steps.generate-matrices.outputs.multisrcMatrix }}
|
||||
env:
|
||||
CI_MODULE_GEN: true
|
||||
steps:
|
||||
- name: Cancel previous runs
|
||||
uses: styfle/cancel-workflow-action@0.9.1
|
||||
|
|
@ -24,16 +32,54 @@ jobs:
|
|||
- name: Validate Gradle Wrapper
|
||||
uses: gradle/wrapper-validation-action@v1
|
||||
|
||||
- name: Set up JDK
|
||||
uses: actions/setup-java@v3
|
||||
with:
|
||||
java-version: 8
|
||||
distribution: adopt
|
||||
|
||||
- name: Copy CI files
|
||||
run: |
|
||||
mkdir -p ~/.gradle
|
||||
cp .github/runner-files/ci-gradle.properties ~/.gradle/gradle.properties
|
||||
|
||||
- name: Generate multisrc sources
|
||||
uses: gradle/gradle-command-action@v2
|
||||
with:
|
||||
arguments: :multisrc:generateExtensions
|
||||
|
||||
- name: Get number of modules
|
||||
run: |
|
||||
set -x
|
||||
./gradlew -q projects | grep '.*extensions\:\(individual\|multisrc\)\:.*\:.*' > projects.txt
|
||||
|
||||
echo "NUM_INDIVIDUAL_MODULES=$(cat projects.txt | grep '.*\:individual\:.*' | wc -l)" >> $GITHUB_ENV
|
||||
echo "NUM_MULTISRC_MODULES=$(cat projects.txt | grep '.*\:multisrc\:.*' | wc -l)" >> $GITHUB_ENV
|
||||
|
||||
- id: generate-matrices
|
||||
name: Create output matrices
|
||||
uses: actions/github-script@v6
|
||||
with:
|
||||
script: |
|
||||
const numIndividualModules = process.env.NUM_INDIVIDUAL_MODULES;
|
||||
const numMultisrcModules = process.env.NUM_MULTISRC_MODULES;
|
||||
const chunkSize = process.env.CI_CHUNK_SIZE;
|
||||
|
||||
const numIndividualChunks = Math.ceil(numIndividualModules / chunkSize);
|
||||
const numMultisrcChunks = Math.ceil(numMultisrcModules / chunkSize);
|
||||
|
||||
console.log(`Individual modules: ${numIndividualModules} (${numIndividualChunks} chunks of ${chunkSize})`);
|
||||
console.log(`Multi-source modules: ${numMultisrcModules} (${numMultisrcChunks} chunks of ${chunkSize})`);
|
||||
|
||||
core.setOutput('individualMatrix', { 'chunk': [...Array(numIndividualChunks).keys()] });
|
||||
core.setOutput('multisrcMatrix', { 'chunk': [...Array(numMultisrcChunks).keys()] });
|
||||
|
||||
build_multisrc:
|
||||
name: Build multisrc modules
|
||||
needs: check_wrapper
|
||||
needs: prepare
|
||||
runs-on: ubuntu-latest
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
lang: [all, en, ar, de, es, fr, id, it, ja, ko, pt, ru, th, tr, vi, zh, bg, hi, pl]
|
||||
# Full list of locales in project, but no APKs are produced for some of them here:
|
||||
# lang: [all, en, ar, ca, de, es, fr, id, it, ja, ko, pt, ru, th, tr, vi, zh, bg, hi, pl]
|
||||
matrix: ${{ fromJSON(needs.prepare.outputs.multisrcMatrix) }}
|
||||
steps:
|
||||
- name: Checkout master branch
|
||||
uses: actions/checkout@v3
|
||||
|
|
@ -53,26 +99,26 @@ jobs:
|
|||
- name: Generate sources from the multi-source library
|
||||
uses: gradle/gradle-command-action@v2
|
||||
env:
|
||||
CI_MULTISRC: "true"
|
||||
CI_MODULE_GEN: "true"
|
||||
with:
|
||||
arguments: :multisrc:generateExtensions -x ktFormat -x ktLint
|
||||
arguments: :multisrc:generateExtensions
|
||||
|
||||
- name: Build "${{ matrix.lang }}" extensions
|
||||
- name: Build extensions (chunk ${{ matrix.chunk }})
|
||||
uses: gradle/gradle-command-action@v2
|
||||
env:
|
||||
CI_MULTISRC: "true"
|
||||
CI_MATRIX_LANG: ${{ matrix.lang }}
|
||||
CI_CHUNK_NUM: ${{ matrix.chunk }}
|
||||
ALIAS: ${{ secrets.ALIAS }}
|
||||
KEY_STORE_PASSWORD: ${{ secrets.KEY_STORE_PASSWORD }}
|
||||
KEY_PASSWORD: ${{ secrets.KEY_PASSWORD }}
|
||||
with:
|
||||
arguments: assembleRelease -x lintKotlin
|
||||
arguments: assembleRelease
|
||||
|
||||
- name: Upload "${{ matrix.lang }}" APKs
|
||||
- name: Upload APKs (chunk ${{ matrix.chunk }})
|
||||
uses: actions/upload-artifact@v2
|
||||
if: "github.repository == 'tachiyomiorg/tachiyomi-extensions'"
|
||||
with:
|
||||
name: "multisrc-${{ matrix.lang }}-apks"
|
||||
name: "multisrc-apks-${{ matrix.chunk }}"
|
||||
path: "**/*.apk"
|
||||
retention-days: 1
|
||||
|
||||
|
|
@ -81,14 +127,10 @@ jobs:
|
|||
|
||||
build_individual:
|
||||
name: Build individual modules
|
||||
needs: check_wrapper
|
||||
needs: prepare
|
||||
runs-on: ubuntu-latest
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
lang: [all, en, ar, ca, de, es, fr, id, it, ja, ko, pt, ru, th, tr, vi, zh]
|
||||
# Full list of locales in project, but no APKs are produced for some of them here:
|
||||
# lang: [all, en, ar, ca, de, es, fr, id, it, ja, ko, pt, ru, th, tr, vi, zh, bg, hi, pl]
|
||||
matrix: ${{ fromJSON(needs.prepare.outputs.individualMatrix) }}
|
||||
steps:
|
||||
- name: Checkout master branch
|
||||
uses: actions/checkout@v3
|
||||
|
|
@ -105,22 +147,22 @@ jobs:
|
|||
cp .github/runner-files/ci-gradle.properties ~/.gradle/gradle.properties
|
||||
echo ${{ secrets.SIGNING_KEY }} | base64 -d > signingkey.jks
|
||||
|
||||
- name: Build "${{ matrix.lang }}" extensions
|
||||
- name: Build extensions (chunk ${{ matrix.chunk }})
|
||||
uses: gradle/gradle-command-action@v2
|
||||
env:
|
||||
CI_MULTISRC: "false"
|
||||
CI_MATRIX_LANG: ${{ matrix.lang }}
|
||||
CI_CHUNK_NUM: ${{ matrix.chunk }}
|
||||
ALIAS: ${{ secrets.ALIAS }}
|
||||
KEY_STORE_PASSWORD: ${{ secrets.KEY_STORE_PASSWORD }}
|
||||
KEY_PASSWORD: ${{ secrets.KEY_PASSWORD }}
|
||||
with:
|
||||
arguments: assembleRelease -x lintKotlin
|
||||
arguments: assembleRelease
|
||||
|
||||
- name: Upload "${{ matrix.lang }}" APKs
|
||||
- name: Upload APKs (chunk ${{ matrix.chunk }})
|
||||
uses: actions/upload-artifact@v2
|
||||
if: "github.repository == 'tachiyomiorg/tachiyomi-extensions'"
|
||||
with:
|
||||
name: "individual-${{ matrix.lang }}-apks"
|
||||
name: "individual-apks-${{ matrix.chunk }}"
|
||||
path: "**/*.apk"
|
||||
retention-days: 1
|
||||
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@ project(":lib-ratelimit").projectDir = File("lib/ratelimit")
|
|||
include(":lib-dataimage")
|
||||
project(":lib-dataimage").projectDir = File("lib/dataimage")
|
||||
|
||||
if (System.getenv("CI") == null) {
|
||||
if (System.getenv("CI") == null || System.getenv("CI_MODULE_GEN") == "true") {
|
||||
// Local development (full project build)
|
||||
|
||||
include(":multisrc")
|
||||
|
|
@ -20,7 +20,7 @@ if (System.getenv("CI") == null) {
|
|||
project(name).projectDir = File("src/${dir.name}/${subdir.name}")
|
||||
}
|
||||
}
|
||||
// Loads generated extensions from multisrc
|
||||
// Loads all generated extensions from multisrc
|
||||
File(rootDir, "generated-src").eachDir { dir ->
|
||||
dir.eachDir { subdir ->
|
||||
val name = ":extensions:multisrc:${dir.name}:${subdir.name}"
|
||||
|
|
@ -44,36 +44,43 @@ if (System.getenv("CI") == null) {
|
|||
// Running in CI (GitHub Actions)
|
||||
|
||||
val isMultisrc = System.getenv("CI_MULTISRC") == "true"
|
||||
val lang = System.getenv("CI_MATRIX_LANG")
|
||||
val chunkSize = System.getenv("CI_CHUNK_SIZE").toInt()
|
||||
val chunk = System.getenv("CI_CHUNK_NUM").toInt()
|
||||
|
||||
if (isMultisrc) {
|
||||
include(":multisrc")
|
||||
project(":multisrc").projectDir = File("multisrc")
|
||||
|
||||
// Loads generated extensions from multisrc
|
||||
File(rootDir, "generated-src").eachDir { dir ->
|
||||
if (dir.name == lang) {
|
||||
dir.eachDir { subdir ->
|
||||
val name = ":extensions:multisrc:${dir.name}:${subdir.name}"
|
||||
File(rootDir, "generated-src").getChunk(chunk, chunkSize)?.forEach {
|
||||
val name = ":extensions:multisrc:${it.parentFile.name}:${it.name}"
|
||||
println(name)
|
||||
include(name)
|
||||
project(name).projectDir = File("generated-src/${dir.name}/${subdir.name}")
|
||||
}
|
||||
}
|
||||
project(name).projectDir = File("generated-src/${it.parentFile.name}/${it.name}")
|
||||
}
|
||||
} else {
|
||||
// Loads all extensions
|
||||
File(rootDir, "src").eachDir { dir ->
|
||||
if (dir.name == lang) {
|
||||
dir.eachDir { subdir ->
|
||||
val name = ":extensions:individual:${dir.name}:${subdir.name}"
|
||||
// Loads individual extensions
|
||||
File(rootDir, "src").getChunk(chunk, chunkSize)?.forEach {
|
||||
val name = ":extensions:individual:${it.parentFile.name}:${it.name}"
|
||||
println(name)
|
||||
include(name)
|
||||
project(name).projectDir = File("src/${dir.name}/${subdir.name}")
|
||||
}
|
||||
}
|
||||
project(name).projectDir = File("src/${it.parentFile.name}/${it.name}")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
inline fun File.eachDir(block: (File) -> Unit) {
|
||||
fun File.getChunk(chunk: Int, chunkSize: Int): List<File>? {
|
||||
return listFiles()
|
||||
// Lang folder
|
||||
?.filter { it.isDirectory }
|
||||
// Extension subfolders
|
||||
?.mapNotNull { dir -> dir.listFiles()?.filter { it.isDirectory } }
|
||||
?.flatten()
|
||||
?.sortedBy { it.name }
|
||||
?.chunked(chunkSize)
|
||||
?.get(chunk)
|
||||
}
|
||||
|
||||
fun File.eachDir(block: (File) -> Unit) {
|
||||
listFiles()?.filter { it.isDirectory }?.forEach { block(it) }
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue