diff options
Diffstat (limited to 'build-logic/automation-plugins')
4 files changed, 296 insertions, 1 deletions
diff --git a/build-logic/automation-plugins/build.gradle.kts b/build-logic/automation-plugins/build.gradle.kts index e28b0255..2375262a 100644 --- a/build-logic/automation-plugins/build.gradle.kts +++ b/build-logic/automation-plugins/build.gradle.kts @@ -1 +1,27 @@ -plugins { `kotlin-dsl` } +/* + * Copyright © 2014-2021 The Android Password Store Authors. All Rights Reserved. + * SPDX-License-Identifier: GPL-3.0-only + */ + +plugins { + `kotlin-dsl` + `kotlin-dsl-precompiled-script-plugins` +} + +gradlePlugin { + plugins { + register("crowdin") { + id = "com.github.android-password-store.crowdin-plugin" + implementationClass = "crowdin.CrowdinDownloadPlugin" + } + register("psl") { + id = "com.github.android-password-store.psl-plugin" + implementationClass = "psl.PublicSuffixListPlugin" + } + } +} + +dependencies { + implementation(libs.build.download) + implementation(libs.build.okhttp) +} diff --git a/build-logic/automation-plugins/src/main/kotlin/crowdin/CrowdinExtension.kt b/build-logic/automation-plugins/src/main/kotlin/crowdin/CrowdinExtension.kt new file mode 100644 index 00000000..f1457673 --- /dev/null +++ b/build-logic/automation-plugins/src/main/kotlin/crowdin/CrowdinExtension.kt @@ -0,0 +1,20 @@ +/* + * Copyright © 2014-2021 The Android Password Store Authors. All Rights Reserved. + * SPDX-License-Identifier: GPL-3.0-only + */ + +package crowdin + +/** Extension for configuring [CrowdinPlugin] */ +interface CrowdinExtension { + + /** Configure the project name on Crowdin */ + var projectName: String + + /** + * Don't delete downloaded and extracted translation archives from build directory. + * + * Useful for debugging. + */ + var skipCleanup: Boolean +} diff --git a/build-logic/automation-plugins/src/main/kotlin/crowdin/CrowdinPlugin.kt b/build-logic/automation-plugins/src/main/kotlin/crowdin/CrowdinPlugin.kt new file mode 100644 index 00000000..580fc939 --- /dev/null +++ b/build-logic/automation-plugins/src/main/kotlin/crowdin/CrowdinPlugin.kt @@ -0,0 +1,127 @@ +/* + * Copyright © 2014-2021 The Android Password Store Authors. All Rights Reserved. + * SPDX-License-Identifier: GPL-3.0-only + */ + +package crowdin + +import de.undercouch.gradle.tasks.download.Download +import java.io.File +import javax.xml.parsers.DocumentBuilderFactory +import okhttp3.OkHttpClient +import okhttp3.Request +import org.gradle.api.GradleException +import org.gradle.api.Plugin +import org.gradle.api.Project +import org.gradle.api.tasks.Copy +import org.gradle.kotlin.dsl.create +import org.gradle.kotlin.dsl.register +import org.w3c.dom.Document + +private const val EXCEPTION_MESSAGE = + """Applying `crowdin-plugin` requires a projectName to be configured via the "crowdin" extension.""" +private const val CROWDIN_BUILD_API_URL = + "https://api.crowdin.com/api/project/%s/export?login=%s&account-key=%s" + +class CrowdinDownloadPlugin : Plugin<Project> { + + override fun apply(project: Project) { + with(project) { + val buildDirectory = layout.buildDirectory.asFile.forUseAtConfigurationTime().get() + val extension = extensions.create<CrowdinExtension>("crowdin") + afterEvaluate { + val projectName = extension.projectName + if (projectName.isEmpty()) { + throw GradleException(EXCEPTION_MESSAGE) + } + tasks.register("buildOnApi") { + doLast { + val login = providers.environmentVariable("CROWDIN_LOGIN").forUseAtConfigurationTime() + val key = + providers.environmentVariable("CROWDIN_PROJECT_KEY").forUseAtConfigurationTime() + if (!login.isPresent) { + throw GradleException("CROWDIN_LOGIN environment variable must be set") + } + if (!key.isPresent) { + throw GradleException("CROWDIN_PROJECT_KEY environment variable must be set") + } + val client = OkHttpClient() + val url = CROWDIN_BUILD_API_URL.format(projectName, login.get(), key.get()) + val request = Request.Builder().url(url).get().build() + client.newCall(request).execute() + } + } + val downloadCrowdin = + tasks.register<Download>("downloadCrowdin") { + dependsOn("buildOnApi") + src("https://crowdin.com/backend/download/project/$projectName.zip") + dest("$buildDirectory/translations.zip") + overwrite(true) + } + val extractCrowdin = + tasks.register<Copy>("extractCrowdin") { + dependsOn(downloadCrowdin) + doFirst { File(buildDir, "translations").deleteRecursively() } + from(zipTree("$buildDirectory/translations.zip")) + into("$buildDirectory/translations") + } + val extractStrings = + tasks.register<Copy>("extractStrings") { + dependsOn(extractCrowdin) + from("$buildDirectory/translations/") + into("${projectDir}/src/") + setFinalizedBy(setOf("removeIncompleteStrings")) + } + tasks.register("removeIncompleteStrings") { + doLast { + val sourceSets = arrayOf("main", "nonFree") + for (sourceSet in sourceSets) { + val stringFiles = + File("${projectDir}/src/$sourceSet").walkTopDown().filter { + it.name == "strings.xml" + } + val sourceFile = + stringFiles.firstOrNull { it.path.endsWith("values/strings.xml") } + ?: throw GradleException("No root strings.xml found in '$sourceSet' sourceSet") + val sourceDoc = parseDocument(sourceFile) + val baselineStringCount = countStrings(sourceDoc) + val threshold = 0.80 * baselineStringCount + stringFiles.forEach { file -> + if (file != sourceFile) { + val doc = parseDocument(file) + val stringCount = countStrings(doc) + if (stringCount < threshold) { + file.delete() + } + } + } + } + } + } + tasks.register("crowdin") { + dependsOn(extractStrings) + if (!extension.skipCleanup) { + doLast { + File("$buildDirectory/translations").deleteRecursively() + File("$buildDirectory/nonFree-translations").deleteRecursively() + File("$buildDirectory/translations.zip").delete() + } + } + } + } + } + } + + private fun parseDocument(file: File): Document { + val dbFactory = DocumentBuilderFactory.newInstance() + val documentBuilder = dbFactory.newDocumentBuilder() + return documentBuilder.parse(file) + } + + private fun countStrings(document: Document): Int { + // Normalization is beneficial for us + // https://stackoverflow.com/questions/13786607/normalization-in-dom-parsing-with-java-how-does-it-work + document.documentElement.normalize() + return document.getElementsByTagName("string").length + } +} diff --git a/build-logic/automation-plugins/src/main/kotlin/psl/PublicSuffixListPlugin.kt b/build-logic/automation-plugins/src/main/kotlin/psl/PublicSuffixListPlugin.kt new file mode 100644 index 00000000..3c003c53 --- /dev/null +++ b/build-logic/automation-plugins/src/main/kotlin/psl/PublicSuffixListPlugin.kt @@ -0,0 +1,122 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +package psl + +import java.io.File +import java.util.TreeSet +import okhttp3.OkHttpClient +import okhttp3.Request +import okio.ByteString +import okio.ByteString.Companion.encodeUtf8 +import okio.buffer +import okio.sink +import org.gradle.api.Plugin +import org.gradle.api.Project + +/** + * Gradle plugin to update the public suffix list used by the `lib-publicsuffixlist` component. + * + * Base on PublicSuffixListGenerator from OkHttp: + * https://github.com/square/okhttp/blob/master/okhttp/src/test/java/okhttp3/internal/publicsuffix/PublicSuffixListGenerator.java + */ +class PublicSuffixListPlugin : Plugin<Project> { + override fun apply(project: Project) { + project.tasks.register("updatePSL") { + doLast { + val filename = project.projectDir.absolutePath + "/src/main/assets/publicsuffixes" + updatePublicSuffixList(filename) + } + } + } + + private fun updatePublicSuffixList(destination: String) { + val list = fetchPublicSuffixList() + writeListToDisk(destination, list) + } + + private fun writeListToDisk(destination: String, data: PublicSuffixListData) { + val fileSink = File(destination).sink() + + fileSink.buffer().use { sink -> + sink.writeInt(data.totalRuleBytes) + + for (domain in data.sortedRules) { + sink.write(domain).writeByte('\n'.toInt()) + } + + sink.writeInt(data.totalExceptionRuleBytes) + + for (domain in data.sortedExceptionRules) { + sink.write(domain).writeByte('\n'.toInt()) + } + } + } + + private fun fetchPublicSuffixList(): PublicSuffixListData { + val client = OkHttpClient.Builder().build() + + val request = + Request.Builder().url("https://publicsuffix.org/list/public_suffix_list.dat").build() + + client.newCall(request).execute().use { response -> + val source = requireNotNull(response.body).source() + + val data = PublicSuffixListData() + + while (!source.exhausted()) { + val line = source.readUtf8LineStrict() + + if (line.trim { it <= ' ' }.isEmpty() || line.startsWith("//")) { + continue + } + + if (line.contains(WILDCARD_CHAR)) { + assertWildcardRule(line) + } + + var rule = line.encodeUtf8() + + if (rule.startsWith(EXCEPTION_RULE_MARKER)) { + rule = rule.substring(1) + // We use '\n' for end of value. + data.totalExceptionRuleBytes += rule.size + 1 + data.sortedExceptionRules.add(rule) + } else { + data.totalRuleBytes += rule.size + 1 // We use '\n' for end of value. + data.sortedRules.add(rule) + } + } + + return data + } + } + + @Suppress("TooGenericExceptionThrown", "ThrowsCount") + private fun assertWildcardRule(rule: String) { + if (rule.indexOf(WILDCARD_CHAR) != 0) { + throw RuntimeException("Wildcard is not not in leftmost position") + } + + if (rule.indexOf(WILDCARD_CHAR, 1) != -1) { + throw RuntimeException("Rule contains multiple wildcards") + } + + if (rule.length == 1) { + throw RuntimeException("Rule wildcards the first level") + } + } + + companion object { + private const val WILDCARD_CHAR = "*" + private val EXCEPTION_RULE_MARKER = "!".encodeUtf8() + } +} + +data class PublicSuffixListData( + var totalRuleBytes: Int = 0, + var totalExceptionRuleBytes: Int = 0, + val sortedRules: TreeSet<ByteString> = TreeSet(), + val sortedExceptionRules: TreeSet<ByteString> = TreeSet() +) |