Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: Allow including binary files when auto build is enabled #5299

Open
wants to merge 1 commit into
base: feature/dev-execution
Choose a base branch
from
Open

feat: Allow including binary files when auto build is enabled #5299

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
67 changes: 51 additions & 16 deletions ...oftware/aws/toolkits/jetbrains/services/amazonqFeatureDev/FeatureDevSessionContextTest.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@

import com.intellij.openapi.vfs.VirtualFile
import com.intellij.testFramework.RuleChain
import org.junit.Assert.assertEquals
import org.junit.Assert.assertFalse
import org.junit.Assert.assertTrue
import org.junit.Before
Expand Down Expand Up @@ -72,7 +73,51 @@ class FeatureDevSessionContextTest : FeatureDevTestBase(HeavyJavaCodeInsightTest
}

@Test
fun testZipProject() {
fun testZipProjectWithoutAutoDev() {
checkZipProject(
false,
setOf(
"src/MyClass.java",
"gradlew",
"gradlew.bat",
"README.md",
"gradle/wrapper/gradle-wrapper.properties",
"builder/GetTestBuilder.java",
"settings.gradle",
"build.gradle",
".gitignore",
)
)
}

@Test
fun testZipProjectWithAutoDev() {
checkZipProject(
true,
setOf(
"src/MyClass.java",
"icons/menu.svg",
"assets/header.jpg",
"gradle/wrapper/gradle-wrapper.jar",
"gradle/wrapper/gradle-wrapper.properties",
"images/logo.png",
"builder/GetTestBuilder.java",
"gradlew",
"README.md",
".gitignore",
"License.md",
"output.bin",
"archive.zip",
"gradlew.bat",
"license.txt",
"build.gradle",
"devfile.yaml",
"settings.gradle"
)
)
}

fun checkZipProject(autoBuildEnabled: Boolean, expectedFiles: Set<String>) {
addFilesToProjectModule(
".gitignore",
".gradle/cached.jar",
Expand All @@ -97,9 +142,12 @@ class FeatureDevSessionContextTest : FeatureDevTestBase(HeavyJavaCodeInsightTest
"build/outputs",
"dist/bundle.js",
"gradle/wrapper/gradle-wrapper.jar",
"devfile.yaml",
)

val zipResult = featureDevSessionContext.getProjectZip(false)
projectRule.fixture.addFileToModule(module, "large-file.txt", "loblob".repeat(1024 * 1024))

val zipResult = featureDevSessionContext.getProjectZip(autoBuildEnabled)
val zipPath = zipResult.payload.path

val zippedFiles = mutableSetOf<String>()
Expand All @@ -111,19 +159,6 @@ class FeatureDevSessionContextTest : FeatureDevTestBase(HeavyJavaCodeInsightTest
}
}

val expectedFiles = setOf(
"src/MyClass.java",
"gradlew",
"gradlew.bat",
"README.md",
"gradle/wrapper/gradle-wrapper.properties",
"builder/GetTestBuilder.java",
"settings.gradle",
"build.gradle",
"gradle/wrapper/gradle-wrapper.jar",
".gitignore",
)

assertTrue(zippedFiles == expectedFiles)
assertEquals(zippedFiles, expectedFiles)
}
}
99 changes: 55 additions & 44 deletions ...ommunity/src/software/aws/toolkits/jetbrains/services/amazonq/FeatureDevSessionContext.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,14 +10,14 @@ import com.intellij.openapi.vfs.VirtualFile
import com.intellij.openapi.vfs.VirtualFileVisitor
import com.intellij.openapi.vfs.isFile
import com.intellij.platform.ide.progress.withBackgroundProgress
import kotlinx.coroutines.async
import kotlinx.coroutines.flow.channelFlow
import kotlinx.coroutines.launch
import kotlinx.coroutines.runBlocking
import kotlinx.coroutines.withContext
import org.apache.commons.codec.digest.DigestUtils
import org.apache.commons.io.FileUtils
import software.aws.toolkits.jetbrains.core.coroutines.getCoroutineBgContext
import software.aws.toolkits.jetbrains.services.amazonq.QConstants.MAX_FILE_SIZE_BYTES
import software.aws.toolkits.jetbrains.services.telemetry.ALLOWED_CODE_EXTENSIONS
import software.aws.toolkits.jetbrains.utils.isDevFile
import software.aws.toolkits.resources.AwsCoreBundle
Expand All @@ -33,7 +33,6 @@ import java.nio.file.Paths
import java.nio.file.StandardCopyOption
import java.util.Base64
import java.util.UUID
import kotlin.coroutines.coroutineContext
import kotlin.io.path.Path
import kotlin.io.path.createParentDirectories
import kotlin.io.path.getPosixFilePermissions
Expand All @@ -47,8 +46,7 @@ class RepoSizeLimitError(override val message: String) : RuntimeException(), Rep
class FeatureDevSessionContext(val project: Project, val maxProjectSizeBytes: Long? = null) {
// TODO: Need to correct this class location in the modules going further to support both amazonq and codescan.

private val requiredFilesForExecution = setOf("gradle/wrapper/gradle-wrapper.jar")
private val additionalGitIgnoreRules = setOf(
private val additionalGitIgnoreFolderRules = setOf(
".aws-sam",
".gem",
".git",
Expand All @@ -58,6 +56,12 @@ class FeatureDevSessionContext(val project: Project, val maxProjectSizeBytes: Lo
".project",
".rvm",
".svn",
"node_modules",
"build",
"dist",
)

private val additionalGitIgnoreBinaryFilesRules = setOf(
"*.zip",
"*.bin",
"*.png",
Expand All @@ -70,9 +74,6 @@ class FeatureDevSessionContext(val project: Project, val maxProjectSizeBytes: Lo
"license.md",
"License.md",
"LICENSE.md",
"node_modules",
"build",
"dist"
)

// well known source files that do not have extensions
Expand All @@ -90,12 +91,17 @@ class FeatureDevSessionContext(val project: Project, val maxProjectSizeBytes: Lo
// selectedSourceFolder: is the directory selected in replacement of the root, this happens when the project is too big to bundle for uploading.
private var _selectedSourceFolder = projectRoot
private var ignorePatternsWithGitIgnore = emptyList<Regex>()
private var ignorePatternsForBinaryFiles = additionalGitIgnoreBinaryFilesRules
.map { convertGitIgnorePatternToRegex(it) }
.mapNotNull { pattern ->
runCatching { Regex(pattern) }.getOrNull()
}
private val gitIgnoreFile = File(selectedSourceFolder.path, ".gitignore")

init {
ignorePatternsWithGitIgnore = try {
buildList {
addAll(additionalGitIgnoreRules.map { convertGitIgnorePatternToRegex(it) })
addAll(additionalGitIgnoreFolderRules.map { convertGitIgnorePatternToRegex(it) })
addAll(parseGitIgnore())
}.mapNotNull { pattern ->
runCatching { Regex(pattern) }.getOrNull()
Expand Down Expand Up @@ -130,35 +136,49 @@ class FeatureDevSessionContext(val project: Project, val maxProjectSizeBytes: Lo
return ALLOWED_CODE_EXTENSIONS.contains(extension)
}

private fun ignoreFileByExtension(file: VirtualFile) =
!isFileExtensionAllowed(file)
fun ignoreFile(file: VirtualFile): Boolean = ignoreFile(file.presentableUrl)

fun ignoreFile(path: String, applyExtraBinaryFilesRules: Boolean = true): Boolean {
val allIgnoreRules = if (applyExtraBinaryFilesRules) ignorePatternsWithGitIgnore + ignorePatternsForBinaryFiles else ignorePatternsWithGitIgnore
val matchedRules = allIgnoreRules.map { pattern ->
// avoid partial match (pattern.containsMatchIn) since it causes us matching files
// against folder patterns. (e.g. settings.gradle ignored by .gradle rule!)
// we convert the glob rules to regex, add a trailing /* to all rules and then match
// entries against them by adding a trailing /.
// TODO: Add unit tests for gitignore matching
val relative = if (path.startsWith(projectRootPath.toString())) Paths.get(path).relativeTo(projectRootPath) else path
pattern.matches("$relative/")
}
return matchedRules.any { it }
}

suspend fun ignoreFile(file: VirtualFile): Boolean = ignoreFile(file.presentableUrl)
private fun wellKnown(file: VirtualFile): Boolean = wellKnownSourceFiles.contains(file.name)

suspend fun ignoreFile(path: String): Boolean {
if (requiredFilesForExecution.any { path.endsWith(it) }) {
private fun shouldIncludeInZipFile(file: VirtualFile, isAutoBuildFeatureEnabled: Boolean): Boolean {
// large files always ignored
if (file.length > MAX_FILE_SIZE_BYTES) {
return false
}
// this method reads like something a JS dev would write and doesn't do what the author thinks
val deferredResults = ignorePatternsWithGitIgnore.map { pattern ->
withContext(coroutineContext) {
// avoid partial match (pattern.containsMatchIn) since it causes us matching files
// against folder patterns. (e.g. settings.gradle ignored by .gradle rule!)
// we convert the glob rules to regex, add a trailing /* to all rules and then match
// entries against them by adding a trailing /.
// TODO: Add unit tests for gitignore matching
val relative = if (path.startsWith(projectRootPath.toString())) Paths.get(path).relativeTo(projectRootPath) else path
async { pattern.matches("$relative/") }
}

// always respect gitignore rules and remove binary files if auto build is disabled
val isFileIgnoredByPattern = ignoreFile(file.path, !isAutoBuildFeatureEnabled)
if (isFileIgnoredByPattern) {
return false
}

// this will serially iterate over and block
// ideally we race the results https://github.com/Kotlin/kotlinx.coroutines/issues/2867
// i.e. Promise.any(...)
return deferredResults.any { it.await() }
}
// all other files are included when auto build enabled
if (isAutoBuildFeatureEnabled) {
return true
}

// when auto build is disabled, only include files with well known extensions and names except "devfile.yam"
if (!isDevFile(file) && (wellKnown(file) || isFileExtensionAllowed(file))) {
return true
}

fun wellKnown(file: VirtualFile): Boolean = wellKnownSourceFiles.contains(file.name)
// Any other files should not be included
return false
}

suspend fun zipFiles(projectRoot: VirtualFile, isAutoBuildFeatureEnabled: Boolean?): File = withContext(getCoroutineBgContext()) {
val files = mutableListOf<VirtualFile>()
Expand All @@ -169,26 +189,17 @@ class FeatureDevSessionContext(val project: Project, val maxProjectSizeBytes: Lo
projectRoot,
object : VirtualFileVisitor<Unit>() {
override fun visitFile(file: VirtualFile): Boolean {
val isWellKnown = runBlocking { wellKnown(file) }
val isFileIgnoredByExtension = runBlocking { ignoreFileByExtension(file) }
// if `isAutoBuildFeatureEnabled` is false, then filter devfile
val isFilterDevFile = if (isAutoBuildFeatureEnabled == true) false else isDevFile(file)
if (file.isDirectory) {
return true
}

if (!isWellKnown && isFileIgnoredByExtension) {
val isIncluded = shouldIncludeInZipFile(file, isAutoBuildFeatureEnabled == true)
if (!isIncluded) {
val extension = file.extension.orEmpty()
ignoredExtensionMap[extension] = (ignoredExtensionMap[extension] ?: 0) + 1
return false
}

if (isFilterDevFile) {
return false
}

val isFileIgnoredByPattern = runBlocking { ignoreFile(file.name) }
if (isFileIgnoredByPattern) {
return false
}

if (file.isFile) {
totalSize += file.length
files.add(file)
Expand Down
1 change: 1 addition & 0 deletions ...re/jetbrains-community/src/software/aws/toolkits/jetbrains/services/amazonq/QConstants.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,4 +6,5 @@ package software.aws.toolkits.jetbrains.services.amazonq
object QConstants {
const val Q_MARKETPLACE_URI = "https://aws.amazon.com/q/developer/"
const val CODEWHISPERER_LOGIN_HELP_URI = "https://docs.aws.amazon.com/toolkit-for-jetbrains/latest/userguide/auth-access.html"
const val MAX_FILE_SIZE_BYTES = 1024 * 1024
}
1 change: 0 additions & 1 deletion ...brains-community/src/software/aws/toolkits/jetbrains/services/telemetry/TelemetryUtils.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -88,7 +88,6 @@ val ALLOWED_CODE_EXTENSIONS = setOf(
"idl",
"ini",
"io",
"jar",
"java",
"jl",
"js",
Expand Down
Loading