Skip to content
This repository was archived by the owner on Nov 22, 2024. It is now read-only.

Fix pod config stacking with empty lists #1036

Merged
andreaTP merged 2 commits into from
Apr 27, 2021
Merged

Fix pod config stacking with empty lists #1036

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
73458be
Fix pod config stacking order
andreaTP Apr 26, 2021
6e353c4
Lists to Optional
andreaTP Apr 26, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions tools/cloudflow-config/src/main/scala/akka/cloudflow/config/CloudflowConfig.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -190,8 +190,8 @@ object CloudflowConfig {

// Container
final case class Container(
env: List[EnvVar] = List(),
ports: List[ContainerPort] = List(),
env: Option[List[EnvVar]] = None,
ports: Option[List[ContainerPort]] = None,
resources: Requirements = Requirements(),
volumeMounts: Map[String, VolumeMount] = Map())

Expand Down
61 changes: 59 additions & 2 deletions tools/cloudflow-config/src/test/scala/akka/cloudflow/config/CloudflowConfigSpec.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -459,8 +459,8 @@ class CloudflowConfigSpec extends AnyFlatSpec with Matchers with OptionValues wi
executor.annotations(AnnotationKey("akey3")) shouldBe AnnotationValue("avalue3")
executor.annotations(AnnotationKey("akey4")) shouldBe AnnotationValue("avalue4")

val driverContainerEnv = driver.containers("container").env.head
val executorContainerEnv = executor.containers("container").env.head
val driverContainerEnv = driver.containers("container").env.head.head
val executorContainerEnv = executor.containers("container").env.head.head

driverContainerEnv.name shouldBe "FOO"
driverContainerEnv.value shouldBe "BAR"
Expand Down Expand Up @@ -615,6 +615,7 @@ class CloudflowConfigSpec extends AnyFlatSpec with Matchers with OptionValues wi
.pods("pod")
.containers("container")
.ports
.get

ports(0).containerPort shouldBe 9001
ports(0).protocol shouldBe "TCP"
Expand Down Expand Up @@ -1007,4 +1008,60 @@ class CloudflowConfigSpec extends AnyFlatSpec with Matchers with OptionValues wi
res.isFailure shouldBe true
res.failure.exception.getMessage.contains("cloudflow.topics.my-topic.topic.replicas") shouldBe true
}

it should "stack pod config in the correct order" in {
// Arrange
val appConfig = ConfigFactory.parseString("""
cloudflow {
# if u remove the streamlet specific config then it works again
streamlets.logger {
kubernetes.pods.pod.containers {
cloudflow {
resources {
requests {
memory = "1G"
}
}
}
}
}
runtimes.akka {
kubernetes.pods.pod.containers {
cloudflow {
resources {
limits {
memory = "3G"
}
}
env = [
{ name = "JAVA_OPTS"
value = "-XX:MaxRAMPercentage=40.0"
}
]
}
}
}
}
""")

// Act
val cloudflowConfig = CloudflowConfig.loadAndValidate(appConfig).get

val podConfig = CloudflowConfig.podsConfig("logger", "akka", cloudflowConfig)

// Assert
podConfig
.getConfigList("kubernetes.pods.pod.containers.cloudflow.env")
.get(0)
.getString("name") shouldBe "JAVA_OPTS"
podConfig
.getConfigList("kubernetes.pods.pod.containers.cloudflow.env")
.get(0)
.getString("value") shouldBe "-XX:MaxRAMPercentage=40.0"
podConfig
.getString("kubernetes.pods.pod.containers.cloudflow.resources.requests.memory") shouldBe "1G"
podConfig
.getString("kubernetes.pods.pod.containers.cloudflow.resources.limits.memory") shouldBe "3G"
}

}
44 changes: 26 additions & 18 deletions tools/cloudflow-operator/src/main/scala/cloudflow/operator/action/runner/Runner.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -384,12 +384,16 @@ object PodsConfig {
*/

private def getContainerConfig(container: CloudflowConfig.Container): ContainerConfig = {
val env = container.env.map { env =>
new EnvVarBuilder()
.withName(env.name)
.withValue(env.value)
.build()
}
val env = container.env
.map {
_.map { env =>
new EnvVarBuilder()
.withName(env.name)
.withValue(env.value)
.build()
}
}
.getOrElse(List())

val resources = {
val limits = {
Expand Down Expand Up @@ -423,18 +427,22 @@ object PodsConfig {
.build()
}.toList

val ports = container.ports.map { port =>
val base = new ContainerPortBuilder()
.withName(port.name.map(_.value).getOrElse(null)) // TODO: check if empty string or null
.withContainerPort(port.containerPort)
.withHostIP(port.hostIP)
.withProtocol(port.protocol)

(port.hostPort match {
case Some(hp) => base.withHostPort(hp)
case _ => base
}).build()
}
val ports = container.ports
.map {
_.map { port =>
val base = new ContainerPortBuilder()
.withName(port.name.map(_.value).getOrElse(null)) // TODO: check if empty string or null
.withContainerPort(port.containerPort)
.withHostIP(port.hostIP)
.withProtocol(port.protocol)

(port.hostPort match {
case Some(hp) => base.withHostPort(hp)
case _ => base
}).build()
}
}
.getOrElse(List())

ContainerConfig(env = env, resources = resources, volumeMounts = volumeMounts, ports = ports)
}
Expand Down
4 changes: 2 additions & 2 deletions tools/tooling/src/main/scala/cli/SamplesGenerator.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ object SamplesGenerator extends App {
"my-pvc" -> PvcVolume(name = "cloudflow-pvc", readOnly = false),
"my-secret" -> SecretVolume(name = "secret.conf")),
containers = Map("my-container" -> Container(
env = List(EnvVar("ENV_VAR_KEY", "ENV_VAR_VALUE")),
env = Some(List(EnvVar("ENV_VAR_KEY", "ENV_VAR_VALUE"))),
resources = Requirements(
requests = Requirement(cpu = Some(Quantity("1")), memory = Some(Quantity("1Gb"))),
limits = Requirement(cpu = Some(Quantity("2")), memory = Some(Quantity("2Gb")))),
Expand All @@ -44,7 +44,7 @@ object SamplesGenerator extends App {
"my-pvc" -> PvcVolume(name = "cloudflow-pvc", readOnly = false),
"my-secret" -> SecretVolume(name = "secret.conf")),
containers = Map("my-container" -> Container(
env = List(EnvVar("ENV_VAR_KEY", "ENV_VAR_VALUE")),
env = Some(List(EnvVar("ENV_VAR_KEY", "ENV_VAR_VALUE"))),
resources = Requirements(
requests = Requirement(cpu = Some(Quantity("1")), memory = Some(Quantity("1Gb"))),
limits = Requirement(cpu = Some(Quantity("2")), memory = Some(Quantity("2Gb")))),
Expand Down