diff --git a/.github/release-drafter.yml b/.github/release-drafter.yml index 975c64a67..0f3a8b314 100644 --- a/.github/release-drafter.yml +++ b/.github/release-drafter.yml @@ -16,8 +16,6 @@ categories: labels: - 'streamlets' - 'akka' - - 'flink' - - 'spark' - title: ':book: Documentation' labels: - 'kind/documentation' diff --git a/.github/workflows/build_and_publish.yaml b/.github/workflows/build_and_publish_docs.yaml similarity index 100% rename from .github/workflows/build_and_publish.yaml rename to .github/workflows/build_and_publish_docs.yaml diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index bd5000803..70231178a 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -82,3 +82,21 @@ jobs: - name: test-maven-examples run: ./scripts/build-mvn-examples.sh test + + fossa-checks: + name: "FOSSA checks" + runs-on: ubuntu-20.04 + steps: + + - uses: actions/checkout@v2 + with: + fetch-depth: 0 + + - name: FOSSA policy check + if: ${{ github.event_name != 'pull_request' }} + run: |- + cd core + curl -H 'Cache-Control: no-cache' https://raw.githubusercontent.com/fossas/spectrometer/master/install.sh | bash + fossa analyze && fossa test + env: + FOSSA_API_KEY: "${{secrets.FOSSA_API_KEY}}" \ No newline at end of file diff --git a/README.md b/README.md index 2c2985692..4872194db 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@

Cloudflow enables users to quickly develop, orchestrate, and operate distributed streaming applications on Kubernetes. Cloudflow allows you to easily break down your streaming application to smaller composable components and wire them together with schema-based contracts. -Cloudflow integrates with popular streaming engines like Akka, Spark and Flink. It also comes with a powerful CLI tool to easily manage, scale and configure your streaming applications at runtime. +It also comes with a powerful CLI tool to easily manage, scale and configure your streaming applications at runtime. With its powerful abstractions, Cloudflow allows to define, build and deploy the most complex streaming applications. - Develop: Focus only on business logic, leave the boilerplate to us. @@ -18,7 +18,7 @@ As data pipelines become first-class citizens in microservices architectures, Cl In a nutshell, Cloudflow is an application development toolkit comprising: - An API definition for `Streamlet`, the core abstraction in Cloudflow. -- An extensible set of runtime implementations for `Streamlet`(s). Cloudflow provides support for popular streaming runtimes, like Spark's Structured Streaming, Flink, and Akka. +- An extensible set of runtime implementations for `Streamlet`(s). - A `Streamlet` composition model driven by a `blueprint` definition. - A sandbox execution mode that accelerates the development and testing of your applications. - A set of `sbt` plugins that are able to package your application into a deployable container. @@ -53,7 +53,7 @@ The underlying data streams are partitioned to allow for parallelism in a distri The `Streamlet` logic can be written using an extensible choice of streaming runtimes, such as Akka Streams and Spark. The lightweight API exposes the raw power of the underlying runtime and its libraries while providing a higher-level abstraction for composing `streamlets` and expressing data schemas. -Your code is written in your familiar Structured Streaming, Flink, or Akka Streams native API. +Your code is written in your familiar API. Applications are deployed as a whole. Cloudflow takes care of deploying the individual `streamlets` and making sure connections get translated into data flowing between them at runtime. diff --git a/core/build.sbt b/core/build.sbt index eba4e6ea1..b772dfc51 100644 --- a/core/build.sbt +++ b/core/build.sbt @@ -364,7 +364,7 @@ lazy val cloudflowAkkaTests = lazy val cloudflowRunner = Project(id = "cloudflow-runner", base = file("cloudflow-runner")) .enablePlugins(BuildInfoPlugin, ScalafmtPlugin) - .dependsOn(cloudflowStreamlets, cloudflowBlueprint) + .dependsOn(cloudflowStreamlets) .settings( scalaVersion := Dependencies.Scala212, crossScalaVersions := Vector(Dependencies.Scala212, Dependencies.Scala213), diff --git a/core/cloudflow-cli/src/main/scala/akka/cli/cloudflow/execution/ConfigureExecution.scala b/core/cloudflow-cli/src/main/scala/akka/cli/cloudflow/execution/ConfigureExecution.scala index 8678e45b0..38c2091fb 100644 --- a/core/cloudflow-cli/src/main/scala/akka/cli/cloudflow/execution/ConfigureExecution.scala +++ b/core/cloudflow-cli/src/main/scala/akka/cli/cloudflow/execution/ConfigureExecution.scala @@ -38,9 +38,9 @@ final case class ConfigureExecution(c: Configure, client: KubeClient, logger: Cl cloudflowConfig, () => client.getKafkaClusters(namespace = c.operatorNamespace).map(parseValues)) - uid <- client.uidCloudflowApp(currentCr.spec.appId, namespace) + uid <- client.uidCloudflowApp(currentCr.getSpec.appId, namespace) _ <- client.configureCloudflowApp( - currentCr.spec.appId, + currentCr.getSpec.appId, namespace, uid, configStr, diff --git a/core/cloudflow-cli/src/main/scala/akka/cli/cloudflow/execution/DeployExecution.scala b/core/cloudflow-cli/src/main/scala/akka/cli/cloudflow/execution/DeployExecution.scala index cf8b6eba5..1839419e3 100644 --- a/core/cloudflow-cli/src/main/scala/akka/cli/cloudflow/execution/DeployExecution.scala +++ b/core/cloudflow-cli/src/main/scala/akka/cli/cloudflow/execution/DeployExecution.scala @@ -29,7 +29,7 @@ final case class DeployExecution(d: Deploy, client: KubeClient, logger: CliLogge import DeployExecution._ private def applicationDescriptorValidation(crApp: App.Cr): Try[Unit] = { - crApp.spec.version match { + crApp.getSpec.version match { case None => Failure(CliException("Application file parse error: spec.version is missing or empty")) @@ -42,7 +42,7 @@ final case class DeployExecution(d: Deploy, client: KubeClient, logger: CliLogge case _ => Failure(CliException("Application file parse error: spec.version is invalid")) } libraryVersion <- Try { - val libraryVersion = crApp.spec.libraryVersion.get + val libraryVersion = crApp.getSpec.libraryVersion.get require { !libraryVersion.contains(' ') } libraryVersion }.recoverWith { @@ -77,7 +77,7 @@ final case class DeployExecution(d: Deploy, client: KubeClient, logger: CliLogge } private def referencedKafkaSecretExists(appCr: App.Cr, kafkaClusters: () => Try[List[String]]): Try[Unit] = { - val expectedClusters = appCr.spec.deployments.flatMap(_.portMappings.values.map(_.cluster)).flatten.distinct + val expectedClusters = appCr.getSpec.deployments.flatMap(_.portMappings.values.map(_.cluster)).flatten.distinct if (expectedClusters.nonEmpty) { (for { @@ -95,11 +95,11 @@ final case class DeployExecution(d: Deploy, client: KubeClient, logger: CliLogge } private def getImageReference(crApp: App.Cr) = { - if (crApp.spec.deployments.size < 1) { + if (crApp.getSpec.deployments.size < 1) { Failure(CliException("The application specification doesn't contains deployments")) } else { // Get the first available image, all images must be present in the same repository. - val imageRef = crApp.spec.deployments(0).image + val imageRef = crApp.getSpec.deployments(0).image Image(imageRef) } @@ -115,14 +115,14 @@ final case class DeployExecution(d: Deploy, client: KubeClient, logger: CliLogge baseApplicationCr <- loadCrFile(d.crFile) localApplicationCr = { d.serviceAccount match { - case Some(sa) => baseApplicationCr.copy(spec = baseApplicationCr.spec.copy(serviceAccount = Some(sa))) + case Some(sa) => baseApplicationCr.copy(_spec = baseApplicationCr.getSpec.copy(serviceAccount = Some(sa))) case _ => baseApplicationCr } } - namespace = d.namespace.getOrElse(localApplicationCr.spec.appId) + namespace = d.namespace.getOrElse(localApplicationCr.getSpec.appId) // update the replicas - currentAppCr <- client.readCloudflowApp(localApplicationCr.spec.appId, namespace) + currentAppCr <- client.readCloudflowApp(localApplicationCr.getSpec.appId, namespace) clusterReplicas = getStreamletsReplicas(currentAppCr) clusterApplicationCr <- updateReplicas(localApplicationCr, clusterReplicas) applicationCrReplicas <- updateReplicas(clusterApplicationCr, d.scales) @@ -154,7 +154,7 @@ final case class DeployExecution(d: Deploy, client: KubeClient, logger: CliLogge }) // Operations on the cluster - name = applicationCr.spec.appId + name = applicationCr.getSpec.appId _ <- client.createNamespace(namespace) _ <- { if (d.noRegistryCredentials) Success(()) @@ -166,7 +166,7 @@ final case class DeployExecution(d: Deploy, client: KubeClient, logger: CliLogge dockerPassword = d.dockerPassword) } } - uid <- client.createCloudflowApp(applicationCr.spec, namespace) + uid <- client.createCloudflowApp(applicationCr.getSpec, namespace) _ <- client.configureCloudflowApp(name, namespace, uid, configStr, logbackContent, streamletsConfigs) } yield { logger.trace("Command Deploy executed successfully") diff --git a/core/cloudflow-cli/src/main/scala/akka/cli/cloudflow/execution/WithConfiguration.scala b/core/cloudflow-cli/src/main/scala/akka/cli/cloudflow/execution/WithConfiguration.scala index af9b3f817..1f79d62b2 100644 --- a/core/cloudflow-cli/src/main/scala/akka/cli/cloudflow/execution/WithConfiguration.scala +++ b/core/cloudflow-cli/src/main/scala/akka/cli/cloudflow/execution/WithConfiguration.scala @@ -87,7 +87,7 @@ trait WithConfiguration { private def validateConfiguredStreamlets(crApp: App.Cr, cloudflowConfig: CloudflowConfig.CloudflowRoot): Try[Unit] = { val configStreamlets = cloudflowConfig.cloudflow.streamlets.keys.toSeq.distinct - val crStreamlets = crApp.spec.streamlets.map(_.name).toSeq.distinct + val crStreamlets = crApp.getSpec.streamlets.map(_.name).toSeq.distinct configStreamlets.diff(crStreamlets) match { case Nil => Success(()) @@ -99,7 +99,7 @@ trait WithConfiguration { private def validateTopicIds(crApp: App.Cr, cloudflowConfig: CloudflowConfig.CloudflowRoot): Try[Unit] = { val configTopics = cloudflowConfig.cloudflow.topics.keys.toSeq.distinct - val crTopics = crApp.spec.deployments.flatMap(_.portMappings.values.map(_.id)).distinct + val crTopics = crApp.getSpec.deployments.flatMap(_.portMappings.values.map(_.id)).distinct configTopics.diff(crTopics) match { case Nil => Success(()) @@ -161,7 +161,7 @@ trait WithConfiguration { def validateConfigParameters(crApp: App.Cr, cloudflowConfig: CloudflowConfig.CloudflowRoot): Try[Unit] = { Try { - crApp.spec.streamlets.foreach { streamlet => + crApp.getSpec.streamlets.foreach { streamlet => cloudflowConfig.cloudflow.streamlets.get(streamlet.name).foreach { s => val configParameters = streamlet.descriptor.configParameters val detectedConfigParameters = @@ -212,8 +212,8 @@ trait WithConfiguration { for { userConfig <- tryUserConfig cloudflowConfig <- CloudflowConfig.loadAndValidate(userConfig) - defaultConfig = CloudflowConfig.defaultConfig(appCr.spec) - defaultMounts = CloudflowConfig.defaultMountsConfig(appCr.spec, allowedRuntimes = List("flink", "spark")) + defaultConfig = CloudflowConfig.defaultConfig(appCr.getSpec) + defaultMounts = CloudflowConfig.defaultMountsConfig(appCr.getSpec, allowedRuntimes = List("flink", "spark")) config = userConfig .withFallback(CloudflowConfig.writeConfig(defaultConfig)) .withFallback(CloudflowConfig.writeConfig(defaultMounts)) @@ -221,7 +221,7 @@ trait WithConfiguration { loggingConfig match { case Some(s) => CloudflowConfig.writeConfig( - CloudflowConfig.loggingMountsConfig(appCr.spec, s"${MurmurHash3.stringHash(s)}")) + CloudflowConfig.loggingMountsConfig(appCr.getSpec, s"${MurmurHash3.stringHash(s)}")) case None => ConfigFactory.empty() } } @@ -327,7 +327,7 @@ trait WithConfiguration { val allReferencedClusters = { appConfig.cloudflow.topics.flatMap { case (_, topic) => topic.cluster } ++ - appCr.spec.deployments.flatMap(_.portMappings.values.flatMap(_.cluster)) ++ + appCr.getSpec.deployments.flatMap(_.portMappings.values.flatMap(_.cluster)) ++ Seq(DefaultConfigurationName) } @@ -345,10 +345,10 @@ trait WithConfiguration { }.toMap } res <- Try { - appCr.spec.deployments.map { deployment => + appCr.getSpec.deployments.map { deployment => val streamletConfig = CloudflowConfig.streamletConfig(deployment.streamletName, deployment.runtime, appConfig) val streamletWithPortMappingsConfig = portMappings(deployment, appConfig, streamletConfig, clustersConfig) - val applicationConfig = applicationRunnerConfig(appCr.name, appCr.spec.appVersion, deployment) + val applicationConfig = applicationRunnerConfig(appCr.name, appCr.getSpec.appVersion, deployment) deployment -> StreamletConfigs( streamlet = streamletWithPortMappingsConfig, runtime = CloudflowConfig.runtimeConfig(deployment.streamletName, deployment.runtime, appConfig), diff --git a/core/cloudflow-cli/src/main/scala/akka/cli/cloudflow/execution/WithUpdateReplicas.scala b/core/cloudflow-cli/src/main/scala/akka/cli/cloudflow/execution/WithUpdateReplicas.scala index b809733f2..51559dcd7 100644 --- a/core/cloudflow-cli/src/main/scala/akka/cli/cloudflow/execution/WithUpdateReplicas.scala +++ b/core/cloudflow-cli/src/main/scala/akka/cli/cloudflow/execution/WithUpdateReplicas.scala @@ -15,7 +15,7 @@ trait WithUpdateReplicas { appCr .map { app => (for { - streamlet <- app.spec.deployments + streamlet <- app.getSpec.deployments } yield { streamlet.replicas match { case Some(r) => Some(streamlet.streamletName -> r) @@ -27,18 +27,18 @@ trait WithUpdateReplicas { } def updateReplicas(crApp: App.Cr, replicas: Map[String, Int]): Try[App.Cr] = { - val allStreamlets = crApp.spec.deployments.map { streamlet => streamlet.streamletName }.distinct + val allStreamlets = crApp.getSpec.deployments.map { streamlet => streamlet.streamletName }.distinct (replicas.keys.toList.distinct.diff(allStreamlets)) match { case Nil => - val clusterDeployments = crApp.spec.deployments.map { streamlet => + val clusterDeployments = crApp.getSpec.deployments.map { streamlet => streamlet.streamletName match { case sname if replicas.contains(sname) => streamlet.copy(replicas = Some(replicas(sname))) case _ => streamlet } } - val res = crApp.copy(spec = crApp.spec.copy(deployments = clusterDeployments)) + val res = crApp.copy(_spec = crApp.getSpec.copy(deployments = clusterDeployments)) Success(res) case missings => Failure( diff --git a/core/cloudflow-cli/src/main/scala/akka/cli/cloudflow/execution/WithUpdateVolumeMounts.scala b/core/cloudflow-cli/src/main/scala/akka/cli/cloudflow/execution/WithUpdateVolumeMounts.scala index b928ac913..cc10a98ff 100644 --- a/core/cloudflow-cli/src/main/scala/akka/cli/cloudflow/execution/WithUpdateVolumeMounts.scala +++ b/core/cloudflow-cli/src/main/scala/akka/cli/cloudflow/execution/WithUpdateVolumeMounts.scala @@ -18,7 +18,7 @@ trait WithUpdateVolumeMounts { for { streamletVolumeNameToPvc <- streamletVolumeNameToPvcMap(crApp, volumeMountsArgs, pvcs) _ <- missingStreamletVolumeMountNames(crApp, streamletVolumeNameToPvc.keys) - } yield crApp.copy(spec = crApp.spec.copy( + } yield crApp.copy(_spec = crApp.getSpec.copy( deployments = updatedDeployments(crApp, streamletVolumeNameToPvc), streamlets = updatedStreamlets(crApp, streamletVolumeNameToPvc))) } @@ -44,11 +44,11 @@ trait WithUpdateVolumeMounts { } val streamletName = parts(0) val volumeMountName = parts(1) - if (crApp.spec.deployments.find(_.streamletName == streamletName).isEmpty) { + if (crApp.getSpec.deployments.find(_.streamletName == streamletName).isEmpty) { throw new CliException(s"Cannot find streamlet '$streamletName' in --volume-mount argument") } - if (crApp.spec.deployments + if (crApp.getSpec.deployments .filter(_.streamletName == streamletName) .flatMap(_.volumeMounts) .find(_.name == volumeMountName) @@ -82,7 +82,7 @@ trait WithUpdateVolumeMounts { } private def streamletVolumeMountNamesInCr(crApp: App.Cr): Set[(String, String)] = { - crApp.spec.deployments + crApp.getSpec.deployments .flatMap(deployment => deployment.volumeMounts.map(vm => deployment.streamletName -> vm.name)) .toSet } @@ -90,7 +90,7 @@ trait WithUpdateVolumeMounts { private def updatedDeployments( crApp: App.Cr, streamletVolumeNameToPvc: Map[(String, String), String]): Seq[App.Deployment] = { - crApp.spec.deployments.map { deployment => + crApp.getSpec.deployments.map { deployment => deployment.copy(volumeMounts = deployment.volumeMounts.map { vmd => streamletVolumeNameToPvc .get((deployment.streamletName, vmd.name)) @@ -103,7 +103,7 @@ trait WithUpdateVolumeMounts { private def updatedStreamlets( crApp: App.Cr, streamletVolumeNameToPvc: Map[(String, String), String]): Seq[App.Streamlet] = { - crApp.spec.streamlets.map { streamlet => + crApp.getSpec.streamlets.map { streamlet => streamlet.copy(descriptor = streamlet.descriptor.copy(volumeMounts = streamlet.descriptor.volumeMounts.map { vmd => streamletVolumeNameToPvc diff --git a/core/cloudflow-cli/src/main/scala/akka/cli/cloudflow/kubeclient/KubeClientFabric8.scala b/core/cloudflow-cli/src/main/scala/akka/cli/cloudflow/kubeclient/KubeClientFabric8.scala index a8817d154..eef084983 100644 --- a/core/cloudflow-cli/src/main/scala/akka/cli/cloudflow/kubeclient/KubeClientFabric8.scala +++ b/core/cloudflow-cli/src/main/scala/akka/cli/cloudflow/kubeclient/KubeClientFabric8.scala @@ -178,17 +178,17 @@ class KubeClientFabric8( .find(_.getMetadata.getName == appName) .getOrElse(throw CliException(s"""Cloudflow application "${appName}" not found""")) - val appStatus: String = Try(app.status.appStatus).toOption.getOrElse("Unknown") + val appStatus: String = Try(app.getStatus.appStatus).toOption.getOrElse("Unknown") val res = models.ApplicationStatus( summary = getCRSummary(app), status = appStatus, // FIXME, remove in a breaking CRD change, the endpoint statuses are not updated anymore. - endpointsStatuses = Try(app.status.endpointStatuses).toOption + endpointsStatuses = Try(app.getStatus.endpointStatuses).toOption .filterNot(_ == null) .map(_.map(getEndpointStatus)) .getOrElse(Seq.empty), - streamletsStatuses = Try(app.status.streamletStatuses).toOption + streamletsStatuses = Try(app.getStatus.streamletStatuses).toOption .filterNot(_ == null) .map(_.map(getStreamletStatus)) .getOrElse(Seq.empty)) @@ -512,7 +512,7 @@ class KubeClientFabric8( endpointStatuses = Seq(), streamletStatuses = Seq()) - App.Cr(spec = spec, metadata = metadata, status = status) + App.Cr(_spec = spec, _metadata = metadata, _status = status) } private def createCFApp(spec: App.Spec, namespace: String): Try[String] = @@ -580,7 +580,7 @@ class KubeClientFabric8( Try { cloudflowApps .inNamespace(namespace) - .withName(app.spec.appId) + .withName(app.getSpec.appId) // NOTE: Patch doesn't work //.patch(app) .replace(app) @@ -663,7 +663,7 @@ private object ModelConversions { models.CRSummary( name = app.name, namespace = app.namespace, - version = app.spec.appVersion, + version = app.getSpec.appVersion, creationTime = app.getMetadata.getCreationTimestamp) } diff --git a/core/cloudflow-cli/src/test/scala/akka/cli/cloudflow/execution/ConfigValidationSpec.scala b/core/cloudflow-cli/src/test/scala/akka/cli/cloudflow/execution/ConfigValidationSpec.scala index 673d56a2f..515f8a35a 100644 --- a/core/cloudflow-cli/src/test/scala/akka/cli/cloudflow/execution/ConfigValidationSpec.scala +++ b/core/cloudflow-cli/src/test/scala/akka/cli/cloudflow/execution/ConfigValidationSpec.scala @@ -16,8 +16,8 @@ class ConfigValidationSpec extends AnyFlatSpec with Matchers with TryValues { def crWithCPDescriptors(cpDescriptors: Seq[App.ConfigParameterDescriptor]) = { App.Cr( - metadata = null, - spec = App.Spec( + _metadata = null, + _spec = App.Spec( appId = "", appVersion = "", deployments = Seq(), diff --git a/core/cloudflow-cli/src/test/scala/akka/cli/cloudflow/execution/WithUpdateVolumeMountsSpec.scala b/core/cloudflow-cli/src/test/scala/akka/cli/cloudflow/execution/WithUpdateVolumeMountsSpec.scala index f541d202d..16085ad14 100644 --- a/core/cloudflow-cli/src/test/scala/akka/cli/cloudflow/execution/WithUpdateVolumeMountsSpec.scala +++ b/core/cloudflow-cli/src/test/scala/akka/cli/cloudflow/execution/WithUpdateVolumeMountsSpec.scala @@ -19,8 +19,8 @@ class WithUpdateVolumesMountsSpec extends AnyFlatSpec with WithUpdateVolumeMount val streamletName = "my-streamlet" def crWithVolumeMounts(volumeMounts: Seq[App.VolumeMountDescriptor]) = { App.Cr( - metadata = null, - spec = App.Spec( + _metadata = null, + _spec = App.Spec( appId = "", appVersion = "", deployments = Seq( @@ -83,13 +83,13 @@ class WithUpdateVolumesMountsSpec extends AnyFlatSpec with WithUpdateVolumeMount // Assert val volumeMountsInDescriptor = - updatedCr.success.value.spec.streamlets + updatedCr.success.value.getSpec.streamlets .find(_.name == streamletName) .map(_.descriptor.volumeMounts) .getOrElse(Seq()) volumeMountsInDescriptor should contain(expectedVolumeMount) val volumeMountsInDeployment = - updatedCr.success.value.spec.deployments + updatedCr.success.value.getSpec.deployments .find(_.streamletName == streamletName) .map(_.volumeMounts) .getOrElse(Seq()) diff --git a/core/cloudflow-config/src/test/scala/akka/cloudflow/config/CloudflowConfigSpec.scala b/core/cloudflow-config/src/test/scala/akka/cloudflow/config/CloudflowConfigSpec.scala index a825a6274..c6c8a3971 100644 --- a/core/cloudflow-config/src/test/scala/akka/cloudflow/config/CloudflowConfigSpec.scala +++ b/core/cloudflow-config/src/test/scala/akka/cloudflow/config/CloudflowConfigSpec.scala @@ -1000,7 +1000,8 @@ class CloudflowConfigSpec extends AnyFlatSpec with Matchers with OptionValues wi val appCr = Serialization.jsonMapper().readValue(crFile, classOf[App.Cr]) // Act - val res = ConfigFactory.empty().withFallback(writeConfig(defaultMountsConfig(appCr.spec, List("flink", "spark")))) + val res = + ConfigFactory.empty().withFallback(writeConfig(defaultMountsConfig(appCr.getSpec, List("flink", "spark")))) def getPvcPath(runtime: String) = s"cloudflow.runtimes.$runtime.kubernetes.pods.pod.volumes.default.pvc.name" diff --git a/core/cloudflow-crd/src/main/scala/akka/datap/crd/App.scala b/core/cloudflow-crd/src/main/scala/akka/datap/crd/App.scala index b51913cca..2c7e8fdb3 100644 --- a/core/cloudflow-crd/src/main/scala/akka/datap/crd/App.scala +++ b/core/cloudflow-crd/src/main/scala/akka/datap/crd/App.scala @@ -92,18 +92,19 @@ object App { @Kind(Kind) @Plural(Plural) final case class Cr( - @JsonProperty("spec") - spec: Spec, + _spec: Spec, @JsonProperty("metadata") - metadata: ObjectMeta, - @JsonProperty("status") - status: AppStatus = null) + _metadata: ObjectMeta, + _status: AppStatus = null) extends CustomResource[Spec, AppStatus] with Namespaced { - this.setMetadata(metadata) - - def name: String = metadata.getName() - def namespace: String = metadata.getNamespace() + this.setMetadata(_metadata) + this.setSpec(_spec) + this.setStatus(_status) + override def initSpec = _spec + override def initStatus = _status + def name: String = getMetadata.getName() + def namespace: String = getMetadata.getNamespace() } @JsonCreator diff --git a/core/cloudflow-operator/src/main/scala/cloudflow/operator/action/Actions.scala b/core/cloudflow-operator/src/main/scala/cloudflow/operator/action/Actions.scala index 24132e279..f55f6a244 100644 --- a/core/cloudflow-operator/src/main/scala/cloudflow/operator/action/Actions.scala +++ b/core/cloudflow-operator/src/main/scala/cloudflow/operator/action/Actions.scala @@ -43,7 +43,7 @@ object Actions { podName: String, podNamespace: String, cause: ObjectReference): Seq[Action] = { - require(currentApp.forall(_.spec.appId == newApp.spec.appId)) + require(currentApp.forall(_.getSpec.appId == newApp.getSpec.appId)) val labels = CloudflowLabels(newApp) val ownerReferences = List(AppOwnerReference(newApp.name, newApp.getMetadata.getUid)) prepareNamespace(newApp, runners, labels, ownerReferences) ++ @@ -52,7 +52,7 @@ object Actions { // If an existing status is there, update status based on app (expected pod counts) // in case pod events do not occur, for instance when a operator delegated to is not responding Option(newApp.getStatus).flatMap { st => - val newStatus = CloudflowStatus.updateApp(newApp, runners).status + val newStatus = CloudflowStatus.updateApp(newApp, runners).getStatus if (newStatus != st) Some(CloudflowStatus.statusUpdateAction(newApp)()) else None }.toList ++ diff --git a/core/cloudflow-operator/src/main/scala/cloudflow/operator/action/CloudflowLabels.scala b/core/cloudflow-operator/src/main/scala/cloudflow/operator/action/CloudflowLabels.scala index 3bab287a9..73e9a8234 100644 --- a/core/cloudflow-operator/src/main/scala/cloudflow/operator/action/CloudflowLabels.scala +++ b/core/cloudflow-operator/src/main/scala/cloudflow/operator/action/CloudflowLabels.scala @@ -41,7 +41,7 @@ object CloudflowLabels { val ConfigUpdateLabel = "com.lightbend.cloudflow/config-update" def apply(app: App.Cr): CloudflowLabels = - CloudflowLabels(app.spec.appId, app.spec.appVersion) + CloudflowLabels(app.getSpec.appId, app.getSpec.appVersion) // The name of the application val Name = "app.kubernetes.io/name" diff --git a/core/cloudflow-operator/src/main/scala/cloudflow/operator/action/CloudflowStatus.scala b/core/cloudflow-operator/src/main/scala/cloudflow/operator/action/CloudflowStatus.scala index 9a6a3382e..4908347aa 100644 --- a/core/cloudflow-operator/src/main/scala/cloudflow/operator/action/CloudflowStatus.scala +++ b/core/cloudflow-operator/src/main/scala/cloudflow/operator/action/CloudflowStatus.scala @@ -154,17 +154,17 @@ object CloudflowStatus { } def pendingAction(app: App.Cr, runners: Map[String, Runner[_]], msg: String): Action = { - log.info(s"Setting pending status for app ${app.spec.appId}") + log.info(s"Setting pending status for app ${app.getSpec.appId}") val newStatus = - freshStatus(app.spec, runners).copy(appStatus = Status.Pending, appMessage = msg) + freshStatus(app.getSpec, runners).copy(appStatus = Status.Pending, appMessage = msg) app.setStatus(newStatus) statusUpdateAction(app)() } def errorAction(app: App.Cr, runners: Map[String, Runner[_]], msg: String): Action = { - log.info(s"Setting error status for app ${app.spec.appId}") - val newStatus = freshStatus(app.spec, runners) + log.info(s"Setting error status for app ${app.getSpec.appId}") + val newStatus = freshStatus(app.getSpec, runners) .copy( appStatus = Status.Error, appMessage = s"An unrecoverable error has occured, please undeploy the application. Reason: ${msg}") @@ -218,7 +218,7 @@ object CloudflowStatus { } def updateApp(newApp: App.Cr, runners: Map[String, Runner[_]]): App.Cr = { - val newStreamletStatuses = createStreamletStatuses(newApp.spec, runners).map { newStreamletStatus => + val newStreamletStatuses = createStreamletStatuses(newApp.getSpec, runners).map { newStreamletStatus => Try { newApp.getStatus.streamletStatuses } .getOrElse(Nil) .find(_.streamletName == newStreamletStatus.streamletName) @@ -228,8 +228,8 @@ object CloudflowStatus { .getOrElse(newStreamletStatus) } val newStatus = newApp.getStatus.copy( - appId = newApp.spec.appId, - appVersion = newApp.spec.appVersion, + appId = newApp.getSpec.appId, + appVersion = newApp.getSpec.appVersion, streamletStatuses = newStreamletStatuses, appStatus = calcAppStatus(newStreamletStatuses)) @@ -288,7 +288,7 @@ object CloudflowStatus { case Some(curr) => curr.setStatus(app.getStatus) - val newAppWithStatus = App.Cr(spec = null, metadata = curr.getMetadata, status = curr.getStatus) + val newAppWithStatus = App.Cr(_spec = null, _metadata = curr.getMetadata, _status = curr.getStatus) Some(current.updateStatus(newAppWithStatus)) case _ => if (retry <= 0) { diff --git a/core/cloudflow-operator/src/main/scala/cloudflow/operator/action/EndpointActions.scala b/core/cloudflow-operator/src/main/scala/cloudflow/operator/action/EndpointActions.scala index bbd32712c..55e57bc22 100644 --- a/core/cloudflow-operator/src/main/scala/cloudflow/operator/action/EndpointActions.scala +++ b/core/cloudflow-operator/src/main/scala/cloudflow/operator/action/EndpointActions.scala @@ -42,13 +42,13 @@ object EndpointActions { app.deployments.flatMap(deployment => deployment.endpoint).toSet val currentEndpoints: Set[App.Endpoint] = - currentApp.map(cr => distinctEndpoints(cr.spec)).getOrElse(Set.empty[App.Endpoint]) - val newEndpoints: Set[App.Endpoint] = distinctEndpoints(newApp.spec) + currentApp.map(cr => distinctEndpoints(cr.getSpec)).getOrElse(Set.empty[App.Endpoint]) + val newEndpoints: Set[App.Endpoint] = distinctEndpoints(newApp.getSpec) val deleteActions = (currentEndpoints -- newEndpoints).flatMap { endpoint: App.Endpoint => Seq( Action.delete[Service]( - Name.ofService(StreamletDeployment.name(newApp.spec.appId, endpoint.streamlet.getOrElse("no-name"))), + Name.ofService(StreamletDeployment.name(newApp.getSpec.appId, endpoint.streamlet.getOrElse("no-name"))), newApp.namespace)) }.toList val createActions = (newEndpoints -- currentEndpoints).flatMap { endpoint => @@ -56,7 +56,7 @@ object EndpointActions { createServiceAction( endpoint, newApp, - StreamletDeployment.name(newApp.spec.appId, endpoint.streamlet.getOrElse("no-name")))) + StreamletDeployment.name(newApp.getSpec.appId, endpoint.streamlet.getOrElse("no-name")))) }.toList deleteActions ++ createActions } diff --git a/core/cloudflow-operator/src/main/scala/cloudflow/operator/action/EventActions.scala b/core/cloudflow-operator/src/main/scala/cloudflow/operator/action/EventActions.scala index 54b544b2c..2ea1d416d 100644 --- a/core/cloudflow-operator/src/main/scala/cloudflow/operator/action/EventActions.scala +++ b/core/cloudflow-operator/src/main/scala/cloudflow/operator/action/EventActions.scala @@ -42,9 +42,9 @@ object EventActions { val (reason, message) = currentApp match { case Some(_) => - ("ApplicationUpdated", s"Updated Cloudflow Application ${app.spec.appId} to namespace ${app.namespace}") + ("ApplicationUpdated", s"Updated Cloudflow Application ${app.getSpec.appId} to namespace ${app.namespace}") case _ => - ("ApplicationDeployed", s"Deployed Cloudflow Application ${app.spec.appId} to namespace ${app.namespace}") + ("ApplicationDeployed", s"Deployed Cloudflow Application ${app.getSpec.appId} to namespace ${app.namespace}") } val deployEvent = @@ -65,8 +65,8 @@ object EventActions { cause: ObjectReference): Seq[Action] = for { currentApp <- currentAppOpt.toVector - streamlet <- app.spec.deployments - currentStreamlet <- currentApp.spec.deployments.find(_.name == streamlet.name) + streamlet <- app.getSpec.deployments + currentStreamlet <- currentApp.getSpec.deployments.find(_.name == streamlet.name) if currentStreamlet.replicas != streamlet.replicas replicas = replicasOrRunnerDefault(streamlet, runners) currentReplicas = replicasOrRunnerDefault(currentStreamlet, runners) @@ -75,7 +75,7 @@ object EventActions { podName = podName, reason = "StreamletScaled", message = - s"Scaled Cloudflow Application ${app.spec.appId} streamlet ${streamlet.name} in namespace ${app.namespace} from ${currentReplicas} to ${replicas}", + s"Scaled Cloudflow Application ${app.getSpec.appId} streamlet ${streamlet.name} in namespace ${app.namespace} from ${currentReplicas} to ${replicas}", objectReference = cause, fieldPath = Some(s"spec.deployments{${streamlet.name}}")) @@ -87,7 +87,7 @@ object EventActions { app = app, podName = podName, reason = "ApplicationUndeployed", - message = s"Undeployed Cloudflow Application ${app.spec.appId} from namespace ${app.namespace}", + message = s"Undeployed Cloudflow Application ${app.getSpec.appId} from namespace ${app.namespace}", objectReference = cause) def streamletChangeEvent(app: App.Cr, streamlet: App.Deployment, podName: String, cause: ObjectReference): Action = @@ -96,7 +96,7 @@ object EventActions { podName = podName, reason = "StreamletConfigurationChanged", message = - s"Changed streamlet configuration of Cloudflow Application ${app.spec.appId} streamlet ${streamlet.name} in namespace ${app.namespace}", + s"Changed streamlet configuration of Cloudflow Application ${app.getSpec.appId} streamlet ${streamlet.name} in namespace ${app.namespace}", objectReference = cause) private[operator] def createEvent( @@ -108,7 +108,7 @@ object EventActions { objectReference: ObjectReference, fieldPath: Option[String] = None): Action = { val eventTime = ZonedDateTime.now() - val metadataName = newEventName(podName, app.spec.appId) + val metadataName = newEventName(podName, app.getSpec.appId) // the object reference fieldPath is irrelevant for application events. fieldPath.foreach(path => objectReference.setFieldPath(path)) diff --git a/core/cloudflow-operator/src/main/scala/cloudflow/operator/action/PrepareNamespaceActions.scala b/core/cloudflow-operator/src/main/scala/cloudflow/operator/action/PrepareNamespaceActions.scala index b31b0ff41..0aac76870 100644 --- a/core/cloudflow-operator/src/main/scala/cloudflow/operator/action/PrepareNamespaceActions.scala +++ b/core/cloudflow-operator/src/main/scala/cloudflow/operator/action/PrepareNamespaceActions.scala @@ -33,7 +33,7 @@ object PrepareNamespaceActions { runners: Map[String, Runner[_]], labels: CloudflowLabels, ownerReferences: List[OwnerReference]): Seq[Action] = - app.spec.streamlets + app.getSpec.streamlets .map(streamlet => streamlet.descriptor.runtime) .distinct .flatMap { runtime => diff --git a/core/cloudflow-operator/src/main/scala/cloudflow/operator/action/TopicActions.scala b/core/cloudflow-operator/src/main/scala/cloudflow/operator/action/TopicActions.scala index 0fd9a07be..06e9b34e4 100644 --- a/core/cloudflow-operator/src/main/scala/cloudflow/operator/action/TopicActions.scala +++ b/core/cloudflow-operator/src/main/scala/cloudflow/operator/action/TopicActions.scala @@ -55,7 +55,7 @@ object TopicActions { def deploymentOf(topicId: String)(deployment: App.Deployment) = deployment.portMappings.values.exists(_.id == topicId) - val managedTopics = distinctTopics(newApp.spec) + val managedTopics = distinctTopics(newApp.getSpec) val labels = CloudflowLabels(newApp) val actions = @@ -63,7 +63,7 @@ object TopicActions { // find a config (secret) for the topic, all deployments should have the same configuration for the same topic, // since topics are defined once, deployments refer to the topics by port mappings. // So it is ok to just take the first one found, that uses the topic. - val appConfigSecretName = newApp.spec.deployments + val appConfigSecretName = newApp.getSpec.deployments .filter(deploymentOf(topic.id)) .headOption .map(_.secretName) diff --git a/core/cloudflow-operator/src/main/scala/cloudflow/operator/action/runner/AkkaRunner.scala b/core/cloudflow-operator/src/main/scala/cloudflow/operator/action/runner/AkkaRunner.scala index 930e9fc8a..ed5f7050f 100644 --- a/core/cloudflow-operator/src/main/scala/cloudflow/operator/action/runner/AkkaRunner.scala +++ b/core/cloudflow-operator/src/main/scala/cloudflow/operator/action/runner/AkkaRunner.scala @@ -58,7 +58,7 @@ final class AkkaRunner(akkaRunnerDefaults: AkkaRunnerDefaults) extends Runner[De Action.delete[Deployment](name, namespace) def appActions(app: App.Cr, labels: CloudflowLabels, ownerReferences: List[OwnerReference]): Seq[Action] = { - app.spec.serviceAccount match { + app.getSpec.serviceAccount match { case Some(_) => Seq() case _ => val roleAkka = akkaRole(app.namespace, labels, ownerReferences) @@ -169,7 +169,7 @@ final class AkkaRunner(akkaRunnerDefaults: AkkaRunnerDefaults) extends Runner[De val labels = CloudflowLabels(app) val ownerReferences = List(AppOwnerReference(app.getMetadata.getName, app.getMetadata.getUid)) - val appId = app.spec.appId + val appId = app.getSpec.appId val podName = Name.ofPod(deployment.name) val k8sStreamletPorts = deployment.endpoint @@ -195,7 +195,7 @@ final class AkkaRunner(akkaRunnerDefaults: AkkaRunnerDefaults) extends Runner[De val podsConfig = getPodsConfig(configSecret) - val streamletToDeploy = app.spec.streamlets.find(streamlet => streamlet.name == deployment.streamletName) + val streamletToDeploy = app.getSpec.streamlets.find(streamlet => streamlet.name == deployment.streamletName) val userConfiguredPorts = getContainerPorts(podsConfig, PodsConfig.CloudflowPodName) // Streamlet volume mounting (Defined by Streamlet.volumeMounts API) @@ -305,7 +305,7 @@ final class AkkaRunner(akkaRunnerDefaults: AkkaRunnerDefaults) extends Runner[De configSecretVolumes val podSpecBuilder = new PodSpecBuilder() - .withServiceAccount(app.spec.serviceAccount.getOrElse(Name.ofServiceAccount)) + .withServiceAccount(app.getSpec.serviceAccount.getOrElse(Name.ofServiceAccount)) .withVolumes(allVolumes.asJava) .withContainers(container) @@ -419,7 +419,7 @@ final class AkkaRunner(akkaRunnerDefaults: AkkaRunnerDefaults) extends Runner[De } private def createEnvironmentVariables(app: App.Cr, podsConfig: PodsConfig) = { - val agentPaths = app.spec.agentPaths + val agentPaths = app.getSpec.agentPaths val prometheusEnvVars = if (agentPaths.contains(PrometheusAgentKey)) { List( new EnvVarBuilder() diff --git a/core/cloudflow-operator/src/main/scala/cloudflow/operator/action/runner/Runner.scala b/core/cloudflow-operator/src/main/scala/cloudflow/operator/action/runner/Runner.scala index 847ee18dc..d7fb9a0f3 100644 --- a/core/cloudflow-operator/src/main/scala/cloudflow/operator/action/runner/Runner.scala +++ b/core/cloudflow-operator/src/main/scala/cloudflow/operator/action/runner/Runner.scala @@ -111,9 +111,9 @@ trait Runner[T <: HasMetadata] { def actions(newApp: App.Cr, currentApp: Option[App.Cr], runners: Map[String, Runner[_]])( implicit ct: ClassTag[T]): Seq[Action] = { - val newDeployments = newApp.spec.deployments.filter(_.runtime == runtime) + val newDeployments = newApp.getSpec.deployments.filter(_.runtime == runtime) - val currentDeployments = currentApp.map(_.spec.deployments.filter(_.runtime == runtime)).getOrElse(Vector()) + val currentDeployments = currentApp.map(_.getSpec.deployments.filter(_.runtime == runtime)).getOrElse(Vector()) val currentDeploymentNames = currentDeployments.map(_.name) val newDeploymentNames = newDeployments.map(_.name) @@ -133,7 +133,7 @@ trait Runner[T <: HasMetadata] { createOrReplaceResource(resource(deployment, newApp, secret)) case None => val msg = - s"Deployment of ${newApp.spec.appId} is pending, secret ${deployment.secretName} is missing for streamlet deployment '${deployment.name}'." + s"Deployment of ${newApp.getSpec.appId} is pending, secret ${deployment.secretName} is missing for streamlet deployment '${deployment.name}'." log.info(msg) CloudflowStatus.pendingAction( newApp, diff --git a/core/cloudflow-operator/src/main/scala/cloudflow/operator/event/AppEvent.scala b/core/cloudflow-operator/src/main/scala/cloudflow/operator/event/AppEvent.scala index f99976d46..fffe41b6d 100644 --- a/core/cloudflow-operator/src/main/scala/cloudflow/operator/event/AppEvent.scala +++ b/core/cloudflow-operator/src/main/scala/cloudflow/operator/event/AppEvent.scala @@ -35,13 +35,13 @@ sealed trait AppEvent { } case class DeployEvent(app: App.Cr, currentApp: Option[App.Cr], cause: ObjectReference) extends AppEvent { - override def toString() = s"DeployEvent for application ${app.spec.appId} in namespace ${app.namespace}" + override def toString() = s"DeployEvent for application ${app.getSpec.appId} in namespace ${app.namespace}" def toActionList(runners: Map[String, runner.Runner[_]], podName: String, podNamespace: String): Seq[Action] = Actions.deploy(app, currentApp, runners, podName, podNamespace, cause) } case class UndeployEvent(app: App.Cr, cause: ObjectReference) extends AppEvent { - override def toString() = s"UndeployEvent for application ${app.spec.appId} in namespace ${app.namespace}" + override def toString() = s"UndeployEvent for application ${app.getSpec.appId} in namespace ${app.namespace}" def toActionList(runners: Map[String, runner.Runner[_]], podName: String, podNamespace: String): Seq[Action] = Actions.undeploy(app, podName, cause) } @@ -62,7 +62,7 @@ object AppEvent { currentApps: Map[String, WatchEvent[App.Cr]], watchEvent: WatchEvent[App.Cr]): (Map[String, WatchEvent[App.Cr]], List[AppEvent]) = { val cr = watchEvent.obj - val appId = cr.spec.appId + val appId = cr.getSpec.appId val currentApp = currentApps.get(appId).map(_.obj) def hasChanged = currentApps.get(appId).forall { _existingEvent => @@ -70,7 +70,7 @@ object AppEvent { existingEventObject.getMetadata.getResourceVersion != watchEvent.obj.getMetadata.getResourceVersion && // the spec must change, otherwise it is not a deploy event (but likely a status update). - existingEventObject.spec != watchEvent.obj.spec + existingEventObject.getSpec != watchEvent.obj.getSpec } watchEvent.eventType match { diff --git a/core/cloudflow-operator/src/main/scala/cloudflow/operator/event/StatusChangeEvent.scala b/core/cloudflow-operator/src/main/scala/cloudflow/operator/event/StatusChangeEvent.scala index 339983fd4..be7b38852 100644 --- a/core/cloudflow-operator/src/main/scala/cloudflow/operator/event/StatusChangeEvent.scala +++ b/core/cloudflow-operator/src/main/scala/cloudflow/operator/event/StatusChangeEvent.scala @@ -85,9 +85,9 @@ object StatusChangeEvent extends Event { case (Some(app), statusChangeEvent) if Option(app.getStatus).flatMap(s => Option(s.appStatus)) != Some(CloudflowStatus.Status.Error) => log.debug( - s"[Status changes] Handling StatusChange for ${app.spec.appId}: ${changeInfo(statusChangeEvent.watchEvent)}.") + s"[Status changes] Handling StatusChange for ${app.getSpec.appId}: ${changeInfo(statusChangeEvent.watchEvent)}.") - val appId = app.spec.appId + val appId = app.getSpec.appId val appCr: App.Cr = currentStatuses .get(appId) @@ -96,12 +96,12 @@ object StatusChangeEvent extends Event { app.setStatus(curr.getStatus) CloudflowStatus.updateApp(app, runners) } else { - app.setStatus(CloudflowStatus.freshStatus(app.spec, runners)) + app.setStatus(CloudflowStatus.freshStatus(app.getSpec, runners)) app } } .getOrElse { - app.setStatus(CloudflowStatus.freshStatus(app.spec, runners)) + app.setStatus(CloudflowStatus.freshStatus(app.getSpec, runners)) app } diff --git a/core/cloudflow-operator/src/main/scala/cloudflow/operator/event/StreamletChangeEvent.scala b/core/cloudflow-operator/src/main/scala/cloudflow/operator/event/StreamletChangeEvent.scala index 99319d63e..3135825ae 100644 --- a/core/cloudflow-operator/src/main/scala/cloudflow/operator/event/StreamletChangeEvent.scala +++ b/core/cloudflow-operator/src/main/scala/cloudflow/operator/event/StreamletChangeEvent.scala @@ -108,10 +108,10 @@ object StreamletChangeEvent extends Event { runners: Map[String, Runner[_]]) = { import streamletChangeEvent._ val secret = watchEvent.obj - app.spec.deployments + app.getSpec.deployments .find(_.streamletName == streamletName) .map { streamletDeployment => - log.info(s"[app: ${app.spec.appId} configuration changed for streamlet $streamletName]") + log.info(s"[app: ${app.getSpec.appId} configuration changed for streamlet $streamletName]") val updateAction = runners .get(streamletDeployment.runtime) .map(_.streamletChangeAction(app, runners, streamletDeployment, secret)) diff --git a/core/cloudflow-operator/src/test/scala/cloudflow/operator/action/AppCrSpec.scala b/core/cloudflow-operator/src/test/scala/cloudflow/operator/action/AppCrSpec.scala index 343293277..ea4651f4e 100644 --- a/core/cloudflow-operator/src/test/scala/cloudflow/operator/action/AppCrSpec.scala +++ b/core/cloudflow-operator/src/test/scala/cloudflow/operator/action/AppCrSpec.scala @@ -69,11 +69,11 @@ class AppCrSpec .value val newApp = mkApp(verifiedBlueprint) - val cr = App.Cr(spec = newApp, metadata = null) + val cr = App.Cr(_spec = newApp, _metadata = null) val mapper = Serialization.jsonMapper() val str = mapper.writeValueAsString(cr) val customResource = mapper.readValue(str, classOf[App.Cr]) - customResource.spec mustBe cr.spec + customResource.getSpec mustBe cr.getSpec } "report its status as Pending when there are no pod statuses yet" in { diff --git a/core/cloudflow-operator/src/test/scala/cloudflow/operator/action/EndpointActionsSpec.scala b/core/cloudflow-operator/src/test/scala/cloudflow/operator/action/EndpointActionsSpec.scala index a52350e5b..64d4aa69f 100644 --- a/core/cloudflow-operator/src/test/scala/cloudflow/operator/action/EndpointActionsSpec.scala +++ b/core/cloudflow-operator/src/test/scala/cloudflow/operator/action/EndpointActionsSpec.scala @@ -64,8 +64,8 @@ class EndpointActionsSpec val currentApp = None val newApp = App.Cr( - spec = CloudflowApplicationSpecBuilder.create(appId, appVersion, image, verifiedBlueprint, agentPaths), - metadata = CloudflowApplicationSpecBuilder.demoMetadata) + _spec = CloudflowApplicationSpecBuilder.create(appId, appVersion, image, verifiedBlueprint, agentPaths), + _metadata = CloudflowApplicationSpecBuilder.demoMetadata) When("endpoint actions are created from a new app") val actions = EndpointActions(newApp, currentApp) @@ -74,12 +74,12 @@ class EndpointActionsSpec val createActions = actions.collect { case c: CreateServiceAction => c } val services = createActions.map(_.service) - val endpoints = newApp.spec.deployments.flatMap(_.endpoint).distinct + val endpoints = newApp.getSpec.deployments.flatMap(_.endpoint).distinct createActions.size mustBe actions.size services.size mustBe endpoints.size services.foreach { service => - assertService(service, newApp.spec) + assertService(service, newApp.getSpec) } } @@ -105,8 +105,8 @@ class EndpointActionsSpec val image = "image-1" val newApp = App.Cr( - spec = CloudflowApplicationSpecBuilder.create(appId, appVersion, image, verifiedBlueprint, agentPaths), - metadata = CloudflowApplicationSpecBuilder.demoMetadata) + _spec = CloudflowApplicationSpecBuilder.create(appId, appVersion, image, verifiedBlueprint, agentPaths), + _metadata = CloudflowApplicationSpecBuilder.demoMetadata) val currentApp = Some(newApp) When("nothing changes in the new app") @@ -136,15 +136,15 @@ class EndpointActionsSpec val newAppVersion = "43-abcdef0" val image = "image-1" val currentApp = App.Cr( - spec = CloudflowApplicationSpecBuilder.create(appId, appVersion, image, verifiedBlueprint, agentPaths), - metadata = CloudflowApplicationSpecBuilder.demoMetadata) + _spec = CloudflowApplicationSpecBuilder.create(appId, appVersion, image, verifiedBlueprint, agentPaths), + _metadata = CloudflowApplicationSpecBuilder.demoMetadata) When("the new app removes the egress") val newBp = bp.disconnect(egressRef.in).remove(egressRef.name) val newApp = App.Cr( - spec = CloudflowApplicationSpecBuilder.create(appId, appVersion, image, newBp.verified.value, agentPaths), - metadata = CloudflowApplicationSpecBuilder.demoMetadata) + _spec = CloudflowApplicationSpecBuilder.create(appId, appVersion, image, newBp.verified.value, agentPaths), + _metadata = CloudflowApplicationSpecBuilder.demoMetadata) val actions = EndpointActions(newApp, Some(currentApp)) Then("delete actions should be created") @@ -174,8 +174,8 @@ class EndpointActionsSpec val appVersion = "42-abcdef0" val image = "image-1" val currentApp = App.Cr( - spec = CloudflowApplicationSpecBuilder.create(appId, appVersion, image, verifiedBlueprint, agentPaths), - metadata = CloudflowApplicationSpecBuilder.demoMetadata) + _spec = CloudflowApplicationSpecBuilder.create(appId, appVersion, image, verifiedBlueprint, agentPaths), + _metadata = CloudflowApplicationSpecBuilder.demoMetadata) When("the new app adds an endpoint, ingress -> egress") val egressRef = egress.ref("egress") @@ -184,8 +184,8 @@ class EndpointActionsSpec .connect(Topic("foos"), egressRef.in) val newAppVersion = "43-abcdef0" val newApp = App.Cr( - spec = CloudflowApplicationSpecBuilder.create(appId, appVersion, image, newBp.verified.value, agentPaths), - metadata = CloudflowApplicationSpecBuilder.demoMetadata) + _spec = CloudflowApplicationSpecBuilder.create(appId, appVersion, image, newBp.verified.value, agentPaths), + _metadata = CloudflowApplicationSpecBuilder.demoMetadata) Then("create actions for service should be created for the new endpoint") val actions = EndpointActions(newApp, Some(currentApp)) @@ -196,7 +196,7 @@ class EndpointActionsSpec services.size mustBe 1 services.foreach { service => - assertService(service, newApp.spec) + assertService(service, newApp.getSpec) } } } diff --git a/core/cloudflow-operator/src/test/scala/cloudflow/operator/action/EventActionsSpec.scala b/core/cloudflow-operator/src/test/scala/cloudflow/operator/action/EventActionsSpec.scala index 5abc1acec..a4513aeb9 100644 --- a/core/cloudflow-operator/src/test/scala/cloudflow/operator/action/EventActionsSpec.scala +++ b/core/cloudflow-operator/src/test/scala/cloudflow/operator/action/EventActionsSpec.scala @@ -64,7 +64,7 @@ class EventActionsSpec "create event resources for a new deployed app" in { Given("a new app") val app = CloudflowApplicationSpecBuilder.create(appId, appVersion, image, verifiedBlueprint, agentPaths) - val appCr = App.Cr(spec = app, metadata = CloudflowApplicationSpecBuilder.demoMetadata) + val appCr = App.Cr(_spec = app, _metadata = CloudflowApplicationSpecBuilder.demoMetadata) When("Event actions are created from a new app") val actions = EventActions.deployEvents(appCr, None, runners, ctx.podNamespace, toObjectReference(appCr)) @@ -81,11 +81,11 @@ class EventActionsSpec "create event resources for an updated app that's already been deployed" in { Given("a new app") val appCr = App.Cr( - spec = CloudflowApplicationSpecBuilder.create(appId, appVersion, image, verifiedBlueprint, agentPaths), - metadata = CloudflowApplicationSpecBuilder.demoMetadata) + _spec = CloudflowApplicationSpecBuilder.create(appId, appVersion, image, verifiedBlueprint, agentPaths), + _metadata = CloudflowApplicationSpecBuilder.demoMetadata) val currentApp = CloudflowApplicationSpecBuilder.create(appId, appVersion, image, verifiedBlueprint, agentPaths) - val currentAppCr = App.Cr(spec = currentApp, metadata = CloudflowApplicationSpecBuilder.demoMetadata) + val currentAppCr = App.Cr(_spec = currentApp, _metadata = CloudflowApplicationSpecBuilder.demoMetadata) When("Event actions are created from a new app") val actions = @@ -105,11 +105,11 @@ class EventActionsSpec val currentApp = CloudflowApplicationSpecBuilder.create(appId, appVersion, image, verifiedBlueprint, agentPaths) val app = App.Cr( - spec = currentApp.copy(deployments = + _spec = currentApp.copy(deployments = currentApp.deployments.map(deployment => deployment.copy(replicas = Some(2)))), - metadata = CloudflowApplicationSpecBuilder.demoMetadata) + _metadata = CloudflowApplicationSpecBuilder.demoMetadata) - val currentAppCr = App.Cr(spec = currentApp, metadata = CloudflowApplicationSpecBuilder.demoMetadata) + val currentAppCr = App.Cr(_spec = currentApp, _metadata = CloudflowApplicationSpecBuilder.demoMetadata) When("Event actions are created from a new app") val actions = @@ -132,7 +132,7 @@ class EventActionsSpec "create event resources when streamlet configuration changes" in { Given("a current app") val currentApp = CloudflowApplicationSpecBuilder.create(appId, appVersion, image, verifiedBlueprint, agentPaths) - val currentAppCr = App.Cr(spec = currentApp, metadata = CloudflowApplicationSpecBuilder.demoMetadata) + val currentAppCr = App.Cr(_spec = currentApp, _metadata = CloudflowApplicationSpecBuilder.demoMetadata) When("Event actions are created for a streamlet") val action = @@ -153,7 +153,7 @@ class EventActionsSpec "create event resources for an app that is undeployed" in { Given("a current app") val currentApp = CloudflowApplicationSpecBuilder.create(appId, appVersion, image, verifiedBlueprint, agentPaths) - val currentAppCr = App.Cr(spec = currentApp, metadata = CloudflowApplicationSpecBuilder.demoMetadata) + val currentAppCr = App.Cr(_spec = currentApp, _metadata = CloudflowApplicationSpecBuilder.demoMetadata) When("Event actions are created for a streamlet") val action = EventActions.undeployEvent(currentAppCr, ctx.podName, toObjectReference(currentAppCr)) diff --git a/core/cloudflow-operator/src/test/scala/cloudflow/operator/action/ResourceNamesSpec.scala b/core/cloudflow-operator/src/test/scala/cloudflow/operator/action/ResourceNamesSpec.scala index 244307417..33b9b570c 100644 --- a/core/cloudflow-operator/src/test/scala/cloudflow/operator/action/ResourceNamesSpec.scala +++ b/core/cloudflow-operator/src/test/scala/cloudflow/operator/action/ResourceNamesSpec.scala @@ -67,8 +67,8 @@ class ResourceNamesSpec .value App.Cr( - spec = CloudflowApplicationSpecBuilder.create(appId, appVersion, image, verifiedBlueprint, agentPaths), - metadata = CloudflowApplicationSpecBuilder.demoMetadata) + _spec = CloudflowApplicationSpecBuilder.create(appId, appVersion, image, verifiedBlueprint, agentPaths), + _metadata = CloudflowApplicationSpecBuilder.demoMetadata) } val akkaRunner = new AkkaRunner(ctx.akkaRunnerDefaults) @@ -93,15 +93,15 @@ class ResourceNamesSpec .value App.Cr( - spec = CloudflowApplicationSpecBuilder.create(appId, appVersion, image, verifiedBlueprint2, agentPaths), - metadata = CloudflowApplicationSpecBuilder.demoMetadata) + _spec = CloudflowApplicationSpecBuilder.create(appId, appVersion, image, verifiedBlueprint2, agentPaths), + _metadata = CloudflowApplicationSpecBuilder.demoMetadata) } val secret = new SecretBuilder().build() "Deployments" should { "have long names truncate to 63 characters when coming from AkkaRunner" in { - val deployment = akkaRunner.resource(testApp01.spec.deployments.head, testApp01, secret) + val deployment = akkaRunner.resource(testApp01.getSpec.deployments.head, testApp01, secret) deployment.getMetadata.getName.length mustEqual 63 } @@ -110,7 +110,7 @@ class ResourceNamesSpec "Pod templates" should { "have long names truncate to 63 characters when coming from AkkaRunner" in { - val deployment = akkaRunner.resource(testApp01.spec.deployments.head, testApp01, secret) + val deployment = akkaRunner.resource(testApp01.getSpec.deployments.head, testApp01, secret) deployment.getSpec.getTemplate.getMetadata.getName.length mustEqual 63 @@ -119,7 +119,7 @@ class ResourceNamesSpec "Containers" should { "have long names truncate to 63 characters when coming from AkkaRunner" in { - val deployment = akkaRunner.resource(testApp01.spec.deployments.head, testApp01, secret) + val deployment = akkaRunner.resource(testApp01.getSpec.deployments.head, testApp01, secret) deployment.getSpec.getTemplate.getSpec.getContainers.asScala.head.getName.length mustEqual 63 @@ -128,7 +128,7 @@ class ResourceNamesSpec "Volumes" should { "have long names truncate to 253 characters when coming from AkkaRunner" in { - val deployment = akkaRunner.resource(testApp01.spec.deployments.head, testApp01, secret) + val deployment = akkaRunner.resource(testApp01.getSpec.deployments.head, testApp01, secret) deployment.getSpec.getTemplate.getSpec.getVolumes.asScala.foreach { vol => assert(vol.getName.length <= 253) @@ -139,7 +139,7 @@ class ResourceNamesSpec "Volume mounts" should { "have long names truncate to 253 characters when coming from AkkaRunner" in { - val deployment = akkaRunner.resource(testApp01.spec.deployments.head, testApp01, secret) + val deployment = akkaRunner.resource(testApp01.getSpec.deployments.head, testApp01, secret) deployment.getSpec.getTemplate.getSpec.getContainers.asScala.head.getVolumeMounts.asScala.foreach { mount => assert(mount.getName.length <= 253) diff --git a/core/cloudflow-operator/src/test/scala/cloudflow/operator/action/TopicActionsSpec.scala b/core/cloudflow-operator/src/test/scala/cloudflow/operator/action/TopicActionsSpec.scala index f8095485e..40596be8c 100644 --- a/core/cloudflow-operator/src/test/scala/cloudflow/operator/action/TopicActionsSpec.scala +++ b/core/cloudflow-operator/src/test/scala/cloudflow/operator/action/TopicActionsSpec.scala @@ -85,10 +85,10 @@ class TopicActionsSpec .collect { case act: CreateOrReplaceAction[TopicActions.TopicResource] => act } .head } - val topics = newApp.spec.deployments.flatMap(_.portMappings.values).distinct + val topics = newApp.getSpec.deployments.flatMap(_.portMappings.values).distinct createActions.size mustBe actions.size // topics must be created to connect ingress, processor, egress - createActions.size mustBe newApp.spec.deployments.flatMap(d => d.portMappings.values.map(_.id)).distinct.size + createActions.size mustBe newApp.getSpec.deployments.flatMap(d => d.portMappings.values.map(_.id)).distinct.size topics.foreach { topic => val resource = createActions .find(_.resource.getMetadata.getName == s"topic-${topic.id}") @@ -96,7 +96,7 @@ class TopicActionsSpec .value .resource .asInstanceOf[TopicActions.TopicResource] - assertTopic(topic, resource, newApp.spec.appId) + assertTopic(topic, resource, newApp.getSpec.appId) } } @@ -129,11 +129,11 @@ class TopicActionsSpec metadata.setNamespace("testing-app") val newApp = App.Cr( - spec = CloudflowApplicationSpecBuilder.create(appId, newAppVersion, image, newBp.verified.value, agentPaths), - metadata = metadata) - val in = newApp.spec.deployments.find(_.streamletName == "processor").value.portMappings(processor.in.name) + _spec = CloudflowApplicationSpecBuilder.create(appId, newAppVersion, image, newBp.verified.value, agentPaths), + _metadata = metadata) + val in = newApp.getSpec.deployments.find(_.streamletName == "processor").value.portMappings(processor.in.name) val savepoint = - newApp.spec.deployments.find(_.streamletName == "processor").value.portMappings(processor.out.name) + newApp.getSpec.deployments.find(_.streamletName == "processor").value.portMappings(processor.out.name) Then("create actions for both topics should be created for the new savepoint between processor and egress") val Seq(foosAction, barsAction) = TopicActions(newApp, runners, ctx.podNamespace) @@ -223,10 +223,10 @@ class TopicActionsSpec .collect { case act: CreateOrReplaceAction[TopicActions.TopicResource] => act } .head } - val topics = newApp.spec.deployments.flatMap(_.portMappings.values).distinct + val topics = newApp.getSpec.deployments.flatMap(_.portMappings.values).distinct createActions.size mustBe actions.size // topics must be created to connect ingress, processor, egress - createActions.size mustBe newApp.spec.deployments.flatMap(d => d.portMappings.values.map(_.id)).distinct.size + createActions.size mustBe newApp.getSpec.deployments.flatMap(d => d.portMappings.values.map(_.id)).distinct.size topics.foreach { topic => val resource = createActions .find(_.resource.getMetadata.getName == s"topic-${topic.id}") @@ -236,7 +236,7 @@ class TopicActionsSpec assertTopic( topic, resource, - newApp.spec.appId, + newApp.getSpec.appId, bootstrapServers = "localhost:19092", partitions = 100, replicas = 3) @@ -293,7 +293,7 @@ class TopicActionsSpec metadata.setNamespace("testing-app") App.Cr( - spec = CloudflowApplicationSpecBuilder.create(appId, appVersion, image, verifiedBlueprint, agentPaths), - metadata = metadata) + _spec = CloudflowApplicationSpecBuilder.create(appId, appVersion, image, verifiedBlueprint, agentPaths), + _metadata = metadata) } } diff --git a/core/cloudflow-operator/src/test/scala/cloudflow/operator/action/runner/AkkaRunnerSpec.scala b/core/cloudflow-operator/src/test/scala/cloudflow/operator/action/runner/AkkaRunnerSpec.scala index f6164e053..2c20ddc2a 100644 --- a/core/cloudflow-operator/src/test/scala/cloudflow/operator/action/runner/AkkaRunnerSpec.scala +++ b/core/cloudflow-operator/src/test/scala/cloudflow/operator/action/runner/AkkaRunnerSpec.scala @@ -69,8 +69,8 @@ class AkkaRunnerSpec .value val app = App.Cr( - spec = CloudflowApplicationSpecBuilder.create(appId, appVersion, image, verifiedBlueprint, agentPaths), - metadata = CloudflowApplicationSpecBuilder.demoMetadata) + _spec = CloudflowApplicationSpecBuilder.create(appId, appVersion, image, verifiedBlueprint, agentPaths), + _metadata = CloudflowApplicationSpecBuilder.demoMetadata) val deployment = App.Deployment( name = appId, diff --git a/core/cloudflow-operator/src/test/scala/cloudflow/operator/action/runner/RunnerActionsSpec.scala b/core/cloudflow-operator/src/test/scala/cloudflow/operator/action/runner/RunnerActionsSpec.scala index 9de1fdb8e..0ab5d187e 100644 --- a/core/cloudflow-operator/src/test/scala/cloudflow/operator/action/runner/RunnerActionsSpec.scala +++ b/core/cloudflow-operator/src/test/scala/cloudflow/operator/action/runner/RunnerActionsSpec.scala @@ -71,8 +71,8 @@ class RunnerActionsSpec val currentApp = None val newApp = App.Cr( - spec = CloudflowApplicationSpecBuilder.create(appId, appVersion, image, verifiedBlueprint, agentPaths), - metadata = CloudflowApplicationSpecBuilder.demoMetadata) + _spec = CloudflowApplicationSpecBuilder.create(appId, appVersion, image, verifiedBlueprint, agentPaths), + _metadata = CloudflowApplicationSpecBuilder.demoMetadata) When("runner actions are created from a new app") val actions = akkaRunner.actions(newApp, currentApp, runners) @@ -92,13 +92,13 @@ class RunnerActionsSpec } val akkaDeployments = createDeploymentActions.map(_.resource) - val streamletDeployments = newApp.spec.deployments + val streamletDeployments = newApp.getSpec.deployments createActions.size + createDeploymentActions.size mustBe actions.size configMaps.size mustBe 0 akkaDeployments.size mustBe streamletDeployments.size akkaDeployments.foreach { deployment => - assertAkkaDeployment(deployment, configMaps, newApp.spec, appId, ctx) + assertAkkaDeployment(deployment, configMaps, newApp.getSpec, appId, ctx) } } @@ -123,8 +123,8 @@ class RunnerActionsSpec val image = "image-1" val newApp = App.Cr( - spec = CloudflowApplicationSpecBuilder.create(appId, appVersion, image, verifiedBlueprint, agentPaths), - metadata = CloudflowApplicationSpecBuilder.demoMetadata) + _spec = CloudflowApplicationSpecBuilder.create(appId, appVersion, image, verifiedBlueprint, agentPaths), + _metadata = CloudflowApplicationSpecBuilder.demoMetadata) val currentApp = Some(newApp) When("nothing changes in the new app") @@ -162,15 +162,15 @@ class RunnerActionsSpec val image = "image-1" val newAppVersion = appVersion // to compare configmap contents easier. val currentApp = App.Cr( - spec = CloudflowApplicationSpecBuilder.create(appId, appVersion, image, verifiedBlueprint, agentPaths), - metadata = CloudflowApplicationSpecBuilder.demoMetadata) + _spec = CloudflowApplicationSpecBuilder.create(appId, appVersion, image, verifiedBlueprint, agentPaths), + _metadata = CloudflowApplicationSpecBuilder.demoMetadata) When("the new app removes the egress") val newBp = bp.disconnect(egressRef.in).remove(egressRef.name) val newApp = App.Cr( - spec = CloudflowApplicationSpecBuilder.create(appId, newAppVersion, image, newBp.verified.value, agentPaths), - metadata = CloudflowApplicationSpecBuilder.demoMetadata) + _spec = CloudflowApplicationSpecBuilder.create(appId, newAppVersion, image, newBp.verified.value, agentPaths), + _metadata = CloudflowApplicationSpecBuilder.demoMetadata) val actions = akkaRunner.actions(newApp, Some(currentApp), runners) Then("delete actions should be created") @@ -194,8 +194,8 @@ class RunnerActionsSpec val appVersion = "42-abcdef0" val image = "image-1" val currentApp = App.Cr( - spec = CloudflowApplicationSpecBuilder.create(appId, appVersion, image, verifiedBlueprint, agentPaths), - metadata = CloudflowApplicationSpecBuilder.demoMetadata) + _spec = CloudflowApplicationSpecBuilder.create(appId, appVersion, image, verifiedBlueprint, agentPaths), + _metadata = CloudflowApplicationSpecBuilder.demoMetadata) When("the new app adds a runner, ingress -> egress") val egressRef = egress.ref("egress") @@ -204,8 +204,8 @@ class RunnerActionsSpec .connect(BTopic("foos"), egressRef.in) val newAppVersion = appVersion // to compare configmap contents easier. val newApp = App.Cr( - spec = CloudflowApplicationSpecBuilder.create(appId, newAppVersion, image, newBp.verified.value, agentPaths), - metadata = CloudflowApplicationSpecBuilder.demoMetadata) + _spec = CloudflowApplicationSpecBuilder.create(appId, newAppVersion, image, newBp.verified.value, agentPaths), + _metadata = CloudflowApplicationSpecBuilder.demoMetadata) Then("create actions for runner resources should be created for the new endpoint") val actions = akkaRunner.actions(newApp, Some(currentApp), runners) @@ -228,7 +228,7 @@ class RunnerActionsSpec akkaDeployments.size mustBe 2 akkaDeployments.foreach { deployment => - assertAkkaDeployment(deployment, configMaps, newApp.spec, appId, ctx) + assertAkkaDeployment(deployment, configMaps, newApp.getSpec, appId, ctx) } } } diff --git a/core/cloudflow-runner/src/main/scala/cloudflow/runner/Runner.scala b/core/cloudflow-runner/src/main/scala/cloudflow/runner/Runner.scala index 213b5b6a2..90415a2af 100644 --- a/core/cloudflow-runner/src/main/scala/cloudflow/runner/Runner.scala +++ b/core/cloudflow-runner/src/main/scala/cloudflow/runner/Runner.scala @@ -19,12 +19,12 @@ package cloudflow.runner import scala.util.{ Failure, Success, Try } import scala.concurrent.Await import scala.concurrent.duration._ +import scala.io.Source import java.nio.file.{ Files, Paths } import org.slf4j.LoggerFactory -import com.typesafe.config.Config +import com.typesafe.config.{ Config, ConfigFactory } import cloudflow.streamlets._ -import cloudflow.blueprint.RunnerConfigUtils._ /** * Runner for cluster deployments. Assumes Linux-style paths! @@ -101,4 +101,43 @@ object Runner extends RunnerConfigResolver with StreamletLoader { """.stripMargin } + def addStorageConfig(config: Config, pvcVolumeMountPath: String): Config = { + val storageConfig = ConfigFactory.parseString(s"""$StorageMountPathKey:"$pvcVolumeMountPath"""") + config.withFallback(storageConfig) + } + + def addPodRuntimeConfig(config: Config, downwardApiVolumeMountPath: String): Config = { + val (name, namespace, uid) = getPodMetadata(downwardApiVolumeMountPath) + val podRuntimeConfig = ConfigFactory.parseString(s""" + |cloudflow.runner.pod: { + | $MetadataName:"$name" + | $MetadataNamespace:"$namespace" + | $MetadataUid:"$uid" + |} + |""".stripMargin) + config.withFallback(podRuntimeConfig) + } + def getPodMetadata(downwardApiVolumeMountPath: String): (String, String, String) = { + val name = readDownwardApi(downwardApiVolumeMountPath, MetadataName) + val namespace = readDownwardApi(downwardApiVolumeMountPath, MetadataNamespace) + val uid = readDownwardApi(downwardApiVolumeMountPath, MetadataUid) + (name, namespace, uid) + } + + private def readDownwardApi(downwardApiVolumeMountPath: String, filename: String): String = { + val path = s"$downwardApiVolumeMountPath/$filename" + Try(Source.fromFile(path).getLines().mkString) match { + case Success(contents) => contents + case Failure(ex) => + throw new Exception( + s"An error occurred while attempting to access the downward API volume mount with path '$path'", + ex) + } + } + + val StorageMountPathKey = "storage.mountPath" + val MetadataName = "metadata.name" + val MetadataNamespace = "metadata.namespace" + val MetadataUid = "metadata.uid" + } diff --git a/core/project/Dependencies.scala b/core/project/Dependencies.scala index 66ed42de4..8b3e70d35 100644 --- a/core/project/Dependencies.scala +++ b/core/project/Dependencies.scala @@ -13,7 +13,7 @@ object Dependencies { val alpakkaKafka = "2.1.1" val akkaMgmt = "1.0.8" val spark = "2.4.5" - val fabric8 = "5.0.0" + val fabric8 = "5.0.3" val jackson = "2.12.6" val jacksonFabric8 = "2.11.4" val slf4j = "1.7.30" @@ -103,7 +103,7 @@ object Dependencies { val mavenPluginAnnotations = "org.apache.maven.plugin-tools" % "maven-plugin-annotations" % "3.6.1" val mavenProject = "org.apache.maven" % "maven-project" % "2.2.1" val mojoExecutor = "org.twdata.maven" % "mojo-executor" % "2.3.1" - val junit = "junit" % "junit" % "4.13" + val junit = "junit" % "junit" % "4.13.1" } object TestDeps { diff --git a/core/project/plugins.sbt b/core/project/plugins.sbt index b03b5b5d8..b9c1464e1 100644 --- a/core/project/plugins.sbt +++ b/core/project/plugins.sbt @@ -18,7 +18,7 @@ libraryDependencies ++= Seq( addSbtPlugin("com.julianpeeters" % "sbt-avrohugger" % "2.0.0-RC25") addSbtPlugin("com.thesamet" % "sbt-protoc" % "1.0.5") -libraryDependencies += "com.thesamet.scalapb" %% "compilerplugin" % "0.11.7" +libraryDependencies += "com.thesamet.scalapb" %% "compilerplugin" % "0.11.8" addSbtPlugin("com.lucidchart" % "sbt-cross" % "4.0") addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "0.15.0") diff --git a/docs/docs-source/README.adoc b/docs/docs-source/README.adoc index ef66ceea6..1e535757f 100644 --- a/docs/docs-source/README.adoc +++ b/docs/docs-source/README.adoc @@ -137,28 +137,30 @@ As a general rule, limit the number of projects at this location: one project pe === Including a file, or file section, in the documentation -For a file at `examples/snippets/modules/ROOT/examples/spark-scala/src/main/scala/com/example/Feature.scala`, the following statement is used to include the whole file: +For a file at `examples/snippets/modules/ROOT/examples/akkastreams-scala/src/main/scala/cloudflow/akkastreamsdoc/RecordSumFlow.scala`, the following statement is used to include the whole file: ``` - include::{cloudflow-examples-version}@docsnippets:ROOT:example$spark-scala/src/main/scala/com/example/Feature.scala[] + include::{cloudflow-examples-version}@docsnippets:ROOT:example$akkastreams-scala/src/main/scala/cloudflow/akkastreamsdoc/RecordSumFlow.scala[] ``` In most case, including the complete file is not optimal. It is better to include only section of it. To do this, use tags. With the following file ``` -package com.example - -class Feature { - //tag::something_interesting[] - val abc = new Greatness() - //end::something_interesting[] -} +object RecordSumFlow extends AkkaStreamlet { + + //tag::definition[] + val recordsInWindowParameter = IntegerConfigParameter( + "records-in-window", + "This value describes how many records of data should be processed together, default 64 records", + Some(64) + ) + //end::definition[] ``` the relevant part is added using: ``` - include::{cloudflow-examples-version}@docsnippets:ROOT:example$spark-scala/src/main/scala/com/example/Feature.scala[tag=something_interesting] + include::{cloudflow-examples-version}@docsnippets:ROOT:example$akkastreams-scala/src/main/scala/cloudflow/akkastreamsdoc/RecordSumFlow.scala[tag=definition] ``` Antora has a good support for the tags: diff --git a/docs/docs-source/docs/modules/ROOT/nav.adoc b/docs/docs-source/docs/modules/ROOT/nav.adoc index 65ffd590b..8ee32a7b7 100644 --- a/docs/docs-source/docs/modules/ROOT/nav.adoc +++ b/docs/docs-source/docs/modules/ROOT/nav.adoc @@ -2,7 +2,6 @@ include::partial$include.adoc[] .Cloudflow Docs - version {cloudflow-version} * xref:index.adoc[Introducing Cloudflow] -* xref:app-development-process.adoc[The Cloudflow development process] * xref:get-started:index.adoc[Getting started with Cloudflow] ** xref:get-started:prepare-development-environment.adoc[Prepare a development environment] @@ -30,12 +29,6 @@ include::partial$include.adoc[] **** xref:develop:akka-streamlet-utilities.adoc[Akka Streamlet utilities] **** xref:develop:clustering-akka-streamlet.adoc[Clustering Akka Streamlet] **** xref:develop:test-akka-streamlet.adoc[Testing an Akka Streamlet] -*** xref:develop:use-spark-streamlets.adoc[Using Spark Streamlets] -**** xref:develop:build-spark-streamlets.adoc[Building a Spark Streamlet] -**** xref:develop:test-spark-streamlet.adoc[Testing a Spark Streamlet] -*** xref:develop:use-flink-streamlets.adoc[Using Flink Streamlets] -**** xref:develop:build-flink-streamlets.adoc[Building a Flink Streamlet] -**** xref:develop:test-flink-streamlet.adoc[Testing a Flink Streamlet] ** xref:develop:cloudflow-configuration.adoc[Configuring Streamlets] *** xref:develop:cloudflow-configuration-kubernetes.adoc[Kubernetes Configuration] *** xref:develop:cloudflow-configuration-scopes.adoc[] @@ -53,8 +46,6 @@ include::partial$include.adoc[] ** xref:administration:versions.adoc[Versions] ** xref:administration:installing-cloudflow.adoc[Installing Cloudflow] ** xref:administration:how-to-install-and-use-strimzi.adoc[Installing Kafka with Strimzi] -** xref:administration:installing-spark-operator.adoc[Adding Spark Support] -** xref:administration:installing-flink-operator.adoc[Adding Flink Support] ** xref:administration:providing-external-access-to-cloudflow-services.adoc[Setting up external access] * xref:project-info:index.adoc[Release notes and upgrading] diff --git a/docs/docs-source/docs/modules/ROOT/pages/app-development-process.adoc b/docs/docs-source/docs/modules/ROOT/pages/app-development-process.adoc deleted file mode 100644 index 4028d624d..000000000 --- a/docs/docs-source/docs/modules/ROOT/pages/app-development-process.adoc +++ /dev/null @@ -1,6 +0,0 @@ -= The Cloudflow development process -:imagesdir: assets/images - -include::ROOT:partial$include.adoc[] -include::{cloudflow-version}@shared-content:ROOT:page$app-development-process.adoc[] - diff --git a/docs/docs-source/docs/modules/ROOT/pages/index.adoc b/docs/docs-source/docs/modules/ROOT/pages/index.adoc index 0001de39e..ef3729e88 100644 --- a/docs/docs-source/docs/modules/ROOT/pages/index.adoc +++ b/docs/docs-source/docs/modules/ROOT/pages/index.adoc @@ -14,7 +14,6 @@ Cloudflow alleviates that pain for distributed streaming applications, which are link:https://www.youtube.com/watch?v=-9pVwCkkE1I[Watch on YouTube ,role=yt-widget] -Cloudflow integrates with popular streaming engines like Akka, Apache Spark, and Apache Flink. Using Cloudflow, you can easily break down your streaming application into small composable components and wire them together with schema-based contracts. With those powerful abstractions, Cloudflow enables you to define, build and deploy the most complex streaming applications. @@ -52,7 +51,7 @@ Cloudflow dramatically accelerates your application development efforts, reducin * The Cloudflow application development toolkit includes: ** An SDK definition for `Streamlet`, the core abstraction in Cloudflow. -** An extensible set of runtime implementations for `Streamlet`(s). Cloudflow provides support for popular streaming runtimes, like Apache Spark Structured Streaming, Apache Flink, and Akka. +** An extensible set of runtime implementations for `Streamlet`(s). ** A `Streamlet` composition model driven by a `blueprint` definition. ** A _Sandbox_ execution mode that accelerates the execution and testing of your applications on your development machine. ** A set of `sbt` plugins for packaging Cloudflow applications into deployable containers and generating an application descriptor that informs the deployment. @@ -73,7 +72,6 @@ NOTE: The default build tool for Cloudflow applications is `sbt` but there is su Choose from the following topics: -* xref:app-development-process.adoc[The Cloudflow development process] * xref:get-started:index.adoc[How to get started with an example application] * xref:develop:index.adoc[How to create Cloudflow applications using streamlets and blueprints]. diff --git a/docs/docs-source/docs/modules/administration/pages/index.adoc b/docs/docs-source/docs/modules/administration/pages/index.adoc index 43772c889..767e0c010 100644 --- a/docs/docs-source/docs/modules/administration/pages/index.adoc +++ b/docs/docs-source/docs/modules/administration/pages/index.adoc @@ -3,15 +3,13 @@ include::ROOT:partial$include.adoc[] -When you develop your Cloudflow application, you have the option to use Akka Streams, Spark, Flink and Kafka. This chapter contains the steps to install Cloudflow, using the `Helm` and `kubectl` command-line tools. It also shows how to install Kafka, Spark, and Flink operators that integrate with Cloudflow and how to expose your services to external access. +This chapter contains the steps to install Cloudflow, using the `Helm` and `kubectl` command-line tools. It also shows how to integrate with Cloudflow and how to expose your services to external access. * xref:installation-prerequisites.adoc[] * xref:versions.adoc[] * xref:installing-cloudflow.adoc[] * xref:how-to-install-and-use-strimzi.adoc[] -* xref:installing-spark-operator.adoc[] -* xref:installing-flink-operator.adoc[] * xref:providing-external-access-to-cloudflow-services.adoc[] For information on upgrading Cloudflow and migrating applications, see: diff --git a/docs/docs-source/docs/modules/administration/pages/installing-flink-operator.adoc b/docs/docs-source/docs/modules/administration/pages/installing-flink-operator.adoc deleted file mode 100644 index b262a7ee9..000000000 --- a/docs/docs-source/docs/modules/administration/pages/installing-flink-operator.adoc +++ /dev/null @@ -1,6 +0,0 @@ -= Adding Flink Support - - -include::ROOT:partial$include.adoc[] - -include::{cloudflow-version}@shared-content:administration:page$installing-flink-operator.adoc[] diff --git a/docs/docs-source/docs/modules/administration/pages/installing-spark-operator.adoc b/docs/docs-source/docs/modules/administration/pages/installing-spark-operator.adoc deleted file mode 100644 index 847fe3c13..000000000 --- a/docs/docs-source/docs/modules/administration/pages/installing-spark-operator.adoc +++ /dev/null @@ -1,5 +0,0 @@ -= Adding Spark Support - - -include::ROOT:partial$include.adoc[] -include::{cloudflow-version}@shared-content:administration:page$installing-spark-operator.adoc[] diff --git a/docs/docs-source/docs/modules/administration/pages/more-storage-options.adoc b/docs/docs-source/docs/modules/administration/pages/more-storage-options.adoc deleted file mode 100644 index 78d29f38d..000000000 --- a/docs/docs-source/docs/modules/administration/pages/more-storage-options.adoc +++ /dev/null @@ -1,6 +0,0 @@ -= More Storage Options - - - -include::ROOT:partial$include.adoc[] -include::{cloudflow-version}@shared-content:administration:page$more-storage-options.adoc[] diff --git a/docs/docs-source/docs/modules/cli/pages/cloudflow.adoc b/docs/docs-source/docs/modules/cli/pages/cloudflow.adoc index 247684483..7a5ff7bab 100644 --- a/docs/docs-source/docs/modules/cli/pages/cloudflow.adoc +++ b/docs/docs-source/docs/modules/cli/pages/cloudflow.adoc @@ -32,6 +32,8 @@ list [options] list available cloudflow applications * `` -n, --namespace ``the namespace to be used + * `` --operator-namespace + ``the namespace where the operator is deployed * `` -o, --output ``available options are: c or classic, t or table, json, yaml == Command: status @@ -45,6 +47,8 @@ show the status of a cloudflow application * `` ``the name of the cloudflow application * `` -n, --namespace ``the namespace to be used + * `` --operator-namespace + ``the namespace where the operator is deployed * `` -o, --output ``available options are: c or classic, t or table, json, yaml == Command: deploy @@ -69,8 +73,11 @@ deploy a cloudflow applications from a cr file * `` config-key ``the configuration keys for the overrides * `` --logback-config ``the logback configuration to be applied - * `` --microservices ``EXPERIMENTAL: Deploy on Akka Cloud Platform + * `` --serviceaccount + ``the serviceaccount to be used * `` -n, --namespace ``the namespace to be used + * `` --operator-namespace + ``the namespace where the operator is deployed * `` -o, --output ``available options are: c or classic, t or table, json, yaml == Command: undeploy @@ -84,6 +91,8 @@ undeploy a cloudflow application * `` ``the name of the cloudflow application * `` -n, --namespace ``the namespace to be used + * `` --operator-namespace + ``the namespace where the operator is deployed * `` -o, --output ``available options are: c or classic, t or table, json, yaml == Command: update-docker-credentials @@ -102,6 +111,8 @@ updates docker registry credentials that are used to pull Cloudflow application * `` -p, --password ``the docker registry password * `` --password-stdin ``Take the docker registry password from std-in * `` -n, --namespace ``the namespace to be used + * `` --operator-namespace + ``the namespace where the operator is deployed * `` -o, --output ``available options are: c or classic, t or table, json, yaml == Command: scale @@ -116,6 +127,8 @@ scales a streamlet of a deployed Cloudflow application to the specified number o * `` ``the cloudflow application * `` = ``Key/value pairs of streamlets replicas * `` -n, --namespace ``the namespace to be used + * `` --operator-namespace + ``the namespace where the operator is deployed * `` -o, --output ``available options are: c or classic, t or table, json, yaml == Command: configure @@ -133,8 +146,11 @@ configures a deployed cloudflow application * `` config-key ``the configuration keys for the overrides * `` --logback-config ``the logback configuration to be applied - * `` --microservices ``EXPERIMENTAL: Deploy on Akka Cloud Platform + * `` --operator-namespace + ``the namespace where the operator is deployed * `` -n, --namespace ``the namespace to be used + * `` --operator-namespace + ``the namespace where the operator is deployed * `` -o, --output ``available options are: c or classic, t or table, json, yaml == Command: configuration @@ -148,4 +164,6 @@ show the current configuration of a cloudflow application * `` ``the name of the cloudflow application * `` -n, --namespace ``the namespace to be used + * `` --operator-namespace + ``the namespace where the operator is deployed * `` -o, --output ``available options are: c or classic, t or table, json, yaml diff --git a/docs/docs-source/docs/modules/develop/pages/build-flink-streamlets.adoc b/docs/docs-source/docs/modules/develop/pages/build-flink-streamlets.adoc deleted file mode 100644 index 2df0386e8..000000000 --- a/docs/docs-source/docs/modules/develop/pages/build-flink-streamlets.adoc +++ /dev/null @@ -1,8 +0,0 @@ -= Building a Flink Streamlet - -:page-supergroup-scala-java: Language - -include::ROOT:partial$include.adoc[] -include::{cloudflow-version}@shared-content:develop:page$build-flink-streamlets.adoc[] - -// Authors: content in this file is included from the ./../shared-content-source directory. Please see the readme in that folder before editing shared content. diff --git a/docs/docs-source/docs/modules/develop/pages/build-spark-streamlets.adoc b/docs/docs-source/docs/modules/develop/pages/build-spark-streamlets.adoc deleted file mode 100644 index e46369984..000000000 --- a/docs/docs-source/docs/modules/develop/pages/build-spark-streamlets.adoc +++ /dev/null @@ -1,8 +0,0 @@ -= Building a Spark Streamlet - - -include::ROOT:partial$include.adoc[] -include::{cloudflow-version}@shared-content:develop:page$build-spark-streamlets.adoc[] - -// Authors: content in this file is included from the ./../shared-content-source directory. Please see the readme in that folder before editing shared content. - diff --git a/docs/docs-source/docs/modules/develop/pages/developing-streamlets.adoc b/docs/docs-source/docs/modules/develop/pages/developing-streamlets.adoc index 742e64833..12b448fa1 100644 --- a/docs/docs-source/docs/modules/develop/pages/developing-streamlets.adoc +++ b/docs/docs-source/docs/modules/develop/pages/developing-streamlets.adoc @@ -6,7 +6,7 @@ include::{cloudflow-version}@shared-content:develop:page$developing-streamlets.a == What's next -Learn more about the specifics of using xref:use-akka-streamlets.adoc[Akka], xref:use-spark-streamlets.adoc[Spark], and xref:use-flink-streamlets.adoc[Flink] streamlets. +Learn more about the specifics of using xref:use-akka-streamlets.adoc[Akka] streamlets. Cloudflow also offers a way to test your application without the need of a cluster in a xref:cloudflow-local-sandbox.adoc[local sandbox]. // Authors: content in this file is included from the ./../shared-content-source directory. Please see the readme in that folder before editing shared content. diff --git a/docs/docs-source/docs/modules/develop/pages/test-flink-streamlet.adoc b/docs/docs-source/docs/modules/develop/pages/test-flink-streamlet.adoc deleted file mode 100644 index 1bf7e9f37..000000000 --- a/docs/docs-source/docs/modules/develop/pages/test-flink-streamlet.adoc +++ /dev/null @@ -1,7 +0,0 @@ -= Testing a Flink Streamlet - - -include::ROOT:partial$include.adoc[] -include::{cloudflow-version}@shared-content:develop:page$test-flink-streamlet.adoc[] - -// Authors: content in this file is included from the ./../shared-content-source directory. Please see the readme in that folder before editing shared content. diff --git a/docs/docs-source/docs/modules/develop/pages/test-spark-streamlet.adoc b/docs/docs-source/docs/modules/develop/pages/test-spark-streamlet.adoc deleted file mode 100644 index a1cf1f30c..000000000 --- a/docs/docs-source/docs/modules/develop/pages/test-spark-streamlet.adoc +++ /dev/null @@ -1,8 +0,0 @@ -= Testing a Spark Streamlet - - -include::ROOT:partial$include.adoc[] -include::{cloudflow-version}@shared-content:develop:page$test-spark-streamlet.adoc[] - -// Authors: content in this file is included from the ./../shared-content-source directory. Please see the readme in that folder before editing shared content. - diff --git a/docs/docs-source/docs/modules/develop/pages/use-flink-streamlets.adoc b/docs/docs-source/docs/modules/develop/pages/use-flink-streamlets.adoc deleted file mode 100644 index 0289e2580..000000000 --- a/docs/docs-source/docs/modules/develop/pages/use-flink-streamlets.adoc +++ /dev/null @@ -1,7 +0,0 @@ -= Using Flink Streamlets - - -include::ROOT:partial$include.adoc[] -include::{cloudflow-version}@shared-content:develop:page$use-flink-streamlets.adoc[] - -// Authors: content in this file is included from the ./../shared-content-source directory. Please see the readme in that folder before editing shared content. \ No newline at end of file diff --git a/docs/docs-source/docs/modules/develop/pages/use-spark-streamlets.adoc b/docs/docs-source/docs/modules/develop/pages/use-spark-streamlets.adoc deleted file mode 100644 index 277524ebe..000000000 --- a/docs/docs-source/docs/modules/develop/pages/use-spark-streamlets.adoc +++ /dev/null @@ -1,7 +0,0 @@ -= Using Spark Streamlets - - -include::ROOT:partial$include.adoc[] -include::{cloudflow-version}@shared-content:develop:page$use-spark-streamlets.adoc[] - -// Authors: content in this file is included from the ./../shared-content-source directory. Please see the readme in that folder before editing shared content. diff --git a/docs/docs-source/docs/modules/get-started/pages/setup-example-project-configure-build.adoc b/docs/docs-source/docs/modules/get-started/pages/setup-example-project-configure-build.adoc index 365127e90..b36daab09 100644 --- a/docs/docs-source/docs/modules/get-started/pages/setup-example-project-configure-build.adoc +++ b/docs/docs-source/docs/modules/get-started/pages/setup-example-project-configure-build.adoc @@ -41,8 +41,7 @@ As we move through the process, the leaf level directories of the above tree wil == The sbt build script -Cloudflow provides sbt plugins for the Akka, Spark and Flink runtimes. -The plugins speed up development by adding the necessary dependencies and abstracting much of the boilerplate necessary to build a complete application. +The Cloudflow plugins speed up development by adding the necessary dependencies and abstracting much of the boilerplate necessary to build a complete application. You can use multiple runtimes in the same application, provided that each runtime is defined in its own sub-project. // TODO: Add here a link to the multi-project support diff --git a/docs/docs-source/docs/modules/project-info/pages/migration-1_3-2_0.adoc b/docs/docs-source/docs/modules/project-info/pages/migration-1_3-2_0.adoc index eed00b9fb..ab3272eb5 100644 --- a/docs/docs-source/docs/modules/project-info/pages/migration-1_3-2_0.adoc +++ b/docs/docs-source/docs/modules/project-info/pages/migration-1_3-2_0.adoc @@ -295,8 +295,6 @@ In this way, we provide the same classpath isolation guarantee when you run your The new `runLocal` implementation reports the output of streamlets grouped by the sub-projects where they belong. As an additional improvement, the execution output also shows an ASCII-art representation of the application graph that connects streamlets and topics. -See xref:ROOT:app-development-process.adoc#local-testing[Local Testing] to learn more about the _sandbox_ provided by Cloudflow. - == Getting Help The Cloudflow community is there to help you out. diff --git a/docs/docs-source/site.yml b/docs/docs-source/site.yml index 39b8badac..7ed871f50 100644 --- a/docs/docs-source/site.yml +++ b/docs/docs-source/site.yml @@ -8,7 +8,7 @@ content: - docs/docs-source/docs - docs/shared-content-source/docs - examples/snippets - branches: [v1.3.3-docs, v2.0.0-docs, v2.0.25-docs, v2.1.0-docs, v2.1.2-docs, v2.2.0-docs, v2.2.2-docs] # versioned content - add branches here + branches: [v1.3.3-docs, v2.0.0-docs, v2.0.25-docs, v2.1.0-docs, v2.1.2-docs, v2.2.0-docs, v2.2.2-docs, v2.3.0-docs] # versioned content - add branches here - url: git@github.com:lightbend/cloudflow.git start-path: docs/homepage-source/docs branches: [main] # should always remain as main diff --git a/docs/homepage-source/docs/modules/ROOT/pages/index.adoc b/docs/homepage-source/docs/modules/ROOT/pages/index.adoc index 90e859719..228e1a897 100644 --- a/docs/homepage-source/docs/modules/ROOT/pages/index.adoc +++ b/docs/homepage-source/docs/modules/ROOT/pages/index.adoc @@ -1,7 +1,7 @@ = Cloudflow -- streaming data pipelines on Kubernetes :page-layout: home -:description: Quickly develop, orchestrate, and operate distributed streaming data pipelines with Apache Spark, Apache Flink, and Akka Streams on Kubernetes -:keywords: spark, kubernetes, stream, streaming, stream processing, apache spark, apache flink, akka, akka streams, akka-streams, pipelines, streaming pipelines, streaming pipelines on kubernetes, developer, streaming applications +:description: Quickly develop, orchestrate, and operate distributed streaming data pipelines on Kubernetes +:keywords: kubernetes, stream, streaming, stream processing, akka, akka streams, akka-streams, pipelines, streaming pipelines, streaming pipelines on kubernetes, developer, streaming applications Cloudflow enables you to quickly develop, orchestrate, and operate distributed streaming applications on Kubernetes. With Cloudflow, streaming applications are comprised of small composable components wired together with schema-based contracts. @@ -15,12 +15,11 @@ With its powerful abstractions, Cloudflow allows you to easily define the most c * Build: We provide a rich development kit for going from business logic to a deployable application. * Deploy: Cloudflow comes with Kubernetes extensions for the cluster and the client to deploy your distributed system with a single command. -Cloudflow integrates with popular streaming engines like Akka, Spark and Flink. -It also comes with a powerful CLI tool to easily manage, scale and configure streaming applications at runtime. +Cloudflow comes with a powerful CLI tool to easily manage, scale and configure streaming applications at runtime. In a nutshell, Cloudflow is an application development toolkit composed of: * A definition for `Streamlet`, the core component model in Cloudflow. -* An extensible runtime for `Streamlet`(s) and out-of-the-box implementations for popular streaming runtimes, like Spark's Structured Streaming, Flink, and Akka. +* An extensible runtime for `Streamlet`(s). * A `Streamlet` composition model driven by a `blueprint` definition. * A _sandbox_ local execution mode that accelerates the development and testing of your applications without requiring a cluster. * A set of `sbt` plugins that are able to package your application into deployable units. @@ -29,7 +28,6 @@ In a nutshell, Cloudflow is an application development toolkit composed of: == Where To Go Next? * link:./docs/current/index.html[Introducing Cloudflow] -* link:./docs/current/app-development-process.html[The Cloudflow development process] * link:./docs/current/get-started/index.html[Getting started with Cloudflow] * link:./docs/current/develop/cloudflow-streamlets.html[Developing applications with Cloudflow] * https://cloudflow.io/docs/current/administration/index.html[Setup Cloudflow on a Kubernetes Cluster] diff --git a/docs/shared-content-source/docs/modules/ROOT/pages/app-development-process.adoc b/docs/shared-content-source/docs/modules/ROOT/pages/app-development-process.adoc deleted file mode 100644 index da9186b9e..000000000 --- a/docs/shared-content-source/docs/modules/ROOT/pages/app-development-process.adoc +++ /dev/null @@ -1,281 +0,0 @@ -:page-partial: -include::ROOT:partial$include.adoc[] - -:imagesdir: assets/images - -To understand the value that Cloudflow offers, let's break down the typical _design-develop-test-package-deploy_  lifecycle of an application and see how Cloudflow helps at each stage to accelerate the end-to-end process. - -== Designing Streaming Applications - -When creating streaming applications, a large portion of the development effort goes into the _mechanics_ of streaming. -These include configuring and setting up connections to the messaging system, serializing and deserializing messages, setting up fault-recovery storage, and more. -All of these mechanics must be addressed before you can finally add the most important and valuable part--business logic. - -[#biz-logic] -.Business Logic in a Streaming Application -[caption="Fig 2. - "] -image::business-logic.png[] - -Cloudflow introduces a new component model, called _Streamlets_. -The Streamlet model offers an abstraction to: - -- Identify the connections of a component as inlets and outlets, relieving you from setting up connections. -- Attach the schema of the data for each connection, providing type safety at compile time, automatic serialization/deserialization, and avoiding potentially critical issues at deployment. -- Provide an entry point for the business logic of the component, allowing you to focus on coding the most valuable business-specific logic. - -== Develop Focusing on Your Business Logic - -In the current version, Cloudflow includes backend `Streamlet` implementations for Akka, Apache Spark - Structured Streaming, and Apache Flink. -Using these implementations you can write business logic in the native API of the backend. -Along with the currently supported built-in integration of Flink and Spark streamlets via the Cloudflow operator, Cloudflow also supports external integrations for these streaming platforms through additional plugins. Using this new integration, users now have more control on deployment and management of Flink and Spark streamlets while still using the same Cloudflow streamlet API for developing their business logic. However, Akka will continue to be supported natively as in earlier versions. -Integration support for Flink and Spark, thus being externalized, makes Cloudflow easily extensible with new streaming backends. The new externalized integration has been marked _Experimental_ in the current version. -For more details on externalized Flink and Spark integrations, please have a look at https://lightbend.github.io/cloudflow-contrib/[Cloudflow Contrib] documentation. - -The following table shows the current support for Java and Scala implementations for each runtime. - -.Scala|Java Support for Streamlet implementations -|=== -|Streamlet API|Scala|Java -|Akka| X | X -|Flink| X | X -|Spark Structured Streaming| X | -|=== - -=== A Streamlet Example - -Let's take a look at the parts of a Streamlet in the following example: - -.Spark Streamlet Example -[source, scala] ----- -include::{cloudflow-examples-version}@docsnippets:ROOT:example$spark-scala/src/main/scala/cloudflow/sparkdoc/MovingAverageSparklet.scala[tag=spark-streamlet-example] ----- - -<1> `SparkStreamlet` is the base class that defines `Streamlet` for the Apache Spark backend. -<2> The `StreamletShape` defines the inlet(s) and outlet(s) of the `Streamlet`, each inlet/outlet is declared with its corresponding data type. -<3> `buildStreamletQueries` is the entry point for the `Streamlet` logic. -<4> The code provided is written in pure Spark Structured Streaming code, minus the boilerplate to create sessions, connections, checkpoints, etc. - -Akka Streams and Flink-based `Streamlets` follow the same pattern. - -== Compose the Application Flow with a Blueprint - -Once you have developed components as Streamlets, you compose the application's end-to-end flow by creating a _blueprint_. -The blueprint declares the `streamlet` instances that belong to an application and how each streamlet instance connects to the others. - -The following example shows a simple _Blueprint_ definition: - -.`blueprint` example -[source, json] ----- -include::{cloudflow-examples-version}@docsnippets:ROOT:example$spark-scala/src/main/blueprint/blueprint.conf[] ----- -<1> `streamlets` section: declares instances from the streamlets available in the application (or its dependencies). -<2> `connections` section: declares how the inlets/outlets of a streamlet should be connected. - -Note that the declaration of instances in the `streamlets` section supports component reuse by letting you define multiple instances of a streamlet, with potentially different configurations. -With the `blueprint` file in place, you can verify whether all components in the application are properly connected. - -The `verifyBlueprint` function, is part of the `sbt-cloudflow` plugin. -The following shows example output from running `sbt verifyBlueprint`: - -[source, bash] ----- -$ sbt verifyBlueprint -[info] Loading settings for project global-plugins from plugins.sbt ... -[info] Loading global plugins from /home/maasg/.sbt/1.0/plugins -[info] Loading settings for project spark-sensors-build from cloudflow-plugins.sbt,plugins.sbt ... -[info] Loading project definition from cloudflow/examples/spark-sensors/project -[info] Loading settings for project sparkSensors from build.sbt ... -[info] Set current project to spark-sensors (in build file:cloudflow/examples/spark-sensors/) -[info] Streamlet 'sensors.MovingAverageSparklet' found -[info] Streamlet 'sensors.SparkConsoleEgress' found -[info] Streamlet 'sensors.SparkRandomGenDataIngress' found -[success] /cloudflow/examples/spark-sensors/src/main/blueprint/blueprint.conf verified. ----- - -The verification uses the schema information provided by the blueprint to check that all connections between streamlets are compatible. Once the blueprint verification succeeds, you know that the components of your streaming application can talk to each other. After verification, you can run the complete application. - -IMPORTANT: The verification task is usually performed as a pre-requisite of other tasks, like `runLocal` or `buildApp` that we see next. While it's possible to run it independently as shown here, it is most often used indirectly as part of those larger build tasks. - - -[[local-testing]] -== Test Locally Using the Sandbox - -Cloudflow comes with a local execution mode called _sandbox_. The sandbox instantiates all streamlets of an application's blueprint with their connections in a single, local JVM. - -See the sandbox in action in the following screencast. - -.Running a Cloudflow App Locally -[source, scala] ----- -$>sbt -sbt>runLocal -... -[info] Streamlet 'carly.aggregator.CallAggregatorConsoleEgress' found -[info] Streamlet 'carly.aggregator.CallRecordGeneratorIngress' found -[info] Streamlet 'carly.aggregator.CallStatsAggregator' found -[success] /cloudflow/examples/call-record-aggregator/call-record-pipeline/src/main/blueprint/blueprint.conf verified. - ┌──────────────┐ ┌──────────────┐ ┌───────────┐ - │cdr-generator1│ │cdr-generator2│ │cdr-ingress│ - └───────┬──────┘ └──────┬───────┘ └─────┬─────┘ - │ │ │ - └─────────┐ │ ┌─────────┘ - │ │ │ - v v v - ┌────────────────────────┐ - │[generated-call-records]│ - └────────────┬───────────┘ - │ - v - ┌─────┐ - │split│ - └─┬─┬─┘ - │ │ - │ └───────────┐ - │ └┐ - v │ - ┌────────────────────┐ │ - │[valid-call-records]│ │ - └─────────┬──────────┘ │ - │ │ - v │ - ┌──────────────┐ │ - │cdr-aggregator│ │ - └─┬────────────┘ │ - │ │ - v v - ┌───────────────────────┐ ┌──────────────────────┐ - │[aggregated-call-stats]│ │[invalid-call-records]│ - └────────────┬──────────┘ └───────┬──────────────┘ - │ │ - v v - ┌──────────────┐ ┌────────────┐ - │console-egress│ │error-egress│ - └──────────────┘ └────────────┘ ----------------------------- Streamlets per project ---------------------------- -spark-aggregation - output file: file:/tmp/cloudflow-local-run7107015168807762297/spark-aggregation-local.log - - cdr-aggregator [carly.aggregator.CallStatsAggregator] - cdr-generator1 [carly.aggregator.CallRecordGeneratorIngress] - cdr-generator2 [carly.aggregator.CallRecordGeneratorIngress] - -akka-java-aggregation-output - output file: file:/tmp/cloudflow-local-run7107015168807762297/akka-java-aggregation-output-local.log - - console-egress [carly.output.AggregateRecordEgress] - error-egress [carly.output.InvalidRecordEgress] - -akka-cdr-ingestor - output file: file:/tmp/cloudflow-local-run7107015168807762297/akka-cdr-ingestor-local.log - - cdr-ingress [carly.ingestor.CallRecordIngress] - - HTTP port [3000] - split [carly.ingestor.CallRecordSplit] - --------------------------------------------------------------------------------- - ------------------------------------- Topics ------------------------------------ -[aggregated-call-stats] -[generated-call-records] -[invalid-call-records] -[valid-call-records] --------------------------------------------------------------------------------- - ------------------------------ Local Configuration ----------------------------- -No local configuration provided. --------------------------------------------------------------------------------- - ------------------------------------- Output ------------------------------------ -Pipeline log output available in folder: /tmp/cloudflow-local-run7107015168807762297 --------------------------------------------------------------------------------- - -Running call-record-aggregator -To terminate, press [ENTER] ----- - -The sandbox provides you with a minimalistic operational version of the complete application. -Use it to exercise the functionality of the application end-to-end and verify that it behaves as expected. -You get a blazing fast feedback loop for the functionality you are developing--removing the need to go through the full package, upload, deploy, and launch process on a remote cluster. - -== Publishing the Application Artifacts - -Once you are confident that the application functions as expected, you can build and publish its artifacts. -Cloudflow applications are packaged as Docker images that contain the necessary dependencies to run the different streamlets on their respective backends. -Depending on the project structure, this process will generate one or more Docker images, at least one for each runtime used in the application. - -Those images get published to the Docker repository of your choice, which must be configured in the project. - -The process of building the application creates and publishes those Docker images. -The final result is an application descriptor encoded as a JSON file that can be used with the `kubectl` Cloudflow plugin to deploy the application to a cluster. - -The `buildApp` task in the `sbt-cloudflow` plugin loads, verifies, creates and publishes the Docker images, and finally generates the JSON descriptor that we can use to deploy the application. -The following abridged example shows the different stages that you can observe in the output of this task. - -.Stages of Building a Cloudflow Application -[source, bash] ----- -$ sbt buildApp -... #<1> -[info] Set current project to root (in build file:/cloudflow/examples/call-record-aggregator/) -[info] Streamlet 'carly.output.AggregateRecordEgress' found -[info] Streamlet 'carly.output.InvalidRecordEgress' found -[info] Streamlet 'carly.ingestor.CallRecordIngress' found -[info] Streamlet 'carly.ingestor.CallRecordSplit' found -[info] Streamlet 'carly.ingestor.CallRecordStreamingIngress' found -... #<2> -[success] /cloudflow/examples/call-record-aggregator/call-record-pipeline/src/main/blueprint/blueprint.conf verified. -... #<3> -[info] Sending build context to Docker daemon 51.97MB -[info] Step 1/4 : FROM lightbend/akka-base:2.0.10-cloudflow-akka-2.6.6-scala-2.12 -[info] ---> bbcdde34a60a -[info] Step 2/4 : USER 185 -[info] Successfully built dc0b6c01246e -[info] Tagging image dc0b6c01246e with name: eu.gcr.io/my-gcp-project/akka-cdr-ingestor:475-2cd83d8 -... -[info] Successfully built and published the following image: -[info] eu.gcr.io/my-gcp-project/akka-cdr-ingestor:475-2cd83d8 -[info] 57ceaacf8169: Pushed -... #<4> -[success] Cloudflow application CR generated in /cloudflow/examples/call-record-aggregator/target/call-record-aggregator.json -[success] Use the following command to deploy the Cloudflow application: -[success] kubectl cloudflow deploy /cloudflow/examples/call-record-aggregator/target/call-record-aggregator.json -[success] Total time: 129 s (02:09), completed Jun 24, 2020 12:58:46 PM ----- -<1> Project gets loaded -<2> The application is validated by combining the info in the blueprint, streamlets, and schema information. -<3> The Docker images are built and published -<4> The application descriptor is generated in the /target folder of the project and we get a hint of the command to use for initialing the deployment - -Next, we see how to use the application descriptor to deploy the application to a cluster. - - -== Deploy with `kubectl` Extensions for a YAML-less Experience - -After testing and packaging, you are ready to deploy the application to a Cloudflow-enabled Kubernetes cluster. A Cloudflow-enabled Kubernetes cluster has Cloudflow installed as well as Spark and/or Flink, if your streamlets use them. In contrast with the usual YAML-full experience that typical K8s deployments require, Cloudflow uses the blueprint information and the streamlet definitions to auto-generate an application deployment. - -Kafka is used by Cloudflow to connect streamlets together with the topology defined in a blueprint. If you intend to connect streamlets in this way, at least one Kafka cluster should be available before you install Cloudflow. Cloudflow may be used without Kafka (for example, when your application contains a single streamlet, or an xref:develop:clustering-akka-streamlet.adoc[Akka cluster]), but if your team intends to connect streamlets together and not include Kafka connection information in each topic they define then it's recommended to define a `default` Kafka cluster at install time. - -Cloudflow also comes with a `kubectl` plugin that augments the capabilities of your local `kubectl` installation to work with Cloudflow applications. -You use your usual `kubectl` commands to `auth` against your target cluster. -Then, with the `kubectl cloudflow` plugin you can deploy and manage a Cloudflow application as a single logical unit. - -[source, bash] ----- -$ kubectl cloudflow deploy /cloudflow/examples/call-record-aggregator/target/call-record-aggregator.json ----- - -This method is not only dev-friendly, but is also compatible with the typical CI/CD deployments. -This allows you to take the application from dev to production in a controlled way. -The deployment procedure will be different with the _cloudflow contrib_ approach where Flink and Spark applications are supported through external plugins. Akka applications will be fully deployed using `kubectl cloudflow` as above. However, for Spark and Flink applications, you need to use an extra plugin and carry out a few extra steps to make them known to the Cloudflow engine. For details please have a look at https://lightbend.github.io/cloudflow-contrib/[Cloudflow Contrib] documentation. - -== Conclusion - -As a developer, Cloudflow gives you a set of powerful tools to accelerate the application development process: - -* The Streamlet API let's you focus on business value and use your knowledge of widely popular streaming runtimes, like Akka Streams, Apache Spark Structured Streaming, and Apache Flink to create full-fledged streaming applications. -* The blueprint lets you easily compose your application with the peace of mind that a verification phase, informed by schema definitions, provides. -* The _sandbox_ lets you exercise the complete application in seconds, giving you a real-time feedback loop to speed up the debugging and validation phases. - -And with a fully developed application, the `kubectl cloudflow` plugin gives you the ability to deploy and control the lifecycle of your application on an enabled K8s cluster. - -Cloudflow takes away the pain of creating and deploying distributed applications on Kubernetes, speeds up your development process, and gives you full control over the operational deployment. -In a nutshell, it gives you distributed application development super-powers on Kubernetes. diff --git a/docs/shared-content-source/docs/modules/administration/pages/installation-prerequisites.adoc b/docs/shared-content-source/docs/modules/administration/pages/installation-prerequisites.adoc index 3ceb7a084..f8690447f 100644 --- a/docs/shared-content-source/docs/modules/administration/pages/installation-prerequisites.adoc +++ b/docs/shared-content-source/docs/modules/administration/pages/installation-prerequisites.adoc @@ -7,7 +7,6 @@ To prepare your environment to install Cloudflow, you will need: * Helm {supported-helm-v} (check with `helm version`) * Kubectl * Kafka -* If using Spark or Flink, storage configuration == Helm @@ -36,45 +35,6 @@ See xref:how-to-install-and-use-strimzi.adoc[Installing Kafka with Strimzi] as a Please make sure to choose a suitable Kafka service that matches your experience with running Kafka in production. -== Storage requirements (for use with Spark or Flink) - -**If you plan to write Cloudflow applications using Spark or Flink**, the Kubernetes cluster will need to have a storage class of the `ReadWriteMany` type installed. - -For testing purposes, we suggest using the NFS Server Provisioner, which can be found here: https://github.com/helm/charts/tree/master/stable/nfs-server-provisioner[NFS Server Provisioner Helm chart] - -We'll install the nfs chart in the `cloudflow` namespace, if it does not exist yet, create the `cloudflow` namespace: - - kubectl create ns cloudflow - - -Add the `Stable` Helm repository and update the local index: - - helm repo add stable https://charts.helm.sh/stable - helm repo update - -Install the NFS Server Provisioner using the following command: - -IMPORTANT: Depending on your Kubernetes configuration, you may want to adjust the values used during the install. -Please see https://github.com/helm/charts/tree/master/stable/nfs-server-provisioner#configuration[NFS Server Provisioner configuration options]. - - helm install nfs-server-provisioner stable/nfs-server-provisioner \ - --namespace cloudflow - -The result of the installation is shown below, the NFS Server provisioner pod is running and the new storage class exists. - ----- -$ kubectl get pods -n cloudflow -NAME READY STATUS RESTARTS AGE -nfs-server-provisioner-0 1/1 Running 0 25s - -$ kubectl get sc -NAME PROVISIONER AGE -nfs cloudflow-nfs 29s -standard (default) kubernetes.io/gce-pd 2m57s ----- - -NOTE: The documented NFS storage class is very portable and has been verified to work on GKE, EKS, AKS and Openshift. - NOTE: The default build tool for Cloudflow applications is `sbt` but there is support for using xref:docs:develop:maven-support.adoc[Maven] to build as well. If you are going to use Maven, make sure to have it https://maven.apache.org/[installed] as well. diff --git a/docs/shared-content-source/docs/modules/administration/pages/installing-cloudflow.adoc b/docs/shared-content-source/docs/modules/administration/pages/installing-cloudflow.adoc index bf5194d24..47e3c7fb9 100644 --- a/docs/shared-content-source/docs/modules/administration/pages/installing-cloudflow.adoc +++ b/docs/shared-content-source/docs/modules/administration/pages/installing-cloudflow.adoc @@ -2,7 +2,6 @@ include::ROOT:partial$include.adoc[] This guide shows how to install Cloudflow, using the `Helm` and `kubectl` command-line tools. -It also shows how to install Kafka, Spark, and Flink operators that integrate with Cloudflow. == Installing Cloudflow @@ -125,17 +124,3 @@ cloudflow-operator-6b7d7cbdfc-xb6w5 1/1 Running 0 ---- You can now deploy an Akka-based Cloudflow application into the cluster as it only requires Kafka to be set up. More on this in the development section of the documentation. - -== Adding Spark support - -If you plan to write applications that utilize Spark, you will need to install the Spark operator before deploying your Cloudflow application. - -Continue with xref:installing-spark-operator.adoc[Adding Spark Support]. - -== Adding Flink support - -If you plan to write applications that utilize Flink you will need to install the Flink operator before deploying your Cloudflow application. - -Continue with xref:installing-flink-operator.adoc[Adding Flink Support]. - - diff --git a/docs/shared-content-source/docs/modules/administration/pages/installing-flink-operator.adoc b/docs/shared-content-source/docs/modules/administration/pages/installing-flink-operator.adoc deleted file mode 100644 index 9fe67d87c..000000000 --- a/docs/shared-content-source/docs/modules/administration/pages/installing-flink-operator.adoc +++ /dev/null @@ -1,31 +0,0 @@ - - -include::ROOT:partial$include.adoc[] - -NOTE: Integration with the Lyft Flink Operator has been deprecated since 2.2.0, and will be removed in version 3.x. Flink integration has moved to the https://github.com/lightbend/cloudflow-contrib[Cloudflow-contrib project]. Please see https://lightbend.github.io/cloudflow-contrib/docs/0.1.1/get-started/index.html[the Cloudflow-contrib getting started guide] for instructions on how to use Flink Native Kubernetes integration. The documentation that follows describes the deprecated feature. - -NOTE: See xref:installation-prerequisites.adoc#_storage_requirements_for_use_with_spark_or_flink[Storage requirements (for use with Spark or Flink)] to setup Storage Requirements for Flink applications. - -If you plan to write applications that utilize Flink you will need to install the Lyft Flink operator before deploying your Cloudflow application. - -Install the Lyft Flink operator using the following Helm command: - ----- -helm install flink-operator \ - https://github.com/lightbend/flink-operator/releases/download/v0.8.2/flink-operator-0.8.2.tgz \ - --namespace cloudflow \ - --set operatorVersion="v0.5.0" ----- - -NOTE: The above Helm command uses a Helm chart for the Lyft Flink operator that is hosted https://github.com/lightbend/flink-operator[here]. - -This completes the installation of the Flink operator. - ----- -$ kubectl get pods -n cloudflow -NAME READY STATUS RESTARTS AGE -cloudflow-operator-6b7d7cbdfc-xb6w5 1/1 Running 0 3m29s -cloudflow-patch-spark-mutatingwebhookconfig-66xxb 0/1 Completed 0 21s -spark-operator-sparkoperator-56d6ffc8cd-cln67 1/1 Running 0 2m29s -flink-operator-7999fdd879-phqlg 1/1 Running 0 1m29s ----- diff --git a/docs/shared-content-source/docs/modules/administration/pages/installing-spark-operator.adoc b/docs/shared-content-source/docs/modules/administration/pages/installing-spark-operator.adoc deleted file mode 100644 index b6e62b0fb..000000000 --- a/docs/shared-content-source/docs/modules/administration/pages/installing-spark-operator.adoc +++ /dev/null @@ -1,171 +0,0 @@ - - -include::ROOT:partial$include.adoc[] - -NOTE: Integration with the Spark Operator has been deprecated since 2.2.0, and will be removed in version 3.x. Spark integration has moved to the https://github.com/lightbend/cloudflow-contrib[Cloudflow-contrib project]. Please see https://lightbend.github.io/cloudflow-contrib/docs/0.1.1/get-started/index.html[the Cloudflow-contrib getting started guide] for instructions on how to use Spark Native Kubernetes integration. The documentation that follows describes the deprecated feature. - -NOTE: See xref:installation-prerequisites.adoc#_storage_requirements_for_use_with_spark_or_flink[Storage requirements (for use with Spark or Flink)] to setup Storage Requirements for Spark applications. - -If you plan to write applications that utilize Spark, you will need to install the Spark operator before deploying your Cloudflow application. - -The only Cloudflow compatible Spark operator is a fork maintained by Lightbend. -See https://github.com/apache/spark/pull/24748[Backported Fix for Spark 2.4.5] for more details. - -Add the Spark Helm chart repository and update the local index. - - helm repo add incubator https://charts.helm.sh/incubator - helm repo update - -Before installing the Helm chart, we will need to prepare a `value.yaml` file, it overrides the default values in the Spark Helm chart. - -[subs="attributes"] ----- -cat > spark-values.yaml << EOF -rbac: - create: true -serviceAccounts: - sparkoperator: - create: true - name: cloudflow-spark-operator - spark: - create: true - name: cloudflow-spark - -enableWebhook: true -enableMetrics: true - -controllerThreads: 10 -installCrds: true -metricsPort: 10254 -metricsEndpoint: "/metrics" -metricsPrefix: "" -resyncInterval: 30 -webhookPort: 8080 -sparkJobNamespace: "" -operatorVersion: latest ---- -kind: Service -apiVersion: v1 -metadata: - name: cloudflow-webhook - labels: - app.kubernetes.io/name: sparkoperator -spec: - selector: - app.kubernetes.io/name: sparkoperator -EOF ----- - -Now we can install the Spark operator using Helm. Note that we use a specific version of the Spark operator Helm chart. - -[source,shell script,subs="attributes"] ----- - helm upgrade -i spark-operator incubator/sparkoperator \ - --values="spark-values.yaml" \ - --namespace cloudflow ----- - -[IMPORTANT] -.*OpenShift* requires a role-binding -==== -If you are installing the Spark operator on *OpenShift*, you need to apply the following `rolebinding` to the cluster: - ----- -cat > spark-rolebinding.yaml << EOF -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: system:openshift:scc:anyuid - namespace: cloudflow -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: system:openshift:scc:anyuid -subjects: -- kind: ServiceAccount - name: cloudflow-spark - namespace: cloudflow -EOF ----- - -To apply the `spark-rolebinding.yaml` file execute the following command. Note that this has *only* to be performed if you are installing Cloudflow on Openshift: - - oc apply -f spark-rolebinding.yaml -==== - -After the Spark operator installation completes, we need to adjust its webhook configuration: - ----- -$ kubectl get pods -n cloudflow -NAME READY STATUS RESTARTS AGE -cloudflow-operator-6b7d7cbdfc-xb6w5 1/1 Running 0 109s -spark-operator-sparkoperator-56d6ffc8cd-cln67 1/1 Running 0 109s ----- - -For this, we will create yet another YAML file that we will apply to the cluster with `kubectl`: ----- -cat > spark-mutating-webhook.yaml << 'EOF2' -apiVersion: batch/v1 -kind: Job -metadata: - name: cloudflow-patch-spark-mutatingwebhookconfig -spec: - template: - spec: - serviceAccountName: cloudflow-operator - restartPolicy: OnFailure - containers: - - name: main - image: alpine:3.12 - command: - - /bin/ash - - "-c" - - | - apk update && apk add wget - wget -q -O /bin/kubectl https://storage.googleapis.com/kubernetes-release/release/v1.19.6/bin/linux/amd64/kubectl && chmod 755 /bin/kubectl - NAME="spark-operator-sparkoperator" - API_VERSION=$(kubectl get deployment -n cloudflow $NAME -o jsonpath='{.apiVersion}') - UUID=$(kubectl get deployment -n cloudflow $NAME -o jsonpath='{.metadata.uid}') - KIND=$(kubectl get deployment -n cloudflow $NAME -o jsonpath='{.kind}') - HOOK_NAME="spark-operator-sparkoperator-webhook-config" - JSON=$(cat <.json == Archetype -You can easily generate a full project with akka, spark and flink streamlets using the Cloudflow Maven Archetype: +You can easily generate a full project with akka streamlets using the Cloudflow Maven Archetype: [source,bash,subs=attributes+] ---- diff --git a/docs/shared-content-source/docs/modules/develop/pages/project-structure.adoc b/docs/shared-content-source/docs/modules/develop/pages/project-structure.adoc index e2fcce6ea..7dad56cf2 100644 --- a/docs/shared-content-source/docs/modules/develop/pages/project-structure.adoc +++ b/docs/shared-content-source/docs/modules/develop/pages/project-structure.adoc @@ -76,22 +76,21 @@ It is an _autoplugin_ that becomes enabled when the `CloudflowApplicationPlugin` === Runtime-specific Plugins -Cloudflow provides out-of-the-box support for Akka, Apache Spark Structured Streaming, and Apache Flink in the form of specific implementations of the Streamlet API and supporting elements, like Docker base images and _runners_ that are able to launch each specific technology stack on Kubernetes. +Cloudflow provides out-of-the-box support for Akka with Docker base images and _runners_ that are able to launch each specific technology stack on Kubernetes. + To develop `Streamlet`{empty}s for a specific runtime, we add the corresponding plugin to the project. To ensure proper isolation of the different runtimes, we require a different structural approach when using only one runtime or when mixing `Streamlet`{empty}s implementations that use two or more runtimes. We explore these differences in the next section. -Before delving into that, here we list the runtime plugins: +The Akka plugins is: -`CloudflowAkkaPlugin`:: SBT plugin for creating Akka-based streamlets. -`CloudflowSparkPlugin`:: SBT plugin for creating Spark Structured Streaming based streamlets. -`CloudflowFlinkPlugin`:: SBT plugin for creating Flink-based streamlets. +`CloudflowAkkaPlugin`:: SBT plugin for creating Akka-based streamlets. [NOTE] ==== -Each runtime plugin uses its own default base image. It's `lightbend/akka-base` for Akka, `lightbend/spark` for Spark and `lightbend/flink` for Flink. It could be the case that you're working against your own container registry and therefore prefer to point to your copy or customized version of our base images. To use your base images, set `cloudflowAkkaBaseImage`, `cloudflowSparkBaseImage`, and `cloudflowFlinkBaseImage` in `settings` of the respective runtime plugins in your application `build.sbt`. +Each runtime plugin uses its own default base image. It's `lightbend/akka-base` for Akka. It could be the case that you're working against your own container registry and therefore prefer to point to your copy or customized version of our base images. To use your base images, set `cloudflowAkkaBaseImage` in `settings` of the `CloudflowAkkaPlugin` in your application `build.sbt`. -In an Akka-based streamlet the default base image value is `lightbend/akka-base:${CloudflowBasePlugin.CloudflowVersion}-cloudflow-akka-$AkkaVersion-scala-${CloudflowBasePlugin.ScalaVersion}`. This may be set to `lightbend/akka-base:2.0.10-cloudflow-akka-2.6.6-scala-2.12` depending on your Cloudflow, Akka and Scala versions. To override this base image in your cloudflow application go to its build.sbt and set up a value in settings such as: +In an any streamlet the default base image value is `adoptopenjdk/openjdk8:alpine`. To override this base image in your cloudflow application go to its build.sbt and set up a value in settings such as: ==== [source, scala] @@ -123,54 +122,7 @@ https://github.com/lightbend/cloudflow/blob/{cloudflow-branch-version}/examples/ Applications that combine multiple streamlet runtimes *must be* organized using `sbt` https://www.scala-sbt.org/1.x/docs/Multi-Project.html[multi-projects builds]. Streamlets for each runtime must added be into their own separate sub-project. -IMPORTANT: The multi-project build organization is necessary to ensure the correct isolation of the dependencies required by each runtime in their own classpath. -Frameworks like Spark and Flink pack a long list of library dependencies and some of them conflict with each other when placed in the same classloader. -Using multi-project builds ensures the correct functioning of the local _sandbox_ and informs the generation of a Docker image per sub-project with the correct dependencies. - -Next to that, we recommend separating the data-oriented schemas into their own sub-project so that streamlet sub-projects can easily import these common data definitions. - -For example, if we have an application that uses Akka and Structured Streaming, we could have the following structure: - -.Project Structure using Multiple Runtimes -[source, text] ----- -. -├── my-application #<1> -│   └── src -│   └── main -│   └── blueprint #<2> -├── datamodel #<3> -│   └── src -│   └── main -│   └── avro #<4> -├── akka-ingestor #<5> -│   └── src -│   ├── main -│   │   └── scala -│   └── test -│   └── scala -├── spark-aggregation #<6> -│   └── src -│   ├── main -│   │   └── scala -│   └── test -│   └── scala -├── project #<7> -│   ├── build.properties -│   └── cloudflow-plugins.sbt -├── build.sbt #<8> -└── target-env.sbt #<9> ----- -<1> The application 'root' project that contains the `blueprint` definition. -<2> The `blueprint.conf` is expected in a `src/main/blueprint` directory by default. -<3> The `datamodel/` folder contains the shared schema definitions. -<4> The `avro/` folder contains the schema definitions in AVRO format. -<5> `akka-ingestor` is a sub-project that contains Akka-based streamlets -<6> `spark-aggregation` is a sub-project that contains Spark-based streamlets -<7> The `project/` folder contains the `cloudflow-plugins.sbt` and other necessary files as we saw previously. -<8> The `target-env.sbt` contains the Docker repository configuration, as we saw at the start of the chapter. - -We can find a `build.sbt` definition that exemplifies the use of a multi-project setup in this https://github.com/lightbend/cloudflow/blob/{cloudflow-branch-version}/examples/call-record-aggregator[Cloudflow example] +You can find how-to on https://lightbend.github.io/cloudflow-contrib/docs/0.2.0/get-started/index.html[Cloudflow contrib project]. == Where to go next? diff --git a/docs/shared-content-source/docs/modules/develop/pages/schema-first-approach.adoc b/docs/shared-content-source/docs/modules/develop/pages/schema-first-approach.adoc index 326e08945..abd5a92c7 100644 --- a/docs/shared-content-source/docs/modules/develop/pages/schema-first-approach.adoc +++ b/docs/shared-content-source/docs/modules/develop/pages/schema-first-approach.adoc @@ -137,12 +137,12 @@ All logic regarding data safety that we discussed for inlets in the last section [[schema-aware]] == Schema-aware `StreamletLogic` -When we implement `StreamletLogic` for a streamlet, we use the inlets and outlets which, as we discussed above, are schema aware. Note that in the following code fragment, all inlet and outlet types are parameterized with domain classes `CallRecord` and `AggregatedCallStats` that have been generated using the schema. +When we implement `StreamletLogic` for a streamlet, we use the inlets and outlets which, as we discussed above, are schema aware. Note that in the following code fragment, the outlet type is parameterized with domain classes `CallRecord` that have been generated using the schema. Here's an example: [source,scala] ---- -include::{cloudflow-examples-version}@docsnippets:ROOT:example$spark-scala/src/main/scala/cloudflow/callrecordaggregator/CallStatsAggregator.scala[tags=docs-schemaAware-example] +include::{cloudflow-examples-version}@docsnippets:ROOT:example$akkastreams-scala/src/main/scala/cloudflow/callrecordaggregator/CallRecordIngress.scala[tags=docs-schemaAware-example] ---- In the above example, we have one inlet that allows data of type `CallRecord` and one outlet that allows data of type `AggregatedCallStats`. Here the user had supplied the schema for both of the above types from which Scala classes have been generated by Cloudflow. And the entire `StreamletLogic` code is based on these two classes - we read `CallRecord` from the inlet, do processing and generate `AggregatedCallStats` to be sent to the outlet. diff --git a/docs/shared-content-source/docs/modules/develop/pages/streamlet-configuration.adoc b/docs/shared-content-source/docs/modules/develop/pages/streamlet-configuration.adoc index 312a4ae4d..d53dd70ac 100644 --- a/docs/shared-content-source/docs/modules/develop/pages/streamlet-configuration.adoc +++ b/docs/shared-content-source/docs/modules/develop/pages/streamlet-configuration.adoc @@ -77,13 +77,6 @@ If we want to write a test for the example streamlet `RecordSumFlow`, we could a include::{cloudflow-examples-version}@docsnippets:ROOT:example$akkastreams-scala/src/test/scala/com/example/SampleSpec.scala[tag=config-value] -- -The Spark testkit has a similar function for adding values to configuration parameters when testing a streamlet. - -[source,scala] --- -include::{cloudflow-examples-version}@docsnippets:ROOT:example$spark-scala/src/test/scala/com/example/SampleSpec.scala[tag=config-value] --- - The Java API is slightly different as you can see in the example below: [source,java] diff --git a/docs/shared-content-source/docs/modules/develop/pages/test-flink-streamlet.adoc b/docs/shared-content-source/docs/modules/develop/pages/test-flink-streamlet.adoc deleted file mode 100644 index 3122e5c39..000000000 --- a/docs/shared-content-source/docs/modules/develop/pages/test-flink-streamlet.adoc +++ /dev/null @@ -1,99 +0,0 @@ -:page-partial: - -include::ROOT:partial$include.adoc[] - -NOTE: Integration with the Lyft Flink Operator has been deprecated since 2.2.0, and will be removed in version 3.x. Flink integration has moved to the https://github.com/lightbend/cloudflow-contrib[Cloudflow-contrib project]. Please see https://lightbend.github.io/cloudflow-contrib/docs/0.1.1/get-started/index.html[the Cloudflow-contrib getting started guide] for instructions on how to use Flink Native Kubernetes integration. The documentation that follows describes the deprecated feature. The FlinkStreamlet API has not changed in cloudflow-contrib, though you do need to use a different dependency and add the `CloudflowNativeFlinkPlugin`, which is described in the https://lightbend.github.io/cloudflow-contrib/docs/0.1.1/get-started/flink-native.html#_building_flink_native_streamlets[Cloudflow contrib documentation for building Flink native streamlets]. - -A `testkit` is provided to make it easier to write unit tests for Flink streamlets. The unit tests are meant to facilitate local testing of streamlets. `FlinkTestkit` offers APIs to write unit tests for Flink streamlets in both Scala and Java. - -== Basic flow of `testkit` APIs in Scala - -Here's the basic flow that you need to follow when writing tests using the `testkit`: - -. Extend the test class with the `FlinkTestkit` abstract class. This abstract class provides the basic initialization and cleanups and the core APIs of the `testkit` -. Create a Flink streamlet -. Setup inlet taps that tap the inlet ports of the streamlet -. Setup outlet taps for outlet ports -. Push data into inlet ports -. Run the streamlet using the `run` method that the testkit offers -. Write assertions to ensure that the expected results match the actual ones - -== Details of the workflow in Scala - -Let's consider an example where we would like to write unit tests for testing a `FlinkStreamlet` that reads data from an inlet, does some processing and writes processed data to an outlet. We will follow the steps that we outlined in the last section. We will use ScalaTest as the testing library. - -=== Setting up a sample FlinkStreamlet - -`FlinkStreamlet` is an abstract class. Let's set up a concrete instance that we would like to test. For more details on how to implement a Flink streamlet, please refer to <<_building_a_flink_streamlet>>. We will now write unit tests for the following `FlinkStreamlet` class. - -[source,scala] ----- -include::{cloudflow-examples-version}@docsnippets:ROOT:example$build-flink-streamlets-scala/step3/src/main/scala/com/example/FlinkProcessor.scala[tag=processor] ----- - -=== The unit test - -Here is a list of imports needed for writing the test suite. - -[source,scala] ----- -include::{cloudflow-examples-version}@docsnippets:ROOT:example$build-flink-streamlets-scala/step3/src/test/scala/com/example/FlinkProcessorSpec.scala[tag=imports] ----- - - -Here's how we would write a unit test using ScalaTest. The various logical steps of the test are annotated with inline comments explaining the rationale behind the step. - -[source,scala] ----- -include::{cloudflow-examples-version}@docsnippets:ROOT:example$build-flink-streamlets-scala/step3/src/test/scala/com/example/FlinkProcessorSpec.scala[tag=test] ----- - -== Basic flow of `testkit` APIs in Java - -Here's the basic flow that you need to follow in Java when writing tests using the `testkit`: - -. Extend the test class with the `JUnitSuite` trait from ScalaTest. -. Instantiate the testkit class -. Create the Flink streamlet that needs to be tested -. Prepare data to be pushed into inlet ports -. Setup inlet taps that tap the inlet ports of the streamlet -. Setup outlet taps for outlet ports -. Run the streamlet using the `run` method that the testkit offers -. Write assertions to ensure that the expected results match the actual ones - -== Details of the workflow in Java - -Let's consider a `FlinkStreamlet` class that reads data from an inlet, process that data, and writes it to an outlet. To write unit tests for this class, we will follow the steps that we outlined in the last section, using ScalaTest as the testing library. - -We will discuss the steps for implementation in both Scala and Java. - -=== Setting up a sample FlinkStreamlet - -`FlinkStreamlet` is an abstract class. Let's set up a concrete instance that we would like to test. For more details on how to implement a Flink streamlet, please refer to <<_building_a_flink_streamlet>>. Here's a sample `FlinkStreamlet` that we would like to write unit tests for. - -[source,java] ----- -include::{cloudflow-examples-version}@docsnippets:ROOT:example$build-flink-streamlets-java/step3/src/main/java/com/example/FlinkProcessor.java[tag=processor] ----- - -=== The unit test - -Here is a list of imports needed for writing the test suite. - -[source,scala] ----- -include::{cloudflow-examples-version}@docsnippets:ROOT:example$build-flink-streamlets-java/step3/src/test/java/com/example/FlinkProcessorTest.java[tag=imports] ----- - - -Here's how we would write a unit test using ScalaTest. The logical steps of the test are annotated with inline comments explaining their rationale. - -[source,scala] ----- -include::{cloudflow-examples-version}@docsnippets:ROOT:example$build-flink-streamlets-java/step3/src/test/java/com/example/FlinkProcessorTest.java[tag=test] ----- - -=== The FlinkTestkit class - -. Provides core APIs like `inletAsTap`, `outletAsTap`, `getInletAsTap` (Java API), `getOutletAsTap` (Java API) and `run` (both Java and Scala APIs). -. Supports adding values for configuration parameters. diff --git a/docs/shared-content-source/docs/modules/develop/pages/test-spark-streamlet.adoc b/docs/shared-content-source/docs/modules/develop/pages/test-spark-streamlet.adoc deleted file mode 100644 index 5b6ee94ef..000000000 --- a/docs/shared-content-source/docs/modules/develop/pages/test-spark-streamlet.adoc +++ /dev/null @@ -1,61 +0,0 @@ -:page-partial: - -include::ROOT:partial$include.adoc[] - -NOTE: Integration with the Spark Operator has been deprecated since 2.2.0, and will be removed in version 3.x. Spark integration has moved to the https://github.com/lightbend/cloudflow-contrib[Cloudflow-contrib project]. Please see https://lightbend.github.io/cloudflow-contrib/docs/0.1.1/get-started/index.html[the Cloudflow-contrib getting started guide] for instructions on how to use Spark Native Kubernetes integration. The documentation that follows describes the deprecated feature. The SparkStreamlet API has not changed in cloudflow-contrib, though you do need to use a different dependency and add the `CloudflowNativeSparkPlugin`, which is described in the https://lightbend.github.io/cloudflow-contrib/docs/0.1.1/get-started/spark-native.html[Cloudflow contrib documentation for building Spark native streamlets]. - -A `testkit` is provided to make it easier to write unit tests for Spark streamlets. The unit tests are meant to facilitate local testing of streamlets. - -== Basic flow of `testkit` APIs - -Here's the basic flow that you need to follow when writing tests using the `testkit`: - -. Extend the test class with the `SparkScalaTestSupport` trait. This trait provides the basic functionalities of managing the `SparkSession`, basic initialization and cleanups and the core APIs of the `testkit`. -. Create a Spark streamlet `testkit` instance. -. Create the Spark streamlet that needs to be tested. -. Setup inlet taps that tap the inlet ports of the streamlet. -. Setup outlet taps for outlet ports. -. Push data into inlet ports. -. Run the streamlet using the `testkit` and the setup inlet taps and outlet taps. -. Write assertions to ensure that the expected results match the actual ones. - -== Details of the workflow - -Let's consider an example where we would like to write unit tests for testing a `SparkStreamlet` that reads data from an inlet, does some processing and writes processed data to an outlet. We will follow the steps that we outlined in the last section. We will use ScalaTest as the testing library. - -=== Setting up a sample SparkStreamlet - -Here is a list of imports needed for writing the test suite. - -[source,scala] ----- -include::{cloudflow-examples-version}@docsnippets:ROOT:example$build-spark-streamlets-scala/step3/src/test/scala/com/example/SparkProcessorSpec.scala[tag=imports] ----- - -`SparkStreamlet` is an abstract class. Let's set up a concrete instance that we would like to test. For more details on how to implement a Spark streamlet, please refer to xref:build-spark-streamlets.adoc[Building a Spark streamlet]. - -[source,scala] ----- -include::{cloudflow-examples-version}@docsnippets:ROOT:example$build-spark-streamlets-scala/step3/src/main/scala/com/example/SparkProcessor.scala[tag=processor] ----- - -=== The unit test - -Here's how we would write a unit test using ScalaTest. The various logical steps of the test are annotated with inline comments explaining the rationale behind the step. - -[source,scala] ----- -include::{cloudflow-examples-version}@docsnippets:ROOT:example$build-spark-streamlets-scala/step3/src/test/scala/com/example/SparkProcessorSpec.scala[tag=test] ----- - -=== The SparkScalaTestSupport trait - -This provides session management and needs to be mixed in with the main test class. This trait provides the following functionalities: - -. Manage a `SparkSession` for all tests, initialized when the test class initialize. -. Cleanup the session using `afterAll`. If you want custom logic for cleanups, override the `afterAll` method and call `super.afterAll()` before adding your custom logic. - -=== The SparkStreamletTestkit class - -. Provide core APIs like `inletAsTap`, `outletAsTap`, `run`. -. Support for adding values for configuration parameters. diff --git a/docs/shared-content-source/docs/modules/develop/pages/use-flink-streamlets.adoc b/docs/shared-content-source/docs/modules/develop/pages/use-flink-streamlets.adoc deleted file mode 100644 index 0c934ebb9..000000000 --- a/docs/shared-content-source/docs/modules/develop/pages/use-flink-streamlets.adoc +++ /dev/null @@ -1,235 +0,0 @@ -:page-partial: - -include::ROOT:partial$include.adoc[] - -NOTE: Integration with the Lyft Flink Operator has been deprecated since 2.2.0, and will be removed in version 3.x. Flink integration has moved to the https://github.com/lightbend/cloudflow-contrib[Cloudflow-contrib project]. Please see https://lightbend.github.io/cloudflow-contrib/docs/0.1.1/get-started/index.html[the Cloudflow-contrib getting started guide] for instructions on how to use Flink Native Kubernetes integration. The documentation that follows describes the deprecated feature. The FlinkStreamlet API has not changed in cloudflow-contrib, though you do need to use a different dependency and add the `CloudflowNativeFlinkPlugin`, which is described in the https://lightbend.github.io/cloudflow-contrib/docs/0.1.1/get-started/flink-native.html#_building_flink_native_streamlets[Cloudflow contrib documentation for building Flink native streamlets]. - - -A Flink streamlet has the following responsibilities: - -* It needs to capture your stream processing logic. -* It needs to publish metadata used by the `sbt-cloudflow` plugin to verify that a blueprint is correct. - This metadata consists of the _shape_ of the streamlet (`StreamletShape`) defined by the inlets and outlets of the streamlet. - Connecting streamlets need to match on inlets and outlets to make a valid cloudflow topology, as mentioned previously in xref:blueprints.adoc[Composing applications using blueprints]. - Cloudflow automatically extracts the shapes from the streamlets to verify this match. -* For the Cloudflow runtime, it needs to provide metadata so that it can be configured, scaled, and run as part of an application. -* The inlets and outlets of a `Streamlet` have two functions: -** To specify to Cloudflow that the streamlet needs certain data streams to exist at runtime, which it will read from and write to, and -** To provide handles inside the stream processing logic to connect to the data streams that Cloudflow provides. The `StreamletLogic` provides methods that take an inlet or outlet argument to read from or write to. These will be the specific data streams that Cloudflow has set up for you. - -The next sections will go into the details of defining a Flink streamlet: - -* defining inlets and outlets -* creating a streamlet shape from the inlets and outlets -* creating a streamlet logic that uses the inlets and outlets to read and write data - -== Inlets and outlets - -A streamlet can have one or more inlets and outlets. -Cloudflow offers classes for `Inlet`{empty}s and `Outlet`{empty}s based on the -codec of the data they manipulate. -Currently, Cloudflow supports Avro, and therefore, the classes are named `AvroInlet` and `AvroOutlet`. -Each outlet also allows the user to define a partitioning function that will be used to partition the data. - -== `StreamletShape` - -The `StreamletShape` captures the connectivity and compatibility details of a Flink-based streamlet. -It captures which—and how many—inlets and outlets the streamlet has. - -The `sbt-cloudflow` plugin extracts—amongst other things—the _shape_ from the streamlet that it finds on the classpath. -This metadata is used to verify that the blueprint connects the streamlets correctly. - -Cloudflow offers an API `StreamletShape` to define various shapes with inlets and outlets. Each inlet and outlet can be defined separately with specific names. -The outlet also allows for the definition of a partitioning function. -This partitioning function is used to distribute the output data among partitions. - -When you build your own Flink streamlet, you need to define the shape. -You will learn how to do this in xref:build-flink-streamlets.adoc[Building a Flink streamlet]. -The next section describes how the `FlinkStreamletLogic` captures stream processing logic. - -== `FlinkStreamletLogic` - -The stream processing logic is captured in the abstract class `FlinkStreamletLogic`. - -The `FlinkStreamletLogic` provides methods to read and write data streams, and provides access to the configuration of the streamlet, among other things. - -A `FlinkStreamletLogic` must provide a method for executing streaming queries that process the Flink computation graph. The method `buildExecutionGraph` has to be overridden by implementation classes that process one or more `DataStream` {empty}s. The resulting graph is then submitted by `executeStreamingQueries` to the Flink runtime `StreamExecutionEnvironment` to generate the final output. -These jobs will be run by the `run` method of `FlinkStreamlet` to produce a `StreamletExecution` class, the `StreamletExecution` class manages the execution of a Flink Job. - -The `FlinkStreamletLogic` may contain instance values since it's only constructed in runtime. The `Streamlet`, however, is also instantiated during compile-time, to extract metadata, and must not contain any instance values. - -== `FlinkStreamletContext` - -The `FlinkStreamletContext` provides the necessary context under which a streamlet runs. -It contains the following context data and contracts: - -* An active `StreamExecutionEnvironment` that will be used to submit streaming jobs to the Flink runtime. -* The Typesafe `Config` loaded from the classpath through a `config` method, which can be used to read configuration settings. -* The name used in the blueprint for the specific instance of this streamlet being run. -* A mapping that gives the name of the Kafka topic from the port name. - -== Lifecycle - -In general terms, when you publish and deploy a Cloudflow application, each streamlet definition becomes a physical deployment as one or more Kubernetes artifacts. -In the case of Flink streamlets, each streamlet is submitted as a Flink application through the Flink Operator as a _Flink Application_ resource. -Upon deployment, it becomes a set of running _pods_, with one _pod_ as a Flink Job Manager and _n-pods_ as Task Managers, where _n_ is the scale factor for that streamlet. - -[NOTE] -==== -The Flink Operator is a Kubernetes operator dedicated to managing Flink applications. -==== -The Flink Operator takes care of monitoring the streamlets and is responsible for their resilient execution, like restarting the Flink application in case of failure. - -[[flink-streamlet-deployment-model]] -image::flink-application-deploy.png[Flink Streamlet Deployment Model,title="Flink Streamlet Deployment Model"] - -In <>, we can visualize the chain of delegation involved in the deployment of a Flink streamlet: - -- The Cloudflow Operator prepares a _Custom Resource_ describing the Flink streamlet and submits it to the Flink Operator -- The Flink Operator processes the _Custom Resource_ and issues the deployment of a Flink job manager pod. -- The Flink Job Manager then requests task manager resources from Kubernetes to deploy the distributed processing. -- Finally, if and when resources are available, the Flink-bound task managers start as Kubernetes pods. The task managers are the components tasked with the actual data processing, while the Job Manager serves as coordinator of the (stream) data process. - -In case you are using the cloudflow-contrib model of integration, you need to go through some additional steps to complete the deployment of your Flink streamlets. This https://lightbend.github.io/cloudflow-contrib/docs/0.0.4/get-started/flink-native.html[section] on cloudflow-contrib has more details. - -If you make any changes to the streamlet and deploy the application again, the existing Flink application will be stopped, and the new version will be started to reflect the changes. -It could be that Flink streamlets are restarted in other ways, for instance, by administrators. - -This means that a streamlet can get stopped, started or restarted at any moment in time. -The next section about message delivery semantics explains the options that are available for how to continue processing data after a restart. - -[[stream-processing-guarantees-flink]] -== Stream processing semantics - -The message processing semantics provided by Flink streamlets are determined by the guarantees provided by the underlying sources, sinks and connectors. Flink offers _at-least-once_ or _exactly_once_ semantics depending on whether checkpointing is enabled. - -If checkpointing is enabled, Flink guarantees end to end exactly-once processing with sources, sinks and connectors. -Checkpointing is enabled by default in Cloudflow, using `exactly_once` checkpointing mode. You can override this using the Configuration system by specific `execution.checkpointing` settings, as shown https://ci.apache.org/projects/flink/flink-docs-release-1.10/ops/config.html#checkpointing[here]. - -When we say “exactly-once semantics”, what we mean is that each incoming event affects the final results exactly once. Even in case of a machine or software failure, there’s no duplicate data and no data that goes unprocessed. This https://flink.apache.org/features/2018/03/01/end-to-end-exactly-once-apache-flink.html[post] explains in detail how `TwoPhaseCommitSinkFunction` implements the two-phase commit protocol and makes it possible to build end-to-end exactly-once applications with Flink and a selection of data sources and sinks, including Apache Kafka versions 0.11 and beyond. - -For more details of how the various types of processing semantics impact Kafka producers and consumers when interacting with Flink, please refer to this https://ci.apache.org/projects/flink/flink-docs-stable/dev/connectors/kafka.html[guide]. - -In Cloudflow, streamlets communicate with each other through Kafka - hence the semantics here depends on the settings that we use in our streamlets implementation. For Flink we use _at-least-once_ in `FlinkKafkaProducer`. To use _exactly-once_ the Kafka broker needs to have the value of `transaction.max.timeout.ms` set to at least _1 hour_, which can have an impact on other runtimes in the application. - -== Reliable restart of stateful processes - -A Pipeline application can be stateful and use the full capabilities of _keyed state_ and _operator state_ in Flink. Stateful operators can be used within a Flink streamlet. All states are stored in state stores deployed on Kubernetes using _Persistent Volume Claims_ (PVCs) backed by a storage class that allows for _access mode_ `ReadWriteMany`. - -To deploy a Flink application it is necessary that a PVC already exists in the namespace for the application. Cloudflow will automatically mount this PVC in the streamlets themselves to allow state storage, as long as it has a `/mnt/flink/storage` mount path. Cloudflow will look for a PVC named `cloudflow-flink` by default that mounts this path. It is also possible to create a differently named PVC, that mounts `/mnt/flink/storage` and mount it using a configuration file, as described in xref:develop:cloudflow-configuration.adoc[The Configuration Model]. The access mode for the PVC has to be `ReadWriteMany`. - -Flink's runtime encodes all state and writes them into checkpoints. And since checkpoints in Flink offer an exactly-once guarantee of semantics, all application state are preserved safely throughout the lifetime of the streamlet. In case of failures, recovery will be done from checkpoints, and there will be no data loss. - -== Adding additional Flink libraries - -If you enable plugin `CloudflowFlinkPlugin`, Cloudflow will add all of the Flink jars, necessary for a typical Flink streamlet implementation. But sometimes -you might need to add additional libraries. When doing so, it is necessary to make sure, that the versions -of the libraries that you are adding have the same version that the rest of of the Flink libraries. -The version used by Cloudflow are currently defined in the variable `cloudflow.sbt.CloudflowFlinkPlugin.FlinkVersion`. -So, if for example, you want to add Azure file system, you can do it as shown below: - ----- -libraryDependencies ++= Seq("org.apache.flink"%"flink-azure-fs-hadoop"%cloudflow.sbt.CloudflowFlinkPlugin.FlinkVersion) ----- - -== Flink local execution - -If you are testing Flink streamlets locally <> you should be aware -of several things: - -* Restartability. Unlike cluster deployment, a local Flink execution will not restart in the case of crashes. -* Checkpointing. By default, in the case of local runs, checkpointing is done in memory, which means that in the case of local runs, checkpointing does not survive beyond an individual execution. -* In the case of local execution, an instance of a Flink server is created for each SBT module for which `CloudflowFlinkPlugin` -is enabled. A single Flink server (in the case of local execution) is running a single streamlet. As a result, if you want to be able to test locally, -each SBT module should contain a single Flink streamlet. If you are developing several Flink streamlets, make sure that -each one of them is in a separate module. - -The checkpointing behaviour in this case (along with many other parameters) can be overwritten -using <>. The default configuration for Flink is https://github.com/apache/flink/blob/master/flink-dist/src/main/resources/flink-conf.yaml[here]. -Complete documentation of configuration parameters, can be found https://ci.apache.org/projects/flink/flink-docs-stable/ops/config.html[here]. - -In addition to "standard" Flink parameters, you can enable the Web UI -when using `runLocal`. If the parameter `local.web` is set in the configuration -(for the Flink runtime or a particular streamlet), the Flink execution is started with the Web UI, as shown in -https://ci.apache.org/projects/flink/flink-docs-release-1.11/ops/deployment/local.html#start-a-local-flink-cluster[here]. By default the Web UI is available -at `http://localhost:8081`, but the port can be overwritten using the configuration system with `rest.port` under `cloudflow.runtimes.flink.config.flink` or specifically for a streamlet under `cloudflow.streamlets..config`. - -An example of modifying some flink configuration is shown below: - -[source,hocon] ----- -cloudflow.streamlets.processor.config { - local.web = on - rest.port = 5000 -} ----- - -NOTE: if you have more then one Flink streamlet in your local project, DO NOT enable web UI in `cloudflow.runtimes.flink.config` - this will set the same UI port for all Flink servers -and will prevent them from starting. Make sure that you are enabling Web UI for individual streamlets and using different ports for them. - -In the above example the Web UI is enabled for the processors streamlet. which will listen on port 5000. -You need to provide the location of the local configuration in your sbt build file: - -[source,scala] ----- -lazy val taxiRidePipeline = appModule("taxi-ride-pipeline") - .enablePlugins(CloudflowApplicationPlugin) - .settings(commonSettings) - .settings( - name := "taxi-ride-fare", - runLocalConfigFile := Some("taxi-ride-pipeline/src/main/resources/local.conf"), - ) ----- - -NOTE: If you set `execution.checkpointing.interval` you need to set all other relevant checkpointing settings, the default checkpointing will not be used if `execution.checkpointing.interval` is set. - -In the above example the `runLocalConfigFile` setting is shown to point to a local.conf in the taxi-ride-fare example project. - -== Azure Blob Storage - -Flink supports several file systems for reading and writing data and for its state backend. In this section, we will look at how to use Microsoft's Azure Blob Storage for these purposes. - -Please refer to the Flink documentation https://ci.apache.org/projects/flink/flink-docs-stable/ops/filesystems/azure.html[Azure Blob Storage] - -To make this work in a Cloudflow context there are a couple of things you need to do as follows. - -=== Checkpoints - -In the configuration file, you need to specify that the `state.backend` is `filesystem`, set `state.checkpoints.dir` to match the location of your Blob, and provide your access key. - -[source,hocon] ----- -cloudflow.runtimes.flink.config { - flink.state.backend = filesystem - flink.state.checkpoints.dir = "wasbs://@$.blob.core.windows.net/" - flink.fs.azure.account.key..blob.core.windows.net = "" -} ----- - -Flink needs `flink-azure-fs-hadoop` support to be built into your Streamlet in order to be able to use Azure as a filesystem backend. To faciliate this simply add the following library dependency to the `build.sbt`. - -[source, scala] ----- -libraryDependencies += "org.apache.flink" % "flink-azure-fs-hadoop" % cloudflow.sbt.CloudflowFlinkPlugin.FlinkVersion ----- - -== Default Cloudflow Checkpointing - -By default Cloudflow sets checkpointing with the values below. - -[source,hocon] ----- -execution.checkpointing.interval = 10000 millis -execution.checkpointing.min-pause = 500 millis -execution.checkpointing.timeout = 10 mins -execution.checkpointing.max-concurrent-checkpoints = 1 -execution.checkpointing.externalized-checkpoint-retention = RETAIN_ON_CANCELLATION ----- - -This parameters can be invalidated by passing at least `execution.checkpointing.interval` setting when xref:develop:cloudflow-configuration.adoc#_runtime_specific_settings[configuring streamlets]. The interval must be changed to any other change take effect. - -It is also possible to disable the settings shown above, and therefore fallback in Flink https://ci.apache.org/projects/flink/flink-docs-release-1.10/ops/config.html#checkpointing[defaults], by configuring the Streamlet or Flink runtimes with `config.cloudflow.checkpointing.default = off`. -You can learn how in xref:develop:cloudflow-configuration.adoc#_runtime_specific_settings[Configuring Streamlets] - - - diff --git a/docs/shared-content-source/docs/modules/develop/pages/use-spark-streamlets.adoc b/docs/shared-content-source/docs/modules/develop/pages/use-spark-streamlets.adoc deleted file mode 100644 index 434b07d0c..000000000 --- a/docs/shared-content-source/docs/modules/develop/pages/use-spark-streamlets.adoc +++ /dev/null @@ -1,244 +0,0 @@ -:page-partial: - -include::ROOT:partial$include.adoc[] - -NOTE: Integration with the Spark Operator has been deprecated since 2.2.0, and will be removed in version 3.x. Spark integration has moved to the https://github.com/lightbend/cloudflow-contrib[Cloudflow-contrib project]. Please see https://lightbend.github.io/cloudflow-contrib/docs/0.1.1/get-started/index.html[the Cloudflow-contrib getting started guide] for instructions on how to use Spark Native Kubernetes integration. The documentation that follows describes the deprecated feature. The SparkStreamlet API has not changed in cloudflow-contrib, though you do need to use a different dependency and add the `CloudflowNativeSparkPlugin`, which is described in the https://lightbend.github.io/cloudflow-contrib/docs/0.1.1/get-started/spark-native.html[Cloudflow contrib documentation for building Spark native streamlets]. - -A Spark streamlet has the following responsibilities: - -* It needs to capture your stream processing logic. -* It needs to publish metadata which will be used by the `sbt-cloudflow` plugin to verify that a blueprint is correct. - This metadata consists of the _shape_ of the streamlet (`StreamletShape`) defined by the inlets and outlets of the streamlet. - Connecting streamlets need to match on inlets and outlets to make a valid cloudflow topology, as mentioned previously in xref:blueprints.adoc[Composing applications using blueprints]. - Cloudflow automatically extracts the shapes from the streamlets to verify this match. -* For the Cloudflow runtime, it needs to provide metadata so that it can be configured, scaled, and run as part of an application. -* The inlets and outlets of a `Streamlet` have two functions: -** To specify to Cloudflow that the streamlet needs certain data streams to exist at runtime, which it will read from and write to, and -** To provide handles inside the stream processing logic to connect to the data streams that Cloudflow provides. The `StreamletLogic` provides methods that take an inlet or outlet argument to read from or write to. These will be the specific data streams that Cloudflow has set up for you. - -The next sections will go into the details of defining a Spark streamlet: - -* defining inlets and outlets -* creating a streamlet shape from the inlets and outlets -* creating a streamlet logic that uses the inlets and outlets to read and write data - -== Inlets and outlets - -A streamlet can have one or more inlets and outlets. -Cloudflow offers classes for `Inlet`{empty}s and `Outlet`{empty}s based on the -codec of the data they manipulate. -Currently, Cloudflow supports Avro and hence the classes are named `AvroInlet` and `AvroOutlet`. -Each outlet also allows the user to define a partitioning function that will be used to partition the data. - -== Usage of different encodings in Spark streamlet - -Currently Cloudflow supports two different data encoding types - Avro and Protobuf. Both can be used for Spark streamlet, -but you should remember several rules. Protobuf support is based on https://scalapb.github.io/docs/sparksql/[Scala-PB Spark], -which comes with its own `implicits` - `scalapb.spark.Implicits._` which is not compatible with `cloudflow.spark.sql.SQLImplicits._`. -When using these `implicits` the following rules have to be observed: - -* If protobuf's encoding is not used, always use `cloudflow.spark.sql.SQLImplicits._`. -* If you are using at least one protobuf encoding, you need to use `scalapb.spark.Implicits._`. -Note that this `implicits` do not include default implicit supporting Scala classes. So if in addition to -protobuf, you are using Avro and or any custom scala types in your Spark processing, you have to add implicit encoders, for these -types. For example, if you are using the following type in your calculations: ----- -case class Rate(timestamp: Timestamp, value: Long) ----- -In your code you should have a definition of the corresponding encoder: ----- -implicit val rateEncoder = Encoders.product[Rate] ----- - -== `StreamletShape` - -The `StreamletShape` captures the connectivity and compatibility details of a Spark-based streamlet. -It captures which—and how many—inlets and outlets the streamlet has. - -The `sbt-cloudflow` plugin extracts—amongst other things—the _shape_ from the streamlet that it finds on the classpath. -This metadata is used to verify that the blueprint connects the streamlets correctly. - -Cloudflow offers an API `StreamletShape` to define various shapes with inlets and outlets. Each inlet and outlet can be defined separately with specific names. -The outlet also allows for the definition of a partitioning function. -This partitioning function is used to distribute the output data among partitions. - -When you build your own Spark streamlet, you need to define the shape. You will learn how to do this in xref:build-spark-streamlets.adoc[Building a Spark streamlet]. The next section describes how the `SparkStreamletLogic` captures stream processing logic. - -== `SparkStreamletLogic` - -The stream processing logic is captured in a `SparkStreamletLogic`, which is an abstract class. - -The `SparkStreamletLogic` provides methods to read and write data streams, and provides access to the configuration of the streamlet, among other things. - -A `SparkStreamletLogic` must setup one or more Structured Streaming Queries, represented by a collection of `StreamingQuery`{empty}s, through the method `buildStreamingQueries`. -These jobs will be run by the `run` method of `SparkStreamlet` to produce a `StreamletExecution`. A `StreamletExecution` is a simple class to manage a collection of `StreamingQuery`{empty}s. - -The `SparkStreamletLogic` is only constructed when the streamlet is run, so it is safe to put instance values and variables in it that you would like to use when the streamlet is running. Note that the `Streamlet` is created for extracting metadata and hence no instance values should be put inside a streamlet. - -== `SparkStreamletContext` - -The `SparkStreamletContext` provides the necessary context under which a streamlet runs. -It contains the following context data and contracts: - -* An active `SparkSession` to run Spark streaming jobs. -* The Typesafe `Config` loaded from the classpath through a `config` method, which can be used to read configuration settings. -* The name used in the blueprint for the specific instance of this streamlet being run. -* A function `checkpointDir` which returns a directory on a persistent storage where checkpoints can be safely kept, making them available across restarts. - -== Controlling Streaming Query execution - -Spark Structured streaming provides a mechanism for controlling the timing of streaming data processing in the form of -https://spark.apache.org/docs/latest/structured-streaming-programming-guide.html#triggers[triggers]. -The trigger settings of a streaming query define whether the query is going to be executed as micro-batch query with a fixed -batch interval or as a continuous processing query. Different kinds of triggers that are supported by Spark are https://spark.apache.org/docs/latest/structured-streaming-programming-guide.html#triggers[here]. - -The `writeStream` method in stream logic allows to specify trigger used for Streamlet execution. - - -== Lifecycle - -In general terms, when you publish and deploy a Cloudflow application, each streamlet definition becomes a physical deployment as one or more Kubernetes artifacts. -In the case of Spark streamlets, each streamlet is submitted as a Spark application through the Spark Operator as a _Spark Application_ resource. -Upon deployment, it becomes a set of running _pods_, with one _pod_ as a Spark Driver and _n-pods_ as executors, where _n_ is the scale factor for that streamlet. - -[NOTE] -==== -The Spark Operator is a Kubernetes operator dedicated to managing Spark applications. -==== -The Spark Operator takes care of monitoring the streamlets and is responsible for their resilient execution, like restarting the Spark application in case of failure. - -[[spark-streamlet-deployment-model]] -image::spark-application-deployment-delegation.png[Spark Streamlet Deployment Model,title="Spark Streamlet Deployment Model"] - -In the Spark streamlet deployment model shown above, we can visualize the chain of delegation involved in the deployment of a Spark streamlet: - -- The Cloudflow Operator prepares a _Custom Resource_ describing the Spark streamlet and submits it to the Spark Operator. -- The Spark Operator processes the _Custom Resource_ and issues the deployment of a Spark driver pod. -- The Spark Driver then requests executor resources from Kubernetes to deploy the distributed processing. -- Finally, if and when resources are available, the Spark-bound executors start as Kubernetes pods. The executors are the components tasked with the actual data processing, while the Spark driver serves as coordinator of the (stream) data process. - -In this architecture, the Spark driver runs the Cloudflow-specific logic that connects the streamlet to our managed data streams, at which point the streamlet starts consuming from inlets. -The streamlet advances through the data streams that are provided on inlets and writes data to outlets. - -In case you are using the cloudflow-contrib model of integration, you need to go through some additional steps to complete the deployment of your Spark streamlets. This https://lightbend.github.io/cloudflow-contrib/docs/0.0.4/get-started/spark-native.html[section] on cloudflow-contrib has more details. - -If you make any changes to the streamlet and deploy the application again, the existing Spark applications will be stopped, and the new version will be started to reflect the changes. -It could be that Spark streamlets are restarted in other ways, for instance by administrators. - -This means that a streamlet can get stopped, started or restarted at any moment in time. -The next section about message delivery semantics explains the options that are available for how to continue processing data after a restart. - -[[message-delivery-semantics-spark]] -== Message delivery semantics - -The message delivery semantics provided by Spark streamlets are determined by the guarantees provided by the underlying Spark sources and sinks used in the streamlet. Recall that we defined the different message delivery guarantees in xref:index.adoc#message-delivery-semantics[Message delivery semantics]. - -Let's consider the following types of streamlets as forming a topology as illustrated in <>: - -* an ingress, a streamlet that reads from an external streaming source and makes data available to the pipeline -* a processor, a streamlet that has an inlet and an outlet - it does domain logic processing on data that it reads and passes the processed data downstream -* an egress, a streamlet that receives data on its inlet and writes to an external sink - -The following sections will use these types of streamlets for describing message delivery semantics. - -[[ing-proc-eggr]] -image::blueprint-simple-plain.png[Ingress-Processor-Egress Streamlets, title="Ingress-Processor-Egress Streamlets"] - -=== Message delivery semantics of Spark-based ingresses - -Spark-based ingresses use a Structured Streaming source to obtain data from an external streaming source and provide it to the Cloudflow application. -In this case, message delivery semantics are dependent on the capabilities of the source. -In general, streaming sources deemed resilient will provide at-least-once semantics in a Pipeline application. - -Refer to the https://spark.apache.org/docs/latest/structured-streaming-programming-guide.html[The Structured Streaming Programming Guide] or to http://shop.oreilly.com/product/0636920047568.do[Stream Processing with Apache Spark] for more detailed documentation about your streaming source of choice. - -=== Message delivery semantics of spark-based processors - -A Spark based _processor_ is a streamlet that receives and produces data internal to the Cloudflow application. -In this scenario, _processors_ consume data from _inlets_ and produce data to _outlets_ using Kafka topics as the underlying physical implementation. - -At the level of a Spark-based processor, this translates to using Kafka sources and sinks. -Given that Kafka is a reliable data source from the Structured Streaming perspective, the message delivery semantics are considered to be _at-least-once_. - -.Testing At-Least-Once Message Delivery -[NOTE] -==== -Bundled with our examples, `spark-resilience-test` can be used to verify that messages are processed with at-least-once semantics even while a part of the pipeline is continuously failing. -==== - -=== Message delivery semantics of spark-based egresses - -The message delivery guarantees of a Spark-based egress are determined by the combination of the Kafka-based inlet and the target system where the egress will produce its data. - -The data provided to the egress is delivered with _at-least-once_ semantics. -The egress is responsible to reliably produce this data to the target system. -The overall message delivery semantics of the Pipeline will depend on the reliability characteristics of this egress. - -In particular, it is possible to 'upgrade' the end-to-end message delivery guarantee to _effectively-exactly-once_ by making the egress idempotent and ensuring that all other streamlets used provide _at-least-once_ semantics. - -For this purpose, we could use Structured Streaming's deduplication feature, or use a target system able to preserve uniqueness of primary keys, such as an RDBMS. - -=== Message delivery semantics and the Kafka retention period - -The _at-least-once_ delivery is guaranteed within the Kafka _retention_ configuration. -This _retention_ is a configuration proper to the Kafka brokers that dictates when old data can be evicted from the log. -If a Spark-based processor is offline for a longer time than the configured _retention_, it will restart from the earliest offset available in the corresponding Kafka topic, which might silently result in data loss. - - -== Reliable restart of stateful processes - -In a Pipeline application, it's possible to use the full capabilities of Structured Streaming to implement the business logic required in a streamlet. -This includes stateful processing, from time-based aggregations to arbitrary stateful computations that use `mapGroupsWithState` and `flatMapGroupsWithState`. - -All stateful processing relies on snapshots and a state store for the bookkeeping of the offsets processed and the computed state at any time. -In Cloudflow, this state store is deployed on Kubernetes using _Persistent Volume Claims_ (PVCs) backed by a storage class that allows for _access mode_ `ReadWriteMany`. - -To deploy a Spark application it is necessary that a PVC already exists in the namespace for the application. Cloudflow will automatically mount this PVC in the streamlets themselves to allow state storage, as long as it has a `/mnt/spark/storage` mount path. Cloudflow will look for a PVC named `cloudflow-spark` by default that mounts this path. It is also possible to create a differently named PVC, that mounts `/mnt/spark/storage` and mount it using a configuration file, as described in xref:develop:cloudflow-configuration.adoc[The Configuration Model]. The access mode for the PVC has to be `ReadWriteMany`. - -We can use the managed storage to safely store checkpoint data. -Checkpoint information is managed by Cloudflow for all streamlets that use the `context.writeStream` method. - -In case we need a directory to store state data that must persist across restarts of the streamlet, we can obtain a directory mounted on the managed PVC using the `context.checkpointDir(name)` method. -This method takes as parameter the `name` of the directory we want for our particular use and returns a path to a persistent storage location. - -When implementing egresses that use Spark's Structured Streaming sinks, ensure that each query uses a unique `name` for its `checkpointDir` - - -In <>, we can see the use of the `checkpointDir` method to provide the checkpoint directory to a console-based egress. - -[[checkpoint-dir-example]] -.Example of the use of `checkpointDir` -======= -[source, scala] ---------------- -include::{cloudflow-examples-version}@docsnippets:ROOT:example$spark-scala/src/main/scala/cloudflow/sparkdoc/SparkConsoleEgress.scala[tag=docs-checkpointDir-example] ---------------- -======= - -.Storage Size is Currently Fixed -[IMPORTANT] -==== -Please note that -- currently -- the volume size allocated for storage is fixed per Cloudflow platform deployment. -==== - -== Checkpoints and application upgrades - -The storage of snapshots and state has certain impact on the upgradability of the Spark-based components. -Once a state representation is deployed, the state definition (schema) may not change. - -Upgrading an application that uses stateful computation requires planning ahead of time to avoid making incompatible changes that prevent recovery using the saved state. -Significant changes to the application logic, addition of sources, or changes to the schema representing the state are not allowed. - -For details on the degrees of freedom and upgrade options, please refer to the https://spark.apache.org/docs/latest/structured-streaming-programming-guide.html#recovery-semantics-after-changes-in-a-streaming-query[Structured Streaming documentation]. - -== Adding additional Spark libraries - -If you enable plugin `CloudflowSparkPlugin`, Cloudflow will add all of the Spark jars, necessary for a typical Spark streamlet implementation. But sometimes -you might need to add additional libraries. When doing so, it is necessary to make sure, that the versions -of the libraries that you are adding have the same version that the rest of of the Spark libraries. -The version used by Cloudflow are currently defined in the variable `cloudflow.sbt.CloudflowSparkPlugin.SparkVersion`. -So, if for example, you want to add local Spark ML library, you can do it as shown below: - ----- -libraryDependencies ++= Seq("org.apache.spark"%%"spark-mllib-local"%cloudflow.sbt.CloudflowSparkPlugin.SparkVersion) ----- diff --git a/examples/README.md b/examples/README.md index c5818ce51..a8ce0208e 100644 --- a/examples/README.md +++ b/examples/README.md @@ -14,8 +14,6 @@ This directory contains examples showcasing different Cloudflow features. ## Examples -- call-record-aggregator - Akka and Spark based Cloudflow Application - sensor-data-scala - A simple Akka based pipeline that processes events from a wind turbine farm. (Scala version) -- spark-sensors - Spark based Cloudflow Application -- taxi-ride - Akka and Flink based Cloudflow Application - tensorflow-akka - A simple pipeline that scores the quality of wines using a TensorFlow model. +- connected-car-cluster-sharding - An example that uses cluster sharding. \ No newline at end of file diff --git a/examples/connected-car-cluster-sharding/akka-connected-car-streamlet/src/test/resources/application.conf b/examples/connected-car-cluster-sharding/akka-connected-car-streamlet/src/test/resources/application.conf index 5b85a0c6d..87ae2265f 100644 --- a/examples/connected-car-cluster-sharding/akka-connected-car-streamlet/src/test/resources/application.conf +++ b/examples/connected-car-cluster-sharding/akka-connected-car-streamlet/src/test/resources/application.conf @@ -2,4 +2,13 @@ akka { actor { provider = "cluster" } + remote { + artery { + transport = tcp + canonical.hostname = "" + canonical.port = 7123 + bind.hostname = "0.0.0.0" + bind.port = 7123 + } + } } diff --git a/examples/connected-car-cluster-sharding/akka-connected-car-streamlet/src/test/scala/connectedcar/streamlet/ConnectedCarClusterTest.scala b/examples/connected-car-cluster-sharding/akka-connected-car-streamlet/src/test/scala/connectedcar/streamlet/ConnectedCarClusterTest.scala index c47f670e6..06dffdcd4 100644 --- a/examples/connected-car-cluster-sharding/akka-connected-car-streamlet/src/test/scala/connectedcar/streamlet/ConnectedCarClusterTest.scala +++ b/examples/connected-car-cluster-sharding/akka-connected-car-streamlet/src/test/scala/connectedcar/streamlet/ConnectedCarClusterTest.scala @@ -27,7 +27,7 @@ class ConnectedCarClusterTest extends AnyWordSpec with Matchers with BeforeAndAf "A ConnectedCarCluster streamlet" should { val testkit = AkkaStreamletTestKit(system) - + // flaky test "Allow for creating a 'flow processor'" in { val record = generateCarERecord() val data = Vector(record) diff --git a/examples/snippets/modules/ROOT/examples/akkastreams-java/build.sbt b/examples/snippets/modules/ROOT/examples/akkastreams-java/build.sbt index fb7e863bf..560989ae8 100644 --- a/examples/snippets/modules/ROOT/examples/akkastreams-java/build.sbt +++ b/examples/snippets/modules/ROOT/examples/akkastreams-java/build.sbt @@ -10,6 +10,7 @@ lazy val sensorData = (project in file(".")) Cloudflow.library.CloudflowAvro, "com.lightbend.akka" %% "akka-stream-alpakka-file" % "1.1.2", "com.typesafe.akka" %% "akka-http-spray-json" % "10.1.12", + "com.typesafe.akka" %% "akka-http-jackson" % "10.1.12", "ch.qos.logback" % "logback-classic" % "1.2.10", "com.typesafe.akka" %% "akka-http-testkit" % "10.1.12" % "test", "org.scalatest" %% "scalatest" % "3.0.8" % "test" diff --git a/examples/snippets/modules/ROOT/examples/akkastreams-scala/src/main/scala/cloudflow/callrecordaggregator/CallRecordIngress.scala b/examples/snippets/modules/ROOT/examples/akkastreams-scala/src/main/scala/cloudflow/callrecordaggregator/CallRecordIngress.scala index e2a158b93..15fd4c41a 100644 --- a/examples/snippets/modules/ROOT/examples/akkastreams-scala/src/main/scala/cloudflow/callrecordaggregator/CallRecordIngress.scala +++ b/examples/snippets/modules/ROOT/examples/akkastreams-scala/src/main/scala/cloudflow/callrecordaggregator/CallRecordIngress.scala @@ -27,8 +27,10 @@ import cloudflow.akkastream.util.scaladsl.HttpServerLogic class CallRecordIngress extends AkkaServerStreamlet { //tag::docs-outlet-partitioner-example[] + //tag::docs-schemaAware-example val out = AvroOutlet[CallRecord]("out").withPartitioner(RoundRobinPartitioner) //end::docs-outlet-partitioner-example[] + //end::docs-schemaAware-example final override val shape = StreamletShape.withOutlets(out) final override def createLogic = HttpServerLogic.default(this, out) diff --git a/examples/snippets/modules/ROOT/examples/base-images-override/build.sbt b/examples/snippets/modules/ROOT/examples/base-images-override/build.sbt index db78411ae..95982831f 100644 --- a/examples/snippets/modules/ROOT/examples/base-images-override/build.sbt +++ b/examples/snippets/modules/ROOT/examples/base-images-override/build.sbt @@ -5,7 +5,7 @@ import sbt.Keys._ lazy val sampleApp = (project in file(".")) .enablePlugins(CloudflowApplicationPlugin) .settings( - cloudflowDockerBaseImage := "myRepositoryUrl/myRepositoryPath:2.0.10-cloudflow-akka-2.6.6-scala-2.12", + cloudflowDockerBaseImage := "myRepositoryUrl/myRepositoryPath:adoptopenjdk/openjdk11:alpine", //end::docs-projectSetup-example[] name := "sample-app", organization := "com.lightbend.cloudflow", diff --git a/examples/snippets/modules/ROOT/examples/build-akka-streamlets-java/build.sbt b/examples/snippets/modules/ROOT/examples/build-akka-streamlets-java/build.sbt index c58058ff4..ad08b2e9c 100644 --- a/examples/snippets/modules/ROOT/examples/build-akka-streamlets-java/build.sbt +++ b/examples/snippets/modules/ROOT/examples/build-akka-streamlets-java/build.sbt @@ -50,6 +50,7 @@ lazy val step2 = appModule("step2") lazy val step3 = appModule("step3") .enablePlugins(CloudflowAkkaPlugin) .settings( + libraryDependencies += Cloudflow.library.CloudflowAvro, Test / parallelExecution := false, Test / fork := true ) diff --git a/examples/templates/single-backend-java/build.sbt b/examples/templates/single-backend-java/build.sbt index 4934fda79..8de6922b4 100644 --- a/examples/templates/single-backend-java/build.sbt +++ b/examples/templates/single-backend-java/build.sbt @@ -2,8 +2,6 @@ import sbt._ import sbt.Keys._ lazy val templateJavaProject = (project in file(".")) - //enable here the backend you want to use in your application: - //CloudflowAkkaStreamsApplicationPlugin, CloudflowSparkApplicationPlugin, CloudflowFlinkApplicationPlugin .enablePlugins(CloudflowAkkaPlugin, CloudflowApplicationPlugin, ScalafmtPlugin) .settings( scalafmtOnCompile := true, diff --git a/examples/templates/single-backend-java/src/main/java/com/example/app/ConsoleOutput.java b/examples/templates/single-backend-java/src/main/java/com/example/app/ConsoleOutput.java index eff1ea92c..f7941488a 100644 --- a/examples/templates/single-backend-java/src/main/java/com/example/app/ConsoleOutput.java +++ b/examples/templates/single-backend-java/src/main/java/com/example/app/ConsoleOutput.java @@ -20,18 +20,11 @@ import akka.stream.*; import akka.stream.javadsl.*; -// pick the streamlet implementation corresponding to your chosen backend -// import cloudflow.spark._ -// import cloudflow.flink._ - import cloudflow.akkastream.*; import cloudflow.akkastream.javadsl.*; import cloudflow.streamlets.*; import cloudflow.streamlets.avro.*; - -// Implement the streamlet extending the corresponding base to the chosen backend: -// AkkaStreamlet, SparkSteamlet, FlinkStreamlet public class ConsoleOutput extends AkkaStreamlet { // Create inputs and outputs by declaring inlets and outlets diff --git a/examples/templates/single-backend-java/src/main/java/com/example/app/DataInput.java b/examples/templates/single-backend-java/src/main/java/com/example/app/DataInput.java index d6531f7ea..bd6260120 100644 --- a/examples/templates/single-backend-java/src/main/java/com/example/app/DataInput.java +++ b/examples/templates/single-backend-java/src/main/java/com/example/app/DataInput.java @@ -29,14 +29,9 @@ import java.util.Random; import scala.concurrent.duration.*; -// pick the streamlet implementation corresponding to your chosen backend -// import cloudflow.spark._ -// import cloudflow.flink._ import cloudflow.akkastream.*; import cloudflow.akkastream.javadsl.*; -// Implement the streamlet extending the corresponding base to the chosen backend: -// AkkaStreamlet, SparkSteamlet, FlinkStreamlet public class DataInput extends AkkaStreamlet { // declare inputs and outputs diff --git a/examples/templates/single-backend-scala/src/main/scala/com/example/app/ConsoleOutput.scala b/examples/templates/single-backend-scala/src/main/scala/com/example/app/ConsoleOutput.scala index 60f5339c4..0414bf78c 100644 --- a/examples/templates/single-backend-scala/src/main/scala/com/example/app/ConsoleOutput.scala +++ b/examples/templates/single-backend-scala/src/main/scala/com/example/app/ConsoleOutput.scala @@ -15,17 +15,11 @@ */ package com.example.app -// pick the streamlet implementation corresponding to your chosen backend -// import cloudflow.spark._ -// import cloudflow.flink._ - import cloudflow.akkastream._ import cloudflow.akkastream.scaladsl._ import cloudflow.streamlets._ import cloudflow.streamlets.avro._ -// Implement the streamlet extending the corresponding base to the chosen backend: -// AkkaStreamlet, SparkSteamlet, FlinkStreamlet class ConsoleOutput extends AkkaStreamlet { // declare inputs, outputs, and a shape diff --git a/examples/templates/single-backend-scala/src/main/scala/com/example/app/DataGenerator.scala b/examples/templates/single-backend-scala/src/main/scala/com/example/app/DataGenerator.scala index baecb6d3e..e0165fe96 100644 --- a/examples/templates/single-backend-scala/src/main/scala/com/example/app/DataGenerator.scala +++ b/examples/templates/single-backend-scala/src/main/scala/com/example/app/DataGenerator.scala @@ -25,14 +25,9 @@ import akka.stream.scaladsl._ import scala.util.Random import scala.concurrent.duration._ -// pick the streamlet implementation corresponding to your chosen backend -// import cloudflow.spark._ -// import cloudflow.flink._ import cloudflow.akkastream._ import cloudflow.akkastream.scaladsl._ -// Implement the streamlet extending the corresponding base to the chosen backend: -// AkkaStreamlet, SparkSteamlet, FlinkStreamlet class DataInput extends AkkaStreamlet { // declare inputs and outputs