How to reference proto files in common sub projects?

1

Hi,

We have refactored the build.sbt file to:


  def scala(project: Project): Project = {
    project.settings(
      ThisBuild / dynverSeparator := "-",
      run / fork := true,
      run / envVars += ("HOST", "0.0.0.0"),
      Compile / run := {
        // needed for the proxy to access the user function on all platforms
        sys.props += "kalix.user-function-interface" -> "0.0.0.0"
        (Compile / run).evaluated
      },
      Compile / scalacOptions ++= Seq(
        "-release:11",
        "-deprecation",
        "-feature",
        "-unchecked",
        "-Xlog-reflective-calls",
        "-Xlint"
      ),

      Compile / javacOptions ++= Seq(
        "-Xlint:unchecked",
        "-Xlint:deprecation",
        "-parameters" // for Jackson
      )
    )
  }


}

object Testing {
  def scalaTest(project: Project): Project = {
    project.settings(
      Test / parallelExecution := false,
      Test / testOptions += Tests.Argument("-oDF"),
      Test / logBuffered := false,
      libraryDependencies ++= basicTestingDependencies
    )
  }
}

object Packaging {

  def docker(project: Project): Project = {
    project.settings(
      dockerBaseImage := "docker.io/library/adoptopenjdk:11-jre-hotspot",
      dockerUsername := sys.props.get("docker.username"),
      dockerRepository := sys.props.get("docker.registry"),
      dockerUpdateLatest := true,
      dockerBuildCommand := {
        if (sys.props("os.arch") != "amd64") {
          // use buildx with platform to build supported amd64 images on other CPU architectures
          // this may require that you have first run 'docker buildx create' to set docker buildx up
          dockerExecCommand.value ++ Seq(
            "buildx",
            "build",
            "--platform=linux/amd64",
            "--load"
          ) ++ dockerBuildOptions.value :+ "."
        } else dockerBuildCommand.value
      }
    )
  }

  def dockerConfiguration(project: Project): Project = {
    project.settings(
      dockerBaseImage := "docker.io/library/eclipse-temurin:17",
      dockerUsername := sys.props.get("docker.username"),
      dockerRepository := sys.props.get("docker.registry"),
      dockerUpdateLatest := true,
      // dockerAlias := dockerAlias.value.withTag(Some("latest"))
      dockerExposedPorts ++= Seq(8080),
      dockerBuildCommand := {
        if (sys.props("os.arch") != "amd64") {
          // use buildx with platform to build supported amd64 images on other CPU architectures
          // this may require that you have first run 'docker buildx create' to set docker buildx up
          dockerExecCommand.value ++ Seq(
            "buildx",
            "build",
            "--platform=linux/amd64",
            "--load"
          ) ++ dockerBuildOptions.value :+ "."
        } else dockerBuildCommand.value
      }
    )
  }
}

object Kalix {

  def service(componentName: String)(project: Project): Project = {
    project.enablePlugins(KalixPlugin, JavaAppPackaging, DockerPlugin)
      .configure(Compilation.scala)
      .configure(Testing.scalaTest)
      .configure(Packaging.docker)
      .settings(
        name := componentName,
        run / fork := true,
        libraryDependencies ++= utilityDependencies ++ loggingDependencies,
        Compile / managedSourceDirectories ++= Seq(
          target.value / "scala-2.13" / "akka-grpc",
          target.value / "scala-2.13" / "src_managed"
        )
      )
  }

  def library(componentName: String)(project: Project): Project = {
    project.enablePlugins(KalixPlugin)
      .configure(Compilation.scala)
      .configure(Testing.scalaTest)
      .settings(
        name := componentName,
        run / fork := true,
        libraryDependencies ++= loggingDependencies,
        Compile / managedSourceDirectories ++= Seq(
          target.value / "scala-2.13" / "akka-grpc",
          target.value / "scala-2.13" / "src_managed"
        )
      )
  }

  def dependsOn(dependency: Project, name: String)(project: Project): Project = {
    project.settings(
      libraryDependencies ++= {
        Seq(
          "com.library" %% name % version.value % "protobuf"
        )
      }
    ).dependsOn(dependency)
  }
}

and define our sub projects with:

lazy val org = project
.in(file(“project1”))
.configure(Kalix.service(“project1”))
// .configure(Kalix.dependsOn(common, “common”))

lazy val event = project
.in(file(“project2”))
.configure(Kalix.service(“project2”))

with .configure(Kalix.dependsOn(common, “common”)), we are getting the following error:

[error] java.lang.RuntimeException: protoc returned exit code: 1
[error] at scala.sys.package$.error(package.scala:30)
[error] at sbtprotoc.ProtocPlugin$.compile(ProtocPlugin.scala:464)
…

How do we reference common shared proto files in a sub project common without requiring the the common subproject to have any services defined which caused the codegen errors?

2

Did you try disabling the Kalix plugin for that non-service module in your sbt build with disablePlugins(KalixPlugin)?

3

I did have it disabled. But how do I reference the proto files in the common from other projects? Those files are not found. Is there a way to a separate sub project to have code only for references from other sub projects? Thanks.

4

I have not tried this out but I think one way would be to add a relative proto path to Compile / PB.protoSources for each service.

Not sure if/how we support picking up protos from the classpath for generation and it will also depend a bit on exactly what you are doing, but I think you may have to dig deeper into the underlying tools for that, for example: loading proto files from artifacts in Akka gRPC docs

5

Thanks Johan. Can you elaborate on how do I add Compile / PB.protoSources to my build.sbt above? I would like to be able to have subprojects that host their own services individually while some of them can share some common code. Like:

lazy val project1 = project
.in(file(“project1”))
.configure(Kalix.service(“project1”))
.configure(Kalix.dependsOn(common, “common”))

lazy val project2 = project
.in(file(“project2”))
.configure(Kalix.service(“project2”))
.configure(Kalix.dependsOn(common, “common”))
.configure(Kalix.dependsOn(project1, “ project1”))

lazy val project3 = project
.in(file(“project3”))
.configure(Kalix.service(“project3”))
.configure(Kalix.dependsOn(common, “common”))
.configure(Kalix.dependsOn(project1, “ project1”))
.configure(Kalix.dependsOn(project2, “ project2”))