Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
102 changes: 98 additions & 4 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,7 @@ jobs:
name: Detect Changed Paths
runs-on: ubuntu-latest
outputs:
compiler: ${{ steps.filter.outputs.compiler }}
cpp: ${{ steps.filter.outputs.cpp }}
cpp_code: ${{ steps.filter.outputs.cpp_code }}
java_code: ${{ steps.filter.outputs.java_code }}
Expand All @@ -68,6 +69,7 @@ jobs:
BASE_REF: ${{ github.base_ref }}
run: |
if [[ "$GITHUB_EVENT_NAME" != "pull_request" ]]; then
echo "compiler=true" >> "$GITHUB_OUTPUT"
echo "cpp=true" >> "$GITHUB_OUTPUT"
echo "cpp_code=true" >> "$GITHUB_OUTPUT"
echo "java_code=true" >> "$GITHUB_OUTPUT"
Expand All @@ -84,6 +86,12 @@ jobs:
git fetch --no-tags --depth=1 origin "$BASE_REF:refs/remotes/origin/$BASE_REF"
changed_files="$(git diff --name-only "origin/$BASE_REF" HEAD)"

if grep -Eq '^(compiler/)' <<< "$changed_files"; then
echo "compiler=true" >> "$GITHUB_OUTPUT"
else
echo "compiler=false" >> "$GITHUB_OUTPUT"
fi

if grep -Eq '^(cpp/|examples/cpp/|benchmarks/cpp/|integration_tests/idl_tests/cpp/|bazel/|BUILD$|WORKSPACE$|MODULE\.bazel$|\.bazelrc$)' <<< "$changed_files"; then
echo "cpp=true" >> "$GITHUB_OUTPUT"
else
Expand Down Expand Up @@ -687,8 +695,88 @@ jobs:
- name: Run CI
run: python ./ci/run_ci.py java --version integration_tests

grpc_tests:
name: Java/Python/Go/Rust gRPC Tests
grpc_java_python_tests:
name: Java/Python gRPC Tests
needs: changes
if: needs.changes.outputs.compiler == 'true'
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v5
- name: Set up JDK 21
uses: actions/setup-java@v4
with:
java-version: 21
distribution: "temurin"
- name: Set up Python 3.11
uses: actions/setup-python@v5
with:
python-version: 3.11
cache: "pip"
- name: Cache Maven local repository
uses: actions/cache@v4
with:
path: ~/.m2/repository
key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
restore-keys: |
${{ runner.os }}-maven-
- name: Install Java artifacts for gRPC tests
run: |
cd java
mvn -T16 --no-transfer-progress clean install -DskipTests -Dmaven.javadoc.skip=true -Dmaven.source.skip=true
- name: Install Python gRPC dependencies
run: |
python -m pip install "grpcio>=1.62.2,<1.71"
python -m pip install -v -e python
- name: Generate gRPC test sources
run: python integration_tests/grpc_tests/generate_grpc.py
- name: Run Java/Python gRPC Tests
run: |
cd integration_tests/grpc_tests/java
mvn -T16 --no-transfer-progress -Dtest=PythonGrpcInteropTest test

grpc_java_go_tests:
name: Java/Go gRPC Tests
needs: changes
if: needs.changes.outputs.compiler == 'true'
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v5
- name: Set up JDK 21
uses: actions/setup-java@v4
with:
java-version: 21
distribution: "temurin"
- name: Set up Python 3.11
uses: actions/setup-python@v5
with:
python-version: 3.11
cache: "pip"
- name: Cache Maven local repository
uses: actions/cache@v4
with:
path: ~/.m2/repository
key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
restore-keys: |
${{ runner.os }}-maven-
- name: Install Java artifacts for gRPC tests
run: |
cd java
mvn -T16 --no-transfer-progress clean install -DskipTests -Dmaven.javadoc.skip=true -Dmaven.source.skip=true
- name: Generate gRPC test sources
run: python integration_tests/grpc_tests/generate_grpc.py
- name: Build Go gRPC peer
run: |
cd integration_tests/grpc_tests/go
go build -o grpc-interop .
- name: Run Java/Go gRPC Tests
run: |
cd integration_tests/grpc_tests/java
mvn -T16 --no-transfer-progress -Dtest=GoGrpcInteropTest test

grpc_java_rust_tests:
name: Java/Rust gRPC Tests
needs: changes
if: needs.changes.outputs.compiler == 'true'
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v5
Expand Down Expand Up @@ -718,8 +806,14 @@ jobs:
run: |
cd java
mvn -T16 --no-transfer-progress clean install -DskipTests -Dmaven.javadoc.skip=true -Dmaven.source.skip=true
- name: Run Java/Python/Go/Rust gRPC Tests
run: ./integration_tests/grpc_tests/run_tests.sh
- name: Generate gRPC test sources
run: python integration_tests/grpc_tests/generate_grpc.py
- name: Build Rust gRPC peer
run: cargo build --manifest-path integration_tests/grpc_tests/rust/Cargo.toml --workspace --quiet
- name: Run Java/Rust gRPC Tests
run: |
cd integration_tests/grpc_tests/java
mvn -T16 --no-transfer-progress -Dtest=RustGrpcInteropTest test

javascript:
name: JavaScript CI
Expand Down
6 changes: 6 additions & 0 deletions ci/release.py
Original file line number Diff line number Diff line change
Expand Up @@ -282,6 +282,12 @@ def bump_rust_version(new_version):
rust_version,
_update_cargo_package_version,
)
_bump_version(
"integration_tests/grpc_tests/rust",
"Cargo.toml",
rust_version,
_update_rust_version,
)


def bump_kotlin_version(new_version):
Expand Down
4 changes: 0 additions & 4 deletions compiler/fory_compiler/generators/go.py
Original file line number Diff line number Diff line change
Expand Up @@ -207,10 +207,6 @@ def generate(self) -> List[GeneratedFile]:
# Generate a single Go file with all types
files.append(self.generate_file())

# Generate gRPC service stubs if requested
if self.options.grpc:
files.extend(self.generate_services())

return files

def get_package_name(self) -> str:
Expand Down
5 changes: 1 addition & 4 deletions compiler/fory_compiler/generators/services/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@
"Shared utilities for gRPC service stub generators."

from enum import Enum
from typing import List, Dict
from typing import Dict
from fory_compiler.ir.ast import RpcMethod


Expand Down Expand Up @@ -49,6 +49,3 @@ def __init__(self):

def add(self, alias: str, import_path: str) -> None:
self._imports[alias] = import_path

def go_imports(self) -> List[str]:
return sorted(self._imports.values())
61 changes: 30 additions & 31 deletions compiler/fory_compiler/generators/services/go.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@

"""Go gRPC service code generator."""

from typing import List
from typing import Dict, List
from fory_compiler.generators.services.base import (
ImportTracker,
StreamingMode,
Expand All @@ -36,8 +36,22 @@ def generate_services(self) -> List[GeneratedFile]:
]
if not local_services:
return []
self._validate_method_name_collisions(local_services)
return [self._generate_grpc_file(local_services)]

def _validate_method_name_collisions(self, services: List[Service]) -> None:
for service in services:
seen: Dict[str, str] = {}
for method in service.methods:
go_name = self.to_pascal_case(method.name)
prior = seen.get(go_name)
if prior is not None:
raise ValueError(
f"Go gRPC method name collision in service {service.name}: "
f"{prior} and {method.name} both generate {go_name}"
)
seen[go_name] = method.name

def _generate_grpc_file(self, services: List[Service]) -> GeneratedFile:
"""Generate one _grpc.go file containing all services in the schema."""
lines: List[str] = []
Expand Down Expand Up @@ -85,7 +99,6 @@ def _build_import_block(self, tracker: ImportTracker) -> List[str]:
'"google.golang.org/grpc/codes"',
'"google.golang.org/grpc/mem"',
'"google.golang.org/grpc/status"',
'"github.com/apache/fory/go/fory"',
]

for alias, path in tracker._imports.items():
Expand Down Expand Up @@ -142,19 +155,16 @@ def _generate_client_struct(self, service: Service) -> List[str]:
lines: List[str] = []
lines.append(f"type {self.to_camel_case(service.name)}Client struct {{")
lines.append("\tcc grpc.ClientConnInterface")
lines.append("\tfory *fory.Fory")
lines.append("}")
lines.append("")
return lines

def _generate_new_client(self, service: Service) -> List[str]:
lines: List[str] = []
lines.append(
f"func New{service.name}Client(cc grpc.ClientConnInterface, f *fory.Fory) {service.name}Client {{"
)
lines.append(
f"\treturn &{self.to_camel_case(service.name)}Client{{cc: cc, fory: f}}"
f"func New{service.name}Client(cc grpc.ClientConnInterface) {service.name}Client {{"
)
lines.append(f"\treturn &{self.to_camel_case(service.name)}Client{{cc: cc}}")
lines.append("}")
lines.append("")
return lines
Expand All @@ -164,31 +174,24 @@ def _generate_codec(self) -> List[str]:
lines.append(
"// CodecV2 implements grpc/encoding.CodecV2 using Fory serialization."
)
lines.append(
"// Pass a configured *fory.Fory instance with all message types registered."
)
lines.append("type CodecV2 struct {")
lines.append("\tFory *fory.Fory")
lines.append("}")
lines.append("// It uses the generated package-level thread-safe Fory runtime.")
lines.append("type CodecV2 struct{}")
lines.append("")
lines.append(
"// Marshal serializes v with Fory. The result is copied before being handed"
"// Marshal serializes v with Fory. The generated thread-safe runtime returns"
)
lines.append(
"// to gRPC because Fory reuses its internal write buffer across calls —"
"// a stable copy before releasing pooled Fory instances, so gRPC never sees"
)
lines.append(
"// streaming handlers may buffer multiple frames before sending, and without"
"// the reusable internal buffers owned by plain fory.Fory runtimes."
)
lines.append("// a copy all frames would alias the last serialized value.")
lines.append("func (c CodecV2) Marshal(v any) (mem.BufferSlice, error) {")
lines.append("\tb, err := c.Fory.Marshal(v)")
lines.append("\tb, err := getFory().Serialize(v)")
lines.append("\tif err != nil {")
lines.append("\t\treturn nil, err")
lines.append("\t}")
lines.append("\tout := make([]byte, len(b))")
lines.append("\tcopy(out, b)")
lines.append("\treturn mem.BufferSlice{mem.NewBuffer(&out, nil)}, nil")
lines.append("\treturn mem.BufferSlice{mem.NewBuffer(&b, nil)}, nil")
lines.append("}")
lines.append("")
lines.append(
Expand All @@ -204,7 +207,7 @@ def _generate_codec(self) -> List[str]:
lines.append("\tfor _, buf := range data {")
lines.append("\t\tn += copy(b[n:], buf.ReadOnlyData())")
lines.append("\t}")
lines.append("\treturn c.Fory.Unmarshal(b, v)")
lines.append("\treturn getFory().Deserialize(b, v)")
lines.append("}")
lines.append("")
lines.append(
Expand All @@ -229,7 +232,7 @@ def _generate_client_methods(
)
lines.append(f"\tout := new({res_type[1:]})")
lines.append(
"\tcallOpts := append([]grpc.CallOption{grpc.ForceCodecV2(CodecV2{Fory: c.fory})}, opts...)"
"\tcallOpts := append([]grpc.CallOption{grpc.ForceCodecV2(CodecV2{})}, opts...)"
)
lines.append(
f'\terr := c.cc.Invoke(ctx, "{self.get_grpc_method_path(service, method)}", in, out, callOpts...)'
Expand All @@ -245,7 +248,7 @@ def _generate_client_methods(
f"func (c *{self.to_camel_case(service.name)}Client) {self.to_pascal_case(method.name)}(ctx context.Context, in {req_type}, opts ...grpc.CallOption) ({service.name}_{self.to_pascal_case(method.name)}Client, error) {{"
)
lines.append(
"\tcallOpts := append([]grpc.CallOption{grpc.ForceCodecV2(CodecV2{Fory: c.fory})}, opts...)"
"\tcallOpts := append([]grpc.CallOption{grpc.ForceCodecV2(CodecV2{})}, opts...)"
)
lines.append(
f'\tstream, err := c.cc.NewStream(ctx, &_{service.name}_serviceDesc.Streams[{stream_index}], "{self.get_grpc_method_path(service, method)}", callOpts...)'
Expand All @@ -271,7 +274,7 @@ def _generate_client_methods(
f"func (c *{self.to_camel_case(service.name)}Client) {self.to_pascal_case(method.name)}(ctx context.Context, opts ...grpc.CallOption) ({service.name}_{self.to_pascal_case(method.name)}Client, error) {{"
)
lines.append(
"\tcallOpts := append([]grpc.CallOption{grpc.ForceCodecV2(CodecV2{Fory: c.fory})}, opts...)"
"\tcallOpts := append([]grpc.CallOption{grpc.ForceCodecV2(CodecV2{})}, opts...)"
)
lines.append(
f'\tstream, err := c.cc.NewStream(ctx, &_{service.name}_serviceDesc.Streams[{stream_index}], "{self.get_grpc_method_path(service, method)}", callOpts...)'
Expand Down Expand Up @@ -683,9 +686,7 @@ def _generate_unary_type_desc(self, service: Service) -> List[str]:
mode = streaming_mode(method)
if mode is StreamingMode.UNARY:
lines.append("\t\t{")
lines.append(
f'\t\t\tMethodName:\t"{self.to_pascal_case(method.name)}",'
)
lines.append(f'\t\t\tMethodName:\t"{method.name}",')
lines.append(
f"\t\t\tHandler:\t_{service.name}_{self.to_pascal_case(method.name)}_Handler,"
)
Expand All @@ -700,9 +701,7 @@ def _generate_stream_type_desc(self, service: Service) -> List[str]:
continue
else:
lines.append("\t\t{")
lines.append(
f'\t\t\tStreamName:\t"{self.to_pascal_case(method.name)}",'
)
lines.append(f'\t\t\tStreamName:\t"{method.name}",')
lines.append(
f"\t\t\tHandler:\t_{service.name}_{self.to_pascal_case(method.name)}_Handler,"
)
Expand Down
Loading
Loading