Refactor to use the crossplane.pythonic package.

Upgrade to use crossplane-sdk-python v0.9.0.
Implement being able to specify the full path to the Composite class.

Author: Patrick J. McNerthney

.github/workflows/ci.yaml

@@ -127,13 +127,13 @@ jobs:
           build-args:
             PYTHON_VERSION=${{ env.PYTHON_VERSION }}
           outputs: type=docker,dest=runtime-${{ matrix.arch }}.tar
-      
+
       - name: Setup the Crossplane CLI
-        run: "curl -sL https://raw.githubusercontent.com/crossplane/crossplane/master/install.sh | sh"
+        run: "mkdir bin && cd bin && curl -sL https://raw.githubusercontent.com/crossplane/crossplane/master/install.sh | sh"
 
       - name: Build Package
-        run: ./crossplane xpkg build --package-file=${{ matrix.arch }}.xpkg --package-root=package/ --embed-runtime-image-tarball=runtime-${{ matrix.arch }}.tar
-      
+        run: bin/crossplane xpkg build --package-file=${{ matrix.arch }}.xpkg --package-root=package/ --embed-runtime-image-tarball=runtime-${{ matrix.arch }}.tar
+
       - name: Upload Single-Platform Package
         uses: actions/upload-artifact@v4
         with:
@@ -165,7 +165,15 @@ jobs:
           merge-multiple: true
 
       - name: Setup the Crossplane CLI
-        run: "curl -sL https://raw.githubusercontent.com/crossplane/crossplane/master/install.sh | sh"
+        run: "mkdir bin && cd bin && curl -sL https://raw.githubusercontent.com/crossplane/crossplane/master/install.sh | sh"
+
+      # If a version wasn't explicitly passed as a workflow_dispatch input we
+      # default to version v0.0.0-<git-commit-date>-<git-short-sha>, for example
+      # v0.0.0-20231101115142-1091066df799. This is a simple implementation of
+      # Go's pseudo-versions: https://go.dev/ref/mod#pseudo-versions.
+      - name: Set Default Multi-Platform Package Version
+        if: env.XPKG_VERSION == ''
+        run: echo "XPKG_VERSION=v0.0.0-$(date -d@$(git show -s --format=%ct) +%Y%m%d%H%M%S)-$(git rev-parse --short=12 HEAD)" >> $GITHUB_ENV
 
       # - name: Login to Upbound
       #   uses: docker/login-action@v3
@@ -175,17 +183,9 @@ jobs:
       #     username: ${{ secrets.XPKG_ACCESS_ID }}
       #     password: ${{ secrets.XPKG_TOKEN }}
 
-      # If a version wasn't explicitly passed as a workflow_dispatch input we
-      # default to version v0.0.0-<git-commit-date>-<git-short-sha>, for example
-      # v0.0.0-20231101115142-1091066df799. This is a simple implementation of
-      # Go's pseudo-versions: https://go.dev/ref/mod#pseudo-versions.
-      # - name: Set Default Multi-Platform Package Version
-      #   if: env.XPKG_VERSION == ''
-      #   run: echo "XPKG_VERSION=v0.0.0-$(date -d@$(git show -s --format=%ct) +%Y%m%d%H%M%S)-$(git rev-parse --short=12 HEAD)" >> $GITHUB_ENV
-
       # - name: Push Multi-Platform Package to Upbound
       #   if: env.XPKG_ACCESS_ID != ''
-      #   run: "./crossplane --verbose xpkg push --package-files $(echo *.xpkg|tr ' ' ,) ${{ env.XPKG }}:${{ env.XPKG_VERSION }}"
+      #   run: "bin/crossplane --verbose xpkg push --package-files $(echo *.xpkg|tr ' ' ,) ${{ env.XPKG }}:${{ env.XPKG_VERSION }}"
 
       - name: Login to GHCR
         uses: docker/login-action@v3
@@ -195,4 +195,4 @@ jobs:
           password: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Push Multi-Platform Package to GHCR
-        run: "./crossplane --verbose xpkg push --package-files $(echo *.xpkg|tr ' ' ,) ${{ env.CROSSPLANE_REGORG }}:${{ env.XPKG_VERSION }}"
+        run: "bin/crossplane --verbose xpkg push --package-files $(echo *.xpkg|tr ' ' ,) ${{ env.CROSSPLANE_REGORG }}:${{ env.XPKG_VERSION }}"

CloudOps.groovy

@@ -27,17 +27,17 @@ RUN \
     && /venv/build/bin/pip install hatch \
     && /venv/build/bin/hatch build -t wheel /whl
 
-# Create a fresh venv and install only the function wheel into it.
+# Create a fresh venv and install only the pythonic wheel into it.
 #RUN --mount=type=cache,target=/root/.cache/pip \
 RUN \
     python3 -m venv /venv/fn \
     && /venv/fn/bin/pip install /whl/*.whl
 
-# Copy the function venv to our runtime stage. It's important that the path be
+# Copy the pythonic venv to our runtime stage. It's important that the path be
 # the same as in the build stage, to avoid shebang paths and symlinks breaking. 
 FROM gcr.io/distroless/python3-debian12 AS image
 WORKDIR /
 USER nonroot:nonroot
 COPY --from=build --chown=nonroot:nonroot /venv/fn /venv/fn
 EXPOSE 9443
-ENTRYPOINT ["/venv/fn/bin/function"]
+ENTRYPOINT ["/venv/fn/bin/pythonic"]

Dockerfile

@@ -35,6 +35,8 @@ spec:
             vpc.spec.forProvider.cidrBlock = self.spec.cidr
             self.status.vpcId = vpc.status.atProvider.vpcId
 ```
+In addtion to an inline script, the python implementation can be specified
+as the complete path to a python class. See [Filing system Composites](#filing-system-composites).
 
 ## Examples
 
@@ -270,6 +272,38 @@ spec:
         self.status.composite = 'Hello, World!'
 ```
 
+## Filing system Composites
+
+Composition Composite implementations can be coded in a stand alone python files
+by configuring the function-pythonic deployment with the code mounted into
+the package-runtime container, and then adding the mount point to the python
+path using the --python-path command line option.
+```yaml
+apiVersion: pkg.crossplane.io/v1beta1
+kind: DeploymentRuntimeConfig
+metadata:
+  name: function-pythonic
+spec:
+  deploymentTemplate:
+    spec:
+      template:
+        spec:
+          containers:
+          - name: package-runtime
+            args:
+            - --debug
+            - --python-path
+            - /mnt/composites
+            volumeMounts:
+            - name: composites
+              mountPath: /mnt/composites
+          volumes:
+          - name: composites
+            configMap:
+              name: pythonic-composites
+```
+See the [filing-system](examples/filing-system) example.
+
 ## Install Additional Python Packages
 
 function-pythonic supports a `--pip-install` command line option which will run pip install

README.md

@@ -0,0 +1,17 @@
+import base64
+
+from .composite import BaseComposite
+from .protobuf import Map, List, Unknown, Yaml, Json
+B64Encode = lambda s: base64.b64encode(s.encode('utf-8')).decode('utf-8')
+B64Decode = lambda s: base64.b64decode(s.encode('utf-8')).decode('utf-8')
+
+__all__ = [
+    'BaseComposite',
+    'Map',
+    'List',
+    'Unknown',
+    'Yaml',
+    'Json',
+    'B64Encode',
+    'B64Decode',
+]

crossplane/pythonic/__init__.py

@@ -2,16 +2,16 @@
 import datetime
 from crossplane.function.proto.v1 import run_function_pb2 as fnv1
 
-import function.protobuf
+from . import protobuf
 
 
 _notset = object()
 
 
 class BaseComposite:
     def __init__(self, request, response, logger):
-        self.request = function.protobuf.Message(None, None, request.DESCRIPTOR, request, 'Function Request')
-        self.response = function.protobuf.Message(None, None, response.DESCRIPTOR, response)
+        self.request = protobuf.Message(None, None, request.DESCRIPTOR, request, 'Function Request')
+        self.response = protobuf.Message(None, None, response.DESCRIPTOR, response)
         self.logger = logger
         self.autoReady = True
         self.credentials = Credentials(self.request)
@@ -54,7 +54,7 @@ def ready(self):
     def ready(self, ready):
         if ready:
             ready = fnv1.Ready.READY_TRUE
-        elif ready == None or (isinstance(ready, function.protobuf.Values) and ready._isUnknown):
+        elif ready == None or (isinstance(ready, protobuf.Values) and ready._isUnknown):
             ready = fnv1.Ready.READY_UNSPECIFIED
         else:
             ready = fnv1.Ready.READY_FALSE
@@ -115,7 +115,6 @@ def __setattr__(self, key, resource):
         self[key] = resource
 
     def __setitem__(self, key, resource):
-        print('SETITEM', key, resource)
         self._composite.response.desired.resources[key].resource = resource
 
     def __delattr__(self, key):
@@ -216,7 +215,7 @@ def ready(self):
     def ready(self, ready):
         if ready:
             ready = fnv1.Ready.READY_TRUE
-        elif ready == None or (isinstance(ready, function.protobuf.Values) and ready._isUnknown):
+        elif ready == None or (isinstance(ready, protobuf.Values) and ready._isUnknown):
             ready = fnv1.Ready.READY_UNSPECIFIED
         else:
             ready = fnv1.Ready.READY_FALSE
@@ -422,7 +421,7 @@ def claim(self, claim):
         if bool(self):
             if claim:
                 self._result.target = fnv1.Target.TARGET_COMPOSITE_AND_CLAIM
-            elif claim == None or (isinstance(claim, function.protobuf.Values) and claim._isUnknown):
+            elif claim == None or (isinstance(claim, protobuf.Values) and claim._isUnknown):
                 self._result.target = fnv1.Target.TARGET_UNSPECIFIED
             else:
                 self._result.target = fnv1.Target.TARGET_COMPOSITE
@@ -461,7 +460,7 @@ def __getitem__(self, type):
         return Condition(self, type)
 
 
-class Condition(function.protobuf.ProtobufValue):
+class Condition(protobuf.ProtobufValue):
     def __init__(self, conditions, type):
         self._conditions = conditions
         self.type = type
@@ -509,7 +508,7 @@ def status(self, status):
             condition.status = fnv1.Status.STATUS_CONDITION_TRUE
         elif status == None:
             condition.status = fnv1.Status.STATUS_CONDITION_UNKNOWN
-        elif isinstance(status, function.protobuf.Values) and status._isUnknown:
+        elif isinstance(status, protobuf.Values) and status._isUnknown:
             condition.status = fnv1.Status.STATUS_CONDITION_UNSPECIFIED
         else:
             condition.status = fnv1.Status.STATUS_CONDITION_FALSE
@@ -556,7 +555,7 @@ def claim(self, claim):
         condition = self._find_condition(True)
         if claim:
             condition.target = fnv1.Target.TARGET_COMPOSITE_AND_CLAIM
-        elif claim == None or (isinstance(claim, function.protobuf.Values) and claim._isUnknown):
+        elif claim == None or (isinstance(claim, protobuf.Values) and claim._isUnknown):
             condition.target = fnv1.Target.TARGET_UNSPECIFIED
         else:
             condition.target = fnv1.Target.TARGET_COMPOSITE

function/__version__.py crossplane/pythonic/__version__.pyfunction/__version__.py renamed to crossplane/pythonic/__version__.py

@@ -2,15 +2,25 @@
 
 import asyncio
 import base64
+import builtins
+import importlib
 import inspect
 
 import grpc
 import crossplane.function.logging
 import crossplane.function.response
 from crossplane.function.proto.v1 import run_function_pb2 as fnv1
 from crossplane.function.proto.v1 import run_function_pb2_grpc as grpcv1
-import function.composite
-import function.protobuf
+from .. import pythonic
+
+builtins.BaseComposite = pythonic.BaseComposite
+builtins.Map = pythonic.Map
+builtins.List = pythonic.List
+builtins.Unknown = pythonic.Unknown
+builtins.Yaml = pythonic.Yaml
+builtins.Json = pythonic.Json
+builtins.B64Encode = pythonic.B64Encode
+builtins.B64Decode = pythonic.B64Decode
 
 
 class FunctionRunner(grpcv1.FunctionRunnerService):
@@ -19,7 +29,7 @@ class FunctionRunner(grpcv1.FunctionRunnerService):
     def __init__(self):
         """Create a new FunctionRunner."""
         self.logger = crossplane.function.logging.get_logger()
-        self.modules = {}
+        self.clazzes = {}
 
     async def RunFunction(
         self, request: fnv1.RunFunctionRequest, _: grpc.aio.ServicerContext
@@ -52,23 +62,47 @@ async def RunFunction(
                 return response
             composite = input['composite']
 
-        module = self.modules.get(composite)
-        if not module:
-            module = Module()
-            try:
-                exec(composite, module.__dict__)
-            except Exception as e:
-                crossplane.function.response.fatal(response, f"Exec exception: {e}")
-                logger.exception('Exec exception')
+        clazz = self.clazzes.get(composite)
+        if not clazz:
+            if '\n' in composite:
+                module = Module()
+                try:
+                    exec(composite, module.__dict__)
+                except Exception as e:
+                    crossplane.function.response.fatal(response, f"Exec exception: {e}")
+                    logger.exception('Exec exception')
+                    return response
+                composite = ['<script>', 'Composite']
+            else:
+                composite = composite.rsplit('.', 1)
+                if len(composite) == 1:
+                    crossplane.function.response.fatal(response, f"Composite class name does not include module: {composite[0]}")
+                    logger.error(f"Composite class name does not include module: {composite[0]}")
+                    return response
+                try:
+                    module = importlib.import_module(composite[0])
+                except Exception as e:
+                    crossplane.function.response.fatal(response, f"Import module exception: {e}")
+                    logger.exception('Import module exception')
+                    return response
+            clazz = getattr(module, composite[1], None)
+            if not clazz:
+                crossplane.function.response.fatal(response, f"{composite[0]} did not define: {composite[1]}")
+                logger.error(f"{composite[0]} did not define: {composite[1]}")
+                return response
+            composite = '.'.join(composite)
+            if not inspect.isclass(clazz):
+                crossplane.function.response.fatal(response, f"{composite} is not a class")
+                logger.error(f"{composite} is not a class")
                 return response
-            if not hasattr(module, 'Composite') or not inspect.isclass(module.Composite):
-                crossplane.function.response.fatal(response, 'Function did not define "class Composite')
-                logger.error('Composite did not define "class Composite"')
+            if not issubclass(clazz, BaseComposite):
+                crossplane.function.response.fatal(response, f"{composite} is not a subclass of BaseComposite")
+                logger.error(f"{composite} is not a subclass of BaseComposite")
                 return response
-            self.modules[composite] = module
+            self.clazzes[composite] = clazz
 
         try:
-            composite = module.Composite(request, response, logger)
+            composite = clazz(request, response, logger)
         except Exception as e:
             crossplane.function.response.fatal(response, f"Instatiate exception: {e}")
             logger.exception('Instatiate exception')
@@ -101,12 +135,4 @@ async def RunFunction(
 
 
 class Module:
-    def __init__(self):
-        self.BaseComposite = function.composite.BaseComposite
-        self.Map = function.protobuf.Map
-        self.List = function.protobuf.List
-        self.Unknown = function.protobuf.Unknown
-        self.Yaml = function.protobuf.Yaml
-        self.Json = function.protobuf.Json
-        self.B64Encode = lambda s: base64.b64encode(s.encode('utf-8')).decode('utf-8')
-        self.B64Decode = lambda s: base64.b64decode(s.encode('utf-8')).decode('utf-8')
+    pass