Major refactor to support Usages

Author: Patrick J. McNerthney

.github/workflows/ci.yaml

@@ -217,8 +217,8 @@ jobs:
   # pushes them as a multi-platform package. We only push the package it the
   # XPKG_ACCESS_ID and XPKG_TOKEN secrets were provided.
   push-xpkg:
-    # Don't publish unless we were run with an explicit version.
-    if: ${{ inputs.version != '' }}
+    # Don't publish unless the main branch or we were run with an explicit version
+    if: ${{ github.ref == 'refs/heads/main' || inputs.version != '' }}
     needs:
       - build-xpkg
     runs-on: ubuntu-24.04

README.md

@@ -95,6 +95,26 @@ overridden for all composed resource by setting the Composite `self.unknownsFata
 to False, or at the individual composed resource level by setting the
 `Resource.unknownsFatal` field to False.
 
+## Usage Dependencies
+
+function-pythonic can be configured to automatically create
+[Crossplane Usages](https://docs.crossplane.io/latest/managed-resources/usages/)
+dependencies between resources. Modifying the above VPC example with:
+```yaml
+self.usages = True
+
+vpc = self.resources.VPC('ec2.aws.crossplane.io/v1beta1', 'VPC')
+vpc.spec.forProvider.region = 'us-east-1
+vpc.spec.forProvider.cidrBlock = '10.0.0.0/16'
+
+subnet = self.resources.SubnetA('ec2.aws.crossplane.io/v1beta1', 'Subnet')
+subnet.spec.forProvider.region = 'us-east-1'
+subnet.spec.forProvider.vpcId = vpc.status.atProvider.vpcId
+subnet.spec.forProvider.availabilityZone = 'us-east-1a'
+subnet.spec.forProvider.cidrBlock = '10.0.0.0/20'
+```
+Will generate the appropriate Crossplane Usage resource.
+
 ## Pythonic access of Protobuf Messages
 
 All Protobuf messages are wrapped by a set of python classes which enable using
@@ -201,6 +221,7 @@ The BaseComposite also provides access to the following Crossplane Function leve
 | self.requireds | Requireds | Request and read additional local Kubernetes resources |
 | self.resources | Resources | Define and process composed resources |
 | self.unknownsFatal | Boolean | Terminate the composition if already created resources are assigned unknown values, default True |
+| self.usages| Boolean | Generate Crossplane Usages for resource dependencies, default False |
 | self.autoReady | Boolean | Perform auto ready processing on all composed resources, default True |
 
 ### Composed Resources
@@ -227,6 +248,7 @@ Resource class:
 | Resource.connection | Connection | The resource connection details |
 | Resource.ready | Boolean | The resource ready state |
 | Resource.unknownsFatal | Boolean | Terminate the composition if this resource has been created and is assigned unknown values, default is Composite.unknownsFatal |
+| Resource.usages | Boolean | Generate Crossplane Usages for this resource, default is Composite.autoReady |
 | Resource.autoReady | Boolean | Perform auto ready processing on this resource, default is Composite.autoReady |
 
 ### Required Resources (AKA Extra Resources)

crossplane/pythonic/__init__.py

@@ -1,10 +1,7 @@
 
-import base64
 
 from .composite import BaseComposite
-from .protobuf import append, Map, List, Unknown, Yaml, Json
-B64Encode = lambda s: base64.b64encode(s.encode('utf-8')).decode('utf-8')
-B64Decode = lambda s: base64.b64decode(s.encode('utf-8')).decode('utf-8')
+from .protobuf import append, Map, List, Unknown, Yaml, Json, B64Encode, B64Decode
 
 __all__ = [
     'BaseComposite',

crossplane/pythonic/composite.py

@@ -31,6 +31,7 @@ def __init__(self, request, logger):
         self.resources = Resources(self)
         self.unknownsFatal = True
         self.autoReady = True
+        self.usages = False
 
         observed = self.request.observed.composite
         desired = self.response.desired.composite
@@ -49,7 +50,7 @@ def __init__(self, request, logger):
     def ttl(self):
         if self.response.meta.ttl.nanos:
             return float(self.response.meta.ttl.seconds) + (float(self.response.meta.ttl.nanos) / 1000000000.0)
-        return self.response.meta.ttl.seconds
+        return int(self.response.meta.ttl.seconds)
 
     @ttl.setter
     def ttl(self, ttl):
@@ -61,7 +62,7 @@ def ttl(self, ttl):
             if ttl.is_integer():
                 self.response.meta.ttl.nanos = 0
             else:
-                self.response.meta.ttl.nanos = int((ttl -  self.response.meta.ttl.seconds) * 1000000000)
+                self.response.meta.ttl.nanos = int((ttl - int(self.response.meta.ttl.seconds)) * 1000000000)
         else:
             raise ValueError('ttl must be an int or float')
 
@@ -78,7 +79,7 @@ def ready(self):
     def ready(self, ready):
         if ready:
             ready = fnv1.Ready.READY_TRUE
-        elif ready == None or (isinstance(ready, protobuf.Values) and ready._isUnknown):
+        elif ready == None or (isinstance(ready, protobuf.Value) and ready._isUnknown):
             ready = fnv1.Ready.READY_UNSPECIFIED
         else:
             ready = fnv1.Ready.READY_FALSE
@@ -184,6 +185,7 @@ def __init__(self, composite, name):
         self.connection = Connection(observed)
         self.unknownsFatal = None
         self.autoReady = None
+        self.usages = None
 
     def __call__(self, apiVersion=_notset, kind=_notset, namespace=_notset, name=_notset):
         self.desired()
@@ -199,15 +201,15 @@ def __call__(self, apiVersion=_notset, kind=_notset, namespace=_notset, name=_no
 
     @property
     def apiVersion(self):
-        return self.observed.apiVersion
+        return self.desired.apiVersion
 
     @apiVersion.setter
     def apiVersion(self, apiVersion):
         self.desired.apiVersion = apiVersion
 
     @property
     def kind(self):
-        return self.observed.kind
+        return self.desired.kind
 
     @kind.setter
     def kind(self, kind):
@@ -265,7 +267,7 @@ def ready(self):
     def ready(self, ready):
         if ready:
             ready = fnv1.Ready.READY_TRUE
-        elif ready == None or (isinstance(ready, protobuf.Values) and ready._isUnknown):
+        elif ready == None or (isinstance(ready, protobuf.Value) and ready._isUnknown):
             ready = fnv1.Ready.READY_UNSPECIFIED
         else:
             ready = fnv1.Ready.READY_FALSE
@@ -376,8 +378,8 @@ def matchLabels(self, labels):
                 elif isinstance(entry, (list, tuple)):
                     self._selector.match_labels.labels[entry[0]] = entry[1]
 
-    def __getitem__(self, key):
-        return RequiredResource(self.name, self._resources.items[key])
+    def __getitem__(self, ix):
+        return RequiredResource(self.name, ix, self._resources.items[ix])
 
     def __bool__(self):
         return bool(self._resources.items)
@@ -391,8 +393,9 @@ def __iter__(self):
 
 
 class RequiredResource:
-    def __init__(self, name, resource):
+    def __init__(self, name, ix, resource):
         self.name = name
+        self.ix = ix
         self.observed = resource.resource
         self.apiVersion = self.observed.apiVersion
         self.kind = self.observed.kind
@@ -487,7 +490,7 @@ def status(self, status):
             condition.status = fnv1.Status.STATUS_CONDITION_TRUE
         elif status == None:
             condition.status = fnv1.Status.STATUS_CONDITION_UNKNOWN
-        elif isinstance(status, protobuf.Values) and status._isUnknown:
+        elif isinstance(status, protobuf.Value) and status._isUnknown:
             condition.status = fnv1.Status.STATUS_CONDITION_UNSPECIFIED
         else:
             condition.status = fnv1.Status.STATUS_CONDITION_FALSE
@@ -521,7 +524,7 @@ def lastTransitionTime(self):
             if observed.type == self.type:
                 time = observed.lastTransitionTime
                 if time:
-                    return datetime.datetime.fromisoformat(time)
+                    return datetime.datetime.fromisoformat(str(time))
         return None
 
     @property
@@ -534,7 +537,7 @@ def claim(self, claim):
         condition = self._find_condition(True)
         if claim:
             condition.target = fnv1.Target.TARGET_COMPOSITE_AND_CLAIM
-        elif claim == None or (isinstance(claim, protobuf.Values) and claim._isUnknown):
+        elif claim == None or (isinstance(claim, protobuf.Value) and claim._isUnknown):
             condition.target = fnv1.Target.TARGET_UNSPECIFIED
         else:
             condition.target = fnv1.Target.TARGET_COMPOSITE
@@ -711,7 +714,7 @@ def claim(self, claim):
         if bool(self):
             if claim:
                 self._result.target = fnv1.Target.TARGET_COMPOSITE_AND_CLAIM
-            elif claim == None or (isinstance(claim, protobuf.Values) and claim._isUnknown):
+            elif claim == None or (isinstance(claim, protobuf.Value) and claim._isUnknown):
                 self._result.target = fnv1.Target.TARGET_UNSPECIFIED
             else:
                 self._result.target = fnv1.Target.TARGET_COMPOSITE

crossplane/pythonic/function.py

@@ -99,7 +99,7 @@ async def run_function(self, request):
             return self.fatal(request, logger, 'Instantiate', e)
 
         step = composite.context._pythonic[step]
-        iteration = (step.iteration or 0) + 1
+        iteration = int(step.iteration) + 1
         step.iteration = iteration
         composite.context.iteration = iteration
         logger.debug(f"Starting compose, {ordinal(len(composite.context._pythonic))} step, {ordinal(iteration)} pass")
@@ -111,7 +111,40 @@ async def run_function(self, request):
         except Exception as e:
             return self.fatal(request, logger, 'Compose', e)
 
-        requested = []
+        if requireds := self.get_requireds(step, composite):
+            logger.info(f"Requireds requested: {','.join(requireds)}")
+        else:
+            self.process_usages(composite)
+            self.process_unknowns(composite)
+            self.process_auto_readies(composite)
+            logger.info('Completed compose')
+
+        return composite.response._message
+
+    def fatal(self, request, logger, message, exception=None):
+        if exception:
+            message += ' exceptiion'
+            logger.exception(message)
+            m = str(exception)
+            if not m:
+                m = exception.__class__.__name__
+            message += ': ' + m
+        else:
+            logger.error(message)
+        return fnv1.RunFunctionResponse(
+            meta=fnv1.ResponseMeta(
+                tag=request.meta.tag,
+            ),
+            results=[
+                fnv1.Result(
+                    severity=fnv1.SEVERITY_FATAL,
+                    message=message,
+                )
+            ]
+        )
+
+    def get_requireds(self, step, composite):
+        requireds = []
         for name, required in composite.requireds:
             if required.apiVersion and required.kind:
                 r = pythonic.Map(apiVersion=required.apiVersion, kind=required.kind)
@@ -123,11 +156,82 @@ async def run_function(self, request):
                     r.matchLabels[key] = value
                 if r != step.requireds[name]:
                     step.requireds[name] = r
-                    requested.append(name)
-        if requested:
-            logger.info(f"Requireds requested: {','.join(requested)}")
-            return composite.response._message
+                    requireds.append(name)
+        return requireds
+
+    def process_usages(self, composite):
+        for _, resource in sorted(entry for entry in composite.resources):
+            dependencies = resource.desired._getDependencies
+            if dependencies:
+                if self.debug:
+                    for destination, source in sorted(dependencies.items()):
+                        destination = self.trimFullName(destination)
+                        source = self.trimFullName(source)
+                        logger.debug(f"Dependency: {destination} = {source}")
+                if resource.usages or (resource.usages is None and composite.usages):
+                    resources = {}
+                    requireds = {}
+                    for destination, source in sorted(dependencies.items()):
+                        name = source.split('.')
+                        if (len(name) > 5 and
+                            name[0] == 'request' and
+                            name[1] == 'observed' and
+                            name[2] == 'resources' and
+                            name[4] == 'resource'
+                        ):
+                            if name[3] not in resources:
+                                resources[name[3]] = []
+                            resources[name[3]].append(f"{'.'.join(destination.split('.')[5:])} = {'.'.join(name[5:])}")
+                        elif (len(name) > 5 and
+                            name[0] == 'request' and
+                            name[1] == 'extra_resources' and
+                            name[3].startswith('items[') and name[3][-1] == ']' and
+                            name[4] == 'resource'
+                        ):
+                            key = (name[2], int(name[3][6:-1]))
+                            if key not in requireds:
+                                requireds[key] = []
+                            requireds[key].append(f"{'.'.join(destination.split('.')[5:])} = [{key[1]}]{'.'.join(name[5:])}")
+                    for name, dependencies in resources.items():
+                        source = composite.resources[name]
+                        name = [resource.name, str(source.kind)]
+                        if source.metadata.namespace:
+                            name.append(str(source.metadata.namespace))
+                        name.append(str(source.observed.metadata.name))
+                        usage = composite.resources['_'.join(name)]('protection.crossplane.io/v1beta1', 'Usage')
+                        if resource.metadata.namespace:
+                            usage.metadata.namespace = resource.metadata.namespace
+                        usage.spec.description = '\n'.join(dependencies)
+                        usage.spec.replayDeletion = True
+                        usage.spec.of.apiVersion = resource.apiVersion
+                        usage.spec.of.kind = resource.kind
+                        usage.spec.of.resourceRef.name = resource.observed.metadata.name
+                        usage.spec.by.apiVersion = source.apiVersion
+                        usage.spec.by.kind = source.kind
+                        if source.metadata.namespace:
+                            usage.spec.by.resourceRef.namespace = source.metadata.namespace
+                        usage.spec.by.resourceRef.name = source.observed.metadata.name
+                    for key, dependencies in requireds.items():
+                        source = composite.requireds[key[0]][key[1]]
+                        name = [resource.name, str(source.kind)]
+                        if source.metadata.namespace:
+                            name.append(str(source.metadata.namespace))
+                        name.append(str(source.metadata.name))
+                        usage = composite.resources['_'.join(name)]('protection.crossplane.io/v1beta1', 'Usage')
+                        if resource.metadata.namespace:
+                            usage.metadata.namespace = resource.metadata.namespace
+                        usage.spec.description = '\n'.join(dependencies)
+                        usage.spec.replayDeletion = True
+                        usage.spec.of.apiVersion = resource.apiVersion
+                        usage.spec.of.kind = resource.kind
+                        usage.spec.of.resourceRef.name = resource.observed.metadata.name
+                        usage.spec.by.apiVersion = source.apiVersion
+                        usage.spec.by.kind = source.kind
+                        if source.metadata.namespace:
+                            usage.spec.by.resourceRef.namespace = source.metadata.namespace
+                        usage.spec.by.resourceRef.name = source.observed.metadata.name
 
+    def process_unknowns(self, composite):
         unknownResources = []
         warningResources = []
         fatalResources = []
@@ -190,42 +294,19 @@ async def run_function(self, request):
         if event:
             event(reason, message)
 
+    def process_auto_readies(self, composite):
         for name, resource in composite.resources:
             if resource.autoReady or (resource.autoReady is None and composite.autoReady):
                 if resource.ready is None:
                     if resource.conditions.Ready.status:
                         resource.ready = True
 
-        logger.info('Completed compose')
-        return composite.response._message
-
-    def fatal(self, request, logger, message, exception=None):
-        if exception:
-            message += ' exceptiion'
-            logger.exception(message)
-            m = str(exception)
-            if not m:
-                m = exception.__class__.__name__
-            message += ': ' + m
-        else:
-            logger.error(message)
-        return fnv1.RunFunctionResponse(
-            meta=fnv1.ResponseMeta(
-                tag=request.meta.tag,
-            ),
-            results=[
-                fnv1.Result(
-                    severity=fnv1.SEVERITY_FATAL,
-                    message=message,
-                )
-            ]
-        )
-
     def trimFullName(self, name):
         name = name.split('.')
         for values in (
+                ('request', 'observed', 'composite', 'resource'),
                 ('request', 'observed', 'resources', None, 'resource'),
-                ('request', 'extra_resources', None, 'items', 'resource'),
+                ('request', 'extra_resources', None, 'items', None, 'resource'),
                 ('response', 'desired', 'resources', None, 'resource'),
         ):
             if len(values) < len(name):