vmware · Agent-Lumi · Jun 17, 2026 · Jun 17, 2026
diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml
@@ -1,4 +1,4 @@
-# Copyright 2023-2025 Broadcom
+# Copyright 2023-2026 Broadcom
 # SPDX-License-Identifier: Apache-2.0
 
 blank_issues_enabled: false

diff --git a/.github/dependabot.yml b/.github/dependabot.yml
@@ -1,4 +1,4 @@
-# Copyright 2023-2025 Broadcom
+# Copyright 2023-2026 Broadcom
 # SPDX-License-Identifier: Apache-2.0
 
 # To get started with Dependabot version updates, you'll need to specify which

diff --git a/.github/workflows/autoupdate.yml b/.github/workflows/autoupdate.yml
@@ -1,4 +1,4 @@
-# Copyright 2023-2025 Broadcom
+# Copyright 2023-2026 Broadcom
 # SPDX-License-Identifier: Apache-2.0
 
 name: autoupdate

diff --git a/.github/workflows/codacy.yml b/.github/workflows/codacy.yml
@@ -1,4 +1,4 @@
-# Copyright 2023-2025 Broadcom
+# Copyright 2023-2026 Broadcom
 # SPDX-License-Identifier: Apache-2.0
 
 # This workflow uses actions that are not certified by GitHub.

diff --git a/.github/workflows/dependabot.autoapprove.yml b/.github/workflows/dependabot.autoapprove.yml
@@ -1,4 +1,4 @@
-# Copyright 2023-2025 Broadcom
+# Copyright 2023-2026 Broadcom
 # SPDX-License-Identifier: Apache-2.0
 
 name: Auto approve

diff --git a/.github/workflows/dependabot.automerge.yml b/.github/workflows/dependabot.automerge.yml
@@ -1,4 +1,4 @@
-# Copyright 2023-2025 Broadcom
+# Copyright 2023-2026 Broadcom
 # SPDX-License-Identifier: Apache-2.0
 
 name: automerge

diff --git a/.github/workflows/format.yml b/.github/workflows/format.yml
@@ -1,4 +1,4 @@
-# Copyright 2023-2025 Broadcom
+# Copyright 2023-2026 Broadcom
 # SPDX-License-Identifier: Apache-2.0
 
 name: Google Java Format

diff --git a/.github/workflows/pr-title-checker.yml b/.github/workflows/pr-title-checker.yml
@@ -1,4 +1,4 @@
-# Copyright 2023-2025 Broadcom
+# Copyright 2023-2026 Broadcom
 # SPDX-License-Identifier: Apache-2.0
 
 name: "PR Title Checker"

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
@@ -1,4 +1,4 @@
-# Copyright 2023-2025 Broadcom
+# Copyright 2023-2026 Broadcom
 # SPDX-License-Identifier: Apache-2.0
 
 # default image used by all pipelines if not overridden per job

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -1,4 +1,4 @@
-# Copyright 2023-2025 Broadcom
+# Copyright 2023-2026 Broadcom
 # SPDX-License-Identifier: Apache-2.0
 
 repos:

diff --git a/cicd/.gitlab-ci-lib.yml b/cicd/.gitlab-ci-lib.yml
@@ -1,4 +1,4 @@
-# Copyright 2023-2025 Broadcom
+# Copyright 2023-2026 Broadcom
 # SPDX-License-Identifier: Apache-2.0
 
 # This is a shared library with common util gitlab "jobs" that can be reused across any project

diff --git a/cicd/build.sh b/cicd/build.sh
@@ -1,5 +1,5 @@
 #!/bin/bash -e
-# Copyright 2023-2025 Broadcom
+# Copyright 2023-2026 Broadcom
 # SPDX-License-Identifier: Apache-2.0
 
 SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"

diff --git a/cicd/docker_parse.sh b/cicd/docker_parse.sh
@@ -1,6 +1,6 @@
 #!/bin/bash
 
-# Copyright 2023-2025 Broadcom
+# Copyright 2023-2026 Broadcom
 # SPDX-License-Identifier: Apache-2.0
 
 # Parses a Docker image name and extracts information such as the registry, username, repository, and tag.

diff --git a/cicd/docker_push_vdk.sh b/cicd/docker_push_vdk.sh
@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 
-# Copyright 2023-2025 Broadcom
+# Copyright 2023-2026 Broadcom
 # SPDX-License-Identifier: Apache-2.0
 
 # This Bash script provides a utility to push a containers image to multiple container registries.

diff --git a/cicd/notify.sh b/cicd/notify.sh
@@ -1,5 +1,5 @@
 #!/bin/bash -e
-# Copyright 2023-2025 Broadcom
+# Copyright 2023-2026 Broadcom
 # SPDX-License-Identifier: Apache-2.0
 
 curl -X POST -H "Content-type: application/json" --data "{\"text\":\"$1\"}" $2
diff --git a/examples/airflow-example/airflow-transform-job/10_transform.py b/examples/airflow-example/airflow-transform-job/10_transform.py
@@ -1,4 +1,4 @@
-# Copyright 2023-2025 Broadcom
+# Copyright 2023-2026 Broadcom
 # SPDX-License-Identifier: Apache-2.0
 from vdk.api.job_input import IJobInput
 

diff --git a/examples/airflow-example/airflow-trino-job1/10_insert_data.py b/examples/airflow-example/airflow-trino-job1/10_insert_data.py
@@ -1,4 +1,4 @@
-# Copyright 2023-2025 Broadcom
+# Copyright 2023-2026 Broadcom
 # SPDX-License-Identifier: Apache-2.0
 import json
 import pathlib

diff --git a/examples/airflow-example/airflow-trino-job2/10_insert_data.py b/examples/airflow-example/airflow-trino-job2/10_insert_data.py
@@ -1,4 +1,4 @@
-# Copyright 2023-2025 Broadcom
+# Copyright 2023-2026 Broadcom
 # SPDX-License-Identifier: Apache-2.0
 import json
 import pathlib

diff --git a/examples/airflow-example/dags/airflow_example_dag.py b/examples/airflow-example/dags/airflow_example_dag.py
@@ -1,4 +1,4 @@
-# Copyright 2023-2025 Broadcom
+# Copyright 2023-2026 Broadcom
 # SPDX-License-Identifier: Apache-2.0
 from datetime import datetime
 

diff --git a/examples/chunker/00_properties.py b/examples/chunker/00_properties.py
@@ -1,4 +1,4 @@
-# Copyright 2023-2025 Broadcom
+# Copyright 2023-2026 Broadcom
 # SPDX-License-Identifier: Apache-2.0
 # Copyright 2021-2024 VMware, Inc.
 # SPDX-License-Identifier: Apache-2.0

diff --git a/examples/chunker/10_chunk_data.py b/examples/chunker/10_chunk_data.py
@@ -1,4 +1,4 @@
-# Copyright 2023-2025 Broadcom
+# Copyright 2023-2026 Broadcom
 # SPDX-License-Identifier: Apache-2.0
 # Copyright 2021-2024 VMware, Inc.
 # SPDX-License-Identifier: Apache-2.0

diff --git a/examples/chunker/config.py b/examples/chunker/config.py
@@ -1,4 +1,4 @@
-# Copyright 2023-2025 Broadcom
+# Copyright 2023-2026 Broadcom
 # SPDX-License-Identifier: Apache-2.0
 # Copyright 2021-2024 VMware, Inc.
 # SPDX-License-Identifier: Apache-2.0

diff --git a/examples/confluence-data-retrieval-example/10_fetch_confluence_document.py b/examples/confluence-data-retrieval-example/10_fetch_confluence_document.py
@@ -1,4 +1,4 @@
-# Copyright 2023-2025 Broadcom
+# Copyright 2023-2026 Broadcom
 # SPDX-License-Identifier: Apache-2.0
 import logging
 import os

diff --git a/examples/confluence-data-retrieval-example/20_fetch_confluence_space.py b/examples/confluence-data-retrieval-example/20_fetch_confluence_space.py
@@ -1,4 +1,4 @@
-# Copyright 2023-2025 Broadcom
+# Copyright 2023-2026 Broadcom
 # SPDX-License-Identifier: Apache-2.0
 import logging
 import os

diff --git a/examples/confluence-reader/confluence_document.py b/examples/confluence-reader/confluence_document.py
@@ -1,4 +1,4 @@
-# Copyright 2023-2025 Broadcom
+# Copyright 2023-2026 Broadcom
 # SPDX-License-Identifier: Apache-2.0
 
 

diff --git a/examples/confluence-reader/fetch_confluence_space.py b/examples/confluence-reader/fetch_confluence_space.py
@@ -1,4 +1,4 @@
-# Copyright 2023-2025 Broadcom
+# Copyright 2023-2026 Broadcom
 # SPDX-License-Identifier: Apache-2.0
 import logging
 import os

diff --git a/examples/dag-example/example-dag/example_dag.py b/examples/dag-example/example-dag/example_dag.py
@@ -1,4 +1,4 @@
-# Copyright 2023-2025 Broadcom
+# Copyright 2023-2026 Broadcom
 # SPDX-License-Identifier: Apache-2.0
 from vdk.plugin.dag.dag_runner import DagInput
 

diff --git a/examples/dag-example/ingest-job1/10_insert_data.py b/examples/dag-example/ingest-job1/10_insert_data.py
@@ -1,4 +1,4 @@
-# Copyright 2023-2025 Broadcom
+# Copyright 2023-2026 Broadcom
 # SPDX-License-Identifier: Apache-2.0
 import json
 import pathlib

diff --git a/examples/dag-example/ingest-job2/10_insert_data.py b/examples/dag-example/ingest-job2/10_insert_data.py
@@ -1,4 +1,4 @@
-# Copyright 2023-2025 Broadcom
+# Copyright 2023-2026 Broadcom
 # SPDX-License-Identifier: Apache-2.0
 import json
 import pathlib

diff --git a/examples/dag-example/read-data-job/10_read.py b/examples/dag-example/read-data-job/10_read.py
@@ -1,4 +1,4 @@
-# Copyright 2023-2025 Broadcom
+# Copyright 2023-2026 Broadcom
 # SPDX-License-Identifier: Apache-2.0
 from vdk.api.job_input import IJobInput
 

diff --git a/examples/dag-with-args-example/dag-job/dag_job.py b/examples/dag-with-args-example/dag-job/dag_job.py
@@ -1,4 +1,4 @@
-# Copyright 2023-2025 Broadcom
+# Copyright 2023-2026 Broadcom
 # SPDX-License-Identifier: Apache-2.0
 from vdk.plugin.dag.dag_runner import DagInput
 

diff --git a/examples/dag-with-args-example/ingest-job-table-one/10_insert_data.py b/examples/dag-with-args-example/ingest-job-table-one/10_insert_data.py
@@ -1,4 +1,4 @@
-# Copyright 2023-2025 Broadcom
+# Copyright 2023-2026 Broadcom
 # SPDX-License-Identifier: Apache-2.0
 import json
 import pathlib

diff --git a/examples/dag-with-args-example/ingest-job-table-two/10_insert_data.py b/examples/dag-with-args-example/ingest-job-table-two/10_insert_data.py
@@ -1,4 +1,4 @@
-# Copyright 2023-2025 Broadcom
+# Copyright 2023-2026 Broadcom
 # SPDX-License-Identifier: Apache-2.0
 import json
 import pathlib

diff --git a/examples/dag-with-args-example/read-job-canada/10_read.py b/examples/dag-with-args-example/read-job-canada/10_read.py
@@ -1,4 +1,4 @@
-# Copyright 2023-2025 Broadcom
+# Copyright 2023-2026 Broadcom
 # SPDX-License-Identifier: Apache-2.0
 from vdk.api.job_input import IJobInput
 

diff --git a/examples/dag-with-args-example/read-job-rest-of-world/10_read.py b/examples/dag-with-args-example/read-job-rest-of-world/10_read.py
@@ -1,4 +1,4 @@
-# Copyright 2023-2025 Broadcom
+# Copyright 2023-2026 Broadcom
 # SPDX-License-Identifier: Apache-2.0
 from vdk.api.job_input import IJobInput
 

diff --git a/examples/dag-with-args-example/read-job-usa/10_read.py b/examples/dag-with-args-example/read-job-usa/10_read.py
@@ -1,4 +1,4 @@
-# Copyright 2023-2025 Broadcom
+# Copyright 2023-2026 Broadcom
 # SPDX-License-Identifier: Apache-2.0
 from vdk.api.job_input import IJobInput
 

diff --git a/examples/energy/dashboard.py b/examples/energy/dashboard.py
@@ -1,4 +1,4 @@
-# Copyright 2023-2025 Broadcom
+# Copyright 2023-2026 Broadcom
 # SPDX-License-Identifier: Apache-2.0
 import math
 

diff --git a/examples/energy/energy/03_ingest_table_natural_gas_prices.py b/examples/energy/energy/03_ingest_table_natural_gas_prices.py
@@ -1,4 +1,4 @@
-# Copyright 2023-2025 Broadcom
+# Copyright 2023-2026 Broadcom
 # SPDX-License-Identifier: Apache-2.0
 import inspect
 import logging

diff --git a/examples/energy/energy/06_ingest_table_create_extremes_index.py b/examples/energy/energy/06_ingest_table_create_extremes_index.py
@@ -1,4 +1,4 @@
-# Copyright 2023-2025 Broadcom
+# Copyright 2023-2026 Broadcom
 # SPDX-License-Identifier: Apache-2.0
 import inspect
 import logging

diff --git a/examples/gdp-execution-id-example/20_python_step.py b/examples/gdp-execution-id-example/20_python_step.py
@@ -1,4 +1,4 @@
-# Copyright 2023-2025 Broadcom
+# Copyright 2023-2026 Broadcom
 # SPDX-License-Identifier: Apache-2.0
 import logging
 

diff --git a/examples/incremental-ingest-from-db-example-notebook/README.md b/examples/incremental-ingest-from-db-example-notebook/README.md
@@ -0,0 +1,74 @@
+# Incremental Ingestion using Job Properties - Colab Notebook
+
+This folder contains a Google Colab notebook that demonstrates incremental ingestion from a local SQLite database using Versatile Data Kit (VDK).
+
+## Overview
+
+The notebook provides an interactive tutorial on how to:
+- Use VDK's job properties to track state between job runs
+- Perform incremental ingestion (only ingesting new/changed records)
+- Configure VDK for SQLite database connections
+- Use VDK's IPython extension in a notebook environment
+
+## Contents
+
+- `incremental-ingest-example-notebook.ipynb` - The Google Colab notebook with step-by-step instructions
+
+## Usage
+
+### Option 1: Run on Google Colab (Recommended)
+
+Click the "Open in Colab" badge at the top of the notebook to run it directly in Google Colab without any local setup.
+
+[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/vmware/versatile-data-kit/blob/main/examples/incremental-ingest-from-db-example-notebook/incremental-ingest-example-notebook.ipynb)
+
+### Option 2: Run Locally
+
+1. Install dependencies:
+   ```bash
+   pip install vdk-ipython vdk-sqlite vdk-ingest-http
+   ```
+
+2. Open the notebook in Jupyter:
+   ```bash
+   jupyter notebook incremental-ingest-example-notebook.ipynb
+   ```
+
+## What You'll Learn
+
+1. How to set up a source SQLite database with sample data
+2. How to configure VDK environment variables for database connections
+3. How to use the `%%vdksql` magic command for SQL operations
+4. How to implement incremental ingestion using job properties
+5. How to use `job_input.get_property()` and `job_input.set_all_properties()`
+6. How to ingest tabular data using `send_tabular_data_for_ingestion()`
+
+## Key Concepts
+
+### Incremental Ingestion
+
+Incremental ingestion is a pattern where only new or changed records are processed, rather than reloading the entire dataset each time. This is achieved by:
+
+- Storing a "watermark" (like the last processed date) in job properties
+- Querying only records newer than the watermark
+- Updating the watermark after successful ingestion
+
+### Job Properties
+
+VDK provides a key-value store for persisting state between job runs:
+
+- `get_property(key, default_value)` - Retrieve a property value
+- `set_all_properties(dict)` - Store multiple property values
+
+## Related Resources
+
+- [Original Example (File-based)](https://github.com/vmware/versatile-data-kit/tree/main/examples/incremental-ingest-from-db-example)
+- [VDK Wiki - Job Properties](https://github.com/vmware/versatile-data-kit/wiki/Job-Properties)
+- [VDK Wiki - Getting Started](https://github.com/vmware/versatile-data-kit/wiki/Getting-Started)
+- [Issue #3060](https://github.com/vmware/versatile-data-kit/issues/3060) - This notebook addresses this issue
+
+## Contributing
+
+This notebook was created as part of the [Google Colab Notebooks for VDK Examples](https://github.com/vmware/versatile-data-kit/milestone/29) milestone.
+
+See the main [CONTRIBUTING.md](../../CONTRIBUTING.md) for contribution guidelines.