diff --git a/.github/workflows/testEndToEnd.yml b/.github/workflows/testEndToEnd.yml index 4d6dd1b0..d1c1a8d7 100644 --- a/.github/workflows/testEndToEnd.yml +++ b/.github/workflows/testEndToEnd.yml @@ -3,18 +3,17 @@ on: pull_request: types: [opened, reopened, synchronize] schedule: - - cron: "44 4 * * *" + - cron: '44 4 * * *' jobs: testEverything: strategy: fail-fast: false matrix: - python_version: ["3.11", "3.12", "3.13"] - operating_system: ["ubuntu-24.04", "macos-15", "windows-2022"] + python_version: ['3.12', '3.13', '3.14', '3.14t'] + operating_system: ['ubuntu-24.04', 'macos-15', 'windows-2022'] #operating_system: ["ubuntu-20.04", "ubuntu-22.04", "macos-latest"] - runs-on: ${{ matrix.operating_system }} steps: #- name: Install Docker for macOS @@ -36,11 +35,10 @@ jobs: uses: actions/setup-python@v5 with: python-version: ${{ matrix.python_version }} - architecture: "x64" - + architecture: 'x64' + - name: Install Poetry - run: - python -m pip install poetry + run: python -m pip install poetry - name: Install contentctl and activate the shell run: | @@ -50,7 +48,7 @@ jobs: - name: Run contentctl init run: | cd my_splunk_content_pack - poetry run contentctl init + poetry run contentctl init - name: Clone the AtomicRedTeam Repo run: | @@ -73,11 +71,10 @@ jobs: run: | cd my_splunk_content_pack poetry run contentctl test --disable-tqdm --post-test-behavior never_pause - + - uses: actions/upload-artifact@v4 with: name: content_pack_${{ matrix.python_version }}_${{ matrix.operating_system }} path: | my_splunk_content_pack/dist/my_splunk_content_pack.tar.gz my_splunk_content_pack/test_results/summary.yml - diff --git a/.github/workflows/test_against_escu.yml b/.github/workflows/test_against_escu.yml index 25cae4c8..5648094e 100644 --- a/.github/workflows/test_against_escu.yml +++ b/.github/workflows/test_against_escu.yml @@ -9,20 +9,19 @@ on: pull_request: types: [opened, reopened, synchronize] schedule: - - cron: "44 4 * * *" + - cron: '44 4 * * *' jobs: smoketest_escu: strategy: fail-fast: false matrix: - python_version: ["3.11", "3.12", "3.13"] - - operating_system: ["ubuntu-24.04", "macos-15"] + python_version: ['3.12', '3.13', '3.14', '3.14t'] + + operating_system: ['ubuntu-24.04', 'macos-15'] # Do not test against ESCU until known character encoding issue is resolved # operating_system: ["ubuntu-20.04", "ubuntu-22.04", "macos-latest", "macos-14", "windows-2022"] - runs-on: ${{ matrix.operating_system }} steps: # Checkout the current branch of contentctl repo @@ -32,7 +31,7 @@ jobs: # Checkout the develop (default) branch of security_content - name: Checkout repo uses: actions/checkout@v5 - with: + with: path: security_content repository: splunk/security_content @@ -41,25 +40,22 @@ jobs: uses: actions/setup-python@v5 with: python-version: ${{ matrix.python_version }} - architecture: "x64" - + architecture: 'x64' + - name: Install Poetry - run: - python -m pip install poetry + run: python -m pip install poetry - name: Install contentctl and activate the shell run: | poetry install --no-interaction - - - name: Clone the AtomicRedTeam Repo and the Mitre/CTI repos for testing enrichments + - name: Clone the AtomicRedTeam Repo and the Mitre/CTI repos for testing enrichments run: | cd security_content git clone --single-branch https://github.com/redcanaryco/atomic-red-team external_repos/atomic-red-team git clone --single-branch https://github.com/mitre/cti external_repos/cti - - # We do not separately run validate and build + # We do not separately run validate and build # since a build ALSO performs a validate - name: Run contentctl build run: | @@ -68,4 +64,3 @@ jobs: # Do not run a test - it will take far too long! # Do not upload any artifacts - diff --git a/contentctl/input/director.py b/contentctl/input/director.py index c6e6b462..30860120 100644 --- a/contentctl/input/director.py +++ b/contentctl/input/director.py @@ -38,6 +38,34 @@ from contentctl.objects.story import Story from contentctl.output.runtime_csv_writer import RuntimeCsvWriter +# Rebuild models that have forward references to resolve circular imports. +# These models have types imported inside TYPE_CHECKING to avoid circular imports +# at module load time. After all modules are loaded, we rebuild the models to +# resolve these forward references. +# We must pass _types_namespace with the actual types so Pydantic can resolve the ForwardRefs. +# +# Order matters: Story must be rebuilt first since Detection depends on it via DetectionTags. +# A common types namespace is used to ensure all forward references can be resolved. + +_types_namespace = { + "Detection": Detection, + "Investigation": Investigation, + "Baseline": Baseline, + "DataSource": DataSource, + "Story": Story, + "Playbook": Playbook, +} + +# Story references Detection, Investigation, Baseline, and DataSource inside TYPE_CHECKING +Story.model_rebuild(_types_namespace=_types_namespace) + +# Detection_Abstract (parent of Detection) references Baseline inside TYPE_CHECKING +# DetectionTags references Story which has forward references +Detection.model_rebuild(_types_namespace=_types_namespace) + +# Playbook references Detection (via PlaybookTag) which needs Baseline resolved +Playbook.model_rebuild(_types_namespace=_types_namespace) + @dataclass class DirectorOutputDto: diff --git a/contentctl/objects/config.py b/contentctl/objects/config.py index 6d785cb0..96113c7a 100644 --- a/contentctl/objects/config.py +++ b/contentctl/objects/config.py @@ -77,14 +77,32 @@ def ensureAppPathExists(self, config: test, stage_file: bool = False): class TestApp(App_Base): model_config = ConfigDict(validate_default=True, arbitrary_types_allowed=True) - hardcoded_path: Optional[Union[FilePath, HttpUrl]] = Field( + # Note: We use Optional[str] here instead of Optional[Union[FilePath, HttpUrl]] + # because tyro treats Union types with distinct classes as requiring subcommand + # selection, which breaks CLI parsing. The model_validator below converts the + # string to the appropriate type (Path or HttpUrl) for internal use. + hardcoded_path: Optional[str] = Field( default=None, description="This may be a relative or absolute link to a file OR an HTTP URL linking to your app.", ) + # Internal storage for the parsed path (not exposed to CLI via PrivateAttr) + _parsed_hardcoded_path: Optional[Union[pathlib.Path, HttpUrl]] = None + + @model_validator(mode="after") + def parse_hardcoded_path(self) -> Self: + """Parse hardcoded_path string as either a file path or HTTP URL.""" + if self.hardcoded_path is None: + self._parsed_hardcoded_path = None + elif self.hardcoded_path.startswith(("http://", "https://")): + self._parsed_hardcoded_path = HttpUrl(self.hardcoded_path) + else: + self._parsed_hardcoded_path = pathlib.Path(self.hardcoded_path) + return self + @field_serializer("hardcoded_path", when_used="always") - def serialize_path(path: Union[AnyUrl, pathlib.Path]) -> str: - return str(path) + def serialize_path(self, path: Optional[str]) -> Optional[str]: + return path def getApp(self, config: test, stage_file: bool = False) -> str: # If the apps directory does not exist, then create it @@ -102,15 +120,17 @@ def getApp(self, config: test, stage_file: bool = False) -> str: f"version[{self.version}] MUST be defined" ) - elif isinstance(self.hardcoded_path, pathlib.Path): - destination = config.getLocalAppDir() / self.hardcoded_path.name + elif isinstance(self._parsed_hardcoded_path, pathlib.Path): + destination = config.getLocalAppDir() / self._parsed_hardcoded_path.name if stage_file: Utils.copy_local_file( - str(self.hardcoded_path), str(destination), verbose_print=True + str(self._parsed_hardcoded_path), + str(destination), + verbose_print=True, ) - elif isinstance(self.hardcoded_path, AnyUrl): - file_url_string = str(self.hardcoded_path) + elif isinstance(self._parsed_hardcoded_path, AnyUrl): + file_url_string = str(self._parsed_hardcoded_path) server_path = pathlib.Path(urlparse(file_url_string).path) destination = config.getLocalAppDir() / server_path.name if stage_file: @@ -795,216 +815,168 @@ def serialize_path(paths: List[FilePath]) -> List[str]: appid="Splunk_SA_CIM", title="Splunk Common Information Model (CIM)", version="5.2.0", - hardcoded_path=HttpUrl( - "https://attack-range-appbinaries.s3.us-west-2.amazonaws.com/Latest/splunk-common-information-model-cim_520.tgz" - ), + hardcoded_path="https://attack-range-appbinaries.s3.us-west-2.amazonaws.com/Latest/splunk-common-information-model-cim_520.tgz", ), TestApp( uid=6553, appid="Splunk_TA_okta_identity_cloud", title="Splunk Add-on for Okta Identity Cloud", version="2.1.0", - hardcoded_path=HttpUrl( - "https://attack-range-appbinaries.s3.us-west-2.amazonaws.com/Latest/splunk-add-on-for-okta-identity-cloud_210.tgz" - ), + hardcoded_path="https://attack-range-appbinaries.s3.us-west-2.amazonaws.com/Latest/splunk-add-on-for-okta-identity-cloud_210.tgz", ), TestApp( uid=6176, appid="Splunk_TA_linux_sysmon", title="Add-on for Linux Sysmon", version="1.0.4", - hardcoded_path=HttpUrl( - "https://attack-range-appbinaries.s3.us-west-2.amazonaws.com/Latest/add-on-for-linux-sysmon_104.tgz" - ), + hardcoded_path="https://attack-range-appbinaries.s3.us-west-2.amazonaws.com/Latest/add-on-for-linux-sysmon_104.tgz", ), TestApp( appid="Splunk_FIX_XMLWINEVENTLOG_HEC_PARSING", title="Splunk Fix XmlWinEventLog HEC Parsing", version="0.1", description="This TA is required for replaying Windows Data into the Test Environment. The Default TA does not include logic for properly splitting multiple log events in a single file. In production environments, this logic is applied by the Universal Forwarder.", - hardcoded_path=HttpUrl( - "https://attack-range-appbinaries.s3.us-west-2.amazonaws.com/Latest/Splunk_TA_fix_windows.tgz" - ), + hardcoded_path="https://attack-range-appbinaries.s3.us-west-2.amazonaws.com/Latest/Splunk_TA_fix_windows.tgz", ), TestApp( uid=742, appid="SPLUNK_ADD_ON_FOR_MICROSOFT_WINDOWS", title="Splunk Add-on for Microsoft Windows", version="8.8.0", - hardcoded_path=HttpUrl( - "https://attack-range-appbinaries.s3.us-west-2.amazonaws.com/Latest/splunk-add-on-for-microsoft-windows_880.tgz" - ), + hardcoded_path="https://attack-range-appbinaries.s3.us-west-2.amazonaws.com/Latest/splunk-add-on-for-microsoft-windows_880.tgz", ), TestApp( uid=5709, appid="Splunk_TA_microsoft_sysmon", title="Splunk Add-on for Sysmon", version="4.0.0", - hardcoded_path=HttpUrl( - "https://attack-range-appbinaries.s3.us-west-2.amazonaws.com/Latest/splunk-add-on-for-sysmon_400.tgz" - ), + hardcoded_path="https://attack-range-appbinaries.s3.us-west-2.amazonaws.com/Latest/splunk-add-on-for-sysmon_400.tgz", ), TestApp( uid=833, appid="Splunk_TA_nix", title="Splunk Add-on for Unix and Linux", version="9.0.0", - hardcoded_path=HttpUrl( - "https://attack-range-appbinaries.s3.us-west-2.amazonaws.com/Latest/splunk-add-on-for-unix-and-linux_900.tgz" - ), + hardcoded_path="https://attack-range-appbinaries.s3.us-west-2.amazonaws.com/Latest/splunk-add-on-for-unix-and-linux_900.tgz", ), TestApp( uid=5579, appid="Splunk_TA_CrowdStrike_FDR", title="Splunk Add-on for CrowdStrike FDR", version="1.5.0", - hardcoded_path=HttpUrl( - "https://attack-range-appbinaries.s3.us-west-2.amazonaws.com/Latest/splunk-add-on-for-crowdstrike-fdr_150.tgz" - ), + hardcoded_path="https://attack-range-appbinaries.s3.us-west-2.amazonaws.com/Latest/splunk-add-on-for-crowdstrike-fdr_150.tgz", ), TestApp( uid=3185, appid="SPLUNK_TA_FOR_IIS", title="Splunk Add-on for Microsoft IIS", version="1.3.0", - hardcoded_path=HttpUrl( - "https://attack-range-appbinaries.s3.us-west-2.amazonaws.com/Latest/splunk-add-on-for-microsoft-iis_130.tgz" - ), + hardcoded_path="https://attack-range-appbinaries.s3.us-west-2.amazonaws.com/Latest/splunk-add-on-for-microsoft-iis_130.tgz", ), TestApp( uid=4242, appid="SPLUNK_TA_FOR_SURICATA", title="TA for Suricata", version="2.3.4", - hardcoded_path=HttpUrl( - "https://attack-range-appbinaries.s3.us-west-2.amazonaws.com/Latest/ta-for-suricata_234.tgz" - ), + hardcoded_path="https://attack-range-appbinaries.s3.us-west-2.amazonaws.com/Latest/ta-for-suricata_234.tgz", ), TestApp( uid=5466, appid="SPLUNK_TA_FOR_ZEEK", title="TA for Zeek", version="1.0.6", - hardcoded_path=HttpUrl( - "https://attack-range-appbinaries.s3.us-west-2.amazonaws.com/Latest/ta-for-zeek_106.tgz" - ), + hardcoded_path="https://attack-range-appbinaries.s3.us-west-2.amazonaws.com/Latest/ta-for-zeek_106.tgz", ), TestApp( uid=3258, appid="SPLUNK_ADD_ON_FOR_NGINX", title="Splunk Add-on for NGINX", version="3.2.2", - hardcoded_path=HttpUrl( - "https://attack-range-appbinaries.s3.us-west-2.amazonaws.com/Latest/splunk-add-on-for-nginx_322.tgz" - ), + hardcoded_path="https://attack-range-appbinaries.s3.us-west-2.amazonaws.com/Latest/splunk-add-on-for-nginx_322.tgz", ), TestApp( uid=5238, appid="SPLUNK_ADD_ON_FOR_STREAM_FORWARDERS", title="Splunk Add-on for Stream Forwarders", version="8.1.1", - hardcoded_path=HttpUrl( - "https://attack-range-appbinaries.s3.us-west-2.amazonaws.com/Latest/splunk-add-on-for-stream-forwarders_811.tgz" - ), + hardcoded_path="https://attack-range-appbinaries.s3.us-west-2.amazonaws.com/Latest/splunk-add-on-for-stream-forwarders_811.tgz", ), TestApp( uid=5234, appid="SPLUNK_ADD_ON_FOR_STREAM_WIRE_DATA", title="Splunk Add-on for Stream Wire Data", version="8.1.1", - hardcoded_path=HttpUrl( - "https://attack-range-appbinaries.s3.us-west-2.amazonaws.com/Latest/splunk-add-on-for-stream-wire-data_811.tgz" - ), + hardcoded_path="https://attack-range-appbinaries.s3.us-west-2.amazonaws.com/Latest/splunk-add-on-for-stream-wire-data_811.tgz", ), TestApp( uid=2757, appid="PALO_ALTO_NETWORKS_ADD_ON_FOR_SPLUNK", title="Palo Alto Networks Add-on for Splunk", version="8.1.1", - hardcoded_path=HttpUrl( - "https://attack-range-appbinaries.s3.us-west-2.amazonaws.com/Latest/palo-alto-networks-add-on-for-splunk_811.tgz" - ), + hardcoded_path="https://attack-range-appbinaries.s3.us-west-2.amazonaws.com/Latest/palo-alto-networks-add-on-for-splunk_811.tgz", ), TestApp( uid=3865, appid="Zscaler_CIM", title="Zscaler Technical Add-On for Splunk", version="4.0.3", - hardcoded_path=HttpUrl( - "https://attack-range-appbinaries.s3.us-west-2.amazonaws.com/Latest/zscaler-technical-add-on-for-splunk_403.tgz" - ), + hardcoded_path="https://attack-range-appbinaries.s3.us-west-2.amazonaws.com/Latest/zscaler-technical-add-on-for-splunk_403.tgz", ), TestApp( uid=3719, appid="SPLUNK_ADD_ON_FOR_AMAZON_KINESIS_FIREHOSE", title="Splunk Add-on for Amazon Kinesis Firehose", version="1.3.2", - hardcoded_path=HttpUrl( - "https://attack-range-appbinaries.s3.us-west-2.amazonaws.com/Latest/splunk-add-on-for-amazon-kinesis-firehose_132.tgz" - ), + hardcoded_path="https://attack-range-appbinaries.s3.us-west-2.amazonaws.com/Latest/splunk-add-on-for-amazon-kinesis-firehose_132.tgz", ), TestApp( uid=1876, appid="Splunk_TA_aws", title="Splunk Add-on for AWS", version="7.5.0", - hardcoded_path=HttpUrl( - "https://attack-range-appbinaries.s3.us-west-2.amazonaws.com/Latest/splunk-add-on-for-amazon-web-services-aws_750.tgz" - ), + hardcoded_path="https://attack-range-appbinaries.s3.us-west-2.amazonaws.com/Latest/splunk-add-on-for-amazon-web-services-aws_750.tgz", ), TestApp( uid=3088, appid="SPLUNK_ADD_ON_FOR_GOOGLE_CLOUD_PLATFORM", title="Splunk Add-on for Google Cloud Platform", version="4.4.0", - hardcoded_path=HttpUrl( - "https://attack-range-appbinaries.s3.us-west-2.amazonaws.com/Latest/splunk-add-on-for-google-cloud-platform_440.tgz" - ), + hardcoded_path="https://attack-range-appbinaries.s3.us-west-2.amazonaws.com/Latest/splunk-add-on-for-google-cloud-platform_440.tgz", ), TestApp( uid=5556, appid="SPLUNK_ADD_ON_FOR_GOOGLE_WORKSPACE", title="Splunk Add-on for Google Workspace", version="2.6.3", - hardcoded_path=HttpUrl( - "https://attack-range-appbinaries.s3.us-west-2.amazonaws.com/Latest/splunk-add-on-for-google-workspace_263.tgz" - ), + hardcoded_path="https://attack-range-appbinaries.s3.us-west-2.amazonaws.com/Latest/splunk-add-on-for-google-workspace_263.tgz", ), TestApp( uid=3110, appid="SPLUNK_TA_MICROSOFT_CLOUD_SERVICES", title="Splunk Add-on for Microsoft Cloud Services", version="5.2.2", - hardcoded_path=HttpUrl( - "https://attack-range-appbinaries.s3.us-west-2.amazonaws.com/Latest/splunk-add-on-for-microsoft-cloud-services_522.tgz" - ), + hardcoded_path="https://attack-range-appbinaries.s3.us-west-2.amazonaws.com/Latest/splunk-add-on-for-microsoft-cloud-services_522.tgz", ), TestApp( uid=4055, appid="SPLUNK_ADD_ON_FOR_MICROSOFT_OFFICE_365", title="Splunk Add-on for Microsoft Office 365", version="4.5.1", - hardcoded_path=HttpUrl( - "https://attack-range-appbinaries.s3.us-west-2.amazonaws.com/Latest/splunk-add-on-for-microsoft-office-365_451.tgz" - ), + hardcoded_path="https://attack-range-appbinaries.s3.us-west-2.amazonaws.com/Latest/splunk-add-on-for-microsoft-office-365_451.tgz", ), TestApp( uid=2890, appid="SPLUNK_MACHINE_LEARNING_TOOLKIT", title="Splunk Machine Learning Toolkit", version="5.4.1", - hardcoded_path=HttpUrl( - "https://attack-range-appbinaries.s3.us-west-2.amazonaws.com/Latest/splunk-machine-learning-toolkit_541.tgz" - ), + hardcoded_path="https://attack-range-appbinaries.s3.us-west-2.amazonaws.com/Latest/splunk-machine-learning-toolkit_541.tgz", ), TestApp( uid=2734, appid="URL_TOOLBOX", title="URL Toolbox", version="1.9.2", - hardcoded_path=HttpUrl( - "https://attack-range-appbinaries.s3.us-west-2.amazonaws.com/Latest/url-toolbox_192.tgz" - ), + hardcoded_path="https://attack-range-appbinaries.s3.us-west-2.amazonaws.com/Latest/url-toolbox_192.tgz", ), ] diff --git a/contentctl/objects/story.py b/contentctl/objects/story.py index 910f6513..819993af 100644 --- a/contentctl/objects/story.py +++ b/contentctl/objects/story.py @@ -1,5 +1,6 @@ from __future__ import annotations +import pathlib import re from functools import cached_property from typing import TYPE_CHECKING, List @@ -22,8 +23,6 @@ from contentctl.objects.detection import Detection from contentctl.objects.investigation import Investigation -import pathlib - from contentctl.objects.enums import ContentStatus from contentctl.objects.security_content_object import SecurityContentObject diff --git a/pyproject.toml b/pyproject.toml index 9c051121..fca4766b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,8 +13,8 @@ contentctl = 'contentctl.contentctl:main' [tool.poetry.dependencies] -python = "^3.11,<3.14" -pydantic = "~2.9.2" +python = "^3.11,<3.15" +pydantic = "^2" PyYAML = "^6.0.2" requests = ">=2.32.4" pycvesearch = "^1.2" @@ -32,7 +32,7 @@ pygit2 = "^1.15.1" #have an issue when parsing an extremely large number of files #(in our testing great than 130) when using the mode:selected #--mode.files command. -tyro = "^0.9.2,<0.9.23" +tyro = "^1" gitpython = "^3.1.43" setuptools = ">=80.9.0" rich = "^14.0.0"